diff --git a/.gitattributes b/.gitattributes index cf582c5c724c295a211628bbfd18e2d261c6c8f8..8e1776c67b5a401618629eeb7d147368941ef18b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -537,3 +537,6 @@ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_python.dll filter=lfs d loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_python.lib filter=lfs diff=lfs merge=lfs -text loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_substrait.dll filter=lfs diff=lfs merge=lfs -text loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_substrait.lib filter=lfs diff=lfs merge=lfs -text +loan-advisor-agent/.venv/Lib/site-packages/pyarrow/lib.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text +loan-advisor-agent/.venv/Lib/site-packages/pyarrow/parquet.dll filter=lfs diff=lfs merge=lfs -text +loan-advisor-agent/.venv/Lib/site-packages/pyarrow/parquet.lib filter=lfs diff=lfs merge=lfs -text diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/api/writer.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/api/writer.h new file mode 100644 index 0000000000000000000000000000000000000000..c6f6f41c61489b809b7e7eee1d5df988be201d4f --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/api/writer.h @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "parquet/api/io.h" +#include "parquet/api/schema.h" +#include "parquet/column_writer.h" +#include "parquet/exception.h" +#include "parquet/file_writer.h" +#include "parquet/statistics.h" diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/reader.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/reader.h new file mode 100644 index 0000000000000000000000000000000000000000..298963a0a0b4bcf7c330d251f146574442ec0133 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/reader.h @@ -0,0 +1,424 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +// N.B. we don't include async_generator.h as it's relatively heavy +#include +#include +#include + +#include "parquet/file_reader.h" +#include "parquet/platform.h" +#include "parquet/properties.h" + +namespace arrow { + +class ChunkedArray; +class KeyValueMetadata; +class RecordBatchReader; +struct Scalar; +class Schema; +class Table; +class RecordBatch; + +} // namespace arrow + +namespace parquet { + +class FileMetaData; +class SchemaDescriptor; + +namespace arrow { + +class ColumnChunkReader; +class ColumnReader; +struct SchemaManifest; +class RowGroupReader; + +/// \brief Arrow read adapter class for deserializing Parquet files as Arrow row batches. +/// +/// This interfaces caters for different use cases and thus provides different +/// interfaces. In its most simplistic form, we cater for a user that wants to +/// read the whole Parquet at once with the `FileReader::ReadTable` method. +/// +/// More advanced users that also want to implement parallelism on top of each +/// single Parquet files should do this on the RowGroup level. For this, they can +/// call `FileReader::RowGroup(i)->ReadTable` to receive only the specified +/// RowGroup as a table. +/// +/// In the most advanced situation, where a consumer wants to independently read +/// RowGroups in parallel and consume each column individually, they can call +/// `FileReader::RowGroup(i)->Column(j)->Read` and receive an `arrow::Column` +/// instance. +/// +/// Finally, one can also get a stream of record batches using +/// `FileReader::GetRecordBatchReader()`. This can internally decode columns +/// in parallel if use_threads was enabled in the ArrowReaderProperties. +/// +/// The parquet format supports an optional integer field_id which can be assigned +/// to a field. Arrow will convert these field IDs to a metadata key named +/// PARQUET:field_id on the appropriate field. +// TODO(wesm): nested data does not always make sense with this user +// interface unless you are only reading a single leaf node from a branch of +// a table. For example: +// +// repeated group data { +// optional group record { +// optional int32 val1; +// optional byte_array val2; +// optional bool val3; +// } +// optional int32 val4; +// } +// +// In the Parquet file, there are 4 leaf nodes: +// +// * data.record.val1 +// * data.record.val2 +// * data.record.val3 +// * data.val4 +// +// When materializing this data in an Arrow array, we would have: +// +// data: list), +// val3: bool, +// >, +// val4: int32 +// >> +// +// However, in the Parquet format, each leaf node has its own repetition and +// definition levels describing the structure of the intermediate nodes in +// this array structure. Thus, we will need to scan the leaf data for a group +// of leaf nodes part of the same type tree to create a single result Arrow +// nested array structure. +// +// This is additionally complicated "chunky" repeated fields or very large byte +// arrays +class PARQUET_EXPORT FileReader { + public: + /// Factory function to create a FileReader from a ParquetFileReader and properties + /// \deprecated Deprecated in 23.0.0. Use arrow::Result version instead. + ARROW_DEPRECATED("Deprecated in 23.0.0. Use arrow::Result version instead.") + static ::arrow::Status Make(::arrow::MemoryPool* pool, + std::unique_ptr reader, + const ArrowReaderProperties& properties, + std::unique_ptr* out); + + /// Factory function to create a FileReader from a ParquetFileReader + /// \deprecated Deprecated in 23.0.0. Use arrow::Result version instead. + ARROW_DEPRECATED("Deprecated in 23.0.0. Use arrow::Result version instead.") + static ::arrow::Status Make(::arrow::MemoryPool* pool, + std::unique_ptr reader, + std::unique_ptr* out); + + /// Factory function to create a FileReader from a ParquetFileReader and properties + static ::arrow::Result> Make( + ::arrow::MemoryPool* pool, std::unique_ptr reader, + const ArrowReaderProperties& properties); + + /// Factory function to create a FileReader from a ParquetFileReader + static ::arrow::Result> Make( + ::arrow::MemoryPool* pool, std::unique_ptr reader); + + // Since the distribution of columns amongst a Parquet file's row groups may + // be uneven (the number of values in each column chunk can be different), we + // provide a column-oriented read interface. The ColumnReader hides the + // details of paging through the file's row groups and yielding + // fully-materialized arrow::Array instances + // + // Returns error status if the column of interest is not flat. + // The indicated column index is relative to the schema + virtual ::arrow::Status GetColumn(int i, std::unique_ptr* out) = 0; + + /// \brief Return arrow schema for all the columns. + virtual ::arrow::Status GetSchema(std::shared_ptr<::arrow::Schema>* out) = 0; + + /// \brief Read column as a whole into a chunked array. + /// + /// The index i refers the index of the top level schema field, which may + /// be nested or flat - e.g. + /// + /// 0 foo.bar + /// foo.bar.baz + /// foo.qux + /// 1 foo2 + /// 2 foo3 + /// + /// i=0 will read the entire foo struct, i=1 the foo2 primitive column etc + virtual ::arrow::Status ReadColumn(int i, + std::shared_ptr<::arrow::ChunkedArray>* out) = 0; + + /// \brief Return a RecordBatchReader of all row groups and columns. + virtual ::arrow::Result> + GetRecordBatchReader() = 0; + + /// \brief Return a RecordBatchReader of row groups selected from row_group_indices. + /// + /// Note that the ordering in row_group_indices matters. FileReaders must outlive + /// their RecordBatchReaders. + /// + /// \returns error Result if row_group_indices contains an invalid index + virtual ::arrow::Result> + GetRecordBatchReader(const std::vector& row_group_indices) = 0; + + /// \brief Return a RecordBatchReader of row groups selected from + /// row_group_indices, whose columns are selected by column_indices. + /// + /// Note that the ordering in row_group_indices and column_indices + /// matter. FileReaders must outlive their RecordBatchReaders. + /// + /// \returns error Result if either row_group_indices or column_indices + /// contains an invalid index + virtual ::arrow::Result> + GetRecordBatchReader(const std::vector& row_group_indices, + const std::vector& column_indices) = 0; + + /// \brief Return a RecordBatchReader of row groups selected from + /// row_group_indices, whose columns are selected by column_indices. + /// + /// Note that the ordering in row_group_indices and column_indices + /// matter. FileReaders must outlive their RecordBatchReaders. + /// + /// \param row_group_indices which row groups to read (order determines read order). + /// \param column_indices which columns to read (order determines output schema). + /// \param[out] out record batch stream from parquet data. + /// + /// \returns error Status if either row_group_indices or column_indices + /// contains an invalid index + /// \deprecated Deprecated in 21.0.0. Use arrow::Result version instead. + ARROW_DEPRECATED("Deprecated in 21.0.0. Use arrow::Result version instead.") + ::arrow::Status GetRecordBatchReader(const std::vector& row_group_indices, + const std::vector& column_indices, + std::shared_ptr<::arrow::RecordBatchReader>* out); + + /// \deprecated Deprecated in 21.0.0. Use arrow::Result version instead. + ARROW_DEPRECATED("Deprecated in 21.0.0. Use arrow::Result version instead.") + ::arrow::Status GetRecordBatchReader(const std::vector& row_group_indices, + std::shared_ptr<::arrow::RecordBatchReader>* out); + + /// \deprecated Deprecated in 21.0.0. Use arrow::Result version instead. + ARROW_DEPRECATED("Deprecated in 21.0.0. Use arrow::Result version instead.") + ::arrow::Status GetRecordBatchReader(std::shared_ptr<::arrow::RecordBatchReader>* out); + + /// \brief Return a generator of record batches. + /// + /// The FileReader must outlive the generator, so this requires that you pass in a + /// shared_ptr. + /// + /// \returns error Result if either row_group_indices or column_indices contains an + /// invalid index + virtual ::arrow::Result< + std::function<::arrow::Future>()>> + GetRecordBatchGenerator(std::shared_ptr reader, + const std::vector row_group_indices, + const std::vector column_indices, + ::arrow::internal::Executor* cpu_executor = NULLPTR, + int64_t rows_to_readahead = 0) = 0; + + /// Read all columns into a Table + virtual ::arrow::Result> ReadTable() = 0; + + /// \deprecated Deprecated in 24.0.0. Use arrow::Result version instead. + ARROW_DEPRECATED("Deprecated in 24.0.0. Use arrow::Result version instead.") + ::arrow::Status ReadTable(std::shared_ptr<::arrow::Table>* out); + + /// \brief Read the given columns into a Table + /// + /// The indicated column indices are relative to the internal representation + /// of the parquet table. For instance : + /// 0 foo.bar + /// foo.bar.baz 0 + /// foo.bar.baz2 1 + /// foo.qux 2 + /// 1 foo2 3 + /// 2 foo3 4 + /// + /// i=0 will read foo.bar.baz, i=1 will read only foo.bar.baz2 and so on. + /// Only leaf fields have indices; foo itself doesn't have an index. + /// To get the index for a particular leaf field, one can use + /// manifest().schema_fields to get the top level fields, and then walk the + /// tree to identify the relevant leaf fields and access its column_index. + /// To get the total number of leaf fields, use FileMetadata.num_columns(). + virtual ::arrow::Result> ReadTable( + const std::vector& column_indices) = 0; + + /// \deprecated Deprecated in 24.0.0. Use arrow::Result version instead. + ARROW_DEPRECATED("Deprecated in 24.0.0. Use arrow::Result version instead.") + ::arrow::Status ReadTable(const std::vector& column_indices, + std::shared_ptr<::arrow::Table>* out); + + /// \brief Read the given row group columns into a Table + virtual ::arrow::Result> ReadRowGroup( + int i, const std::vector& column_indices) = 0; + + /// \brief Read the given row group into a Table + virtual ::arrow::Result> ReadRowGroup(int i) = 0; + + /// \brief Read the given row groups columns into a Table + virtual ::arrow::Result> ReadRowGroups( + const std::vector& row_groups, const std::vector& column_indices) = 0; + + /// \brief Read the given row groups into a Table + virtual ::arrow::Result> ReadRowGroups( + const std::vector& row_groups) = 0; + + /// \deprecated Deprecated in 24.0.0. Use arrow::Result version instead. + ARROW_DEPRECATED("Deprecated in 24.0.0. Use arrow::Result version instead.") + ::arrow::Status ReadRowGroup(int i, const std::vector& column_indices, + std::shared_ptr<::arrow::Table>* out); + + /// \deprecated Deprecated in 24.0.0. Use arrow::Result version instead. + ARROW_DEPRECATED("Deprecated in 24.0.0. Use arrow::Result version instead.") + ::arrow::Status ReadRowGroup(int i, std::shared_ptr<::arrow::Table>* out); + + /// \deprecated Deprecated in 24.0.0. Use arrow::Result version instead. + ARROW_DEPRECATED("Deprecated in 24.0.0. Use arrow::Result version instead.") + ::arrow::Status ReadRowGroups(const std::vector& row_groups, + const std::vector& column_indices, + std::shared_ptr<::arrow::Table>* out); + + /// \deprecated Deprecated in 24.0.0. Use arrow::Result version instead. + ARROW_DEPRECATED("Deprecated in 24.0.0. Use arrow::Result version instead.") + ::arrow::Status ReadRowGroups(const std::vector& row_groups, + std::shared_ptr<::arrow::Table>* out); + + /// \brief Scan file contents with one thread, return number of rows + virtual ::arrow::Status ScanContents(std::vector columns, + const int32_t column_batch_size, + int64_t* num_rows) = 0; + + /// \brief Return a reader for the RowGroup, this object must not outlive the + /// FileReader. + virtual std::shared_ptr RowGroup(int row_group_index) = 0; + + /// \brief The number of row groups in the file + virtual int num_row_groups() const = 0; + + virtual ParquetFileReader* parquet_reader() const = 0; + + /// Set whether to use multiple threads during reads of multiple columns. + /// By default only one thread is used. + virtual void set_use_threads(bool use_threads) = 0; + + /// Set number of records to read per batch for the RecordBatchReader. + virtual void set_batch_size(int64_t batch_size) = 0; + + virtual const ArrowReaderProperties& properties() const = 0; + + virtual const SchemaManifest& manifest() const = 0; + + virtual ~FileReader() = default; +}; + +class RowGroupReader { + public: + virtual ~RowGroupReader() = default; + virtual std::shared_ptr Column(int column_index) = 0; + virtual ::arrow::Status ReadTable(const std::vector& column_indices, + std::shared_ptr<::arrow::Table>* out) = 0; + virtual ::arrow::Status ReadTable(std::shared_ptr<::arrow::Table>* out) = 0; + + private: + struct Iterator; +}; + +class ColumnChunkReader { + public: + virtual ~ColumnChunkReader() = default; + virtual ::arrow::Status Read(std::shared_ptr<::arrow::ChunkedArray>* out) = 0; +}; + +// At this point, the column reader is a stream iterator. It only knows how to +// read the next batch of values for a particular column from the file until it +// runs out. +// +// We also do not expose any internal Parquet details, such as row groups. This +// might change in the future. +class PARQUET_EXPORT ColumnReader { + public: + virtual ~ColumnReader() = default; + + // Scan the next array of the indicated size. The actual size of the + // returned array may be less than the passed size depending how much data is + // available in the file. + // + // When all the data in the file has been exhausted, the result is set to + // nullptr. + // + // Returns Status::OK on a successful read, including if you have exhausted + // the data available in the file. + virtual ::arrow::Status NextBatch(int64_t batch_size, + std::shared_ptr<::arrow::ChunkedArray>* out) = 0; +}; + +/// \brief Experimental helper class for bindings (like Python) that struggle +/// either with std::move or C++ exceptions +class PARQUET_EXPORT FileReaderBuilder { + public: + FileReaderBuilder(); + + /// Create FileReaderBuilder from Arrow file and optional properties / metadata + ::arrow::Status Open(std::shared_ptr<::arrow::io::RandomAccessFile> file, + const ReaderProperties& properties = default_reader_properties(), + std::shared_ptr metadata = NULLPTR); + + /// Create FileReaderBuilder from file path and optional properties / metadata + ::arrow::Status OpenFile(const std::string& path, bool memory_map = false, + const ReaderProperties& props = default_reader_properties(), + std::shared_ptr metadata = NULLPTR); + + ParquetFileReader* raw_reader() { return raw_reader_.get(); } + + /// Set Arrow MemoryPool for memory allocation + FileReaderBuilder* memory_pool(::arrow::MemoryPool* pool); + /// Set Arrow reader properties + FileReaderBuilder* properties(const ArrowReaderProperties& arg_properties); + /// Build FileReader instance + ::arrow::Status Build(std::unique_ptr* out); + ::arrow::Result> Build(); + + private: + ::arrow::MemoryPool* pool_; + ArrowReaderProperties properties_; + std::unique_ptr raw_reader_; +}; + +/// \defgroup parquet-arrow-reader-factories Factory functions for Parquet Arrow readers +/// +/// @{ + +/// \brief Build FileReader from Arrow file and MemoryPool +/// +/// Advanced settings are supported through the FileReaderBuilder class. +PARQUET_EXPORT +::arrow::Result> OpenFile( + std::shared_ptr<::arrow::io::RandomAccessFile>, ::arrow::MemoryPool* allocator); + +/// @} + +PARQUET_EXPORT +::arrow::Status StatisticsAsScalars(const Statistics& Statistics, + std::shared_ptr<::arrow::Scalar>* min, + std::shared_ptr<::arrow::Scalar>* max); + +} // namespace arrow +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/schema.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/schema.h new file mode 100644 index 0000000000000000000000000000000000000000..1e1f29635b8b8e91801b651669faf5f9bac1477b --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/schema.h @@ -0,0 +1,184 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "arrow/result.h" +#include "arrow/status.h" +#include "arrow/type.h" +#include "arrow/type_fwd.h" + +#include "parquet/level_conversion.h" +#include "parquet/platform.h" +#include "parquet/schema.h" + +namespace parquet { + +class ArrowReaderProperties; +class ArrowWriterProperties; +class WriterProperties; + +namespace arrow { + +/// \defgroup arrow-to-parquet-schema-conversion Functions to convert an Arrow +/// schema into a Parquet schema. +/// +/// @{ + +PARQUET_EXPORT +::arrow::Status FieldToNode(const std::shared_ptr<::arrow::Field>& field, + const WriterProperties& properties, + const ArrowWriterProperties& arrow_properties, + schema::NodePtr* out); + +PARQUET_EXPORT +::arrow::Status ToParquetSchema(const ::arrow::Schema* arrow_schema, + const WriterProperties& properties, + const ArrowWriterProperties& arrow_properties, + std::shared_ptr* out); + +PARQUET_EXPORT +::arrow::Status ToParquetSchema(const ::arrow::Schema* arrow_schema, + const WriterProperties& properties, + std::shared_ptr* out); + +/// @} + +/// \defgroup parquet-to-arrow-schema-conversion Functions to convert a Parquet +/// schema into an Arrow schema. +/// +/// @{ + +PARQUET_EXPORT +::arrow::Status FromParquetSchema( + const SchemaDescriptor* parquet_schema, const ArrowReaderProperties& properties, + const std::shared_ptr& key_value_metadata, + std::shared_ptr<::arrow::Schema>* out); + +PARQUET_EXPORT +::arrow::Status FromParquetSchema(const SchemaDescriptor* parquet_schema, + const ArrowReaderProperties& properties, + std::shared_ptr<::arrow::Schema>* out); + +PARQUET_EXPORT +::arrow::Status FromParquetSchema(const SchemaDescriptor* parquet_schema, + std::shared_ptr<::arrow::Schema>* out); + +/// @} + +/// \brief Bridge between an arrow::Field and parquet column indices. +struct PARQUET_EXPORT SchemaField { + std::shared_ptr<::arrow::Field> field; + std::vector children; + + // Only set for leaf nodes + int column_index = -1; + + parquet::internal::LevelInfo level_info; + + bool is_leaf() const { return column_index != -1; } +}; + +/// \brief Bridge between a parquet Schema and an arrow Schema. +/// +/// Expose parquet columns as a tree structure. Useful traverse and link +/// between arrow's Schema and parquet's Schema. +struct PARQUET_EXPORT SchemaManifest { + static ::arrow::Status Make( + const SchemaDescriptor* schema, + const std::shared_ptr& metadata, + const ArrowReaderProperties& properties, SchemaManifest* manifest); + + const SchemaDescriptor* descr; + std::shared_ptr<::arrow::Schema> origin_schema; + std::shared_ptr schema_metadata; + std::vector schema_fields; + + std::unordered_map column_index_to_field; + std::unordered_map child_to_parent; + + ::arrow::Status GetColumnField(int column_index, const SchemaField** out) const { + auto it = column_index_to_field.find(column_index); + if (it == column_index_to_field.end()) { + return ::arrow::Status::KeyError("Column index ", column_index, + " not found in schema manifest, may be malformed"); + } + *out = it->second; + return ::arrow::Status::OK(); + } + + const SchemaField* GetParent(const SchemaField* field) const { + // Returns nullptr also if not found + auto it = child_to_parent.find(field); + if (it == child_to_parent.end()) { + return NULLPTR; + } + return it->second; + } + + /// Coalesce a list of field indices (relative to the equivalent arrow::Schema) which + /// correspond to the column root (first node below the parquet schema's root group) of + /// each leaf referenced in column_indices. + /// + /// For example, for leaves `a.b.c`, `a.b.d.e`, and `i.j.k` (column_indices=[0,1,3]) + /// the roots are `a` and `i` (return=[0,2]). + /// + /// root + /// -- a <------ + /// -- -- b | | + /// -- -- -- c | + /// -- -- -- d | + /// -- -- -- -- e + /// -- f + /// -- -- g + /// -- -- -- h + /// -- i <--- + /// -- -- j | + /// -- -- -- k + ::arrow::Result> GetFieldIndices( + const std::vector& column_indices) const { + const schema::GroupNode* group = descr->group_node(); + std::unordered_set already_added; + + std::vector out; + for (int column_idx : column_indices) { + if (column_idx < 0 || column_idx >= descr->num_columns()) { + return ::arrow::Status::IndexError("Column index ", column_idx, " is not valid"); + } + + auto field_node = descr->GetColumnRoot(column_idx); + auto field_idx = group->FieldIndex(*field_node); + if (field_idx == -1) { + return ::arrow::Status::IndexError("Column index ", column_idx, " is not valid"); + } + + if (already_added.insert(field_idx).second) { + out.push_back(field_idx); + } + } + return out; + } +}; + +} // namespace arrow +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/test_util.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/test_util.h new file mode 100644 index 0000000000000000000000000000000000000000..3be799c6b10124fbb24fd9c21e21d9949a883035 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/test_util.h @@ -0,0 +1,487 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "arrow/array.h" +#include "arrow/array/builder_binary.h" +#include "arrow/array/builder_decimal.h" +#include "arrow/array/builder_primitive.h" +#include "arrow/testing/gtest_util.h" +#include "arrow/testing/random.h" +#include "arrow/type_fwd.h" +#include "arrow/type_traits.h" +#include "arrow/util/decimal.h" +#include "arrow/util/float16.h" +#include "parquet/column_reader.h" +#include "parquet/test_util.h" + +namespace parquet { + +using internal::RecordReader; + +namespace arrow { + +using ::arrow::Array; +using ::arrow::ChunkedArray; +using ::arrow::Status; + +template > +struct DecimalWithPrecisionAndScale { + using type = T; + static_assert(PRECISION >= T::kMinPrecision && PRECISION <= T::kMaxPrecision, + "Invalid precision value"); + static constexpr ::arrow::Type::type type_id = T::type_id; + static constexpr int32_t precision = PRECISION; + static constexpr int32_t scale = PRECISION - 1; +}; +template +using Decimal32WithPrecisionAndScale = + DecimalWithPrecisionAndScale<::arrow::Decimal32Type, PRECISION>; +template +using Decimal64WithPrecisionAndScale = + DecimalWithPrecisionAndScale<::arrow::Decimal64Type, PRECISION>; +template +using Decimal128WithPrecisionAndScale = + DecimalWithPrecisionAndScale<::arrow::Decimal128Type, PRECISION>; +template +using Decimal256WithPrecisionAndScale = + DecimalWithPrecisionAndScale<::arrow::Decimal256Type, PRECISION>; + +template +::arrow::enable_if_floating_point NonNullArray( + size_t size, std::shared_ptr* out) { + using c_type = typename ArrowType::c_type; + std::vector values; + if constexpr (::arrow::is_half_float_type::value) { + values.resize(size); + test::random_float16_numbers(static_cast(size), 0, ::arrow::util::Float16(0.0f), + ::arrow::util::Float16(1.0f), values.data()); + } else { + ::arrow::random_real(size, 0, static_cast(0), static_cast(1), + &values); + } + ::arrow::NumericBuilder builder; + RETURN_NOT_OK(builder.AppendValues(values.data(), values.size())); + return builder.Finish(out); +} + +template +::arrow::enable_if_integer NonNullArray(size_t size, + std::shared_ptr* out) { + std::vector values; + ::arrow::randint(size, 0, 64, &values); + + // Passing data type so this will work with TimestampType too + ::arrow::NumericBuilder builder(std::make_shared(), + ::arrow::default_memory_pool()); + RETURN_NOT_OK(builder.AppendValues(values.data(), values.size())); + return builder.Finish(out); +} + +template +::arrow::enable_if_date NonNullArray(size_t size, + std::shared_ptr* out) { + std::vector values; + ::arrow::randint(size, 0, 24, &values); + for (size_t i = 0; i < size; i++) { + values[i] *= 86400000; + } + + // Passing data type so this will work with TimestampType too + ::arrow::NumericBuilder builder(std::make_shared(), + ::arrow::default_memory_pool()); + RETURN_NOT_OK(builder.AppendValues(values.data(), values.size())); + return builder.Finish(out); +} + +template +::arrow::enable_if_base_binary NonNullArray( + size_t size, std::shared_ptr* out) { + using BuilderType = typename ::arrow::TypeTraits::BuilderType; + BuilderType builder; + for (size_t i = 0; i < size; i++) { + RETURN_NOT_OK(builder.Append("test-string")); + } + return builder.Finish(out); +} + +template +::arrow::enable_if_fixed_size_binary NonNullArray( + size_t size, std::shared_ptr* out) { + using BuilderType = typename ::arrow::TypeTraits::BuilderType; + // set byte_width to the length of "fixed": 5 + // todo: find a way to generate test data with more diversity. + BuilderType builder(::arrow::fixed_size_binary(5)); + for (size_t i = 0; i < size; i++) { + RETURN_NOT_OK(builder.Append("fixed")); + } + return builder.Finish(out); +} + +template +static void random_decimals(int64_t n, uint32_t seed, int32_t precision, uint8_t* out) { + auto gen = ::arrow::random::RandomArrayGenerator(seed); + std::shared_ptr decimals; + if constexpr (byte_width == 4) { + decimals = gen.Decimal32(::arrow::decimal32(precision, 0), n); + } else if constexpr (byte_width == 8) { + decimals = gen.Decimal64(::arrow::decimal64(precision, 0), n); + } else if constexpr (byte_width == 16) { + decimals = gen.Decimal128(::arrow::decimal128(precision, 0), n); + } else { + decimals = gen.Decimal256(::arrow::decimal256(precision, 0), n); + } + std::memcpy(out, decimals->data()->GetValues(1, 0), byte_width * n); +} + +template +::arrow::enable_if_t>, + Status> +NonNullArray(size_t size, std::shared_ptr* out) { + constexpr int32_t kDecimalPrecision = precision; + constexpr int32_t kDecimalScale = ArrowType::scale; + + const auto type = + std::make_shared(kDecimalPrecision, kDecimalScale); + const int32_t byte_width = type->byte_width(); + + constexpr int32_t seed = 0; + + ARROW_ASSIGN_OR_RAISE(auto out_buf, ::arrow::AllocateBuffer(size * byte_width)); + random_decimals(size, seed, kDecimalPrecision, + out_buf->mutable_data()); + + using Builder = typename ::arrow::TypeTraits::BuilderType; + Builder builder(type); + RETURN_NOT_OK(builder.AppendValues(out_buf->data(), size)); + return builder.Finish(out); +} + +template +::arrow::enable_if_boolean NonNullArray(size_t size, + std::shared_ptr* out) { + std::vector values; + ::arrow::randint(size, 0, 1, &values); + ::arrow::BooleanBuilder builder; + RETURN_NOT_OK(builder.AppendValues(values.data(), values.size())); + return builder.Finish(out); +} + +// This helper function only supports (size/2) nulls. +template +::arrow::enable_if_floating_point NullableArray( + size_t size, size_t num_nulls, uint32_t seed, std::shared_ptr* out) { + using c_type = typename ArrowType::c_type; + std::vector values; + if constexpr (::arrow::is_half_float_type::value) { + values.resize(size); + test::random_float16_numbers(static_cast(size), 0, ::arrow::util::Float16(-1e4f), + ::arrow::util::Float16(1e4f), values.data()); + } else { + ::arrow::random_real(size, seed, static_cast(-1e10), + static_cast(1e10), &values); + } + std::vector valid_bytes(size, 1); + + for (size_t i = 0; i < num_nulls; i++) { + valid_bytes[i * 2] = 0; + } + + ::arrow::NumericBuilder builder; + if (values.size() > 0) { + RETURN_NOT_OK(builder.AppendValues(values.data(), values.size(), valid_bytes.data())); + } + return builder.Finish(out); +} + +// This helper function only supports (size/2) nulls. +template +::arrow::enable_if_integer NullableArray(size_t size, size_t num_nulls, + uint32_t seed, + std::shared_ptr* out) { + std::vector values; + + // Seed is random in Arrow right now + (void)seed; + ::arrow::randint(size, 0, 64, &values); + std::vector valid_bytes(size, 1); + + for (size_t i = 0; i < num_nulls; i++) { + valid_bytes[i * 2] = 0; + } + + // Passing data type so this will work with TimestampType too + ::arrow::NumericBuilder builder(std::make_shared(), + ::arrow::default_memory_pool()); + RETURN_NOT_OK(builder.AppendValues(values.data(), values.size(), valid_bytes.data())); + return builder.Finish(out); +} + +template +::arrow::enable_if_date NullableArray(size_t size, size_t num_nulls, + uint32_t seed, + std::shared_ptr* out) { + std::vector values; + + // Seed is random in Arrow right now + (void)seed; + ::arrow::randint(size, 0, 24, &values); + for (size_t i = 0; i < size; i++) { + values[i] *= 86400000; + } + std::vector valid_bytes(size, 1); + + for (size_t i = 0; i < num_nulls; i++) { + valid_bytes[i * 2] = 0; + } + + // Passing data type so this will work with TimestampType too + ::arrow::NumericBuilder builder(std::make_shared(), + ::arrow::default_memory_pool()); + RETURN_NOT_OK(builder.AppendValues(values.data(), values.size(), valid_bytes.data())); + return builder.Finish(out); +} + +// This helper function only supports (size/2) nulls yet. +template +::arrow::enable_if_base_binary NullableArray( + size_t size, size_t num_nulls, uint32_t seed, std::shared_ptr<::arrow::Array>* out) { + std::vector valid_bytes(size, 1); + + for (size_t i = 0; i < num_nulls; i++) { + valid_bytes[i * 2] = 0; + } + + using BuilderType = typename ::arrow::TypeTraits::BuilderType; + BuilderType builder; + + const int kBufferSize = 10; + uint8_t buffer[kBufferSize]; + for (size_t i = 0; i < size; i++) { + if (!valid_bytes[i]) { + RETURN_NOT_OK(builder.AppendNull()); + } else { + ::arrow::random_bytes(kBufferSize, seed + static_cast(i), buffer); + if (ArrowType::is_utf8) { + // Trivially force data to be valid UTF8 by making it all ASCII + for (auto& byte : buffer) { + byte &= 0x7f; + } + } + RETURN_NOT_OK(builder.Append(buffer, kBufferSize)); + } + } + return builder.Finish(out); +} + +// This helper function only supports (size/2) nulls yet, +// same as NullableArray(..) +template +::arrow::enable_if_fixed_size_binary NullableArray( + size_t size, size_t num_nulls, uint32_t seed, std::shared_ptr<::arrow::Array>* out) { + std::vector valid_bytes(size, 1); + + for (size_t i = 0; i < num_nulls; i++) { + valid_bytes[i * 2] = 0; + } + + using BuilderType = typename ::arrow::TypeTraits::BuilderType; + const int byte_width = 10; + BuilderType builder(::arrow::fixed_size_binary(byte_width)); + + const int kBufferSize = byte_width; + uint8_t buffer[kBufferSize]; + for (size_t i = 0; i < size; i++) { + if (!valid_bytes[i]) { + RETURN_NOT_OK(builder.AppendNull()); + } else { + ::arrow::random_bytes(kBufferSize, seed + static_cast(i), buffer); + RETURN_NOT_OK(builder.Append(buffer)); + } + } + return builder.Finish(out); +} + +template +::arrow::enable_if_t>, + Status> +NullableArray(size_t size, size_t num_nulls, uint32_t seed, + std::shared_ptr<::arrow::Array>* out) { + std::vector valid_bytes(size, '\1'); + + for (size_t i = 0; i < num_nulls; ++i) { + valid_bytes[i * 2] = '\0'; + } + + constexpr int32_t kDecimalPrecision = precision; + constexpr int32_t kDecimalScale = ArrowType::scale; + + const auto type = + std::make_shared(kDecimalPrecision, kDecimalScale); + const int32_t byte_width = type->byte_width(); + + ARROW_ASSIGN_OR_RAISE(auto out_buf, ::arrow::AllocateBuffer(size * byte_width)); + random_decimals(size, seed, precision, + out_buf->mutable_data()); + + using Builder = typename ::arrow::TypeTraits::BuilderType; + Builder builder(type); + RETURN_NOT_OK(builder.AppendValues(out_buf->data(), size, valid_bytes.data())); + return builder.Finish(out); +} + +// This helper function only supports (size/2) nulls yet. +template +::arrow::enable_if_boolean NullableArray(size_t size, size_t num_nulls, + uint32_t seed, + std::shared_ptr* out) { + std::vector values; + + // Seed is random in Arrow right now + (void)seed; + + ::arrow::randint(size, 0, 1, &values); + std::vector valid_bytes(size, 1); + + for (size_t i = 0; i < num_nulls; i++) { + valid_bytes[i * 2] = 0; + } + + ::arrow::BooleanBuilder builder; + RETURN_NOT_OK(builder.AppendValues(values.data(), values.size(), valid_bytes.data())); + return builder.Finish(out); +} + +/// Wrap an Array into a ListArray by splitting it up into size lists. +/// +/// This helper function only supports (size/2) nulls. +Status MakeListArray(const std::shared_ptr& values, int64_t size, + int64_t null_count, const std::string& item_name, + bool nullable_values, std::shared_ptr<::arrow::ListArray>* out) { + // We always include an empty list + int64_t non_null_entries = size - null_count - 1; + int64_t length_per_entry = values->length() / non_null_entries; + + auto offsets = AllocateBuffer(); + RETURN_NOT_OK(offsets->Resize((size + 1) * sizeof(int32_t))); + int32_t* offsets_ptr = reinterpret_cast(offsets->mutable_data()); + + auto null_bitmap = AllocateBuffer(); + int64_t bitmap_size = ::arrow::bit_util::BytesForBits(size); + RETURN_NOT_OK(null_bitmap->Resize(bitmap_size)); + uint8_t* null_bitmap_ptr = null_bitmap->mutable_data(); + memset(null_bitmap_ptr, 0, bitmap_size); + + int32_t current_offset = 0; + for (int64_t i = 0; i < size; i++) { + offsets_ptr[i] = current_offset; + if (!(((i % 2) == 0) && ((i / 2) < null_count))) { + // Non-null list (list with index 1 is always empty). + ::arrow::bit_util::SetBit(null_bitmap_ptr, i); + if (i != 1) { + current_offset += static_cast(length_per_entry); + } + } + } + offsets_ptr[size] = static_cast(values->length()); + + auto value_field = ::arrow::field(item_name, values->type(), nullable_values); + *out = std::make_shared<::arrow::ListArray>(::arrow::list(value_field), size, offsets, + values, null_bitmap, null_count); + + return Status::OK(); +} + +// Make an array containing only empty lists, with a null values array +Status MakeEmptyListsArray(int64_t size, std::shared_ptr* out_array) { + // Allocate an offsets buffer containing only zeroes + const int64_t offsets_nbytes = (size + 1) * sizeof(int32_t); + ARROW_ASSIGN_OR_RAISE(auto offsets_buffer, ::arrow::AllocateBuffer(offsets_nbytes)); + memset(offsets_buffer->mutable_data(), 0, offsets_nbytes); + + auto value_field = + ::arrow::field("item", ::arrow::float64(), false /* nullable_values */); + auto list_type = ::arrow::list(value_field); + + std::vector> child_buffers = {nullptr /* null bitmap */, + nullptr /* values */}; + auto child_data = + ::arrow::ArrayData::Make(value_field->type(), 0, std::move(child_buffers)); + + std::vector> buffers = {nullptr /* bitmap */, + std::move(offsets_buffer)}; + auto array_data = ::arrow::ArrayData::Make(list_type, size, std::move(buffers)); + array_data->child_data.push_back(child_data); + + *out_array = ::arrow::MakeArray(array_data); + return Status::OK(); +} + +std::shared_ptr<::arrow::Table> MakeSimpleTable( + const std::shared_ptr& values, bool nullable) { + auto schema = ::arrow::schema({::arrow::field("col", values->type(), nullable)}); + return ::arrow::Table::Make(schema, {values}); +} + +std::shared_ptr<::arrow::Table> MakeSimpleTable(const std::shared_ptr& values, + bool nullable) { + auto carr = std::make_shared<::arrow::ChunkedArray>(values); + return MakeSimpleTable(carr, nullable); +} + +template +void ExpectArray(T* expected, Array* result) { + auto p_array = static_cast<::arrow::PrimitiveArray*>(result); + for (int i = 0; i < result->length(); i++) { + EXPECT_EQ(expected[i], reinterpret_cast(p_array->values()->data())[i]); + } +} + +template +void ExpectArrayT(void* expected, Array* result) { + ::arrow::PrimitiveArray* p_array = static_cast<::arrow::PrimitiveArray*>(result); + for (int64_t i = 0; i < result->length(); i++) { + EXPECT_EQ(reinterpret_cast(expected)[i], + reinterpret_cast( + p_array->values()->data())[i]); + } +} + +template <> +void ExpectArrayT<::arrow::BooleanType>(void* expected, Array* result) { + ::arrow::BooleanBuilder builder; + ARROW_EXPECT_OK( + builder.AppendValues(reinterpret_cast(expected), result->length())); + + std::shared_ptr expected_array; + ARROW_EXPECT_OK(builder.Finish(&expected_array)); + EXPECT_TRUE(result->Equals(*expected_array)); +} + +} // namespace arrow + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/writer.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/writer.h new file mode 100644 index 0000000000000000000000000000000000000000..46f993e1f9d3189be83f99c40a600a5ad9439921 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/writer.h @@ -0,0 +1,176 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "parquet/platform.h" +#include "parquet/properties.h" + +namespace arrow { + +class Array; +class ChunkedArray; +class RecordBatch; +class Schema; +class Table; + +} // namespace arrow + +namespace parquet { + +class FileMetaData; +class ParquetFileWriter; + +namespace arrow { + +/// \brief Iterative FileWriter class +/// +/// For basic usage, can write a Table at a time, creating one or more row +/// groups per write call. +/// +/// For advanced usage, can write column-by-column: Start a new RowGroup or +/// Chunk with NewRowGroup, then write column-by-column the whole column chunk. +/// +/// If PARQUET:field_id is present as a metadata key on a field, and the corresponding +/// value is a nonnegative integer, then it will be used as the field_id in the parquet +/// file. +class PARQUET_EXPORT FileWriter { + public: + static ::arrow::Status Make(MemoryPool* pool, std::unique_ptr writer, + std::shared_ptr<::arrow::Schema> schema, + std::shared_ptr arrow_properties, + std::unique_ptr* out); + + /// \brief Try to create an Arrow to Parquet file writer. + /// + /// \param schema schema of data that will be passed. + /// \param pool memory pool to use. + /// \param sink output stream to write Parquet data. + /// \param properties general Parquet writer properties. + /// \param arrow_properties Arrow-specific writer properties. + /// + /// \since 11.0.0 + static ::arrow::Result> Open( + const ::arrow::Schema& schema, MemoryPool* pool, + std::shared_ptr<::arrow::io::OutputStream> sink, + std::shared_ptr properties = default_writer_properties(), + std::shared_ptr arrow_properties = + default_arrow_writer_properties()); + + /// Return the Arrow schema to be written to. + virtual std::shared_ptr<::arrow::Schema> schema() const = 0; + + /// \brief Write a Table to Parquet. + /// + /// \param table Arrow table to write. + /// \param chunk_size maximum number of rows to write per row group. + virtual ::arrow::Status WriteTable( + const ::arrow::Table& table, int64_t chunk_size = DEFAULT_MAX_ROW_GROUP_LENGTH) = 0; + + /// \brief Start a new row group. + /// + /// Returns an error if not all columns have been written. + virtual ::arrow::Status NewRowGroup() = 0; + + /// \brief Write ColumnChunk in row group using an array. + virtual ::arrow::Status WriteColumnChunk(const ::arrow::Array& data) = 0; + + /// \brief Write ColumnChunk in row group using slice of a ChunkedArray + virtual ::arrow::Status WriteColumnChunk( + const std::shared_ptr<::arrow::ChunkedArray>& data, int64_t offset, + int64_t size) = 0; + + /// \brief Write ColumnChunk in a row group using a ChunkedArray + virtual ::arrow::Status WriteColumnChunk( + const std::shared_ptr<::arrow::ChunkedArray>& data) = 0; + + /// \brief Start a new buffered row group. + /// + /// Returns an error if not all columns have been written. + virtual ::arrow::Status NewBufferedRowGroup() = 0; + + /// \brief Write a RecordBatch into the buffered row group. + /// + /// Multiple RecordBatches can be written into the same row group + /// through this method. + /// + /// WriterProperties.max_row_group_length() is respected and a new + /// row group will be created if the current row group exceeds the + /// limit. + /// + /// Batches get flushed to the output stream once NewBufferedRowGroup() + /// or Close() is called. + /// + /// WARNING: If you are writing multiple files in parallel in the same + /// executor, deadlock may occur if ArrowWriterProperties::use_threads + /// is set to true to write columns in parallel. Please disable use_threads + /// option in this case. + virtual ::arrow::Status WriteRecordBatch(const ::arrow::RecordBatch& batch) = 0; + + /// \brief Write the footer and close the file. + virtual ::arrow::Status Close() = 0; + virtual ~FileWriter(); + + virtual MemoryPool* memory_pool() const = 0; + /// \brief Add key-value metadata to the file. + /// \param[in] key_value_metadata the metadata to add. + /// \note This will overwrite any existing metadata with the same key. + /// \return Error if Close() has been called. + /// + /// WARNING: If `store_schema` is enabled, `ARROW:schema` would be stored + /// in the key-value metadata. Overwriting this key would result in + /// `store_schema` being unusable during read. + virtual ::arrow::Status AddKeyValueMetadata( + const std::shared_ptr& key_value_metadata) = 0; + /// \brief Return the file metadata, only available after calling Close(). + virtual const std::shared_ptr metadata() const = 0; +}; + +/// \brief Write Parquet file metadata only to indicated Arrow OutputStream +PARQUET_EXPORT +::arrow::Status WriteFileMetaData(const FileMetaData& file_metadata, + ::arrow::io::OutputStream* sink); + +/// \brief Write metadata-only Parquet file to indicated Arrow OutputStream +PARQUET_EXPORT +::arrow::Status WriteMetaDataFile(const FileMetaData& file_metadata, + ::arrow::io::OutputStream* sink); + +/// \brief Write a Table to Parquet. +/// +/// This writes one table in a single shot. To write a Parquet file with +/// multiple tables iteratively, see parquet::arrow::FileWriter. +/// +/// \param table Table to write. +/// \param pool memory pool to use. +/// \param sink output stream to write Parquet data. +/// \param chunk_size maximum number of rows to write per row group. +/// \param properties general Parquet writer properties. +/// \param arrow_properties Arrow-specific writer properties. +::arrow::Status PARQUET_EXPORT +WriteTable(const ::arrow::Table& table, MemoryPool* pool, + std::shared_ptr<::arrow::io::OutputStream> sink, + int64_t chunk_size = DEFAULT_MAX_ROW_GROUP_LENGTH, + std::shared_ptr properties = default_writer_properties(), + std::shared_ptr arrow_properties = + default_arrow_writer_properties()); + +} // namespace arrow +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/benchmark_util.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/benchmark_util.h new file mode 100644 index 0000000000000000000000000000000000000000..8a56bb3afd212167cf242c136fd8191cfd8fb367 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/benchmark_util.h @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "parquet/types.h" + +namespace parquet::benchmarks { + +template +void GenerateBenchmarkData(uint32_t size, uint32_t seed, T* data, + std::vector* heap, uint32_t data_string_length); + +#define _GENERATE_BENCHMARK_DATA_DECL(KLASS) \ + template <> \ + void GenerateBenchmarkData(uint32_t size, uint32_t seed, KLASS* data, \ + std::vector* heap, uint32_t data_string_length); + +_GENERATE_BENCHMARK_DATA_DECL(int32_t) +_GENERATE_BENCHMARK_DATA_DECL(int64_t) +_GENERATE_BENCHMARK_DATA_DECL(float) +_GENERATE_BENCHMARK_DATA_DECL(double) +_GENERATE_BENCHMARK_DATA_DECL(ByteArray) +_GENERATE_BENCHMARK_DATA_DECL(FLBA) +_GENERATE_BENCHMARK_DATA_DECL(Int96) + +#undef _GENERATE_BENCHMARK_DATA_DECL + +} // namespace parquet::benchmarks diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/bloom_filter.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/bloom_filter.h new file mode 100644 index 0000000000000000000000000000000000000000..9c7e079cb68623ba256961b4164ec6e55b3beb67 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/bloom_filter.h @@ -0,0 +1,392 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "arrow/util/bit_util.h" +#include "arrow/util/logging.h" +#include "parquet/encryption/type_fwd.h" +#include "parquet/hasher.h" +#include "parquet/platform.h" +#include "parquet/types.h" + +namespace parquet { + +// A Bloom filter is a compact structure to indicate whether an item is not in a set or +// probably in a set. The Bloom filter usually consists of a bit set that represents a +// set of elements, a hash strategy and a Bloom filter algorithm. +class PARQUET_EXPORT BloomFilter { + public: + // Maximum Bloom filter size, it sets to HDFS default block size 128MB + // This value will be reconsidered when implementing Bloom filter producer. + static constexpr uint32_t kMaximumBloomFilterBytes = 128 * 1024 * 1024; + + /// Determine whether an element exist in set or not. + /// + /// @param hash the element to contain. + /// @return false if value is definitely not in set, and true means PROBABLY + /// in set. + virtual bool FindHash(uint64_t hash) const = 0; + + /// Insert element to set represented by Bloom filter bitset. + /// @param hash the hash of value to insert into Bloom filter. + virtual void InsertHash(uint64_t hash) = 0; + + /// Insert elements to set represented by Bloom filter bitset. + /// @param hashes the hash values to insert into Bloom filter. + /// @param num_values the number of hash values to insert. + virtual void InsertHashes(const uint64_t* hashes, int num_values) = 0; + + /// Write this Bloom filter to an output stream. A Bloom filter structure should + /// include bitset length, hash strategy, algorithm, and bitset. + /// + /// @param sink the output stream to write + virtual void WriteTo(ArrowOutputStream* sink) const = 0; + + /// Get the number of bytes of bitset + virtual uint32_t GetBitsetSize() const = 0; + + /// Compute hash for 32 bits value by using its plain encoding result. + /// + /// @param value the value to hash. + /// @return hash result. + virtual uint64_t Hash(int32_t value) const = 0; + + /// Compute hash for 64 bits value by using its plain encoding result. + /// + /// @param value the value to hash. + /// @return hash result. + virtual uint64_t Hash(int64_t value) const = 0; + + /// Compute hash for float value by using its plain encoding result. + /// + /// @param value the value to hash. + /// @return hash result. + virtual uint64_t Hash(float value) const = 0; + + /// Compute hash for double value by using its plain encoding result. + /// + /// @param value the value to hash. + /// @return hash result. + virtual uint64_t Hash(double value) const = 0; + + /// Compute hash for Int96 value by using its plain encoding result. + /// + /// @param value the value to hash. + /// @return hash result. + virtual uint64_t Hash(const Int96* value) const = 0; + uint64_t Hash(const Int96& value) const { return Hash(&value); } + + /// Compute hash for ByteArray value by using its plain encoding result. + /// + /// @param value the value to hash. + /// @return hash result. + virtual uint64_t Hash(const ByteArray* value) const = 0; + uint64_t Hash(const ByteArray& value) const { return Hash(&value); } + + /// Compute hash for fixed byte array value by using its plain encoding result. + /// + /// @param value the value address. + /// @param len the value length. + /// @return hash result. + virtual uint64_t Hash(const FLBA* value, uint32_t len) const = 0; + uint64_t Hash(const FLBA& value, uint32_t len) const { return Hash(&value, len); } + + /// Compute hash for std::string_view value by using its plain encoding result. + /// + /// @param value the value to hash. + /// @return hash result. + virtual uint64_t Hash(std::string_view value) const = 0; + + /// Batch compute hashes for 32 bits values by using its plain encoding result. + /// + /// @param values values a pointer to the values to hash. + /// @param num_values the number of values to hash. + /// @param hashes a pointer to the output hash values, its length should be equal to + /// num_values. + virtual void Hashes(const int32_t* values, int num_values, uint64_t* hashes) const = 0; + + /// Batch compute hashes for 64 bits values by using its plain encoding result. + /// + /// @param values values a pointer to the values to hash. + /// @param num_values the number of values to hash. + /// @param hashes a pointer to the output hash values, its length should be equal to + /// num_values. + virtual void Hashes(const int64_t* values, int num_values, uint64_t* hashes) const = 0; + + /// Batch compute hashes for float values by using its plain encoding result. + /// + /// @param values values a pointer to the values to hash. + /// @param num_values the number of values to hash. + /// @param hashes a pointer to the output hash values, its length should be equal to + /// num_values. + virtual void Hashes(const float* values, int num_values, uint64_t* hashes) const = 0; + + /// Batch compute hashes for double values by using its plain encoding result. + /// + /// @param values values a pointer to the values to hash. + /// @param num_values the number of values to hash. + /// @param hashes a pointer to the output hash values, its length should be equal to + /// num_values. + virtual void Hashes(const double* values, int num_values, uint64_t* hashes) const = 0; + + /// Batch compute hashes for Int96 values by using its plain encoding result. + /// + /// @param values values a pointer to the values to hash. + /// @param num_values the number of values to hash. + /// @param hashes a pointer to the output hash values, its length should be equal to + /// num_values. + virtual void Hashes(const Int96* values, int num_values, uint64_t* hashes) const = 0; + + /// Batch compute hashes for ByteArray values by using its plain encoding result. + /// + /// @param values values a pointer to the values to hash. + /// @param num_values the number of values to hash. + /// @param hashes a pointer to the output hash values, its length should be equal to + /// num_values. + virtual void Hashes(const ByteArray* values, int num_values, + uint64_t* hashes) const = 0; + + /// Batch compute hashes for fixed byte array values by using its plain encoding result. + /// + /// @param values values a pointer to the values to hash. + /// @param type_len the value length. + /// @param num_values the number of values to hash. + /// @param hashes a pointer to the output hash values, its length should be equal to + /// num_values. + virtual void Hashes(const FLBA* values, uint32_t type_len, int num_values, + uint64_t* hashes) const = 0; + + virtual ~BloomFilter() = default; + + protected: + // Hash strategy available for Bloom filter. + enum class HashStrategy : uint32_t { XXHASH = 0 }; + + // Bloom filter algorithm. + enum class Algorithm : uint32_t { BLOCK = 0 }; + + enum class CompressionStrategy : uint32_t { UNCOMPRESSED = 0 }; +}; + +/// The BlockSplitBloomFilter is implemented using block-based Bloom filters from +/// Putze et al.'s "Cache-,Hash- and Space-Efficient Bloom filters". The basic idea is to +/// hash the item to a tiny Bloom filter which size fit a single cache line or smaller. +/// +/// This implementation sets 8 bits in each tiny Bloom filter. Each tiny Bloom +/// filter is 32 bytes to take advantage of 32-byte SIMD instructions. +class PARQUET_EXPORT BlockSplitBloomFilter : public BloomFilter { + public: + /// The constructor of BlockSplitBloomFilter. It uses XXH64 as hash function. + /// + /// \param pool memory pool to use. + explicit BlockSplitBloomFilter( + ::arrow::MemoryPool* pool = ::arrow::default_memory_pool()); + + /// Initialize the BlockSplitBloomFilter. The range of num_bytes should be within + /// [kMinimumBloomFilterBytes, kMaximumBloomFilterBytes], it will be + /// rounded up/down to lower/upper bound if num_bytes is out of range and also + /// will be rounded up to a power of 2. + /// + /// @param num_bytes The number of bytes to store Bloom filter bitset. + void Init(uint32_t num_bytes); + + /// Initialize the BlockSplitBloomFilter. It copies the bitset as underlying + /// bitset because the given bitset may not satisfy the 32-byte alignment requirement + /// which may lead to segfault when performing SIMD instructions. It is the caller's + /// responsibility to free the bitset passed in. This is used when reconstructing + /// a Bloom filter from a parquet file. + /// + /// @param bitset The given bitset to initialize the Bloom filter. + /// @param num_bytes The number of bytes of given bitset. + void Init(const uint8_t* bitset, uint32_t num_bytes); + + /// Minimum Bloom filter size, it sets to 32 bytes to fit a tiny Bloom filter. + static constexpr uint32_t kMinimumBloomFilterBytes = 32; + + /// Calculate optimal size according to the number of distinct values and false + /// positive probability. + /// + /// @param ndv The number of distinct values. + /// @param fpp The false positive probability. + /// @return it always return a value between kMinimumBloomFilterBytes and + /// kMaximumBloomFilterBytes, and the return value is always a power of 2 + static uint32_t OptimalNumOfBytes(uint32_t ndv, double fpp) { + uint32_t optimal_num_of_bits = OptimalNumOfBits(ndv, fpp); + ARROW_DCHECK(::arrow::bit_util::IsMultipleOf8(optimal_num_of_bits)); + return optimal_num_of_bits >> 3; + } + + /// Calculate optimal size according to the number of distinct values and false + /// positive probability. + /// + /// @param ndv The number of distinct values. + /// @param fpp The false positive probability. + /// @return it always return a value between kMinimumBloomFilterBytes * 8 and + /// kMaximumBloomFilterBytes * 8, and the return value is always a power of 16 + static uint32_t OptimalNumOfBits(uint32_t ndv, double fpp) { + ARROW_DCHECK(fpp > 0.0 && fpp < 1.0); + const double m = -8.0 * ndv / log(1 - pow(fpp, 1.0 / 8)); + uint32_t num_bits; + + // Handle overflow. + if (m < 0 || m > kMaximumBloomFilterBytes << 3) { + num_bits = static_cast(kMaximumBloomFilterBytes << 3); + } else { + num_bits = static_cast(m); + } + + // Round up to lower bound + if (num_bits < kMinimumBloomFilterBytes << 3) { + num_bits = kMinimumBloomFilterBytes << 3; + } + + // Get next power of 2 if bits is not power of 2. + if ((num_bits & (num_bits - 1)) != 0) { + num_bits = static_cast(::arrow::bit_util::NextPower2(num_bits)); + } + + // Round down to upper bound + if (num_bits > kMaximumBloomFilterBytes << 3) { + num_bits = kMaximumBloomFilterBytes << 3; + } + + return num_bits; + } + + bool FindHash(uint64_t hash) const override; + void InsertHash(uint64_t hash) override; + void InsertHashes(const uint64_t* hashes, int num_values) override; + void WriteTo(ArrowOutputStream* sink) const override; + uint32_t GetBitsetSize() const override { return num_bytes_; } + + uint64_t Hash(int32_t value) const override { return hasher_->Hash(value); } + uint64_t Hash(int64_t value) const override { return hasher_->Hash(value); } + uint64_t Hash(float value) const override { return hasher_->Hash(value); } + uint64_t Hash(double value) const override { return hasher_->Hash(value); } + uint64_t Hash(const Int96* value) const override { return hasher_->Hash(value); } + uint64_t Hash(const ByteArray* value) const override { return hasher_->Hash(value); } + uint64_t Hash(const FLBA* value, uint32_t len) const override { + return hasher_->Hash(value, len); + } + uint64_t Hash(std::string_view value) const override { return hasher_->Hash(value); } + + void Hashes(const int32_t* values, int num_values, uint64_t* hashes) const override { + hasher_->Hashes(values, num_values, hashes); + } + void Hashes(const int64_t* values, int num_values, uint64_t* hashes) const override { + hasher_->Hashes(values, num_values, hashes); + } + void Hashes(const float* values, int num_values, uint64_t* hashes) const override { + hasher_->Hashes(values, num_values, hashes); + } + void Hashes(const double* values, int num_values, uint64_t* hashes) const override { + hasher_->Hashes(values, num_values, hashes); + } + void Hashes(const Int96* values, int num_values, uint64_t* hashes) const override { + hasher_->Hashes(values, num_values, hashes); + } + void Hashes(const ByteArray* values, int num_values, uint64_t* hashes) const override { + hasher_->Hashes(values, num_values, hashes); + } + void Hashes(const FLBA* values, uint32_t type_len, int num_values, + uint64_t* hashes) const override { + hasher_->Hashes(values, type_len, num_values, hashes); + } + + uint64_t Hash(const int32_t* value) const { return hasher_->Hash(*value); } + uint64_t Hash(const int64_t* value) const { return hasher_->Hash(*value); } + uint64_t Hash(const float* value) const { return hasher_->Hash(*value); } + uint64_t Hash(const double* value) const { return hasher_->Hash(*value); } + + /// Deserialize the Bloom filter from an input stream. It is used when reconstructing + /// a Bloom filter from a parquet filter. + /// + /// @param properties The parquet reader properties. + /// @param input_stream The input stream from which to construct the bloom filter. + /// @param bloom_filter_length The length of the serialized bloom filter including + /// header. + /// @return The BlockSplitBloomFilter. + static BlockSplitBloomFilter Deserialize( + const ReaderProperties& properties, ArrowInputStream* input_stream, + std::optional bloom_filter_length = std::nullopt); + + /// Deserialize an encrypted Bloom filter from an input stream. + /// + /// The same metadata decryptor is used for both the serialized header and bitset, + /// while switching module AADs between the two encrypted modules. + /// + /// @param properties The parquet reader properties. + /// @param input_stream The input stream from which to construct the bloom filter. + /// @param bloom_filter_length The length of the serialized bloom filter including + /// header. + /// @param decryptor Decryptor for encrypted Bloom filter modules. + /// @param row_group_ordinal Ordinal of the row group containing this Bloom filter. + /// @param column_ordinal Ordinal of the column containing this Bloom filter. + /// @return The BlockSplitBloomFilter. + static BlockSplitBloomFilter DeserializeEncrypted( + const ReaderProperties& properties, ArrowInputStream* input_stream, + std::optional bloom_filter_length, Decryptor* decryptor, + int16_t row_group_ordinal, int16_t column_ordinal); + + private: + inline void InsertHashImpl(uint64_t hash); + + // Bytes in a tiny Bloom filter block. + static constexpr int kBytesPerFilterBlock = 32; + + // The number of bits to be set in each tiny Bloom filter + static constexpr int kBitsSetPerBlock = 8; + + // A mask structure used to set bits in each tiny Bloom filter. + struct BlockMask { + uint32_t item[kBitsSetPerBlock]; + }; + + // The block-based algorithm needs eight odd SALT values to calculate eight indexes + // of bit to set, one bit in each 32-bit word. + static constexpr uint32_t SALT[kBitsSetPerBlock] = { + 0x47b6137bU, 0x44974d91U, 0x8824ad5bU, 0xa2b7289dU, + 0x705495c7U, 0x2df1424bU, 0x9efc4947U, 0x5c6bfb31U}; + + // Memory pool to allocate aligned buffer for bitset + ::arrow::MemoryPool* pool_; + + // The underlying buffer of bitset. + std::shared_ptr data_; + + // The number of bytes of Bloom filter bitset. + uint32_t num_bytes_; + + // Hash strategy used in this Bloom filter. + HashStrategy hash_strategy_; + + // Algorithm used in this Bloom filter. + Algorithm algorithm_; + + // Compression used in this Bloom filter. + CompressionStrategy compression_strategy_; + + // The hash pointer points to actual hash class used. + std::unique_ptr hasher_; +}; + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/bloom_filter_reader.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/bloom_filter_reader.h new file mode 100644 index 0000000000000000000000000000000000000000..a9ce847ef08dd5b04e587c4aa2d512cff6ca8279 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/bloom_filter_reader.h @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "arrow/io/interfaces.h" +#include "parquet/properties.h" +#include "parquet/type_fwd.h" + +namespace parquet { + +class InternalFileDecryptor; +class BloomFilter; + +class PARQUET_EXPORT RowGroupBloomFilterReader { + public: + virtual ~RowGroupBloomFilterReader() = default; + + /// \brief Read bloom filter of a column chunk. + /// + /// \param[in] i column ordinal of the column chunk. + /// \returns bloom filter of the column or nullptr if it does not exist. + /// \throws ParquetException if the index is out of bound, or read bloom + /// filter failed. + virtual std::unique_ptr GetColumnBloomFilter(int i) = 0; +}; + +/// \brief Interface for reading the bloom filter for a Parquet file. +class PARQUET_EXPORT BloomFilterReader { + public: + virtual ~BloomFilterReader() = default; + + /// \brief Create a BloomFilterReader instance. + /// \returns a BloomFilterReader instance. + /// WARNING: The returned BloomFilterReader references to all the input parameters, so + /// it must not outlive all of the input parameters. Usually these input parameters + /// come from the same ParquetFileReader object, so it must not outlive the reader + /// that creates this BloomFilterReader. + static std::unique_ptr Make( + std::shared_ptr<::arrow::io::RandomAccessFile> input, + std::shared_ptr file_metadata, const ReaderProperties& properties, + std::shared_ptr file_decryptor = NULLPTR); + + /// \brief Get the bloom filter reader of a specific row group. + /// \param[in] i row group ordinal to get bloom filter reader. + /// \returns RowGroupBloomFilterReader of the specified row group. A nullptr may or may + /// not be returned if the bloom filter for the row group is unavailable. It + /// is the caller's responsibility to check the return value of follow-up calls + /// to the RowGroupBloomFilterReader. + /// \throws ParquetException if the index is out of bound. + virtual std::shared_ptr RowGroup(int i) = 0; +}; + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/bloom_filter_writer.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/bloom_filter_writer.h new file mode 100644 index 0000000000000000000000000000000000000000..2a78596de129bb6edd0cbad7b7f168e894d290a4 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/bloom_filter_writer.h @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "arrow/type_fwd.h" + +#include "parquet/bloom_filter.h" +#include "parquet/index_location.h" +#include "parquet/type_fwd.h" + +namespace parquet { + +/// \brief Writer for updating a bloom filter with values of a specific Parquet type. +/// \note Boolean type is not supported. +template +class PARQUET_EXPORT TypedBloomFilterWriter { + public: + using T = typename ParquetType::c_type; + + /// \param descr The descriptor of the column to write. Must outlive this writer. + /// \param bloom_filter The bloom filter to update. Must outlive this writer. + TypedBloomFilterWriter(const ColumnDescriptor* descr, BloomFilter* bloom_filter); + + /// \brief Update the bloom filter with typed values. + /// + /// \param values The values to update the bloom filter with. + /// \param num_values The number of values to update the bloom filter with. + void Update(const T* values, int64_t num_values); + + /// \brief Update the bloom filter with typed values that have spaces. + /// + /// \param values The values to update the bloom filter with. + /// \param num_values The number of values to update the bloom filter with. + /// \param valid_bits The validity bitmap of the values. + /// \param valid_bits_offset The offset of the validity bitmap. + void UpdateSpaced(const T* values, int64_t num_values, const uint8_t* valid_bits, + int64_t valid_bits_offset); + + /// \brief Update the bloom filter with an Arrow array. + /// + /// \param values The Arrow array to update the bloom filter with. + void Update(const ::arrow::Array& values); + + private: + const ColumnDescriptor* descr_; + BloomFilter* bloom_filter_; +}; + +/// \brief Interface for building bloom filters of a parquet file. +class PARQUET_EXPORT BloomFilterBuilder { + public: + virtual ~BloomFilterBuilder() = default; + + /// \brief API to create a BloomFilterBuilder. + /// + /// \param schema The schema of the file and it must outlive the created builder. + /// \param properties Properties to get bloom filter options. It must outlive the + /// created builder. + static std::unique_ptr Make(const SchemaDescriptor* schema, + const WriterProperties* properties); + + /// \brief Start a new row group to write bloom filters, meaning that next calls + /// to `CreateBloomFilter` will create bloom filters for the new row group. + /// + /// \throws ParquetException if `WriteTo()` has been called already. + virtual void AppendRowGroup() = 0; + + /// \brief Create a BloomFilter of the column ordinal of the current row group. + /// + /// \param column_ordinal Column ordinal for the bloom filter. + /// \return created BloomFilter whose ownership belongs to the builder, or nullptr if + /// bloom filter is not enabled for the column. + /// \throws ParquetException if any condition is violated: + /// - `AppendRowGroup()` has not been called yet + /// - The column ordinal is out of bound + /// - Bloom filter already exists for the column + /// - `WriteTo()` has been called + virtual BloomFilter* CreateBloomFilter(int32_t column_ordinal) = 0; + + /// \brief Write all bloom filters to sink. + /// + /// The bloom filters cannot be modified after this method is called. + /// + /// \param[in] sink The output stream to write the bloom filters. + /// \return The location of all bloom filters. + /// \throws ParquetException if `WriteTo()` has been called. + virtual IndexLocations WriteTo(::arrow::io::OutputStream* sink) = 0; +}; + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_page.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_page.h new file mode 100644 index 0000000000000000000000000000000000000000..a4b6e041f042fa46ab92e463de75d01600e01029 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_page.h @@ -0,0 +1,181 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// This module defines an abstract interface for iterating through pages in a +// Parquet column chunk within a row group. It could be extended in the future +// to iterate through all data pages in all chunks in a file. + +#pragma once + +#include +#include +#include +#include + +#include "parquet/size_statistics.h" +#include "parquet/statistics.h" +#include "parquet/types.h" + +namespace parquet { + +// TODO: Parallel processing is not yet safe because of memory-ownership +// semantics (the PageReader may or may not own the memory referenced by a +// page) +// +// TODO(wesm): In the future Parquet implementations may store the crc code +// in format::PageHeader. parquet-mr currently does not, so we also skip it +// here, both on the read and write path +class Page { + public: + Page(const std::shared_ptr& buffer, PageType::type type) + : buffer_(buffer), type_(type) {} + + virtual ~Page() = default; + + PageType::type type() const { return type_; } + + std::shared_ptr buffer() const { return buffer_; } + + // @returns: a pointer to the page's data + const uint8_t* data() const { return buffer_->data(); } + + // @returns: the total size in bytes of the page's data buffer + int32_t size() const { return static_cast(buffer_->size()); } + + private: + std::shared_ptr buffer_; + PageType::type type_; +}; + +/// \brief Base type for DataPageV1 and DataPageV2 including common attributes +class DataPage : public Page { + public: + int32_t num_values() const { return num_values_; } + Encoding::type encoding() const { return encoding_; } + int64_t uncompressed_size() const { return uncompressed_size_; } + const EncodedStatistics& statistics() const { return statistics_; } + /// Return the row ordinal within the row group to the first row in the data page. + /// Currently it is only present from data pages created by ColumnWriter in order + /// to collect page index. + std::optional first_row_index() const { return first_row_index_; } + const SizeStatistics& size_statistics() const { return size_statistics_; } + + virtual ~DataPage() = default; + + protected: + DataPage(PageType::type type, const std::shared_ptr& buffer, int32_t num_values, + Encoding::type encoding, int64_t uncompressed_size, + EncodedStatistics statistics, std::optional first_row_index, + SizeStatistics size_statistics) + : Page(buffer, type), + num_values_(num_values), + encoding_(encoding), + uncompressed_size_(uncompressed_size), + statistics_(std::move(statistics)), + first_row_index_(std::move(first_row_index)), + size_statistics_(std::move(size_statistics)) {} + + int32_t num_values_; + Encoding::type encoding_; + int64_t uncompressed_size_; + EncodedStatistics statistics_; + /// Row ordinal within the row group to the first row in the data page. + std::optional first_row_index_; + SizeStatistics size_statistics_; +}; + +class DataPageV1 : public DataPage { + public: + DataPageV1(const std::shared_ptr& buffer, int32_t num_values, + Encoding::type encoding, Encoding::type definition_level_encoding, + Encoding::type repetition_level_encoding, int64_t uncompressed_size, + EncodedStatistics statistics = EncodedStatistics(), + std::optional first_row_index = std::nullopt, + SizeStatistics size_statistics = SizeStatistics()) + : DataPage(PageType::DATA_PAGE, buffer, num_values, encoding, uncompressed_size, + std::move(statistics), std::move(first_row_index), + std::move(size_statistics)), + definition_level_encoding_(definition_level_encoding), + repetition_level_encoding_(repetition_level_encoding) {} + + Encoding::type repetition_level_encoding() const { return repetition_level_encoding_; } + + Encoding::type definition_level_encoding() const { return definition_level_encoding_; } + + private: + Encoding::type definition_level_encoding_; + Encoding::type repetition_level_encoding_; +}; + +class DataPageV2 : public DataPage { + public: + DataPageV2(const std::shared_ptr& buffer, int32_t num_values, int32_t num_nulls, + int32_t num_rows, Encoding::type encoding, + int32_t definition_levels_byte_length, int32_t repetition_levels_byte_length, + int64_t uncompressed_size, bool is_compressed = false, + EncodedStatistics statistics = EncodedStatistics(), + std::optional first_row_index = std::nullopt, + SizeStatistics size_statistics = SizeStatistics()) + : DataPage(PageType::DATA_PAGE_V2, buffer, num_values, encoding, uncompressed_size, + std::move(statistics), std::move(first_row_index), + std::move(size_statistics)), + num_nulls_(num_nulls), + num_rows_(num_rows), + definition_levels_byte_length_(definition_levels_byte_length), + repetition_levels_byte_length_(repetition_levels_byte_length), + is_compressed_(is_compressed) {} + + int32_t num_nulls() const { return num_nulls_; } + + int32_t num_rows() const { return num_rows_; } + + int32_t definition_levels_byte_length() const { return definition_levels_byte_length_; } + + int32_t repetition_levels_byte_length() const { return repetition_levels_byte_length_; } + + bool is_compressed() const { return is_compressed_; } + + private: + int32_t num_nulls_; + int32_t num_rows_; + int32_t definition_levels_byte_length_; + int32_t repetition_levels_byte_length_; + bool is_compressed_; +}; + +class DictionaryPage : public Page { + public: + DictionaryPage(const std::shared_ptr& buffer, int32_t num_values, + Encoding::type encoding, bool is_sorted = false) + : Page(buffer, PageType::DICTIONARY_PAGE), + num_values_(num_values), + encoding_(encoding), + is_sorted_(is_sorted) {} + + int32_t num_values() const { return num_values_; } + + Encoding::type encoding() const { return encoding_; } + + bool is_sorted() const { return is_sorted_; } + + private: + int32_t num_values_; + Encoding::type encoding_; + bool is_sorted_; +}; + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_reader.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_reader.h new file mode 100644 index 0000000000000000000000000000000000000000..66540d5687935a7886a46318b9e76e3f35b319aa --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_reader.h @@ -0,0 +1,458 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "arrow/type_fwd.h" +#include "arrow/util/macros.h" +#include "parquet/exception.h" +#include "parquet/level_conversion.h" +#include "parquet/metadata.h" +#include "parquet/platform.h" +#include "parquet/properties.h" +#include "parquet/schema.h" +#include "parquet/types.h" + +namespace arrow { + +namespace bit_util { +class BitReader; +} // namespace bit_util + +namespace util { +template +class RleBitPackedDecoder; +} // namespace util + +} // namespace arrow + +namespace parquet { + +class Decryptor; +class Page; + +// 16 MB is the default maximum page header size +static constexpr uint32_t kDefaultMaxPageHeaderSize = 16 * 1024 * 1024; + +// 16 KB is the default expected page header size +static constexpr uint32_t kDefaultPageHeaderSize = 16 * 1024; + +// \brief DataPageStats stores encoded statistics and number of values/rows for +// a page. +struct PARQUET_EXPORT DataPageStats { + DataPageStats(const EncodedStatistics* encoded_statistics, int32_t num_values, + std::optional num_rows) + : encoded_statistics(encoded_statistics), + num_values(num_values), + num_rows(num_rows) {} + + // Encoded statistics extracted from the page header. + // Nullptr if there are no statistics in the page header. + const EncodedStatistics* encoded_statistics; + // Number of values stored in the page. Filled for both V1 and V2 data pages. + // For repeated fields, this can be greater than number of rows. For + // non-repeated fields, this will be the same as the number of rows. + int32_t num_values; + // Number of rows stored in the page. std::nullopt if not available. + std::optional num_rows; +}; + +class PARQUET_EXPORT LevelDecoder { + public: + LevelDecoder(); + ~LevelDecoder(); + + // Initialize the LevelDecoder state with new data + // and return the number of bytes consumed + int SetData(Encoding::type encoding, int16_t max_level, int num_buffered_values, + const uint8_t* data, int32_t data_size); + + void SetDataV2(int32_t num_bytes, int16_t max_level, int num_buffered_values, + const uint8_t* data); + + // Decodes a batch of levels into an array and returns the number of levels decoded + int Decode(int batch_size, int16_t* levels); + + private: + int bit_width_; + int num_values_remaining_; + Encoding::type encoding_; + std::unique_ptr<::arrow::util::RleBitPackedDecoder> rle_decoder_; + std::unique_ptr<::arrow::bit_util::BitReader> bit_packed_decoder_; + int16_t max_level_; +}; + +struct CryptoContext { + bool start_decrypt_with_dictionary_page = false; + int16_t row_group_ordinal = -1; + int16_t column_ordinal = -1; + std::function()> meta_decryptor_factory; + std::function()> data_decryptor_factory; +}; + +// Abstract page iterator interface. This way, we can feed column pages to the +// ColumnReader through whatever mechanism we choose +class PARQUET_EXPORT PageReader { + using DataPageFilter = std::function; + + public: + virtual ~PageReader() = default; + + static std::unique_ptr Open( + std::shared_ptr stream, int64_t total_num_values, + Compression::type codec, bool always_compressed = false, + ::arrow::MemoryPool* pool = ::arrow::default_memory_pool(), + const CryptoContext* ctx = NULLPTR); + static std::unique_ptr Open(std::shared_ptr stream, + int64_t total_num_values, + Compression::type codec, + const ReaderProperties& properties, + bool always_compressed = false, + const CryptoContext* ctx = NULLPTR); + + // If data_page_filter is present (not null), NextPage() will call the + // callback function exactly once per page in the order the pages appear in + // the column. If the callback function returns true the page will be + // skipped. The callback will be called only if the page type is DATA_PAGE or + // DATA_PAGE_V2. Dictionary pages will not be skipped. + // Caller is responsible for checking that statistics are correct using + // ApplicationVersion::HasCorrectStatistics(). + // \note API EXPERIMENTAL + void set_data_page_filter(DataPageFilter data_page_filter) { + data_page_filter_ = std::move(data_page_filter); + } + + // @returns: shared_ptr(nullptr) on EOS, std::shared_ptr + // containing new Page otherwise + // + // The returned Page may contain references that aren't guaranteed to live + // beyond the next call to NextPage(). + virtual std::shared_ptr NextPage() = 0; + + virtual void set_max_page_header_size(uint32_t size) = 0; + + protected: + // Callback that decides if we should skip a page or not. + DataPageFilter data_page_filter_; +}; + +class PARQUET_EXPORT ColumnReader { + public: + virtual ~ColumnReader() = default; + + static std::shared_ptr Make( + const ColumnDescriptor* descr, std::unique_ptr pager, + ::arrow::MemoryPool* pool = ::arrow::default_memory_pool()); + + // Returns true if there are still values in this column. + virtual bool HasNext() = 0; + + virtual Type::type type() const = 0; + + virtual const ColumnDescriptor* descr() const = 0; + + // Get the encoding that can be exposed by this reader. If it returns + // dictionary encoding, then ReadBatchWithDictionary can be used to read data. + // + // \note API EXPERIMENTAL + virtual ExposedEncoding GetExposedEncoding() = 0; + + protected: + friend class RowGroupReader; + // Set the encoding that can be exposed by this reader. + // + // \note API EXPERIMENTAL + virtual void SetExposedEncoding(ExposedEncoding encoding) = 0; +}; + +// API to read values from a single column. This is a main client facing API. +template +class TypedColumnReader : public ColumnReader { + public: + using T = typename DType::c_type; + + // Read a batch of repetition levels, definition levels, and values from the + // column. + // + // Since null values are not stored in the values, the number of values read + // may be less than the number of repetition and definition levels. With + // nested data this is almost certainly true. + // + // Set def_levels or rep_levels to nullptr if you want to skip reading them. + // This is only safe if you know through some other source that there are no + // undefined values. + // + // To fully exhaust a row group, you must read batches until the number of + // values read reaches the number of stored values according to the metadata. + // + // This API is the same for both V1 and V2 of the DataPage + // + // @returns: actual number of levels read (see values_read for number of values read) + virtual int64_t ReadBatch(int64_t batch_size, int16_t* def_levels, int16_t* rep_levels, + T* values, int64_t* values_read) = 0; + + // Skip reading values. This method will work for both repeated and + // non-repeated fields. Note that this method is skipping values and not + // records. This distinction is important for repeated fields, meaning that + // we are not skipping over the values to the next record. For example, + // consider the following two consecutive records containing one repeated field: + // {[1, 2, 3]}, {[4, 5]}. If we Skip(2), our next read value will be 3, which + // is inside the first record. + // Returns the number of values skipped. + virtual int64_t Skip(int64_t num_values_to_skip) = 0; + + // Read a batch of repetition levels, definition levels, and indices from the + // column. And read the dictionary if a dictionary page is encountered during + // reading pages. This API is similar to ReadBatch(), with ability to read + // dictionary and indices. It is only valid to call this method when the reader can + // expose dictionary encoding. (i.e., the reader's GetExposedEncoding() returns + // DICTIONARY). + // + // The dictionary is read along with the data page. When there's no data page, + // the dictionary won't be returned. + // + // @param batch_size The batch size to read + // @param[out] def_levels The Parquet definition levels. + // @param[out] rep_levels The Parquet repetition levels. + // @param[out] indices The dictionary indices. + // @param[out] indices_read The number of indices read. + // @param[out] dict The pointer to dictionary values. It will return nullptr if + // there's no data page. Each column chunk only has one dictionary page. The dictionary + // is owned by the reader, so the caller is responsible for copying the dictionary + // values before the reader gets destroyed. + // @param[out] dict_len The dictionary length. It will return 0 if there's no data + // page. + // @returns: actual number of levels read (see indices_read for number of + // indices read + // + // \note API EXPERIMENTAL + virtual int64_t ReadBatchWithDictionary(int64_t batch_size, int16_t* def_levels, + int16_t* rep_levels, int32_t* indices, + int64_t* indices_read, const T** dict, + int32_t* dict_len) = 0; +}; + +namespace internal { + +/// \brief Stateful column reader that delimits semantic records for both flat +/// and nested columns +/// +/// \note API EXPERIMENTAL +/// \since 1.3.0 +class PARQUET_EXPORT RecordReader { + public: + /// \brief Creates a record reader. + /// @param descr Column descriptor + /// @param leaf_info Level info, used to determine if a column is nullable or not + /// @param pool Memory pool to use for buffering values and rep/def levels + /// @param read_dictionary True if reading directly as Arrow dictionary-encoded + /// @param read_dense_for_nullable True if reading dense and not leaving space for null + /// values + /// @param arrow_type Which type to read this column as (optional). Currently + /// only used for byte array columns (see BinaryRecordReader::GetBuilderChunks). + static std::shared_ptr Make( + const ColumnDescriptor* descr, LevelInfo leaf_info, + ::arrow::MemoryPool* pool = ::arrow::default_memory_pool(), + bool read_dictionary = false, bool read_dense_for_nullable = false, + const std::shared_ptr<::arrow::DataType>& arrow_type = NULLPTR); + + virtual ~RecordReader() = default; + + /// \brief Attempt to read indicated number of records from column chunk + /// Note that for repeated fields, a record may have more than one value + /// and all of them are read. If read_dense_for_nullable() it will + /// not leave any space for null values. Otherwise, it will read spaced. + /// \return number of records read + virtual int64_t ReadRecords(int64_t num_records) = 0; + + /// \brief Attempt to skip indicated number of records from column chunk. + /// Note that for repeated fields, a record may have more than one value + /// and all of them are skipped. + /// \return number of records skipped + virtual int64_t SkipRecords(int64_t num_records) = 0; + + /// \brief Pre-allocate space for data. Results in better flat read performance + virtual void Reserve(int64_t num_values) = 0; + + /// \brief Clear consumed values and repetition/definition levels as the + /// result of calling ReadRecords + /// For FLBA and ByteArray types, call GetBuilderChunks() to reset them. + virtual void Reset() = 0; + + /// \brief Transfer filled values buffer to caller. A new one will be + /// allocated in subsequent ReadRecords calls + virtual std::shared_ptr ReleaseValues() = 0; + + /// \brief Transfer filled validity bitmap buffer to caller. A new one will + /// be allocated in subsequent ReadRecords calls + virtual std::shared_ptr ReleaseIsValid() = 0; + + /// \brief Return true if the record reader has more internal data yet to + /// process + virtual bool HasMoreData() const = 0; + + /// \brief Advance record reader to the next row group. Must be set before + /// any records could be read/skipped. + /// \param[in] reader obtained from RowGroupReader::GetColumnPageReader + virtual void SetPageReader(std::unique_ptr reader) = 0; + + /// \brief Returns the underlying column reader's descriptor. + virtual const ColumnDescriptor* descr() const = 0; + + virtual void DebugPrintState() = 0; + + /// \brief Returns the dictionary owned by the current decoder. Throws an + /// exception if the current decoder is not for dictionary encoding. The caller is + /// responsible for casting the returned pointer to proper type depending on the + /// column's physical type. An example: + /// const ByteArray* dict = reinterpret_cast(ReadDictionary(&len)); + /// or: + /// const float* dict = reinterpret_cast(ReadDictionary(&len)); + /// \param[out] dictionary_length The number of dictionary entries. + virtual const void* ReadDictionary(int32_t* dictionary_length) = 0; + + /// \brief Decoded definition levels + int16_t* def_levels() const { + return reinterpret_cast(def_levels_->mutable_data()); + } + + /// \brief Decoded repetition levels + int16_t* rep_levels() const { + return reinterpret_cast(rep_levels_->mutable_data()); + } + + /// \brief Decoded values, including nulls, if any + /// FLBA and ByteArray types do not use this array and read into their own + /// builders. + uint8_t* values() const { return values_->mutable_data(); } + + /// \brief Number of values written, including space left for nulls if any. + /// If this Reader was constructed with read_dense_for_nullable(), there is no space for + /// nulls and null_count() will be 0. There is no read-ahead/buffering for values. For + /// FLBA and ByteArray types this value reflects the values written with the last + /// ReadRecords call since those readers will reset the values after each call. + int64_t values_written() const { return values_written_; } + + /// \brief Number of definition / repetition levels (from those that have + /// been decoded) that have been consumed inside the reader. + int64_t levels_position() const { return levels_position_; } + + /// \brief Number of definition / repetition levels that have been written + /// internally in the reader. This may be larger than values_written() because + /// for repeated fields we need to look at the levels in advance to figure out + /// the record boundaries. + int64_t levels_written() const { return levels_written_; } + + /// \brief Number of nulls in the leaf that we have read so far into the + /// values vector. This is only valid when !read_dense_for_nullable(). When + /// read_dense_for_nullable() it will always be 0. + int64_t null_count() const { return null_count_; } + + /// \brief True if the leaf values are nullable + bool nullable_values() const { return nullable_values_; } + + /// \brief True if reading directly as Arrow dictionary-encoded + bool read_dictionary() const { return read_dictionary_; } + + /// \brief True if reading dense for nullable columns. + bool read_dense_for_nullable() const { return read_dense_for_nullable_; } + + protected: + /// \brief Indicates if we can have nullable values. Note that repeated fields + /// may or may not be nullable. + bool nullable_values_; + + bool at_record_start_; + int64_t records_read_; + + /// \brief Stores values. These values are populated based on each ReadRecords + /// call. No extra values are buffered for the next call. SkipRecords will not + /// add any value to this buffer. + std::shared_ptr<::arrow::ResizableBuffer> values_; + /// \brief False for FIXED_LEN_BYTE_ARRAY and BYTE_ARRAY, in which case we + /// don't allocate the values buffer and we directly read into builder classes. + bool uses_values_; + + /// \brief Values that we have read into 'values_' + 'null_count_'. + int64_t values_written_; + int64_t values_capacity_; + int64_t null_count_; + + /// \brief Each bit corresponds to one element in 'values_' and specifies if it + /// is null or not null. + /// + /// Not set if leaf type is not nullable or read_dense_for_nullable_ is true. + std::shared_ptr<::arrow::ResizableBuffer> valid_bits_; + + /// \brief Buffer for definition levels. May contain more levels than + /// is actually read. This is because we read levels ahead to + /// figure out record boundaries for repeated fields. + /// For flat required fields, 'def_levels_' and 'rep_levels_' are not + /// populated. For non-repeated fields 'rep_levels_' is not populated. + /// 'def_levels_' and 'rep_levels_' must be of the same size if present. + std::shared_ptr<::arrow::ResizableBuffer> def_levels_; + /// \brief Buffer for repetition levels. Only populated for repeated + /// fields. + std::shared_ptr<::arrow::ResizableBuffer> rep_levels_; + + /// \brief Number of definition / repetition levels that have been written + /// internally in the reader. This may be larger than values_written() since + /// for repeated fields we need to look at the levels in advance to figure out + /// the record boundaries. + int64_t levels_written_; + /// \brief Position of the next level that should be consumed. + int64_t levels_position_; + int64_t levels_capacity_; + + bool read_dictionary_ = false; + // If true, we will not leave any space for the null values in the values_ + // vector or fill nulls values in BinaryRecordReader/DictionaryRecordReader. + // + // If read_dense_for_nullable_ is true, the BinaryRecordReader/DictionaryRecordReader + // might still populate the validity bitmap buffer. + bool read_dense_for_nullable_ = false; +}; + +class BinaryRecordReader : virtual public RecordReader { + public: + virtual std::vector> GetBuilderChunks() = 0; +}; + +/// \brief Read records directly to dictionary-encoded Arrow form (int32 +/// indices). Only valid for BYTE_ARRAY columns +class DictionaryRecordReader : virtual public RecordReader { + public: + virtual std::shared_ptr<::arrow::ChunkedArray> GetResult() = 0; +}; + +} // namespace internal + +using BoolReader = TypedColumnReader; +using Int32Reader = TypedColumnReader; +using Int64Reader = TypedColumnReader; +using Int96Reader = TypedColumnReader; +using FloatReader = TypedColumnReader; +using DoubleReader = TypedColumnReader; +using ByteArrayReader = TypedColumnReader; +using FixedLenByteArrayReader = TypedColumnReader; + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_scanner.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_scanner.h new file mode 100644 index 0000000000000000000000000000000000000000..085eb045b7446c10faee3c5bd4df93e816321863 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_scanner.h @@ -0,0 +1,264 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +#include "parquet/column_reader.h" +#include "parquet/exception.h" +#include "parquet/platform.h" +#include "parquet/schema.h" +#include "parquet/types.h" + +namespace parquet { + +static constexpr int64_t DEFAULT_SCANNER_BATCH_SIZE = 128; + +class PARQUET_EXPORT Scanner { + public: + explicit Scanner(std::shared_ptr reader, + int64_t batch_size = DEFAULT_SCANNER_BATCH_SIZE, + ::arrow::MemoryPool* pool = ::arrow::default_memory_pool()) + : batch_size_(batch_size), + level_offset_(0), + levels_buffered_(0), + value_buffer_(AllocateBuffer(pool)), + value_offset_(0), + values_buffered_(0), + reader_(std::move(reader)) { + def_levels_.resize( + descr()->max_definition_level() > 0 ? static_cast(batch_size_) : 0); + rep_levels_.resize( + descr()->max_repetition_level() > 0 ? static_cast(batch_size_) : 0); + } + + virtual ~Scanner() {} + + static std::shared_ptr Make( + std::shared_ptr col_reader, + int64_t batch_size = DEFAULT_SCANNER_BATCH_SIZE, + ::arrow::MemoryPool* pool = ::arrow::default_memory_pool()); + + virtual void PrintNext(std::ostream& out, int width, bool with_levels = false) = 0; + + bool HasNext() { return level_offset_ < levels_buffered_ || reader_->HasNext(); } + + const ColumnDescriptor* descr() const { return reader_->descr(); } + + int64_t batch_size() const { return batch_size_; } + + void SetBatchSize(int64_t batch_size) { batch_size_ = batch_size; } + + protected: + int64_t batch_size_; + + std::vector def_levels_; + std::vector rep_levels_; + int level_offset_; + int levels_buffered_; + + std::shared_ptr value_buffer_; + int value_offset_; + int64_t values_buffered_; + std::shared_ptr reader_; +}; + +template +class PARQUET_TEMPLATE_CLASS_EXPORT TypedScanner : public Scanner { + public: + typedef typename DType::c_type T; + + explicit TypedScanner(std::shared_ptr reader, + int64_t batch_size = DEFAULT_SCANNER_BATCH_SIZE, + ::arrow::MemoryPool* pool = ::arrow::default_memory_pool()) + : Scanner(std::move(reader), batch_size, pool) { + typed_reader_ = static_cast*>(reader_.get()); + int value_byte_size = type_traits::value_byte_size; + PARQUET_THROW_NOT_OK(value_buffer_->Resize(batch_size_ * value_byte_size)); + values_ = reinterpret_cast(value_buffer_->mutable_data()); + } + + virtual ~TypedScanner() {} + + bool NextLevels(int16_t* def_level, int16_t* rep_level) { + if (level_offset_ == levels_buffered_) { + levels_buffered_ = static_cast( + typed_reader_->ReadBatch(static_cast(batch_size_), def_levels_.data(), + rep_levels_.data(), values_, &values_buffered_)); + + value_offset_ = 0; + level_offset_ = 0; + if (!levels_buffered_) { + return false; + } + } + *def_level = descr()->max_definition_level() > 0 ? def_levels_[level_offset_] : 0; + *rep_level = descr()->max_repetition_level() > 0 ? rep_levels_[level_offset_] : 0; + level_offset_++; + return true; + } + + bool Next(T* val, int16_t* def_level, int16_t* rep_level, bool* is_null) { + if (level_offset_ == levels_buffered_) { + if (!HasNext()) { + // Out of data pages + return false; + } + } + + NextLevels(def_level, rep_level); + *is_null = *def_level < descr()->max_definition_level(); + + if (*is_null) { + return true; + } + + if (value_offset_ == values_buffered_) { + throw ParquetException("Value was non-null, but has not been buffered"); + } + *val = values_[value_offset_++]; + return true; + } + + // Returns true if there is a next value + bool NextValue(T* val, bool* is_null) { + if (level_offset_ == levels_buffered_) { + if (!HasNext()) { + // Out of data pages + return false; + } + } + + // Out of values + int16_t def_level = -1; + int16_t rep_level = -1; + NextLevels(&def_level, &rep_level); + *is_null = def_level < descr()->max_definition_level(); + + if (*is_null) { + return true; + } + + if (value_offset_ == values_buffered_) { + throw ParquetException("Value was non-null, but has not been buffered"); + } + *val = values_[value_offset_++]; + return true; + } + + virtual void PrintNext(std::ostream& out, int width, bool with_levels = false) { + T val{}; + int16_t def_level = -1; + int16_t rep_level = -1; + bool is_null = false; + char buffer[80]; + + if (!Next(&val, &def_level, &rep_level, &is_null)) { + throw ParquetException("No more values buffered"); + } + + if (with_levels) { + out << " D:" << def_level << " R:" << rep_level << " "; + if (!is_null) { + out << "V:"; + } + } + + if (is_null) { + std::string null_fmt = format_fwf(width); + snprintf(buffer, sizeof(buffer), null_fmt.c_str(), "NULL"); + } else { + FormatValue(&val, buffer, sizeof(buffer), width); + } + out << buffer; + } + + private: + // The ownership of this object is expressed through the reader_ variable in the base + TypedColumnReader* typed_reader_; + + inline void FormatValue(void* val, char* buffer, int bufsize, int width); + + T* values_; +}; + +template +inline void TypedScanner::FormatValue(void* val, char* buffer, int bufsize, + int width) { + std::string fmt = format_fwf(width); + snprintf(buffer, bufsize, fmt.c_str(), *reinterpret_cast(val)); +} + +template <> +inline void TypedScanner::FormatValue(void* val, char* buffer, int bufsize, + int width) { + std::string fmt = format_fwf(width); + std::string result = Int96ToString(*reinterpret_cast(val)); + snprintf(buffer, bufsize, fmt.c_str(), result.c_str()); +} + +template <> +inline void TypedScanner::FormatValue(void* val, char* buffer, int bufsize, + int width) { + std::string fmt = format_fwf(width); + std::string result = ByteArrayToString(*reinterpret_cast(val)); + snprintf(buffer, bufsize, fmt.c_str(), result.c_str()); +} + +template <> +inline void TypedScanner::FormatValue(void* val, char* buffer, int bufsize, + int width) { + std::string fmt = format_fwf(width); + std::string result = FixedLenByteArrayToString( + *reinterpret_cast(val), descr()->type_length()); + snprintf(buffer, bufsize, fmt.c_str(), result.c_str()); +} + +typedef TypedScanner BoolScanner; +typedef TypedScanner Int32Scanner; +typedef TypedScanner Int64Scanner; +typedef TypedScanner Int96Scanner; +typedef TypedScanner FloatScanner; +typedef TypedScanner DoubleScanner; +typedef TypedScanner ByteArrayScanner; +typedef TypedScanner FixedLenByteArrayScanner; + +template +int64_t ScanAll(int32_t batch_size, int16_t* def_levels, int16_t* rep_levels, + uint8_t* values, int64_t* values_buffered, + parquet::ColumnReader* reader) { + typedef typename RType::T Type; + auto typed_reader = static_cast(reader); + auto vals = reinterpret_cast(&values[0]); + return typed_reader->ReadBatch(batch_size, def_levels, rep_levels, vals, + values_buffered); +} + +int64_t PARQUET_EXPORT ScanAllValues(int32_t batch_size, int16_t* def_levels, + int16_t* rep_levels, uint8_t* values, + int64_t* values_buffered, + parquet::ColumnReader* reader); + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_writer.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_writer.h new file mode 100644 index 0000000000000000000000000000000000000000..7051053fd1ea43fd14033b8b98793cf1fa333fd7 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_writer.h @@ -0,0 +1,317 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "arrow/type_fwd.h" +#include "arrow/util/compression.h" +#include "parquet/exception.h" +#include "parquet/platform.h" +#include "parquet/types.h" + +namespace arrow { + +class Array; + +namespace bit_util { +class BitWriter; +} // namespace bit_util + +namespace util { +class RleBitPackedEncoder; +class CodecOptions; +} // namespace util + +} // namespace arrow + +namespace parquet { + +struct ArrowWriteContext; +class ColumnChunkMetaDataBuilder; +class ColumnDescriptor; +class ColumnIndexBuilder; +class DataPage; +class DictionaryPage; +class Encryptor; +class OffsetIndexBuilder; +class WriterProperties; + +class PARQUET_EXPORT LevelEncoder { + public: + LevelEncoder(); + ~LevelEncoder(); + + static int MaxBufferSize(Encoding::type encoding, int16_t max_level, + int num_buffered_values); + + // Initialize the LevelEncoder. + void Init(Encoding::type encoding, int16_t max_level, int num_buffered_values, + uint8_t* data, int data_size); + + // Encodes a batch of levels from an array and returns the number of levels encoded + int Encode(int batch_size, const int16_t* levels); + + int32_t len() { + if (encoding_ != Encoding::RLE) { + throw ParquetException("Only implemented for RLE encoding"); + } + return rle_length_; + } + + private: + int bit_width_; + int rle_length_; + Encoding::type encoding_; + std::unique_ptr<::arrow::util::RleBitPackedEncoder> rle_encoder_; + std::unique_ptr<::arrow::bit_util::BitWriter> bit_packed_encoder_; +}; + +class PARQUET_EXPORT PageWriter { + public: + virtual ~PageWriter() {} + + static std::unique_ptr Open( + std::shared_ptr sink, Compression::type codec, + ColumnChunkMetaDataBuilder* metadata, int16_t row_group_ordinal = -1, + int16_t column_chunk_ordinal = -1, + ::arrow::MemoryPool* pool = ::arrow::default_memory_pool(), + bool buffered_row_group = false, + std::shared_ptr header_encryptor = NULLPTR, + std::shared_ptr data_encryptor = NULLPTR, + bool page_write_checksum_enabled = false, + // column_index_builder MUST outlive the PageWriter + ColumnIndexBuilder* column_index_builder = NULLPTR, + // offset_index_builder MUST outlive the PageWriter + OffsetIndexBuilder* offset_index_builder = NULLPTR, + const CodecOptions& codec_options = CodecOptions{}); + + // The Column Writer decides if dictionary encoding is used if set and + // if the dictionary encoding has fallen back to default encoding on reaching dictionary + // page limit + virtual void Close(bool has_dictionary, bool fallback) = 0; + + // Return the number of uncompressed bytes written (including header size) + virtual int64_t WriteDataPage(const DataPage& page) = 0; + + // Return the number of uncompressed bytes written (including header size) + virtual int64_t WriteDictionaryPage(const DictionaryPage& page) = 0; + + /// \brief The total number of bytes written as serialized data and + /// dictionary pages to the sink so far. + virtual int64_t total_compressed_bytes_written() const = 0; + + virtual bool has_compressor() = 0; + + virtual void Compress(const Buffer& src_buffer, ResizableBuffer* dest_buffer) = 0; +}; + +class PARQUET_EXPORT ColumnWriter { + public: + virtual ~ColumnWriter() = default; + + static std::shared_ptr Make(ColumnChunkMetaDataBuilder*, + std::unique_ptr, + const WriterProperties* properties, + BloomFilter* bloom_filter = NULLPTR); + + /// \brief Closes the ColumnWriter, commits any buffered values to pages. + /// \return Total size of the column in bytes + virtual int64_t Close() = 0; + + /// \brief The physical Parquet type of the column + virtual Type::type type() const = 0; + + /// \brief The schema for the column + virtual const ColumnDescriptor* descr() const = 0; + + /// \brief The number of rows written so far + virtual int64_t rows_written() const = 0; + + /// \brief The total size of the compressed pages + page headers. Values + /// are still buffered and not written to a pager yet + /// + /// So in un-buffered mode, it always returns 0 + virtual int64_t total_compressed_bytes() const = 0; + + /// \brief The total number of bytes written as serialized data and + /// dictionary pages to the ColumnChunk so far + /// These bytes are uncompressed bytes. + virtual int64_t total_bytes_written() const = 0; + + /// \brief The total number of bytes written as serialized data and + /// dictionary pages to the ColumnChunk so far. + /// If the column is uncompressed, the value would be equal to + /// total_bytes_written(). + virtual int64_t total_compressed_bytes_written() const = 0; + + /// \brief Estimated size of the values that are not written to a page yet. + virtual int64_t estimated_buffered_value_bytes() const = 0; + + /// \brief Estimated size of the definition levels that are not written to a page yet. + virtual int64_t estimated_buffered_def_level_bytes() const = 0; + + /// \brief Estimated size of the repetition levels that are not written to a page yet. + virtual int64_t estimated_buffered_rep_level_bytes() const = 0; + + /// \brief Estimated size of the dictionary that are not written to a page yet. + virtual int64_t estimated_buffered_dict_bytes() const = 0; + + /// \brief The file-level writer properties + virtual const WriterProperties* properties() = 0; + + /// \brief Add key-value metadata to the ColumnChunk. + /// \param[in] key_value_metadata the metadata to add. + /// \note This will overwrite any existing metadata with the same key. + /// \throw ParquetException if Close() has been called. + virtual void AddKeyValueMetadata( + const std::shared_ptr& key_value_metadata) = 0; + + /// \brief Reset the ColumnChunk key-value metadata. + /// \throw ParquetException if Close() has been called. + virtual void ResetKeyValueMetadata() = 0; + + /// \brief Write Apache Arrow columnar data directly to ColumnWriter. Returns + /// error status if the array data type is not compatible with the concrete + /// writer type. + /// + /// leaf_array is always a primitive (possibly dictionary encoded type). + /// Leaf_field_nullable indicates whether the leaf array is considered nullable + /// according to its schema in a Table or its parent array. + virtual ::arrow::Status WriteArrow(const int16_t* def_levels, const int16_t* rep_levels, + int64_t num_levels, const ::arrow::Array& leaf_array, + ArrowWriteContext* ctx, + bool leaf_field_nullable) = 0; +}; + +// API to write values to a single column. This is the main client facing API. +template +class TypedColumnWriter : public ColumnWriter { + public: + using T = typename DType::c_type; + + // Write a batch of repetition levels, definition levels, and values to the + // column. + // `num_values` is the number of logical leaf values. + // `def_levels` (resp. `rep_levels`) can be null if the column's max definition level + // (resp. max repetition level) is 0. + // If not null, each of `def_levels` and `rep_levels` must have at least + // `num_values`. + // + // The number of physical values written (taken from `values`) is returned. + // It can be smaller than `num_values` is there are some undefined values. + virtual int64_t WriteBatch(int64_t num_values, const int16_t* def_levels, + const int16_t* rep_levels, const T* values) = 0; + + /// Write a batch of repetition levels, definition levels, and values to the + /// column. + /// + /// In comparison to WriteBatch the length of repetition and definition levels + /// is the same as of the number of values read for max_definition_level == 1. + /// In the case of max_definition_level > 1, the repetition and definition + /// levels are larger than the values but the values include the null entries + /// with definition_level == (max_definition_level - 1). Thus we have to differentiate + /// in the parameters of this function if the input has the length of num_values or the + /// _number of rows in the lowest nesting level_. + /// + /// In the case that the most inner node in the Parquet is required, the _number of rows + /// in the lowest nesting level_ is equal to the number of non-null values. If the + /// inner-most schema node is optional, the _number of rows in the lowest nesting level_ + /// also includes all values with definition_level == (max_definition_level - 1). + /// + /// @param num_values number of levels to write. + /// @param def_levels The Parquet definition levels, length is num_values + /// @param rep_levels The Parquet repetition levels, length is num_values + /// @param valid_bits Bitmap that indicates if the row is null on the lowest nesting + /// level. The length is number of rows in the lowest nesting level. + /// @param valid_bits_offset The offset in bits of the valid_bits where the + /// first relevant bit resides. + /// @param values The values in the lowest nested level including + /// spacing for nulls on the lowest levels; input has the length + /// of the number of rows on the lowest nesting level. + virtual void WriteBatchSpaced(int64_t num_values, const int16_t* def_levels, + const int16_t* rep_levels, const uint8_t* valid_bits, + int64_t valid_bits_offset, const T* values) = 0; +}; + +using BoolWriter = TypedColumnWriter; +using Int32Writer = TypedColumnWriter; +using Int64Writer = TypedColumnWriter; +using Int96Writer = TypedColumnWriter; +using FloatWriter = TypedColumnWriter; +using DoubleWriter = TypedColumnWriter; +using ByteArrayWriter = TypedColumnWriter; +using FixedLenByteArrayWriter = TypedColumnWriter; + +namespace internal { + +/** + * Timestamp conversion constants + */ +constexpr int64_t kJulianEpochOffsetDays = INT64_C(2440588); + +template +inline void ArrowTimestampToImpalaTimestamp(const int64_t time, Int96* impala_timestamp) { + auto julian_days = static_cast(time / UnitPerDay + kJulianEpochOffsetDays); + int64_t last_day_units = time % UnitPerDay; + if (last_day_units < 0) { + --julian_days; + last_day_units += UnitPerDay; + } + impala_timestamp->value[2] = static_cast(julian_days); + uint64_t last_day_nanos = static_cast(last_day_units) * NanosecondsPerUnit; + // impala_timestamp will be unaligned every other entry so do memcpy instead + // of assign and reinterpret cast to avoid undefined behavior. + std::memcpy(impala_timestamp, &last_day_nanos, sizeof(uint64_t)); +} + +constexpr int64_t kSecondsInNanos = INT64_C(1000000000); + +inline void SecondsToImpalaTimestamp(const int64_t seconds, Int96* impala_timestamp) { + ArrowTimestampToImpalaTimestamp(seconds, + impala_timestamp); +} + +constexpr int64_t kMillisecondsInNanos = kSecondsInNanos / INT64_C(1000); + +inline void MillisecondsToImpalaTimestamp(const int64_t milliseconds, + Int96* impala_timestamp) { + ArrowTimestampToImpalaTimestamp( + milliseconds, impala_timestamp); +} + +constexpr int64_t kMicrosecondsInNanos = kMillisecondsInNanos / INT64_C(1000); + +inline void MicrosecondsToImpalaTimestamp(const int64_t microseconds, + Int96* impala_timestamp) { + ArrowTimestampToImpalaTimestamp( + microseconds, impala_timestamp); +} + +constexpr int64_t kNanosecondsInNanos = INT64_C(1); + +inline void NanosecondsToImpalaTimestamp(const int64_t nanoseconds, + Int96* impala_timestamp) { + ArrowTimestampToImpalaTimestamp( + nanoseconds, impala_timestamp); +} + +} // namespace internal +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encoding.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encoding.h new file mode 100644 index 0000000000000000000000000000000000000000..4dc45defe9633cbb8114a88950d097102dec40de --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encoding.h @@ -0,0 +1,476 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "arrow/type_fwd.h" + +#include "parquet/exception.h" +#include "parquet/platform.h" +#include "parquet/types.h" + +namespace arrow { +template +class Dictionary32Builder; +} + +namespace parquet { + +template +class TypedEncoder; + +using BooleanEncoder = TypedEncoder; +using Int32Encoder = TypedEncoder; +using Int64Encoder = TypedEncoder; +using Int96Encoder = TypedEncoder; +using FloatEncoder = TypedEncoder; +using DoubleEncoder = TypedEncoder; +using ByteArrayEncoder = TypedEncoder; +using FLBAEncoder = TypedEncoder; + +template +class TypedDecoder; + +class BooleanDecoder; +using Int32Decoder = TypedDecoder; +using Int64Decoder = TypedDecoder; +using Int96Decoder = TypedDecoder; +using FloatDecoder = TypedDecoder; +using DoubleDecoder = TypedDecoder; +using ByteArrayDecoder = TypedDecoder; +class FLBADecoder; + +template +struct EncodingTraits; + +template <> +struct EncodingTraits { + using Encoder = BooleanEncoder; + using Decoder = BooleanDecoder; + + using ArrowType = ::arrow::BooleanType; + using Accumulator = ::arrow::BooleanBuilder; + struct DictAccumulator {}; +}; + +template <> +struct EncodingTraits { + using Encoder = Int32Encoder; + using Decoder = Int32Decoder; + + using ArrowType = ::arrow::Int32Type; + using Accumulator = ::arrow::NumericBuilder<::arrow::Int32Type>; + using DictAccumulator = ::arrow::Dictionary32Builder<::arrow::Int32Type>; +}; + +template <> +struct EncodingTraits { + using Encoder = Int64Encoder; + using Decoder = Int64Decoder; + + using ArrowType = ::arrow::Int64Type; + using Accumulator = ::arrow::NumericBuilder<::arrow::Int64Type>; + using DictAccumulator = ::arrow::Dictionary32Builder<::arrow::Int64Type>; +}; + +template <> +struct EncodingTraits { + using Encoder = Int96Encoder; + using Decoder = Int96Decoder; + + struct Accumulator {}; + struct DictAccumulator {}; +}; + +template <> +struct EncodingTraits { + using Encoder = FloatEncoder; + using Decoder = FloatDecoder; + + using ArrowType = ::arrow::FloatType; + using Accumulator = ::arrow::NumericBuilder<::arrow::FloatType>; + using DictAccumulator = ::arrow::Dictionary32Builder<::arrow::FloatType>; +}; + +template <> +struct EncodingTraits { + using Encoder = DoubleEncoder; + using Decoder = DoubleDecoder; + + using ArrowType = ::arrow::DoubleType; + using Accumulator = ::arrow::NumericBuilder<::arrow::DoubleType>; + using DictAccumulator = ::arrow::Dictionary32Builder<::arrow::DoubleType>; +}; + +template <> +struct EncodingTraits { + using Encoder = ByteArrayEncoder; + using Decoder = ByteArrayDecoder; + + /// \brief Internal helper class for decoding BYTE_ARRAY data + /// + /// This class allows the caller to choose the concrete Arrow data type + /// by passing a corresponding `ArrayBuilder`. + /// Supported `ArrayBuilder` classes are `BinaryBuilder`, `LargeBinaryBuilder` + /// and `BinaryViewBuilder`. + /// If the builder is a `BinaryBuilder`, `chunks` can accumulate several + /// arrays as needed to work around the 32-bit offset limit. + struct Accumulator { + std::unique_ptr<::arrow::ArrayBuilder> builder; + std::vector> chunks; + }; + using DictAccumulator = ::arrow::Dictionary32Builder<::arrow::BinaryType>; +}; + +template <> +struct EncodingTraits { + using Encoder = FLBAEncoder; + using Decoder = FLBADecoder; + + using ArrowType = ::arrow::FixedSizeBinaryType; + using Accumulator = ::arrow::FixedSizeBinaryBuilder; + using DictAccumulator = ::arrow::Dictionary32Builder<::arrow::FixedSizeBinaryType>; +}; + +class ColumnDescriptor; + +// Untyped base for all encoders +class Encoder { + public: + virtual ~Encoder() = default; + + virtual int64_t EstimatedDataEncodedSize() = 0; + virtual std::shared_ptr FlushValues() = 0; + virtual Encoding::type encoding() const = 0; + + virtual void Put(const ::arrow::Array& values) = 0; + + // Report the number of bytes written to the encoder since the last report. + // It only works for BYTE_ARRAY type and throw for other types. + // This call is not idempotent since it resets the internal counter. + virtual int64_t ReportUnencodedDataBytes() = 0; + + virtual MemoryPool* memory_pool() const = 0; +}; + +// Base class for value encoders. Since encoders may or not have state (e.g., +// dictionary encoding) we use a class instance to maintain any state. +// +// Encode interfaces are internal, subject to change without deprecation. +template +class TypedEncoder : virtual public Encoder { + public: + using T = typename DType::c_type; + + using Encoder::Put; + + virtual void Put(const T* src, int num_values) = 0; + + virtual void Put(const std::vector& src, int num_values = -1); + + virtual void PutSpaced(const T* src, int num_values, const uint8_t* valid_bits, + int64_t valid_bits_offset) = 0; +}; + +template +void TypedEncoder::Put(const std::vector& src, int num_values) { + if (num_values == -1) { + num_values = static_cast(src.size()); + } + Put(src.data(), num_values); +} + +template <> +inline void TypedEncoder::Put(const std::vector& src, int num_values) { + // NOTE(wesm): This stub is here only to satisfy the compiler; it is + // overridden later with the actual implementation +} + +// Base class for dictionary encoders +template +class DictEncoder : virtual public TypedEncoder { + public: + /// Writes out any buffered indices to buffer preceded by the bit width of this data. + /// Returns the number of bytes written. + /// If the supplied buffer is not big enough, returns -1. + /// buffer must be preallocated with buffer_len bytes. Use EstimatedDataEncodedSize() + /// to size buffer. + virtual int WriteIndices(uint8_t* buffer, int buffer_len) = 0; + + virtual int dict_encoded_size() const = 0; + + virtual int bit_width() const = 0; + + /// Writes out the encoded dictionary to buffer. buffer must be preallocated to + /// dict_encoded_size() bytes. + virtual void WriteDict(uint8_t* buffer) const = 0; + + virtual int num_entries() const = 0; + + /// \brief EXPERIMENTAL: Append dictionary indices into the encoder. It is + /// assumed (without any boundschecking) that the indices reference + /// preexisting dictionary values + /// \param[in] indices the dictionary index values. Only Int32Array currently + /// supported + virtual void PutIndices(const ::arrow::Array& indices) = 0; + + /// \brief EXPERIMENTAL: Append dictionary into encoder, inserting indices + /// separately. Currently throws exception if the current dictionary memo is + /// non-empty + /// \param[in] values the dictionary values. Only valid for certain + /// Parquet/Arrow type combinations, like BYTE_ARRAY/BinaryArray + virtual void PutDictionary(const ::arrow::Array& values) = 0; +}; + +// ---------------------------------------------------------------------- +// Value decoding + +class Decoder { + public: + virtual ~Decoder() = default; + + // Sets the data for a new page. This will be called multiple times on the same + // decoder and should reset all internal state. + // + // `num_values` comes from the data page header, and may be greater than the number of + // physical values in the data buffer if there are some omitted (null) values. + // `len`, on the other hand, is the size in bytes of the data buffer and + // directly relates to the number of physical values. + virtual void SetData(int num_values, const uint8_t* data, int len) = 0; + + // Returns the number of values left (for the last call to SetData()). This is + // the number of values left in this page. + virtual int values_left() const = 0; + virtual Encoding::type encoding() const = 0; +}; + +template +class TypedDecoder : virtual public Decoder { + public: + using T = typename DType::c_type; + + /// \brief Decode values into a buffer + /// + /// Subclasses may override the more specialized Decode methods below. + /// + /// \param[in] buffer destination for decoded values + /// \param[in] max_values maximum number of values to decode + /// \return The number of values decoded. Should be identical to max_values except + /// at the end of the current data page. + virtual int Decode(T* buffer, int max_values) = 0; + + /// \brief Decode the values in this data page but leave spaces for null entries. + /// + /// \param[in] buffer destination for decoded values + /// \param[in] num_values size of the def_levels and buffer arrays including the number + /// of null slots + /// \param[in] null_count number of null slots + /// \param[in] valid_bits bitmap data indicating position of valid slots + /// \param[in] valid_bits_offset offset into valid_bits + /// \return The number of values decoded, including nulls. + virtual int DecodeSpaced(T* buffer, int num_values, int null_count, + const uint8_t* valid_bits, int64_t valid_bits_offset) = 0; + + /// \brief Decode into an ArrayBuilder or other accumulator + /// + /// \param[in] num_values number of values to decode, including null slots + /// \param[in] null_count number of null slots + /// \param[in] valid_bits validity bitmap + /// \param[in] valid_bits_offset bit offset to start into the validity bitmap + /// \param[in] out accumulator to decode into + /// + /// This function assumes the definition levels were already decoded + /// as a validity bitmap in the given `valid_bits`. `null_count` + /// is the number of 0s in `valid_bits`. + /// `valid_bits` must at least `valid_bits_offset + num_values` bits. + /// As a space optimization, it is allowed for `valid_bits` to be null + /// if `null_count` is zero. + /// This function throws a ParquetException if there are less than + /// `num_values - null_count` values left to decode. + /// + /// \return The number of non-null values decoded + virtual int DecodeArrow(int num_values, int null_count, const uint8_t* valid_bits, + int64_t valid_bits_offset, + typename EncodingTraits::Accumulator* out) = 0; + + /// \brief Decode into an ArrayBuilder or other accumulator ignoring nulls + /// + /// \param[in] num_values number of values to decode + /// \param[in] out accumulator to decode into + /// + /// \return The number of values decoded + int DecodeArrowNonNull(int num_values, + typename EncodingTraits::Accumulator* out) { + return DecodeArrow(num_values, 0, /*valid_bits=*/NULLPTR, 0, out); + } + + /// \brief Decode into a DictionaryBuilder + /// + /// This function assumes the definition levels were already decoded + /// as a validity bitmap in the given `valid_bits`. `null_count` + /// is the number of 0s in `valid_bits`. + /// As a space optimization, it is allowed for `valid_bits` to be null + /// if `null_count` is zero. + /// + /// \return number of values decoded + virtual int DecodeArrow(int num_values, int null_count, const uint8_t* valid_bits, + int64_t valid_bits_offset, + typename EncodingTraits::DictAccumulator* builder) = 0; + + /// \brief Decode into a DictionaryBuilder ignoring nulls + /// + /// \return number of values decoded + int DecodeArrowNonNull(int num_values, + typename EncodingTraits::DictAccumulator* builder) { + return DecodeArrow(num_values, 0, /*valid_bits=*/NULLPTR, 0, builder); + } +}; + +template +class DictDecoder : virtual public TypedDecoder { + public: + using T = typename DType::c_type; + + virtual void SetDict(TypedDecoder* dictionary) = 0; + + /// \brief Insert dictionary values into the Arrow dictionary builder's memo, + /// but do not append any indices + virtual void InsertDictionary(::arrow::ArrayBuilder* builder) = 0; + + /// \brief Decode only dictionary indices and append to dictionary + /// builder. The builder must have had the dictionary from this decoder + /// inserted already. + /// + /// \warning Remember to reset the builder each time the dict decoder is initialized + /// with a new dictionary page + virtual int DecodeIndicesSpaced(int num_values, int null_count, + const uint8_t* valid_bits, int64_t valid_bits_offset, + ::arrow::ArrayBuilder* builder) = 0; + + /// \brief Decode only dictionary indices (no nulls) + /// + /// \warning Remember to reset the builder each time the dict decoder is initialized + /// with a new dictionary page + virtual int DecodeIndices(int num_values, ::arrow::ArrayBuilder* builder) = 0; + + /// \brief Decode only dictionary indices (no nulls). Same as above + /// DecodeIndices but target is an array instead of a builder. + /// + /// \note API EXPERIMENTAL + virtual int DecodeIndices(int num_values, int32_t* indices) = 0; + + /// \brief Get dictionary. The reader will call this API when it encounters a + /// new dictionary. + /// + /// @param[out] dictionary The pointer to dictionary values. Dictionary is owned by + /// the decoder and is destroyed when the decoder is destroyed. + /// @param[out] dictionary_length The dictionary length. + /// + /// \note API EXPERIMENTAL + virtual void GetDictionary(const T** dictionary, int32_t* dictionary_length) = 0; +}; + +// ---------------------------------------------------------------------- +// TypedEncoder specializations, traits, and factory functions + +class BooleanDecoder : virtual public TypedDecoder { + public: + using TypedDecoder::Decode; + + /// \brief Decode and bit-pack values into a buffer + /// + /// \param[in] buffer destination for decoded values + /// This buffer will contain bit-packed values. If + /// max_values is not a multiple of 8, the trailing bits + /// of the last byte will be undefined. + /// \param[in] max_values max values to decode. + /// \return The number of values decoded. Should be identical to max_values except + /// at the end of the current data page. + virtual int Decode(uint8_t* buffer, int max_values) = 0; +}; + +class FLBADecoder : virtual public TypedDecoder { + public: + using TypedDecoder::DecodeSpaced; + + // TODO(wesm): As possible follow-up to PARQUET-1508, we should examine if + // there is value in adding specialized read methods for + // FIXED_LEN_BYTE_ARRAY. If only Decimal data can occur with this data type + // then perhaps not +}; + +PARQUET_EXPORT +std::unique_ptr MakeEncoder( + Type::type type_num, Encoding::type encoding, bool use_dictionary = false, + const ColumnDescriptor* descr = NULLPTR, + ::arrow::MemoryPool* pool = ::arrow::default_memory_pool()); + +template +std::unique_ptr::Encoder> MakeTypedEncoder( + Encoding::type encoding, bool use_dictionary = false, + const ColumnDescriptor* descr = NULLPTR, + ::arrow::MemoryPool* pool = ::arrow::default_memory_pool()) { + using OutType = typename EncodingTraits::Encoder; + std::unique_ptr base = + MakeEncoder(DType::type_num, encoding, use_dictionary, descr, pool); + return std::unique_ptr(dynamic_cast(base.release())); +} + +PARQUET_EXPORT +std::unique_ptr MakeDecoder( + Type::type type_num, Encoding::type encoding, const ColumnDescriptor* descr = NULLPTR, + ::arrow::MemoryPool* pool = ::arrow::default_memory_pool()); + +namespace detail { + +PARQUET_EXPORT +std::unique_ptr MakeDictDecoder(Type::type type_num, + const ColumnDescriptor* descr, + ::arrow::MemoryPool* pool); + +} // namespace detail + +template +std::unique_ptr> MakeDictDecoder( + const ColumnDescriptor* descr = NULLPTR, + ::arrow::MemoryPool* pool = ::arrow::default_memory_pool()) { + using OutType = DictDecoder; + auto decoder = detail::MakeDictDecoder(DType::type_num, descr, pool); + return std::unique_ptr(dynamic_cast(decoder.release())); +} + +template +std::unique_ptr::Decoder> MakeTypedDecoder( + Encoding::type encoding, const ColumnDescriptor* descr = NULLPTR, + ::arrow::MemoryPool* pool = ::arrow::default_memory_pool()) { + using OutType = typename EncodingTraits::Decoder; + std::unique_ptr base = MakeDecoder(DType::type_num, encoding, descr, pool); + return std::unique_ptr(dynamic_cast(base.release())); +} + +/// Return the list of supported encodings for the given physical type +/// +/// Only non-dictionary encodings are returned. +PARQUET_EXPORT +std::vector SupportedEncodings(Type::type physical_type); + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/crypto_factory.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/crypto_factory.h new file mode 100644 index 0000000000000000000000000000000000000000..97dc3930d0bc0056e53fdfc7cf403a43a11a1d81 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/crypto_factory.h @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "parquet/encryption/encryption.h" +#include "parquet/encryption/file_key_wrapper.h" +#include "parquet/encryption/key_toolkit.h" +#include "parquet/encryption/kms_client_factory.h" +#include "parquet/platform.h" + +namespace parquet::encryption { + +static constexpr ParquetCipher::type kDefaultEncryptionAlgorithm = + ParquetCipher::AES_GCM_V1; +static constexpr bool kDefaultPlaintextFooter = false; +static constexpr bool kDefaultDoubleWrapping = true; +static constexpr double kDefaultCacheLifetimeSeconds = 600; // 10 minutes +static constexpr bool kDefaultInternalKeyMaterial = true; +static constexpr bool kDefaultUniformEncryption = false; +static constexpr int32_t kDefaultDataKeyLengthBits = 128; + +struct PARQUET_EXPORT EncryptionConfiguration { + explicit EncryptionConfiguration(const std::string& footer_key) + : footer_key(footer_key) {} + + /// ID of the master key for footer encryption/signing + std::string footer_key; + + /// List of columns to encrypt, with column master key IDs (see HIVE-21848). + /// Format: "columnKeyID:colName,colName;columnKeyID:colName..." + /// Either + /// (1) column_keys must be set + /// or + /// (2) uniform_encryption must be set to true + /// If none of (1) and (2) are true, or if both are true, an exception will be + /// thrown. + std::string column_keys; + + /// Encrypt footer and all columns with the same encryption key. + bool uniform_encryption = kDefaultUniformEncryption; + + /// Parquet encryption algorithm. Can be "AES_GCM_V1" (default), or "AES_GCM_CTR_V1". + ParquetCipher::type encryption_algorithm = kDefaultEncryptionAlgorithm; + + /// Write files with plaintext footer. + /// The default is false - files are written with encrypted footer. + bool plaintext_footer = kDefaultPlaintextFooter; + + /// Use double wrapping - where data encryption keys (DEKs) are encrypted with key + /// encryption keys (KEKs), which in turn are encrypted with master keys. + /// The default is true. If set to false, use single wrapping - where DEKs are + /// encrypted directly with master keys. + bool double_wrapping = kDefaultDoubleWrapping; + + /// Lifetime of cached entities (key encryption keys, local wrapping keys, KMS client + /// objects). + /// The default is 600 (10 minutes). + double cache_lifetime_seconds = kDefaultCacheLifetimeSeconds; + + /// Store key material inside Parquet file footers; this mode doesn’t produce + /// additional files. By default, true. If set to false, key material is stored in + /// separate files in the same folder, which enables key rotation for immutable + /// Parquet files. + bool internal_key_material = kDefaultInternalKeyMaterial; + + /// Length of data encryption keys (DEKs), randomly generated by parquet key + /// management tools. Can be 128, 192 or 256 bits. + /// The default is 128 bits. + int32_t data_key_length_bits = kDefaultDataKeyLengthBits; +}; + +struct PARQUET_EXPORT DecryptionConfiguration { + /// Lifetime of cached entities (key encryption keys, local wrapping keys, KMS client + /// objects). + /// The default is 600 (10 minutes). + double cache_lifetime_seconds = kDefaultCacheLifetimeSeconds; +}; + +/// This is a core class, that translates the parameters of high level encryption (like +/// the names of encrypted columns, names of master keys, etc), into parameters of low +/// level encryption (like the key metadata, DEK, etc). A factory that produces the low +/// level FileEncryptionProperties and FileDecryptionProperties objects, from the high +/// level parameters. +class PARQUET_EXPORT CryptoFactory { + public: + /// a KmsClientFactory object must be registered via this method before calling any of + /// GetFileEncryptionProperties()/GetFileDecryptionProperties() methods. + void RegisterKmsClientFactory(std::shared_ptr kms_client_factory); + + /// Get the encryption properties for a Parquet file. + /// If external key material is used then a file system and path to the + /// parquet file must be provided. + std::shared_ptr GetFileEncryptionProperties( + const KmsConnectionConfig& kms_connection_config, + const EncryptionConfiguration& encryption_config, const std::string& file_path = "", + const std::shared_ptr<::arrow::fs::FileSystem>& file_system = NULLPTR); + + /// Get decryption properties for a Parquet file. + /// If external key material is used then a file system and path to the + /// parquet file must be provided. + std::shared_ptr GetFileDecryptionProperties( + const KmsConnectionConfig& kms_connection_config, + const DecryptionConfiguration& decryption_config, const std::string& file_path = "", + const std::shared_ptr<::arrow::fs::FileSystem>& file_system = NULLPTR); + + void RemoveCacheEntriesForToken(const std::string& access_token) { + key_toolkit_->RemoveCacheEntriesForToken(access_token); + } + + void RemoveCacheEntriesForAllTokens() { + key_toolkit_->RemoveCacheEntriesForAllTokens(); + } + + /// Rotates master encryption keys for a Parquet file that uses external key material. + /// In single wrapping mode, data encryption keys are decrypted with the old master keys + /// and then re-encrypted with new master keys. + /// In double wrapping mode, key encryption keys are decrypted with the old master keys + /// and then re-encrypted with new master keys. + /// This relies on the KMS supporting versioning, such that the old master key is + /// used when unwrapping a key, and the latest version is used when wrapping a key. + void RotateMasterKeys(const KmsConnectionConfig& kms_connection_config, + const std::string& parquet_file_path, + const std::shared_ptr<::arrow::fs::FileSystem>& file_system, + bool double_wrapping = kDefaultDoubleWrapping, + double cache_lifetime_seconds = kDefaultCacheLifetimeSeconds); + + private: + ColumnPathToEncryptionPropertiesMap GetColumnEncryptionProperties( + int dek_length, const std::string& column_keys, FileKeyWrapper* key_wrapper); + + /// Key utilities object for kms client initialization and cache control + std::shared_ptr key_toolkit_ = std::make_shared(); +}; + +} // namespace parquet::encryption diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/encryption.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/encryption.h new file mode 100644 index 0000000000000000000000000000000000000000..677882b8b8b11bcecaa0022dd03de3950772dc2b --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/encryption.h @@ -0,0 +1,434 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "arrow/util/secure_string.h" +#include "parquet/exception.h" +#include "parquet/schema.h" +#include "parquet/types.h" + +namespace parquet { + +static constexpr ParquetCipher::type kDefaultEncryptionAlgorithm = + ParquetCipher::AES_GCM_V1; +static constexpr int32_t kMaximalAadMetadataLength = 256; +static constexpr bool kDefaultEncryptedFooter = true; +static constexpr bool kDefaultCheckSignature = true; +static constexpr bool kDefaultAllowPlaintextFiles = false; +static constexpr int32_t kAadFileUniqueLength = 8; + +class ColumnDecryptionProperties; +using ColumnPathToDecryptionPropertiesMap = + std::map>; + +class ColumnEncryptionProperties; +using ColumnPathToEncryptionPropertiesMap = + std::map>; + +class PARQUET_EXPORT DecryptionKeyRetriever { + public: + /// \brief Retrieve a key. + virtual ::arrow::util::SecureString GetKey(const std::string& key_id) = 0; + + virtual ~DecryptionKeyRetriever() {} +}; + +/// Simple integer key retriever +class PARQUET_EXPORT IntegerKeyIdRetriever : public DecryptionKeyRetriever { + public: + void PutKey(uint32_t key_id, ::arrow::util::SecureString key); + + ::arrow::util::SecureString GetKey(const std::string& key_id_string) override { + // key_id_string is string but for IntegerKeyIdRetriever it encodes + // a native-endian 32 bit unsigned integer key_id + uint32_t key_id; + assert(key_id_string.size() == sizeof(key_id)); + memcpy(&key_id, key_id_string.data(), sizeof(key_id)); + + return key_map_.at(key_id); + } + + private: + std::map key_map_; +}; + +// Simple string key retriever +class PARQUET_EXPORT StringKeyIdRetriever : public DecryptionKeyRetriever { + public: + void PutKey(std::string key_id, ::arrow::util::SecureString key); + ::arrow::util::SecureString GetKey(const std::string& key_id) override; + + private: + std::map key_map_; +}; + +class PARQUET_EXPORT HiddenColumnException : public ParquetException { + public: + explicit HiddenColumnException(const std::string& columnPath) + : ParquetException(columnPath.c_str()) {} +}; + +class PARQUET_EXPORT KeyAccessDeniedException : public ParquetException { + public: + explicit KeyAccessDeniedException(const std::string& columnPath) + : ParquetException(columnPath.c_str()) {} +}; + +inline std::span str2span(const std::string& str) { + if (str.empty()) { + return {}; + } + + return {reinterpret_cast(str.data()), str.size()}; +} + +class PARQUET_EXPORT ColumnEncryptionProperties { + public: + class PARQUET_EXPORT Builder { + public: + PARQUET_DEPRECATED("name argument is ignored, use default constructor instead") + explicit Builder(const std::string& name) : encrypted_(true) {} + + PARQUET_DEPRECATED("path argument is ignored, use default constructor instead") + explicit Builder(const schema::ColumnPath& path) : encrypted_(true) {} + + Builder() = default; + + /// Set a column-specific key. + /// If key is not set on an encrypted column, the column will + /// be encrypted with the footer key. + /// keyBytes Key length must be either 16, 24 or 32 bytes. + /// Caller is responsible for wiping out the input key array. + Builder* key(::arrow::util::SecureString column_key); + + /// Set a key retrieval metadata. + /// use either key_metadata() or key_id(), not both + Builder* key_metadata(std::string key_metadata); + + /// A convenience function to set key metadata using a string id. + /// Set a key retrieval metadata (converted from String). + /// use either key_metadata() or key_id(), not both + /// key_id will be converted to metadata (UTF-8 array). + Builder* key_id(std::string key_id); + + std::shared_ptr build() { + return std::shared_ptr( + new ColumnEncryptionProperties(encrypted_, key_, key_metadata_)); + } + + private: + bool encrypted_ = true; + ::arrow::util::SecureString key_; + std::string key_metadata_; + }; + + bool is_encrypted() const { return encrypted_; } + bool is_encrypted_with_footer_key() const { return encrypted_with_footer_key_; } + const ::arrow::util::SecureString& key() const { return key_; } + const std::string& key_metadata() const { return key_metadata_; } + + static std::shared_ptr Unencrypted(); + static std::shared_ptr WithFooterKey(); + static std::shared_ptr WithColumnKey( + ::arrow::util::SecureString key, std::string key_metadata = ""); + + private: + bool encrypted_; + bool encrypted_with_footer_key_; + ::arrow::util::SecureString key_; + std::string key_metadata_; + explicit ColumnEncryptionProperties(bool encrypted, ::arrow::util::SecureString key, + std::string key_metadata); +}; + +class PARQUET_EXPORT ColumnDecryptionProperties { + public: + class PARQUET_EXPORT Builder { + public: + explicit Builder(std::string name) : column_path_(std::move(name)) {} + + explicit Builder(const schema::ColumnPath& path) : Builder(path.ToDotString()) {} + + /// Set an explicit column key. If applied on a file that contains + /// key metadata for this column the metadata will be ignored, + /// the column will be decrypted with this key. + /// key length must be either 16, 24 or 32 bytes. + Builder* key(::arrow::util::SecureString key); + + std::shared_ptr build(); + + private: + std::string column_path_; + ::arrow::util::SecureString key_; + }; + + const std::string& column_path() const { return column_path_; } + const ::arrow::util::SecureString& key() const { return key_; } + + private: + std::string column_path_; + ::arrow::util::SecureString key_; + + /// This class is only required for setting explicit column decryption keys - + /// to override key retriever (or to provide keys when key metadata and/or + /// key retriever are not available) + explicit ColumnDecryptionProperties(std::string column_path, + ::arrow::util::SecureString key); +}; + +class PARQUET_EXPORT AADPrefixVerifier { + public: + /// Verifies identity (AAD Prefix) of individual file, + /// or of file collection in a data set. + /// Throws exception if an AAD prefix is wrong. + /// In a data set, AAD Prefixes should be collected, + /// and then checked for missing files. + virtual void Verify(const std::string& aad_prefix) = 0; + virtual ~AADPrefixVerifier() {} +}; + +class PARQUET_EXPORT FileDecryptionProperties { + public: + class PARQUET_EXPORT Builder { + public: + Builder() { + check_plaintext_footer_integrity_ = kDefaultCheckSignature; + plaintext_files_allowed_ = kDefaultAllowPlaintextFiles; + } + + /// Set an explicit footer key. If applied on a file that contains + /// footer key metadata the metadata will be ignored, the footer + /// will be decrypted/verified with this key. + /// If explicit key is not set, footer key will be fetched from + /// key retriever. + /// With explicit keys or AAD prefix, new encryption properties object must be + /// created for each encrypted file. + /// Explicit encryption keys (footer and column) are cloned. + /// Upon completion of file reading, the cloned encryption keys in the properties + /// will be wiped out (array values set to 0). + /// Caller is responsible for wiping out the input key array. + /// param footerKey Key length must be either 16, 24 or 32 bytes. + Builder* footer_key(::arrow::util::SecureString footer_key); + + /// Set explicit column keys (decryption properties). + /// Its also possible to set a key retriever on this property object. + /// Upon file decryption, availability of explicit keys is checked before + /// invocation of the retriever callback. + /// If an explicit key is available for a footer or a column, + /// its key metadata will be ignored. + Builder* column_keys( + ColumnPathToDecryptionPropertiesMap column_decryption_properties); + + /// Set a key retriever callback. Its also possible to + /// set explicit footer or column keys on this file property object. + /// Upon file decryption, availability of explicit keys is checked before + /// invocation of the retriever callback. + /// If an explicit key is available for a footer or a column, + /// its key metadata will be ignored. + Builder* key_retriever(std::shared_ptr key_retriever); + + /// Skip integrity verification of plaintext footers. + /// If not called, integrity of plaintext footers will be checked in runtime, + /// and an exception will be thrown in the following situations: + /// - footer signing key is not available + /// (not passed, or not found by key retriever) + /// - footer content and signature don't match + Builder* disable_footer_signature_verification() { + check_plaintext_footer_integrity_ = false; + return this; + } + + /// Explicitly supply the file AAD prefix. + /// A must when a prefix is used for file encryption, but not stored in file. + /// If AAD prefix is stored in file, it will be compared to the explicitly + /// supplied value and an exception will be thrown if they differ. + Builder* aad_prefix(std::string aad_prefix); + + /// Set callback for verification of AAD Prefixes stored in file. + Builder* aad_prefix_verifier(std::shared_ptr aad_prefix_verifier); + + /// By default, reading plaintext (unencrypted) files is not + /// allowed when using a decryptor + /// - in order to detect files that were not encrypted by mistake. + /// However, the default behavior can be overridden by calling this method. + /// The caller should use then a different method to ensure encryption + /// of files with sensitive data. + Builder* plaintext_files_allowed() { + plaintext_files_allowed_ = true; + return this; + } + + std::shared_ptr build() { + return std::shared_ptr(new FileDecryptionProperties( + footer_key_, key_retriever_, check_plaintext_footer_integrity_, aad_prefix_, + aad_prefix_verifier_, column_decryption_properties_, plaintext_files_allowed_)); + } + + private: + ::arrow::util::SecureString footer_key_; + std::string aad_prefix_; + std::shared_ptr aad_prefix_verifier_; + ColumnPathToDecryptionPropertiesMap column_decryption_properties_; + + std::shared_ptr key_retriever_; + bool check_plaintext_footer_integrity_; + bool plaintext_files_allowed_; + }; + + const ::arrow::util::SecureString& column_key(const std::string& column_path) const; + + const ::arrow::util::SecureString& footer_key() const { return footer_key_; } + + const std::string& aad_prefix() const { return aad_prefix_; } + + const std::shared_ptr& key_retriever() const { + return key_retriever_; + } + + bool check_plaintext_footer_integrity() const { + return check_plaintext_footer_integrity_; + } + + bool plaintext_files_allowed() const { return plaintext_files_allowed_; } + + const std::shared_ptr& aad_prefix_verifier() const { + return aad_prefix_verifier_; + } + + private: + ::arrow::util::SecureString footer_key_; + std::string aad_prefix_; + std::shared_ptr aad_prefix_verifier_; + ColumnPathToDecryptionPropertiesMap column_decryption_properties_; + std::shared_ptr key_retriever_; + bool check_plaintext_footer_integrity_; + bool plaintext_files_allowed_; + + FileDecryptionProperties( + ::arrow::util::SecureString footer_key, + std::shared_ptr key_retriever, + bool check_plaintext_footer_integrity, std::string aad_prefix, + std::shared_ptr aad_prefix_verifier, + ColumnPathToDecryptionPropertiesMap column_decryption_properties, + bool plaintext_files_allowed); +}; + +class PARQUET_EXPORT FileEncryptionProperties { + public: + class PARQUET_EXPORT Builder { + public: + explicit Builder(::arrow::util::SecureString footer_key) + : parquet_cipher_(kDefaultEncryptionAlgorithm), + encrypted_footer_(kDefaultEncryptedFooter), + footer_key_(std::move(footer_key)) { + store_aad_prefix_in_file_ = false; + } + + /// Create files with plaintext footer. + /// If not called, the files will be created with encrypted footer (default). + Builder* set_plaintext_footer() { + encrypted_footer_ = false; + return this; + } + + /// Set encryption algorithm. + /// If not called, files will be encrypted with AES_GCM_V1 (default). + Builder* algorithm(ParquetCipher::type parquet_cipher) { + parquet_cipher_ = parquet_cipher; + return this; + } + + /// Set a key retrieval metadata (converted from String). + /// use either footer_key_metadata or footer_key_id, not both. + Builder* footer_key_id(std::string key_id); + + /// Set a key retrieval metadata. + /// use either footer_key_metadata or footer_key_id, not both. + Builder* footer_key_metadata(std::string footer_key_metadata); + + /// Set the file AAD Prefix. + Builder* aad_prefix(std::string aad_prefix); + + /// Skip storing AAD Prefix in file. + /// If not called, and if AAD Prefix is set, it will be stored. + Builder* disable_aad_prefix_storage(); + + /// Set the list of encrypted columns and their properties (keys etc). + /// If not called, all columns will be encrypted with the footer key. + /// If called, the file columns not in the list will be left unencrypted. + Builder* encrypted_columns(ColumnPathToEncryptionPropertiesMap encrypted_columns); + + std::shared_ptr build() { + return std::shared_ptr(new FileEncryptionProperties( + parquet_cipher_, footer_key_, footer_key_metadata_, encrypted_footer_, + aad_prefix_, store_aad_prefix_in_file_, encrypted_columns_)); + } + + private: + ParquetCipher::type parquet_cipher_; + bool encrypted_footer_; + ::arrow::util::SecureString footer_key_; + std::string footer_key_metadata_; + + std::string aad_prefix_; + bool store_aad_prefix_in_file_; + ColumnPathToEncryptionPropertiesMap encrypted_columns_; + }; + + bool encrypted_footer() const { return encrypted_footer_; } + + EncryptionAlgorithm algorithm() const { return algorithm_; } + + const ::arrow::util::SecureString& footer_key() const { return footer_key_; } + + const std::string& footer_key_metadata() const { return footer_key_metadata_; } + + const std::string& file_aad() const { return file_aad_; } + + std::shared_ptr column_encryption_properties( + const std::string& column_path); + + const ColumnPathToEncryptionPropertiesMap& encrypted_columns() const { + return encrypted_columns_; + } + + private: + EncryptionAlgorithm algorithm_; + ::arrow::util::SecureString footer_key_; + std::string footer_key_metadata_; + bool encrypted_footer_; + std::string file_aad_; + std::string aad_prefix_; + bool store_aad_prefix_in_file_; + ColumnPathToEncryptionPropertiesMap encrypted_columns_; + + FileEncryptionProperties(ParquetCipher::type cipher, + ::arrow::util::SecureString footer_key, + std::string footer_key_metadata, bool encrypted_footer, + std::string aad_prefix, bool store_aad_prefix_in_file, + ColumnPathToEncryptionPropertiesMap encrypted_columns); +}; + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_key_material_store.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_key_material_store.h new file mode 100644 index 0000000000000000000000000000000000000000..185d9df90a88fc5d4588100b23eea23f81c733d0 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_key_material_store.h @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "arrow/filesystem/filesystem.h" +#include "parquet/platform.h" + +namespace parquet::encryption { + +/// Stores encryption key material outside the Parquet file, for example in a separate +/// small file in the same folder. This is important for “key rotation”, when MEKs have to +/// be changed (if compromised; or periodically, just in case) - without modifying the +/// Parquet files (often immutable). +class PARQUET_EXPORT FileKeyMaterialStore { + public: + /// Add key material for one encryption key. + virtual void AddKeyMaterial(std::string key_id_in_file, std::string key_material) = 0; + + /// Get key material + virtual std::string GetKeyMaterial(std::string key_id_in_file) = 0; + + /// After key material was added for all keys in the given Parquet file, + /// save material in persistent store. + virtual void SaveMaterial() = 0; + + /// Remove key material from persistent store. Used in key rotation. + virtual void RemoveMaterial() = 0; + + /// Move key material to another store. Used in key rotation. + virtual void MoveMaterialTo(std::shared_ptr target_key_store) = 0; + + /// Returns the Set of all key IDs in this store (for the given Parquet file) + virtual std::vector GetKeyIDSet() = 0; + + virtual ~FileKeyMaterialStore() {} +}; + +} // namespace parquet::encryption diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_key_unwrapper.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_key_unwrapper.h new file mode 100644 index 0000000000000000000000000000000000000000..3fb26bf02f7e45c4ab93d6efa7d60b056e3662a4 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_key_unwrapper.h @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "arrow/util/concurrent_map.h" +#include "arrow/util/secure_string.h" + +#include "parquet/encryption/encryption.h" +#include "parquet/encryption/file_system_key_material_store.h" +#include "parquet/encryption/key_material.h" +#include "parquet/encryption/key_toolkit.h" +#include "parquet/encryption/key_toolkit_internal.h" +#include "parquet/encryption/kms_client.h" +#include "parquet/platform.h" + +namespace parquet::encryption { + +// This class will retrieve the key from "key metadata", following these steps: +// 1. Parse "key metadata" (see structure in KeyMetadata class). +// 2. Retrieve "key material" which can be stored inside or outside "key metadata". +// 3. Unwrap the "data encryption key" from "key material". There are 2 modes: +// 3.1. single wrapping: decrypt the wrapped "data encryption key" directly with "master +// encryption key" 3.2. double wrapping: 2 steps: 3.2.1. "key encryption key" is decrypted +// with "master encryption key" 3.2.2. "data encryption key" is decrypted with the above +// "key encryption key" +class PARQUET_EXPORT FileKeyUnwrapper : public DecryptionKeyRetriever { + public: + /// key_toolkit and kms_connection_config is to get KmsClient from cache or create + /// KmsClient if it's not in the cache yet. cache_entry_lifetime_seconds is life time of + /// KmsClient in the cache. + /// If the file uses external key material then the Parquet file path and file + /// system must be specified. + FileKeyUnwrapper(std::shared_ptr key_toolkit, + const KmsConnectionConfig& kms_connection_config, + double cache_lifetime_seconds, const std::string& file_path = "", + const std::shared_ptr<::arrow::fs::FileSystem>& file_system = NULLPTR); + + /// Constructor overload that takes a raw pointer to the KeyToolkit + FileKeyUnwrapper(KeyToolkit* key_toolkit, + const KmsConnectionConfig& kms_connection_config, + double cache_lifetime_seconds, const std::string& file_path = "", + const std::shared_ptr<::arrow::fs::FileSystem>& file_system = NULLPTR); + + /// Constructor overload that takes a raw pointer to the KeyToolkit and + /// accepts an existing key_material_store rather than using + /// the file path and file system to create one when needed. + FileKeyUnwrapper(KeyToolkit* key_toolkit, + const KmsConnectionConfig& kms_connection_config, + double cache_lifetime_seconds, + std::shared_ptr key_material_store); + + /// Get the data key from key metadata + ::arrow::util::SecureString GetKey(const std::string& key_metadata_bytes) override; + + /// Get the data key along with the master key id from key material + KeyWithMasterId GetDataEncryptionKey(const KeyMaterial& key_material); + + private: + FileKeyUnwrapper(std::shared_ptr key_toolkit_owner, KeyToolkit* key_toolkit, + const KmsConnectionConfig& kms_connection_config, + double cache_lifetime_seconds, + std::shared_ptr key_material_store, + const std::string& file_path, + const std::shared_ptr<::arrow::fs::FileSystem>& file_system); + + std::shared_ptr GetKmsClientFromConfigOrKeyMaterial( + const KeyMaterial& key_material); + + /// A map of Key Encryption Key (KEK) ID -> KEK bytes, for the current token + std::shared_ptr<::arrow::util::ConcurrentMap> + kek_per_kek_id_; + std::shared_ptr key_toolkit_owner_; + KeyToolkit* key_toolkit_; + KmsConnectionConfig kms_connection_config_; + const double cache_entry_lifetime_seconds_; + std::shared_ptr key_material_store_; + const std::string file_path_; + std::shared_ptr<::arrow::fs::FileSystem> file_system_; +}; + +} // namespace parquet::encryption diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_key_wrapper.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_key_wrapper.h new file mode 100644 index 0000000000000000000000000000000000000000..b0ddbd5ea630d6ef25ec5405875b1817ef5f6c88 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_key_wrapper.h @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "arrow/util/concurrent_map.h" + +#include "parquet/encryption/file_key_material_store.h" +#include "parquet/encryption/key_encryption_key.h" +#include "parquet/encryption/key_toolkit.h" +#include "parquet/encryption/kms_client.h" +#include "parquet/platform.h" + +namespace parquet::encryption { + +// This class will generate "key metadata" from "data encryption key" and "master key", +// following these steps: +// 1. Wrap "data encryption key". There are 2 modes: +// 1.1. single wrapping: encrypt "data encryption key" directly with "master encryption +// key" +// 1.2. double wrapping: 2 steps: +// 1.2.1. "key encryption key" is randomized (see KeyEncryptionKey class) +// 1.2.2. "data encryption key" is encrypted with the above "key encryption key" +// 2. Create "key material" (see structure in KeyMaterial class) +// 3. Create "key metadata" with "key material" inside or a reference to outside "key +// material" (see structure in KeyMetadata class). +class PARQUET_EXPORT FileKeyWrapper { + public: + static constexpr int kKeyEncryptionKeyLength = 16; + static constexpr int kKeyEncryptionKeyIdLength = 16; + + /// key_toolkit and kms_connection_config is to get KmsClient from the cache or create + /// KmsClient if it's not in the cache yet. cache_entry_lifetime_seconds is life time of + /// KmsClient in the cache. key_material_store is to store "key material" outside + /// parquet file, NULL if "key material" is stored inside parquet file. + FileKeyWrapper(KeyToolkit* key_toolkit, + const KmsConnectionConfig& kms_connection_config, + std::shared_ptr key_material_store, + double cache_entry_lifetime_seconds, bool double_wrapping); + + /// Creates key_metadata field for a given data key, via wrapping the key with the + /// master key. + /// When external key material is used, an identifier is usually generated automatically + /// but may be specified explicitly to support key rotation, + /// which requires keeping the same identifiers. + std::string GetEncryptionKeyMetadata(const ::arrow::util::SecureString& data_key, + const std::string& master_key_id, + bool is_footer_key, + std::string key_id_in_file = ""); + + private: + KeyEncryptionKey CreateKeyEncryptionKey(const std::string& master_key_id); + + /// A map of Master Encryption Key ID -> KeyEncryptionKey, for the current token + std::shared_ptr<::arrow::util::ConcurrentMap> + kek_per_master_key_id_; + + std::shared_ptr kms_client_; + KmsConnectionConfig kms_connection_config_; + std::shared_ptr key_material_store_; + const double cache_entry_lifetime_seconds_; + const bool double_wrapping_; + uint16_t key_counter_; +}; + +} // namespace parquet::encryption diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_system_key_material_store.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_system_key_material_store.h new file mode 100644 index 0000000000000000000000000000000000000000..1f35e87bde790f97502cd04bf6d2ac04fbd3adeb --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_system_key_material_store.h @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "arrow/filesystem/filesystem.h" + +#include "parquet/encryption/file_key_material_store.h" +#include "parquet/exception.h" + +namespace parquet::encryption { + +/// A FileKeyMaterialStore that stores key material in a file system file in the same +/// folder as the Parquet file. +class PARQUET_EXPORT FileSystemKeyMaterialStore : public FileKeyMaterialStore { + public: + static constexpr const char kKeyMaterialFilePrefix[] = "_KEY_MATERIAL_FOR_"; + static constexpr const char kTempFilePrefix[] = "_TMP"; + static constexpr const char kKeyMaterialFileSuffix[] = ".json"; + + FileSystemKeyMaterialStore() {} + FileSystemKeyMaterialStore(std::string key_material_file_path, + std::shared_ptr<::arrow::fs::FileSystem> file_system); + + /// Creates a new file system key material store for a parquet file. + /// When use_tmp_prefix is true, files are saved with an extra _TMP prefix so they don't + /// conflict with existing external material files. This is useful during key rotation + /// so that temporary key material files can be created while using the existing key + /// material, before moving the key material to the non-temporary location. + static std::shared_ptr Make( + std::string parquet_file_path, std::shared_ptr<::arrow::fs::FileSystem> file_system, + bool use_tmp_prefix); + + /// Add key material for one encryption key. + void AddKeyMaterial(std::string key_id_in_file, std::string key_material) { + key_material_map_.emplace(std::move(key_id_in_file), std::move(key_material)); + } + + /// Get key material + std::string GetKeyMaterial(std::string key_id_in_file) { + if (key_material_map_.empty()) { + LoadKeyMaterialMap(); + } + auto found = key_material_map_.find(key_id_in_file); + if (found == key_material_map_.end()) { + throw ParquetException("Invalid key id"); + } + return found->second; + } + + /// After key material was added for all keys in the given Parquet file, + /// save material in persistent store. + void SaveMaterial(); + + /// Remove key material from persistent store. Used in key rotation. + void RemoveMaterial(); + + /// Move key material to another store. Used in key rotation. + void MoveMaterialTo(std::shared_ptr target_key_store); + + /// Returns the Set of all key IDs in this store (for the given Parquet file) + std::vector GetKeyIDSet(); + + private: + std::string GetStorageFilePath() { return key_material_file_path_; } + + std::string BuildKeyMaterialMapJson(); + void LoadKeyMaterialMap(); + std::string key_material_file_path_; + std::shared_ptr<::arrow::fs::FileSystem> file_system_; + /// Maps ID of a key in Parquet file and key material + std::unordered_map key_material_map_; +}; + +} // namespace parquet::encryption diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_encryption_key.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_encryption_key.h new file mode 100644 index 0000000000000000000000000000000000000000..d53fc25f2ef6f6e777840166c016eaeccaae1aa1 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_encryption_key.h @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "arrow/util/base64.h" +#include "arrow/util/secure_string.h" + +namespace parquet::encryption { + +// In the double wrapping mode, each "data encryption key" (DEK) is encrypted with a “key +// encryption key” (KEK), that in turn is encrypted with a "master encryption key" (MEK). +// In a writer process, a random KEK is generated for each MEK ID, and cached in a map. This allows to perform an interaction with a KMS server only once for each +// MEK, in order to wrap its KEK. "Data encryption key" (DEK) wrapping is performed +// locally, and does not involve an interaction with a KMS server. +class KeyEncryptionKey { + public: + KeyEncryptionKey(::arrow::util::SecureString kek_bytes, std::string kek_id, + std::string encoded_wrapped_kek) + : kek_bytes_(std::move(kek_bytes)), + kek_id_(std::move(kek_id)), + encoded_kek_id_(::arrow::util::base64_encode(kek_id_)), + encoded_wrapped_kek_(std::move(encoded_wrapped_kek)) {} + + const ::arrow::util::SecureString& kek_bytes() const { return kek_bytes_; } + + const std::string& kek_id() const { return kek_id_; } + + const std::string& encoded_kek_id() const { return encoded_kek_id_; } + + const std::string& encoded_wrapped_kek() const { return encoded_wrapped_kek_; } + + private: + ::arrow::util::SecureString kek_bytes_; + std::string kek_id_; + std::string encoded_kek_id_; + std::string encoded_wrapped_kek_; +}; + +} // namespace parquet::encryption diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_material.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_material.h new file mode 100644 index 0000000000000000000000000000000000000000..8b4f91247329c7f8bec3bbc74ec8b1dfa490495f --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_material.h @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "parquet/platform.h" + +namespace arrow { +namespace json { +namespace internal { +class ObjectParser; +} // namespace internal +} // namespace json +} // namespace arrow + +namespace parquet::encryption { + +// KeyMaterial class represents the "key material", keeping the information that allows +// readers to recover an encryption key (see description of the KeyMetadata class). The +// keytools package (PARQUET-1373) implements the "envelope encryption" pattern, in a +// "single wrapping" or "double wrapping" mode. In the single wrapping mode, the key +// material is generated by encrypting the "data encryption key" (DEK) by a "master key". +// In the double wrapping mode, the key material is generated by encrypting the DEK by a +// "key encryption key" (KEK), that in turn is encrypted by a "master key". +// +// Key material is kept in a flat json object, with the following fields: +// 1. "keyMaterialType" - a String, with the type of key material. In the current +// version, only one value is allowed - "PKMT1" (stands +// for "parquet key management tools, version 1"). For external key material storage, +// this field is written in both "key metadata" and "key material" jsons. For internal +// key material storage, this field is written only once in the common json. +// 2. "isFooterKey" - a boolean. If true, means that the material belongs to a file footer +// key, and keeps additional information (such as +// KMS instance ID and URL). If false, means that the material belongs to a column +// key. +// 3. "kmsInstanceID" - a String, with the KMS Instance ID. Written only in footer key +// material. +// 4. "kmsInstanceURL" - a String, with the KMS Instance URL. Written only in footer key +// material. +// 5. "masterKeyID" - a String, with the ID of the master key used to generate the +// material. +// 6. "wrappedDEK" - a String, with the wrapped DEK (base64 encoding). +// 7. "doubleWrapping" - a boolean. If true, means that the material was generated in +// double wrapping mode. +// If false - in single wrapping mode. +// 8. "keyEncryptionKeyID" - a String, with the ID of the KEK used to generate the +// material. Written only in double wrapping mode. +// 9. "wrappedKEK" - a String, with the wrapped KEK (base64 encoding). Written only in +// double wrapping mode. +class PARQUET_EXPORT KeyMaterial { + public: + // these fields are defined in a specification and should never be changed + static constexpr const char kKeyMaterialTypeField[] = "keyMaterialType"; + static constexpr const char kKeyMaterialType1[] = "PKMT1"; + + static constexpr const char kFooterKeyIdInFile[] = "footerKey"; + static constexpr const char kColumnKeyIdInFilePrefix[] = "columnKey"; + + static constexpr const char kIsFooterKeyField[] = "isFooterKey"; + static constexpr const char kDoubleWrappingField[] = "doubleWrapping"; + static constexpr const char kKmsInstanceIdField[] = "kmsInstanceID"; + static constexpr const char kKmsInstanceUrlField[] = "kmsInstanceURL"; + static constexpr const char kMasterKeyIdField[] = "masterKeyID"; + static constexpr const char kWrappedDataEncryptionKeyField[] = "wrappedDEK"; + static constexpr const char kKeyEncryptionKeyIdField[] = "keyEncryptionKeyID"; + static constexpr const char kWrappedKeyEncryptionKeyField[] = "wrappedKEK"; + + public: + KeyMaterial() = default; + + static KeyMaterial Parse(const std::string& key_material_string); + + static KeyMaterial Parse( + const ::arrow::json::internal::ObjectParser* key_material_json); + + /// This method returns a json string that will be stored either inside a parquet file + /// or in a key material store outside the parquet file. + static std::string SerializeToJson(bool is_footer_key, + const std::string& kms_instance_id, + const std::string& kms_instance_url, + const std::string& master_key_id, + bool is_double_wrapped, const std::string& kek_id, + const std::string& encoded_wrapped_kek, + const std::string& encoded_wrapped_dek, + bool is_internal_storage); + + bool is_footer_key() const { return is_footer_key_; } + bool is_double_wrapped() const { return is_double_wrapped_; } + const std::string& master_key_id() const { return master_key_id_; } + const std::string& wrapped_dek() const { return encoded_wrapped_dek_; } + const std::string& kek_id() const { return kek_id_; } + const std::string& wrapped_kek() const { return encoded_wrapped_kek_; } + const std::string& kms_instance_id() const { return kms_instance_id_; } + const std::string& kms_instance_url() const { return kms_instance_url_; } + + private: + KeyMaterial(bool is_footer_key, const std::string& kms_instance_id, + const std::string& kms_instance_url, const std::string& master_key_id, + bool is_double_wrapped, const std::string& kek_id, + const std::string& encoded_wrapped_kek, + const std::string& encoded_wrapped_dek); + + bool is_footer_key_; + std::string kms_instance_id_; + std::string kms_instance_url_; + std::string master_key_id_; + bool is_double_wrapped_; + std::string kek_id_; + std::string encoded_wrapped_kek_; + std::string encoded_wrapped_dek_; +}; + +} // namespace parquet::encryption diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_metadata.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_metadata.h new file mode 100644 index 0000000000000000000000000000000000000000..78a0acc7503e15d4595677ef9da28cabdc902de9 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_metadata.h @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "parquet/encryption/key_material.h" +#include "parquet/exception.h" +#include "parquet/platform.h" + +namespace parquet::encryption { + +// Parquet encryption specification defines "key metadata" as an arbitrary byte array, +// generated by file writers for each encryption key, and passed to the low level API for +// storage in the file footer. The "key metadata" field is made available to file readers +// to enable recovery of the key. This interface can be utilized for implementation +// of any key management scheme. +// +// The keytools package (PARQUET-1373) implements one approach, of many possible, to key +// management and to generation of the "key metadata" fields. This approach, based on the +// "envelope encryption" pattern, allows integration with KMS servers. It keeps the actual +// material, required to recover a key, in a "key material" object (see the KeyMaterial +// class for details). This class is implemented to support version 1 of the parquet key +// management tools specification. +// +// KeyMetadata writes (and reads) the "key metadata" field as a flat json object, +// with the following fields: +// 1. "keyMaterialType" - a String, with the type of key material. +// 2. "internalStorage" - a boolean. If true, means that "key material" is kept inside the +// "key metadata" field. If false, "key material" is kept externally (outside Parquet +// files) - in this case, "key metadata" keeps a reference to the external "key material". +// 3. "keyReference" - a String, with the reference to the external "key material". +// Written only if internalStorage is false. +// +// If internalStorage is true, "key material" is a part of "key metadata", and the json +// keeps additional fields, described in the KeyMaterial class. +class PARQUET_EXPORT KeyMetadata { + public: + static constexpr const char kKeyMaterialInternalStorageField[] = "internalStorage"; + static constexpr const char kKeyReferenceField[] = "keyReference"; + + /// key_metadata_bytes is the key metadata field stored in the parquet file, + /// in the serialized json object format. + static KeyMetadata Parse(const std::string& key_metadata_bytes); + + static std::string CreateSerializedForExternalMaterial( + const std::string& key_reference); + + bool key_material_stored_internally() const { return is_internal_storage_; } + + const KeyMaterial& key_material() const { + if (!is_internal_storage_) { + throw ParquetException("key material is stored externally."); + } + return ::std::get(key_material_or_reference_); + } + + const std::string& key_reference() const { + if (is_internal_storage_) { + throw ParquetException("key material is stored internally."); + } + return ::std::get(key_material_or_reference_); + } + + private: + explicit KeyMetadata(const KeyMaterial& key_material); + explicit KeyMetadata(const std::string& key_reference); + + bool is_internal_storage_; + /// If is_internal_storage_ is true, KeyMaterial is set, + /// else a string referencing to an outside "key material" is set. + ::std::variant key_material_or_reference_; +}; + +} // namespace parquet::encryption diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_toolkit.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_toolkit.h new file mode 100644 index 0000000000000000000000000000000000000000..74d7731e8c4b8edc7ac93bde2919e0d9528897ab --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_toolkit.h @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "parquet/encryption/key_encryption_key.h" +#include "parquet/encryption/kms_client.h" +#include "parquet/encryption/kms_client_factory.h" +#include "parquet/encryption/two_level_cache_with_expiration.h" +#include "parquet/platform.h" + +namespace parquet::encryption { + +static constexpr uint64_t kCacheCleanPeriodForKeyRotation = 60 * 60; // 1 hour + +// KeyToolkit is a utility that keeps various tools for key management (such as key +// rotation, kms client instantiation, cache control, etc), plus a number of auxiliary +// classes for internal use. +class PARQUET_EXPORT KeyToolkit { + public: + KeyToolkit() { last_cache_clean_for_key_rotation_time_ = {}; } + + /// KMS client two level cache: token -> KMSInstanceId -> KmsClient + TwoLevelCacheWithExpiration>& kms_client_cache_per_token() { + return kms_client_cache_; + } + /// Key encryption key two level cache for wrapping: token -> MasterEncryptionKeyId -> + /// KeyEncryptionKey + TwoLevelCacheWithExpiration& kek_write_cache_per_token() { + return key_encryption_key_write_cache_; + } + + /// Key encryption key two level cache for unwrapping: token -> KeyEncryptionKeyId -> + /// KeyEncryptionKeyBytes + TwoLevelCacheWithExpiration<::arrow::util::SecureString>& kek_read_cache_per_token() { + return key_encryption_key_read_cache_; + } + + std::shared_ptr GetKmsClient( + const KmsConnectionConfig& kms_connection_config, double cache_entry_lifetime_ms); + + /// Flush any caches that are tied to the (compromised) access_token + void RemoveCacheEntriesForToken(const std::string& access_token); + + void RemoveCacheEntriesForAllTokens(); + + void RegisterKmsClientFactory(std::shared_ptr kms_client_factory) { + if (kms_client_factory_ != NULLPTR) { + throw ParquetException("KMS client factory has already been registered."); + } + kms_client_factory_ = std::move(kms_client_factory); + } + + /// Key rotation. In the single wrapping mode, decrypts data keys with old master keys, + /// then encrypts them with new master keys. In the double wrapping mode, decrypts KEKs + /// (key encryption keys) with old master keys, generates new KEKs and encrypts them + /// with new master keys. Works only if key material is not stored internally in file + /// footers. Not supported in local key wrapping mode. Method can be run by multiple + /// threads, but each thread must work on different files. + void RotateMasterKeys(const KmsConnectionConfig& kms_connection_config, + const std::string& parquet_file_path, + const std::shared_ptr<::arrow::fs::FileSystem>& file_system, + bool double_wrapping, double cache_lifetime_seconds); + + private: + TwoLevelCacheWithExpiration> kms_client_cache_; + TwoLevelCacheWithExpiration key_encryption_key_write_cache_; + TwoLevelCacheWithExpiration<::arrow::util::SecureString> key_encryption_key_read_cache_; + std::shared_ptr kms_client_factory_; + mutable ::arrow::util::Mutex last_cache_clean_for_key_rotation_time_mutex_; + internal::TimePoint last_cache_clean_for_key_rotation_time_; +}; + +// "data encryption key" and "master key identifier" are paired together as output when +// parsing from "key material" +class PARQUET_EXPORT KeyWithMasterId { + public: + KeyWithMasterId(::arrow::util::SecureString key_bytes, std::string master_id) + : key_bytes_(std::move(key_bytes)), master_id_(std::move(master_id)) {} + + const ::arrow::util::SecureString& data_key() const { return key_bytes_; } + const std::string& master_id() const { return master_id_; } + + private: + ::arrow::util::SecureString key_bytes_; + std::string master_id_; +}; + +} // namespace parquet::encryption diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/kms_client.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/kms_client.h new file mode 100644 index 0000000000000000000000000000000000000000..519638cf22f4d2a9e970d57e3363f51b6596d0bd --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/kms_client.h @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "arrow/util/mutex.h" +#include "arrow/util/secure_string.h" + +#include "parquet/exception.h" +#include "parquet/platform.h" + +namespace parquet::encryption { + +/// This class wraps the key access token of a KMS server. If your token changes over +/// time, you should keep the reference to the KeyAccessToken object and call Refresh() +/// method every time you have a new token. +class PARQUET_EXPORT KeyAccessToken { + public: + KeyAccessToken() = default; + + explicit KeyAccessToken(const std::string value) : value_(value) {} + + void Refresh(const std::string& new_value) { + auto lock = mutex_.Lock(); + value_ = new_value; + } + + const std::string& value() const { + auto lock = mutex_.Lock(); + return value_; + } + + private: + std::string value_; + mutable ::arrow::util::Mutex mutex_; +}; + +struct PARQUET_EXPORT KmsConnectionConfig { + std::string kms_instance_id; + std::string kms_instance_url; + /// If the access token is changed in the future, you should keep a reference to + /// this object and call Refresh() on it whenever there is a new access token. + std::shared_ptr refreshable_key_access_token; + std::unordered_map custom_kms_conf; + + KmsConnectionConfig(); + + const std::string& key_access_token() const { + if (refreshable_key_access_token == NULLPTR || + refreshable_key_access_token->value().empty()) { + throw ParquetException("key access token is not set!"); + } + return refreshable_key_access_token->value(); + } + + void SetDefaultIfEmpty(); +}; + +class PARQUET_EXPORT KmsClient { + public: + static constexpr const char kKmsInstanceIdDefault[] = "DEFAULT"; + static constexpr const char kKmsInstanceUrlDefault[] = "DEFAULT"; + static constexpr const char kKeyAccessTokenDefault[] = "DEFAULT"; + + /// \brief Wraps a key. + /// + /// Encrypts it with the master key, encodes the result + /// and potentially adds a KMS-specific metadata. + virtual std::string WrapKey(const ::arrow::util::SecureString& key_bytes, + const std::string& master_key_identifier) = 0; + + /// \brief Decrypts (unwraps) a key with the master key. + virtual ::arrow::util::SecureString UnwrapKey( + const std::string& wrapped_key, const std::string& master_key_identifier) = 0; + + virtual ~KmsClient() {} +}; + +} // namespace parquet::encryption diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/kms_client_factory.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/kms_client_factory.h new file mode 100644 index 0000000000000000000000000000000000000000..fd590eca122da0853fee666ab4bc0dd9078ddfbc --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/kms_client_factory.h @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "parquet/encryption/kms_client.h" +#include "parquet/platform.h" + +namespace parquet::encryption { + +class PARQUET_EXPORT KmsClientFactory { + public: + explicit KmsClientFactory(bool wrap_locally = false) : wrap_locally_(wrap_locally) {} + + virtual ~KmsClientFactory() = default; + + virtual std::shared_ptr CreateKmsClient( + const KmsConnectionConfig& kms_connection_config) = 0; + + protected: + bool wrap_locally_; +}; + +} // namespace parquet::encryption diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/local_wrap_kms_client.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/local_wrap_kms_client.h new file mode 100644 index 0000000000000000000000000000000000000000..924fb3bc03fc3cbef450707d24356bdf4505f35d --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/local_wrap_kms_client.h @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "arrow/util/concurrent_map.h" + +#include "parquet/encryption/kms_client.h" +#include "parquet/platform.h" + +namespace parquet::encryption { + +/// This class supports local wrapping mode, master keys will be fetched from the KMS +/// server and used to encrypt other keys (data encryption keys or key encryption keys). +class PARQUET_EXPORT LocalWrapKmsClient : public KmsClient { + public: + static constexpr const char kLocalWrapNoKeyVersion[] = "NO_VERSION"; + + explicit LocalWrapKmsClient(const KmsConnectionConfig& kms_connection_config); + + std::string WrapKey(const ::arrow::util::SecureString& key_bytes, + const std::string& master_key_identifier) override; + + ::arrow::util::SecureString UnwrapKey( + const std::string& wrapped_key, const std::string& master_key_identifier) override; + + protected: + /// Get master key from the remote KMS server. + /// Note: this function might be called by multiple threads + virtual const ::arrow::util::SecureString& GetMasterKeyFromServer( + const std::string& master_key_identifier) = 0; + + private: + /// KMS systems wrap keys by encrypting them by master keys, and attaching additional + /// information (such as the version number of the masker key) to the result of + /// encryption. The master key version is required in key rotation. Currently, the + /// local wrapping mode does not support key rotation (because not all KMS systems allow + /// to fetch a master key by its ID and version number). Still, the local wrapping mode + /// adds a placeholder for the master key version, that will enable support for key + /// rotation in this mode in the future, with appropriate KMS systems. This will also + /// enable backward compatibility, where future readers will be able to extract master + /// key version in the files written by the current code. + /// + /// LocalKeyWrap class writes (and reads) the "key wrap" as a flat json with the + /// following fields: + /// 1. "masterKeyVersion" - a String, with the master key version. In the current + /// version, only one value is allowed - "NO_VERSION". + /// 2. "encryptedKey" - a String, with the key encrypted by the master key + /// (base64-encoded). + class LocalKeyWrap { + public: + static constexpr const char kLocalWrapKeyVersionField[] = "masterKeyVersion"; + static constexpr const char kLocalWrapEncryptedKeyField[] = "encryptedKey"; + + LocalKeyWrap(std::string master_key_version, std::string encrypted_encoded_key); + + static std::string CreateSerialized(const std::string& encrypted_encoded_key); + + static LocalKeyWrap Parse(const std::string& wrapped_key); + + const std::string& master_key_version() const { return master_key_version_; } + + const std::string& encrypted_encoded_key() const { return encrypted_encoded_key_; } + + private: + std::string encrypted_encoded_key_; + std::string master_key_version_; + }; + + const ::arrow::util::SecureString& GetKeyFromServer(const std::string& key_identifier); + + protected: + KmsConnectionConfig kms_connection_config_; + ::arrow::util::ConcurrentMap + master_key_cache_; +}; + +} // namespace parquet::encryption diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/test_encryption_util.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/test_encryption_util.h new file mode 100644 index 0000000000000000000000000000000000000000..c063d2c54e3ef3efce9f411362334839c9b42a6a --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/test_encryption_util.h @@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// This module defines an abstract interface for iterating through pages in a +// Parquet column chunk within a row group. It could be extended in the future +// to iterate through all data pages in all chunks in a file. + +#pragma once + +#include +#include +#include + +#include + +#include "arrow/filesystem/filesystem.h" +#include "arrow/filesystem/localfs.h" +#include "arrow/status.h" +#include "arrow/util/io_util.h" +#include "arrow/util/secure_string.h" + +#include "parquet/encryption/encryption.h" +#include "parquet/test_util.h" + +namespace parquet { +class ParquetFileReader; +namespace encryption::test { + +using ::arrow::internal::TemporaryDir; +using ::arrow::util::SecureString; + +constexpr int kFixedLength = 10; + +const SecureString kFooterEncryptionKey("0123456789012345"); +const SecureString kColumnEncryptionKey1("1234567890123450"); +const SecureString kColumnEncryptionKey2("1234567890123451"); +const SecureString kColumnEncryptionKey3("1234567890123452"); +const char kFileName[] = "tester"; + +// Get the path of file inside parquet test data directory +std::string data_file(const char* file); + +// A temporary directory that contains the encrypted files generated in the tests. +extern std::unique_ptr temp_dir; + +inline ::arrow::Result> temp_data_dir() { + return TemporaryDir::Make("parquet-encryption-test-"); +} + +const char kDoubleFieldName[] = "double_field"; +const char kFloatFieldName[] = "float_field"; +const char kBooleanFieldName[] = "boolean_field"; +const char kInt32FieldName[] = "int32_field"; +const char kInt64FieldName[] = "int64_field"; +const char kInt96FieldName[] = "int96_field"; +const char kByteArrayFieldName[] = "ba_field"; +const char kFixedLenByteArrayFieldName[] = "flba_field"; + +const char kFooterMasterKey[] = "0123456789012345"; +const char kFooterMasterKeyId[] = "kf"; +const char* const kColumnMasterKeys[] = {"1234567890123450", "1234567890123451", + "1234567890123452", "1234567890123453", + "1234567890123454", "1234567890123455"}; +const char* const kColumnMasterKeyIds[] = {"kc1", "kc2", "kc3", "kc4", "kc5", "kc6"}; + +// New master key values used to simulate key rotation +const char kNewFooterMasterKey[] = "9123456789012345"; +const char* const kNewColumnMasterKeys[] = {"9234567890123450", "9234567890123451", + "9234567890123452", "9234567890123453", + "9234567890123454", "9234567890123455"}; + +// The result of this function will be used to set into TestOnlyInMemoryKmsClientFactory +// as the key mapping to look at. +std::unordered_map BuildKeyMap(const char* const* column_ids, + const char* const* column_keys, + const char* footer_id, + const char* footer_key); + +// The result of this function will be used to set into EncryptionConfiguration +// as column keys. +std::string BuildColumnKeyMapping(); + +// FileEncryptor and FileDecryptor are helper classes to write/read an encrypted parquet +// file corresponding to each pair of FileEncryptionProperties/FileDecryptionProperties. +// FileEncryptor writes the file with fixed data values and FileDecryptor reads the file +// and verify the correctness of data values. +class FileEncryptor { + public: + FileEncryptor(); + + void EncryptFile( + std::string file, + std::shared_ptr encryption_configurations); + + private: + std::shared_ptr SetupEncryptionSchema(); + + int num_rowgroups_ = 5; + int rows_per_rowgroup_ = 50; + std::shared_ptr schema_; +}; + +class FileDecryptor { + public: + void DecryptFile( + const std::string& file_name, + const std::shared_ptr& file_decryption_properties); + void DecryptPageIndex( + const std::string& file_name, + const std::shared_ptr& file_decryption_properties); + + private: + void CheckFile( + parquet::ParquetFileReader* file_reader, + const std::shared_ptr& file_decryption_properties); + void CheckPageIndex( + parquet::ParquetFileReader* file_reader, + const std::shared_ptr& file_decryption_properties); +}; + +} // namespace encryption::test +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/test_in_memory_kms.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/test_in_memory_kms.h new file mode 100644 index 0000000000000000000000000000000000000000..ddf0c9ca9d0876786f79c44974e8d1bfbe83a97e --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/test_in_memory_kms.h @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "arrow/util/base64.h" + +#include "parquet/encryption/kms_client_factory.h" +#include "parquet/encryption/local_wrap_kms_client.h" +#include "parquet/platform.h" + +namespace parquet::encryption { + +// This is a mock class, built for testing only. Don't use it as an example of +// LocalWrapKmsClient implementation. +class TestOnlyLocalWrapInMemoryKms : public LocalWrapKmsClient { + public: + explicit TestOnlyLocalWrapInMemoryKms(const KmsConnectionConfig& kms_connection_config); + + static void InitializeMasterKeys( + const std::unordered_map& + master_keys_map); + + protected: + const ::arrow::util::SecureString& GetMasterKeyFromServer( + const std::string& master_key_identifier) override; + + private: + static std::unordered_map master_key_map_; +}; + +// This is a mock class, built for testing only. Don't use it as an example of KmsClient +// implementation. +class TestOnlyInServerWrapKms : public KmsClient { + public: + static void InitializeMasterKeys( + const std::unordered_map& + master_keys_map); + + std::string WrapKey(const ::arrow::util::SecureString& key_bytes, + const std::string& master_key_identifier) override; + + ::arrow::util::SecureString UnwrapKey( + const std::string& wrapped_key, const std::string& master_key_identifier) override; + + static void StartKeyRotation( + const std::unordered_map& + new_master_keys_map); + static void FinishKeyRotation(); + + private: + ::arrow::util::SecureString GetMasterKeyFromServer( + const std::string& master_key_identifier); + + // Different wrapping and unwrapping key maps to imitate versioning + // and support key rotation. + static std::unordered_map + unwrapping_master_key_map_; + static std::unordered_map + wrapping_master_key_map_; +}; + +// This is a mock class, built for testing only. Don't use it as an example of +// KmsClientFactory implementation. +class TestOnlyInMemoryKmsClientFactory : public KmsClientFactory { + public: + TestOnlyInMemoryKmsClientFactory( + bool wrap_locally, + const std::unordered_map& master_keys_map) + : KmsClientFactory(wrap_locally) { + TestOnlyLocalWrapInMemoryKms::InitializeMasterKeys(master_keys_map); + TestOnlyInServerWrapKms::InitializeMasterKeys(master_keys_map); + } + + std::shared_ptr CreateKmsClient( + const KmsConnectionConfig& kms_connection_config) { + if (wrap_locally_) { + return std::make_shared(kms_connection_config); + } else { + return std::make_shared(); + } + } +}; + +} // namespace parquet::encryption diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/two_level_cache_with_expiration.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/two_level_cache_with_expiration.h new file mode 100644 index 0000000000000000000000000000000000000000..b21fa8658ba1b29f41a8d49f147e1e7f9fbbd2a2 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/two_level_cache_with_expiration.h @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "arrow/util/concurrent_map.h" +#include "arrow/util/mutex.h" + +namespace parquet::encryption { + +using ::arrow::util::ConcurrentMap; + +namespace internal { + +using TimePoint = + std::chrono::time_point>; + +inline TimePoint CurrentTimePoint() { return std::chrono::system_clock::now(); } + +template +class ExpiringCacheEntry { + public: + ExpiringCacheEntry() = default; + + ExpiringCacheEntry(E cached_item, double expiration_interval_seconds) + : expiration_timestamp_(CurrentTimePoint() + + std::chrono::duration(expiration_interval_seconds)), + cached_item_(std::move(cached_item)) {} + + bool IsExpired() const { + const auto now = CurrentTimePoint(); + return (now > expiration_timestamp_); + } + + E cached_item() { return cached_item_; } + + private: + const TimePoint expiration_timestamp_; + E cached_item_; +}; + +// This class is to avoid the below warning when compiling KeyToolkit class with VS2015 +// warning C4503: decorated name length exceeded, name was truncated +template +class ExpiringCacheMapEntry { + public: + ExpiringCacheMapEntry() = default; + + explicit ExpiringCacheMapEntry( + std::shared_ptr> cached_item, + double expiration_interval_seconds) + : map_cache_(cached_item, expiration_interval_seconds) {} + + bool IsExpired() { return map_cache_.IsExpired(); } + + std::shared_ptr> cached_item() { + return map_cache_.cached_item(); + } + + private: + // ConcurrentMap object may be accessed and modified at many places at the same time, + // from multiple threads, or even removed from cache. + ExpiringCacheEntry>> map_cache_; +}; + +} // namespace internal + +// Two-level cache with expiration of internal caches according to token lifetime. +// External cache is per token, internal is per string key. +// Wrapper class around: +// std::unordered_map>> +// This cache is safe to be shared between threads. +template +class TwoLevelCacheWithExpiration { + public: + TwoLevelCacheWithExpiration() { + last_cache_cleanup_timestamp_ = internal::CurrentTimePoint(); + } + + std::shared_ptr> GetOrCreateInternalCache( + const std::string& access_token, double cache_entry_lifetime_seconds) { + auto lock = mutex_.Lock(); + + auto external_cache_entry = cache_.find(access_token); + if (external_cache_entry == cache_.end() || + external_cache_entry->second.IsExpired()) { + cache_.insert({access_token, internal::ExpiringCacheMapEntry( + std::make_shared>(), + cache_entry_lifetime_seconds)}); + } + + return cache_[access_token].cached_item(); + } + + void CheckCacheForExpiredTokens(double cache_cleanup_period_seconds = 0.0) { + auto lock = mutex_.Lock(); + + const auto now = internal::CurrentTimePoint(); + if (now > (last_cache_cleanup_timestamp_ + + std::chrono::duration(cache_cleanup_period_seconds))) { + RemoveExpiredEntriesNoMutex(); + last_cache_cleanup_timestamp_ = now; + } + } + + void Remove(const std::string& access_token) { + auto lock = mutex_.Lock(); + cache_.erase(access_token); + } + + void Clear() { + auto lock = mutex_.Lock(); + cache_.clear(); + } + + private: + void RemoveExpiredEntriesNoMutex() { + for (auto it = cache_.begin(); it != cache_.end();) { + if (it->second.IsExpired()) { + it = cache_.erase(it); + } else { + ++it; + } + } + } + std::unordered_map> cache_; + internal::TimePoint last_cache_cleanup_timestamp_; + ::arrow::util::Mutex mutex_; +}; + +} // namespace parquet::encryption diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/type_fwd.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/type_fwd.h new file mode 100644 index 0000000000000000000000000000000000000000..cc7e24396101f3e6cc7188e59215e0553fe99a99 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/type_fwd.h @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +namespace parquet { + +class Decryptor; +class Encryptor; + +class InternalFileDecryptor; +class InternalFileEncryptor; + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/exception.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/exception.h new file mode 100644 index 0000000000000000000000000000000000000000..99f596b4f61ba0dea2bbc74749850bf341f513f0 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/exception.h @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "arrow/type_fwd.h" +#include "arrow/util/string_util.h" +#include "parquet/platform.h" + +#ifdef _MSC_VER +# pragma warning(push) +// Disable warning for STL types usage in DLL interface +// https://web.archive.org/web/20130317015847/http://connect.microsoft.com/VisualStudio/feedback/details/696593/vc-10-vs-2010-basic-string-exports +# pragma warning(disable : 4275 4251) +// Disable diamond inheritance warnings +# pragma warning(disable : 4250) +// Disable macro redefinition warnings +# pragma warning(disable : 4005) +// Disable extern before exported template warnings +# pragma warning(disable : 4910) +#endif + +// PARQUET-1085 +#if !defined(ARROW_UNUSED) +# define ARROW_UNUSED(x) UNUSED(x) +#endif + +// Parquet exception to Arrow Status + +#define BEGIN_PARQUET_CATCH_EXCEPTIONS try { +#define END_PARQUET_CATCH_EXCEPTIONS \ + } \ + catch (const ::parquet::ParquetStatusException& e) { \ + return e.status(); \ + } \ + catch (const ::parquet::ParquetException& e) { \ + return ::arrow::Status::IOError(e.what()); \ + } + +// clang-format off + +#define PARQUET_CATCH_NOT_OK(s) \ + BEGIN_PARQUET_CATCH_EXCEPTIONS \ + (s); \ + END_PARQUET_CATCH_EXCEPTIONS + +// clang-format on + +#define PARQUET_CATCH_AND_RETURN(s) \ + BEGIN_PARQUET_CATCH_EXCEPTIONS \ + return (s); \ + END_PARQUET_CATCH_EXCEPTIONS + +// Arrow Status to Parquet exception + +#define PARQUET_IGNORE_NOT_OK(s) \ + do { \ + ::arrow::Status _s = ::arrow::ToStatus(s); \ + ARROW_UNUSED(_s); \ + } while (0) + +#define PARQUET_THROW_NOT_OK(s) \ + do { \ + ::arrow::Status _s = ::arrow::ToStatus(s); \ + if (!_s.ok()) { \ + throw ::parquet::ParquetStatusException(std::move(_s)); \ + } \ + } while (0) + +#define PARQUET_ASSIGN_OR_THROW_IMPL(status_name, lhs, rexpr) \ + auto status_name = (rexpr); \ + PARQUET_THROW_NOT_OK(status_name.status()); \ + lhs = std::move(status_name).ValueOrDie(); + +#define PARQUET_ASSIGN_OR_THROW(lhs, rexpr) \ + PARQUET_ASSIGN_OR_THROW_IMPL(ARROW_ASSIGN_OR_RAISE_NAME(_error_or_value, __COUNTER__), \ + lhs, rexpr); + +namespace parquet { + +class PARQUET_EXPORT ParquetException : public std::exception { + public: + PARQUET_NORETURN static void EofException(const std::string& msg = "") { + static std::string prefix = "Unexpected end of stream"; + if (msg.empty()) { + throw ParquetException(prefix); + } + throw ParquetException(prefix, ": ", msg); + } + + PARQUET_NORETURN static void NYI(const std::string& msg = "") { + throw ParquetException("Not yet implemented: ", msg, "."); + } + + template + explicit ParquetException(Args&&... args) + : msg_(::arrow::internal::JoinToString(std::forward(args)...)) {} + + explicit ParquetException(std::string msg) : msg_(std::move(msg)) {} + + explicit ParquetException(const char* msg, const std::exception&) : msg_(msg) {} + + ParquetException(const ParquetException&) = default; + ParquetException& operator=(const ParquetException&) = default; + ParquetException(ParquetException&&) = default; + ParquetException& operator=(ParquetException&&) = default; + + const char* what() const noexcept override { return msg_.c_str(); } + + private: + std::string msg_; +}; + +// Support printing a ParquetException. +// This is needed for clang-on-MSVC as there operator<< is not defined for +// std::exception. +PARQUET_EXPORT +std::ostream& operator<<(std::ostream& os, const ParquetException& exception); + +class ParquetStatusException : public ParquetException { + public: + explicit ParquetStatusException(::arrow::Status status) + : ParquetException(status.ToString()), status_(std::move(status)) {} + + const ::arrow::Status& status() const { return status_; } + + private: + ::arrow::Status status_; +}; + +// This class exists for the purpose of detecting an invalid or corrupted file. +class ParquetInvalidOrCorruptedFileException : public ParquetStatusException { + public: + ParquetInvalidOrCorruptedFileException(const ParquetInvalidOrCorruptedFileException&) = + default; + + template ::value, + int>::type = 0, + typename... Args> + explicit ParquetInvalidOrCorruptedFileException(Arg arg, Args&&... args) + : ParquetStatusException(::arrow::Status::Invalid(std::forward(arg), + std::forward(args)...)) {} +}; + +template +void ThrowNotOk(StatusReturnBlock&& b) { + PARQUET_THROW_NOT_OK(b()); +} + +} // namespace parquet + +#ifdef _MSC_VER +# pragma warning(pop) +#endif diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/file_reader.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/file_reader.h new file mode 100644 index 0000000000000000000000000000000000000000..c74b867dbebd239fdeb31d7811a6c385e85fae80 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/file_reader.h @@ -0,0 +1,257 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "arrow/io/caching.h" +#include "arrow/util/type_fwd.h" +#include "parquet/metadata.h" // IWYU pragma: keep +#include "parquet/platform.h" +#include "parquet/properties.h" + +namespace parquet { + +class ColumnReader; +class FileMetaData; +class PageIndexReader; +class BloomFilterReader; +class PageReader; +class RowGroupMetaData; + +namespace internal { +class RecordReader; +} + +class PARQUET_EXPORT RowGroupReader { + public: + // Forward declare a virtual class 'Contents' to aid dependency injection and more + // easily create test fixtures + // An implementation of the Contents class is defined in the .cc file + struct Contents { + virtual ~Contents() {} + virtual std::unique_ptr GetColumnPageReader(int i) = 0; + virtual const RowGroupMetaData* metadata() const = 0; + virtual const ReaderProperties* properties() const = 0; + }; + + explicit RowGroupReader(std::unique_ptr contents); + + // Returns the rowgroup metadata + const RowGroupMetaData* metadata() const; + + // Construct a ColumnReader for the indicated row group-relative + // column. Ownership is shared with the RowGroupReader. + std::shared_ptr Column(int i); + + // EXPERIMENTAL: Construct a RecordReader for the indicated column of the row group. + // Ownership is shared with the RowGroupReader. + std::shared_ptr RecordReader(int i, + bool read_dictionary = false); + + // Construct a ColumnReader, trying to enable exposed encoding. + // + // For dictionary encoding, currently we only support column chunks that are fully + // dictionary encoded, i.e., all data pages in the column chunk are dictionary encoded. + // If a column chunk uses dictionary encoding but then falls back to plain encoding, the + // encoding will not be exposed. + // + // The returned column reader provides an API GetExposedEncoding() for the + // users to check the exposed encoding and determine how to read the batches. + // + // \note API EXPERIMENTAL + std::shared_ptr ColumnWithExposeEncoding( + int i, ExposedEncoding encoding_to_expose); + + // Construct a RecordReader, trying to enable exposed encoding. + // + // For dictionary encoding, currently we only support column chunks that are + // fully dictionary encoded byte arrays. The caller should verify if the reader can read + // and expose the dictionary by checking the reader's read_dictionary(). If a column + // chunk uses dictionary encoding but then falls back to plain encoding, the returned + // reader will read decoded data without exposing the dictionary. + // + // \note API EXPERIMENTAL + std::shared_ptr RecordReaderWithExposeEncoding( + int i, ExposedEncoding encoding_to_expose); + + std::unique_ptr GetColumnPageReader(int i); + + private: + // Holds a pointer to an instance of Contents implementation + std::unique_ptr contents_; +}; + +class PARQUET_EXPORT ParquetFileReader { + public: + // Declare a virtual class 'Contents' to aid dependency injection and more + // easily create test fixtures + // An implementation of the Contents class is defined in the .cc file + struct PARQUET_EXPORT Contents { + static std::unique_ptr Open( + std::shared_ptr<::arrow::io::RandomAccessFile> source, + const ReaderProperties& props = default_reader_properties(), + std::shared_ptr metadata = NULLPTR); + + static ::arrow::Future> OpenAsync( + std::shared_ptr<::arrow::io::RandomAccessFile> source, + const ReaderProperties& props = default_reader_properties(), + std::shared_ptr metadata = NULLPTR); + + virtual ~Contents() = default; + // Perform any cleanup associated with the file contents + virtual void Close() = 0; + virtual std::shared_ptr GetRowGroup(int i) = 0; + virtual std::shared_ptr metadata() const = 0; + virtual std::shared_ptr GetPageIndexReader() = 0; + virtual BloomFilterReader& GetBloomFilterReader() = 0; + }; + + ParquetFileReader(); + ~ParquetFileReader(); + + // Create a file reader instance from an Arrow file object. Thread-safety is + // the responsibility of the file implementation + static std::unique_ptr Open( + std::shared_ptr<::arrow::io::RandomAccessFile> source, + const ReaderProperties& props = default_reader_properties(), + std::shared_ptr metadata = NULLPTR); + + // API Convenience to open a serialized Parquet file on disk, using Arrow IO + // interfaces. + static std::unique_ptr OpenFile( + const std::string& path, bool memory_map = false, + const ReaderProperties& props = default_reader_properties(), + std::shared_ptr metadata = NULLPTR); + + // Asynchronously open a file reader from an Arrow file object. + // Does not throw - all errors are reported through the Future. + static ::arrow::Future> OpenAsync( + std::shared_ptr<::arrow::io::RandomAccessFile> source, + const ReaderProperties& props = default_reader_properties(), + std::shared_ptr metadata = NULLPTR); + + void Open(std::unique_ptr contents); + void Close(); + + // The RowGroupReader is owned by the FileReader + std::shared_ptr RowGroup(int i); + + // Returns the file metadata. Only one instance is ever created + std::shared_ptr metadata() const; + + /// Returns the PageIndexReader. Only one instance is ever created. + /// + /// If the file does not have the page index, nullptr may be returned. + /// Because it pays to check existence of page index in the file, it + /// is possible to return a non null value even if page index does + /// not exist. It is the caller's responsibility to check the return + /// value and follow-up calls to PageIndexReader. + /// + /// WARNING: The returned PageIndexReader must not outlive the ParquetFileReader. + /// Initialize GetPageIndexReader() is not thread-safety. + std::shared_ptr GetPageIndexReader(); + + /// Returns the BloomFilterReader. Only one instance is ever created. + /// + /// WARNING: The returned BloomFilterReader must not outlive the ParquetFileReader. + /// Initialize GetBloomFilterReader() is not thread-safety. + BloomFilterReader& GetBloomFilterReader(); + + /// Pre-buffer the specified column indices in all row groups. + /// + /// Readers can optionally call this to cache the necessary slices + /// of the file in-memory before deserialization. Arrow readers can + /// automatically do this via an option. This is intended to + /// increase performance when reading from high-latency filesystems + /// (e.g. Amazon S3). + /// + /// After calling this, creating readers for row groups/column + /// indices that were not buffered may fail. Creating multiple + /// readers for the a subset of the buffered regions is + /// acceptable. This may be called again to buffer a different set + /// of row groups/columns. + /// + /// If memory usage is a concern, note that data will remain + /// buffered in memory until either \a PreBuffer() is called again, + /// or the reader itself is destructed. Reading - and buffering - + /// only one row group at a time may be useful. + /// + /// This method may throw. + void PreBuffer(const std::vector& row_groups, + const std::vector& column_indices, + const ::arrow::io::IOContext& ctx, + const ::arrow::io::CacheOptions& options); + + /// Retrieve the list of byte ranges that would need to be read to retrieve + /// the data for the specified row groups and column indices. + /// + /// A reader can optionally call this if they wish to handle their own + /// caching and management of file reads (or offload them to other readers). + /// Unlike PreBuffer, this method will not perform any actual caching or + /// reads, instead just using the file metadata to determine the byte ranges + /// that would need to be read if you were to consume the entirety of the column + /// chunks for the provided columns in the specified row groups. + /// + /// If row_groups or column_indices are empty, then the result of this will be empty. + /// + /// hole_size_limit represents the maximum distance, in bytes, between two + /// consecutive ranges; beyond this value, ranges will not be combined. The default + /// value is 1MB. + /// + /// range_size_limit is the maximum size in bytes of a combined range; if combining + /// two consecutive ranges would produce a range larger than this, they are not + /// combined. The default values is 64MB. This *must* be larger than hole_size_limit. + /// + /// This will not take into account page indexes or any other predicate push down + /// benefits that may be available. + ::arrow::Result> GetReadRanges( + const std::vector& row_groups, const std::vector& column_indices, + int64_t hole_size_limit = 1024 * 1024, int64_t range_size_limit = 64 * 1024 * 1024); + + /// Wait for the specified row groups and column indices to be pre-buffered. + /// + /// After the returned Future completes, reading the specified row + /// groups/columns will not block. + /// + /// PreBuffer must be called first. This method does not throw. + ::arrow::Future<> WhenBuffered(const std::vector& row_groups, + const std::vector& column_indices) const; + + private: + // Holds a pointer to an instance of Contents implementation + std::unique_ptr contents_; +}; + +// Read only Parquet file metadata +std::shared_ptr PARQUET_EXPORT +ReadMetaData(const std::shared_ptr<::arrow::io::RandomAccessFile>& source); + +/// \brief Scan all values in file. Useful for performance testing +/// \param[in] columns the column numbers to scan. If empty scans all +/// \param[in] column_batch_size number of values to read at a time when scanning column +/// \param[in] reader a ParquetFileReader instance +/// \return number of semantic rows in file +PARQUET_EXPORT +int64_t ScanFileContents(std::vector columns, const int32_t column_batch_size, + ParquetFileReader* reader); + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/file_writer.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/file_writer.h new file mode 100644 index 0000000000000000000000000000000000000000..5261e4d1adba8df8d88d6a0a1475988bd476029e --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/file_writer.h @@ -0,0 +1,260 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "parquet/metadata.h" +#include "parquet/platform.h" +#include "parquet/properties.h" +#include "parquet/schema.h" + +namespace parquet { + +class ColumnWriter; + +// FIXME: copied from reader-internal.cc +static constexpr uint8_t kParquetMagic[4] = {'P', 'A', 'R', '1'}; +static constexpr uint8_t kParquetEMagic[4] = {'P', 'A', 'R', 'E'}; + +class PARQUET_EXPORT RowGroupWriter { + public: + // Estimated uncompressed byte sizes of data buffered by column writers + // that have not yet been serialized into pages. + struct BufferedStats { + int64_t def_level_bytes = 0; + int64_t rep_level_bytes = 0; + int64_t value_bytes = 0; + int64_t dict_bytes = 0; + }; + + // Forward declare a virtual class 'Contents' to aid dependency injection and more + // easily create test fixtures + // An implementation of the Contents class is defined in the .cc file + struct Contents { + virtual ~Contents() = default; + virtual int num_columns() const = 0; + virtual int64_t num_rows() const = 0; + + // to be used only with ParquetFileWriter::AppendRowGroup + virtual ColumnWriter* NextColumn() = 0; + // to be used only with ParquetFileWriter::AppendBufferedRowGroup + virtual ColumnWriter* column(int i) = 0; + + virtual int current_column() const = 0; + virtual void Close() = 0; + + /// \brief total uncompressed bytes written by the page writer + virtual int64_t total_bytes_written() const = 0; + /// \brief total bytes still compressed but not written by the page writer + virtual int64_t total_compressed_bytes() const = 0; + /// \brief total compressed bytes written by the page writer + virtual int64_t total_compressed_bytes_written() const = 0; + /// \brief Estimated sizes of buffered data (levels, values, dict) not yet + /// written to pages. + virtual BufferedStats EstimatedBufferedStats() const = 0; + + virtual bool buffered() const = 0; + }; + + explicit RowGroupWriter(std::unique_ptr contents); + + /// Construct a ColumnWriter for the indicated row group-relative column. + /// + /// To be used only with ParquetFileWriter::AppendRowGroup + /// Ownership is solely within the RowGroupWriter. The ColumnWriter is only + /// valid until the next call to NextColumn or Close. As the contents are + /// directly written to the sink, once a new column is started, the contents + /// of the previous one cannot be modified anymore. + ColumnWriter* NextColumn(); + /// Index of currently written column. Equal to -1 if NextColumn() + /// has not been called yet. + int current_column(); + void Close(); + + int num_columns() const; + + /// Construct a ColumnWriter for the indicated row group column. + /// + /// To be used only with ParquetFileWriter::AppendBufferedRowGroup + /// Ownership is solely within the RowGroupWriter. The ColumnWriter is + /// valid until Close. The contents are buffered in memory and written to sink + /// on Close + ColumnWriter* column(int i); + + /** + * Number of rows that shall be written as part of this RowGroup. + */ + int64_t num_rows() const; + + /// \brief total uncompressed bytes written by the page writer + int64_t total_bytes_written() const; + /// \brief total bytes still compressed but not written by the page writer. + /// It will always return 0 from the SerializedPageWriter. + int64_t total_compressed_bytes() const; + /// \brief total compressed bytes written by the page writer + int64_t total_compressed_bytes_written() const; + /// \brief Estimated sizes of buffered data (levels, values, dict) not yet + /// written to pages. + BufferedStats estimated_buffered_stats() const; + + /// Returns whether the current RowGroupWriter is in the buffered mode and is created + /// by calling ParquetFileWriter::AppendBufferedRowGroup. + bool buffered() const; + + private: + // Holds a pointer to an instance of Contents implementation + std::unique_ptr contents_; +}; + +PARQUET_EXPORT +void WriteFileMetaData(const FileMetaData& file_metadata, + ::arrow::io::OutputStream* sink); + +PARQUET_EXPORT +void WriteMetaDataFile(const FileMetaData& file_metadata, + ::arrow::io::OutputStream* sink); + +PARQUET_EXPORT +void WriteEncryptedFileMetadata(const FileMetaData& file_metadata, + ArrowOutputStream* sink, + const std::shared_ptr& encryptor, + bool encrypt_footer); + +PARQUET_EXPORT +void WriteEncryptedFileMetadata(const FileMetaData& file_metadata, + ::arrow::io::OutputStream* sink, + const std::shared_ptr& encryptor = NULLPTR, + bool encrypt_footer = false); +PARQUET_EXPORT +void WriteFileCryptoMetaData(const FileCryptoMetaData& crypto_metadata, + ::arrow::io::OutputStream* sink); + +class PARQUET_EXPORT ParquetFileWriter { + public: + // Forward declare a virtual class 'Contents' to aid dependency injection and more + // easily create test fixtures + // An implementation of the Contents class is defined in the .cc file + struct Contents { + Contents(std::shared_ptr<::parquet::schema::GroupNode> schema, + std::shared_ptr key_value_metadata) + : schema_(), key_value_metadata_(std::move(key_value_metadata)) { + schema_.Init(std::move(schema)); + } + virtual ~Contents() {} + // Perform any cleanup associated with the file contents + virtual void Close() = 0; + + virtual RowGroupWriter* AppendRowGroup() = 0; + virtual RowGroupWriter* AppendBufferedRowGroup() = 0; + + virtual int64_t num_rows() const = 0; + virtual int num_columns() const = 0; + virtual int num_row_groups() const = 0; + + virtual const std::shared_ptr& properties() const = 0; + + const std::shared_ptr& key_value_metadata() const { + return key_value_metadata_; + } + + virtual void AddKeyValueMetadata( + const std::shared_ptr& key_value_metadata) = 0; + + // Return const-pointer to make it clear that this object is not to be copied + const SchemaDescriptor* schema() const { return &schema_; } + + SchemaDescriptor schema_; + + /// This should be the only place this is stored. Everything else is a const reference + std::shared_ptr key_value_metadata_; + + const std::shared_ptr& metadata() const { return file_metadata_; } + std::shared_ptr file_metadata_; + }; + + ParquetFileWriter(); + ~ParquetFileWriter(); + + static std::unique_ptr Open( + std::shared_ptr<::arrow::io::OutputStream> sink, + std::shared_ptr schema, + std::shared_ptr properties = default_writer_properties(), + std::shared_ptr key_value_metadata = NULLPTR); + + void Open(std::unique_ptr contents); + void Close(); + + /// Construct a RowGroupWriter with an arbitrary number of rows. + /// + /// Ownership is solely within the ParquetFileWriter. The RowGroupWriter is only valid + /// until the next call to AppendRowGroup or AppendBufferedRowGroup or Close. + RowGroupWriter* AppendRowGroup(); + + /// Construct a RowGroupWriter that buffers all the values until the RowGroup is ready. + /// Use this if you want to write a RowGroup based on a certain size + /// + /// Ownership is solely within the ParquetFileWriter. The RowGroupWriter is only valid + /// until the next call to AppendRowGroup or AppendBufferedRowGroup or Close. + RowGroupWriter* AppendBufferedRowGroup(); + + /// \brief Add key-value metadata to the file. + /// \param[in] key_value_metadata the metadata to add. + /// \note This will overwrite any existing metadata with the same key(s). + /// \throw ParquetException if Close() has been called. + void AddKeyValueMetadata( + const std::shared_ptr& key_value_metadata); + + /// Number of columns. + /// + /// This number is fixed during the lifetime of the writer as it is determined via + /// the schema. + int num_columns() const; + + /// Number of rows in the yet started RowGroups. + /// + /// Changes on the addition of a new RowGroup. + int64_t num_rows() const; + + /// Number of started RowGroups. + int num_row_groups() const; + + /// Configuration passed to the writer, e.g. the used Parquet format version. + const std::shared_ptr& properties() const; + + /// Returns the file schema descriptor + const SchemaDescriptor* schema() const; + + /// Returns a column descriptor in schema + const ColumnDescriptor* descr(int i) const; + + /// Returns the file custom metadata + const std::shared_ptr& key_value_metadata() const; + + /// Returns the file metadata, only available after calling Close(). + const std::shared_ptr metadata() const; + + private: + // Holds a pointer to an instance of Contents implementation + std::unique_ptr contents_; + std::shared_ptr file_metadata_; +}; + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/geospatial/statistics.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/geospatial/statistics.h new file mode 100644 index 0000000000000000000000000000000000000000..cc7b0b4822f2dba08023981f8feed30c4567a8a2 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/geospatial/statistics.h @@ -0,0 +1,198 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "parquet/platform.h" +#include "parquet/types.h" + +namespace parquet::geospatial { + +/// \brief The maximum number of dimensions represented by a geospatial type +/// (i.e., X, Y, Z, and M) +inline constexpr int kMaxDimensions = 4; + +/// \brief NaN, used to represent bounds for which predicate pushdown cannnot +/// be applied (e.g., because a writer did not provide bounds for a given dimension) +inline constexpr double kNaN = std::numeric_limits::quiet_NaN(); + +/// \brief Structure represented encoded statistics to be written to and read from Parquet +/// serialized metadata. +/// +/// See the Parquet Thrift definition and GeoStatistics for the specific definition +/// of field values. +struct PARQUET_EXPORT EncodedGeoStatistics { + bool xy_bounds_present{false}; + double xmin{kNaN}; + double xmax{kNaN}; + double ymin{kNaN}; + double ymax{kNaN}; + + bool z_bounds_present{false}; + double zmin{kNaN}; + double zmax{kNaN}; + + bool m_bounds_present{false}; + double mmin{kNaN}; + double mmax{kNaN}; + + bool geospatial_types_present() const { return !geospatial_types.empty(); } + std::vector geospatial_types; +}; + +class GeoStatisticsImpl; + +/// \brief Base type for computing geospatial column statistics while writing a file +/// or representing them when reading a file +/// +/// These statistics track the minimum and maximum value (omitting NaN values) of the +/// four possible dimensions (X, Y, Z, and M) and the distinct set of geometry +/// type/dimension combinations (e.g., point XY, linestring XYZM) present in the data. +/// Any of these individual components may be "invalid": for example, when reading a +/// Parquet file, information about individual components obtained from the column +/// chunk metadata may have been missing or deemed unusable. Orthogonally, +/// any of these individual components may be "empty": for example, when using +/// GeoStatistics to accumulate bounds whilst writing, if all geometries in a column chunk +/// are null, all ranges (X, Y, Z, and M) will be empty. If all geometries in a column +/// chunk contain only XY coordinates (the most common case), the Z and M ranges will +/// be empty but the X and Y ranges will contain finite bounds. Empty ranges are +/// considered "valid" because they are known to represent exactly zero values (in +/// contrast to an invalid range, whose contents is completely unknown). These concepts +/// are all necessary for this object to accurately represent (1) accumulated or partially +/// accumulated statistics during the writing process and (2) deserialized statistics read +/// from the column chunk metadata during the reading process. +/// +/// EXPERIMENTAL +class PARQUET_EXPORT GeoStatistics { + public: + GeoStatistics(); + explicit GeoStatistics(const EncodedGeoStatistics& encoded); + + ~GeoStatistics(); + + /// \brief Return true if bounds, geometry types, and validity are identical + bool Equals(const GeoStatistics& other) const; + + /// \brief Update these statistics based on previously calculated or decoded statistics + /// + /// Merging statistics with wraparound X values is not currently supported. Merging + /// two GeoStatistics where one or both has a wraparound X range will result in these + /// statistics having an X dimension marked as invalid. + void Merge(const GeoStatistics& other); + + /// \brief Update these statistics based on values + void Update(const ByteArray* values, int64_t num_values); + + /// \brief Update these statistics based on the non-null elements of values + void UpdateSpaced(const ByteArray* values, const uint8_t* valid_bits, + int64_t valid_bits_offset, int64_t num_spaced_values, + int64_t num_values); + + /// \brief Update these statistics based on the non-null elements of values + /// + /// Currently, BinaryArray and LargeBinaryArray input is supported. + void Update(const ::arrow::Array& values); + + /// \brief Return these statistics to an empty state + void Reset(); + + /// \brief Encode the statistics for serializing to Thrift + /// + /// If invalid WKB was encountered or if the statistics contain NaN + /// for any reason, Encode() will return nullopt to indicate that + /// statistics should not be written to thrift. + std::optional Encode() const; + + /// \brief Returns false if invalid WKB was encountered + bool is_valid() const; + + /// \brief Reset existing statistics and populate them from previously-encoded ones + void Decode(const EncodedGeoStatistics& encoded); + + /// \brief Minimum values in XYZM order + /// + /// For dimensions where dimension_valid() is false, the value will be NaN. For + /// dimensions where dimension_empty() is true, the value will be +Inf. + /// + /// For the first dimension (X) only, wraparound bounds apply where xmin > xmax. In this + /// case, these bounds represent the union of the intervals [xmax, Inf] and [-Inf, + /// xmin]. This implementation does not yet generate these types of bounds but they may + /// be encountered in statistics when reading a Parquet file. + std::array lower_bound() const; + + /// \brief Maximum values in XYZM order + /// + /// For dimensions where dimension_valid() is false, the value will be NaN. For + /// dimensions where dimension_empty() is true, the value will be -Inf. + /// + /// For the first dimension (X) only, wraparound bounds apply where xmin > xmax. In this + /// case, these bounds represent the union of the intervals [xmax, Inf] and [-Inf, + /// xmin]. This implementation does not yet generate these types of bounds but they may + /// be encountered in statistics when reading a Parquet file. + std::array upper_bound() const; + + /// \brief Dimension emptiness in XYZM order + /// + /// True for a given dimension if and only if zero non-NaN values were encountered + /// in that dimension and dimension_valid() is true for that dimension. + /// + /// When calculating statistics, zero or more of these values may be true because + /// this implementation calculates bounds for all dimensions; however, it may be + /// true that zero coordinates were encountered in a given dimension. For example, + /// dimension_empty() will return four true values if Update() was not called + /// or if Update() was called with only null values. If Update() was provided + /// one or more geometries with X and Y dimensions but not Z or M dimensions, + /// dimension_empty() will return true, true, false, false. + /// + /// For statistics read from a Parquet file, dimension_empty() will always contain + /// false values because there is no mechanism to communicate an empty interval + /// in the Thrift metadata. + std::array dimension_empty() const; + + /// \brief Dimension validity (i.e. presence) in XYZM order + /// + /// When calculating statistics, this will always be true because this implementation + /// calculates statistics for all dimensions. When reading a Parquet file, one or more + /// of these values may be false because the file may not have provided bounds for all + /// dimensions. + /// + /// See documentation for dimension_empty(), lower_bound(), and/or upper_bound() for the + /// canonical values of those outputs for the dimensions where dimension_valid() is + /// false. + std::array dimension_valid() const; + + /// \brief Return the geometry type codes + /// + /// This implementation always returns sorted output with no duplicates. When + /// calculating statistics, a value will always be returned (although the returned + /// vector may be empty if Update() was never called or was only called with null + /// values). When reading a Parquet file, std::nullopt may be returned because + /// the file may not have provided this information. + std::optional> geometry_types() const; + + /// \brief Return a string representation of these statistics + std::string ToString() const; + + private: + std::unique_ptr impl_; +}; + +} // namespace parquet::geospatial diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/hasher.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/hasher.h new file mode 100644 index 0000000000000000000000000000000000000000..feae2e53c243f44dcf42803ea87a29d80637736d --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/hasher.h @@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include "parquet/types.h" + +namespace parquet { +// Abstract class for hash +class PARQUET_EXPORT Hasher { + public: + /// Compute hash for 32 bits value by using its plain encoding result. + /// + /// @param value the value to hash. + /// @return hash result. + virtual uint64_t Hash(int32_t value) const = 0; + + /// Compute hash for 64 bits value by using its plain encoding result. + /// + /// @param value the value to hash. + /// @return hash result. + virtual uint64_t Hash(int64_t value) const = 0; + + /// Compute hash for float value by using its plain encoding result. + /// + /// @param value the value to hash. + /// @return hash result. + virtual uint64_t Hash(float value) const = 0; + + /// Compute hash for double value by using its plain encoding result. + /// + /// @param value the value to hash. + /// @return hash result. + virtual uint64_t Hash(double value) const = 0; + + /// Compute hash for Int96 value by using its plain encoding result. + /// + /// @param value the value to hash. + /// @return hash result. + virtual uint64_t Hash(const Int96* value) const = 0; + + /// Compute hash for ByteArray value by using its plain encoding result. + /// + /// @param value the value to hash. + /// @return hash result. + virtual uint64_t Hash(const ByteArray* value) const = 0; + + /// Compute hash for fixed byte array value by using its plain encoding result. + /// + /// @param value the value address. + /// @param len the value length. + virtual uint64_t Hash(const FLBA* value, uint32_t len) const = 0; + + /// Compute hash for std::string_view value by using its plain encoding result. + /// + /// @param value the value to hash. + virtual uint64_t Hash(std::string_view value) const = 0; + + /// Batch compute hashes for 32 bits values by using its plain encoding result. + /// + /// @param values a pointer to the values to hash. + /// @param num_values the number of values to hash. + /// @param hashes a pointer to the output hash values, its length should be equal to + /// num_values. + virtual void Hashes(const int32_t* values, int num_values, uint64_t* hashes) const = 0; + + /// Batch compute hashes for 64 bits values by using its plain encoding result. + /// + /// @param values a pointer to the values to hash. + /// @param num_values the number of values to hash. + /// @param hashes a pointer to the output hash values, its length should be equal to + /// num_values. + virtual void Hashes(const int64_t* values, int num_values, uint64_t* hashes) const = 0; + + /// Batch compute hashes for float values by using its plain encoding result. + /// + /// @param values a pointer to the values to hash. + /// @param num_values the number of values to hash. + /// @param hashes a pointer to the output hash values, its length should be equal to + /// num_values. + virtual void Hashes(const float* values, int num_values, uint64_t* hashes) const = 0; + + /// Batch compute hashes for double values by using its plain encoding result. + /// + /// @param values a pointer to the values to hash. + /// @param num_values the number of values to hash. + /// @param hashes a pointer to the output hash values, its length should be equal to + /// num_values. + virtual void Hashes(const double* values, int num_values, uint64_t* hashes) const = 0; + + /// Batch compute hashes for Int96 values by using its plain encoding result. + /// + /// @param values a pointer to the values to hash. + /// @param num_values the number of values to hash. + /// @param hashes a pointer to the output hash values, its length should be equal to + /// num_values. + virtual void Hashes(const Int96* values, int num_values, uint64_t* hashes) const = 0; + + /// Batch compute hashes for ByteArray values by using its plain encoding result. + /// + /// @param values a pointer to the values to hash. + /// @param num_values the number of values to hash. + /// @param hashes a pointer to the output hash values, its length should be equal to + /// num_values. + virtual void Hashes(const ByteArray* values, int num_values, + uint64_t* hashes) const = 0; + + /// Batch compute hashes for fixed byte array values by using its plain encoding result. + /// + /// @param values the value address. + /// @param type_len the value length. + /// @param num_values the number of values to hash. + /// @param hashes a pointer to the output hash values, its length should be equal to + /// num_values. + virtual void Hashes(const FLBA* values, uint32_t type_len, int num_values, + uint64_t* hashes) const = 0; + + virtual ~Hasher() = default; +}; + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/index_location.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/index_location.h new file mode 100644 index 0000000000000000000000000000000000000000..5205586636f9962488a9ffd05336e0892b4c026f --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/index_location.h @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "parquet/platform.h" + +namespace parquet { + +enum class PARQUET_EXPORT IndexKind : uint8_t { + kColumnIndex, + kOffsetIndex, + kBloomFilter, +}; + +/// \brief Location to a Parquet index (page index or bloom filter). +struct PARQUET_EXPORT IndexLocation { + // File offset of the given index, in bytes + int64_t offset; + // Length of the given index, in bytes + int32_t length; +}; + +/// \brief Identifier for a column chunk. +struct PARQUET_EXPORT ColumnChunkId { + int32_t row_group_index; + int32_t column_index; +}; + +using IndexLocations = std::vector>; + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_comparison.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_comparison.h new file mode 100644 index 0000000000000000000000000000000000000000..2529bec7b5e454fbbd77e49e9629a50a9e766526 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_comparison.h @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include +#include + +#include "parquet/platform.h" + +namespace parquet::internal { + +/// Builds a bitmap where each set bit indicates the corresponding level is greater +/// than rhs. +uint64_t PARQUET_EXPORT GreaterThanBitmap(const int16_t* levels, int64_t num_levels, + int16_t rhs); + +struct MinMax { + int16_t min; + int16_t max; +}; + +MinMax FindMinMax(const int16_t* levels, int64_t num_levels); + +} // namespace parquet::internal diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_comparison_inc.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_comparison_inc.h new file mode 100644 index 0000000000000000000000000000000000000000..52f1a8c14fb945f0099bd6c51a42c9504539801b --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_comparison_inc.h @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include "arrow/util/bit_util.h" +#include "arrow/util/endian.h" +#include "parquet/level_comparison.h" + +// Used to make sure ODR rule isn't violated. +#ifndef PARQUET_IMPL_NAMESPACE +# error "PARQUET_IMPL_NAMESPACE must be defined" +#endif +namespace parquet::internal::PARQUET_IMPL_NAMESPACE { +/// Builds a bitmap by applying predicate to the level vector provided. +/// +/// \param[in] levels Rep or def level array. +/// \param[in] num_levels The number of levels to process (must be [0, 64]) +/// \param[in] predicate The predicate to apply (must have the signature `bool +/// predicate(int16_t)`. +/// \returns The bitmap using least significant "bit" ordering. +/// +template +inline uint64_t LevelsToBitmap(const int16_t* levels, int64_t num_levels, + Predicate predicate) { + // Both clang and GCC can vectorize this automatically with SSE4/AVX2. + uint64_t mask = 0; + for (int x = 0; x < num_levels; x++) { + mask |= static_cast(predicate(levels[x]) ? 1 : 0) << x; + } + return ::arrow::bit_util::ToLittleEndian(mask); +} + +inline MinMax FindMinMaxImpl(const int16_t* levels, int64_t num_levels) { + MinMax out{std::numeric_limits::max(), std::numeric_limits::min()}; + for (int x = 0; x < num_levels; x++) { + out.min = std::min(levels[x], out.min); + out.max = std::max(levels[x], out.max); + } + return out; +} + +inline uint64_t GreaterThanBitmapImpl(const int16_t* levels, int64_t num_levels, + int16_t rhs) { + return LevelsToBitmap(levels, num_levels, [rhs](int16_t value) { return value > rhs; }); +} + +} // namespace parquet::internal::PARQUET_IMPL_NAMESPACE diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_conversion.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_conversion.h new file mode 100644 index 0000000000000000000000000000000000000000..3aebd72bd595815493bc75b51a4925c3b18c09eb --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_conversion.h @@ -0,0 +1,216 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "arrow/util/endian.h" +#include "parquet/platform.h" +#include "parquet/schema.h" + +namespace parquet::internal { + +struct PARQUET_EXPORT LevelInfo { + LevelInfo() + : null_slot_usage(1), def_level(0), rep_level(0), repeated_ancestor_def_level(0) {} + LevelInfo(int32_t null_slots, int32_t definition_level, int32_t repetition_level, + int32_t repeated_ancestor_definition_level) + : null_slot_usage(null_slots), + def_level(static_cast(definition_level)), + rep_level(static_cast(repetition_level)), + repeated_ancestor_def_level( + static_cast(repeated_ancestor_definition_level)) {} + + bool operator==(const LevelInfo& b) const { + return null_slot_usage == b.null_slot_usage && def_level == b.def_level && + rep_level == b.rep_level && + repeated_ancestor_def_level == b.repeated_ancestor_def_level; + } + + bool HasNullableValues() const { return repeated_ancestor_def_level < def_level; } + + // How many slots an undefined but present (i.e. null) element in + // parquet consumes when decoding to Arrow. + // "Slot" is used in the same context as the Arrow specification + // (i.e. a value holder). + // This is only ever >1 for descendents of FixedSizeList. + int32_t null_slot_usage = 1; + + // The definition level at which the value for the field + // is considered not null (definition levels greater than + // or equal to this value indicate a not-null + // value for the field). For list fields definition levels + // greater than or equal to this field indicate a present, + // possibly null, child value. + int16_t def_level = 0; + + // The repetition level corresponding to this element + // or the closest repeated ancestor. Any repetition + // level less than this indicates either a new list OR + // an empty list (which is determined in conjunction + // with definition levels). + int16_t rep_level = 0; + + // The definition level indicating the level at which the closest + // repeated ancestor is not empty. This is used to discriminate + // between a value less than |def_level| being null or excluded entirely. + // For instance if we have an arrow schema like: + // list(struct(f0: int)). Then then there are the following + // definition levels: + // 0 = null list + // 1 = present but empty list. + // 2 = a null value in the list + // 3 = a non null struct but null integer. + // 4 = a present integer. + // When reconstructing, the struct and integer arrays' + // repeated_ancestor_def_level would be 2. Any + // def_level < 2 indicates that there isn't a corresponding + // child value in the list. + // i.e. [null, [], [null], [{f0: null}], [{f0: 1}]] + // has the def levels [0, 1, 2, 3, 4]. The actual + // struct array is only of length 3: [not-set, set, set] and + // the int array is also of length 3: [N/A, null, 1]. + // + int16_t repeated_ancestor_def_level = 0; + + /// Increments levels according to the cardinality of node. + void Increment(const schema::Node& node) { + if (node.is_repeated()) { + IncrementRepeated(); + return; + } + if (node.is_optional()) { + IncrementOptional(); + return; + } + } + + /// Increments level for a optional node. + void IncrementOptional() { def_level++; } + + /// Increments levels for the repeated node. Returns + /// the previous ancestor_list_def_level. + int16_t IncrementRepeated() { + int16_t last_repeated_ancestor = repeated_ancestor_def_level; + + // Repeated fields add both a repetition and definition level. This is used + // to distinguish between an empty list and a list with an item in it. + ++rep_level; + ++def_level; + // For levels >= repeated_ancestor_def_level it indicates the list was + // non-null and had at least one element. This is important + // for later decoding because we need to add a slot for these + // values. for levels < current_def_level no slots are added + // to arrays. + repeated_ancestor_def_level = def_level; + return last_repeated_ancestor; + } + + // Calculates and returns LevelInfo for a column descriptor. + static LevelInfo ComputeLevelInfo(const ColumnDescriptor* descr) { + LevelInfo level_info; + level_info.def_level = descr->max_definition_level(); + level_info.rep_level = descr->max_repetition_level(); + + int16_t min_spaced_def_level = descr->max_definition_level(); + const ::parquet::schema::Node* node = descr->schema_node().get(); + while (node && !node->is_repeated()) { + if (node->is_optional()) { + min_spaced_def_level--; + } + node = node->parent(); + } + level_info.repeated_ancestor_def_level = min_spaced_def_level; + return level_info; + } + + friend std::ostream& operator<<(std::ostream& os, const LevelInfo& levels) { + // This print method is to silence valgrind issues. What's printed + // is not important because all asserts happen directly on + // members. + os << "{def=" << levels.def_level << ", rep=" << levels.rep_level + << ", repeated_ancestor_def=" << levels.repeated_ancestor_def_level; + if (levels.null_slot_usage > 1) { + os << ", null_slot_usage=" << levels.null_slot_usage; + } + os << "}"; + return os; + } +}; + +// Input/Output structure for reconstructed validity bitmaps. +struct PARQUET_EXPORT ValidityBitmapInputOutput { + // Input only. + // The maximum number of values_read expected (actual + // values read must be less than or equal to this value). + // If this number is exceeded methods will throw a + // ParquetException. Exceeding this limit indicates + // either a corrupt or incorrectly written file. + int64_t values_read_upper_bound = 0; + // Output only. The number of values added to the encountered + // (this is logically the count of the number of elements + // for an Arrow array). + int64_t values_read = 0; + // Input/Output. The number of nulls encountered. + int64_t null_count = 0; + // Output only. The validity bitmap to populate. Maybe be null only + // for DefRepLevelsToListInfo (if all that is needed is list offsets). + uint8_t* valid_bits = NULLPTR; + // Input only, offset into valid_bits to start at. + int64_t valid_bits_offset = 0; +}; + +// Converts def_levels to validity bitmaps for non-list arrays and structs that have +// at least one member that is not a list and has no list descendents. +// For lists use DefRepLevelsToList and structs where all descendants contain +// a list use DefRepLevelsToBitmap. +void PARQUET_EXPORT DefLevelsToBitmap(const int16_t* def_levels, int64_t num_def_levels, + LevelInfo level_info, + ValidityBitmapInputOutput* output); + +// Reconstructs a validity bitmap and list offsets for a list arrays based on +// def/rep levels. The first element of offsets will not be modified if rep_levels +// starts with a new list. The first element of offsets will be used when calculating +// the next offset. See documentation onf DefLevelsToBitmap for when to use this +// method vs the other ones in this file for reconstruction. +// +// Offsets must be sized to 1 + values_read_upper_bound. +void PARQUET_EXPORT DefRepLevelsToList(const int16_t* def_levels, + const int16_t* rep_levels, int64_t num_def_levels, + LevelInfo level_info, + ValidityBitmapInputOutput* output, + int32_t* offsets); +void PARQUET_EXPORT DefRepLevelsToList(const int16_t* def_levels, + const int16_t* rep_levels, int64_t num_def_levels, + LevelInfo level_info, + ValidityBitmapInputOutput* output, + int64_t* offsets); + +// Reconstructs a validity bitmap for a struct every member is a list or has +// a list descendant. See documentation on DefLevelsToBitmap for when more +// details on this method compared to the other ones defined above. +void PARQUET_EXPORT DefRepLevelsToBitmap(const int16_t* def_levels, + const int16_t* rep_levels, + int64_t num_def_levels, LevelInfo level_info, + ValidityBitmapInputOutput* output); + +// This is exposed to ensure we can properly test a software simulated pext function +// (i.e. it isn't hidden by runtime dispatch). +uint64_t PARQUET_EXPORT TestOnlyExtractBitsSoftware(uint64_t bitmap, uint64_t selection); + +} // namespace parquet::internal diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_conversion_inc.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_conversion_inc.h new file mode 100644 index 0000000000000000000000000000000000000000..39071697da10edc32a140253b50e5d2743f4df7a --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_conversion_inc.h @@ -0,0 +1,356 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include "parquet/level_conversion.h" + +#include +#include +#include +#include + +#include "arrow/util/bit_run_reader.h" +#include "arrow/util/bit_util.h" +#include "arrow/util/bitmap_writer.h" +#include "arrow/util/logging.h" +#include "arrow/util/simd.h" +#include "parquet/exception.h" +#include "parquet/level_comparison.h" + +#ifndef PARQUET_IMPL_NAMESPACE +# error "PARQUET_IMPL_NAMESPACE must be defined" +#endif + +namespace parquet::internal::PARQUET_IMPL_NAMESPACE { + +// clang-format off +/* Python code to generate lookup table: + +kLookupBits = 5 +count = 0 +print('constexpr int kLookupBits = {};'.format(kLookupBits)) +print('constexpr uint8_t kPextTable[1 << kLookupBits][1 << kLookupBits] = {') +print(' ', end = '') +for mask in range(1 << kLookupBits): + for data in range(1 << kLookupBits): + bit_value = 0 + bit_len = 0 + for i in range(kLookupBits): + if mask & (1 << i): + bit_value |= (((data >> i) & 1) << bit_len) + bit_len += 1 + out = '0x{:02X},'.format(bit_value) + count += 1 + if count % (1 << kLookupBits) == 1: + print(' {') + if count % 8 == 1: + print(' ', end = '') + if count % 8 == 0: + print(out, end = '\n') + else: + print(out, end = ' ') + if count % (1 << kLookupBits) == 0: + print(' },', end = '') +print('\n};') + +*/ +// clang-format on + +constexpr int kLookupBits = 5; +constexpr uint8_t kPextTable[1 << kLookupBits][1 << kLookupBits] = { + { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }, + { + 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, + 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, + 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, + }, + { + 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, + 0x01, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, + 0x01, 0x01, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01, + }, + { + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, + 0x03, 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, + 0x02, 0x03, 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03, + }, + { + 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, + 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, + }, + { + 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02, 0x03, 0x00, 0x01, 0x00, + 0x01, 0x02, 0x03, 0x02, 0x03, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, + 0x02, 0x03, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02, 0x03, + }, + { + 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x00, 0x00, 0x01, + 0x01, 0x02, 0x02, 0x03, 0x03, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, + 0x03, 0x03, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, + }, + { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x00, 0x01, 0x02, + 0x03, 0x04, 0x05, 0x06, 0x07, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, + 0x06, 0x07, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + }, + { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + }, + { + 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02, + 0x03, 0x02, 0x03, 0x02, 0x03, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, + 0x00, 0x01, 0x02, 0x03, 0x02, 0x03, 0x02, 0x03, 0x02, 0x03, + }, + { + 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, + 0x03, 0x02, 0x02, 0x03, 0x03, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, + 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x02, 0x02, 0x03, 0x03, + }, + { + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, + 0x07, 0x04, 0x05, 0x06, 0x07, 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, + 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x04, 0x05, 0x06, 0x07, + }, + { + 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02, 0x02, + 0x02, 0x03, 0x03, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, + 0x01, 0x01, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, + }, + { + 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02, 0x03, 0x04, 0x05, 0x04, + 0x05, 0x06, 0x07, 0x06, 0x07, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, + 0x02, 0x03, 0x04, 0x05, 0x04, 0x05, 0x06, 0x07, 0x06, 0x07, + }, + { + 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x04, 0x04, 0x05, + 0x05, 0x06, 0x06, 0x07, 0x07, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, + 0x03, 0x03, 0x04, 0x04, 0x05, 0x05, 0x06, 0x06, 0x07, 0x07, + }, + { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, + 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, + 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + }, + { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + }, + { + 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, + 0x01, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02, 0x03, 0x02, 0x03, + 0x02, 0x03, 0x02, 0x03, 0x02, 0x03, 0x02, 0x03, 0x02, 0x03, + }, + { + 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, + 0x01, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x02, 0x02, + 0x03, 0x03, 0x02, 0x02, 0x03, 0x03, 0x02, 0x02, 0x03, 0x03, + }, + { + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, + 0x03, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x04, 0x05, + 0x06, 0x07, 0x04, 0x05, 0x06, 0x07, 0x04, 0x05, 0x06, 0x07, + }, + { + 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, + 0x03, 0x03, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, + }, + { + 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02, 0x03, 0x00, 0x01, 0x00, + 0x01, 0x02, 0x03, 0x02, 0x03, 0x04, 0x05, 0x04, 0x05, 0x06, 0x07, + 0x06, 0x07, 0x04, 0x05, 0x04, 0x05, 0x06, 0x07, 0x06, 0x07, + }, + { + 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x00, 0x00, 0x01, + 0x01, 0x02, 0x02, 0x03, 0x03, 0x04, 0x04, 0x05, 0x05, 0x06, 0x06, + 0x07, 0x07, 0x04, 0x04, 0x05, 0x05, 0x06, 0x06, 0x07, 0x07, + }, + { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x00, 0x01, 0x02, + 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, + 0x0E, 0x0F, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + }, + { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, + 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, + }, + { + 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02, + 0x03, 0x02, 0x03, 0x02, 0x03, 0x04, 0x05, 0x04, 0x05, 0x04, 0x05, + 0x04, 0x05, 0x06, 0x07, 0x06, 0x07, 0x06, 0x07, 0x06, 0x07, + }, + { + 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, + 0x03, 0x02, 0x02, 0x03, 0x03, 0x04, 0x04, 0x05, 0x05, 0x04, 0x04, + 0x05, 0x05, 0x06, 0x06, 0x07, 0x07, 0x06, 0x06, 0x07, 0x07, + }, + { + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, + 0x07, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x08, 0x09, + 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x0C, 0x0D, 0x0E, 0x0F, + }, + { + 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02, 0x02, + 0x02, 0x03, 0x03, 0x03, 0x03, 0x04, 0x04, 0x04, 0x04, 0x05, 0x05, + 0x05, 0x05, 0x06, 0x06, 0x06, 0x06, 0x07, 0x07, 0x07, 0x07, + }, + { + 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02, 0x03, 0x04, 0x05, 0x04, + 0x05, 0x06, 0x07, 0x06, 0x07, 0x08, 0x09, 0x08, 0x09, 0x0A, 0x0B, + 0x0A, 0x0B, 0x0C, 0x0D, 0x0C, 0x0D, 0x0E, 0x0F, 0x0E, 0x0F, + }, + { + 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x04, 0x04, 0x05, + 0x05, 0x06, 0x06, 0x07, 0x07, 0x08, 0x08, 0x09, 0x09, 0x0A, 0x0A, + 0x0B, 0x0B, 0x0C, 0x0C, 0x0D, 0x0D, 0x0E, 0x0E, 0x0F, 0x0F, + }, + { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, + 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, + 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, + }, +}; + +inline uint64_t ExtractBitsSoftware(uint64_t bitmap, uint64_t select_bitmap) { + // A software emulation of _pext_u64 + + // These checks should be inline and are likely to be common cases. + if (select_bitmap == ~uint64_t{0}) { + return bitmap; + } else if (select_bitmap == 0) { + return 0; + } + + // Fallback to lookup table method + uint64_t bit_value = 0; + int bit_len = 0; + constexpr uint8_t kLookupMask = (1U << kLookupBits) - 1; + while (select_bitmap != 0) { + const auto mask_len = std::popcount(select_bitmap & kLookupMask); + const uint64_t value = kPextTable[select_bitmap & kLookupMask][bitmap & kLookupMask]; + bit_value |= (value << bit_len); + bit_len += mask_len; + bitmap >>= kLookupBits; + select_bitmap >>= kLookupBits; + } + return bit_value; +} + +#ifdef ARROW_HAVE_BMI2 + +// Use _pext_u64 on 64-bit builds, _pext_u32 on 32-bit builds, +# if UINTPTR_MAX == 0xFFFFFFFF + +using extract_bitmap_t = uint32_t; +inline extract_bitmap_t ExtractBits(extract_bitmap_t bitmap, + extract_bitmap_t select_bitmap) { + return _pext_u32(bitmap, select_bitmap); +} + +# else + +using extract_bitmap_t = uint64_t; +inline extract_bitmap_t ExtractBits(extract_bitmap_t bitmap, + extract_bitmap_t select_bitmap) { + return _pext_u64(bitmap, select_bitmap); +} + +# endif + +#else // !defined(ARROW_HAVE_BMI2) + +// Use 64-bit pext emulation when BMI2 isn't available. +using extract_bitmap_t = uint64_t; +inline extract_bitmap_t ExtractBits(extract_bitmap_t bitmap, + extract_bitmap_t select_bitmap) { + return ExtractBitsSoftware(bitmap, select_bitmap); +} + +#endif + +static constexpr int64_t kExtractBitsSize = 8 * sizeof(extract_bitmap_t); + +template +int64_t DefLevelsBatchToBitmap(const int16_t* def_levels, const int64_t batch_size, + int64_t upper_bound_remaining, LevelInfo level_info, + ::arrow::internal::FirstTimeBitmapWriter* writer) { + ARROW_DCHECK_LE(batch_size, kExtractBitsSize); + + // Greater than level_info.def_level - 1 implies >= the def_level + auto defined_bitmap = static_cast(::arrow::bit_util::FromLittleEndian( + internal::GreaterThanBitmap(def_levels, batch_size, level_info.def_level - 1))); + + if (has_repeated_parent) { + // Greater than level_info.repeated_ancestor_def_level - 1 implies >= the + // repeated_ancestor_def_level + auto present_bitmap = static_cast( + ::arrow::bit_util::FromLittleEndian(internal::GreaterThanBitmap( + def_levels, batch_size, level_info.repeated_ancestor_def_level - 1))); + auto selected_bits = ExtractBits(defined_bitmap, present_bitmap); + int64_t selected_count = std::popcount(present_bitmap); + if (ARROW_PREDICT_FALSE(selected_count > upper_bound_remaining)) { + throw ParquetException("Values read exceeded upper bound"); + } + writer->AppendWord(selected_bits, selected_count); + return std::popcount(selected_bits); + } else { + if (ARROW_PREDICT_FALSE(batch_size > upper_bound_remaining)) { + std::stringstream ss; + ss << "Values read exceeded upper bound"; + throw ParquetException(ss.str()); + } + + writer->AppendWord(defined_bitmap, batch_size); + return std::popcount(defined_bitmap); + } +} + +template +void DefLevelsToBitmapSimd(const int16_t* def_levels, int64_t num_def_levels, + LevelInfo level_info, ValidityBitmapInputOutput* output) { + ::arrow::internal::FirstTimeBitmapWriter writer( + output->valid_bits, + /*start_offset=*/output->valid_bits_offset, + /*length=*/output->values_read_upper_bound); + int64_t set_count = 0; + output->values_read = 0; + int64_t values_read_remaining = output->values_read_upper_bound; + while (num_def_levels > kExtractBitsSize) { + set_count += DefLevelsBatchToBitmap( + def_levels, kExtractBitsSize, values_read_remaining, level_info, &writer); + def_levels += kExtractBitsSize; + num_def_levels -= kExtractBitsSize; + values_read_remaining = output->values_read_upper_bound - writer.position(); + } + set_count += DefLevelsBatchToBitmap( + def_levels, num_def_levels, values_read_remaining, level_info, &writer); + + output->values_read = writer.position(); + output->null_count += output->values_read - set_count; + writer.Finish(); +} + +} // namespace parquet::internal::PARQUET_IMPL_NAMESPACE diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/metadata.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/metadata.h new file mode 100644 index 0000000000000000000000000000000000000000..eb752f5a3e5a62d4af4eae00a7a70e1a8fcecb80 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/metadata.h @@ -0,0 +1,536 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "parquet/encryption/type_fwd.h" +#include "parquet/index_location.h" +#include "parquet/platform.h" +#include "parquet/properties.h" +#include "parquet/type_fwd.h" +namespace parquet { + +using KeyValueMetadata = ::arrow::KeyValueMetadata; + +class PARQUET_EXPORT ApplicationVersion { + public: + // Known Versions with Issues + static const ApplicationVersion& PARQUET_251_FIXED_VERSION(); + static const ApplicationVersion& PARQUET_816_FIXED_VERSION(); + static const ApplicationVersion& PARQUET_CPP_FIXED_STATS_VERSION(); + static const ApplicationVersion& PARQUET_MR_FIXED_STATS_VERSION(); + static const ApplicationVersion& PARQUET_CPP_10353_FIXED_VERSION(); + + // Application that wrote the file. e.g. "IMPALA" + std::string application_; + // Build name + std::string build_; + + // Version of the application that wrote the file, expressed as + // (..). Unmatched parts default to 0. + // "1.2.3" => {1, 2, 3} + // "1.2" => {1, 2, 0} + // "1.2-cdh5" => {1, 2, 0} + struct { + int major; + int minor; + int patch; + std::string unknown; + std::string pre_release; + std::string build_info; + } version; + + ApplicationVersion() = default; + explicit ApplicationVersion(const std::string& created_by); + ApplicationVersion(std::string application, int major, int minor, int patch); + + // Returns true if version is strictly less than other_version + bool VersionLt(const ApplicationVersion& other_version) const; + + // Returns true if version is strictly equal with other_version + bool VersionEq(const ApplicationVersion& other_version) const; + + // Checks if the Version has the correct statistics for a given column + bool HasCorrectStatistics(Type::type primitive, const EncodedStatistics& statistics, + SortOrder::type sort_order = SortOrder::SIGNED) const; +}; + +class PARQUET_EXPORT ColumnCryptoMetaData { + public: + static std::unique_ptr Make(const uint8_t* metadata); + ~ColumnCryptoMetaData(); + + bool Equals(const ColumnCryptoMetaData& other) const; + + std::shared_ptr path_in_schema() const; + bool encrypted_with_footer_key() const; + const std::string& key_metadata() const; + + private: + explicit ColumnCryptoMetaData(const uint8_t* metadata); + + class ColumnCryptoMetaDataImpl; + std::unique_ptr impl_; +}; + +/// \brief Public struct for Thrift PageEncodingStats in ColumnChunkMetaData +struct PageEncodingStats { + PageType::type page_type; + Encoding::type encoding; + int32_t count; +}; + +/// \brief ColumnChunkMetaData is a proxy around format::ColumnChunkMetaData. +class PARQUET_EXPORT ColumnChunkMetaData { + public: + // API convenience to get a MetaData accessor + static std::unique_ptr Make( + const void* metadata, const ColumnDescriptor* descr, + const ReaderProperties& properties = default_reader_properties(), + const ApplicationVersion* writer_version = NULLPTR, int16_t row_group_ordinal = -1, + int16_t column_ordinal = -1, + std::shared_ptr file_decryptor = NULLPTR); + + ~ColumnChunkMetaData(); + + bool Equals(const ColumnChunkMetaData& other) const; + + // Byte offset of `ColumnMetaData` in `file_path()`. + // + // Note that the meaning of this field has been inconsistent among implementations + // so its use has since been deprecated in the Parquet specification. Modern + // implementations will set this to `0` to indicate that the `ColumnMetaData` is solely + // contained in the `ColumnChunk` struct. + int64_t file_offset() const; + + // parameter is only used when a dataset is spread across multiple files + const std::string& file_path() const; + + // column metadata + Type::type type() const; + int64_t num_values() const; + std::shared_ptr path_in_schema() const; + bool is_stats_set() const; + bool is_geo_stats_set() const; + std::shared_ptr statistics() const; + std::shared_ptr encoded_statistics() const; + std::shared_ptr size_statistics() const; + std::shared_ptr geo_statistics() const; + + Compression::type compression() const; + // Indicate if the ColumnChunk compression is supported by the current + // compiled parquet library. + bool can_decompress() const; + + const std::vector& encodings() const; + const std::vector& encoding_stats() const; + std::optional bloom_filter_offset() const; + std::optional bloom_filter_length() const; + bool has_dictionary_page() const; + int64_t dictionary_page_offset() const; + int64_t data_page_offset() const; + bool has_index_page() const; + int64_t index_page_offset() const; + int64_t total_compressed_size() const; + int64_t total_uncompressed_size() const; + std::unique_ptr crypto_metadata() const; + std::optional GetColumnIndexLocation() const; + std::optional GetOffsetIndexLocation() const; + const std::shared_ptr& key_value_metadata() const; + + private: + explicit ColumnChunkMetaData( + const void* metadata, const ColumnDescriptor* descr, int16_t row_group_ordinal, + int16_t column_ordinal, const ReaderProperties& properties, + const ApplicationVersion* writer_version = NULLPTR, + std::shared_ptr file_decryptor = NULLPTR); + // PIMPL Idiom + class ColumnChunkMetaDataImpl; + std::unique_ptr impl_; +}; + +/// \brief RowGroupMetaData is a proxy around format::RowGroupMetaData. +class PARQUET_EXPORT RowGroupMetaData { + public: + /// \brief Create a RowGroupMetaData from a serialized thrift message. + static std::unique_ptr Make( + const void* metadata, const SchemaDescriptor* schema, + const ReaderProperties& properties = default_reader_properties(), + const ApplicationVersion* writer_version = NULLPTR, + std::shared_ptr file_decryptor = NULLPTR); + + ~RowGroupMetaData(); + + bool Equals(const RowGroupMetaData& other) const; + + /// \brief The number of columns in this row group. The order must match the + /// parent's column ordering. + int num_columns() const; + + /// \brief Return the ColumnChunkMetaData of the corresponding column ordinal. + /// + /// WARNING, the returned object references memory location in it's parent + /// (RowGroupMetaData) object. Hence, the parent must outlive the returned + /// object. + /// + /// \param[in] index of the ColumnChunkMetaData to retrieve. + /// + /// \throws ParquetException if the index is out of bound. + std::unique_ptr ColumnChunk(int index) const; + + /// \brief Number of rows in this row group. + int64_t num_rows() const; + + /// \brief Total byte size of all the uncompressed column data in this row group. + int64_t total_byte_size() const; + + /// \brief Total byte size of all the compressed (and potentially encrypted) + /// column data in this row group. + /// + /// This information is optional and may be 0 if omitted. + int64_t total_compressed_size() const; + + /// \brief Byte offset from beginning of file to first page (data or + /// dictionary) in this row group + /// + /// The file_offset field that this method exposes is optional. This method + /// will return 0 if that field is not set to a meaningful value. + int64_t file_offset() const; + // Return const-pointer to make it clear that this object is not to be copied + const SchemaDescriptor* schema() const; + // Indicate if all of the RowGroup's ColumnChunks can be decompressed. + bool can_decompress() const; + // Sorting columns of the row group if any. + std::vector sorting_columns() const; + + private: + explicit RowGroupMetaData( + const void* metadata, const SchemaDescriptor* schema, + const ReaderProperties& properties, + const ApplicationVersion* writer_version = NULLPTR, + std::shared_ptr file_decryptor = NULLPTR); + // PIMPL Idiom + class RowGroupMetaDataImpl; + std::unique_ptr impl_; +}; + +class FileMetaDataBuilder; + +/// \brief FileMetaData is a proxy around format::FileMetaData. +class PARQUET_EXPORT FileMetaData { + public: + /// \brief Create a FileMetaData from a serialized thrift message. + static std::shared_ptr Make( + const void* serialized_metadata, uint32_t* inout_metadata_len, + const ReaderProperties& properties = default_reader_properties(), + std::shared_ptr file_decryptor = NULLPTR); + + ~FileMetaData(); + + bool Equals(const FileMetaData& other) const; + + /// \brief The number of parquet "leaf" columns. + /// + /// Parquet thrift definition requires that nested schema elements are + /// flattened. This method returns the number of columns in the flattened + /// version. + /// For instance, if the schema looks like this : + /// 0 foo.bar + /// foo.bar.baz 0 + /// foo.bar.baz2 1 + /// foo.qux 2 + /// 1 foo2 3 + /// 2 foo3 4 + /// This method will return 5, because there are 5 "leaf" fields (so 5 + /// flattened fields) + int num_columns() const; + + /// \brief The number of flattened schema elements. + /// + /// Parquet thrift definition requires that nested schema elements are + /// flattened. This method returns the total number of elements in the + /// flattened list. + int num_schema_elements() const; + + /// \brief The total number of rows. + /// + /// If the FileMetaData was obtained by calling `SubSet()`, this is the total + /// number of rows in the selected row groups. + int64_t num_rows() const; + + /// \brief The number of row groups in the file. + /// + /// If the FileMetaData was obtained by calling `SubSet()`, this is the number + /// of selected row groups. + int num_row_groups() const; + + /// \brief Return the RowGroupMetaData of the corresponding row group ordinal. + /// + /// WARNING, the returned object references memory location in it's parent + /// (FileMetaData) object. Hence, the parent must outlive the returned object. + /// + /// \param[in] index of the RowGroup to retrieve. + /// + /// \throws ParquetException if the index is out of bound. + std::unique_ptr RowGroup(int index) const; + + /// \brief Return the "version" of the file + /// + /// WARNING: The value returned by this method is unreliable as 1) the Parquet + /// file metadata stores the version as a single integer and 2) some producers + /// are known to always write a hardcoded value. Therefore, you cannot use + /// this value to know which features are used in the file. + ParquetVersion::type version() const; + + /// \brief Return the application's user-agent string of the writer. + const std::string& created_by() const; + + /// \brief Return the application's version of the writer. + const ApplicationVersion& writer_version() const; + + /// \brief Size of the original thrift encoded metadata footer. + uint32_t size() const; + + /// \brief Indicate if all of the FileMetaData's RowGroups can be decompressed. + /// + /// This will return false if any of the RowGroup's page is compressed with a + /// compression format which is not compiled in the current parquet library. + bool can_decompress() const; + + bool is_encryption_algorithm_set() const; + EncryptionAlgorithm encryption_algorithm() const; + const std::string& footer_signing_key_metadata() const; + + PARQUET_DEPRECATED( + "Deprecated in 24.0.0. If you need this functionality, please report an issue.") + bool VerifySignature(const void* signature); + + void WriteTo(::arrow::io::OutputStream* dst, + const std::shared_ptr& encryptor = NULLPTR) const; + + /// \brief Return Thrift-serialized representation of the metadata as a + /// string + std::string SerializeToString() const; + + // Return const-pointer to make it clear that this object is not to be copied + const SchemaDescriptor* schema() const; + + const std::shared_ptr& key_value_metadata() const; + + /// \brief Set a path to all ColumnChunk for all RowGroups. + /// + /// Commonly used by systems (Dask, Spark) who generates an metadata-only + /// parquet file. The path is usually relative to said index file. + /// + /// \param[in] path to set. + void set_file_path(const std::string& path); + + /// \brief Merge row groups from another metadata file into this one. + /// + /// The schema of the input FileMetaData must be equal to the + /// schema of this object. + /// + /// This is used by systems who creates an aggregate metadata-only file by + /// concatenating the row groups of multiple files. This newly created + /// metadata file acts as an index of all available row groups. + /// + /// \param[in] other FileMetaData to merge the row groups from. + /// + /// \throws ParquetException if schemas are not equal. + void AppendRowGroups(const FileMetaData& other); + + /// \brief Return a FileMetaData containing a subset of the row groups in this + /// FileMetaData. + std::shared_ptr Subset(const std::vector& row_groups) const; + + /// \brief Serialize metadata unencrypted as string + /// + /// \param[in] scrub whether to remove sensitive information from the metadata. + /// \param[in] debug whether to serialize the metadata as Thrift (if false) or + /// debug text (if true). + std::string SerializeUnencrypted(bool scrub, bool debug) const; + + private: + friend FileMetaDataBuilder; + friend class SerializedFile; + friend class SerializedRowGroup; + + explicit FileMetaData(const void* serialized_metadata, uint32_t* metadata_len, + const ReaderProperties& properties, + std::shared_ptr file_decryptor = NULLPTR); + + void set_file_decryptor(std::shared_ptr file_decryptor); + const std::shared_ptr& file_decryptor() const; + + // Verify the signature of a plaintext footer. + static bool VerifySignature(std::span serialized_metadata, + std::span signature, + InternalFileDecryptor* file_decryptor); + + // PIMPL Idiom + FileMetaData(); + class FileMetaDataImpl; + std::unique_ptr impl_; +}; + +class PARQUET_EXPORT FileCryptoMetaData { + public: + // API convenience to get a MetaData accessor + static std::shared_ptr Make( + const uint8_t* serialized_metadata, uint32_t* metadata_len, + const ReaderProperties& properties = default_reader_properties()); + ~FileCryptoMetaData(); + + EncryptionAlgorithm encryption_algorithm() const; + const std::string& key_metadata() const; + + void WriteTo(::arrow::io::OutputStream* dst) const; + + private: + friend FileMetaDataBuilder; + FileCryptoMetaData(const uint8_t* serialized_metadata, uint32_t* metadata_len, + const ReaderProperties& properties); + + // PIMPL Idiom + FileCryptoMetaData(); + class FileCryptoMetaDataImpl; + std::unique_ptr impl_; +}; + +// Builder API +class PARQUET_EXPORT ColumnChunkMetaDataBuilder { + public: + // API convenience to get a MetaData reader + static std::unique_ptr Make( + std::shared_ptr props, const ColumnDescriptor* column); + + static std::unique_ptr Make( + std::shared_ptr props, const ColumnDescriptor* column, + void* contents); + + ~ColumnChunkMetaDataBuilder(); + + // column chunk + // Used when a dataset is spread across multiple files + void set_file_path(const std::string& path); + + // column metadata + void SetStatistics(const EncodedStatistics& stats); + void SetSizeStatistics(const SizeStatistics& size_stats); + + // column geometry statistics + void SetGeoStatistics(const geospatial::EncodedGeoStatistics& geo_stats); + + void SetKeyValueMetadata(std::shared_ptr key_value_metadata); + + // get the column descriptor + const ColumnDescriptor* descr() const; + + int64_t total_compressed_size() const; + // commit the metadata + + void Finish(int64_t num_values, int64_t dictionary_page_offset, + int64_t index_page_offset, int64_t data_page_offset, + int64_t compressed_size, int64_t uncompressed_size, bool has_dictionary, + bool dictionary_fallback, + const std::map& dict_encoding_stats_, + const std::map& data_encoding_stats_, + const std::shared_ptr& encryptor = NULLPTR); + + // The metadata contents, suitable for passing to ColumnChunkMetaData::Make + const void* contents() const; + + // For writing metadata at end of column chunk + void WriteTo(::arrow::io::OutputStream* sink); + + private: + explicit ColumnChunkMetaDataBuilder(std::shared_ptr props, + const ColumnDescriptor* column); + explicit ColumnChunkMetaDataBuilder(std::shared_ptr props, + const ColumnDescriptor* column, void* contents); + // PIMPL Idiom + class ColumnChunkMetaDataBuilderImpl; + std::unique_ptr impl_; +}; + +class PARQUET_EXPORT RowGroupMetaDataBuilder { + public: + // API convenience to get a MetaData reader + static std::unique_ptr Make( + std::shared_ptr props, const SchemaDescriptor* schema_, + void* contents); + + ~RowGroupMetaDataBuilder(); + + ColumnChunkMetaDataBuilder* NextColumnChunk(); + int num_columns(); + int64_t num_rows(); + int current_column() const; + + void set_num_rows(int64_t num_rows); + + // commit the metadata + void Finish(int64_t total_bytes_written, int16_t row_group_ordinal = -1); + + private: + explicit RowGroupMetaDataBuilder(std::shared_ptr props, + const SchemaDescriptor* schema_, void* contents); + // PIMPL Idiom + class RowGroupMetaDataBuilderImpl; + std::unique_ptr impl_; +}; + +class PARQUET_EXPORT FileMetaDataBuilder { + public: + // API convenience to get a MetaData builder + static std::unique_ptr Make( + const SchemaDescriptor* schema, std::shared_ptr props); + + ~FileMetaDataBuilder(); + + // The prior RowGroupMetaDataBuilder (if any) is destroyed + RowGroupMetaDataBuilder* AppendRowGroup(); + + // Set locations of all column chunks of a specific index kind. + void SetIndexLocations(IndexKind kind, const IndexLocations& locations); + + // Complete the Thrift structure + std::unique_ptr Finish( + const std::shared_ptr& key_value_metadata = NULLPTR); + + // crypto metadata + std::unique_ptr GetCryptoMetaData(); + + private: + explicit FileMetaDataBuilder(const SchemaDescriptor* schema, + std::shared_ptr props); + // PIMPL Idiom + class FileMetaDataBuilderImpl; + std::unique_ptr impl_; +}; + +PARQUET_EXPORT std::string ParquetVersionToString(ParquetVersion::type ver); + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/page_index.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/page_index.h new file mode 100644 index 0000000000000000000000000000000000000000..409aa277e733266694b73b19669ffd7ebc57aa40 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/page_index.h @@ -0,0 +1,391 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "arrow/io/interfaces.h" +#include "parquet/encryption/type_fwd.h" +#include "parquet/index_location.h" +#include "parquet/type_fwd.h" +#include "parquet/types.h" + +#include +#include + +namespace parquet { + +/// \brief ColumnIndex is a proxy around format::ColumnIndex. +class PARQUET_EXPORT ColumnIndex { + public: + /// \brief Create a ColumnIndex from a serialized thrift message. + static std::unique_ptr Make(const ColumnDescriptor& descr, + const void* serialized_index, + uint32_t index_len, + const ReaderProperties& properties, + Decryptor* decryptor = NULLPTR); + + virtual ~ColumnIndex() = default; + + /// \brief A bitmap with a bit set for each data page that has only null values. + /// + /// The length of this vector is equal to the number of data pages in the column. + virtual const std::vector& null_pages() const = 0; + + /// \brief A vector of encoded lower bounds for each data page in this column. + /// + /// `null_pages` should be inspected first, as only pages with non-null values + /// may have their lower bounds populated. + virtual const std::vector& encoded_min_values() const = 0; + + /// \brief A vector of encoded upper bounds for each data page in this column. + /// + /// `null_pages` should be inspected first, as only pages with non-null values + /// may have their upper bounds populated. + virtual const std::vector& encoded_max_values() const = 0; + + /// \brief The ordering of lower and upper bounds. + /// + /// The boundary order applies across all lower bounds, and all upper bounds, + /// respectively. However, the order between lower bounds and upper bounds + /// cannot be derived from this. + virtual BoundaryOrder::type boundary_order() const = 0; + + /// \brief Whether per-page null count information is available. + virtual bool has_null_counts() const = 0; + + /// \brief An optional vector with the number of null values in each data page. + /// + /// `has_null_counts` should be called first to determine if this information is + /// available. + virtual const std::vector& null_counts() const = 0; + + /// \brief A vector of page indices for non-null pages. + virtual const std::vector& non_null_page_indices() const = 0; + + /// \brief Whether definition level histogram is available. + virtual bool has_definition_level_histograms() const = 0; + + /// \brief Whether repetition level histogram is available. + virtual bool has_repetition_level_histograms() const = 0; + + /// \brief List of definition level histograms for each page concatenated together. + virtual const std::vector& definition_level_histograms() const = 0; + + /// \brief List of repetition level histograms for each page concatenated together. + virtual const std::vector& repetition_level_histograms() const = 0; +}; + +/// \brief Typed implementation of ColumnIndex. +template +class PARQUET_EXPORT TypedColumnIndex : public ColumnIndex { + public: + using T = typename DType::c_type; + + /// \brief A vector of lower bounds for each data page in this column. + /// + /// This is like `encoded_min_values`, but with the values decoded according to + /// the column's physical type. + /// `min_values` and `max_values` can be used together with `boundary_order` + /// in order to prune some data pages when searching for specific values. + virtual const std::vector& min_values() const = 0; + + /// \brief A vector of upper bounds for each data page in this column. + /// + /// Just like `min_values`, but for upper bounds instead of lower bounds. + virtual const std::vector& max_values() const = 0; +}; + +using BoolColumnIndex = TypedColumnIndex; +using Int32ColumnIndex = TypedColumnIndex; +using Int64ColumnIndex = TypedColumnIndex; +using FloatColumnIndex = TypedColumnIndex; +using DoubleColumnIndex = TypedColumnIndex; +using ByteArrayColumnIndex = TypedColumnIndex; +using FLBAColumnIndex = TypedColumnIndex; + +/// \brief PageLocation is a proxy around format::PageLocation. +struct PARQUET_EXPORT PageLocation { + /// File offset of the data page. + int64_t offset; + /// Total compressed size of the data page and header. + int32_t compressed_page_size; + /// Row id of the first row in the page within the row group. + int64_t first_row_index; +}; + +/// \brief OffsetIndex is a proxy around format::OffsetIndex. +class PARQUET_EXPORT OffsetIndex { + public: + /// \brief Create a OffsetIndex from a serialized thrift message. + static std::unique_ptr Make(const void* serialized_index, + uint32_t index_len, + const ReaderProperties& properties, + Decryptor* decryptor = NULLPTR); + + virtual ~OffsetIndex() = default; + + /// \brief A vector of locations for each data page in this column. + virtual const std::vector& page_locations() const = 0; + + /// \brief A vector of unencoded/uncompressed size of each page for BYTE_ARRAY types, + /// or empty for other types. + virtual const std::vector& unencoded_byte_array_data_bytes() const = 0; +}; + +/// \brief Interface for reading the page index for a Parquet row group. +class PARQUET_EXPORT RowGroupPageIndexReader { + public: + virtual ~RowGroupPageIndexReader() = default; + + /// \brief Read column index of a column chunk. + /// + /// \param[in] i column ordinal of the column chunk. + /// \returns column index of the column or nullptr if it does not exist. + /// \throws ParquetException if the index is out of bound. + virtual std::shared_ptr GetColumnIndex(int32_t i) = 0; + + /// \brief Read offset index of a column chunk. + /// + /// \param[in] i column ordinal of the column chunk. + /// \returns offset index of the column or nullptr if it does not exist. + /// \throws ParquetException if the index is out of bound. + virtual std::shared_ptr GetOffsetIndex(int32_t i) = 0; +}; + +struct PageIndexSelection { + /// Specifies whether to read the column index. + bool column_index = false; + /// Specifies whether to read the offset index. + bool offset_index = false; +}; + +PARQUET_EXPORT +std::ostream& operator<<(std::ostream& out, const PageIndexSelection& params); + +struct RowGroupIndexReadRange { + /// Base start and total size of column index of all column chunks in a row group. + /// If none of the column chunks have column index, it is set to std::nullopt. + std::optional<::arrow::io::ReadRange> column_index = std::nullopt; + /// Base start and total size of offset index of all column chunks in a row group. + /// If none of the column chunks have offset index, it is set to std::nullopt. + std::optional<::arrow::io::ReadRange> offset_index = std::nullopt; +}; + +/// \brief Interface for reading the page index for a Parquet file. +class PARQUET_EXPORT PageIndexReader { + public: + virtual ~PageIndexReader() = default; + + /// \brief Create a PageIndexReader instance. + /// \returns a PageIndexReader instance. + /// WARNING: The returned PageIndexReader references to all the input parameters, so + /// it must not outlive all of the input parameters. Usually these input parameters + /// come from the same ParquetFileReader object, so it must not outlive the reader + /// that creates this PageIndexReader. + static std::shared_ptr Make( + ::arrow::io::RandomAccessFile* input, std::shared_ptr file_metadata, + const ReaderProperties& properties, + InternalFileDecryptor* file_decryptor = NULLPTR); + + /// \brief Get the page index reader of a specific row group. + /// \param[in] i row group ordinal to get page index reader. + /// \returns RowGroupPageIndexReader of the specified row group. A nullptr may or may + /// not be returned if the page index for the row group is unavailable. It is + /// the caller's responsibility to check the return value of follow-up calls + /// to the RowGroupPageIndexReader. + /// \throws ParquetException if the index is out of bound. + virtual std::shared_ptr RowGroup(int i) = 0; + + /// \brief Advise the reader which part of page index will be read later. + /// + /// The PageIndexReader can optionally prefetch and cache page index that + /// may be read later to get better performance. + /// + /// The contract of this function is as below: + /// 1) If WillNeed() has not been called for a specific row group and the page index + /// exists, follow-up calls to get column index or offset index of all columns in + /// this row group SHOULD NOT FAIL, but the performance may not be optimal. + /// 2) If WillNeed() has been called for a specific row group, follow-up calls to get + /// page index are limited to columns and index type requested by WillNeed(). + /// So it MAY FAIL if columns that are not requested by WillNeed() are requested. + /// 3) Later calls to WillNeed() MAY OVERRIDE previous calls of same row groups. + /// For example, + /// 1) If WillNeed() is not called for row group 0, then follow-up calls to read + /// column index and/or offset index of all columns of row group 0 should not + /// fail if its page index exists. + /// 2) If WillNeed() is called for columns 0 and 1 for row group 0, then follow-up + /// call to read page index of column 2 for row group 0 MAY FAIL even if its + /// page index exists. + /// 3) If WillNeed() is called for row group 0 with offset index only, then + /// follow-up call to read column index of row group 0 MAY FAIL even if + /// the column index of this column exists. + /// 4) If WillNeed() is called for columns 0 and 1 for row group 0, then later + /// call to WillNeed() for columns 1 and 2 for row group 0. The later one + /// overrides previous call and only columns 1 and 2 of row group 0 are allowed + /// to access. + /// + /// \param[in] row_group_indices list of row group ordinal to read page index later. + /// \param[in] column_indices list of column ordinal to read page index later. If it is + /// empty, it means all columns in the row group will be read. + /// \param[in] selection which kind of page index is required later. + virtual void WillNeed(const std::vector& row_group_indices, + const std::vector& column_indices, + const PageIndexSelection& selection) = 0; + + /// \brief Advise the reader page index of these row groups will not be read anymore. + /// + /// The PageIndexReader implementation has the opportunity to cancel any prefetch or + /// release resource that are related to these row groups. + /// + /// \param[in] row_group_indices list of row group ordinal that whose page index will + /// not be accessed anymore. + virtual void WillNotNeed(const std::vector& row_group_indices) = 0; + + /// \brief Determine the column index and offset index ranges for the given row group. + /// + /// \param[in] row_group_metadata row group metadata to get column chunk metadata. + /// \param[in] columns list of column ordinals to get page index. If the list is empty, + /// it means all columns in the row group. + /// \returns RowGroupIndexReadRange of the specified row group. Throws ParquetException + /// if the selected column ordinal is out of bound or metadata of page index + /// is corrupted. + static RowGroupIndexReadRange DeterminePageIndexRangesInRowGroup( + const RowGroupMetaData& row_group_metadata, const std::vector& columns); +}; + +/// \brief Interface for collecting column index of data pages in a column chunk. +class PARQUET_EXPORT ColumnIndexBuilder { + public: + /// \brief API convenience to create a ColumnIndexBuilder. + static std::unique_ptr Make(const ColumnDescriptor* descr); + + virtual ~ColumnIndexBuilder() = default; + + /// \brief Add statistics of a data page. + /// + /// If the ColumnIndexBuilder has seen any corrupted statistics, it will + /// not update statistics anymore. + /// + /// \param stats Page statistics in the encoded form. + /// \param size_stats Size statistics of the page if available. + virtual void AddPage(const EncodedStatistics& stats, + const SizeStatistics& size_stats) = 0; + + /// \brief Complete the column index. + /// + /// Once called, AddPage() can no longer be called. + /// WriteTo() and Build() can only called after Finish() has been called. + virtual void Finish() = 0; + + /// \brief Serialize the column index thrift message. + /// + /// If the ColumnIndexBuilder has seen any corrupted statistics, it will + /// not write any data to the sink. + /// + /// \param[out] sink output stream to write the serialized message. + /// \param[in] encryptor encryptor to encrypt the serialized column index. + virtual void WriteTo(::arrow::io::OutputStream* sink, + Encryptor* encryptor = NULLPTR) const = 0; + + /// \brief Create a ColumnIndex directly. + /// + /// \return If the ColumnIndexBuilder has seen any corrupted statistics, it simply + /// returns nullptr. Otherwise the column index is built and returned. + virtual std::unique_ptr Build() const = 0; +}; + +/// \brief Interface for collecting offset index of data pages in a column chunk. +class PARQUET_EXPORT OffsetIndexBuilder { + public: + /// \brief API convenience to create a OffsetIndexBuilder. + static std::unique_ptr Make(); + + virtual ~OffsetIndexBuilder() = default; + + /// \brief Add page location and size stats of a data page. + virtual void AddPage(int64_t offset, int32_t compressed_page_size, + int64_t first_row_index, + std::optional unencoded_byte_array_length = {}) = 0; + + /// \brief Add page location and size stats of a data page. + void AddPage(const PageLocation& page_location, const SizeStatistics& size_stats); + + /// \brief Complete the offset index. + /// + /// In the buffered row group mode, data pages are flushed into memory + /// sink and the OffsetIndexBuilder has only collected the relative offset + /// which requires adjustment once they are flushed to the file. + /// + /// \param final_position Final stream offset to add for page offset adjustment. + virtual void Finish(int64_t final_position) = 0; + + /// \brief Serialize the offset index thrift message. + /// + /// \param[out] sink output stream to write the serialized message. + /// \param[in] encryptor encryptor to encrypt the serialized offset index. + virtual void WriteTo(::arrow::io::OutputStream* sink, + Encryptor* encryptor = NULLPTR) const = 0; + + /// \brief Create an OffsetIndex directly. + virtual std::unique_ptr Build() const = 0; +}; + +/// \brief Interface for collecting page index of a parquet file. +class PARQUET_EXPORT PageIndexBuilder { + public: + /// \brief API convenience to create a PageIndexBuilder. + static std::unique_ptr Make( + const SchemaDescriptor* schema, InternalFileEncryptor* file_encryptor = NULLPTR); + + virtual ~PageIndexBuilder() = default; + + /// \brief Start a new row group. + virtual void AppendRowGroup() = 0; + + /// \brief Get the ColumnIndexBuilder from column ordinal. + /// + /// \param i Column ordinal. + /// \return ColumnIndexBuilder for the column and its memory ownership belongs to + /// the PageIndexBuilder. + virtual ColumnIndexBuilder* GetColumnIndexBuilder(int32_t i) = 0; + + /// \brief Get the OffsetIndexBuilder from column ordinal. + /// + /// \param i Column ordinal. + /// \return OffsetIndexBuilder for the column and its memory ownership belongs to + /// the PageIndexBuilder. + virtual OffsetIndexBuilder* GetOffsetIndexBuilder(int32_t i) = 0; + + /// \brief Complete the page index builder and no more write is allowed. + virtual void Finish() = 0; + + struct WriteResult { + IndexLocations column_index_locations; + IndexLocations offset_index_locations; + }; + + /// \brief Serialize the page index thrift message. + /// + /// Only valid column indexes and offset indexes are serialized and their locations + /// are set. + /// + /// \param[out] sink The output stream to write the page index. + /// \return The location of all page indexes. + virtual WriteResult WriteTo(::arrow::io::OutputStream* sink) const = 0; +}; + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/parquet_version.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/parquet_version.h new file mode 100644 index 0000000000000000000000000000000000000000..7b8e661ee1a2dfb81ccfe589757cafa3c7a9618a --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/parquet_version.h @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef PARQUET_VERSION_H +#define PARQUET_VERSION_H + +#define PARQUET_VERSION_MAJOR 24 +#define PARQUET_VERSION_MINOR 0 +#define PARQUET_VERSION_PATCH 0 + +#define PARQUET_SO_VERSION "2400" +#define PARQUET_FULL_SO_VERSION "2400.0.0" + +// define the parquet created by version +#define CREATED_BY_VERSION "parquet-cpp-arrow version 24.0.0" + +#endif // PARQUET_VERSION_H diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/platform.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/platform.h new file mode 100644 index 0000000000000000000000000000000000000000..9ece8ae5550d8ec97051b6c58e6ae56abf40b2b3 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/platform.h @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "arrow/buffer.h" // IWYU pragma: export +#include "arrow/io/interfaces.h" // IWYU pragma: export +#include "arrow/status.h" // IWYU pragma: export +#include "arrow/type_fwd.h" // IWYU pragma: export +#include "arrow/util/macros.h" // IWYU pragma: export + +#if defined(_WIN32) || defined(__CYGWIN__) + +# ifdef _MSC_VER +# pragma warning(push) +// Disable warning for STL types usage in DLL interface +// https://web.archive.org/web/20130317015847/http://connect.microsoft.com/VisualStudio/feedback/details/696593/vc-10-vs-2010-basic-string-exports +# pragma warning(disable : 4275 4251) +// Disable diamond inheritance warnings +# pragma warning(disable : 4250) +// Disable macro redefinition warnings +# pragma warning(disable : 4005) +// Disable extern before exported template warnings +# pragma warning(disable : 4910) +# else +# pragma GCC diagnostic ignored "-Wattributes" +# endif + +# ifdef PARQUET_STATIC +# define PARQUET_EXPORT +# elif defined(PARQUET_EXPORTING) +# define PARQUET_EXPORT __declspec(dllexport) +# else +# define PARQUET_EXPORT __declspec(dllimport) +# endif + +# define PARQUET_NO_EXPORT + +#else // Not Windows +# ifndef PARQUET_EXPORT +# define PARQUET_EXPORT __attribute__((visibility("default"))) +# endif +# ifndef PARQUET_NO_EXPORT +# define PARQUET_NO_EXPORT __attribute__((visibility("hidden"))) +# endif +#endif // Non-Windows + +// This is a complicated topic, some reading on it: +// http://www.codesynthesis.com/~boris/blog/2010/01/18/dll-export-cxx-templates/ +#if defined(_MSC_VER) || defined(__clang__) +# define PARQUET_TEMPLATE_CLASS_EXPORT +# define PARQUET_TEMPLATE_EXPORT PARQUET_EXPORT +#else +# define PARQUET_TEMPLATE_CLASS_EXPORT PARQUET_EXPORT +# define PARQUET_TEMPLATE_EXPORT +#endif + +#define PARQUET_DISALLOW_COPY_AND_ASSIGN ARROW_DISALLOW_COPY_AND_ASSIGN + +#define PARQUET_NORETURN ARROW_NORETURN +#define PARQUET_DEPRECATED ARROW_DEPRECATED + +// If ARROW_VALGRIND set when compiling unit tests, also define +// PARQUET_VALGRIND +#ifdef ARROW_VALGRIND +# define PARQUET_VALGRIND +#endif + +namespace parquet { + +using Buffer = ::arrow::Buffer; +using Codec = ::arrow::util::Codec; +using CodecOptions = ::arrow::util::CodecOptions; +using Compression = ::arrow::Compression; +using MemoryPool = ::arrow::MemoryPool; +using MutableBuffer = ::arrow::MutableBuffer; +using ResizableBuffer = ::arrow::ResizableBuffer; +using ResizableBuffer = ::arrow::ResizableBuffer; +using ArrowInputFile = ::arrow::io::RandomAccessFile; +using ArrowInputStream = ::arrow::io::InputStream; +using ArrowOutputStream = ::arrow::io::OutputStream; + +constexpr int64_t kDefaultOutputStreamSize = 1024; + +constexpr int16_t kNonPageOrdinal = static_cast(-1); + +PARQUET_EXPORT +std::shared_ptr<::arrow::io::BufferOutputStream> CreateOutputStream( + ::arrow::MemoryPool* pool = ::arrow::default_memory_pool()); + +PARQUET_EXPORT +std::shared_ptr AllocateBuffer( + ::arrow::MemoryPool* pool = ::arrow::default_memory_pool(), int64_t size = 0); + +} // namespace parquet + +#ifdef _MSC_VER +# pragma warning(pop) +#endif diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/printer.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/printer.h new file mode 100644 index 0000000000000000000000000000000000000000..697d21e24cb5a07f535a472e130052d81e473167 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/printer.h @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "parquet/platform.h" + +namespace parquet { + +class ParquetFileReader; + +class PARQUET_EXPORT ParquetFilePrinter { + private: + ParquetFileReader* fileReader; + + public: + explicit ParquetFilePrinter(ParquetFileReader* reader) : fileReader(reader) {} + ~ParquetFilePrinter() = default; + + void DebugPrint(std::ostream& stream, std::list selected_columns, + bool print_values = false, bool format_dump = false, + bool print_key_value_metadata = false, + const char* filename = "No Name"); + + void JSONPrint(std::ostream& stream, std::list selected_columns, + const char* filename = "No Name"); +}; + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/properties.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/properties.h new file mode 100644 index 0000000000000000000000000000000000000000..fee0360a572500a1a032900b3fe0cd9bd9909f2b --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/properties.h @@ -0,0 +1,1529 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "arrow/io/caching.h" +#include "arrow/type_fwd.h" +#include "arrow/util/compression.h" +#include "arrow/util/type_fwd.h" +#include "parquet/encryption/encryption.h" +#include "parquet/exception.h" +#include "parquet/parquet_version.h" +#include "parquet/platform.h" +#include "parquet/schema.h" +#include "parquet/type_fwd.h" +#include "parquet/types.h" + +namespace parquet { + +/// Controls serialization format of data pages. parquet-format v2.0.0 +/// introduced a new data page metadata type DataPageV2 and serialized page +/// structure (for example, encoded levels are no longer compressed). Prior to +/// the completion of PARQUET-457 in 2020, this library did not implement +/// DataPageV2 correctly, so if you use the V2 data page format, you may have +/// forward compatibility issues (older versions of the library will be unable +/// to read the files). Note that some Parquet implementations do not implement +/// DataPageV2 at all. +enum class ParquetDataPageVersion { V1, V2 }; + +/// Controls the level of size statistics that are written to the file. +enum class SizeStatisticsLevel : uint8_t { + // No size statistics are written. + None = 0, + // Only column chunk size statistics are written. + ColumnChunk, + // Both size statistics in the column chunk and page index are written. + PageAndColumnChunk +}; + +/// Align the default buffer size to a small multiple of a page size. +constexpr int64_t kDefaultBufferSize = 4096 * 4; + +constexpr int32_t kDefaultThriftStringSizeLimit = 100 * 1000 * 1000; +// Structs in the thrift definition are relatively large (at least 300 bytes). +// This limits total memory to the same order of magnitude as +// kDefaultStringSizeLimit. +constexpr int32_t kDefaultThriftContainerSizeLimit = 1000 * 1000; + +// PARQUET-978: Minimize footer reads by reading 64 KB from the end of the file +constexpr int64_t kDefaultFooterReadSize = 64 * 1024; + +class PARQUET_EXPORT ReaderProperties { + public: + explicit ReaderProperties(MemoryPool* pool = ::arrow::default_memory_pool()) + : pool_(pool) {} + + MemoryPool* memory_pool() const { return pool_; } + + std::shared_ptr GetStream(std::shared_ptr source, + int64_t start, int64_t num_bytes); + + /// Buffered stream reading allows the user to control the memory usage of + /// parquet readers. This ensure that all `RandomAccessFile::ReadAt` calls are + /// wrapped in a buffered reader that uses a fix sized buffer (of size + /// `buffer_size()`) instead of the full size of the ReadAt. + /// + /// The primary reason for this control knobs is for resource control and not + /// performance. + bool is_buffered_stream_enabled() const { return buffered_stream_enabled_; } + /// Enable buffered stream reading. + void enable_buffered_stream() { buffered_stream_enabled_ = true; } + /// Disable buffered stream reading. + void disable_buffered_stream() { buffered_stream_enabled_ = false; } + + bool read_dense_for_nullable() const { return read_dense_for_nullable_; } + void enable_read_dense_for_nullable() { read_dense_for_nullable_ = true; } + void disable_read_dense_for_nullable() { read_dense_for_nullable_ = false; } + + /// Return the size of the buffered stream buffer. + int64_t buffer_size() const { return buffer_size_; } + /// Set the size of the buffered stream buffer in bytes. + void set_buffer_size(int64_t size) { buffer_size_ = size; } + + /// \brief Return the size limit on thrift strings. + /// + /// This limit helps prevent space and time bombs in files, but may need to + /// be increased in order to read files with especially large headers. + int32_t thrift_string_size_limit() const { return thrift_string_size_limit_; } + /// Set the size limit on thrift strings. + void set_thrift_string_size_limit(int32_t size) { thrift_string_size_limit_ = size; } + + /// \brief Return the size limit on thrift containers. + /// + /// This limit helps prevent space and time bombs in files, but may need to + /// be increased in order to read files with especially large headers. + int32_t thrift_container_size_limit() const { return thrift_container_size_limit_; } + /// Set the size limit on thrift containers. + void set_thrift_container_size_limit(int32_t size) { + thrift_container_size_limit_ = size; + } + + /// Set the decryption properties. + void file_decryption_properties(std::shared_ptr decryption) { + file_decryption_properties_ = std::move(decryption); + } + /// Return the decryption properties. + const std::shared_ptr& file_decryption_properties() const { + return file_decryption_properties_; + } + + bool page_checksum_verification() const { return page_checksum_verification_; } + void set_page_checksum_verification(bool check_crc) { + page_checksum_verification_ = check_crc; + } + + // Set the default read size to read the footer from a file. For high latency + // file systems and files with large metadata (>64KB) this can increase performance + // by reducing the number of round-trips to retrieve the entire file metadata. + void set_footer_read_size(size_t size) { footer_read_size_ = size; } + size_t footer_read_size() const { return footer_read_size_; } + + private: + MemoryPool* pool_; + int64_t buffer_size_ = kDefaultBufferSize; + int32_t thrift_string_size_limit_ = kDefaultThriftStringSizeLimit; + int32_t thrift_container_size_limit_ = kDefaultThriftContainerSizeLimit; + bool buffered_stream_enabled_ = false; + bool page_checksum_verification_ = false; + // Used with a RecordReader. + bool read_dense_for_nullable_ = false; + size_t footer_read_size_ = kDefaultFooterReadSize; + std::shared_ptr file_decryption_properties_; +}; + +ReaderProperties PARQUET_EXPORT default_reader_properties(); + +static constexpr int64_t kDefaultDataPageSize = 1024 * 1024; +static constexpr int64_t kDefaultMaxRowsPerPage = 20'000; +static constexpr bool DEFAULT_IS_DICTIONARY_ENABLED = true; +static constexpr int64_t DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT = kDefaultDataPageSize; +static constexpr int64_t DEFAULT_WRITE_BATCH_SIZE = 1024; +static constexpr int64_t DEFAULT_MAX_ROW_GROUP_LENGTH = 1024 * 1024; +static constexpr bool DEFAULT_ARE_STATISTICS_ENABLED = true; +static constexpr int64_t DEFAULT_MAX_STATISTICS_SIZE = 4096; +static constexpr Encoding::type DEFAULT_ENCODING = Encoding::UNKNOWN; +static const char DEFAULT_CREATED_BY[] = CREATED_BY_VERSION; +static constexpr Compression::type DEFAULT_COMPRESSION_TYPE = Compression::UNCOMPRESSED; +static constexpr bool DEFAULT_IS_PAGE_INDEX_ENABLED = true; +static constexpr SizeStatisticsLevel DEFAULT_SIZE_STATISTICS_LEVEL = + SizeStatisticsLevel::PageAndColumnChunk; + +struct PARQUET_EXPORT BloomFilterOptions { + /// Expected number of distinct values (NDV) in the bloom filter. + /// + /// Bloom filters are most effective for high-cardinality columns. A good default + /// is to set ndv equal to the number of rows. Lower values reduce disk usage but + /// may not be worthwhile for very small NDVs. + /// + /// Increasing ndv (without increasing fpp) increases disk and memory usage. + int32_t ndv = 1 << 20; + + /// False-positive probability (FPP) of the bloom filter. + /// + /// Lower FPP values require more disk and memory space. For a fixed ndv, the + /// space requirement grows roughly proportional to log(1/fpp). Recommended + /// values are 0.1, 0.05, or 0.01. Very small values are counterproductive as + /// the bitset may exceed the size of the actual data. Set ndv appropriately + /// to minimize space usage. + /// + /// Below is a table to demonstrate estimated size using common values. + /// + /// | ndv | fpp | bits/key | size | + /// |:-----------|:------|:---------|:----------| + /// | 100,000 | 0.10 | 10.5 | 128 KiB | + /// | 100,000 | 0.05 | 10.5 | 128 KiB | + /// | 100,000 | 0.01 | 10.5 | 128 KiB | + /// | 1,000,000 | 0.10 | 8.4 | 1024 KiB | + /// | 1,000,000 | 0.05 | 8.4 | 1024 KiB | + /// | 1,000,000 | 0.01 | 16.8 | 2048 KiB | + /// | 10,000,000 | 0.10 | 6.7 | 8192 KiB | + /// | 10,000,000 | 0.05 | 13.4 | 16384 KiB | + /// | 10,000,000 | 0.01 | 13.4 | 16384 KiB | + double fpp = 0.05; +}; + +class PARQUET_EXPORT ColumnProperties { + public: + ColumnProperties(Encoding::type encoding = DEFAULT_ENCODING, + Compression::type codec = DEFAULT_COMPRESSION_TYPE, + bool dictionary_enabled = DEFAULT_IS_DICTIONARY_ENABLED, + bool statistics_enabled = DEFAULT_ARE_STATISTICS_ENABLED, + size_t max_stats_size = DEFAULT_MAX_STATISTICS_SIZE, + bool page_index_enabled = DEFAULT_IS_PAGE_INDEX_ENABLED) + : encoding_(encoding), + codec_(codec), + dictionary_enabled_(dictionary_enabled), + statistics_enabled_(statistics_enabled), + max_stats_size_(max_stats_size), + page_index_enabled_(page_index_enabled) {} + + void set_encoding(Encoding::type encoding) { encoding_ = encoding; } + + void set_compression(Compression::type codec) { codec_ = codec; } + + void set_dictionary_enabled(bool dictionary_enabled) { + dictionary_enabled_ = dictionary_enabled; + } + + void set_statistics_enabled(bool statistics_enabled) { + statistics_enabled_ = statistics_enabled; + } + + void set_max_statistics_size(size_t max_stats_size) { + max_stats_size_ = max_stats_size; + } + + void set_compression_level(int compression_level) { + if (!codec_options_) { + codec_options_ = std::make_shared(); + } + codec_options_->compression_level = compression_level; + } + + void set_codec_options(const std::shared_ptr& codec_options) { + codec_options_ = codec_options; + } + + void set_page_index_enabled(bool page_index_enabled) { + page_index_enabled_ = page_index_enabled; + } + + void set_bloom_filter_options(const BloomFilterOptions& bloom_filter_options) { + if (bloom_filter_options.fpp >= 1.0 || bloom_filter_options.fpp <= 0.0) { + throw ParquetException( + "Bloom filter false positive probability must be in (0.0, 1.0), got " + + std::to_string(bloom_filter_options.fpp)); + } + bloom_filter_options_ = bloom_filter_options; + } + + Encoding::type encoding() const { return encoding_; } + + Compression::type compression() const { return codec_; } + + bool dictionary_enabled() const { return dictionary_enabled_; } + + bool statistics_enabled() const { return statistics_enabled_; } + + size_t max_statistics_size() const { return max_stats_size_; } + + int compression_level() const { + if (!codec_options_) { + return ::arrow::util::kUseDefaultCompressionLevel; + } + return codec_options_->compression_level; + } + + const std::shared_ptr& codec_options() const { return codec_options_; } + + bool page_index_enabled() const { return page_index_enabled_; } + + std::optional bloom_filter_options() const { + return bloom_filter_options_; + } + + bool bloom_filter_enabled() const { return bloom_filter_options_.has_value(); } + + private: + Encoding::type encoding_; + Compression::type codec_; + bool dictionary_enabled_; + bool statistics_enabled_; + size_t max_stats_size_; + std::shared_ptr codec_options_; + bool page_index_enabled_; + std::optional bloom_filter_options_; +}; + +// EXPERIMENTAL: Options for content-defined chunking. +/// +/// Content-defined chunking is an experimental feature that optimizes parquet +/// files for content addressable storage (CAS) systems by writing data pages +/// according to content-defined chunk boundaries. This allows for more +/// efficient deduplication of data across files, hence more efficient network +/// transfers and storage. +/// Each content-defined chunk is written as a separate parquet data page. The +/// following options control the chunks' size and the chunking process. Note +/// that the chunk size is calculated based on the logical value of the data, +/// before any encoding or compression is applied. +struct PARQUET_EXPORT CdcOptions { + /// Minimum chunk size in bytes, default is 256 KiB + /// The rolling hash will not be updated until this size is reached for each chunk. + /// Note that all data sent through the hash function is counted towards the chunk + /// size, including definition and repetition levels if present. + int64_t min_chunk_size = 256 * 1024; + /// Maximum chunk size in bytes, default is 1024 KiB + /// The chunker will create a new chunk whenever the chunk size exceeds this value. + /// Note that the parquet writer has a related `pagesize` property that controls + /// the maximum size of a parquet data page after encoding. While setting + /// `pagesize` to a smaller value than `max_chunk_size` doesn't affect the + /// chunking effectiveness, it results in more small parquet data pages. + int64_t max_chunk_size = 1024 * 1024; + /// Number of bit adjustment to the gearhash mask in order to center the chunk size + /// around the average size more aggressively, default is 0 + /// Increasing the normalization level increases the probability of finding a chunk, + /// improving the deduplication ratio, but also increasing the number of small chunks + /// resulting in many small parquet data pages. The default value provides a good + /// balance between deduplication ratio and fragmentation. + /// Use norm_level=1 or norm_level=2 to reach a higher deduplication ratio at the + /// expense of fragmentation. Negative values can also be used to reduce the + /// probability of finding a chunk, resulting in larger chunks and fewer data pages. + /// Note that values outside [-3, 3] are not recommended, prefer using the default + /// value of 0 for most use cases. + int norm_level = 0; +}; + +class PARQUET_EXPORT WriterProperties { + public: + class PARQUET_EXPORT Builder { + public: + Builder() + : pool_(::arrow::default_memory_pool()), + dictionary_pagesize_limit_(DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT), + write_batch_size_(DEFAULT_WRITE_BATCH_SIZE), + max_row_group_length_(DEFAULT_MAX_ROW_GROUP_LENGTH), + pagesize_(kDefaultDataPageSize), + max_rows_per_page_(kDefaultMaxRowsPerPage), + version_(ParquetVersion::PARQUET_2_6), + data_page_version_(ParquetDataPageVersion::V1), + created_by_(DEFAULT_CREATED_BY), + store_decimal_as_integer_(false), + page_checksum_enabled_(false), + size_statistics_level_(DEFAULT_SIZE_STATISTICS_LEVEL), + content_defined_chunking_enabled_(false), + content_defined_chunking_options_({}) {} + + explicit Builder(const WriterProperties& properties) + : pool_(properties.memory_pool()), + dictionary_pagesize_limit_(properties.dictionary_pagesize_limit()), + write_batch_size_(properties.write_batch_size()), + max_row_group_length_(properties.max_row_group_length()), + pagesize_(properties.data_pagesize()), + max_rows_per_page_(properties.max_rows_per_page()), + version_(properties.version()), + data_page_version_(properties.data_page_version()), + created_by_(properties.created_by()), + store_decimal_as_integer_(properties.store_decimal_as_integer()), + page_checksum_enabled_(properties.page_checksum_enabled()), + size_statistics_level_(properties.size_statistics_level()), + sorting_columns_(properties.sorting_columns()), + default_column_properties_(properties.default_column_properties()), + content_defined_chunking_enabled_( + properties.content_defined_chunking_enabled()), + content_defined_chunking_options_( + properties.content_defined_chunking_options()) { + CopyColumnSpecificProperties(properties); + } + + /// \brief EXPERIMENTAL: Use content-defined page chunking for all columns. + /// + /// Optimize parquet files for content addressable storage (CAS) systems by writing + /// data pages according to content-defined chunk boundaries. This allows for more + /// efficient deduplication of data across files, hence more efficient network + /// transfers and storage. The chunking is based on a rolling hash algorithm that + /// identifies chunk boundaries based on the actual content of the data. + /// + /// Note that only the WriteArrow() interface is supported at the moment. + Builder* enable_content_defined_chunking() { + content_defined_chunking_enabled_ = true; + return this; + } + + /// \brief EXPERIMENTAL: Disable content-defined page chunking for all columns. + Builder* disable_content_defined_chunking() { + content_defined_chunking_enabled_ = false; + return this; + } + + /// \brief EXPERIMENTAL: Specify content-defined chunking options, see CdcOptions. + Builder* content_defined_chunking_options(const CdcOptions& options) { + content_defined_chunking_options_ = options; + return this; + } + + /// Specify the memory pool for the writer. Default default_memory_pool. + Builder* memory_pool(MemoryPool* pool) { + pool_ = pool; + return this; + } + + /// Enable dictionary encoding in general for all columns. Default + /// enabled. + Builder* enable_dictionary() { + default_column_properties_.set_dictionary_enabled(true); + return this; + } + + /// Disable dictionary encoding in general for all columns. Default + /// enabled. + Builder* disable_dictionary() { + default_column_properties_.set_dictionary_enabled(false); + return this; + } + + /// Enable dictionary encoding for column specified by `path`. Default + /// enabled. + Builder* enable_dictionary(const std::string& path) { + dictionary_enabled_[path] = true; + return this; + } + + /// Enable dictionary encoding for column specified by `path`. Default + /// enabled. + Builder* enable_dictionary(const std::shared_ptr& path) { + return this->enable_dictionary(path->ToDotString()); + } + + /// Disable dictionary encoding for column specified by `path`. Default + /// enabled. + Builder* disable_dictionary(const std::string& path) { + dictionary_enabled_[path] = false; + return this; + } + + /// Disable dictionary encoding for column specified by `path`. Default + /// enabled. + Builder* disable_dictionary(const std::shared_ptr& path) { + return this->disable_dictionary(path->ToDotString()); + } + + /// Specify the dictionary page size limit per row group. Default 1MB. + Builder* dictionary_pagesize_limit(int64_t dictionary_psize_limit) { + dictionary_pagesize_limit_ = dictionary_psize_limit; + return this; + } + + /// Specify the write batch size while writing batches of Arrow values + /// into Parquet. Default 1024. + Builder* write_batch_size(int64_t write_batch_size) { + write_batch_size_ = write_batch_size; + return this; + } + + /// Specify the max number of rows to put in a single row group. + /// Default 1Mi rows. + Builder* max_row_group_length(int64_t max_row_group_length) { + max_row_group_length_ = max_row_group_length; + return this; + } + + /// Specify the data page size. + /// Default 1MB. + Builder* data_pagesize(int64_t pg_size) { + pagesize_ = pg_size; + return this; + } + + /// Specify the maximum number of rows per data page. + /// Default 20K rows. + Builder* max_rows_per_page(int64_t max_rows) { + max_rows_per_page_ = max_rows; + return this; + } + + /// Specify the data page version. + /// Default V1. + Builder* data_page_version(ParquetDataPageVersion data_page_version) { + data_page_version_ = data_page_version; + return this; + } + + /// Specify the Parquet file version. + /// Default PARQUET_2_6. + Builder* version(ParquetVersion::type version) { + version_ = version; + return this; + } + + Builder* created_by(const std::string& created_by) { + created_by_ = created_by; + return this; + } + + Builder* enable_page_checksum() { + page_checksum_enabled_ = true; + return this; + } + + Builder* disable_page_checksum() { + page_checksum_enabled_ = false; + return this; + } + + /// \brief Define the encoding that is used when we don't utilise dictionary encoding. + // + /// This is only applied if dictionary encoding is disabled. If the dictionary grows + /// too large we always fall back to the PLAIN encoding. + Builder* encoding(Encoding::type encoding_type) { + if (encoding_type == Encoding::PLAIN_DICTIONARY || + encoding_type == Encoding::RLE_DICTIONARY) { + throw ParquetException("Can't use dictionary encoding as fallback encoding"); + } + + default_column_properties_.set_encoding(encoding_type); + return this; + } + + /// \brief Define the encoding that is used when we don't utilise dictionary encoding. + // + /// This is only applied if dictionary encoding is disabled. If the dictionary grows + /// too large we always fall back to the PLAIN encoding. + Builder* encoding(const std::string& path, Encoding::type encoding_type) { + if (encoding_type == Encoding::PLAIN_DICTIONARY || + encoding_type == Encoding::RLE_DICTIONARY) { + throw ParquetException("Can't use dictionary encoding as fallback encoding"); + } + + encodings_[path] = encoding_type; + return this; + } + + /// \brief Define the encoding that is used when we don't utilise dictionary encoding. + // + /// This is only applied if dictionary encoding is disabled. If the dictionary grows + /// too large we always fall back to the PLAIN encoding. + Builder* encoding(const std::shared_ptr& path, + Encoding::type encoding_type) { + return this->encoding(path->ToDotString(), encoding_type); + } + + /// Specify compression codec in general for all columns. + /// Default UNCOMPRESSED. + Builder* compression(Compression::type codec) { + default_column_properties_.set_compression(codec); + return this; + } + + /// Specify max statistics size to store min max value. + /// Default 4KB. + Builder* max_statistics_size(size_t max_stats_sz) { + default_column_properties_.set_max_statistics_size(max_stats_sz); + return this; + } + + /// Specify compression codec for the column specified by `path`. + /// Default UNCOMPRESSED. + Builder* compression(const std::string& path, Compression::type codec) { + codecs_[path] = codec; + return this; + } + + /// Specify compression codec for the column specified by `path`. + /// Default UNCOMPRESSED. + Builder* compression(const std::shared_ptr& path, + Compression::type codec) { + return this->compression(path->ToDotString(), codec); + } + + /// \brief Specify the default compression level for the compressor in + /// every column. In case a column does not have an explicitly specified + /// compression level, the default one would be used. + /// + /// The provided compression level is compressor specific. The user would + /// have to familiarize oneself with the available levels for the selected + /// compressor. If the compressor does not allow for selecting different + /// compression levels, calling this function would not have any effect. + /// Parquet and Arrow do not validate the passed compression level. If no + /// level is selected by the user or if the special + /// std::numeric_limits::min() value is passed, then Arrow selects the + /// compression level. + /// + /// If other compressor-specific options need to be set in addition to the compression + /// level, use the codec_options method. + Builder* compression_level(int compression_level) { + default_column_properties_.set_compression_level(compression_level); + return this; + } + + /// \brief Specify a compression level for the compressor for the column + /// described by path. + /// + /// The provided compression level is compressor specific. The user would + /// have to familiarize oneself with the available levels for the selected + /// compressor. If the compressor does not allow for selecting different + /// compression levels, calling this function would not have any effect. + /// Parquet and Arrow do not validate the passed compression level. If no + /// level is selected by the user or if the special + /// std::numeric_limits::min() value is passed, then Arrow selects the + /// compression level. + Builder* compression_level(const std::string& path, int compression_level) { + if (!codec_options_[path]) { + codec_options_[path] = std::make_shared(); + } + codec_options_[path]->compression_level = compression_level; + return this; + } + + /// \brief Specify a compression level for the compressor for the column + /// described by path. + /// + /// The provided compression level is compressor specific. The user would + /// have to familiarize oneself with the available levels for the selected + /// compressor. If the compressor does not allow for selecting different + /// compression levels, calling this function would not have any effect. + /// Parquet and Arrow do not validate the passed compression level. If no + /// level is selected by the user or if the special + /// std::numeric_limits::min() value is passed, then Arrow selects the + /// compression level. + Builder* compression_level(const std::shared_ptr& path, + int compression_level) { + return this->compression_level(path->ToDotString(), compression_level); + } + + /// \brief Specify the default codec options for the compressor in + /// every column. + /// + /// The codec options allow configuring the compression level as well + /// as other codec-specific options. + Builder* codec_options( + const std::shared_ptr<::arrow::util::CodecOptions>& codec_options) { + default_column_properties_.set_codec_options(codec_options); + return this; + } + + /// \brief Specify the codec options for the compressor for the column + /// described by path. + Builder* codec_options( + const std::string& path, + const std::shared_ptr<::arrow::util::CodecOptions>& codec_options) { + codec_options_[path] = codec_options; + return this; + } + + /// \brief Specify the codec options for the compressor for the column + /// described by path. + Builder* codec_options( + const std::shared_ptr& path, + const std::shared_ptr<::arrow::util::CodecOptions>& codec_options) { + return this->codec_options(path->ToDotString(), codec_options); + } + + /// Define the file encryption properties. + /// Default NULL. + Builder* encryption( + std::shared_ptr file_encryption_properties) { + file_encryption_properties_ = std::move(file_encryption_properties); + return this; + } + + /// Enable statistics in general. + /// Default enabled. + Builder* enable_statistics() { + default_column_properties_.set_statistics_enabled(true); + return this; + } + + /// Disable statistics in general. + /// Default enabled. + Builder* disable_statistics() { + default_column_properties_.set_statistics_enabled(false); + return this; + } + + /// Enable statistics for the column specified by `path`. + /// Default enabled. + Builder* enable_statistics(const std::string& path) { + statistics_enabled_[path] = true; + return this; + } + + /// Enable statistics for the column specified by `path`. + /// Default enabled. + Builder* enable_statistics(const std::shared_ptr& path) { + return this->enable_statistics(path->ToDotString()); + } + + /// Define the sorting columns. + /// Default empty. + /// + /// If sorting columns are set, user should ensure that records + /// are sorted by sorting columns. Otherwise, the storing data + /// will be inconsistent with sorting_columns metadata. + Builder* set_sorting_columns(std::vector sorting_columns) { + sorting_columns_ = std::move(sorting_columns); + return this; + } + + /// Disable statistics for the column specified by `path`. + /// Default enabled. + Builder* disable_statistics(const std::string& path) { + statistics_enabled_[path] = false; + return this; + } + + /// Disable statistics for the column specified by `path`. + /// Default enabled. + Builder* disable_statistics(const std::shared_ptr& path) { + return this->disable_statistics(path->ToDotString()); + } + + /// Disable bloom filter for the column specified by `path`. + /// Default disabled. + Builder* disable_bloom_filter(const std::string& path) { + bloom_filter_options_.erase(path); + return this; + } + + /// Disable bloom filter for the column specified by `path`. + /// Default disabled. + Builder* disable_bloom_filter(const std::shared_ptr& path) { + return this->disable_bloom_filter(path->ToDotString()); + } + + /// Enable bloom filter for the column specified by `path`. + /// + /// Default disabled. + /// + /// \note Bloom filter is not supported for boolean columns. ParquetException will + /// be thrown during write if the column is of boolean type. + Builder* enable_bloom_filter(const std::string& path, + const BloomFilterOptions& bloom_filter_options) { + bloom_filter_options_[path] = bloom_filter_options; + return this; + } + + /// Enable bloom filter for the column specified by `path`. + /// + /// Default disabled. + /// + /// \note Bloom filter is not supported for boolean columns. ParquetException will + /// be thrown during write if the column is of boolean type. + Builder* enable_bloom_filter(const std::shared_ptr& path, + const BloomFilterOptions& bloom_filter_options) { + return this->enable_bloom_filter(path->ToDotString(), bloom_filter_options); + } + + /// Allow decimals with 1 <= precision <= 18 to be stored as integers. + /// + /// In Parquet, DECIMAL can be stored in any of the following physical types: + /// - int32: for 1 <= precision <= 9. + /// - int64: for 10 <= precision <= 18. + /// - fixed_len_byte_array: precision is limited by the array size. + /// Length n can store <= floor(log_10(2^(8*n - 1) - 1)) base-10 digits. + /// - binary: precision is unlimited. The minimum number of bytes to store + /// the unscaled value is used. + /// + /// By default, this is DISABLED and all decimal types annotate fixed_len_byte_array. + /// + /// When enabled, the C++ writer will use following physical types to store decimals: + /// - int32: for 1 <= precision <= 9. + /// - int64: for 10 <= precision <= 18. + /// - fixed_len_byte_array: for precision > 18. + /// + /// As a consequence, decimal columns stored in integer types are more compact. + Builder* enable_store_decimal_as_integer() { + store_decimal_as_integer_ = true; + return this; + } + + /// Disable decimal logical type with 1 <= precision <= 18 to be stored + /// as integer physical type. + /// + /// Default disabled. + Builder* disable_store_decimal_as_integer() { + store_decimal_as_integer_ = false; + return this; + } + + /// Enable writing page index in general for all columns. Default enabled. + /// + /// Writing statistics to the page index disables the old method of writing + /// statistics to each data page header. + /// The page index makes filtering more efficient than the page header, as + /// it gathers all the statistics for a Parquet file in a single place, + /// avoiding scattered I/O. + /// + /// Please check the link below for more details: + /// https://github.com/apache/parquet-format/blob/master/PageIndex.md + Builder* enable_write_page_index() { + default_column_properties_.set_page_index_enabled(true); + return this; + } + + /// Disable writing page index in general for all columns. Default enabled. + Builder* disable_write_page_index() { + default_column_properties_.set_page_index_enabled(false); + return this; + } + + /// Enable writing page index for column specified by `path`. Default enabled. + Builder* enable_write_page_index(const std::string& path) { + page_index_enabled_[path] = true; + return this; + } + + /// Enable writing page index for column specified by `path`. Default enabled. + Builder* enable_write_page_index(const std::shared_ptr& path) { + return this->enable_write_page_index(path->ToDotString()); + } + + /// Disable writing page index for column specified by `path`. Default enabled. + Builder* disable_write_page_index(const std::string& path) { + page_index_enabled_[path] = false; + return this; + } + + /// Disable writing page index for column specified by `path`. Default enabled. + Builder* disable_write_page_index(const std::shared_ptr& path) { + return this->disable_write_page_index(path->ToDotString()); + } + + /// \brief Set the level to write size statistics for all columns. Default is + /// PageAndColumnChunk. + /// + /// \param level The level to write size statistics. Note that if page index is not + /// enabled, page level size statistics will not be written even if the level + /// is set to PageAndColumnChunk. + Builder* set_size_statistics_level(SizeStatisticsLevel level) { + size_statistics_level_ = level; + return this; + } + + /// \brief Build the WriterProperties with the builder parameters. + /// \return The WriterProperties defined by the builder. + std::shared_ptr build() { + std::unordered_map column_properties; + auto get = [&](const std::string& key) -> ColumnProperties& { + auto it = column_properties.find(key); + if (it == column_properties.end()) + return column_properties[key] = default_column_properties_; + else + return it->second; + }; + + for (const auto& item : encodings_) get(item.first).set_encoding(item.second); + for (const auto& item : codecs_) get(item.first).set_compression(item.second); + for (const auto& item : codec_options_) + get(item.first).set_codec_options(item.second); + for (const auto& item : dictionary_enabled_) + get(item.first).set_dictionary_enabled(item.second); + for (const auto& item : statistics_enabled_) + get(item.first).set_statistics_enabled(item.second); + for (const auto& item : page_index_enabled_) + get(item.first).set_page_index_enabled(item.second); + for (const auto& item : bloom_filter_options_) + get(item.first).set_bloom_filter_options(item.second); + + return std::shared_ptr(new WriterProperties( + pool_, dictionary_pagesize_limit_, write_batch_size_, max_row_group_length_, + pagesize_, max_rows_per_page_, version_, created_by_, page_checksum_enabled_, + size_statistics_level_, std::move(file_encryption_properties_), + default_column_properties_, column_properties, data_page_version_, + store_decimal_as_integer_, std::move(sorting_columns_), + content_defined_chunking_enabled_, content_defined_chunking_options_)); + } + + private: + void CopyColumnSpecificProperties(const WriterProperties& properties); + + MemoryPool* pool_; + int64_t dictionary_pagesize_limit_; + int64_t write_batch_size_; + int64_t max_row_group_length_; + int64_t pagesize_; + int64_t max_rows_per_page_; + ParquetVersion::type version_; + ParquetDataPageVersion data_page_version_; + std::string created_by_; + bool store_decimal_as_integer_; + bool page_checksum_enabled_; + SizeStatisticsLevel size_statistics_level_; + + std::shared_ptr file_encryption_properties_; + + // If empty, there is no sorting columns. + std::vector sorting_columns_; + + // Settings used for each column unless overridden in any of the maps below + ColumnProperties default_column_properties_; + std::unordered_map encodings_; + std::unordered_map codecs_; + std::unordered_map> codec_options_; + std::unordered_map dictionary_enabled_; + std::unordered_map statistics_enabled_; + std::unordered_map page_index_enabled_; + std::unordered_map bloom_filter_options_; + + bool content_defined_chunking_enabled_; + CdcOptions content_defined_chunking_options_; + }; + + inline MemoryPool* memory_pool() const { return pool_; } + + inline int64_t dictionary_pagesize_limit() const { return dictionary_pagesize_limit_; } + + inline int64_t write_batch_size() const { return write_batch_size_; } + + inline int64_t max_row_group_length() const { return max_row_group_length_; } + + inline int64_t data_pagesize() const { return pagesize_; } + + inline int64_t max_rows_per_page() const { return max_rows_per_page_; } + + inline ParquetDataPageVersion data_page_version() const { + return parquet_data_page_version_; + } + + inline ParquetVersion::type version() const { return parquet_version_; } + + inline std::string created_by() const { return parquet_created_by_; } + + inline bool store_decimal_as_integer() const { return store_decimal_as_integer_; } + + inline bool page_checksum_enabled() const { return page_checksum_enabled_; } + + inline bool content_defined_chunking_enabled() const { + return content_defined_chunking_enabled_; + } + inline CdcOptions content_defined_chunking_options() const { + return content_defined_chunking_options_; + } + + inline SizeStatisticsLevel size_statistics_level() const { + return size_statistics_level_; + } + + inline Encoding::type dictionary_index_encoding() const { + if (parquet_version_ == ParquetVersion::PARQUET_1_0) { + return Encoding::PLAIN_DICTIONARY; + } else { + return Encoding::RLE_DICTIONARY; + } + } + + inline Encoding::type dictionary_page_encoding() const { + if (parquet_version_ == ParquetVersion::PARQUET_1_0) { + return Encoding::PLAIN_DICTIONARY; + } else { + return Encoding::PLAIN; + } + } + + const ColumnProperties& column_properties( + const std::shared_ptr& path) const { + auto it = column_properties_.find(path->ToDotString()); + if (it != column_properties_.end()) return it->second; + return default_column_properties_; + } + + Encoding::type encoding(const std::shared_ptr& path) const { + return column_properties(path).encoding(); + } + + Compression::type compression(const std::shared_ptr& path) const { + return column_properties(path).compression(); + } + + int compression_level(const std::shared_ptr& path) const { + return column_properties(path).compression_level(); + } + + const std::shared_ptr codec_options( + const std::shared_ptr& path) const { + return column_properties(path).codec_options(); + } + + bool dictionary_enabled(const std::shared_ptr& path) const { + return column_properties(path).dictionary_enabled(); + } + + const std::vector& sorting_columns() const { return sorting_columns_; } + + bool statistics_enabled(const std::shared_ptr& path) const { + return column_properties(path).statistics_enabled(); + } + + size_t max_statistics_size(const std::shared_ptr& path) const { + return column_properties(path).max_statistics_size(); + } + + bool page_index_enabled(const std::shared_ptr& path) const { + return column_properties(path).page_index_enabled(); + } + + bool page_index_enabled() const { + if (default_column_properties_.page_index_enabled()) { + return true; + } + for (const auto& item : column_properties_) { + if (item.second.page_index_enabled()) { + return true; + } + } + return false; + } + + // Return whether bloom filter is enabled for any column. + bool bloom_filter_enabled() const { + return std::any_of(column_properties_.cbegin(), column_properties_.cend(), + [](const auto& p) { return p.second.bloom_filter_enabled(); }); + } + + std::optional bloom_filter_options( + const std::shared_ptr& path) const { + return column_properties(path).bloom_filter_options(); + } + + inline FileEncryptionProperties* file_encryption_properties() const { + return file_encryption_properties_.get(); + } + + std::shared_ptr column_encryption_properties( + const std::string& path) const { + if (file_encryption_properties_) { + return file_encryption_properties_->column_encryption_properties(path); + } else { + return NULLPTR; + } + } + + // \brief Return the default column properties + const ColumnProperties& default_column_properties() const { + return default_column_properties_; + } + + private: + explicit WriterProperties( + MemoryPool* pool, int64_t dictionary_pagesize_limit, int64_t write_batch_size, + int64_t max_row_group_length, int64_t pagesize, int64_t max_rows_per_page, + ParquetVersion::type version, const std::string& created_by, + bool page_write_checksum_enabled, SizeStatisticsLevel size_statistics_level, + std::shared_ptr file_encryption_properties, + const ColumnProperties& default_column_properties, + const std::unordered_map& column_properties, + ParquetDataPageVersion data_page_version, bool store_short_decimal_as_integer, + std::vector sorting_columns, bool content_defined_chunking_enabled, + CdcOptions content_defined_chunking_options) + : pool_(pool), + dictionary_pagesize_limit_(dictionary_pagesize_limit), + write_batch_size_(write_batch_size), + max_row_group_length_(max_row_group_length), + pagesize_(pagesize), + max_rows_per_page_(max_rows_per_page), + parquet_data_page_version_(data_page_version), + parquet_version_(version), + parquet_created_by_(created_by), + store_decimal_as_integer_(store_short_decimal_as_integer), + page_checksum_enabled_(page_write_checksum_enabled), + size_statistics_level_(size_statistics_level), + file_encryption_properties_(file_encryption_properties), + sorting_columns_(std::move(sorting_columns)), + default_column_properties_(default_column_properties), + column_properties_(column_properties), + content_defined_chunking_enabled_(content_defined_chunking_enabled), + content_defined_chunking_options_(content_defined_chunking_options) {} + + MemoryPool* pool_; + int64_t dictionary_pagesize_limit_; + int64_t write_batch_size_; + int64_t max_row_group_length_; + int64_t pagesize_; + int64_t max_rows_per_page_; + ParquetDataPageVersion parquet_data_page_version_; + ParquetVersion::type parquet_version_; + std::string parquet_created_by_; + bool store_decimal_as_integer_; + bool page_checksum_enabled_; + SizeStatisticsLevel size_statistics_level_; + + std::shared_ptr file_encryption_properties_; + + std::vector sorting_columns_; + + ColumnProperties default_column_properties_; + std::unordered_map column_properties_; + + bool content_defined_chunking_enabled_; + CdcOptions content_defined_chunking_options_; +}; + +PARQUET_EXPORT const std::shared_ptr& default_writer_properties(); + +// ---------------------------------------------------------------------- +// Properties specific to Apache Arrow columnar read and write + +static constexpr bool kArrowDefaultUseThreads = false; + +// Default number of rows to read when using ::arrow::RecordBatchReader +static constexpr int64_t kArrowDefaultBatchSize = 64 * 1024; + +constexpr inline ::arrow::Type::type kArrowDefaultBinaryType = ::arrow::Type::BINARY; +constexpr inline ::arrow::Type::type kArrowDefaultListType = ::arrow::Type::LIST; + +/// EXPERIMENTAL: Properties for configuring FileReader behavior. +class PARQUET_EXPORT ArrowReaderProperties { + public: + explicit ArrowReaderProperties(bool use_threads = kArrowDefaultUseThreads) + : use_threads_(use_threads), + read_dict_indices_(), + batch_size_(kArrowDefaultBatchSize), + pre_buffer_(true), + cache_options_(::arrow::io::CacheOptions::LazyDefaults()), + coerce_int96_timestamp_unit_(::arrow::TimeUnit::NANO), + binary_type_(kArrowDefaultBinaryType), + list_type_(kArrowDefaultListType), + arrow_extensions_enabled_(false), + should_load_statistics_(false), + smallest_decimal_enabled_(false) {} + + /// \brief Set whether to use the IO thread pool to parse columns in parallel. + /// + /// Default is false. + void set_use_threads(bool use_threads) { use_threads_ = use_threads; } + /// Return whether will use multiple threads. + bool use_threads() const { return use_threads_; } + + /// \brief Set whether to read a particular column as dictionary encoded. + /// + /// If the file metadata contains a serialized Arrow schema, then ... + //// + /// This is only supported for columns with a Parquet physical type of + /// BYTE_ARRAY, such as string or binary types. + void set_read_dictionary(int column_index, bool read_dict) { + if (read_dict) { + read_dict_indices_.insert(column_index); + } else { + read_dict_indices_.erase(column_index); + } + } + /// Return whether the column at the index will be read as dictionary. + bool read_dictionary(int column_index) const { + if (read_dict_indices_.find(column_index) != read_dict_indices_.end()) { + return true; + } else { + return false; + } + } + + /// \brief Set the Arrow binary type to read BYTE_ARRAY columns as. + /// + /// Allowed values are Type::BINARY, Type::LARGE_BINARY and Type::BINARY_VIEW. + /// Default is Type::BINARY. + /// + /// If a BYTE_ARRAY column has the STRING logical type, it is read as the + /// Arrow string type corresponding to the configured binary type (for example + /// Type::LARGE_STRING if the configured binary type is Type::LARGE_BINARY). + /// + /// However, if a serialized Arrow schema is found in the Parquet metadata, + /// this setting is ignored and the Arrow schema takes precedence + /// (see ArrowWriterProperties::store_schema). + void set_binary_type(::arrow::Type::type value) { binary_type_ = value; } + /// Return the Arrow binary type to read BYTE_ARRAY columns as. + ::arrow::Type::type binary_type() const { return binary_type_; } + + /// \brief Set the Arrow list type to read Parquet list columns as. + /// + /// Allowed values are Type::LIST and Type::LARGE_LIST. + /// Default is Type::LIST. + /// + /// However, if a serialized Arrow schema is found in the Parquet metadata, + /// this setting is ignored and the Arrow schema takes precedence + /// (see ArrowWriterProperties::store_schema). + void set_list_type(::arrow::Type::type value) { list_type_ = value; } + /// Return the Arrow list type to read Parquet list columns as. + ::arrow::Type::type list_type() const { return list_type_; } + + /// \brief Set the maximum number of rows to read into a record batch. + /// + /// Will only be fewer rows when there are no more rows in the file. + /// Note that some APIs such as ReadTable may ignore this setting. + void set_batch_size(int64_t batch_size) { batch_size_ = batch_size; } + /// Return the batch size in rows. + /// + /// Note that some APIs such as ReadTable may ignore this setting. + int64_t batch_size() const { return batch_size_; } + + /// Enable read coalescing (default false). + /// + /// When enabled, the Arrow reader will pre-buffer necessary regions + /// of the file in-memory. This is intended to improve performance on + /// high-latency filesystems (e.g. Amazon S3). + void set_pre_buffer(bool pre_buffer) { pre_buffer_ = pre_buffer; } + /// Return whether read coalescing is enabled. + bool pre_buffer() const { return pre_buffer_; } + + /// Set options for read coalescing. This can be used to tune the + /// implementation for characteristics of different filesystems. + void set_cache_options(::arrow::io::CacheOptions options) { cache_options_ = options; } + /// Return the options for read coalescing. + const ::arrow::io::CacheOptions& cache_options() const { return cache_options_; } + + /// Set execution context for read coalescing. + void set_io_context(const ::arrow::io::IOContext& ctx) { io_context_ = ctx; } + /// Return the execution context used for read coalescing. + const ::arrow::io::IOContext& io_context() const { return io_context_; } + + /// Set timestamp unit to use for deprecated INT96-encoded timestamps + /// (default is NANO). + void set_coerce_int96_timestamp_unit(::arrow::TimeUnit::type unit) { + coerce_int96_timestamp_unit_ = unit; + } + + ::arrow::TimeUnit::type coerce_int96_timestamp_unit() const { + return coerce_int96_timestamp_unit_; + } + + /// Enable Parquet-supported Arrow extension types. + /// + /// When enabled, Parquet logical types will be mapped to their corresponding Arrow + /// extension types at read time, if such exist. Currently only arrow::extension::json() + /// extension type is supported. Columns whose LogicalType is JSON will be interpreted + /// as arrow::extension::json(), with storage type inferred from the serialized Arrow + /// schema if present, or `utf8` by default. + void set_arrow_extensions_enabled(bool extensions_enabled) { + arrow_extensions_enabled_ = extensions_enabled; + } + bool get_arrow_extensions_enabled() const { return arrow_extensions_enabled_; } + + /// \brief Set whether to load statistics as much as possible. + /// + /// Default is false. + void set_should_load_statistics(bool should_load_statistics) { + should_load_statistics_ = should_load_statistics; + } + /// Return whether loading statistics as much as possible. + bool should_load_statistics() const { return should_load_statistics_; } + + /// \brief Set whether to infer Decimal32/64 from Parquet decimal logical types. + /// + /// Default is false for compatibility, meaning that only Decimal128 and Decimal256 + /// can be inferred. + void set_smallest_decimal_enabled(bool smallest_decimal_enable) { + smallest_decimal_enabled_ = smallest_decimal_enable; + } + /// \brief Whether to infer Decimal32/64 from Parquet decimal logical types. + /// + /// When enabled, Parquet decimal columns will be inferred as the smallest possible + /// Arrow Decimal type. + /// When disabled, Parquet decimal columns will be inferred as either Decimal128 or + /// Decimal256, but not Decimal32/64. + /// + /// Note: if an Arrow schema is found in the Parquet metadata, it will take priority and + /// this setting will be ignored. + bool smallest_decimal_enabled() const { return smallest_decimal_enabled_; } + + private: + bool use_threads_; + std::unordered_set read_dict_indices_; + int64_t batch_size_; + bool pre_buffer_; + ::arrow::io::IOContext io_context_; + ::arrow::io::CacheOptions cache_options_; + ::arrow::TimeUnit::type coerce_int96_timestamp_unit_; + ::arrow::Type::type binary_type_; + ::arrow::Type::type list_type_; + bool arrow_extensions_enabled_; + bool should_load_statistics_; + bool smallest_decimal_enabled_; +}; + +/// EXPERIMENTAL: Constructs the default ArrowReaderProperties +PARQUET_EXPORT +ArrowReaderProperties default_arrow_reader_properties(); + +class PARQUET_EXPORT ArrowWriterProperties { + public: + enum EngineVersion { + V1, // Supports only nested lists. + V2 // Full support for all nesting combinations + }; + class Builder { + public: + Builder() + : write_timestamps_as_int96_(false), + coerce_timestamps_enabled_(false), + coerce_timestamps_unit_(::arrow::TimeUnit::SECOND), + truncated_timestamps_allowed_(false), + store_schema_(false), + compliant_nested_types_(true), + engine_version_(V2), + use_threads_(kArrowDefaultUseThreads), + executor_(NULLPTR), + write_time_adjusted_to_utc_(false) {} + + /// \brief Disable writing legacy int96 timestamps (default disabled). + Builder* disable_deprecated_int96_timestamps() { + write_timestamps_as_int96_ = false; + return this; + } + + /// \brief Enable writing legacy int96 timestamps (default disabled). + /// + /// May be turned on to write timestamps compatible with older Parquet writers. + /// This takes precedent over coerce_timestamps. + Builder* enable_deprecated_int96_timestamps() { + write_timestamps_as_int96_ = true; + return this; + } + + /// \brief Coerce all timestamps to the specified time unit. + /// \param unit time unit to truncate to. + /// For Parquet versions 1.0 and 2.4, nanoseconds are casted to microseconds. + Builder* coerce_timestamps(::arrow::TimeUnit::type unit) { + coerce_timestamps_enabled_ = true; + coerce_timestamps_unit_ = unit; + return this; + } + + /// \brief Allow loss of data when truncating timestamps. + /// + /// This is disallowed by default and an error will be returned. + Builder* allow_truncated_timestamps() { + truncated_timestamps_allowed_ = true; + return this; + } + + /// \brief Disallow loss of data when truncating timestamps (default). + Builder* disallow_truncated_timestamps() { + truncated_timestamps_allowed_ = false; + return this; + } + + /// \brief EXPERIMENTAL: Write binary serialized Arrow schema to the file, + /// to enable certain read options (like "read_dictionary") to be set + /// automatically + Builder* store_schema() { + store_schema_ = true; + return this; + } + + /// \brief When enabled, will not preserve Arrow field names for list types. + /// + /// Instead of using the field names Arrow uses for the values array of + /// list types (default "item"), will use "element", as is specified in + /// the Parquet spec. + /// + /// This is enabled by default. + Builder* enable_compliant_nested_types() { + compliant_nested_types_ = true; + return this; + } + + /// Preserve Arrow list field name. + Builder* disable_compliant_nested_types() { + compliant_nested_types_ = false; + return this; + } + + /// Set the version of the Parquet writer engine. + Builder* set_engine_version(EngineVersion version) { + engine_version_ = version; + return this; + } + + /// \brief Set whether to use multiple threads to write columns + /// in parallel in the buffered row group mode. + /// + /// WARNING: If writing multiple files in parallel in the same + /// executor, deadlock may occur if use_threads is true. Please + /// disable it in this case. + /// + /// Default is false. + Builder* set_use_threads(bool use_threads) { + use_threads_ = use_threads; + return this; + } + + /// \brief Set the executor to write columns in parallel in the + /// buffered row group mode. + /// + /// Default is nullptr and the default cpu executor will be used. + Builder* set_executor(::arrow::internal::Executor* executor) { + executor_ = executor; + return this; + } + + /// \brief Set the value of isAdjustedTOUTC when writing a TIME column + /// + /// Default is false because Arrow TIME data is expressed in an unspecified timezone. + /// Note this setting doesn't affect TIMESTAMP data. + Builder* set_time_adjusted_to_utc(bool adjusted) { + write_time_adjusted_to_utc_ = adjusted; + return this; + } + + /// Create the final properties. + std::shared_ptr build() { + return std::shared_ptr(new ArrowWriterProperties( + write_timestamps_as_int96_, coerce_timestamps_enabled_, coerce_timestamps_unit_, + truncated_timestamps_allowed_, store_schema_, compliant_nested_types_, + engine_version_, use_threads_, executor_, write_time_adjusted_to_utc_)); + } + + private: + bool write_timestamps_as_int96_; + + bool coerce_timestamps_enabled_; + ::arrow::TimeUnit::type coerce_timestamps_unit_; + bool truncated_timestamps_allowed_; + + bool store_schema_; + bool compliant_nested_types_; + EngineVersion engine_version_; + + bool use_threads_; + ::arrow::internal::Executor* executor_; + + bool write_time_adjusted_to_utc_; + }; + + bool support_deprecated_int96_timestamps() const { return write_timestamps_as_int96_; } + + bool coerce_timestamps_enabled() const { return coerce_timestamps_enabled_; } + ::arrow::TimeUnit::type coerce_timestamps_unit() const { + return coerce_timestamps_unit_; + } + + bool truncated_timestamps_allowed() const { return truncated_timestamps_allowed_; } + + bool store_schema() const { return store_schema_; } + + /// \brief Enable nested type naming according to the parquet specification. + /// + /// Older versions of arrow wrote out field names for nested lists based on the name + /// of the field. According to the parquet specification they should always be + /// "element". + bool compliant_nested_types() const { return compliant_nested_types_; } + + /// \brief The underlying engine version to use when writing Arrow data. + /// + /// V2 is currently the latest V1 is considered deprecated but left in + /// place in case there are bugs detected in V2. + EngineVersion engine_version() const { return engine_version_; } + + /// \brief Returns whether the writer will use multiple threads + /// to write columns in parallel in the buffered row group mode. + bool use_threads() const { return use_threads_; } + + /// \brief Returns the executor used to write columns in parallel. + ::arrow::internal::Executor* executor() const; + + /// \brief The value of isAdjustedTOUTC when writing a TIME column + /// + /// Note this setting doesn't affect TIMESTAMP data. + bool write_time_adjusted_to_utc() const { return write_time_adjusted_to_utc_; } + + private: + explicit ArrowWriterProperties(bool write_nanos_as_int96, + bool coerce_timestamps_enabled, + ::arrow::TimeUnit::type coerce_timestamps_unit, + bool truncated_timestamps_allowed, bool store_schema, + bool compliant_nested_types, + EngineVersion engine_version, bool use_threads, + ::arrow::internal::Executor* executor, + bool write_time_adjusted_to_utc) + : write_timestamps_as_int96_(write_nanos_as_int96), + coerce_timestamps_enabled_(coerce_timestamps_enabled), + coerce_timestamps_unit_(coerce_timestamps_unit), + truncated_timestamps_allowed_(truncated_timestamps_allowed), + store_schema_(store_schema), + compliant_nested_types_(compliant_nested_types), + engine_version_(engine_version), + use_threads_(use_threads), + executor_(executor), + write_time_adjusted_to_utc_(write_time_adjusted_to_utc) {} + + const bool write_timestamps_as_int96_; + const bool coerce_timestamps_enabled_; + const ::arrow::TimeUnit::type coerce_timestamps_unit_; + const bool truncated_timestamps_allowed_; + const bool store_schema_; + const bool compliant_nested_types_; + const EngineVersion engine_version_; + const bool use_threads_; + ::arrow::internal::Executor* executor_; + const bool write_time_adjusted_to_utc_; +}; + +/// \brief State object used for writing Arrow data directly to a Parquet +/// column chunk. API possibly not stable +struct ArrowWriteContext { + ArrowWriteContext(MemoryPool* memory_pool, ArrowWriterProperties* properties) + : memory_pool(memory_pool), + properties(properties), + data_buffer(AllocateBuffer(memory_pool)), + def_levels_buffer(AllocateBuffer(memory_pool)) {} + + template + ::arrow::Status GetScratchData(const int64_t num_values, T** out) { + ARROW_RETURN_NOT_OK(this->data_buffer->Resize(num_values * sizeof(T), false)); + *out = reinterpret_cast(this->data_buffer->mutable_data()); + return ::arrow::Status::OK(); + } + + MemoryPool* memory_pool; + const ArrowWriterProperties* properties; + + // Buffer used for storing the data of an array converted to the physical type + // as expected by parquet-cpp. + std::shared_ptr data_buffer; + + // We use the shared ownership of this buffer + std::shared_ptr def_levels_buffer; +}; + +PARQUET_EXPORT +std::shared_ptr default_arrow_writer_properties(); + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/schema.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/schema.h new file mode 100644 index 0000000000000000000000000000000000000000..b0ec9463d789e8be1b1896ba805bb7c8207a8e12 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/schema.h @@ -0,0 +1,494 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// This module contains the logical parquet-cpp types (independent of Thrift +// structures), schema nodes, and related type tools + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "parquet/platform.h" +#include "parquet/types.h" +#include "parquet/windows_fixup.h" // for OPTIONAL + +namespace parquet { + +class SchemaDescriptor; + +namespace schema { + +class Node; + +// List encodings: using the terminology from Impala to define different styles +// of representing logical lists (a.k.a. ARRAY types) in Parquet schemas. Since +// the converted type named in the Parquet metadata is ConvertedType::LIST we +// use that terminology here. It also helps distinguish from the *_ARRAY +// primitive types. +// +// One-level encoding: Only allows required lists with required cells +// repeated value_type name +// +// Two-level encoding: Enables optional lists with only required cells +// group list +// repeated value_type item +// +// Three-level encoding: Enables optional lists with optional cells +// group bag +// repeated group list +// value_type item +// +// 2- and 1-level encoding are respectively equivalent to 3-level encoding with +// the non-repeated nodes set to required. +// +// The "official" encoding recommended in the Parquet spec is the 3-level, and +// we use that as the default when creating list types. For semantic completeness +// we allow the other two. Since all types of encodings will occur "in the +// wild" we need to be able to interpret the associated definition levels in +// the context of the actual encoding used in the file. +// +// NB: Some Parquet writers may not set ConvertedType::LIST on the repeated +// SchemaElement, which could make things challenging if we are trying to infer +// that a sequence of nodes semantically represents an array according to one +// of these encodings (versus a struct containing an array). We should refuse +// the temptation to guess, as they say. +struct ListEncoding { + enum type { ONE_LEVEL, TWO_LEVEL, THREE_LEVEL }; +}; + +class PARQUET_EXPORT ColumnPath { + public: + ColumnPath() : path_() {} + explicit ColumnPath(const std::vector& path) : path_(path) {} + explicit ColumnPath(std::vector&& path) : path_(std::move(path)) {} + + static std::shared_ptr FromDotString(const std::string& dotstring); + static std::shared_ptr FromNode(const Node& node); + + std::shared_ptr extend(const std::string& node_name) const; + std::string ToDotString() const; + const std::vector& ToDotVector() const; + + protected: + std::vector path_; +}; + +// Base class for logical schema types. A type has a name, repetition level, +// and optionally a logical type (ConvertedType in Parquet metadata parlance) +class PARQUET_EXPORT Node { + public: + enum type { PRIMITIVE, GROUP }; + + virtual ~Node() {} + + bool is_primitive() const { return type_ == Node::PRIMITIVE; } + + bool is_group() const { return type_ == Node::GROUP; } + + bool is_optional() const { return repetition_ == Repetition::OPTIONAL; } + + bool is_repeated() const { return repetition_ == Repetition::REPEATED; } + + bool is_required() const { return repetition_ == Repetition::REQUIRED; } + + virtual bool Equals(const Node* other) const = 0; + + const std::string& name() const { return name_; } + + Node::type node_type() const { return type_; } + + Repetition::type repetition() const { return repetition_; } + + ConvertedType::type converted_type() const { return converted_type_; } + + const std::shared_ptr& logical_type() const { return logical_type_; } + + /// \brief The field_id value for the serialized SchemaElement. If the + /// field_id is less than 0 (e.g. -1), it will not be set when serialized to + /// Thrift. + int field_id() const { return field_id_; } + + const Node* parent() const { return parent_; } + + const std::shared_ptr path() const; + + virtual void ToParquet(void* element) const = 0; + + // Node::Visitor abstract class for walking schemas with the visitor pattern + class Visitor { + public: + virtual ~Visitor() {} + + virtual void Visit(Node* node) = 0; + }; + class ConstVisitor { + public: + virtual ~ConstVisitor() {} + + virtual void Visit(const Node* node) = 0; + }; + + virtual void Visit(Visitor* visitor) = 0; + virtual void VisitConst(ConstVisitor* visitor) const = 0; + + protected: + friend class GroupNode; + + Node(Node::type type, const std::string& name, Repetition::type repetition, + ConvertedType::type converted_type = ConvertedType::NONE, int field_id = -1) + : type_(type), + name_(name), + repetition_(repetition), + converted_type_(converted_type), + field_id_(field_id), + parent_(NULLPTR) {} + + Node(Node::type type, const std::string& name, Repetition::type repetition, + std::shared_ptr logical_type, int field_id = -1) + : type_(type), + name_(name), + repetition_(repetition), + logical_type_(std::move(logical_type)), + field_id_(field_id), + parent_(NULLPTR) {} + + Node::type type_; + std::string name_; + Repetition::type repetition_; + ConvertedType::type converted_type_{ConvertedType::NONE}; + std::shared_ptr logical_type_; + int field_id_; + // Nodes should not be shared, they have a single parent. + const Node* parent_; + + bool EqualsInternal(const Node* other) const; + void SetParent(const Node* p_parent); + + private: + PARQUET_DISALLOW_COPY_AND_ASSIGN(Node); +}; + +// Save our breath all over the place with these typedefs +using NodePtr = std::shared_ptr; +using NodeVector = std::vector; + +// A type that is one of the primitive Parquet storage types. In addition to +// the other type metadata (name, repetition level, logical type), also has the +// physical storage type and their type-specific metadata (byte width, decimal +// parameters) +class PARQUET_EXPORT PrimitiveNode : public Node { + public: + static std::unique_ptr FromParquet(const void* opaque_element); + + // A field_id -1 (or any negative value) will be serialized as null in Thrift + static inline NodePtr Make(const std::string& name, Repetition::type repetition, + Type::type type, + ConvertedType::type converted_type = ConvertedType::NONE, + int length = -1, int precision = -1, int scale = -1, + int field_id = -1) { + return NodePtr(new PrimitiveNode(name, repetition, type, converted_type, length, + precision, scale, field_id)); + } + + // If no logical type, pass LogicalType::None() or nullptr + // A field_id -1 (or any negative value) will be serialized as null in Thrift + static inline NodePtr Make(const std::string& name, Repetition::type repetition, + std::shared_ptr logical_type, + Type::type primitive_type, int primitive_length = -1, + int field_id = -1) { + return NodePtr(new PrimitiveNode(name, repetition, std::move(logical_type), + primitive_type, primitive_length, field_id)); + } + + bool Equals(const Node* other) const override; + + Type::type physical_type() const { return physical_type_; } + + ColumnOrder column_order() const { return column_order_; } + + void SetColumnOrder(ColumnOrder column_order) { column_order_ = column_order; } + + int32_t type_length() const { return type_length_; } + + const DecimalMetadata& decimal_metadata() const { return decimal_metadata_; } + + void ToParquet(void* element) const override; + void Visit(Visitor* visitor) override; + void VisitConst(ConstVisitor* visitor) const override; + + private: + PrimitiveNode(const std::string& name, Repetition::type repetition, Type::type type, + ConvertedType::type converted_type = ConvertedType::NONE, int length = -1, + int precision = -1, int scale = -1, int field_id = -1); + + PrimitiveNode(const std::string& name, Repetition::type repetition, + std::shared_ptr logical_type, + Type::type primitive_type, int primitive_length = -1, int field_id = -1); + + Type::type physical_type_; + int32_t type_length_; + DecimalMetadata decimal_metadata_; + ColumnOrder column_order_; + + // For FIXED_LEN_BYTE_ARRAY + void SetTypeLength(int32_t length) { type_length_ = length; } + + bool EqualsInternal(const PrimitiveNode* other) const; + + FRIEND_TEST(TestPrimitiveNode, Attrs); + FRIEND_TEST(TestPrimitiveNode, Equals); + FRIEND_TEST(TestPrimitiveNode, PhysicalLogicalMapping); + FRIEND_TEST(TestPrimitiveNode, FromParquet); +}; + +class PARQUET_EXPORT GroupNode : public Node { + public: + static std::unique_ptr FromParquet(const void* opaque_element, + NodeVector fields = {}); + + // A field_id -1 (or any negative value) will be serialized as null in Thrift + static inline NodePtr Make(const std::string& name, Repetition::type repetition, + const NodeVector& fields, + ConvertedType::type converted_type = ConvertedType::NONE, + int field_id = -1) { + return NodePtr(new GroupNode(name, repetition, fields, converted_type, field_id)); + } + + // If no logical type, pass nullptr + // A field_id -1 (or any negative value) will be serialized as null in Thrift + static inline NodePtr Make(const std::string& name, Repetition::type repetition, + const NodeVector& fields, + std::shared_ptr logical_type, + int field_id = -1) { + return NodePtr( + new GroupNode(name, repetition, fields, std::move(logical_type), field_id)); + } + + bool Equals(const Node* other) const override; + + const NodePtr& field(int i) const { return fields_[i]; } + // Get the index of a field by its name, or negative value if not found. + // If several fields share the same name, it is unspecified which one + // is returned. + int FieldIndex(const std::string& name) const; + // Get the index of a field by its node, or negative value if not found. + int FieldIndex(const Node& node) const; + + int field_count() const { return static_cast(fields_.size()); } + + void ToParquet(void* element) const override; + void Visit(Visitor* visitor) override; + void VisitConst(ConstVisitor* visitor) const override; + + /// \brief Return true if this node or any child node has REPEATED repetition + /// type + bool HasRepeatedFields() const; + + private: + GroupNode(const std::string& name, Repetition::type repetition, + const NodeVector& fields, + ConvertedType::type converted_type = ConvertedType::NONE, int field_id = -1); + + GroupNode(const std::string& name, Repetition::type repetition, + const NodeVector& fields, std::shared_ptr logical_type, + int field_id = -1); + + NodeVector fields_; + bool EqualsInternal(const GroupNode* other) const; + + // Mapping between field name to the field index + std::unordered_multimap field_name_to_idx_; + + FRIEND_TEST(TestGroupNode, Attrs); + FRIEND_TEST(TestGroupNode, Equals); + FRIEND_TEST(TestGroupNode, FieldIndex); + FRIEND_TEST(TestGroupNode, FieldIndexDuplicateName); +}; + +// ---------------------------------------------------------------------- +// Convenience primitive type factory functions + +#define PRIMITIVE_FACTORY(FuncName, TYPE) \ + static inline NodePtr FuncName(const std::string& name, \ + Repetition::type repetition = Repetition::OPTIONAL, \ + int field_id = -1) { \ + return PrimitiveNode::Make(name, repetition, Type::TYPE, ConvertedType::NONE, \ + /*length=*/-1, /*precision=*/-1, /*scale=*/-1, field_id); \ + } + +PRIMITIVE_FACTORY(Boolean, BOOLEAN) +PRIMITIVE_FACTORY(Int32, INT32) +PRIMITIVE_FACTORY(Int64, INT64) +PRIMITIVE_FACTORY(Int96, INT96) +PRIMITIVE_FACTORY(Float, FLOAT) +PRIMITIVE_FACTORY(Double, DOUBLE) +PRIMITIVE_FACTORY(ByteArray, BYTE_ARRAY) + +void PARQUET_EXPORT PrintSchema(const schema::Node* schema, std::ostream& stream, + int indent_width = 2); + +} // namespace schema + +// The ColumnDescriptor encapsulates information necessary to interpret +// primitive column data in the context of a particular schema. We have to +// examine the node structure of a column's path to the root in the schema tree +// to be able to reassemble the nested structure from the repetition and +// definition levels. +class PARQUET_EXPORT ColumnDescriptor { + public: + ColumnDescriptor(schema::NodePtr node, int16_t max_definition_level, + int16_t max_repetition_level, + const SchemaDescriptor* schema_descr = NULLPTR); + + bool Equals(const ColumnDescriptor& other) const; + + int16_t max_definition_level() const { return max_definition_level_; } + + int16_t max_repetition_level() const { return max_repetition_level_; } + + Type::type physical_type() const { return primitive_node_->physical_type(); } + + ConvertedType::type converted_type() const { return primitive_node_->converted_type(); } + + const std::shared_ptr& logical_type() const { + return primitive_node_->logical_type(); + } + + ColumnOrder column_order() const { return primitive_node_->column_order(); } + + SortOrder::type sort_order() const { + const auto& la = logical_type(); + auto pt = physical_type(); + return la ? GetSortOrder(la, pt) : GetSortOrder(converted_type(), pt); + } + + const std::string& name() const { return primitive_node_->name(); } + + const std::shared_ptr path() const; + + const schema::NodePtr& schema_node() const { return node_; } + + std::string ToString() const; + + int type_length() const; + + int type_precision() const; + + int type_scale() const; + + private: + schema::NodePtr node_; + const schema::PrimitiveNode* primitive_node_; + + int16_t max_definition_level_; + int16_t max_repetition_level_; +}; + +// Container for the converted Parquet schema with a computed information from +// the schema analysis needed for file reading +// +// * Column index to Node +// * Max repetition / definition levels for each primitive node +// +// The ColumnDescriptor objects produced by this class can be used to assist in +// the reconstruction of fully materialized data structures from the +// repetition-definition level encoding of nested data +// +// TODO(wesm): this object can be recomputed from a Schema +class PARQUET_EXPORT SchemaDescriptor { + public: + SchemaDescriptor() = default; + ~SchemaDescriptor() = default; + + // Analyze the schema + void Init(std::unique_ptr schema); + void Init(schema::NodePtr schema); + + const ColumnDescriptor* Column(int i) const; + + // Get the index of a column by its dotstring path, or negative value if not found. + // If several columns share the same dotstring path, it is unspecified which one + // is returned. + int ColumnIndex(const std::string& node_path) const; + // Get the index of a column by its node, or negative value if not found. + int ColumnIndex(const schema::Node& node) const; + + bool Equals(const SchemaDescriptor& other, std::ostream* diff_output = NULLPTR) const; + + // The number of physical columns appearing in the file + int num_columns() const { return static_cast(leaves_.size()); } + + const schema::NodePtr& schema_root() const { return schema_; } + + const schema::GroupNode* group_node() const { return group_node_; } + + // Returns the root (child of the schema root) node of the leaf(column) node + const schema::Node* GetColumnRoot(int i) const; + + const std::string& name() const { return group_node_->name(); } + + std::string ToString() const; + + void updateColumnOrders(const std::vector& column_orders); + + /// \brief Return column index corresponding to a particular + /// PrimitiveNode. Returns -1 if not found + int GetColumnIndex(const schema::PrimitiveNode& node) const; + + /// \brief Return true if any field or their children have REPEATED repetition + /// type + bool HasRepeatedFields() const; + + private: + friend class ColumnDescriptor; + + // Root Node + schema::NodePtr schema_; + // Root Node + // Would never be NULLPTR. + const schema::GroupNode* group_node_; + + void BuildTree(const schema::NodePtr& node, int16_t max_def_level, + int16_t max_rep_level, const schema::NodePtr& base); + + // Result of leaf node / tree analysis + std::vector leaves_; + + std::unordered_map node_to_leaf_index_; + + // Mapping between leaf nodes and root group of leaf (first node + // below the schema's root group) + // + // For example, the leaf `a.b.c.d` would have a link back to `a` + // + // -- a <------ + // -- -- b | + // -- -- -- c | + // -- -- -- -- d + std::unordered_map leaf_to_base_; + + // Mapping between ColumnPath DotString to the leaf index + std::unordered_multimap leaf_to_idx_; +}; + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/size_statistics.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/size_statistics.h new file mode 100644 index 0000000000000000000000000000000000000000..8f1652e2b11a88983c99a551c25601fb396599de --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/size_statistics.h @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "parquet/platform.h" +#include "parquet/type_fwd.h" + +namespace parquet { + +/// A structure for capturing metadata for estimating the unencoded, +/// uncompressed size of data written. This is useful for readers to estimate +/// how much memory is needed to reconstruct data in their memory model and for +/// fine-grained filter push down on nested structures (the histograms contained +/// in this structure can help determine the number of nulls at a particular +/// nesting level and maximum length of lists). +struct PARQUET_EXPORT SizeStatistics { + /// When present, there is expected to be one element corresponding to each + /// definition (i.e. size=max definition+1) where each element + /// represents the number of times the definition level was observed in the + /// data. + /// + /// This field may be omitted (a.k.a. zero-length vector) if max_definition_level + /// is 0 without loss of information. + std::vector definition_level_histogram; + + /// Same as definition_level_histogram except for repetition levels. + /// + /// This field may be omitted (a.k.a. zero-length vector) if max_repetition_level + /// is 0 without loss of information. + std::vector repetition_level_histogram; + + /// The number of physical bytes stored for BYTE_ARRAY data values assuming + /// no encoding. This is exclusive of the bytes needed to store the length of + /// each byte array. In other words, this field is equivalent to the `(size + /// of PLAIN-ENCODING the byte array values) - (4 bytes * number of values + /// written)`. To determine unencoded sizes of other types readers can use + /// schema information multiplied by the number of non-null and null values. + /// The number of null/non-null values can be inferred from the histograms + /// below. + /// + /// For example, if a column chunk is dictionary-encoded with dictionary + /// ["a", "bc", "cde"], and a data page contains the indices [0, 0, 1, 2], + /// then this value for that data page should be 7 (1 + 1 + 2 + 3). + /// + /// This field should only be set for types that use BYTE_ARRAY as their + /// physical type. + std::optional unencoded_byte_array_data_bytes; + + /// \brief Check if the SizeStatistics is set. + bool is_set() const { + return !repetition_level_histogram.empty() || !definition_level_histogram.empty() || + unencoded_byte_array_data_bytes.has_value(); + } + + /// \brief Increment the unencoded byte array data bytes. + void IncrementUnencodedByteArrayDataBytes(int64_t value); + + /// \brief Merge two SizeStatistics. + /// \throws ParquetException if SizeStatistics to merge is not compatible. + void Merge(const SizeStatistics& other); + + /// \brief Validate the SizeStatistics + /// \throws ParquetException if the histograms don't have the right length, + /// or if unencoded_byte_array_data_bytes is present for a non-BYTE_ARRAY column. + void Validate(const ColumnDescriptor* descr) const; + + /// \brief Reset the SizeStatistics to be empty. + void Reset(); + + /// \brief Make an empty SizeStatistics object for specific type. + static std::unique_ptr Make(const ColumnDescriptor* descr); +}; + +PARQUET_EXPORT +std::ostream& operator<<(std::ostream&, const SizeStatistics&); + +PARQUET_EXPORT +void UpdateLevelHistogram(std::span levels, std::span histogram); + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/statistics.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/statistics.h new file mode 100644 index 0000000000000000000000000000000000000000..f1efcbf483f827f2df54481fc870a2cfc2987e67 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/statistics.h @@ -0,0 +1,441 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "parquet/platform.h" +#include "parquet/types.h" + +namespace arrow { + +class Array; +class BinaryArray; + +} // namespace arrow + +namespace parquet { + +class ColumnDescriptor; + +// ---------------------------------------------------------------------- +// Value comparator interfaces + +/// \brief Base class for value comparators. Generally used with +/// TypedComparator +class PARQUET_EXPORT Comparator { + public: + virtual ~Comparator() {} + + /// \brief Create a comparator explicitly from physical type and + /// sort order + /// \param[in] physical_type the physical type for the typed + /// comparator + /// \param[in] sort_order either SortOrder::SIGNED or + /// SortOrder::UNSIGNED + /// \param[in] type_length for FIXED_LEN_BYTE_ARRAY only + static std::shared_ptr Make(Type::type physical_type, + SortOrder::type sort_order, + int type_length = -1); + + /// \brief Create typed comparator inferring default sort order from + /// ColumnDescriptor + /// \param[in] descr the Parquet column schema + static std::shared_ptr Make(const ColumnDescriptor* descr); +}; + +/// \brief Interface for comparison of physical types according to the +/// semantics of a particular logical type. +template +class TypedComparator : public Comparator { + public: + using T = typename DType::c_type; + + /// \brief Scalar comparison of two elements, return true if first + /// is strictly less than the second + virtual bool Compare(const T& a, const T& b) const = 0; + + /// \brief Compute maximum and minimum elements in a batch of + /// elements without any nulls + virtual std::pair GetMinMax(const T* values, int64_t length) const = 0; + + /// \brief Compute minimum and maximum elements from an Arrow array. Only + /// valid for certain Parquet Type / Arrow Type combinations, like BYTE_ARRAY + /// / arrow::BinaryArray + virtual std::pair GetMinMax(const ::arrow::Array& values) const = 0; + + /// \brief Compute maximum and minimum elements in a batch of + /// elements with accompanying bitmap indicating which elements are + /// included (bit set) and excluded (bit not set) + /// + /// \param[in] values the sequence of values + /// \param[in] length the length of the sequence + /// \param[in] valid_bits a bitmap indicating which elements are + /// included (1) or excluded (0) + /// \param[in] valid_bits_offset the bit offset into the bitmap of + /// the first element in the sequence + virtual std::pair GetMinMaxSpaced(const T* values, int64_t length, + const uint8_t* valid_bits, + int64_t valid_bits_offset) const = 0; +}; + +/// \brief Typed version of Comparator::Make +template +std::shared_ptr> MakeComparator(Type::type physical_type, + SortOrder::type sort_order, + int type_length = -1) { + return std::static_pointer_cast>( + Comparator::Make(physical_type, sort_order, type_length)); +} + +/// \brief Typed version of Comparator::Make +template +std::shared_ptr> MakeComparator(const ColumnDescriptor* descr) { + return std::static_pointer_cast>(Comparator::Make(descr)); +} + +// ---------------------------------------------------------------------- + +/// \brief Structure represented encoded statistics to be written to +/// and read from Parquet serialized metadata. +class PARQUET_EXPORT EncodedStatistics { + std::string max_, min_; + bool is_signed_ = false; + + public: + EncodedStatistics() = default; + + const std::string& max() const { return max_; } + const std::string& min() const { return min_; } + + std::optional is_max_value_exact; + std::optional is_min_value_exact; + + int64_t null_count = 0; + int64_t distinct_count = 0; + + bool has_min = false; + bool has_max = false; + bool has_null_count = false; + bool has_distinct_count = false; + + // When all values in the statistics are null, it is set to true. + // Otherwise, at least one value is not null, or we are not sure at all. + // Page index requires this information to decide whether a data page + // is a null page or not. + bool all_null_value = false; + + // From parquet-mr + // Don't write stats larger than the max size rather than truncating. The + // rationale is that some engines may use the minimum value in the page as + // the true minimum for aggregations and there is no way to mark that a + // value has been truncated and is a lower bound and not in the page. + void ApplyStatSizeLimits(size_t length) { + if (max_.length() > length) { + has_max = false; + max_.clear(); + is_max_value_exact = std::nullopt; + } + if (min_.length() > length) { + has_min = false; + min_.clear(); + is_min_value_exact = std::nullopt; + } + } + + // Clear Min Max. + void ClearMinMax() { + has_max = false; + max_.clear(); + has_min = false; + min_.clear(); + } + + bool is_set() const { + return has_min || has_max || has_null_count || has_distinct_count; + } + + bool is_signed() const { return is_signed_; } + + void set_is_signed(bool is_signed) { is_signed_ = is_signed; } + + EncodedStatistics& set_max(std::string value) { + max_ = std::move(value); + has_max = true; + return *this; + } + + EncodedStatistics& set_min(std::string value) { + min_ = std::move(value); + has_min = true; + return *this; + } + + EncodedStatistics& set_null_count(int64_t value) { + null_count = value; + has_null_count = true; + return *this; + } + + EncodedStatistics& set_distinct_count(int64_t value) { + distinct_count = value; + has_distinct_count = true; + return *this; + } +}; + +/// \brief Base type for computing column statistics while writing a file +class PARQUET_EXPORT Statistics { + public: + virtual ~Statistics() {} + + /// \brief Create a new statistics instance given a column schema + /// definition + /// \param[in] descr the column schema + /// \param[in] pool a memory pool to use for any memory allocations, optional + static std::shared_ptr Make( + const ColumnDescriptor* descr, + ::arrow::MemoryPool* pool = ::arrow::default_memory_pool()); + + /// \brief Create a new statistics instance given a column schema + /// definition and preexisting state + /// \param[in] descr the column schema + /// \param[in] encoded_min the encoded minimum value + /// \param[in] encoded_max the encoded maximum value + /// \param[in] num_values total number of values + /// \param[in] null_count number of null values + /// \param[in] distinct_count number of distinct values + /// \param[in] has_min_max whether the min/max statistics are set + /// \param[in] has_null_count whether the null_count statistics are set + /// \param[in] has_distinct_count whether the distinct_count statistics are set + /// \param[in] pool a memory pool to use for any memory allocations, optional + static std::shared_ptr Make( + const ColumnDescriptor* descr, const std::string& encoded_min, + const std::string& encoded_max, int64_t num_values, int64_t null_count, + int64_t distinct_count, bool has_min_max, bool has_null_count, + bool has_distinct_count, + ::arrow::MemoryPool* pool = ::arrow::default_memory_pool()); + + /// \brief Create a new statistics instance given a column schema + /// definition and preexisting state + /// \param[in] descr the column schema + /// \param[in] encoded_min the encoded minimum value + /// \param[in] encoded_max the encoded maximum value + /// \param[in] num_values total number of values + /// \param[in] null_count number of null values + /// \param[in] distinct_count number of distinct values + /// \param[in] has_min_max whether the min/max statistics are set + /// \param[in] has_null_count whether the null_count statistics are set + /// \param[in] has_distinct_count whether the distinct_count statistics are set + /// \param[in] is_min_value_exact whether the min value is exact + /// \param[in] is_max_value_exact whether the max value is exact + /// \param[in] pool a memory pool to use for any memory allocations, optional + static std::shared_ptr Make( + const ColumnDescriptor* descr, const std::string& encoded_min, + const std::string& encoded_max, int64_t num_values, int64_t null_count, + int64_t distinct_count, bool has_min_max, bool has_null_count, + bool has_distinct_count, std::optional is_min_value_exact, + std::optional is_max_value_exact, + ::arrow::MemoryPool* pool = ::arrow::default_memory_pool()); + + // Helper function to convert EncodedStatistics to Statistics. + // EncodedStatistics does not contain number of non-null values, and it can be + // passed using the num_values parameter. + static std::shared_ptr Make( + const ColumnDescriptor* descr, const EncodedStatistics* encoded_statistics, + int64_t num_values = -1, + ::arrow::MemoryPool* pool = ::arrow::default_memory_pool()); + + /// \brief Return true if the count of null values is set + virtual bool HasNullCount() const = 0; + + /// \brief The number of null values, may not be set + virtual int64_t null_count() const = 0; + + /// \brief Return true if the count of distinct values is set + virtual bool HasDistinctCount() const = 0; + + /// \brief The number of distinct values, may not be set + virtual int64_t distinct_count() const = 0; + + /// \brief The number of non-null values in the column + virtual int64_t num_values() const = 0; + + /// \brief Return true if both min and max statistics are set. Obtain + /// with TypedStatistics::min and max + virtual bool HasMinMax() const = 0; + + /// \brief Reset state of object to initial (no data observed) state + virtual void Reset() = 0; + + /// \brief Plain-encoded minimum value + virtual std::string EncodeMin() const = 0; + + /// \brief Plain-encoded maximum value + virtual std::string EncodeMax() const = 0; + + /// \brief Return the minimum value exact flag if set. + /// It will be true if there was no truncation. + virtual std::optional is_min_value_exact() const = 0; + + /// \brief Return the maximum value exact flag if set. + /// It will be true if there was no truncation. + virtual std::optional is_max_value_exact() const = 0; + + /// \brief The finalized encoded form of the statistics for transport + virtual EncodedStatistics Encode() = 0; + + /// \brief The physical type of the column schema + virtual Type::type physical_type() const = 0; + + /// \brief The full type descriptor from the column schema + virtual const ColumnDescriptor* descr() const = 0; + + /// \brief Check two Statistics for equality + virtual bool Equals(const Statistics& other) const = 0; + + protected: + static std::shared_ptr Make(Type::type physical_type, const void* min, + const void* max, int64_t num_values, + int64_t null_count, int64_t distinct_count); +}; + +/// \brief A typed implementation of Statistics +template +class TypedStatistics : public Statistics { + public: + using T = typename DType::c_type; + + /// \brief The current minimum value + virtual const T& min() const = 0; + + /// \brief The current maximum value + virtual const T& max() const = 0; + + /// \brief Update state with state of another Statistics object + virtual void Merge(const TypedStatistics& other) = 0; + + /// \brief Batch statistics update + virtual void Update(const T* values, int64_t num_values, int64_t null_count) = 0; + + /// \brief Batch statistics update with supplied validity bitmap + /// \param[in] values pointer to column values + /// \param[in] valid_bits Pointer to bitmap representing if values are non-null. + /// \param[in] valid_bits_offset Offset offset into valid_bits where the slice of + /// data begins. + /// \param[in] num_spaced_values The length of values in values/valid_bits to inspect + /// when calculating statistics. This can be smaller than + /// num_values+null_count as null_count can include nulls + /// from parents while num_spaced_values does not. + /// \param[in] num_values Number of values that are not null. + /// \param[in] null_count Number of values that are null. + virtual void UpdateSpaced(const T* values, const uint8_t* valid_bits, + int64_t valid_bits_offset, int64_t num_spaced_values, + int64_t num_values, int64_t null_count) = 0; + + /// \brief EXPERIMENTAL: Update statistics with an Arrow array without + /// conversion to a primitive Parquet C type. Only implemented for certain + /// Parquet type / Arrow type combinations like BYTE_ARRAY / + /// arrow::BinaryArray + /// + /// If update_counts is true then the null_count and num_values will be updated + /// based on the null_count of values. Set to false if these are updated + /// elsewhere (e.g. when updating a dictionary where the counts are taken from + /// the indices and not the values) + virtual void Update(const ::arrow::Array& values, bool update_counts = true) = 0; + + /// \brief Set min and max values to particular values + virtual void SetMinMax(const T& min, const T& max) = 0; + + /// \brief Increments the null count directly + /// Use Update to extract the null count from data. Use this if you determine + /// the null count through some other means (e.g. dictionary arrays where the + /// null count is determined from the indices) + virtual void IncrementNullCount(int64_t n) = 0; + + /// \brief Increments the number of values directly + /// The same note on IncrementNullCount applies here + virtual void IncrementNumValues(int64_t n) = 0; +}; + +using BoolStatistics = TypedStatistics; +using Int32Statistics = TypedStatistics; +using Int64Statistics = TypedStatistics; +using FloatStatistics = TypedStatistics; +using DoubleStatistics = TypedStatistics; +using ByteArrayStatistics = TypedStatistics; +using FLBAStatistics = TypedStatistics; + +/// \brief Typed version of Statistics::Make +template +std::shared_ptr> MakeStatistics( + const ColumnDescriptor* descr, + ::arrow::MemoryPool* pool = ::arrow::default_memory_pool()) { + return std::static_pointer_cast>(Statistics::Make(descr, pool)); +} + +/// \brief Create Statistics initialized to a particular state +/// \param[in] min the minimum value +/// \param[in] max the minimum value +/// \param[in] num_values number of values +/// \param[in] null_count number of null values +/// \param[in] distinct_count number of distinct values +template +std::shared_ptr> MakeStatistics(const typename DType::c_type& min, + const typename DType::c_type& max, + int64_t num_values, + int64_t null_count, + int64_t distinct_count) { + return std::static_pointer_cast>(Statistics::Make( + DType::type_num, &min, &max, num_values, null_count, distinct_count)); +} + +/// \brief Typed version of Statistics::Make +template +std::shared_ptr> MakeStatistics( + const ColumnDescriptor* descr, const std::string& encoded_min, + const std::string& encoded_max, int64_t num_values, int64_t null_count, + int64_t distinct_count, bool has_min_max, bool has_null_count, + bool has_distinct_count, ::arrow::MemoryPool* pool = ::arrow::default_memory_pool()) { + return std::static_pointer_cast>(Statistics::Make( + descr, encoded_min, encoded_max, num_values, null_count, distinct_count, + has_min_max, has_null_count, has_distinct_count, + /*is_min_value_exact=*/std::nullopt, /*is_max_value_exact=*/std::nullopt, pool)); +} + +/// \brief Typed version of Statistics::Make +template +std::shared_ptr> MakeStatistics( + const ColumnDescriptor* descr, const std::string& encoded_min, + const std::string& encoded_max, int64_t num_values, int64_t null_count, + int64_t distinct_count, bool has_min_max, bool has_null_count, + bool has_distinct_count, std::optional is_min_value_exact, + std::optional is_max_value_exact, + ::arrow::MemoryPool* pool = ::arrow::default_memory_pool()) { + return std::static_pointer_cast>( + Statistics::Make(descr, encoded_min, encoded_max, num_values, null_count, + distinct_count, has_min_max, has_null_count, has_distinct_count, + is_min_value_exact, is_max_value_exact, pool)); +} + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/stream_reader.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/stream_reader.h new file mode 100644 index 0000000000000000000000000000000000000000..ef570f3d1b9fe1770f856c600988e850566cd447 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/stream_reader.h @@ -0,0 +1,304 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "parquet/column_reader.h" +#include "parquet/file_reader.h" +#include "parquet/stream_writer.h" + +namespace parquet { + +/// \brief A class for reading Parquet files using an output stream type API. +/// +/// The values given must be of the correct type i.e. the type must +/// match the file schema exactly otherwise a ParquetException will be +/// thrown. +/// +/// The user must explicitly advance to the next row using the +/// EndRow() function or EndRow input manipulator. +/// +/// Required and optional fields are supported: +/// - Required fields are read using operator>>(T) +/// - Optional fields are read with +/// operator>>(std::optional) +/// +/// Note that operator>>(std::optional) can be used to read +/// required fields. +/// +/// Similarly operator>>(T) can be used to read optional fields. +/// However, if the value is not present then a ParquetException will +/// be raised. +/// +/// Currently there is no support for repeated fields. +/// +class PARQUET_EXPORT StreamReader { + public: + template + using optional = ::std::optional; + + // N.B. Default constructed objects are not usable. This + // constructor is provided so that the object may be move + // assigned afterwards. + StreamReader() = default; + + /// Reader must have at least one field defined in its schema. + explicit StreamReader(std::unique_ptr reader); + + ~StreamReader() = default; + + bool eof() const { return eof_; } + + int current_column() const { return column_index_; } + + int64_t current_row() const { return current_row_; } + + int num_columns() const; + + int64_t num_rows() const; + + // Moving is possible. + StreamReader(StreamReader&&) = default; + StreamReader& operator=(StreamReader&&) = default; + + // Copying is not allowed. + StreamReader(const StreamReader&) = delete; + StreamReader& operator=(const StreamReader&) = delete; + + StreamReader& operator>>(bool& v); + + StreamReader& operator>>(int8_t& v); + + StreamReader& operator>>(uint8_t& v); + + StreamReader& operator>>(int16_t& v); + + StreamReader& operator>>(uint16_t& v); + + StreamReader& operator>>(int32_t& v); + + StreamReader& operator>>(uint32_t& v); + + StreamReader& operator>>(int64_t& v); + + StreamReader& operator>>(uint64_t& v); + + StreamReader& operator>>(std::chrono::milliseconds& v); + + StreamReader& operator>>(std::chrono::microseconds& v); + + StreamReader& operator>>(float& v); + + StreamReader& operator>>(double& v); + + StreamReader& operator>>(char& v); + + template + StreamReader& operator>>(char (&v)[N]) { + ReadFixedLength(v, N); + return *this; + } + + template + StreamReader& operator>>(std::array& v) { + ReadFixedLength(v.data(), static_cast(N)); + return *this; + } + + // N.B. Cannot allow for reading to a arbitrary char pointer as the + // length cannot be verified. Also it would overshadow the + // char[N] input operator. + // StreamReader& operator>>(char * v); + + StreamReader& operator>>(std::string& v); + + StreamReader& operator>>(::arrow::Decimal128& v); + + // Input operators for optional fields. + + StreamReader& operator>>(optional& v); + + StreamReader& operator>>(optional& v); + + StreamReader& operator>>(optional& v); + + StreamReader& operator>>(optional& v); + + StreamReader& operator>>(optional& v); + + StreamReader& operator>>(optional& v); + + StreamReader& operator>>(optional& v); + + StreamReader& operator>>(optional& v); + + StreamReader& operator>>(optional& v); + + StreamReader& operator>>(optional& v); + + StreamReader& operator>>(optional& v); + + StreamReader& operator>>(optional& v); + + StreamReader& operator>>(optional& v); + + StreamReader& operator>>(optional& v); + + StreamReader& operator>>(optional& v); + + StreamReader& operator>>(optional<::arrow::Decimal128>& v); + + template + StreamReader& operator>>(optional>& v) { + CheckColumn(Type::FIXED_LEN_BYTE_ARRAY, ConvertedType::NONE, N); + FixedLenByteArray flba; + if (ReadOptional(&flba)) { + v = std::array{}; + std::memcpy(v->data(), flba.ptr, N); + } else { + v.reset(); + } + return *this; + } + + /// \brief Terminate current row and advance to next one. + /// \throws ParquetException if all columns in the row were not + /// read or skipped. + void EndRow(); + + /// \brief Skip the data in the next columns. + /// If the number of columns exceeds the columns remaining on the + /// current row then skipping is terminated - it does _not_ continue + /// skipping columns on the next row. + /// Skipping of columns still requires the use 'EndRow' even if all + /// remaining columns were skipped. + /// \return Number of columns actually skipped. + int64_t SkipColumns(int64_t num_columns_to_skip); + + /// \brief Skip the data in the next rows. + /// Skipping of rows is not allowed if reading of data for the + /// current row is not finished. + /// Skipping of rows will be terminated if the end of file is + /// reached. + /// \return Number of rows actually skipped. + int64_t SkipRows(int64_t num_rows_to_skip); + + protected: + [[noreturn]] void ThrowReadFailedException( + const std::shared_ptr& node); + + template + void Read(T* v) { + const auto& node = nodes_[column_index_]; + auto reader = static_cast(column_readers_[column_index_++].get()); + int16_t def_level; + int16_t rep_level; + int64_t values_read; + + reader->ReadBatch(kBatchSizeOne, &def_level, &rep_level, v, &values_read); + + if (values_read != 1) { + ThrowReadFailedException(node); + } + } + + template + void Read(T* v) { + const auto& node = nodes_[column_index_]; + auto reader = static_cast(column_readers_[column_index_++].get()); + int16_t def_level; + int16_t rep_level; + ReadType tmp; + int64_t values_read; + + reader->ReadBatch(kBatchSizeOne, &def_level, &rep_level, &tmp, &values_read); + + if (values_read == 1) { + *v = tmp; + } else { + ThrowReadFailedException(node); + } + } + + template + void ReadOptional(optional* v) { + const auto& node = nodes_[column_index_]; + auto reader = static_cast(column_readers_[column_index_++].get()); + int16_t def_level; + int16_t rep_level; + ReadType tmp; + int64_t values_read; + + reader->ReadBatch(kBatchSizeOne, &def_level, &rep_level, &tmp, &values_read); + + if (values_read == 1) { + *v = T(tmp); + } else if ((values_read == 0) && (def_level == 0)) { + v->reset(); + } else { + ThrowReadFailedException(node); + } + } + + void ReadFixedLength(char* ptr, int len); + + void Read(ByteArray* v); + + void Read(FixedLenByteArray* v); + + bool ReadOptional(ByteArray* v); + + bool ReadOptional(FixedLenByteArray* v); + + void NextRowGroup(); + + void CheckColumn(Type::type physical_type, ConvertedType::type converted_type, + int length = 0); + + void SkipRowsInColumn(ColumnReader* reader, int64_t num_rows_to_skip); + + void SetEof(); + + private: + std::unique_ptr file_reader_; + std::shared_ptr file_metadata_; + std::shared_ptr row_group_reader_; + std::vector> column_readers_; + std::vector> nodes_; + + bool eof_{true}; + int row_group_index_{0}; + int column_index_{0}; + int64_t current_row_{0}; + int64_t row_group_row_offset_{0}; + + static constexpr int64_t kBatchSizeOne = 1; +}; // namespace parquet + +PARQUET_EXPORT +StreamReader& operator>>(StreamReader&, EndRowType); + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/stream_writer.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/stream_writer.h new file mode 100644 index 0000000000000000000000000000000000000000..023aad7a415d71a357829fe37e39909abdb6268f --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/stream_writer.h @@ -0,0 +1,251 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "parquet/column_writer.h" +#include "parquet/file_writer.h" + +namespace parquet { + +/// \brief A class for writing Parquet files using an output stream type API. +/// +/// The values given must be of the correct type i.e. the type must +/// match the file schema exactly otherwise a ParquetException will be +/// thrown. +/// +/// The user must explicitly indicate the end of the row using the +/// EndRow() function or EndRow output manipulator. +/// +/// A maximum row group size can be configured, the default size is +/// 512MB. Alternatively the row group size can be set to zero and the +/// user can create new row groups by calling the EndRowGroup() +/// function or using the EndRowGroup output manipulator. +/// +/// Required and optional fields are supported: +/// - Required fields are written using operator<<(T) +/// - Optional fields are written using +/// operator<<(std::optional). +/// +/// Note that operator<<(T) can be used to write optional fields. +/// +/// Similarly, operator<<(std::optional) can be used to +/// write required fields. However if the optional parameter does not +/// have a value (i.e. it is nullopt) then a ParquetException will be +/// raised. +/// +/// Currently there is no support for repeated fields. +/// +class PARQUET_EXPORT StreamWriter { + public: + template + using optional = ::std::optional; + + // N.B. Default constructed objects are not usable. This + // constructor is provided so that the object may be move + // assigned afterwards. + StreamWriter() = default; + + explicit StreamWriter(std::unique_ptr writer); + + ~StreamWriter() = default; + + static void SetDefaultMaxRowGroupSize(int64_t max_size); + + void SetMaxRowGroupSize(int64_t max_size); + + int current_column() const { return column_index_; } + + int64_t current_row() const { return current_row_; } + + int num_columns() const; + + // Moving is possible. + StreamWriter(StreamWriter&&) = default; + StreamWriter& operator=(StreamWriter&&) = default; + + // Copying is not allowed. + StreamWriter(const StreamWriter&) = delete; + StreamWriter& operator=(const StreamWriter&) = delete; + + /// \brief Output operators for required fields. + /// These can also be used for optional fields when a value must be set. + StreamWriter& operator<<(bool v); + + StreamWriter& operator<<(int8_t v); + + StreamWriter& operator<<(uint8_t v); + + StreamWriter& operator<<(int16_t v); + + StreamWriter& operator<<(uint16_t v); + + StreamWriter& operator<<(int32_t v); + + StreamWriter& operator<<(uint32_t v); + + StreamWriter& operator<<(int64_t v); + + StreamWriter& operator<<(uint64_t v); + + StreamWriter& operator<<(const std::chrono::milliseconds& v); + + StreamWriter& operator<<(const std::chrono::microseconds& v); + + StreamWriter& operator<<(float v); + + StreamWriter& operator<<(double v); + + StreamWriter& operator<<(char v); + + /// \brief Helper class to write fixed length strings. + /// This is useful as the standard string view (such as + /// std::string_view) is for variable length data. + struct PARQUET_EXPORT FixedStringView { + FixedStringView() = default; + + explicit FixedStringView(const char* data_ptr); + + FixedStringView(const char* data_ptr, std::size_t data_len); + + const char* data{NULLPTR}; + std::size_t size{0}; + }; + + /// \brief Output operators for fixed length strings. + template + StreamWriter& operator<<(const char (&v)[N]) { + return WriteFixedLength(v, N); + } + template + StreamWriter& operator<<(const std::array& v) { + return WriteFixedLength(v.data(), N); + } + StreamWriter& operator<<(FixedStringView v); + + /// \brief Output operators for variable length strings. + StreamWriter& operator<<(const char* v); + StreamWriter& operator<<(const std::string& v); + StreamWriter& operator<<(::std::string_view v); + + /// \brief Helper class to write variable length raw data. + using RawDataView = std::span; + + /// \brief Output operators for variable length raw data. + StreamWriter& operator<<(RawDataView v); + + /// \brief Output operator for optional fields. + template + StreamWriter& operator<<(const optional& v) { + if (v) { + return operator<<(*v); + } + SkipOptionalColumn(); + return *this; + } + + /// \brief Skip the next N columns of optional data. If there are + /// less than N columns remaining then the excess columns are + /// ignored. + /// \throws ParquetException if there is an attempt to skip any + /// required column. + /// \return Number of columns actually skipped. + int64_t SkipColumns(int num_columns_to_skip); + + /// \brief Terminate the current row and advance to next one. + /// \throws ParquetException if all columns in the row were not + /// written or skipped. + void EndRow(); + + /// \brief Terminate the current row group and create new one. + void EndRowGroup(); + + protected: + template + StreamWriter& Write(const T v) { + auto writer = static_cast(row_group_writer_->column(column_index_++)); + + writer->WriteBatch(kBatchSizeOne, &kDefLevelOne, &kRepLevelZero, &v); + + if (max_row_group_size_ > 0) { + row_group_size_ += writer->estimated_buffered_value_bytes(); + } + return *this; + } + + StreamWriter& WriteVariableLength(const char* data_ptr, std::size_t data_len, + ConvertedType::type converted_type); + + StreamWriter& WriteFixedLength(const char* data_ptr, std::size_t data_len); + + void CheckColumn(Type::type physical_type, ConvertedType::type converted_type, + int length = -1); + + /// \brief Skip the next column which must be optional. + /// \throws ParquetException if the next column does not exist or is + /// not optional. + void SkipOptionalColumn(); + + void WriteNullValue(ColumnWriter* writer); + + private: + using node_ptr_type = std::shared_ptr; + + struct null_deleter { + void operator()(void*) {} + }; + + int32_t column_index_{0}; + int64_t current_row_{0}; + int64_t row_group_size_{0}; + int64_t max_row_group_size_{default_row_group_size_}; + + std::unique_ptr file_writer_; + std::unique_ptr row_group_writer_; + std::vector nodes_; + + static constexpr int16_t kDefLevelZero = 0; + static constexpr int16_t kDefLevelOne = 1; + static constexpr int16_t kRepLevelZero = 0; + static constexpr int64_t kBatchSizeOne = 1; + + static int64_t default_row_group_size_; +}; + +struct PARQUET_EXPORT EndRowType {}; +constexpr EndRowType EndRow = {}; + +struct PARQUET_EXPORT EndRowGroupType {}; +constexpr EndRowGroupType EndRowGroup = {}; + +PARQUET_EXPORT +StreamWriter& operator<<(StreamWriter&, EndRowType); + +PARQUET_EXPORT +StreamWriter& operator<<(StreamWriter&, EndRowGroupType); + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/test_util.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/test_util.h new file mode 100644 index 0000000000000000000000000000000000000000..b7229444e25b5707fb06022896e30d4a3d129ede --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/test_util.h @@ -0,0 +1,893 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// This module defines an abstract interface for iterating through pages in a +// Parquet column chunk within a row group. It could be extended in the future +// to iterate through all data pages in all chunks in a file. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +#include "arrow/extension_type.h" +#include "arrow/io/memory.h" +#include "arrow/testing/util.h" +#include "arrow/util/endian.h" +#include "arrow/util/float16.h" + +#include "parquet/column_page.h" +#include "parquet/column_reader.h" +#include "parquet/column_writer.h" +#include "parquet/encoding.h" +#include "parquet/platform.h" + +// https://github.com/google/googletest/pull/2904 might not be available +// in our version of gtest/gmock +#define EXPECT_THROW_THAT(callable, ex_type, property) \ + EXPECT_THROW( \ + try { (callable)(); } catch (const ex_type& err) { \ + EXPECT_THAT(err, (property)); \ + throw; \ + }, \ + ex_type) + +namespace parquet { + +static constexpr int FLBA_LENGTH = 12; + +inline bool operator==(const FixedLenByteArray& a, const FixedLenByteArray& b) { + return 0 == memcmp(a.ptr, b.ptr, FLBA_LENGTH); +} + +namespace test { + +typedef ::testing::Types + ParquetTypes; + +class ParquetTestException : public parquet::ParquetException { + using ParquetException::ParquetException; +}; + +const char* get_data_dir(); +std::string get_bad_data_dir(); + +std::string get_data_file(const std::string& filename, bool is_good = true); + +template +static inline void assert_vector_equal(const std::vector& left, + const std::vector& right) { + ASSERT_EQ(left.size(), right.size()); + + for (size_t i = 0; i < left.size(); ++i) { + ASSERT_EQ(left[i], right[i]) << i; + } +} + +template +static inline bool vector_equal(const std::vector& left, const std::vector& right) { + if (left.size() != right.size()) { + return false; + } + + for (size_t i = 0; i < left.size(); ++i) { + if (left[i] != right[i]) { + std::cerr << "index " << i << " left was " << left[i] << " right was " << right[i] + << std::endl; + return false; + } + } + + return true; +} + +template +static std::vector slice(const std::vector& values, int start, int end) { + if (end < start) { + return std::vector(0); + } + + std::vector out(end - start); + for (int i = start; i < end; ++i) { + out[i - start] = values[i]; + } + return out; +} + +void random_bytes(int n, uint32_t seed, std::vector* out); +void random_bools(int n, double p, uint32_t seed, bool* out); + +template +inline void random_numbers(int n, uint32_t seed, T min_value, T max_value, T* out) { + std::default_random_engine gen(seed); + std::uniform_int_distribution d(min_value, max_value); + for (int i = 0; i < n; ++i) { + out[i] = d(gen); + } +} + +template <> +inline void random_numbers(int n, uint32_t seed, float min_value, float max_value, + float* out) { + std::default_random_engine gen(seed); + std::uniform_real_distribution d(min_value, max_value); + for (int i = 0; i < n; ++i) { + out[i] = d(gen); + } +} + +template <> +inline void random_numbers(int n, uint32_t seed, double min_value, double max_value, + double* out) { + std::default_random_engine gen(seed); + std::uniform_real_distribution d(min_value, max_value); + for (int i = 0; i < n; ++i) { + out[i] = d(gen); + } +} + +void random_Int96_numbers(int n, uint32_t seed, int32_t min_value, int32_t max_value, + Int96* out); + +void random_float16_numbers(int n, uint32_t seed, ::arrow::util::Float16 min_value, + ::arrow::util::Float16 max_value, uint16_t* out); + +void random_fixed_byte_array(int n, uint32_t seed, uint8_t* buf, int len, FLBA* out); + +void random_byte_array(int n, uint32_t seed, uint8_t* buf, ByteArray* out, int min_size, + int max_size); + +void random_byte_array(int n, uint32_t seed, uint8_t* buf, ByteArray* out, int max_size); + +void prefixed_random_byte_array(int n, uint32_t seed, uint8_t* buf, ByteArray* out, + int min_size, int max_size, double prefixed_probability); + +void prefixed_random_byte_array(int n, uint32_t seed, uint8_t* buf, int len, FLBA* out, + double prefixed_probability); + +template +std::shared_ptr EncodeValues(Encoding::type encoding, bool use_dictionary, + const Sequence& values, int length, + const ColumnDescriptor* descr) { + auto encoder = MakeTypedEncoder(encoding, use_dictionary, descr); + encoder->Put(values, length); + return encoder->FlushValues(); +} + +template +static void InitValues(int num_values, uint32_t seed, std::vector& values, + std::vector& buffer) { + random_numbers(num_values, seed, std::numeric_limits::min(), + std::numeric_limits::max(), values.data()); +} + +template +static void InitValues(int num_values, std::vector& values, + std::vector& buffer) { + InitValues(num_values, 0, values, buffer); +} + +template +static void InitDictValues(int num_values, int num_dicts, std::vector& values, + std::vector& buffer) { + int repeat_factor = num_values / num_dicts; + InitValues(num_dicts, values, buffer); + // add some repeated values + for (int j = 1; j < repeat_factor; ++j) { + for (int i = 0; i < num_dicts; ++i) { + std::memcpy(&values[num_dicts * j + i], &values[i], sizeof(T)); + } + } + // computed only dict_per_page * repeat_factor - 1 values < num_values + // compute remaining + for (int i = num_dicts * repeat_factor; i < num_values; ++i) { + std::memcpy(&values[i], &values[i - num_dicts * repeat_factor], sizeof(T)); + } +} + +template <> +inline void InitDictValues(int num_values, int num_dicts, std::vector& values, + std::vector& buffer) { + // No op for bool +} + +class MockPageReader : public PageReader { + public: + explicit MockPageReader(const std::vector>& pages) + : pages_(pages), page_index_(0) {} + + std::shared_ptr NextPage() override { + if (page_index_ == static_cast(pages_.size())) { + // EOS to consumer + return std::shared_ptr(nullptr); + } + return pages_[page_index_++]; + } + + // No-op + void set_max_page_header_size(uint32_t size) override {} + + private: + std::vector> pages_; + int page_index_; +}; + +// TODO(wesm): this is only used for testing for now. Refactor to form part of +// primary file write path +template +class DataPageBuilder { + public: + using c_type = typename Type::c_type; + + // This class writes data and metadata to the passed inputs + explicit DataPageBuilder(ArrowOutputStream* sink) + : sink_(sink), + num_values_(0), + encoding_(Encoding::PLAIN), + definition_level_encoding_(Encoding::RLE), + repetition_level_encoding_(Encoding::RLE), + have_def_levels_(false), + have_rep_levels_(false), + have_values_(false) {} + + void AppendDefLevels(const std::vector& levels, int16_t max_level, + Encoding::type encoding = Encoding::RLE) { + AppendLevels(levels, max_level, encoding); + + num_values_ = std::max(static_cast(levels.size()), num_values_); + definition_level_encoding_ = encoding; + have_def_levels_ = true; + } + + void AppendRepLevels(const std::vector& levels, int16_t max_level, + Encoding::type encoding = Encoding::RLE) { + AppendLevels(levels, max_level, encoding); + + num_values_ = std::max(static_cast(levels.size()), num_values_); + repetition_level_encoding_ = encoding; + have_rep_levels_ = true; + } + + void AppendValues(const ColumnDescriptor* d, const std::vector& values, + Encoding::type encoding = Encoding::PLAIN) { + std::shared_ptr values_sink = EncodeValues( + encoding, false, values.data(), static_cast(values.size()), d); + PARQUET_THROW_NOT_OK(sink_->Write(values_sink->data(), values_sink->size())); + + num_values_ = std::max(static_cast(values.size()), num_values_); + encoding_ = encoding; + have_values_ = true; + } + + int32_t num_values() const { return num_values_; } + + Encoding::type encoding() const { return encoding_; } + + Encoding::type rep_level_encoding() const { return repetition_level_encoding_; } + + Encoding::type def_level_encoding() const { return definition_level_encoding_; } + + private: + ArrowOutputStream* sink_; + + int32_t num_values_; + Encoding::type encoding_; + Encoding::type definition_level_encoding_; + Encoding::type repetition_level_encoding_; + + bool have_def_levels_; + bool have_rep_levels_; + bool have_values_; + + // Used internally for both repetition and definition levels + void AppendLevels(const std::vector& levels, int16_t max_level, + Encoding::type encoding) { + if (encoding != Encoding::RLE) { + ParquetException::NYI("only rle encoding currently implemented"); + } + + std::vector encode_buffer(LevelEncoder::MaxBufferSize( + Encoding::RLE, max_level, static_cast(levels.size()))); + + // We encode into separate memory from the output stream because the + // RLE-encoded bytes have to be preceded in the stream by their absolute + // size. + LevelEncoder encoder; + encoder.Init(encoding, max_level, static_cast(levels.size()), + encode_buffer.data(), static_cast(encode_buffer.size())); + + encoder.Encode(static_cast(levels.size()), levels.data()); + + int32_t rle_bytes = encoder.len(); + int32_t rle_bytes_le = ::arrow::bit_util::ToLittleEndian(rle_bytes); + PARQUET_THROW_NOT_OK( + sink_->Write(reinterpret_cast(&rle_bytes_le), sizeof(int32_t))); + PARQUET_THROW_NOT_OK(sink_->Write(encode_buffer.data(), rle_bytes)); + } +}; + +template <> +inline void DataPageBuilder::AppendValues(const ColumnDescriptor* d, + const std::vector& values, + Encoding::type encoding) { + if (encoding != Encoding::PLAIN) { + ParquetException::NYI("only plain encoding currently implemented"); + } + + auto encoder = MakeTypedEncoder(Encoding::PLAIN, false, d); + dynamic_cast(encoder.get()) + ->Put(values, static_cast(values.size())); + std::shared_ptr buffer = encoder->FlushValues(); + PARQUET_THROW_NOT_OK(sink_->Write(buffer->data(), buffer->size())); + + num_values_ = std::max(static_cast(values.size()), num_values_); + encoding_ = encoding; + have_values_ = true; +} + +template +static std::shared_ptr MakeDataPage( + const ColumnDescriptor* d, const std::vector& values, + int num_vals, Encoding::type encoding, const uint8_t* indices, int indices_size, + const std::vector& def_levels, int16_t max_def_level, + const std::vector& rep_levels, int16_t max_rep_level) { + int num_values = 0; + + auto page_stream = CreateOutputStream(); + test::DataPageBuilder page_builder(page_stream.get()); + + if (!rep_levels.empty()) { + page_builder.AppendRepLevels(rep_levels, max_rep_level); + } + if (!def_levels.empty()) { + page_builder.AppendDefLevels(def_levels, max_def_level); + } + + if (encoding == Encoding::PLAIN) { + page_builder.AppendValues(d, values, encoding); + num_values = std::max(page_builder.num_values(), num_vals); + } else { // DICTIONARY PAGES + PARQUET_THROW_NOT_OK(page_stream->Write(indices, indices_size)); + num_values = std::max(page_builder.num_values(), num_vals); + } + + PARQUET_ASSIGN_OR_THROW(auto buffer, page_stream->Finish()); + + return std::make_shared(buffer, num_values, encoding, + page_builder.def_level_encoding(), + page_builder.rep_level_encoding(), buffer->size()); +} + +template +class DictionaryPageBuilder { + public: + typedef typename TYPE::c_type TC; + static constexpr int TN = TYPE::type_num; + using SpecializedEncoder = typename EncodingTraits::Encoder; + + // This class writes data and metadata to the passed inputs + explicit DictionaryPageBuilder(const ColumnDescriptor* d) + : num_dict_values_(0), have_values_(false) { + auto encoder = MakeTypedEncoder(Encoding::PLAIN, true, d); + dict_traits_ = dynamic_cast*>(encoder.get()); + encoder_.reset(dynamic_cast(encoder.release())); + } + + ~DictionaryPageBuilder() {} + + std::shared_ptr AppendValues(const std::vector& values) { + int num_values = static_cast(values.size()); + // Dictionary encoding + encoder_->Put(values.data(), num_values); + num_dict_values_ = dict_traits_->num_entries(); + have_values_ = true; + return encoder_->FlushValues(); + } + + std::shared_ptr WriteDict() { + std::shared_ptr dict_buffer = + AllocateBuffer(::arrow::default_memory_pool(), dict_traits_->dict_encoded_size()); + dict_traits_->WriteDict(dict_buffer->mutable_data()); + return dict_buffer; + } + + int32_t num_values() const { return num_dict_values_; } + + private: + DictEncoder* dict_traits_; + std::unique_ptr encoder_; + int32_t num_dict_values_; + bool have_values_; +}; + +template <> +inline DictionaryPageBuilder::DictionaryPageBuilder( + const ColumnDescriptor* d) { + ParquetException::NYI("only plain encoding currently implemented for boolean"); +} + +template <> +inline std::shared_ptr DictionaryPageBuilder::WriteDict() { + ParquetException::NYI("only plain encoding currently implemented for boolean"); + return nullptr; +} + +template <> +inline std::shared_ptr DictionaryPageBuilder::AppendValues( + const std::vector& values) { + ParquetException::NYI("only plain encoding currently implemented for boolean"); + return nullptr; +} + +template +inline static std::shared_ptr MakeDictPage( + const ColumnDescriptor* d, const std::vector& values, + const std::vector& values_per_page, Encoding::type encoding, + std::vector>& rle_indices) { + test::DictionaryPageBuilder page_builder(d); + int num_pages = static_cast(values_per_page.size()); + int value_start = 0; + + for (int i = 0; i < num_pages; i++) { + rle_indices.push_back(page_builder.AppendValues( + slice(values, value_start, value_start + values_per_page[i]))); + value_start += values_per_page[i]; + } + + auto buffer = page_builder.WriteDict(); + + return std::make_shared(buffer, page_builder.num_values(), + Encoding::PLAIN); +} + +// Given def/rep levels and values create multiple dict pages +template +inline static void PaginateDict(const ColumnDescriptor* d, + const std::vector& values, + const std::vector& def_levels, + int16_t max_def_level, + const std::vector& rep_levels, + int16_t max_rep_level, int num_levels_per_page, + const std::vector& values_per_page, + std::vector>& pages, + Encoding::type encoding = Encoding::RLE_DICTIONARY) { + int num_pages = static_cast(values_per_page.size()); + std::vector> rle_indices; + std::shared_ptr dict_page = + MakeDictPage(d, values, values_per_page, encoding, rle_indices); + pages.push_back(dict_page); + int def_level_start = 0; + int def_level_end = 0; + int rep_level_start = 0; + int rep_level_end = 0; + for (int i = 0; i < num_pages; i++) { + if (max_def_level > 0) { + def_level_start = i * num_levels_per_page; + def_level_end = (i + 1) * num_levels_per_page; + } + if (max_rep_level > 0) { + rep_level_start = i * num_levels_per_page; + rep_level_end = (i + 1) * num_levels_per_page; + } + std::shared_ptr data_page = MakeDataPage( + d, {}, values_per_page[i], encoding, rle_indices[i]->data(), + static_cast(rle_indices[i]->size()), + slice(def_levels, def_level_start, def_level_end), max_def_level, + slice(rep_levels, rep_level_start, rep_level_end), max_rep_level); + pages.push_back(data_page); + } +} + +// Given def/rep levels and values create multiple plain pages +template +static inline void PaginatePlain(const ColumnDescriptor* d, + const std::vector& values, + const std::vector& def_levels, + int16_t max_def_level, + const std::vector& rep_levels, + int16_t max_rep_level, int num_levels_per_page, + const std::vector& values_per_page, + std::vector>& pages, + Encoding::type encoding = Encoding::PLAIN) { + int num_pages = static_cast(values_per_page.size()); + int def_level_start = 0; + int def_level_end = 0; + int rep_level_start = 0; + int rep_level_end = 0; + int value_start = 0; + for (int i = 0; i < num_pages; i++) { + if (max_def_level > 0) { + def_level_start = i * num_levels_per_page; + def_level_end = (i + 1) * num_levels_per_page; + } + if (max_rep_level > 0) { + rep_level_start = i * num_levels_per_page; + rep_level_end = (i + 1) * num_levels_per_page; + } + std::shared_ptr page = MakeDataPage( + d, slice(values, value_start, value_start + values_per_page[i]), + values_per_page[i], encoding, nullptr, 0, + slice(def_levels, def_level_start, def_level_end), max_def_level, + slice(rep_levels, rep_level_start, rep_level_end), max_rep_level); + pages.push_back(page); + value_start += values_per_page[i]; + } +} + +// Generates pages from randomly generated data +template +static inline int MakePages(const ColumnDescriptor* d, int num_pages, int levels_per_page, + std::vector& def_levels, + std::vector& rep_levels, + std::vector& values, + std::vector& buffer, + std::vector>& pages, + Encoding::type encoding = Encoding::PLAIN, + uint32_t seed = 0) { + int num_levels = levels_per_page * num_pages; + int num_values = 0; + int16_t zero = 0; + int16_t max_def_level = d->max_definition_level(); + int16_t max_rep_level = d->max_repetition_level(); + std::vector values_per_page(num_pages, levels_per_page); + // Create definition levels + if (max_def_level > 0 && num_levels != 0) { + def_levels.resize(num_levels); + random_numbers(num_levels, seed, zero, max_def_level, def_levels.data()); + for (int p = 0; p < num_pages; p++) { + int num_values_per_page = 0; + for (int i = 0; i < levels_per_page; i++) { + if (def_levels[i + p * levels_per_page] == max_def_level) { + num_values_per_page++; + num_values++; + } + } + values_per_page[p] = num_values_per_page; + } + } else { + num_values = num_levels; + } + // Create repetition levels + if (max_rep_level > 0 && num_levels != 0) { + rep_levels.resize(num_levels); + // Using a different seed so that def_levels and rep_levels are different. + random_numbers(num_levels, seed + 789, zero, max_rep_level, rep_levels.data()); + // The generated levels are random. Force the very first page to start with a new + // record. + rep_levels[0] = 0; + // For a null value, rep_levels and def_levels are both 0. + // If we have a repeated value right after this, it needs to start with + // rep_level = 0 to indicate a new record. + for (int i = 0; i < num_levels - 1; ++i) { + if (rep_levels[i] == 0 && def_levels[i] == 0) { + rep_levels[i + 1] = 0; + } + } + } + // Create values + values.resize(num_values); + if (encoding == Encoding::PLAIN) { + InitValues(num_values, values, buffer); + PaginatePlain(d, values, def_levels, max_def_level, rep_levels, max_rep_level, + levels_per_page, values_per_page, pages); + } else if (encoding == Encoding::RLE_DICTIONARY || + encoding == Encoding::PLAIN_DICTIONARY) { + // Calls InitValues and repeats the data + InitDictValues(num_values, levels_per_page, values, buffer); + PaginateDict(d, values, def_levels, max_def_level, rep_levels, max_rep_level, + levels_per_page, values_per_page, pages); + } + + return num_values; +} + +// ---------------------------------------------------------------------- +// Test data generation + +template <> +void inline InitValues(int num_values, uint32_t seed, std::vector& values, + std::vector& buffer) { + values = {}; + if (seed == 0) { + seed = static_cast(::arrow::random_seed()); + } + ::arrow::random_is_valid(num_values, 0.5, &values, static_cast(seed)); +} + +template <> +inline void InitValues(int num_values, uint32_t seed, + std::vector& values, + std::vector& buffer) { + int max_byte_array_len = 12; + int num_bytes = static_cast(max_byte_array_len + sizeof(uint32_t)); + size_t nbytes = num_values * num_bytes; + buffer.resize(nbytes); + random_byte_array(num_values, seed, buffer.data(), values.data(), max_byte_array_len); +} + +inline void InitWideByteArrayValues(int num_values, std::vector& values, + std::vector& buffer, int min_len, + int max_len) { + int num_bytes = static_cast(max_len + sizeof(uint32_t)); + size_t nbytes = num_values * num_bytes; + buffer.resize(nbytes); + random_byte_array(num_values, 0, buffer.data(), values.data(), min_len, max_len); +} + +template <> +inline void InitValues(int num_values, uint32_t seed, std::vector& values, + std::vector& buffer) { + size_t nbytes = num_values * FLBA_LENGTH; + buffer.resize(nbytes); + random_fixed_byte_array(num_values, seed, buffer.data(), FLBA_LENGTH, values.data()); +} + +template <> +inline void InitValues(int num_values, uint32_t seed, std::vector& values, + std::vector& buffer) { + random_Int96_numbers(num_values, seed, std::numeric_limits::min(), + std::numeric_limits::max(), values.data()); +} + +inline std::string TestColumnName(int i) { + std::stringstream col_name; + col_name << "column_" << i; + return col_name.str(); +} + +// This class lives here because of its dependency on the InitValues specializations. +template +class PrimitiveTypedTest : public ::testing::Test { + public: + using c_type = typename TestType::c_type; + + virtual void SetUpSchema(Repetition::type repetition, int num_columns) { + std::vector fields; + + for (int i = 0; i < num_columns; ++i) { + std::string name = TestColumnName(i); + fields.push_back(schema::PrimitiveNode::Make(name, repetition, TestType::type_num, + ConvertedType::NONE, FLBA_LENGTH)); + } + node_ = schema::GroupNode::Make("schema", Repetition::REQUIRED, fields); + schema_.Init(node_); + } + + void SetUpSchema(Repetition::type repetition) { this->SetUpSchema(repetition, 1); } + + void GenerateData(int64_t num_values, uint32_t seed = 0); + void SetupValuesOut(int64_t num_values); + void SyncValuesOut(); + + protected: + schema::NodePtr node_; + SchemaDescriptor schema_; + + // Input buffers + std::vector values_; + + std::vector def_levels_; + + std::vector buffer_; + // Pointer to the values, needed as we cannot use std::vector::data() + c_type* values_ptr_; + std::vector bool_buffer_; + + // Output buffers + std::vector values_out_; + std::vector bool_buffer_out_; + c_type* values_out_ptr_; +}; + +template +inline void PrimitiveTypedTest::SyncValuesOut() {} + +template <> +inline void PrimitiveTypedTest::SyncValuesOut() { + std::vector::const_iterator source_iterator = bool_buffer_out_.begin(); + std::vector::iterator destination_iterator = values_out_.begin(); + while (source_iterator != bool_buffer_out_.end()) { + *destination_iterator++ = *source_iterator++ != 0; + } +} + +template +inline void PrimitiveTypedTest::SetupValuesOut(int64_t num_values) { + values_out_.clear(); + values_out_.resize(num_values); + values_out_ptr_ = values_out_.data(); +} + +template <> +inline void PrimitiveTypedTest::SetupValuesOut(int64_t num_values) { + values_out_.clear(); + values_out_.resize(num_values); + + bool_buffer_out_.clear(); + bool_buffer_out_.resize(num_values); + // Write once to all values so we can copy it without getting Valgrind errors + // about uninitialised values. + std::fill(bool_buffer_out_.begin(), bool_buffer_out_.end(), true); + values_out_ptr_ = reinterpret_cast(bool_buffer_out_.data()); +} + +template +inline void PrimitiveTypedTest::GenerateData(int64_t num_values, + uint32_t seed) { + def_levels_.resize(num_values); + values_.resize(num_values); + + InitValues(static_cast(num_values), seed, values_, buffer_); + values_ptr_ = values_.data(); + + std::fill(def_levels_.begin(), def_levels_.end(), 1); +} + +template <> +inline void PrimitiveTypedTest::GenerateData(int64_t num_values, + uint32_t seed) { + def_levels_.resize(num_values); + values_.resize(num_values); + + InitValues(static_cast(num_values), seed, values_, buffer_); + bool_buffer_.resize(num_values); + std::copy(values_.begin(), values_.end(), bool_buffer_.begin()); + values_ptr_ = reinterpret_cast(bool_buffer_.data()); + + std::fill(def_levels_.begin(), def_levels_.end(), 1); +} + +// ---------------------------------------------------------------------- +// test data generation + +template +inline void GenerateData(int num_values, T* out, std::vector* heap) { + // seed the prng so failure is deterministic + random_numbers(num_values, 0, std::numeric_limits::min(), + std::numeric_limits::max(), out); +} + +template +inline void GenerateBoundData(int num_values, T* out, T min, T max, + std::vector* heap) { + // seed the prng so failure is deterministic + random_numbers(num_values, 0, min, max, out); +} + +template <> +inline void GenerateData(int num_values, bool* out, std::vector* heap) { + // seed the prng so failure is deterministic + random_bools(num_values, 0.5, 0, out); +} + +template <> +inline void GenerateData(int num_values, Int96* out, std::vector* heap) { + // seed the prng so failure is deterministic + random_Int96_numbers(num_values, 0, std::numeric_limits::min(), + std::numeric_limits::max(), out); +} + +template <> +inline void GenerateData(int num_values, ByteArray* out, + std::vector* heap) { + int max_byte_array_len = 12; + heap->resize(num_values * max_byte_array_len); + // seed the prng so failure is deterministic + random_byte_array(num_values, 0, heap->data(), out, 2, max_byte_array_len); +} + +// Generate ByteArray or FLBA data where there is a given probability +// for each value to share a common prefix with its predecessor. +// This is useful to exercise prefix-based encodings such as DELTA_BYTE_ARRAY. +template +inline void GeneratePrefixedData(int num_values, T* out, std::vector* heap, + double prefixed_probability); + +template <> +inline void GeneratePrefixedData(int num_values, ByteArray* out, + std::vector* heap, + double prefixed_probability) { + int max_byte_array_len = 12; + heap->resize(num_values * max_byte_array_len); + // seed the prng so failure is deterministic + prefixed_random_byte_array(num_values, /*seed=*/0, heap->data(), out, /*min_size=*/2, + /*max_size=*/max_byte_array_len, prefixed_probability); +} + +static constexpr int kGenerateDataFLBALength = 8; + +template <> +inline void GeneratePrefixedData(int num_values, FLBA* out, + std::vector* heap, + double prefixed_probability) { + heap->resize(num_values * kGenerateDataFLBALength); + // seed the prng so failure is deterministic + prefixed_random_byte_array(num_values, /*seed=*/0, heap->data(), + kGenerateDataFLBALength, out, prefixed_probability); +} + +template <> +inline void GenerateData(int num_values, FLBA* out, std::vector* heap) { + heap->resize(num_values * kGenerateDataFLBALength); + // seed the prng so failure is deterministic + random_fixed_byte_array(num_values, 0, heap->data(), kGenerateDataFLBALength, out); +} + +// ---------------------------------------------------------------------- +// Test utility functions for geometry + +#if ARROW_LITTLE_ENDIAN +static constexpr uint8_t kWkbNativeEndianness = 0x01; +#else +static constexpr uint8_t kWkbNativeEndianness = 0x00; +#endif + +/// \brief Number of bytes in a WKB Point with X and Y dimensions (uint8_t endian, +/// uint32_t geometry type, 2 * double coordinates) +static constexpr int kWkbPointXYSize = 21; + +std::string MakeWKBPoint(const std::vector& xyzm, bool has_z, bool has_m); + +std::optional> GetWKBPointCoordinateXY(const ByteArray& value); + +// A minimal version of a geoarrow.wkb extension type to test interoperability +class GeoArrowWkbExtensionType : public ::arrow::ExtensionType { + public: + explicit GeoArrowWkbExtensionType(std::shared_ptr<::arrow::DataType> storage_type, + std::string metadata) + : ::arrow::ExtensionType(std::move(storage_type)), metadata_(std::move(metadata)) {} + + std::string extension_name() const override { return "geoarrow.wkb"; } + + std::string Serialize() const override { return metadata_; } + + ::arrow::Result> Deserialize( + std::shared_ptr<::arrow::DataType> storage_type, + const std::string& serialized_data) const override { + return std::make_shared(std::move(storage_type), + serialized_data); + } + + std::shared_ptr<::arrow::Array> MakeArray( + std::shared_ptr<::arrow::ArrayData> data) const override { + return std::make_shared<::arrow::ExtensionArray>(data); + } + + bool ExtensionEquals(const ExtensionType& other) const override { + return other.extension_name() == extension_name() && other.Serialize() == Serialize(); + } + + private: + std::string metadata_; +}; + +std::shared_ptr<::arrow::DataType> geoarrow_wkb( + std::string metadata = "{}", + const std::shared_ptr<::arrow::DataType> storage = ::arrow::binary()); + +std::shared_ptr<::arrow::DataType> geoarrow_wkb_lonlat( + const std::shared_ptr<::arrow::DataType> storage = ::arrow::binary()); + +} // namespace test +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/type_fwd.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/type_fwd.h new file mode 100644 index 0000000000000000000000000000000000000000..6b02381125621aa90ac8bec6a6ec0c9656d14304 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/type_fwd.h @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +namespace parquet { + +/// \brief Feature selection when writing Parquet files +/// +/// `ParquetVersion::type` governs which data types are allowed and how they +/// are represented. For example, uint32_t data will be written differently +/// depending on this value (as INT64 for PARQUET_1_0, as UINT32 for other +/// versions). +/// +/// However, some features - such as compression algorithms, encryption, +/// or the improved "v2" data page format - must be enabled separately in +/// ArrowWriterProperties. +struct ParquetVersion { + enum type : int { + /// Enable only pre-2.2 Parquet format features when writing + /// + /// This setting is useful for maximum compatibility with legacy readers. + /// Note that logical types may still be emitted, as long they have a + /// corresponding converted type. + PARQUET_1_0, + + /// Enable Parquet format 2.4 and earlier features when writing + /// + /// This enables UINT32 as well as logical types which don't have + /// a corresponding converted type. + /// + /// Note: Parquet format 2.4.0 was released in October 2017. + PARQUET_2_4, + + /// Enable Parquet format 2.6 and earlier features when writing + /// + /// This enables the NANOS time unit in addition to the PARQUET_2_4 + /// features. + /// + /// Note: Parquet format 2.6.0 was released in September 2018. + PARQUET_2_6, + + /// Enable latest Parquet format 2.x features + /// + /// This value is equal to the greatest 2.x version supported by + /// this library. + PARQUET_2_LATEST = PARQUET_2_6 + }; +}; + +class FileMetaData; +class FileCryptoMetaData; +class RowGroupMetaData; + +class ColumnDescriptor; +class SchemaDescriptor; + +class ReaderProperties; +class ArrowReaderProperties; + +class WriterProperties; +class WriterPropertiesBuilder; +class ArrowWriterProperties; +class ArrowWriterPropertiesBuilder; + +class EncodedStatistics; +class Statistics; +struct SizeStatistics; + +namespace geospatial { +class GeoStatistics; +struct EncodedGeoStatistics; +} // namespace geospatial + +class ColumnIndex; +class OffsetIndex; +class BloomFilter; +struct BloomFilterOptions; +struct ColumnChunkId; +struct IndexLocation; + +namespace arrow { + +class FileWriter; +class FileReader; + +} // namespace arrow + +namespace schema { +class ColumnPath; +} // namespace schema + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/types.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/types.h new file mode 100644 index 0000000000000000000000000000000000000000..a1b07ff5c0cb5a81bcefb96a14f6e4cc77a5d06a --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/types.h @@ -0,0 +1,886 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "parquet/platform.h" +#include "parquet/type_fwd.h" +#include "parquet/windows_fixup.h" // for OPTIONAL + +namespace arrow::util { + +class Codec; + +} // namespace arrow::util + +namespace parquet { + +// ---------------------------------------------------------------------- +// Metadata enums to match Thrift metadata +// +// The reason we maintain our own enums is to avoid transitive dependency on +// the compiled Thrift headers (and thus thrift/Thrift.h) for users of the +// public API. After building parquet-cpp, you should not need to include +// Thrift headers in your application. This means some boilerplate to convert +// between our types and Parquet's Thrift types. +// +// We can also add special values like NONE to distinguish between metadata +// values being set and not set. As an example consider ConvertedType and +// CompressionCodec + +// Mirrors parquet::Type +struct Type { + enum type { + BOOLEAN = 0, + INT32 = 1, + INT64 = 2, + INT96 = 3, + FLOAT = 4, + DOUBLE = 5, + BYTE_ARRAY = 6, + FIXED_LEN_BYTE_ARRAY = 7, + // Should always be last element. + UNDEFINED = 8 + }; +}; + +// Mirrors parquet::ConvertedType +struct ConvertedType { + enum type { + NONE, // Not a real converted type, but means no converted type is specified + UTF8, + MAP, + MAP_KEY_VALUE, + LIST, + ENUM, + DECIMAL, + DATE, + TIME_MILLIS, + TIME_MICROS, + TIMESTAMP_MILLIS, + TIMESTAMP_MICROS, + UINT_8, + UINT_16, + UINT_32, + UINT_64, + INT_8, + INT_16, + INT_32, + INT_64, + JSON, + BSON, + INTERVAL, + // DEPRECATED INVALID ConvertedType for all-null data. + // Only useful for reading legacy files written out by interim Parquet C++ releases. + // For writing, always emit LogicalType::Null instead. + // See PARQUET-1990. + NA = 25, + UNDEFINED = 26 // Not a real converted type; should always be last element + }; +}; + +// forward declaration +namespace format { + +class LogicalType; + +} + +// Mirrors parquet::FieldRepetitionType +struct Repetition { + enum type { REQUIRED = 0, OPTIONAL = 1, REPEATED = 2, /*Always last*/ UNDEFINED = 3 }; +}; + +// Reference: +// parquet-mr/parquet-hadoop/src/main/java/org/apache/parquet/ +// format/converter/ParquetMetadataConverter.java +// Sort order for page and column statistics. Types are associated with sort +// orders (e.g., UTF8 columns should use UNSIGNED) and column stats are +// aggregated using a sort order. As of parquet-format version 2.3.1, the +// order used to aggregate stats is always SIGNED and is not stored in the +// Parquet file. These stats are discarded for types that need unsigned. +// See PARQUET-686. +struct SortOrder { + enum type { SIGNED, UNSIGNED, UNKNOWN }; +}; + +namespace schema { + +struct DecimalMetadata { + bool isset; + int32_t scale; + int32_t precision; +}; + +} // namespace schema + +/// \brief Implementation of parquet.thrift LogicalType types. +class PARQUET_EXPORT LogicalType { + public: + struct Type { + enum type { + UNDEFINED = 0, // Not a real logical type + STRING = 1, + MAP, + LIST, + ENUM, + DECIMAL, + DATE, + TIME, + TIMESTAMP, + INTERVAL, + INT, + NIL, // Thrift NullType: annotates data that is always null + JSON, + BSON, + UUID, + FLOAT16, + GEOMETRY, + GEOGRAPHY, + VARIANT, + NONE // Not a real logical type; should always be last element + }; + }; + + struct TimeUnit { + enum unit { UNKNOWN = 0, MILLIS = 1, MICROS, NANOS }; + }; + + enum class EdgeInterpolationAlgorithm { + UNKNOWN = 0, + SPHERICAL = 1, + VINCENTY = 2, + THOMAS = 3, + ANDOYER = 4, + KARNEY = 5 + }; + + /// \brief The latest supported Variant specification version by this library + static constexpr int8_t kVariantSpecVersion = 1; + + /// \brief If possible, return a logical type equivalent to the given legacy + /// converted type (and decimal metadata if applicable). + static std::shared_ptr FromConvertedType( + const parquet::ConvertedType::type converted_type, + const parquet::schema::DecimalMetadata converted_decimal_metadata = {false, -1, + -1}); + + /// \brief Return the logical type represented by the Thrift intermediary object. + static std::shared_ptr FromThrift( + const parquet::format::LogicalType& thrift_logical_type); + + /// \brief Return the explicitly requested logical type. + static std::shared_ptr String(); + static std::shared_ptr Map(); + static std::shared_ptr List(); + static std::shared_ptr Enum(); + static std::shared_ptr Decimal(int32_t precision, int32_t scale = 0); + static std::shared_ptr Date(); + static std::shared_ptr Time(bool is_adjusted_to_utc, + LogicalType::TimeUnit::unit time_unit); + + /// \brief Create a Timestamp logical type + /// \param[in] is_adjusted_to_utc set true if the data is UTC-normalized + /// \param[in] time_unit the resolution of the timestamp + /// \param[in] is_from_converted_type if true, the timestamp was generated + /// by translating a legacy converted type of TIMESTAMP_MILLIS or + /// TIMESTAMP_MICROS. Default is false. + /// \param[in] force_set_converted_type if true, always set the + /// legacy ConvertedType TIMESTAMP_MICROS and TIMESTAMP_MILLIS + /// metadata. Default is false + static std::shared_ptr Timestamp( + bool is_adjusted_to_utc, LogicalType::TimeUnit::unit time_unit, + bool is_from_converted_type = false, bool force_set_converted_type = false); + + static std::shared_ptr Interval(); + static std::shared_ptr Int(int bit_width, bool is_signed); + + /// \brief Create a logical type for data that's always null + /// + /// Any physical type can be annotated with this logical type. + static std::shared_ptr Null(); + + static std::shared_ptr JSON(); + static std::shared_ptr BSON(); + static std::shared_ptr UUID(); + static std::shared_ptr Float16(); + static std::shared_ptr Variant( + int8_t specVersion = kVariantSpecVersion); + + static std::shared_ptr Geometry(std::string crs = ""); + + static std::shared_ptr Geography( + std::string crs = "", LogicalType::EdgeInterpolationAlgorithm algorithm = + EdgeInterpolationAlgorithm::SPHERICAL); + + /// \brief Create a placeholder for when no logical type is specified + static std::shared_ptr None(); + + /// \brief Return true if this logical type is consistent with the given underlying + /// physical type. + bool is_applicable(parquet::Type::type primitive_type, + int32_t primitive_length = -1) const; + + /// \brief Return true if this logical type is equivalent to the given legacy converted + /// type (and decimal metadata if applicable). + bool is_compatible(parquet::ConvertedType::type converted_type, + parquet::schema::DecimalMetadata converted_decimal_metadata = { + false, -1, -1}) const; + + /// \brief If possible, return the legacy converted type (and decimal metadata if + /// applicable) equivalent to this logical type. + parquet::ConvertedType::type ToConvertedType( + parquet::schema::DecimalMetadata* out_decimal_metadata) const; + + /// \brief Return a printable representation of this logical type. + std::string ToString() const; + + /// \brief Return a JSON representation of this logical type. + std::string ToJSON() const; + + /// \brief Return a serializable Thrift object for this logical type. + parquet::format::LogicalType ToThrift() const; + + /// \brief Return true if the given logical type is equivalent to this logical type. + bool Equals(const LogicalType& other) const; + + /// \brief Return the enumerated type of this logical type. + LogicalType::Type::type type() const; + + /// \brief Return the appropriate sort order for this logical type. + SortOrder::type sort_order() const; + + // Type checks ... + bool is_string() const; + bool is_map() const; + bool is_list() const; + bool is_enum() const; + bool is_decimal() const; + bool is_date() const; + bool is_time() const; + bool is_timestamp() const; + bool is_interval() const; + bool is_int() const; + bool is_null() const; + bool is_JSON() const; + bool is_BSON() const; + bool is_UUID() const; + bool is_float16() const; + bool is_geometry() const; + bool is_geography() const; + bool is_variant() const; + bool is_none() const; + /// \brief Return true if this logical type is of a known type. + bool is_valid() const; + bool is_invalid() const; + /// \brief Return true if this logical type is suitable for a schema GroupNode. + bool is_nested() const; + bool is_nonnested() const; + /// \brief Return true if this logical type is included in the Thrift output for its + /// node. + bool is_serialized() const; + + LogicalType(const LogicalType&) = delete; + LogicalType& operator=(const LogicalType&) = delete; + virtual ~LogicalType() noexcept; + + protected: + LogicalType(); + + class Impl; + std::unique_ptr impl_; +}; + +/// \brief Allowed for physical type BYTE_ARRAY, must be encoded as UTF-8. +class PARQUET_EXPORT StringLogicalType : public LogicalType { + public: + static std::shared_ptr Make(); + + private: + StringLogicalType() = default; +}; + +/// \brief Allowed for group nodes only. +class PARQUET_EXPORT MapLogicalType : public LogicalType { + public: + static std::shared_ptr Make(); + + private: + MapLogicalType() = default; +}; + +/// \brief Allowed for group nodes only. +class PARQUET_EXPORT ListLogicalType : public LogicalType { + public: + static std::shared_ptr Make(); + + private: + ListLogicalType() = default; +}; + +/// \brief Allowed for physical type BYTE_ARRAY, must be encoded as UTF-8. +class PARQUET_EXPORT EnumLogicalType : public LogicalType { + public: + static std::shared_ptr Make(); + + private: + EnumLogicalType() = default; +}; + +/// \brief Allowed for physical type INT32, INT64, FIXED_LEN_BYTE_ARRAY, or BYTE_ARRAY, +/// depending on the precision. +class PARQUET_EXPORT DecimalLogicalType : public LogicalType { + public: + static std::shared_ptr Make(int32_t precision, int32_t scale = 0); + int32_t precision() const; + int32_t scale() const; + + private: + DecimalLogicalType() = default; +}; + +/// \brief Allowed for physical type INT32. +class PARQUET_EXPORT DateLogicalType : public LogicalType { + public: + static std::shared_ptr Make(); + + private: + DateLogicalType() = default; +}; + +/// \brief Allowed for physical type INT32 (for MILLIS) or INT64 (for MICROS and NANOS). +class PARQUET_EXPORT TimeLogicalType : public LogicalType { + public: + static std::shared_ptr Make(bool is_adjusted_to_utc, + LogicalType::TimeUnit::unit time_unit); + bool is_adjusted_to_utc() const; + LogicalType::TimeUnit::unit time_unit() const; + + private: + TimeLogicalType() = default; +}; + +/// \brief Allowed for physical type INT64. +class PARQUET_EXPORT TimestampLogicalType : public LogicalType { + public: + static std::shared_ptr Make(bool is_adjusted_to_utc, + LogicalType::TimeUnit::unit time_unit, + bool is_from_converted_type = false, + bool force_set_converted_type = false); + bool is_adjusted_to_utc() const; + LogicalType::TimeUnit::unit time_unit() const; + + /// \brief If true, will not set LogicalType in Thrift metadata + bool is_from_converted_type() const; + + /// \brief If true, will set ConvertedType for micros and millis + /// resolution in legacy ConvertedType Thrift metadata + bool force_set_converted_type() const; + + private: + TimestampLogicalType() = default; +}; + +/// \brief Allowed for physical type FIXED_LEN_BYTE_ARRAY with length 12 +class PARQUET_EXPORT IntervalLogicalType : public LogicalType { + public: + static std::shared_ptr Make(); + + private: + IntervalLogicalType() = default; +}; + +/// \brief Allowed for physical type INT32 (for bit widths 8, 16, and 32) and INT64 +/// (for bit width 64). +class PARQUET_EXPORT IntLogicalType : public LogicalType { + public: + static std::shared_ptr Make(int bit_width, bool is_signed); + int bit_width() const; + bool is_signed() const; + + private: + IntLogicalType() = default; +}; + +/// \brief Allowed for any physical type. +class PARQUET_EXPORT NullLogicalType : public LogicalType { + public: + static std::shared_ptr Make(); + + private: + NullLogicalType() = default; +}; + +/// \brief Allowed for physical type BYTE_ARRAY. +class PARQUET_EXPORT JSONLogicalType : public LogicalType { + public: + static std::shared_ptr Make(); + + private: + JSONLogicalType() = default; +}; + +/// \brief Allowed for physical type BYTE_ARRAY. +class PARQUET_EXPORT BSONLogicalType : public LogicalType { + public: + static std::shared_ptr Make(); + + private: + BSONLogicalType() = default; +}; + +/// \brief Allowed for physical type FIXED_LEN_BYTE_ARRAY with length 16, +/// must encode raw UUID bytes. +class PARQUET_EXPORT UUIDLogicalType : public LogicalType { + public: + static std::shared_ptr Make(); + + private: + UUIDLogicalType() = default; +}; + +/// \brief Allowed for physical type FIXED_LEN_BYTE_ARRAY with length 2, +/// must encode raw FLOAT16 bytes. +class PARQUET_EXPORT Float16LogicalType : public LogicalType { + public: + static std::shared_ptr Make(); + + private: + Float16LogicalType() = default; +}; + +class PARQUET_EXPORT GeometryLogicalType : public LogicalType { + public: + static std::shared_ptr Make(std::string crs = ""); + + const std::string& crs() const; + + private: + GeometryLogicalType() = default; +}; + +class PARQUET_EXPORT GeographyLogicalType : public LogicalType { + public: + static std::shared_ptr Make( + std::string crs = "", LogicalType::EdgeInterpolationAlgorithm algorithm = + EdgeInterpolationAlgorithm::SPHERICAL); + + const std::string& crs() const; + LogicalType::EdgeInterpolationAlgorithm algorithm() const; + std::string_view algorithm_name() const; + + private: + GeographyLogicalType() = default; +}; + +/// \brief Allowed for group nodes only. +class PARQUET_EXPORT VariantLogicalType : public LogicalType { + public: + static std::shared_ptr Make( + int8_t specVersion = kVariantSpecVersion); + + int8_t spec_version() const; + + private: + VariantLogicalType() = default; +}; + +/// \brief Allowed for any physical type. +class PARQUET_EXPORT NoLogicalType : public LogicalType { + public: + static std::shared_ptr Make(); + + private: + NoLogicalType() = default; +}; + +// Internal API, for unrecognized logical types +class PARQUET_EXPORT UndefinedLogicalType : public LogicalType { + public: + static std::shared_ptr Make(); + + private: + UndefinedLogicalType() = default; +}; + +// Data encodings. Mirrors parquet::Encoding +struct Encoding { + enum type { + PLAIN = 0, + PLAIN_DICTIONARY = 2, + RLE = 3, + BIT_PACKED = 4, + DELTA_BINARY_PACKED = 5, + DELTA_LENGTH_BYTE_ARRAY = 6, + DELTA_BYTE_ARRAY = 7, + RLE_DICTIONARY = 8, + BYTE_STREAM_SPLIT = 9, + // Should always be last element (except UNKNOWN) + UNDEFINED = 10, + UNKNOWN = 999 + }; +}; + +// Exposed data encodings. It is the encoding of the data read from the file, +// rather than the encoding of the data in the file. E.g., the data encoded as +// RLE_DICTIONARY in the file can be read as dictionary indices by RLE +// decoding, in which case the data read from the file is DICTIONARY encoded. +enum class ExposedEncoding { + NO_ENCODING = 0, // data is not encoded, i.e. already decoded during reading + DICTIONARY = 1 +}; + +/// \brief Return true if Parquet supports indicated compression type +PARQUET_EXPORT +bool IsCodecSupported(Compression::type codec); + +PARQUET_EXPORT +std::unique_ptr GetCodec(Compression::type codec); + +PARQUET_EXPORT +std::unique_ptr GetCodec(Compression::type codec, + const CodecOptions& codec_options); + +PARQUET_EXPORT +std::unique_ptr GetCodec(Compression::type codec, int compression_level); + +struct ParquetCipher { + enum type { AES_GCM_V1 = 0, AES_GCM_CTR_V1 = 1 }; +}; + +struct AadMetadata { + std::string aad_prefix; + std::string aad_file_unique; + bool supply_aad_prefix; +}; + +struct EncryptionAlgorithm { + ParquetCipher::type algorithm; + AadMetadata aad; +}; + +// parquet::PageType +struct PageType { + enum type { + DATA_PAGE, + INDEX_PAGE, + DICTIONARY_PAGE, + DATA_PAGE_V2, + // Should always be last element + UNDEFINED + }; +}; + +bool PageCanUseChecksum(PageType::type pageType); + +class ColumnOrder { + public: + enum type { UNDEFINED, TYPE_DEFINED_ORDER }; + explicit ColumnOrder(ColumnOrder::type column_order) : column_order_(column_order) {} + // Default to Type Defined Order + ColumnOrder() : column_order_(type::TYPE_DEFINED_ORDER) {} + ColumnOrder::type get_order() { return column_order_; } + + static ColumnOrder undefined_; + static ColumnOrder type_defined_; + + private: + ColumnOrder::type column_order_; +}; + +/// \brief BoundaryOrder is a proxy around format::BoundaryOrder. +struct BoundaryOrder { + enum type { + Unordered = 0, + Ascending = 1, + Descending = 2, + // Should always be last element + UNDEFINED = 3 + }; +}; + +/// \brief SortingColumn is a proxy around format::SortingColumn. +struct PARQUET_EXPORT SortingColumn { + // The column index (in this row group) + int32_t column_idx; + + // If true, indicates this column is sorted in descending order. + bool descending; + + // If true, nulls will come before non-null values, otherwise, nulls go at the end. + bool nulls_first; +}; + +inline bool operator==(const SortingColumn& left, const SortingColumn& right) { + return left.nulls_first == right.nulls_first && left.descending == right.descending && + left.column_idx == right.column_idx; +} + +inline bool operator!=(const SortingColumn& left, const SortingColumn& right) { + return !(left == right); +} + +// ---------------------------------------------------------------------- + +struct ByteArray { + ByteArray() : len(0), ptr(NULLPTR) {} + ByteArray(uint32_t len, const uint8_t* ptr) : len(len), ptr(ptr) {} + + ByteArray(::std::string_view view) // NOLINT implicit conversion + : ByteArray(static_cast(view.size()), + reinterpret_cast(view.data())) {} + + explicit operator std::string_view() const { + return std::string_view{reinterpret_cast(ptr), len}; + } + + uint32_t len; + const uint8_t* ptr; +}; + +inline bool operator==(const ByteArray& left, const ByteArray& right) { + return left.len == right.len && + (left.len == 0 || std::memcmp(left.ptr, right.ptr, left.len) == 0); +} + +inline bool operator!=(const ByteArray& left, const ByteArray& right) { + return !(left == right); +} + +struct FixedLenByteArray { + FixedLenByteArray() : ptr(NULLPTR) {} + explicit FixedLenByteArray(const uint8_t* ptr) : ptr(ptr) {} + const uint8_t* ptr; +}; + +using FLBA = FixedLenByteArray; + +// Julian day at unix epoch. +// +// The Julian Day Number (JDN) is the integer assigned to a whole solar day in +// the Julian day count starting from noon Universal time, with Julian day +// number 0 assigned to the day starting at noon on Monday, January 1, 4713 BC, +// proleptic Julian calendar (November 24, 4714 BC, in the proleptic Gregorian +// calendar), +constexpr int64_t kJulianToUnixEpochDays = INT64_C(2440588); +constexpr int64_t kSecondsPerDay = INT64_C(60 * 60 * 24); +constexpr int64_t kMillisecondsPerDay = kSecondsPerDay * INT64_C(1000); +constexpr int64_t kMicrosecondsPerDay = kMillisecondsPerDay * INT64_C(1000); +constexpr int64_t kNanosecondsPerDay = kMicrosecondsPerDay * INT64_C(1000); + +ARROW_PACKED_START(struct, Int96) { std::array value; }; +ARROW_PACKED_END +static_assert(sizeof(Int96) == 12, "Int96 not packed to 12 bytes"); +static_assert(alignof(Int96) <= 4, "Int96 alignment too large"); + +inline bool operator==(const Int96& left, const Int96& right) { + return left.value == right.value; +} + +inline bool operator!=(const Int96& left, const Int96& right) { return !(left == right); } + +static inline std::string ByteArrayToString(const ByteArray& a) { + return std::string(reinterpret_cast(a.ptr), a.len); +} + +static inline void Int96SetNanoSeconds(parquet::Int96& i96, int64_t nanoseconds) { + std::memcpy(&i96.value, &nanoseconds, sizeof(nanoseconds)); +} + +struct DecodedInt96 { + uint64_t days_since_epoch; + uint64_t nanoseconds; +}; + +static inline DecodedInt96 DecodeInt96Timestamp(const parquet::Int96& i96) { + // We do the computations in the unsigned domain to avoid unsigned behaviour + // on overflow. + DecodedInt96 result; + result.days_since_epoch = i96.value[2] - static_cast(kJulianToUnixEpochDays); + result.nanoseconds = 0; + + memcpy(&result.nanoseconds, &i96.value, sizeof(uint64_t)); + return result; +} + +static inline int64_t Int96GetNanoSeconds(const parquet::Int96& i96) { + const auto decoded = DecodeInt96Timestamp(i96); + return static_cast(decoded.days_since_epoch * kNanosecondsPerDay + + decoded.nanoseconds); +} + +static inline int64_t Int96GetMicroSeconds(const parquet::Int96& i96) { + const auto decoded = DecodeInt96Timestamp(i96); + uint64_t microseconds = decoded.nanoseconds / static_cast(1000); + return static_cast(decoded.days_since_epoch * kMicrosecondsPerDay + + microseconds); +} + +static inline int64_t Int96GetMilliSeconds(const parquet::Int96& i96) { + const auto decoded = DecodeInt96Timestamp(i96); + uint64_t milliseconds = decoded.nanoseconds / static_cast(1000000); + return static_cast(decoded.days_since_epoch * kMillisecondsPerDay + + milliseconds); +} + +static inline int64_t Int96GetSeconds(const parquet::Int96& i96) { + const auto decoded = DecodeInt96Timestamp(i96); + uint64_t seconds = decoded.nanoseconds / static_cast(1000000000); + return static_cast(decoded.days_since_epoch * kSecondsPerDay + seconds); +} + +static inline std::string Int96ToString(const Int96& a) { + std::ostringstream result; + std::copy(a.value.begin(), a.value.end(), std::ostream_iterator(result, " ")); + return result.str(); +} + +static inline std::string FixedLenByteArrayToString(const FixedLenByteArray& a, int len) { + std::ostringstream result; + std::copy(a.ptr, a.ptr + len, std::ostream_iterator(result, " ")); + return result.str(); +} + +template +struct type_traits {}; + +template <> +struct type_traits { + using value_type = bool; + + static constexpr int value_byte_size = 1; + static constexpr const char* printf_code = "d"; +}; + +template <> +struct type_traits { + using value_type = int32_t; + + static constexpr int value_byte_size = 4; + static constexpr const char* printf_code = "d"; +}; + +template <> +struct type_traits { + using value_type = int64_t; + + static constexpr int value_byte_size = 8; + static constexpr const char* printf_code = + (sizeof(long) == 64) ? "ld" : "lld"; // NOLINT: runtime/int +}; + +template <> +struct type_traits { + using value_type = Int96; + + static constexpr int value_byte_size = 12; + static constexpr const char* printf_code = "s"; +}; + +template <> +struct type_traits { + using value_type = float; + + static constexpr int value_byte_size = 4; + static constexpr const char* printf_code = "f"; +}; + +template <> +struct type_traits { + using value_type = double; + + static constexpr int value_byte_size = 8; + static constexpr const char* printf_code = "lf"; +}; + +template <> +struct type_traits { + using value_type = ByteArray; + + static constexpr int value_byte_size = sizeof(ByteArray); + static constexpr const char* printf_code = "s"; +}; + +template <> +struct type_traits { + using value_type = FixedLenByteArray; + + static constexpr int value_byte_size = sizeof(FixedLenByteArray); + static constexpr const char* printf_code = "s"; +}; + +template +struct PhysicalType { + using c_type = typename type_traits::value_type; + static constexpr Type::type type_num = TYPE; +}; + +using BooleanType = PhysicalType; +using Int32Type = PhysicalType; +using Int64Type = PhysicalType; +using Int96Type = PhysicalType; +using FloatType = PhysicalType; +using DoubleType = PhysicalType; +using ByteArrayType = PhysicalType; +using FLBAType = PhysicalType; + +template +inline std::string format_fwf(int width) { + std::stringstream ss; + ss << "%-" << width << type_traits::printf_code; + return ss.str(); +} + +PARQUET_EXPORT std::string EncodingToString(Encoding::type t); + +PARQUET_EXPORT std::string ConvertedTypeToString(ConvertedType::type t); + +PARQUET_EXPORT std::string TypeToString(Type::type t); + +PARQUET_EXPORT std::string TypeToString(Type::type t, int type_length); + +PARQUET_EXPORT std::string FormatStatValue( + Type::type parquet_type, ::std::string_view val, + const std::shared_ptr& logical_type = NULLPTR); + +PARQUET_EXPORT int GetTypeByteSize(Type::type t); + +PARQUET_EXPORT SortOrder::type DefaultSortOrder(Type::type primitive); + +PARQUET_EXPORT SortOrder::type GetSortOrder(ConvertedType::type converted, + Type::type primitive); + +PARQUET_EXPORT SortOrder::type GetSortOrder( + const std::shared_ptr& logical_type, Type::type primitive); + +// PLAIN_DICTIONARY is deprecated but used to be used as a dictionary index +// encoding. +constexpr bool IsDictionaryIndexEncoding(Encoding::type e) { + return e == Encoding::RLE_DICTIONARY || e == Encoding::PLAIN_DICTIONARY; +} + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/visit_type_inline.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/visit_type_inline.h new file mode 100644 index 0000000000000000000000000000000000000000..fae348d9b791f584851a3a1b1cbda93fded44d18 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/visit_type_inline.h @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "arrow/util/macros.h" +#include "parquet/exception.h" +#include "parquet/types.h" + +namespace parquet { + +#define TYPE_VISIT_INLINE(TYPE_VALUE) \ + case Type::TYPE_VALUE: { \ + const PhysicalType* concrete_ptr = NULLPTR; \ + return std::forward(visitor)(concrete_ptr, std::forward(args)...); \ + } + +/// \brief Call `visitor` with a nullptr of the corresponding concrete type class +/// \tparam ARGS Additional arguments, if any, will be passed to the visitor after +/// the type argument +/// +/// The intent is for this to be called on a generic lambda +/// that may internally use `if constexpr` or similar constructs. +template +auto VisitType(Type::type type, VISITOR&& visitor, ARGS&&... args) + -> decltype(std::forward(visitor)(std::declval*>(), + args...)) { + switch (type) { + TYPE_VISIT_INLINE(INT32) + TYPE_VISIT_INLINE(INT64) + TYPE_VISIT_INLINE(INT96) + TYPE_VISIT_INLINE(FLOAT) + TYPE_VISIT_INLINE(DOUBLE) + TYPE_VISIT_INLINE(FIXED_LEN_BYTE_ARRAY) + TYPE_VISIT_INLINE(BYTE_ARRAY) + TYPE_VISIT_INLINE(BOOLEAN) + default: + throw ParquetException("Invalid Type::type"); + } +} + +#undef TYPE_VISIT_INLINE + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/windows_compatibility.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/windows_compatibility.h new file mode 100644 index 0000000000000000000000000000000000000000..44c62ba31b5bb96cabf34d6f9e14ae2a94332e44 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/windows_compatibility.h @@ -0,0 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "arrow/util/windows_compatibility.h" +#include "parquet/windows_fixup.h" diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/windows_fixup.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/windows_fixup.h new file mode 100644 index 0000000000000000000000000000000000000000..b4e0aed3a0ee17dc668bff59270e15466244f625 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/windows_fixup.h @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// This header needs to be included multiple times. + +#include "arrow/util/windows_fixup.h" + +#ifdef _WIN32 + +// parquet.thrift's OPTIONAL RepetitionType conflicts with a Windows #define +# ifdef OPTIONAL +# undef OPTIONAL +# endif + +#endif // _WIN32 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/xxhasher.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/xxhasher.h new file mode 100644 index 0000000000000000000000000000000000000000..baccc60d8ff14db3d6552abb861ba1637b720a42 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/xxhasher.h @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "parquet/hasher.h" +#include "parquet/platform.h" +#include "parquet/types.h" + +namespace parquet { + +class PARQUET_EXPORT XxHasher : public Hasher { + public: + uint64_t Hash(int32_t value) const override; + uint64_t Hash(int64_t value) const override; + uint64_t Hash(float value) const override; + uint64_t Hash(double value) const override; + uint64_t Hash(const Int96* value) const override; + uint64_t Hash(const ByteArray* value) const override; + uint64_t Hash(const FLBA* val, uint32_t len) const override; + uint64_t Hash(std::string_view value) const override; + + void Hashes(const int32_t* values, int num_values, uint64_t* hashes) const override; + void Hashes(const int64_t* values, int num_values, uint64_t* hashes) const override; + void Hashes(const float* values, int num_values, uint64_t* hashes) const override; + void Hashes(const double* values, int num_values, uint64_t* hashes) const override; + void Hashes(const Int96* values, int num_values, uint64_t* hashes) const override; + void Hashes(const ByteArray* values, int num_values, uint64_t* hashes) const override; + void Hashes(const FLBA* values, uint32_t type_len, int num_values, + uint64_t* hashes) const override; + + static constexpr int kParquetBloomXxHashSeed = 0; +}; + +} // namespace parquet diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/__init__.pxd b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/__init__.pxd new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/common.pxd b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/common.pxd new file mode 100644 index 0000000000000000000000000000000000000000..65cafc5fa857ed8dcf685f6e33bb22ba5bbd478f --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/common.pxd @@ -0,0 +1,155 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# distutils: language = c++ + +from libc.stdint cimport * + +from libcpp cimport bool as c_bool, nullptr +from libcpp.functional cimport function +from libcpp.memory cimport (shared_ptr, unique_ptr, make_shared, + static_pointer_cast, dynamic_pointer_cast) +from libcpp.optional cimport nullopt, optional +from libcpp.string cimport string as c_string +from libcpp.utility cimport move, pair +from libcpp.vector cimport vector +from libcpp.unordered_map cimport unordered_map +from libcpp.unordered_set cimport unordered_set + +from cpython cimport PyObject +from cpython.datetime cimport PyDateTime_DateTime +cimport cpython + + +cdef extern from "" namespace "std" nogil: + # Needed until https://github.com/cython/cython/issues/6651 is fixed + cdef cppclass cpp_string_view "std::string_view": + string_view() + string_view(const char*) + string_view(c_string&) + size_t size() + bint empty() + const char* data() + + +cdef extern from * namespace "arrow::py" nogil: + """ + #include + #include + #include + #include + + namespace arrow { + namespace py { + + template + std::shared_ptr to_shared(std::unique_ptr& t) { + return std::move(t); + } + template + std::shared_ptr to_shared(std::unique_ptr&& t) { + return std::move(t); + } + + // Needed until https://github.com/cython/cython/issues/6651 is fixed + inline std::string to_string(std::string_view s) { + return std::string(s); + } + + } // namespace py + } // namespace arrow + """ + cdef shared_ptr[T] to_shared" arrow::py::to_shared"[T](unique_ptr[T]) + cdef c_string to_string(cpp_string_view s) + +cdef extern from "arrow/python/platform.h": + pass + +cdef extern from "": + void Py_XDECREF(PyObject* o) + Py_ssize_t Py_REFCNT(PyObject* o) + +cdef extern from "arrow/api.h" namespace "arrow" nogil: + # We can later add more of the common status factory methods as needed + cdef CStatus CStatus_OK "arrow::Status::OK"() + + cdef CStatus CStatus_Invalid "arrow::Status::Invalid"() + cdef CStatus CStatus_NotImplemented \ + "arrow::Status::NotImplemented"(const c_string& msg) + cdef CStatus CStatus_UnknownError \ + "arrow::Status::UnknownError"(const c_string& msg) + + cdef cppclass CStatus "arrow::Status": + CStatus() + + c_string ToString() + c_string message() + shared_ptr[CStatusDetail] detail() + + c_bool ok() + c_bool IsIOError() + c_bool IsOutOfMemory() + c_bool IsInvalid() + c_bool IsKeyError() + c_bool IsNotImplemented() + c_bool IsTypeError() + c_bool IsCapacityError() + c_bool IsIndexError() + c_bool IsSerializationError() + c_bool IsCancelled() + + void Warn() + + cdef cppclass CStatusDetail "arrow::StatusDetail": + c_string ToString() + + +cdef extern from "arrow/result.h" namespace "arrow" nogil: + cdef cppclass CResult "arrow::Result"[T]: + CResult() + CResult(CStatus) + CResult(T) + c_bool ok() + CStatus status() + CStatus Value(T*) + T operator*() + + +cdef extern from "arrow/util/future.h" namespace "arrow" nogil: + cdef cppclass CFuture "arrow::Future"[T]: + CFuture() + + +cdef extern from "arrow/python/async.h" namespace "arrow::py" nogil: + # BindFuture's third argument is really a C++ callable with + # the signature `object(T*)`, but Cython does not allow declaring that. + # We use an ellipsis as a workaround. + # Another possibility is to type-erase the argument by making it + # `object(void*)`, but it would lose compile-time C++ type safety. + void BindFuture[T](CFuture[T], object cb, ...) + + +cdef extern from "arrow/python/common.h" namespace "arrow::py" nogil: + T GetResultValue[T](CResult[T]) except * + cdef function[F] BindFunction[F](void* unbound, object bound, ...) + + +cdef inline object PyObject_to_object(PyObject* o): + # Cast to "object" increments reference count + cdef object result = o + cpython.Py_DECREF(result) + return result diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow.pxd b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow.pxd new file mode 100644 index 0000000000000000000000000000000000000000..40a8c9afb842ec5f4e3c3977486084f427401d24 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow.pxd @@ -0,0 +1,3281 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# distutils: language = c++ + +from pyarrow.includes.common cimport * + + +cdef extern from "arrow/util/key_value_metadata.h" namespace "arrow" nogil: + cdef cppclass CKeyValueMetadata" arrow::KeyValueMetadata": + CKeyValueMetadata() + CKeyValueMetadata(const unordered_map[c_string, c_string]&) + CKeyValueMetadata(const vector[c_string]& keys, + const vector[c_string]& values) + + void reserve(int64_t n) + int64_t size() const + c_string key(int64_t i) const + c_string value(int64_t i) const + int FindKey(const c_string& key) const + + shared_ptr[CKeyValueMetadata] Copy() const + c_bool Equals(const CKeyValueMetadata& other) + void Append(const c_string& key, const c_string& value) + void ToUnorderedMap(unordered_map[c_string, c_string]*) const + c_string ToString() const + + CResult[c_string] Get(const c_string& key) const + CStatus Delete(const c_string& key) + CStatus Set(const c_string& key, const c_string& value) + c_bool Contains(const c_string& key) const + + +cdef extern from "arrow/util/decimal.h" namespace "arrow" nogil: + cdef cppclass CDecimal32" arrow::Decimal32": + c_string ToString(int32_t scale) const + + +cdef extern from "arrow/util/decimal.h" namespace "arrow" nogil: + cdef cppclass CDecimal64" arrow::Decimal64": + c_string ToString(int32_t scale) const + + +cdef extern from "arrow/util/decimal.h" namespace "arrow" nogil: + cdef cppclass CDecimal128" arrow::Decimal128": + c_string ToString(int32_t scale) const + + +cdef extern from "arrow/util/decimal.h" namespace "arrow" nogil: + cdef cppclass CDecimal256" arrow::Decimal256": + c_string ToString(int32_t scale) const + + +cdef extern from "arrow/config.h" namespace "arrow" nogil: + cdef cppclass CCppBuildInfo "arrow::BuildInfo": + int version + int version_major + int version_minor + int version_patch + c_string version_string + c_string so_version + c_string full_so_version + c_string compiler_id + c_string compiler_version + c_string compiler_flags + c_string git_id + c_string git_description + c_string package_kind + c_string build_type + + const CCppBuildInfo& GetCppBuildInfo "arrow::GetBuildInfo"() + + cdef cppclass CRuntimeInfo" arrow::RuntimeInfo": + c_string simd_level + c_string detected_simd_level + + CRuntimeInfo GetRuntimeInfo() + + cdef cppclass CGlobalOptions" arrow::GlobalOptions": + optional[c_string] timezone_db_path + + CStatus Initialize(const CGlobalOptions& options) + + +cdef extern from "arrow/util/future.h" namespace "arrow" nogil: + cdef cppclass CFuture_Void" arrow::Future<>": + CStatus status() + + +cdef extern from "" namespace "std" nogil: + cdef cppclass CArrayStatisticsCountType" std::variant": + CArrayStatisticsCountType() + CArrayStatisticsCountType(int64_t) + CArrayStatisticsCountType(double) + + cdef cppclass CArrayStatisticsValueType" std::variant": + CArrayStatisticsValueType() + CArrayStatisticsValueType(c_bool) + CArrayStatisticsValueType(int64_t) + CArrayStatisticsValueType(uint64_t) + CArrayStatisticsValueType(double) + CArrayStatisticsValueType(c_string) + + +cdef extern from "arrow/api.h" namespace "arrow" nogil: + cdef enum Type" arrow::Type::type": + _Type_NA" arrow::Type::NA" + + _Type_BOOL" arrow::Type::BOOL" + + _Type_UINT8" arrow::Type::UINT8" + _Type_INT8" arrow::Type::INT8" + _Type_UINT16" arrow::Type::UINT16" + _Type_INT16" arrow::Type::INT16" + _Type_UINT32" arrow::Type::UINT32" + _Type_INT32" arrow::Type::INT32" + _Type_UINT64" arrow::Type::UINT64" + _Type_INT64" arrow::Type::INT64" + + _Type_HALF_FLOAT" arrow::Type::HALF_FLOAT" + _Type_FLOAT" arrow::Type::FLOAT" + _Type_DOUBLE" arrow::Type::DOUBLE" + + _Type_DECIMAL32" arrow::Type::DECIMAL32" + _Type_DECIMAL64" arrow::Type::DECIMAL64" + _Type_DECIMAL128" arrow::Type::DECIMAL128" + _Type_DECIMAL256" arrow::Type::DECIMAL256" + + _Type_DATE32" arrow::Type::DATE32" + _Type_DATE64" arrow::Type::DATE64" + _Type_TIMESTAMP" arrow::Type::TIMESTAMP" + _Type_TIME32" arrow::Type::TIME32" + _Type_TIME64" arrow::Type::TIME64" + _Type_DURATION" arrow::Type::DURATION" + _Type_INTERVAL_MONTH_DAY_NANO" arrow::Type::INTERVAL_MONTH_DAY_NANO" + _Type_INTERVAL_DAY_TIME" arrow::Type::INTERVAL_DAY_TIME" + _Type_INTERVAL_MONTHS" arrow::Type::INTERVAL_MONTHS" + + _Type_BINARY" arrow::Type::BINARY" + _Type_STRING" arrow::Type::STRING" + _Type_LARGE_BINARY" arrow::Type::LARGE_BINARY" + _Type_LARGE_STRING" arrow::Type::LARGE_STRING" + _Type_FIXED_SIZE_BINARY" arrow::Type::FIXED_SIZE_BINARY" + _Type_BINARY_VIEW" arrow::Type::BINARY_VIEW" + _Type_STRING_VIEW" arrow::Type::STRING_VIEW" + + _Type_LIST" arrow::Type::LIST" + _Type_LARGE_LIST" arrow::Type::LARGE_LIST" + _Type_FIXED_SIZE_LIST" arrow::Type::FIXED_SIZE_LIST" + _Type_LIST_VIEW" arrow::Type::LIST_VIEW" + _Type_LARGE_LIST_VIEW" arrow::Type::LARGE_LIST_VIEW" + _Type_STRUCT" arrow::Type::STRUCT" + _Type_SPARSE_UNION" arrow::Type::SPARSE_UNION" + _Type_DENSE_UNION" arrow::Type::DENSE_UNION" + _Type_DICTIONARY" arrow::Type::DICTIONARY" + _Type_RUN_END_ENCODED" arrow::Type::RUN_END_ENCODED" + _Type_MAP" arrow::Type::MAP" + + _Type_EXTENSION" arrow::Type::EXTENSION" + + cdef enum UnionMode" arrow::UnionMode::type": + _UnionMode_SPARSE" arrow::UnionMode::SPARSE" + _UnionMode_DENSE" arrow::UnionMode::DENSE" + + cdef enum TimeUnit" arrow::TimeUnit::type": + TimeUnit_SECOND" arrow::TimeUnit::SECOND" + TimeUnit_MILLI" arrow::TimeUnit::MILLI" + TimeUnit_MICRO" arrow::TimeUnit::MICRO" + TimeUnit_NANO" arrow::TimeUnit::NANO" + + cdef cppclass CBufferSpec" arrow::DataTypeLayout::BufferSpec": + pass + + cdef cppclass CDataTypeLayout" arrow::DataTypeLayout": + vector[CBufferSpec] buffers + optional[CBufferSpec] variadic_spec + c_bool has_dictionary + + cdef cppclass CDataType" arrow::DataType": + Type id() + + c_bool Equals(const CDataType& other, c_bool check_metadata) + c_bool Equals(const shared_ptr[CDataType]& other, c_bool check_metadata) + + shared_ptr[CField] field(int i) + const vector[shared_ptr[CField]] fields() + int num_fields() + CDataTypeLayout layout() + c_string ToString() + + c_bool is_primitive(Type type) + c_bool is_numeric(Type type) + + cdef cppclass CArrayStatistics" arrow::ArrayStatistics": + optional[CArrayStatisticsCountType] null_count + optional[CArrayStatisticsCountType] distinct_count + optional[CArrayStatisticsValueType] min + c_bool is_min_exact + optional[CArrayStatisticsValueType] max + c_bool is_max_exact + + c_bool Equals(const CArrayStatistics& statistics) const + + cdef cppclass CArrayData" arrow::ArrayData": + shared_ptr[CDataType] type + int64_t length + int64_t null_count + int64_t offset + vector[shared_ptr[CBuffer]] buffers + vector[shared_ptr[CArrayData]] child_data + shared_ptr[CArrayData] dictionary + + @staticmethod + shared_ptr[CArrayData] Make(const shared_ptr[CDataType]& type, + int64_t length, + vector[shared_ptr[CBuffer]]& buffers, + int64_t null_count, + int64_t offset) + + @staticmethod + shared_ptr[CArrayData] MakeWithChildren" Make"( + const shared_ptr[CDataType]& type, + int64_t length, + vector[shared_ptr[CBuffer]]& buffers, + vector[shared_ptr[CArrayData]]& child_data, + int64_t null_count, + int64_t offset) + + @staticmethod + shared_ptr[CArrayData] MakeWithChildrenAndDictionary" Make"( + const shared_ptr[CDataType]& type, + int64_t length, + vector[shared_ptr[CBuffer]]& buffers, + vector[shared_ptr[CArrayData]]& child_data, + shared_ptr[CArrayData]& dictionary, + int64_t null_count, + int64_t offset) + + cdef cppclass CArray" arrow::Array": + shared_ptr[CDataType] type() + + int64_t length() + int64_t null_count() + int64_t offset() + Type type_id() + + int num_fields() + + CResult[shared_ptr[CScalar]] GetScalar(int64_t i) const + + c_string Diff(const CArray& other) + c_bool Equals(const CArray& arr) + c_bool IsNull(int i) + + shared_ptr[CArrayData] data() + + shared_ptr[CArray] Slice(int64_t offset) + shared_ptr[CArray] Slice(int64_t offset, int64_t length) + + CStatus Validate() const + CStatus ValidateFull() const + CResult[shared_ptr[CArray]] View(const shared_ptr[CDataType]& type) + + CDeviceAllocationType device_type() + CResult[shared_ptr[CArray]] CopyTo(const shared_ptr[CMemoryManager]& to) const + + const shared_ptr[CArrayStatistics]& statistics() const + + shared_ptr[CArray] MakeArray(const shared_ptr[CArrayData]& data) + CResult[shared_ptr[CArray]] MakeArrayOfNull( + const shared_ptr[CDataType]& type, int64_t length, CMemoryPool* pool) + + CResult[shared_ptr[CArray]] MakeArrayFromScalar( + const CScalar& scalar, int64_t length, CMemoryPool* pool) + + CStatus DebugPrint(const CArray& arr, int indent) + + cdef cppclass CFixedWidthType" arrow::FixedWidthType"(CDataType): + int bit_width() + int byte_width() + + cdef cppclass CNullArray" arrow::NullArray"(CArray): + CNullArray(int64_t length) + + cdef cppclass CDictionaryArray" arrow::DictionaryArray"(CArray): + CDictionaryArray(const shared_ptr[CDataType]& type, + const shared_ptr[CArray]& indices, + const shared_ptr[CArray]& dictionary) + CDictionaryArray(const shared_ptr[CArrayData]& data) + + @staticmethod + CResult[shared_ptr[CArray]] FromArrays( + const shared_ptr[CDataType]& type, + const shared_ptr[CArray]& indices, + const shared_ptr[CArray]& dictionary) + + shared_ptr[CArray] indices() + shared_ptr[CArray] dictionary() + + cdef cppclass CDate32Type" arrow::Date32Type"(CFixedWidthType): + pass + + cdef cppclass CDate64Type" arrow::Date64Type"(CFixedWidthType): + pass + + cdef cppclass CTimestampType" arrow::TimestampType"(CFixedWidthType): + CTimestampType(TimeUnit unit) + TimeUnit unit() + const c_string& timezone() + + cdef cppclass CTime32Type" arrow::Time32Type"(CFixedWidthType): + TimeUnit unit() + + cdef cppclass CTime64Type" arrow::Time64Type"(CFixedWidthType): + TimeUnit unit() + + shared_ptr[CDataType] ctime32" arrow::time32"(TimeUnit unit) + shared_ptr[CDataType] ctime64" arrow::time64"(TimeUnit unit) + + cdef cppclass CDurationType" arrow::DurationType"(CFixedWidthType): + TimeUnit unit() + + shared_ptr[CDataType] cduration" arrow::duration"(TimeUnit unit) + + cdef cppclass CDictionaryType" arrow::DictionaryType"(CFixedWidthType): + CDictionaryType(const shared_ptr[CDataType]& index_type, + const shared_ptr[CDataType]& value_type, + c_bool ordered) + + shared_ptr[CDataType] index_type() + shared_ptr[CDataType] value_type() + c_bool ordered() + + shared_ptr[CDataType] ctimestamp" arrow::timestamp"(TimeUnit unit) + shared_ptr[CDataType] ctimestamp" arrow::timestamp"( + TimeUnit unit, const c_string& timezone) + + cdef cppclass CMemoryPool" arrow::MemoryPool": + int64_t bytes_allocated() + int64_t max_memory() + int64_t total_bytes_allocated() + int64_t num_allocations() + c_string backend_name() + void ReleaseUnused() + void PrintStats() + + cdef cppclass CLoggingMemoryPool" arrow::LoggingMemoryPool"(CMemoryPool): + CLoggingMemoryPool(CMemoryPool*) + + cdef cppclass CProxyMemoryPool" arrow::ProxyMemoryPool"(CMemoryPool): + CProxyMemoryPool(CMemoryPool*) + + ctypedef enum CDeviceAllocationType "arrow::DeviceAllocationType": + CDeviceAllocationType_kCPU "arrow::DeviceAllocationType::kCPU" + CDeviceAllocationType_kCUDA "arrow::DeviceAllocationType::kCUDA" + CDeviceAllocationType_kCUDA_HOST "arrow::DeviceAllocationType::kCUDA_HOST" + CDeviceAllocationType_kOPENCL "arrow::DeviceAllocationType::kOPENCL" + CDeviceAllocationType_kVULKAN "arrow::DeviceAllocationType::kVULKAN" + CDeviceAllocationType_kMETAL "arrow::DeviceAllocationType::kMETAL" + CDeviceAllocationType_kVPI "arrow::DeviceAllocationType::kVPI" + CDeviceAllocationType_kROCM "arrow::DeviceAllocationType::kROCM" + CDeviceAllocationType_kROCM_HOST "arrow::DeviceAllocationType::kROCM_HOST" + CDeviceAllocationType_kEXT_DEV "arrow::DeviceAllocationType::kEXT_DEV" + CDeviceAllocationType_kCUDA_MANAGED "arrow::DeviceAllocationType::kCUDA_MANAGED" + CDeviceAllocationType_kONEAPI "arrow::DeviceAllocationType::kONEAPI" + CDeviceAllocationType_kWEBGPU "arrow::DeviceAllocationType::kWEBGPU" + CDeviceAllocationType_kHEXAGON "arrow::DeviceAllocationType::kHEXAGON" + + cdef cppclass CDevice" arrow::Device": + const char* type_name() + c_string ToString() + c_bool Equals(const CDevice& other) + int64_t device_id() + c_bool is_cpu() const + shared_ptr[CMemoryManager] default_memory_manager() + CDeviceAllocationType device_type() + + cdef cppclass CMemoryManager" arrow::MemoryManager": + const shared_ptr[CDevice] device() + c_bool is_cpu() const + + shared_ptr[CMemoryManager] c_default_cpu_memory_manager \ + " arrow::default_cpu_memory_manager"() + + cdef cppclass CBuffer" arrow::Buffer": + CBuffer(const uint8_t* data, int64_t size) + const uint8_t* data() + uint8_t* mutable_data() + uintptr_t address() + uintptr_t mutable_address() + int64_t size() + shared_ptr[CBuffer] parent() + c_bool is_cpu() const + c_bool is_mutable() const + c_string ToHexString() + c_bool Equals(const CBuffer& other) + shared_ptr[CDevice] device() + const shared_ptr[CMemoryManager] memory_manager() + CDeviceAllocationType device_type() + + CResult[shared_ptr[CBuffer]] SliceBufferSafe( + const shared_ptr[CBuffer]& buffer, int64_t offset) + CResult[shared_ptr[CBuffer]] SliceBufferSafe( + const shared_ptr[CBuffer]& buffer, int64_t offset, int64_t length) + + cdef cppclass CMutableBuffer" arrow::MutableBuffer"(CBuffer): + CMutableBuffer(const uint8_t* data, int64_t size) + + cdef cppclass CResizableBuffer" arrow::ResizableBuffer"(CMutableBuffer): + CStatus Resize(const int64_t new_size, c_bool shrink_to_fit) + CStatus Reserve(const int64_t new_size) + + CResult[unique_ptr[CBuffer]] AllocateBuffer(const int64_t size, + CMemoryPool* pool) + + CResult[unique_ptr[CResizableBuffer]] AllocateResizableBuffer( + const int64_t size, CMemoryPool* pool) + + cdef cppclass CSyncEvent" arrow::Device::SyncEvent": + pass + + cdef cppclass CDevice" arrow::Device": + pass + + cdef CMemoryPool* c_default_memory_pool" arrow::default_memory_pool"() + cdef CMemoryPool* c_system_memory_pool" arrow::system_memory_pool"() + cdef CStatus c_jemalloc_memory_pool" arrow::jemalloc_memory_pool"( + CMemoryPool** out) + cdef CStatus c_mimalloc_memory_pool" arrow::mimalloc_memory_pool"( + CMemoryPool** out) + cdef vector[c_string] c_supported_memory_backends \ + " arrow::SupportedMemoryBackendNames"() + + CStatus c_jemalloc_set_decay_ms" arrow::jemalloc_set_decay_ms"(int ms) + + cdef cppclass CListType" arrow::ListType"(CDataType): + CListType(const shared_ptr[CDataType]& value_type) + CListType(const shared_ptr[CField]& field) + shared_ptr[CDataType] value_type() + shared_ptr[CField] value_field() + + cdef cppclass CLargeListType" arrow::LargeListType"(CDataType): + CLargeListType(const shared_ptr[CDataType]& value_type) + CLargeListType(const shared_ptr[CField]& field) + shared_ptr[CDataType] value_type() + shared_ptr[CField] value_field() + + cdef cppclass CListViewType" arrow::ListViewType"(CDataType): + CListViewType(const shared_ptr[CDataType]& value_type) + CListViewType(const shared_ptr[CField]& field) + shared_ptr[CDataType] value_type() + shared_ptr[CField] value_field() + + cdef cppclass CLargeListViewType" arrow::LargeListViewType"(CDataType): + CLargeListViewType(const shared_ptr[CDataType]& value_type) + CLargeListViewType(const shared_ptr[CField]& field) + shared_ptr[CDataType] value_type() + shared_ptr[CField] value_field() + + cdef cppclass CMapType" arrow::MapType"(CDataType): + CMapType(const shared_ptr[CField]& key_field, + const shared_ptr[CField]& item_field, c_bool keys_sorted) + shared_ptr[CDataType] key_type() + shared_ptr[CField] key_field() + shared_ptr[CDataType] item_type() + shared_ptr[CField] item_field() + c_bool keys_sorted() + + cdef cppclass CFixedSizeListType" arrow::FixedSizeListType"(CDataType): + CFixedSizeListType(const shared_ptr[CDataType]& value_type, + int32_t list_size) + CFixedSizeListType(const shared_ptr[CField]& field, int32_t list_size) + shared_ptr[CDataType] value_type() + shared_ptr[CField] value_field() + int32_t list_size() + + cdef cppclass CStringType" arrow::StringType"(CDataType): + pass + + cdef cppclass CFixedSizeBinaryType \ + " arrow::FixedSizeBinaryType"(CFixedWidthType): + CFixedSizeBinaryType(int byte_width) + int byte_width() + int bit_width() + + cdef cppclass CDecimal32Type \ + " arrow::Decimal32Type"(CFixedSizeBinaryType): + CDecimal32Type(int precision, int scale) + int precision() + int scale() + + cdef cppclass CDecimal64Type \ + " arrow::Decimal64Type"(CFixedSizeBinaryType): + CDecimal64Type(int precision, int scale) + int precision() + int scale() + + cdef cppclass CDecimal128Type \ + " arrow::Decimal128Type"(CFixedSizeBinaryType): + CDecimal128Type(int precision, int scale) + int precision() + int scale() + + cdef cppclass CDecimal256Type \ + " arrow::Decimal256Type"(CFixedSizeBinaryType): + CDecimal256Type(int precision, int scale) + int precision() + int scale() + + cdef cppclass CRunEndEncodedType " arrow::RunEndEncodedType"(CDataType): + CRunEndEncodedType(const shared_ptr[CDataType]& run_end_type, + const shared_ptr[CDataType]& value_type) + const shared_ptr[CDataType]& run_end_type() + const shared_ptr[CDataType]& value_type() + + cdef cppclass CField" arrow::Field": + cppclass CMergeOptions "MergeOptions": + CMergeOptions() + c_bool promote_nullability + + @staticmethod + CMergeOptions Defaults() + + @staticmethod + CMergeOptions Permissive() + + const c_string& name() + shared_ptr[CDataType] type() + c_bool nullable() + + c_string ToString() + c_bool Equals(const CField& other, c_bool check_metadata) + + shared_ptr[const CKeyValueMetadata] metadata() + + CField(const c_string& name, const shared_ptr[CDataType]& type, + c_bool nullable) + + CField(const c_string& name, const shared_ptr[CDataType]& type, + c_bool nullable, const shared_ptr[CKeyValueMetadata]& metadata) + + # Removed const in Cython so don't have to cast to get code to generate + shared_ptr[CField] AddMetadata( + const shared_ptr[CKeyValueMetadata]& metadata) + shared_ptr[CField] WithMetadata( + const shared_ptr[CKeyValueMetadata]& metadata) + shared_ptr[CField] RemoveMetadata() + shared_ptr[CField] WithType(const shared_ptr[CDataType]& type) + shared_ptr[CField] WithName(const c_string& name) + shared_ptr[CField] WithNullable(c_bool nullable) + vector[shared_ptr[CField]] Flatten() + + cdef cppclass CFieldRef" arrow::FieldRef": + CFieldRef() + CFieldRef(c_string name) + CFieldRef(int index) + CFieldRef(vector[CFieldRef]) + + @staticmethod + CResult[CFieldRef] FromDotPath(c_string& dot_path) + const c_string* name() const + + cdef cppclass CFieldRefHash" arrow::FieldRef::Hash": + pass + + cdef cppclass CStructType" arrow::StructType"(CDataType): + CStructType(const vector[shared_ptr[CField]]& fields) + + shared_ptr[CField] GetFieldByName(const c_string& name) + vector[shared_ptr[CField]] GetAllFieldsByName(const c_string& name) + int GetFieldIndex(const c_string& name) + vector[int] GetAllFieldIndices(const c_string& name) + + cdef cppclass CUnionType" arrow::UnionType"(CDataType): + UnionMode mode() + const vector[int8_t]& type_codes() + const vector[int]& child_ids() + + cdef shared_ptr[CDataType] CMakeSparseUnionType" arrow::sparse_union"( + vector[shared_ptr[CField]] fields, + vector[int8_t] type_codes) + + cdef shared_ptr[CDataType] CMakeDenseUnionType" arrow::dense_union"( + vector[shared_ptr[CField]] fields, + vector[int8_t] type_codes) + + cdef shared_ptr[CDataType] CMakeRunEndEncodedType" arrow::run_end_encoded"( + shared_ptr[CDataType] run_end_type, + shared_ptr[CDataType] value_type) + + cdef shared_ptr[CDataType] CMakeListViewType" arrow::list_view"( + shared_ptr[CField] value_type) + + cdef shared_ptr[CDataType] CMakeLargeListViewType" arrow::large_list_view"( + shared_ptr[CField] value_type) + + cdef cppclass CSchema" arrow::Schema": + CSchema(const vector[shared_ptr[CField]]& fields) + CSchema(const vector[shared_ptr[CField]]& fields, + const shared_ptr[const CKeyValueMetadata]& metadata) + + # Does not actually exist, but gets Cython to not complain + CSchema(const vector[shared_ptr[CField]]& fields, + const shared_ptr[CKeyValueMetadata]& metadata) + + c_bool Equals(const CSchema& other, c_bool check_metadata) + + shared_ptr[CField] field(int i) + shared_ptr[const CKeyValueMetadata] metadata() + shared_ptr[CField] GetFieldByName(const c_string& name) + vector[shared_ptr[CField]] GetAllFieldsByName(const c_string& name) + int GetFieldIndex(const c_string& name) + vector[int] GetAllFieldIndices(const c_string& name) + const vector[shared_ptr[CField]] fields() + int num_fields() + c_string ToString() + + CResult[shared_ptr[CSchema]] AddField(int i, + const shared_ptr[CField]& field) + CResult[shared_ptr[CSchema]] RemoveField(int i) + CResult[shared_ptr[CSchema]] SetField(int i, + const shared_ptr[CField]& field) + + # Removed const in Cython so don't have to cast to get code to generate + shared_ptr[CSchema] AddMetadata( + const shared_ptr[CKeyValueMetadata]& metadata) + shared_ptr[CSchema] WithMetadata( + const shared_ptr[CKeyValueMetadata]& metadata) + shared_ptr[CSchema] RemoveMetadata() + + CResult[shared_ptr[CSchema]] UnifySchemas( + const vector[shared_ptr[CSchema]]& schemas, + CField.CMergeOptions field_merge_options) + + cdef cppclass PrettyPrintOptions: + PrettyPrintOptions() + PrettyPrintOptions(int indent_arg) + PrettyPrintOptions(int indent_arg, int window_arg) + int indent + int indent_size + int window + int container_window + c_string null_rep + c_bool skip_new_lines + c_bool truncate_metadata + c_bool show_field_metadata + c_bool show_schema_metadata + int element_size_limit + + @staticmethod + PrettyPrintOptions Defaults() + + CStatus PrettyPrint(const CArray& schema, + const PrettyPrintOptions& options, + c_string* result) + CStatus PrettyPrint(const CChunkedArray& schema, + const PrettyPrintOptions& options, + c_string* result) + CStatus PrettyPrint(const CSchema& schema, + const PrettyPrintOptions& options, + c_string* result) + + cdef cppclass CBooleanArray" arrow::BooleanArray"(CArray): + c_bool Value(int i) + int64_t false_count() + int64_t true_count() + + cdef cppclass CUInt8Array" arrow::UInt8Array"(CArray): + uint8_t Value(int i) + + cdef cppclass CInt8Array" arrow::Int8Array"(CArray): + int8_t Value(int i) + + cdef cppclass CUInt16Array" arrow::UInt16Array"(CArray): + uint16_t Value(int i) + + cdef cppclass CInt16Array" arrow::Int16Array"(CArray): + int16_t Value(int i) + + cdef cppclass CUInt32Array" arrow::UInt32Array"(CArray): + uint32_t Value(int i) + + cdef cppclass CInt32Array" arrow::Int32Array"(CArray): + int32_t Value(int i) + + cdef cppclass CUInt64Array" arrow::UInt64Array"(CArray): + uint64_t Value(int i) + + cdef cppclass CInt64Array" arrow::Int64Array"(CArray): + int64_t Value(int i) + + cdef cppclass CDate32Array" arrow::Date32Array"(CArray): + int32_t Value(int i) + + cdef cppclass CDate64Array" arrow::Date64Array"(CArray): + int64_t Value(int i) + + cdef cppclass CTime32Array" arrow::Time32Array"(CArray): + int32_t Value(int i) + + cdef cppclass CTime64Array" arrow::Time64Array"(CArray): + int64_t Value(int i) + + cdef cppclass CTimestampArray" arrow::TimestampArray"(CArray): + int64_t Value(int i) + + cdef cppclass CDurationArray" arrow::DurationArray"(CArray): + int64_t Value(int i) + + cdef cppclass CMonthDayNanoIntervalArray \ + "arrow::MonthDayNanoIntervalArray"(CArray): + pass + + cdef cppclass CHalfFloatArray" arrow::HalfFloatArray"(CArray): + uint16_t Value(int i) + + cdef cppclass CFloatArray" arrow::FloatArray"(CArray): + float Value(int i) + + cdef cppclass CDoubleArray" arrow::DoubleArray"(CArray): + double Value(int i) + + cdef cppclass CFixedSizeBinaryArray" arrow::FixedSizeBinaryArray"(CArray): + const uint8_t* GetValue(int i) + + cdef cppclass CDecimal32Array" arrow::Decimal32Array"( + CFixedSizeBinaryArray + ): + c_string FormatValue(int i) + + cdef cppclass CDecimal64Array" arrow::Decimal64Array"( + CFixedSizeBinaryArray + ): + c_string FormatValue(int i) + + cdef cppclass CDecimal128Array" arrow::Decimal128Array"( + CFixedSizeBinaryArray + ): + c_string FormatValue(int i) + + cdef cppclass CDecimal256Array" arrow::Decimal256Array"( + CFixedSizeBinaryArray + ): + c_string FormatValue(int i) + + cdef cppclass CListArray" arrow::ListArray"(CArray): + @staticmethod + CResult[shared_ptr[CArray]] FromArrays( + const CArray& offsets, + const CArray& values, + CMemoryPool* pool, + shared_ptr[CBuffer] null_bitmap, + ) + + @staticmethod + CResult[shared_ptr[CArray]] FromArraysAndType" FromArrays"( + shared_ptr[CDataType], + const CArray& offsets, + const CArray& values, + CMemoryPool* pool, + shared_ptr[CBuffer] null_bitmap, + ) + + const int32_t* raw_value_offsets() + int32_t value_offset(int i) + int32_t value_length(int i) + shared_ptr[CArray] values() + shared_ptr[CArray] offsets() + shared_ptr[CDataType] value_type() + + cdef cppclass CLargeListArray" arrow::LargeListArray"(CArray): + @staticmethod + CResult[shared_ptr[CArray]] FromArrays( + const CArray& offsets, + const CArray& values, + CMemoryPool* pool, + shared_ptr[CBuffer] null_bitmap + ) + + @staticmethod + CResult[shared_ptr[CArray]] FromArraysAndType" FromArrays"( + shared_ptr[CDataType], + const CArray& offsets, + const CArray& values, + CMemoryPool* pool, + shared_ptr[CBuffer] null_bitmap + ) + + int64_t value_offset(int i) + int64_t value_length(int i) + shared_ptr[CArray] values() + shared_ptr[CArray] offsets() + shared_ptr[CDataType] value_type() + + cdef cppclass CFixedSizeListArray" arrow::FixedSizeListArray"(CArray): + @staticmethod + CResult[shared_ptr[CArray]] FromArrays( + const shared_ptr[CArray]& values, + int32_t list_size, + shared_ptr[CBuffer] null_bitmap) + + @staticmethod + CResult[shared_ptr[CArray]] FromArraysAndType" FromArrays"( + const shared_ptr[CArray]& values, + shared_ptr[CDataType], + shared_ptr[CBuffer] null_bitmap) + + int64_t value_offset(int i) + int64_t value_length(int i) + shared_ptr[CArray] values() + shared_ptr[CDataType] value_type() + + cdef cppclass CListViewArray" arrow::ListViewArray"(CArray): + @staticmethod + CResult[shared_ptr[CArray]] FromArrays( + const CArray& offsets, + const CArray& sizes, + const CArray& values, + CMemoryPool* pool, + shared_ptr[CBuffer] null_bitmap, + ) + + @staticmethod + CResult[shared_ptr[CArray]] FromArraysAndType" FromArrays"( + shared_ptr[CDataType], + const CArray& offsets, + const CArray& sizes, + const CArray& values, + CMemoryPool* pool, + shared_ptr[CBuffer] null_bitmap, + ) + + CResult[shared_ptr[CArray]] Flatten( + CMemoryPool* pool + ) + + const int32_t* raw_value_offsets() + const int32_t* raw_value_sizes() + int32_t value_offset(int i) + int32_t value_length(int i) + shared_ptr[CArray] values() + shared_ptr[CArray] offsets() + shared_ptr[CArray] sizes() + shared_ptr[CDataType] value_type() + + cdef cppclass CLargeListViewArray" arrow::LargeListViewArray"(CArray): + @staticmethod + CResult[shared_ptr[CArray]] FromArrays( + const CArray& offsets, + const CArray& sizes, + const CArray& values, + CMemoryPool* pool, + shared_ptr[CBuffer] null_bitmap, + ) + + @staticmethod + CResult[shared_ptr[CArray]] FromArraysAndType" FromArrays"( + shared_ptr[CDataType], + const CArray& offsets, + const CArray& sizes, + const CArray& values, + CMemoryPool* pool, + shared_ptr[CBuffer] null_bitmap, + ) + + CResult[shared_ptr[CArray]] Flatten( + CMemoryPool* pool + ) + + int64_t value_offset(int i) + int64_t value_length(int i) + shared_ptr[CArray] values() + shared_ptr[CArray] offsets() + shared_ptr[CArray] sizes() + shared_ptr[CDataType] value_type() + + cdef cppclass CMapArray" arrow::MapArray"(CArray): + @staticmethod + CResult[shared_ptr[CArray]] FromArrays( + const shared_ptr[CArray]& offsets, + const shared_ptr[CArray]& keys, + const shared_ptr[CArray]& items, + CMemoryPool* pool, + const shared_ptr[CBuffer] null_bitmap, + ) + + @staticmethod + CResult[shared_ptr[CArray]] FromArraysAndType" FromArrays"( + shared_ptr[CDataType], + const shared_ptr[CArray]& offsets, + const shared_ptr[CArray]& keys, + const shared_ptr[CArray]& items, + CMemoryPool* pool, + const shared_ptr[CBuffer] null_bitmap, + ) + + shared_ptr[CArray] keys() + shared_ptr[CArray] items() + CMapType* map_type() + int64_t value_offset(int i) + int64_t value_length(int i) + shared_ptr[CArray] values() + shared_ptr[CDataType] value_type() + + cdef cppclass CUnionArray" arrow::UnionArray"(CArray): + shared_ptr[CBuffer] type_codes() + int8_t* raw_type_codes() + int child_id(int64_t index) + shared_ptr[CArray] field(int pos) + const CArray* UnsafeField(int pos) + UnionMode mode() + + cdef cppclass CSparseUnionArray" arrow::SparseUnionArray"(CUnionArray): + @staticmethod + CResult[shared_ptr[CArray]] Make( + const CArray& type_codes, + const vector[shared_ptr[CArray]]& children, + const vector[c_string]& field_names, + const vector[int8_t]& type_codes) + + cdef cppclass CDenseUnionArray" arrow::DenseUnionArray"(CUnionArray): + @staticmethod + CResult[shared_ptr[CArray]] Make( + const CArray& type_codes, + const CArray& value_offsets, + const vector[shared_ptr[CArray]]& children, + const vector[c_string]& field_names, + const vector[int8_t]& type_codes) + + int32_t value_offset(int i) + shared_ptr[CBuffer] value_offsets() + + cdef cppclass CBinaryArray" arrow::BinaryArray"(CArray): + const uint8_t* GetValue(int i, int32_t* length) + shared_ptr[CBuffer] value_data() + int32_t value_offset(int64_t i) + int32_t value_length(int64_t i) + int32_t total_values_length() + + cdef cppclass CLargeBinaryArray" arrow::LargeBinaryArray"(CArray): + const uint8_t* GetValue(int i, int64_t* length) + shared_ptr[CBuffer] value_data() + int64_t value_offset(int64_t i) + int64_t value_length(int64_t i) + int64_t total_values_length() + + cdef cppclass CStringArray" arrow::StringArray"(CBinaryArray): + CStringArray(int64_t length, shared_ptr[CBuffer] value_offsets, + shared_ptr[CBuffer] data, + shared_ptr[CBuffer] null_bitmap, + int64_t null_count, + int64_t offset) + + c_string GetString(int i) + + cdef cppclass CLargeStringArray" arrow::LargeStringArray" \ + (CLargeBinaryArray): + CLargeStringArray(int64_t length, shared_ptr[CBuffer] value_offsets, + shared_ptr[CBuffer] data, + shared_ptr[CBuffer] null_bitmap, + int64_t null_count, + int64_t offset) + + c_string GetString(int i) + + cdef cppclass CStructArray" arrow::StructArray"(CArray): + CStructArray(shared_ptr[CDataType]& type, int64_t length, + vector[shared_ptr[CArray]]& children, + shared_ptr[CBuffer] null_bitmap=nullptr, + int64_t null_count=-1, + int64_t offset=0) + + # XXX Cython crashes if default argument values are declared here + # https://github.com/cython/cython/issues/2167 + @staticmethod + CResult[shared_ptr[CArray]] MakeFromFieldNames "Make"( + vector[shared_ptr[CArray]] children, + vector[c_string] field_names, + shared_ptr[CBuffer] null_bitmap, + int64_t null_count, + int64_t offset) + + @staticmethod + CResult[shared_ptr[CArray]] MakeFromFields "Make"( + vector[shared_ptr[CArray]] children, + vector[shared_ptr[CField]] fields, + shared_ptr[CBuffer] null_bitmap, + int64_t null_count, + int64_t offset) + + shared_ptr[CArray] field(int pos) + shared_ptr[CArray] GetFieldByName(const c_string& name) const + CResult[shared_ptr[CArray]] GetFlattenedField(int index, CMemoryPool* pool) const + + CResult[vector[shared_ptr[CArray]]] Flatten(CMemoryPool* pool) + + cdef cppclass CRunEndEncodedArray" arrow::RunEndEncodedArray"(CArray): + @staticmethod + CResult[shared_ptr[CRunEndEncodedArray]] Make( + const shared_ptr[CDataType]& type, + int64_t logical_length, + const shared_ptr[CArray]& run_ends, + const shared_ptr[CArray]& values, + int64_t logical_offset) + + @staticmethod + CResult[shared_ptr[CRunEndEncodedArray]] MakeFromArrays "Make"( + int64_t logical_length, + const shared_ptr[CArray]& run_ends, + const shared_ptr[CArray]& values, + int64_t logical_offset) + + shared_ptr[CArray]& run_ends() + shared_ptr[CArray]& values() + + int64_t FindPhysicalOffset() + int64_t FindPhysicalLength() + + cdef cppclass CChunkedArray" arrow::ChunkedArray": + CChunkedArray(const vector[shared_ptr[CArray]]& arrays) + CChunkedArray(const vector[shared_ptr[CArray]]& arrays, + const shared_ptr[CDataType]& type) + + @staticmethod + CResult[shared_ptr[CChunkedArray]] Make(vector[shared_ptr[CArray]] chunks, + shared_ptr[CDataType] type) + int64_t length() + int64_t null_count() + int num_chunks() + c_bool Equals(const CChunkedArray& other) + + shared_ptr[CArray] chunk(int i) + shared_ptr[CDataType] type() + CResult[shared_ptr[CScalar]] GetScalar(int64_t index) const + shared_ptr[CChunkedArray] Slice(int64_t offset, int64_t length) const + shared_ptr[CChunkedArray] Slice(int64_t offset) const + + CResult[vector[shared_ptr[CChunkedArray]]] Flatten(CMemoryPool* pool) + + c_bool is_cpu() const + + CStatus Validate() const + CStatus ValidateFull() const + + cdef cppclass CRecordBatch" arrow::RecordBatch": + @staticmethod + shared_ptr[CRecordBatch] Make( + const shared_ptr[CSchema]& schema, int64_t num_rows, + const vector[shared_ptr[CArray]]& columns) + + CResult[shared_ptr[CStructArray]] ToStructArray() const + + @staticmethod + CResult[shared_ptr[CRecordBatch]] FromStructArray( + const shared_ptr[CArray]& array) + + c_bool Equals(const CRecordBatch& other, c_bool check_metadata) + + shared_ptr[CSchema] schema() + shared_ptr[CArray] column(int i) + const c_string& column_name(int i) + + CResult[shared_ptr[CRecordBatch]] AddColumn( + int i, shared_ptr[CField] field, shared_ptr[CArray] column) + CResult[shared_ptr[CRecordBatch]] RemoveColumn(int i) + CResult[shared_ptr[CRecordBatch]] SetColumn( + int i, shared_ptr[CField] field, shared_ptr[CArray] column) + + const vector[shared_ptr[CArray]]& columns() + + CResult[shared_ptr[CRecordBatch]] RenameColumns(const vector[c_string]&) + CResult[shared_ptr[CRecordBatch]] SelectColumns(const vector[int]&) + + int num_columns() + int64_t num_rows() + + CDeviceAllocationType device_type() + + CStatus Validate() const + CStatus ValidateFull() const + + shared_ptr[CRecordBatch] ReplaceSchemaMetadata( + const shared_ptr[CKeyValueMetadata]& metadata) + + shared_ptr[CRecordBatch] Slice(int64_t offset) + shared_ptr[CRecordBatch] Slice(int64_t offset, int64_t length) + + CResult[shared_ptr[CRecordBatch]] CopyTo(const shared_ptr[CMemoryManager]& to) const + + CResult[shared_ptr[CTensor]] ToTensor(c_bool null_to_nan, c_bool row_major, + CMemoryPool* pool) const + + cdef cppclass CRecordBatchWithMetadata" arrow::RecordBatchWithMetadata": + shared_ptr[CRecordBatch] batch + # The struct in C++ does not actually have these two `const` qualifiers, but + # adding `const` gets Cython to not complain + shared_ptr[const CKeyValueMetadata] custom_metadata + + cdef cppclass CTable" arrow::Table": + CTable(const shared_ptr[CSchema]& schema, + const vector[shared_ptr[CChunkedArray]]& columns) + + @staticmethod + shared_ptr[CTable] Make( + const shared_ptr[CSchema]& schema, + const vector[shared_ptr[CChunkedArray]]& columns) + + @staticmethod + shared_ptr[CTable] MakeWithRows "Make"( + const shared_ptr[CSchema]& schema, + const vector[shared_ptr[CChunkedArray]]& columns, + int64_t num_rows) + + @staticmethod + shared_ptr[CTable] MakeFromArrays" Make"( + const shared_ptr[CSchema]& schema, + const vector[shared_ptr[CArray]]& arrays) + + @staticmethod + CResult[shared_ptr[CTable]] FromRecordBatchReader( + CRecordBatchReader *reader) + + @staticmethod + CResult[shared_ptr[CTable]] FromRecordBatches( + const shared_ptr[CSchema]& schema, + const vector[shared_ptr[CRecordBatch]]& batches) + + int num_columns() + int64_t num_rows() + + c_bool Equals(const CTable& other, c_bool check_metadata) + + shared_ptr[CSchema] schema() + shared_ptr[CChunkedArray] column(int i) + shared_ptr[CField] field(int i) + + CResult[shared_ptr[CTable]] AddColumn( + int i, shared_ptr[CField] field, shared_ptr[CChunkedArray] column) + CResult[shared_ptr[CTable]] RemoveColumn(int i) + CResult[shared_ptr[CTable]] SetColumn( + int i, shared_ptr[CField] field, shared_ptr[CChunkedArray] column) + + vector[c_string] ColumnNames() + CResult[shared_ptr[CTable]] RenameColumns(const vector[c_string]&) + CResult[shared_ptr[CTable]] SelectColumns(const vector[int]&) + + CResult[shared_ptr[CTable]] Flatten(CMemoryPool* pool) + + CResult[shared_ptr[CTable]] CombineChunks(CMemoryPool* pool) + + CStatus Validate() const + CStatus ValidateFull() const + + shared_ptr[CTable] ReplaceSchemaMetadata( + const shared_ptr[CKeyValueMetadata]& metadata) + + shared_ptr[CTable] Slice(int64_t offset) + shared_ptr[CTable] Slice(int64_t offset, int64_t length) + + cdef cppclass CRecordBatchReader" arrow::RecordBatchReader": + shared_ptr[CSchema] schema() + CStatus Close() + CResult[CRecordBatchWithMetadata] ReadNext() + CStatus ReadNext(shared_ptr[CRecordBatch]* batch) + CResult[shared_ptr[CTable]] ToTable() + + cdef cppclass TableBatchReader(CRecordBatchReader): + TableBatchReader(const CTable& table) + TableBatchReader(shared_ptr[CTable] table) + void set_chunksize(int64_t chunksize) + + cdef cppclass CTensor" arrow::Tensor": + shared_ptr[CDataType] type() + shared_ptr[CBuffer] data() + + const vector[int64_t]& shape() + const vector[int64_t]& strides() + int64_t size() + + int ndim() + const vector[c_string]& dim_names() + const c_string& dim_name(int i) + + c_bool is_mutable() + c_bool is_contiguous() + Type type_id() + c_bool Equals(const CTensor& other) + + cdef cppclass CSparseIndex" arrow::SparseIndex": + pass + + cdef cppclass CSparseCOOIndex" arrow::SparseCOOIndex": + c_bool is_canonical() + + cdef cppclass CSparseCOOTensor" arrow::SparseCOOTensor": + shared_ptr[CDataType] type() + shared_ptr[CBuffer] data() + CResult[shared_ptr[CTensor]] ToTensor() + + shared_ptr[CSparseIndex] sparse_index() + + const vector[int64_t]& shape() + int64_t size() + int64_t non_zero_length() + + int ndim() + const vector[c_string]& dim_names() + const c_string& dim_name(int i) + + c_bool is_mutable() + Type type_id() + c_bool Equals(const CSparseCOOTensor& other) + + cdef cppclass CSparseCSRMatrix" arrow::SparseCSRMatrix": + shared_ptr[CDataType] type() + shared_ptr[CBuffer] data() + CResult[shared_ptr[CTensor]] ToTensor() + + const vector[int64_t]& shape() + int64_t size() + int64_t non_zero_length() + + int ndim() + const vector[c_string]& dim_names() + const c_string& dim_name(int i) + + c_bool is_mutable() + Type type_id() + c_bool Equals(const CSparseCSRMatrix& other) + + cdef cppclass CSparseCSCMatrix" arrow::SparseCSCMatrix": + shared_ptr[CDataType] type() + shared_ptr[CBuffer] data() + CResult[shared_ptr[CTensor]] ToTensor() + + const vector[int64_t]& shape() + int64_t size() + int64_t non_zero_length() + + int ndim() + const vector[c_string]& dim_names() + const c_string& dim_name(int i) + + c_bool is_mutable() + Type type_id() + c_bool Equals(const CSparseCSCMatrix& other) + + cdef cppclass CSparseCSFTensor" arrow::SparseCSFTensor": + shared_ptr[CDataType] type() + shared_ptr[CBuffer] data() + CResult[shared_ptr[CTensor]] ToTensor() + + const vector[int64_t]& shape() + int64_t size() + int64_t non_zero_length() + + int ndim() + const vector[c_string]& dim_names() + const c_string& dim_name(int i) + + c_bool is_mutable() + Type type_id() + c_bool Equals(const CSparseCSFTensor& other) + + cdef cppclass CScalar" arrow::Scalar": + CScalar(shared_ptr[CDataType]) + + shared_ptr[CDataType] type + c_bool is_valid + + c_string ToString() const + c_bool Equals(const CScalar& other) const + CStatus Validate() const + CStatus ValidateFull() const + + cdef cppclass CScalarHash" arrow::Scalar::Hash": + size_t operator()(const shared_ptr[CScalar]& scalar) const + + cdef cppclass CNullScalar" arrow::NullScalar"(CScalar): + CNullScalar() + + cdef cppclass CBooleanScalar" arrow::BooleanScalar"(CScalar): + CBooleanScalar(c_bool value) + c_bool value + + cdef cppclass CInt8Scalar" arrow::Int8Scalar"(CScalar): + int8_t value + + cdef cppclass CUInt8Scalar" arrow::UInt8Scalar"(CScalar): + uint8_t value + + cdef cppclass CInt16Scalar" arrow::Int16Scalar"(CScalar): + int16_t value + + cdef cppclass CUInt16Scalar" arrow::UInt16Scalar"(CScalar): + uint16_t value + + cdef cppclass CInt32Scalar" arrow::Int32Scalar"(CScalar): + int32_t value + + cdef cppclass CUInt32Scalar" arrow::UInt32Scalar"(CScalar): + uint32_t value + + cdef cppclass CInt64Scalar" arrow::Int64Scalar"(CScalar): + int64_t value + + cdef cppclass CUInt64Scalar" arrow::UInt64Scalar"(CScalar): + uint64_t value + + cdef cppclass CHalfFloatScalar" arrow::HalfFloatScalar"(CScalar): + uint16_t value + + cdef cppclass CFloatScalar" arrow::FloatScalar"(CScalar): + float value + + cdef cppclass CDoubleScalar" arrow::DoubleScalar"(CScalar): + double value + + cdef cppclass CDecimal32Scalar" arrow::Decimal32Scalar"(CScalar): + CDecimal32 value + + cdef cppclass CDecimal64Scalar" arrow::Decimal64Scalar"(CScalar): + CDecimal64 value + + cdef cppclass CDecimal128Scalar" arrow::Decimal128Scalar"(CScalar): + CDecimal128 value + + cdef cppclass CDecimal256Scalar" arrow::Decimal256Scalar"(CScalar): + CDecimal256 value + + cdef cppclass CDate32Scalar" arrow::Date32Scalar"(CScalar): + int32_t value + + cdef cppclass CDate64Scalar" arrow::Date64Scalar"(CScalar): + int64_t value + + cdef cppclass CTime32Scalar" arrow::Time32Scalar"(CScalar): + int32_t value + + cdef cppclass CTime64Scalar" arrow::Time64Scalar"(CScalar): + int64_t value + + cdef cppclass CTimestampScalar" arrow::TimestampScalar"(CScalar): + int64_t value + + cdef cppclass CDurationScalar" arrow::DurationScalar"(CScalar): + int64_t value + + cdef cppclass CMonthDayNanoIntervalScalar \ + "arrow::MonthDayNanoIntervalScalar"(CScalar): + pass + + cdef cppclass CBaseBinaryScalar" arrow::BaseBinaryScalar"(CScalar): + shared_ptr[CBuffer] value + + cdef cppclass CBaseListScalar" arrow::BaseListScalar"(CScalar): + shared_ptr[CArray] value + + cdef cppclass CListScalar" arrow::ListScalar"(CBaseListScalar): + pass + + cdef cppclass CListViewScalar" arrow::ListViewScalar"(CBaseListScalar): + pass + + cdef cppclass CLargeListViewScalar" arrow::LargeListViewScalar"(CBaseListScalar): + pass + + cdef cppclass CMapScalar" arrow::MapScalar"(CListScalar): + pass + + cdef cppclass CStructScalar" arrow::StructScalar"(CScalar): + vector[shared_ptr[CScalar]] value + CResult[shared_ptr[CScalar]] field(CFieldRef ref) const + + cdef cppclass CDictionaryScalarIndexAndDictionary \ + "arrow::DictionaryScalar::ValueType": + shared_ptr[CScalar] index + shared_ptr[CArray] dictionary + + cdef cppclass CDictionaryScalar" arrow::DictionaryScalar"(CScalar): + CDictionaryScalar(CDictionaryScalarIndexAndDictionary value, + shared_ptr[CDataType], c_bool is_valid) + CDictionaryScalarIndexAndDictionary value + + CResult[shared_ptr[CScalar]] GetEncodedValue() + + cdef cppclass CUnionScalar" arrow::UnionScalar"(CScalar): + int8_t type_code + + cdef cppclass CDenseUnionScalar" arrow::DenseUnionScalar"(CUnionScalar): + shared_ptr[CScalar] value + + cdef cppclass CSparseUnionScalar" arrow::SparseUnionScalar"(CUnionScalar): + vector[shared_ptr[CScalar]] value + int child_id + + cdef cppclass CRunEndEncodedScalar" arrow::RunEndEncodedScalar"(CScalar): + shared_ptr[CScalar] value + + cdef cppclass CExtensionScalar" arrow::ExtensionScalar"(CScalar): + CExtensionScalar(shared_ptr[CScalar] storage, + shared_ptr[CDataType], c_bool is_valid) + shared_ptr[CScalar] value + + shared_ptr[CScalar] MakeScalar[Value](Value value) + + cdef cppclass CConcatenateTablesOptions" arrow::ConcatenateTablesOptions": + c_bool unify_schemas + CField.CMergeOptions field_merge_options + + @staticmethod + CConcatenateTablesOptions Defaults() + + CResult[shared_ptr[CTable]] ConcatenateTables( + const vector[shared_ptr[CTable]]& tables, + CConcatenateTablesOptions options, + CMemoryPool* memory_pool) + + CResult[shared_ptr[CRecordBatch]] ConcatenateRecordBatches( + const vector[shared_ptr[CRecordBatch]]& batches, + CMemoryPool* memory_pool) + + cdef cppclass CDictionaryUnifier" arrow::DictionaryUnifier": + @staticmethod + CResult[shared_ptr[CChunkedArray]] UnifyChunkedArray( + shared_ptr[CChunkedArray] array, CMemoryPool* pool) + + @staticmethod + CResult[shared_ptr[CTable]] UnifyTable( + const CTable& table, CMemoryPool* pool) + + shared_ptr[CScalar] MakeNullScalar(shared_ptr[CDataType] type) + + +cdef extern from "arrow/c/dlpack_abi.h" nogil: + ctypedef enum DLDeviceType: + kDLCPU = 1 + + ctypedef struct DLDevice: + DLDeviceType device_type + int32_t device_id + + ctypedef struct DLManagedTensor: + void (*deleter)(DLManagedTensor*) + + +cdef extern from "arrow/c/dlpack.h" namespace "arrow::dlpack" nogil: + CResult[DLManagedTensor*] ExportArrayToDLPack" arrow::dlpack::ExportArray"( + const shared_ptr[CArray]& arr) + CResult[DLManagedTensor*] ExportTensorToDLPack" arrow::dlpack::ExportTensor"( + const shared_ptr[CTensor]& tensor) + + CResult[DLDevice] ExportDevice(const shared_ptr[CArray]& arr) + CResult[DLDevice] ExportDevice(const shared_ptr[CTensor]& tensor) + + +cdef extern from "arrow/builder.h" namespace "arrow" nogil: + + cdef cppclass CArrayBuilder" arrow::ArrayBuilder": + CArrayBuilder(shared_ptr[CDataType], CMemoryPool* pool) + + int64_t length() + int64_t null_count() + CStatus AppendNull() + CStatus Finish(shared_ptr[CArray]* out) + CStatus Reserve(int64_t additional_capacity) + + cdef cppclass CBooleanBuilder" arrow::BooleanBuilder"(CArrayBuilder): + CBooleanBuilder(CMemoryPool* pool) + CStatus Append(const c_bool val) + CStatus Append(const uint8_t val) + + cdef cppclass CInt8Builder" arrow::Int8Builder"(CArrayBuilder): + CInt8Builder(CMemoryPool* pool) + CStatus Append(const int8_t value) + + cdef cppclass CInt16Builder" arrow::Int16Builder"(CArrayBuilder): + CInt16Builder(CMemoryPool* pool) + CStatus Append(const int16_t value) + + cdef cppclass CInt32Builder" arrow::Int32Builder"(CArrayBuilder): + CInt32Builder(CMemoryPool* pool) + CStatus Append(const int32_t value) + + cdef cppclass CInt64Builder" arrow::Int64Builder"(CArrayBuilder): + CInt64Builder(CMemoryPool* pool) + CStatus Append(const int64_t value) + + cdef cppclass CUInt8Builder" arrow::UInt8Builder"(CArrayBuilder): + CUInt8Builder(CMemoryPool* pool) + CStatus Append(const uint8_t value) + + cdef cppclass CUInt16Builder" arrow::UInt16Builder"(CArrayBuilder): + CUInt16Builder(CMemoryPool* pool) + CStatus Append(const uint16_t value) + + cdef cppclass CUInt32Builder" arrow::UInt32Builder"(CArrayBuilder): + CUInt32Builder(CMemoryPool* pool) + CStatus Append(const uint32_t value) + + cdef cppclass CUInt64Builder" arrow::UInt64Builder"(CArrayBuilder): + CUInt64Builder(CMemoryPool* pool) + CStatus Append(const uint64_t value) + + cdef cppclass CHalfFloatBuilder" arrow::HalfFloatBuilder"(CArrayBuilder): + CHalfFloatBuilder(CMemoryPool* pool) + + cdef cppclass CFloatBuilder" arrow::FloatBuilder"(CArrayBuilder): + CFloatBuilder(CMemoryPool* pool) + CStatus Append(const float value) + + cdef cppclass CDoubleBuilder" arrow::DoubleBuilder"(CArrayBuilder): + CDoubleBuilder(CMemoryPool* pool) + CStatus Append(const double value) + + cdef cppclass CBinaryBuilder" arrow::BinaryBuilder"(CArrayBuilder): + CArrayBuilder(shared_ptr[CDataType], CMemoryPool* pool) + CStatus Append(const char* value, int32_t length) + + cdef cppclass CStringBuilder" arrow::StringBuilder"(CBinaryBuilder): + CStringBuilder(CMemoryPool* pool) + CStatus Append(const c_string& value) + + cdef cppclass CBinaryViewBuilder" arrow::BinaryViewBuilder"(CArrayBuilder): + CBinaryViewBuilder(shared_ptr[CDataType], CMemoryPool* pool) + CStatus Append(const char* value, int32_t length) + + cdef cppclass CStringViewBuilder" arrow::StringViewBuilder"(CBinaryViewBuilder): + CStringViewBuilder(CMemoryPool* pool) + CStatus Append(const c_string& value) + + cdef cppclass CTimestampBuilder "arrow::TimestampBuilder"(CArrayBuilder): + CTimestampBuilder(const shared_ptr[CDataType] typ, CMemoryPool* pool) + CStatus Append(const int64_t value) + + cdef cppclass CDate32Builder "arrow::Date32Builder"(CArrayBuilder): + CDate32Builder(CMemoryPool* pool) + CStatus Append(const int32_t value) + + cdef cppclass CDate64Builder "arrow::Date64Builder"(CArrayBuilder): + CDate64Builder(CMemoryPool* pool) + CStatus Append(const int64_t value) + + +# Use typedef to emulate syntax for std::function +ctypedef void CallbackTransform(object, const shared_ptr[CBuffer]& src, + shared_ptr[CBuffer]* dest) + +ctypedef CResult[shared_ptr[CInputStream]] StreamWrapFunc( + shared_ptr[CInputStream]) + + +cdef extern from "arrow/util/cancel.h" namespace "arrow" nogil: + cdef cppclass CStopToken "arrow::StopToken": + CStatus Poll() + c_bool IsStopRequested() + + cdef cppclass CStopSource "arrow::StopSource": + CStopToken token() + + CResult[CStopSource*] SetSignalStopSource() + void ResetSignalStopSource() + + CStatus RegisterCancellingSignalHandler(vector[int] signals) + void UnregisterCancellingSignalHandler() + + +cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: + cdef enum FileMode" arrow::io::FileMode::type": + FileMode_READ" arrow::io::FileMode::READ" + FileMode_WRITE" arrow::io::FileMode::WRITE" + FileMode_READWRITE" arrow::io::FileMode::READWRITE" + + cdef enum ObjectType" arrow::io::ObjectType::type": + ObjectType_FILE" arrow::io::ObjectType::FILE" + ObjectType_DIRECTORY" arrow::io::ObjectType::DIRECTORY" + + cdef cppclass CIOContext" arrow::io::IOContext": + CIOContext() + CIOContext(CStopToken) + CIOContext(CMemoryPool*) + CIOContext(CMemoryPool*, CStopToken) + + CIOContext c_default_io_context "arrow::io::default_io_context"() + int GetIOThreadPoolCapacity() + CStatus SetIOThreadPoolCapacity(int threads) + + cdef cppclass FileStatistics: + int64_t size + ObjectType kind + + cdef cppclass FileInterface: + CStatus Close() + CResult[int64_t] Tell() + FileMode mode() + c_bool closed() + + cdef cppclass Readable: + # put overload under a different name to avoid cython bug with multiple + # layers of inheritance + CResult[shared_ptr[CBuffer]] ReadBuffer" Read"(int64_t nbytes) + CResult[int64_t] Read(int64_t nbytes, uint8_t* out) + + cdef cppclass Seekable: + CStatus Seek(int64_t position) + + cdef cppclass Writable: + CStatus WriteBuffer" Write"(shared_ptr[CBuffer] data) + CStatus Write(const uint8_t* data, int64_t nbytes) + CStatus Flush() + + cdef cppclass CCacheOptions "arrow::io::CacheOptions": + int64_t hole_size_limit + int64_t range_size_limit + c_bool lazy + int64_t prefetch_limit + c_bool Equals "operator==" (CCacheOptions other) + + @staticmethod + CCacheOptions MakeFromNetworkMetrics(int64_t time_to_first_byte_millis, + int64_t transfer_bandwidth_mib_per_sec, + double ideal_bandwidth_utilization_frac, + int64_t max_ideal_request_size_mib) + + @staticmethod + CCacheOptions LazyDefaults() + + cdef cppclass COutputStream" arrow::io::OutputStream"(FileInterface, + Writable): + pass + + cdef cppclass CInputStream" arrow::io::InputStream"(FileInterface, + Readable): + CResult[shared_ptr[const CKeyValueMetadata]] ReadMetadata() + + cdef cppclass CRandomAccessFile" arrow::io::RandomAccessFile"(CInputStream, + Seekable): + CResult[int64_t] GetSize() + + @staticmethod + CResult[shared_ptr[CInputStream]] GetStream( + shared_ptr[CRandomAccessFile] file, + int64_t file_offset, + int64_t nbytes) + + CResult[int64_t] ReadAt(int64_t position, int64_t nbytes, + uint8_t* buffer) + CResult[shared_ptr[CBuffer]] ReadAt(int64_t position, int64_t nbytes) + c_bool supports_zero_copy() + + cdef cppclass WritableFile(COutputStream, Seekable): + CStatus WriteAt(int64_t position, const uint8_t* data, + int64_t nbytes) + + cdef cppclass ReadWriteFileInterface(CRandomAccessFile, + WritableFile): + pass + + cdef cppclass CIOFileSystem" arrow::io::FileSystem": + CStatus Stat(const c_string& path, FileStatistics* stat) + + cdef cppclass FileOutputStream(COutputStream): + @staticmethod + CResult[shared_ptr[COutputStream]] Open(const c_string& path) + + @staticmethod + CResult[shared_ptr[COutputStream]] OpenWithAppend" Open"( + const c_string& path, c_bool append) + + int file_descriptor() + + cdef cppclass ReadableFile(CRandomAccessFile): + @staticmethod + CResult[shared_ptr[ReadableFile]] Open(const c_string& path) + + @staticmethod + CResult[shared_ptr[ReadableFile]] Open(const c_string& path, + CMemoryPool* memory_pool) + + int file_descriptor() + + cdef cppclass CMemoryMappedFile \ + " arrow::io::MemoryMappedFile"(ReadWriteFileInterface): + + @staticmethod + CResult[shared_ptr[CMemoryMappedFile]] Create(const c_string& path, + int64_t size) + + @staticmethod + CResult[shared_ptr[CMemoryMappedFile]] Open(const c_string& path, + FileMode mode) + + CStatus Resize(int64_t size) + + int file_descriptor() + + cdef cppclass CCompressedInputStream \ + " arrow::io::CompressedInputStream"(CInputStream): + @staticmethod + CResult[shared_ptr[CCompressedInputStream]] Make( + CCodec* codec, shared_ptr[CInputStream] raw) + + cdef cppclass CCompressedOutputStream \ + " arrow::io::CompressedOutputStream"(COutputStream): + @staticmethod + CResult[shared_ptr[CCompressedOutputStream]] Make( + CCodec* codec, shared_ptr[COutputStream] raw) + + cdef cppclass CBufferedInputStream \ + " arrow::io::BufferedInputStream"(CInputStream): + + @staticmethod + CResult[shared_ptr[CBufferedInputStream]] Create( + int64_t buffer_size, CMemoryPool* pool, + shared_ptr[CInputStream] raw) + + CResult[shared_ptr[CInputStream]] Detach() + + cdef cppclass CBufferedOutputStream \ + " arrow::io::BufferedOutputStream"(COutputStream): + + @staticmethod + CResult[shared_ptr[CBufferedOutputStream]] Create( + int64_t buffer_size, CMemoryPool* pool, + shared_ptr[COutputStream] raw) + + CResult[shared_ptr[COutputStream]] Detach() + + cdef cppclass CTransformInputStreamVTable \ + "arrow::py::TransformInputStreamVTable": + CTransformInputStreamVTable() + function[CallbackTransform] transform + + shared_ptr[CInputStream] MakeTransformInputStream \ + "arrow::py::MakeTransformInputStream"( + shared_ptr[CInputStream] wrapped, CTransformInputStreamVTable vtable, + object method_arg) + + shared_ptr[function[StreamWrapFunc]] MakeStreamTransformFunc \ + "arrow::py::MakeStreamTransformFunc"( + CTransformInputStreamVTable vtable, + object method_arg) + + # ---------------------------------------------------------------------- + # HDFS + + CStatus HaveLibHdfs() + CStatus HaveLibHdfs3() + + cdef enum HdfsDriver" arrow::io::HdfsDriver": + HdfsDriver_LIBHDFS" arrow::io::HdfsDriver::LIBHDFS" + HdfsDriver_LIBHDFS3" arrow::io::HdfsDriver::LIBHDFS3" + + cdef cppclass HdfsConnectionConfig: + c_string host + int port + c_string user + c_string kerb_ticket + unordered_map[c_string, c_string] extra_conf + HdfsDriver driver + + cdef cppclass HdfsPathInfo: + ObjectType kind + c_string name + c_string owner + c_string group + int32_t last_modified_time + int32_t last_access_time + int64_t size + int16_t replication + int64_t block_size + int16_t permissions + + cdef cppclass HdfsReadableFile(CRandomAccessFile): + pass + + cdef cppclass HdfsOutputStream(COutputStream): + pass + + cdef cppclass CIOHadoopFileSystem \ + "arrow::io::HadoopFileSystem"(CIOFileSystem): + @staticmethod + CStatus Connect(const HdfsConnectionConfig* config, + shared_ptr[CIOHadoopFileSystem]* client) + + CStatus MakeDirectory(const c_string& path) + + CStatus Delete(const c_string& path, c_bool recursive) + + CStatus Disconnect() + + c_bool Exists(const c_string& path) + + CStatus Chmod(const c_string& path, int mode) + CStatus Chown(const c_string& path, const char* owner, + const char* group) + + CStatus GetCapacity(int64_t* nbytes) + CStatus GetUsed(int64_t* nbytes) + + CStatus ListDirectory(const c_string& path, + vector[HdfsPathInfo]* listing) + + CStatus GetPathInfo(const c_string& path, HdfsPathInfo* info) + + CStatus Rename(const c_string& src, const c_string& dst) + + CStatus OpenReadable(const c_string& path, + shared_ptr[HdfsReadableFile]* handle) + + CStatus OpenWritable(const c_string& path, c_bool append, + int32_t buffer_size, int16_t replication, + int64_t default_block_size, + shared_ptr[HdfsOutputStream]* handle) + + cdef cppclass CBufferReader \ + " arrow::io::BufferReader"(CRandomAccessFile): + CBufferReader(const shared_ptr[CBuffer]& buffer) + CBufferReader(const uint8_t* data, int64_t nbytes) + + cdef cppclass CBufferOutputStream \ + " arrow::io::BufferOutputStream"(COutputStream): + CBufferOutputStream(const shared_ptr[CResizableBuffer]& buffer) + + cdef cppclass CMockOutputStream \ + " arrow::io::MockOutputStream"(COutputStream): + CMockOutputStream() + int64_t GetExtentBytesWritten() + + cdef cppclass CFixedSizeBufferWriter \ + " arrow::io::FixedSizeBufferWriter"(WritableFile): + CFixedSizeBufferWriter(const shared_ptr[CBuffer]& buffer) + + void set_memcopy_threads(int num_threads) + void set_memcopy_blocksize(int64_t blocksize) + void set_memcopy_threshold(int64_t threshold) + + +cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: + cdef enum MessageType" arrow::ipc::MessageType": + MessageType_SCHEMA" arrow::ipc::MessageType::SCHEMA" + MessageType_RECORD_BATCH" arrow::ipc::MessageType::RECORD_BATCH" + MessageType_DICTIONARY_BATCH \ + " arrow::ipc::MessageType::DICTIONARY_BATCH" + + # TODO: use "cpdef enum class" to automatically get a Python wrapper? + # See + # https://github.com/cython/cython/commit/2c7c22f51405299a4e247f78edf52957d30cf71d#diff-61c1365c0f761a8137754bb3a73bfbf7 + ctypedef enum CMetadataVersion" arrow::ipc::MetadataVersion": + CMetadataVersion_V1" arrow::ipc::MetadataVersion::V1" + CMetadataVersion_V2" arrow::ipc::MetadataVersion::V2" + CMetadataVersion_V3" arrow::ipc::MetadataVersion::V3" + CMetadataVersion_V4" arrow::ipc::MetadataVersion::V4" + CMetadataVersion_V5" arrow::ipc::MetadataVersion::V5" + + cdef cppclass CIpcWriteOptions" arrow::ipc::IpcWriteOptions": + c_bool allow_64bit + int max_recursion_depth + int32_t alignment + c_bool write_legacy_ipc_format + CMemoryPool* memory_pool + CMetadataVersion metadata_version + shared_ptr[CCodec] codec + c_bool use_threads + c_bool emit_dictionary_deltas + c_bool unify_dictionaries + + CIpcWriteOptions() + CIpcWriteOptions(CIpcWriteOptions) + + @staticmethod + CIpcWriteOptions Defaults() + + ctypedef enum CAlignment" arrow::ipc::Alignment": + CAlignment_Any" arrow::ipc::Alignment::kAnyAlignment" + CAlignment_DataTypeSpecific" arrow::ipc::Alignment::kDataTypeSpecificAlignment" + CAlignment_64Byte" arrow::ipc::Alignment::k64ByteAlignment" + + cdef cppclass CIpcReadOptions" arrow::ipc::IpcReadOptions": + int max_recursion_depth + CMemoryPool* memory_pool + vector[int] included_fields + c_bool use_threads + c_bool ensure_native_endian + CAlignment ensure_alignment + + @staticmethod + CIpcReadOptions Defaults() + + cdef cppclass CIpcWriteStats" arrow::ipc::WriteStats": + int64_t num_messages + int64_t num_record_batches + int64_t num_dictionary_batches + int64_t num_dictionary_deltas + int64_t num_replaced_dictionaries + + cdef cppclass CIpcReadStats" arrow::ipc::ReadStats": + int64_t num_messages + int64_t num_record_batches + int64_t num_dictionary_batches + int64_t num_dictionary_deltas + int64_t num_replaced_dictionaries + + cdef cppclass CDictionaryMemo" arrow::ipc::DictionaryMemo": + pass + + cdef cppclass CIpcPayload" arrow::ipc::IpcPayload": + MessageType type + shared_ptr[CBuffer] metadata + vector[shared_ptr[CBuffer]] body_buffers + int64_t body_length + + cdef cppclass CMessage" arrow::ipc::Message": + CResult[unique_ptr[CMessage]] Open(shared_ptr[CBuffer] metadata, + shared_ptr[CBuffer] body) + + shared_ptr[CBuffer] body() + + c_bool Equals(const CMessage& other) + + shared_ptr[CBuffer] metadata() + CMetadataVersion metadata_version() + MessageType type() + + CStatus SerializeTo(COutputStream* stream, + const CIpcWriteOptions& options, + int64_t* output_length) + + c_string FormatMessageType(MessageType type) + + cdef cppclass CMessageReader" arrow::ipc::MessageReader": + @staticmethod + unique_ptr[CMessageReader] Open(const shared_ptr[CInputStream]& stream) + + CResult[unique_ptr[CMessage]] ReadNextMessage() + + cdef cppclass CRecordBatchWriter" arrow::ipc::RecordBatchWriter": + CStatus Close() + CStatus WriteRecordBatch(const CRecordBatch& batch) + CStatus WriteRecordBatch( + const CRecordBatch& batch, + const shared_ptr[const CKeyValueMetadata]& metadata) + CStatus WriteTable(const CTable& table, int64_t max_chunksize) + + CIpcWriteStats stats() + + cdef cppclass CRecordBatchStreamReader \ + " arrow::ipc::RecordBatchStreamReader"(CRecordBatchReader): + @staticmethod + CResult[shared_ptr[CRecordBatchReader]] Open( + const shared_ptr[CInputStream], const CIpcReadOptions&) + + @staticmethod + CResult[shared_ptr[CRecordBatchReader]] Open2" Open"( + unique_ptr[CMessageReader] message_reader, + const CIpcReadOptions& options) + + CIpcReadStats stats() + + cdef cppclass CRecordBatchFileReader \ + " arrow::ipc::RecordBatchFileReader": + @staticmethod + CResult[shared_ptr[CRecordBatchFileReader]] Open( + CRandomAccessFile* file, + const CIpcReadOptions& options) + + @staticmethod + CResult[shared_ptr[CRecordBatchFileReader]] Open2" Open"( + CRandomAccessFile* file, int64_t footer_offset, + const CIpcReadOptions& options) + + shared_ptr[CSchema] schema() + + int num_record_batches() + + CResult[shared_ptr[CRecordBatch]] ReadRecordBatch(int i) + + CResult[CRecordBatchWithMetadata] ReadRecordBatchWithCustomMetadata(int i) + + CIpcReadStats stats() + + shared_ptr[const CKeyValueMetadata] metadata() + + CResult[shared_ptr[CRecordBatchWriter]] MakeStreamWriter( + shared_ptr[COutputStream] sink, const shared_ptr[CSchema]& schema, + CIpcWriteOptions& options) + + CResult[shared_ptr[CRecordBatchWriter]] MakeFileWriter( + shared_ptr[COutputStream] sink, const shared_ptr[CSchema]& schema, + CIpcWriteOptions& options, shared_ptr[const CKeyValueMetadata] metadata) + + CResult[unique_ptr[CMessage]] ReadMessage(CInputStream* stream, + CMemoryPool* pool) + + CStatus GetRecordBatchSize(const CRecordBatch& batch, int64_t* size) + CStatus GetTensorSize(const CTensor& tensor, int64_t* size) + + CStatus WriteTensor(const CTensor& tensor, COutputStream* dst, + int32_t* metadata_length, + int64_t* body_length) + + CResult[shared_ptr[CTensor]] ReadTensor(CInputStream* stream) + + CResult[shared_ptr[CRecordBatch]] ReadRecordBatch( + const CMessage& message, const shared_ptr[CSchema]& schema, + CDictionaryMemo* dictionary_memo, + const CIpcReadOptions& options) + + CResult[shared_ptr[CBuffer]] SerializeSchema( + const CSchema& schema, CMemoryPool* pool) + + CResult[shared_ptr[CBuffer]] SerializeRecordBatch( + const CRecordBatch& schema, const CIpcWriteOptions& options) + + CResult[shared_ptr[CSchema]] ReadSchema(const CMessage& message, + CDictionaryMemo* dictionary_memo) + + CResult[shared_ptr[CSchema]] ReadSchema(CInputStream* stream, + CDictionaryMemo* dictionary_memo) + + CResult[shared_ptr[CRecordBatch]] ReadRecordBatch( + const shared_ptr[CSchema]& schema, + CDictionaryMemo* dictionary_memo, + const CIpcReadOptions& options, + CInputStream* stream) + + CStatus AlignStream(CInputStream* stream, int64_t alignment) + CStatus AlignStream(COutputStream* stream, int64_t alignment) + + cdef CStatus GetRecordBatchPayload \ + " arrow::ipc::GetRecordBatchPayload"( + const CRecordBatch& batch, + const CIpcWriteOptions& options, + CIpcPayload* out) + + +cdef extern from "arrow/util/value_parsing.h" namespace "arrow" nogil: + cdef cppclass CTimestampParser" arrow::TimestampParser": + const char* kind() const + const char* format() const + + @staticmethod + shared_ptr[CTimestampParser] MakeStrptime(c_string format) + + @staticmethod + shared_ptr[CTimestampParser] MakeISO8601() + + +cdef extern from "arrow/csv/api.h" namespace "arrow::csv" nogil: + + cdef cppclass CCSVInvalidRow" arrow::csv::InvalidRow": + int32_t expected_columns + int32_t actual_columns + int64_t number + c_string text + + ctypedef enum CInvalidRowResult" arrow::csv::InvalidRowResult": + CInvalidRowResult_Error" arrow::csv::InvalidRowResult::Error" + CInvalidRowResult_Skip" arrow::csv::InvalidRowResult::Skip" + + ctypedef CInvalidRowResult CInvalidRowHandler(const CCSVInvalidRow&) + + +cdef extern from "arrow/csv/api.h" namespace "arrow::csv" nogil: + + ctypedef enum CQuotingStyle "arrow::csv::QuotingStyle": + CQuotingStyle_Needed "arrow::csv::QuotingStyle::Needed" + CQuotingStyle_AllValid "arrow::csv::QuotingStyle::AllValid" + CQuotingStyle_None "arrow::csv::QuotingStyle::None" + + cdef cppclass CCSVParseOptions" arrow::csv::ParseOptions": + unsigned char delimiter + c_bool quoting + unsigned char quote_char + c_bool double_quote + c_bool escaping + unsigned char escape_char + c_bool newlines_in_values + c_bool ignore_empty_lines + function[CInvalidRowHandler] invalid_row_handler + + CCSVParseOptions() + CCSVParseOptions(CCSVParseOptions) + + @staticmethod + CCSVParseOptions Defaults() + + CStatus Validate() + + cdef cppclass CCSVConvertOptions" arrow::csv::ConvertOptions": + c_bool check_utf8 + unordered_map[c_string, shared_ptr[CDataType]] column_types + vector[c_string] null_values + vector[c_string] true_values + vector[c_string] false_values + c_bool strings_can_be_null + c_bool quoted_strings_can_be_null + vector[shared_ptr[CTimestampParser]] timestamp_parsers + + c_bool auto_dict_encode + int32_t auto_dict_max_cardinality + unsigned char decimal_point + + vector[c_string] include_columns + c_bool include_missing_columns + + CCSVConvertOptions() + CCSVConvertOptions(CCSVConvertOptions) + + @staticmethod + CCSVConvertOptions Defaults() + + CStatus Validate() + + cdef cppclass CCSVReadOptions" arrow::csv::ReadOptions": + c_bool use_threads + int32_t block_size + int32_t skip_rows + int32_t skip_rows_after_names + vector[c_string] column_names + c_bool autogenerate_column_names + + CCSVReadOptions() + CCSVReadOptions(CCSVReadOptions) + + @staticmethod + CCSVReadOptions Defaults() + + CStatus Validate() + + cdef cppclass CCSVWriteOptions" arrow::csv::WriteOptions": + c_bool include_header + int32_t batch_size + unsigned char delimiter + CQuotingStyle quoting_style + CQuotingStyle quoting_header + CIOContext io_context + + CCSVWriteOptions() + CCSVWriteOptions(CCSVWriteOptions) + + @staticmethod + CCSVWriteOptions Defaults() + + CStatus Validate() + + cdef cppclass CCSVReader" arrow::csv::TableReader": + @staticmethod + CResult[shared_ptr[CCSVReader]] Make( + CIOContext, shared_ptr[CInputStream], + CCSVReadOptions, CCSVParseOptions, CCSVConvertOptions) + + CResult[shared_ptr[CTable]] Read() + + cdef cppclass CCSVStreamingReader" arrow::csv::StreamingReader"( + CRecordBatchReader): + @staticmethod + CResult[shared_ptr[CCSVStreamingReader]] Make( + CIOContext, shared_ptr[CInputStream], + CCSVReadOptions, CCSVParseOptions, CCSVConvertOptions) + + cdef CStatus WriteCSV(CTable&, CCSVWriteOptions& options, COutputStream*) + cdef CStatus WriteCSV( + CRecordBatch&, CCSVWriteOptions& options, COutputStream*) + cdef CResult[shared_ptr[CRecordBatchWriter]] MakeCSVWriter( + shared_ptr[COutputStream], shared_ptr[CSchema], + CCSVWriteOptions& options) + + +cdef extern from "arrow/json/options.h" nogil: + + ctypedef enum CUnexpectedFieldBehavior \ + "arrow::json::UnexpectedFieldBehavior": + CUnexpectedFieldBehavior_Ignore \ + "arrow::json::UnexpectedFieldBehavior::Ignore" + CUnexpectedFieldBehavior_Error \ + "arrow::json::UnexpectedFieldBehavior::Error" + CUnexpectedFieldBehavior_InferType \ + "arrow::json::UnexpectedFieldBehavior::InferType" + + cdef cppclass CJSONReadOptions" arrow::json::ReadOptions": + c_bool use_threads + int32_t block_size + + @staticmethod + CJSONReadOptions Defaults() + + cdef cppclass CJSONParseOptions" arrow::json::ParseOptions": + shared_ptr[CSchema] explicit_schema + c_bool newlines_in_values + CUnexpectedFieldBehavior unexpected_field_behavior + + @staticmethod + CJSONParseOptions Defaults() + + +cdef extern from "arrow/json/reader.h" namespace "arrow::json" nogil: + + cdef cppclass CJSONReader" arrow::json::TableReader": + @staticmethod + CResult[shared_ptr[CJSONReader]] Make( + CMemoryPool*, shared_ptr[CInputStream], + CJSONReadOptions, CJSONParseOptions) + + CResult[shared_ptr[CTable]] Read() + + cdef cppclass CJSONStreamingReader" arrow::json::StreamingReader"( + CRecordBatchReader): + @staticmethod + CResult[shared_ptr[CJSONStreamingReader]] Make( + shared_ptr[CInputStream], + CJSONReadOptions, CJSONParseOptions, CIOContext) + + +cdef extern from "arrow/util/thread_pool.h" namespace "arrow::internal" nogil: + + cdef cppclass CExecutor "arrow::internal::Executor": + pass + + cdef cppclass CThreadPool "arrow::internal::ThreadPool"(CExecutor): + @staticmethod + CResult[shared_ptr[CThreadPool]] Make(int threads) + + CThreadPool* GetCpuThreadPool() + + +cdef extern from "arrow/compute/api.h" namespace "arrow::compute" nogil: + + CStatus InitializeCompute " arrow::compute::Initialize"() + + cdef cppclass CExecBatch "arrow::compute::ExecBatch": + vector[CDatum] values + int64_t length + + cdef cppclass CExecContext" arrow::compute::ExecContext": + CExecContext() + CExecContext(CMemoryPool* pool) + CExecContext(CMemoryPool* pool, CExecutor* exc) + + CMemoryPool* memory_pool() const + CExecutor* executor() + + cdef cppclass CKernelSignature" arrow::compute::KernelSignature": + c_string ToString() const + + cdef cppclass CKernel" arrow::compute::Kernel": + shared_ptr[CKernelSignature] signature + + cdef cppclass CArrayKernel" arrow::compute::ArrayKernel"(CKernel): + pass + + cdef cppclass CScalarKernel" arrow::compute::ScalarKernel"(CArrayKernel): + pass + + cdef cppclass CVectorKernel" arrow::compute::VectorKernel"(CArrayKernel): + pass + + cdef cppclass CScalarAggregateKernel \ + " arrow::compute::ScalarAggregateKernel"(CKernel): + pass + + cdef cppclass CHashAggregateKernel \ + " arrow::compute::HashAggregateKernel"(CKernel): + pass + + cdef cppclass CArity" arrow::compute::Arity": + int num_args + c_bool is_varargs + + CArity() + + CArity(int num_args, c_bool is_varargs) + + cdef enum FunctionKind" arrow::compute::Function::Kind": + FunctionKind_SCALAR" arrow::compute::Function::SCALAR" + FunctionKind_VECTOR" arrow::compute::Function::VECTOR" + FunctionKind_SCALAR_AGGREGATE \ + " arrow::compute::Function::SCALAR_AGGREGATE" + FunctionKind_HASH_AGGREGATE \ + " arrow::compute::Function::HASH_AGGREGATE" + FunctionKind_META \ + " arrow::compute::Function::META" + + cdef cppclass CFunctionDoc" arrow::compute::FunctionDoc": + c_string summary + c_string description + vector[c_string] arg_names + c_string options_class + c_bool options_required + + cdef cppclass CFunctionOptionsType" arrow::compute::FunctionOptionsType": + const char* type_name() const + + cdef cppclass CFunctionOptions" arrow::compute::FunctionOptions": + const CFunctionOptionsType* options_type() const + const char* type_name() const + c_bool Equals(const CFunctionOptions& other) const + c_string ToString() const + unique_ptr[CFunctionOptions] Copy() const + CResult[shared_ptr[CBuffer]] Serialize() const + + @staticmethod + CResult[unique_ptr[CFunctionOptions]] Deserialize( + const c_string& type_name, const CBuffer& buffer) + + cdef cppclass CFunction" arrow::compute::Function": + const c_string& name() const + FunctionKind kind() const + const CArity& arity() const + const CFunctionDoc& doc() const + int num_kernels() const + CResult[CDatum] Execute(const vector[CDatum]& args, + const CFunctionOptions* options, + CExecContext* ctx) const + CResult[CDatum] Execute(const CExecBatch& args, + const CFunctionOptions* options, + CExecContext* ctx) const + + cdef cppclass CScalarFunction" arrow::compute::ScalarFunction"(CFunction): + vector[const CScalarKernel*] kernels() const + + cdef cppclass CVectorFunction" arrow::compute::VectorFunction"(CFunction): + vector[const CVectorKernel*] kernels() const + + cdef cppclass CScalarAggregateFunction \ + " arrow::compute::ScalarAggregateFunction"(CFunction): + vector[const CScalarAggregateKernel*] kernels() const + + cdef cppclass CHashAggregateFunction \ + " arrow::compute::HashAggregateFunction"(CFunction): + vector[const CHashAggregateKernel*] kernels() const + + cdef cppclass CMetaFunction" arrow::compute::MetaFunction"(CFunction): + pass + + cdef cppclass CFunctionRegistry" arrow::compute::FunctionRegistry": + CResult[shared_ptr[CFunction]] GetFunction( + const c_string& name) const + vector[c_string] GetFunctionNames() const + int num_functions() const + + CFunctionRegistry* GetFunctionRegistry() + + cdef cppclass CElementWiseAggregateOptions \ + "arrow::compute::ElementWiseAggregateOptions"(CFunctionOptions): + CElementWiseAggregateOptions(c_bool skip_nulls) + c_bool skip_nulls + + ctypedef enum CRoundMode \ + "arrow::compute::RoundMode": + CRoundMode_DOWN \ + "arrow::compute::RoundMode::DOWN" + CRoundMode_UP \ + "arrow::compute::RoundMode::UP" + CRoundMode_TOWARDS_ZERO \ + "arrow::compute::RoundMode::TOWARDS_ZERO" + CRoundMode_TOWARDS_INFINITY \ + "arrow::compute::RoundMode::TOWARDS_INFINITY" + CRoundMode_HALF_DOWN \ + "arrow::compute::RoundMode::HALF_DOWN" + CRoundMode_HALF_UP \ + "arrow::compute::RoundMode::HALF_UP" + CRoundMode_HALF_TOWARDS_ZERO \ + "arrow::compute::RoundMode::HALF_TOWARDS_ZERO" + CRoundMode_HALF_TOWARDS_INFINITY \ + "arrow::compute::RoundMode::HALF_TOWARDS_INFINITY" + CRoundMode_HALF_TO_EVEN \ + "arrow::compute::RoundMode::HALF_TO_EVEN" + CRoundMode_HALF_TO_ODD \ + "arrow::compute::RoundMode::HALF_TO_ODD" + + cdef cppclass CRoundOptions \ + "arrow::compute::RoundOptions"(CFunctionOptions): + CRoundOptions(int64_t ndigits, CRoundMode round_mode) + int64_t ndigits + CRoundMode round_mode + + cdef cppclass CRoundBinaryOptions \ + "arrow::compute::RoundBinaryOptions"(CFunctionOptions): + CRoundBinaryOptions(CRoundMode round_mode) + CRoundMode round_mode + + ctypedef enum CCalendarUnit \ + "arrow::compute::CalendarUnit": + CCalendarUnit_NANOSECOND \ + "arrow::compute::CalendarUnit::NANOSECOND" + CCalendarUnit_MICROSECOND \ + "arrow::compute::CalendarUnit::MICROSECOND" + CCalendarUnit_MILLISECOND \ + "arrow::compute::CalendarUnit::MILLISECOND" + CCalendarUnit_SECOND \ + "arrow::compute::CalendarUnit::SECOND" + CCalendarUnit_MINUTE \ + "arrow::compute::CalendarUnit::MINUTE" + CCalendarUnit_HOUR \ + "arrow::compute::CalendarUnit::HOUR" + CCalendarUnit_DAY \ + "arrow::compute::CalendarUnit::DAY" + CCalendarUnit_WEEK \ + "arrow::compute::CalendarUnit::WEEK" + CCalendarUnit_MONTH \ + "arrow::compute::CalendarUnit::MONTH" + CCalendarUnit_QUARTER \ + "arrow::compute::CalendarUnit::QUARTER" + CCalendarUnit_YEAR \ + "arrow::compute::CalendarUnit::YEAR" + + cdef cppclass CRoundTemporalOptions \ + "arrow::compute::RoundTemporalOptions"(CFunctionOptions): + CRoundTemporalOptions(int multiple, CCalendarUnit unit, + c_bool week_starts_monday, + c_bool ceil_is_strictly_greater, + c_bool calendar_based_origin) + int multiple + CCalendarUnit unit + c_bool week_starts_monday + c_bool ceil_is_strictly_greater + c_bool calendar_based_origin + + cdef cppclass CRoundToMultipleOptions \ + "arrow::compute::RoundToMultipleOptions"(CFunctionOptions): + CRoundToMultipleOptions(shared_ptr[CScalar] multiple, CRoundMode round_mode) + shared_ptr[CScalar] multiple + CRoundMode round_mode + + cdef enum CJoinNullHandlingBehavior \ + "arrow::compute::JoinOptions::NullHandlingBehavior": + CJoinNullHandlingBehavior_EMIT_NULL \ + "arrow::compute::JoinOptions::EMIT_NULL" + CJoinNullHandlingBehavior_SKIP \ + "arrow::compute::JoinOptions::SKIP" + CJoinNullHandlingBehavior_REPLACE \ + "arrow::compute::JoinOptions::REPLACE" + + cdef cppclass CJoinOptions \ + "arrow::compute::JoinOptions"(CFunctionOptions): + CJoinOptions(CJoinNullHandlingBehavior null_handling, + c_string null_replacement) + CJoinNullHandlingBehavior null_handling + c_string null_replacement + + cdef cppclass CMatchSubstringOptions \ + "arrow::compute::MatchSubstringOptions"(CFunctionOptions): + CMatchSubstringOptions(c_string pattern, c_bool ignore_case) + c_string pattern + c_bool ignore_case + + cdef cppclass CTrimOptions \ + "arrow::compute::TrimOptions"(CFunctionOptions): + CTrimOptions(c_string characters) + c_string characters + + cdef cppclass CPadOptions \ + "arrow::compute::PadOptions"(CFunctionOptions): + CPadOptions(int64_t width, c_string padding, c_bool lean_left_on_odd_padding) + int64_t width + c_string padding + c_bool lean_left_on_odd_padding + + cdef cppclass CZeroFillOptions \ + "arrow::compute::ZeroFillOptions"(CFunctionOptions): + CZeroFillOptions(int64_t width, c_string padding) + int64_t width + c_string padding + + cdef cppclass CSliceOptions \ + "arrow::compute::SliceOptions"(CFunctionOptions): + CSliceOptions(int64_t start, int64_t stop, int64_t step) + int64_t start + int64_t stop + int64_t step + + cdef cppclass CListSliceOptions \ + "arrow::compute::ListSliceOptions"(CFunctionOptions): + CListSliceOptions(int64_t start, optional[int64_t] stop, + int64_t step, + optional[c_bool] return_fixed_size_list) + int64_t start + optional[int64_t] stop + int64_t step + optional[c_bool] return_fixed_size_list + + cdef cppclass CSplitOptions \ + "arrow::compute::SplitOptions"(CFunctionOptions): + CSplitOptions(int64_t max_splits, c_bool reverse) + int64_t max_splits + c_bool reverse + + cdef cppclass CSplitPatternOptions \ + "arrow::compute::SplitPatternOptions"(CFunctionOptions): + CSplitPatternOptions(c_string pattern, int64_t max_splits, + c_bool reverse) + int64_t max_splits + c_bool reverse + c_string pattern + + cdef cppclass CReplaceSliceOptions \ + "arrow::compute::ReplaceSliceOptions"(CFunctionOptions): + CReplaceSliceOptions(int64_t start, int64_t stop, c_string replacement) + int64_t start + int64_t stop + c_string replacement + + cdef cppclass CReplaceSubstringOptions \ + "arrow::compute::ReplaceSubstringOptions"(CFunctionOptions): + CReplaceSubstringOptions(c_string pattern, c_string replacement, + int64_t max_replacements) + c_string pattern + c_string replacement + int64_t max_replacements + + cdef cppclass CExtractRegexOptions \ + "arrow::compute::ExtractRegexOptions"(CFunctionOptions): + CExtractRegexOptions(c_string pattern) + c_string pattern + + cdef cppclass CExtractRegexSpanOptions \ + "arrow::compute::ExtractRegexSpanOptions"(CFunctionOptions): + CExtractRegexSpanOptions(c_string pattern) + c_string pattern + + cdef cppclass CCastOptions" arrow::compute::CastOptions"(CFunctionOptions): + CCastOptions() + CCastOptions(c_bool safe) + CCastOptions(CCastOptions options) + + @staticmethod + CCastOptions Safe() + + @staticmethod + CCastOptions Unsafe() + shared_ptr[CDataType] to_type + c_bool allow_int_overflow + c_bool allow_time_truncate + c_bool allow_time_overflow + c_bool allow_decimal_truncate + c_bool allow_float_truncate + c_bool allow_invalid_utf8 + + cdef enum CFilterNullSelectionBehavior \ + "arrow::compute::FilterOptions::NullSelectionBehavior": + CFilterNullSelectionBehavior_DROP \ + "arrow::compute::FilterOptions::DROP" + CFilterNullSelectionBehavior_EMIT_NULL \ + "arrow::compute::FilterOptions::EMIT_NULL" + + cdef cppclass CFilterOptions \ + " arrow::compute::FilterOptions"(CFunctionOptions): + CFilterOptions() + CFilterOptions(CFilterNullSelectionBehavior null_selection_behavior) + CFilterNullSelectionBehavior null_selection_behavior + + cdef enum CDictionaryEncodeNullEncodingBehavior \ + "arrow::compute::DictionaryEncodeOptions::NullEncodingBehavior": + CDictionaryEncodeNullEncodingBehavior_ENCODE \ + "arrow::compute::DictionaryEncodeOptions::ENCODE" + CDictionaryEncodeNullEncodingBehavior_MASK \ + "arrow::compute::DictionaryEncodeOptions::MASK" + + cdef cppclass CDictionaryEncodeOptions \ + "arrow::compute::DictionaryEncodeOptions"(CFunctionOptions): + CDictionaryEncodeOptions( + CDictionaryEncodeNullEncodingBehavior null_encoding) + CDictionaryEncodeNullEncodingBehavior null_encoding + + cdef cppclass CRunEndEncodeOptions \ + "arrow::compute::RunEndEncodeOptions"(CFunctionOptions): + CRunEndEncodeOptions() + CRunEndEncodeOptions(shared_ptr[CDataType] run_end_type) + shared_ptr[CDataType] run_end_type + + cdef cppclass CTakeOptions \ + " arrow::compute::TakeOptions"(CFunctionOptions): + CTakeOptions(c_bool boundscheck) + c_bool boundscheck + + cdef cppclass CInversePermutationOptions \ + "arrow::compute::InversePermutationOptions"(CFunctionOptions): + CInversePermutationOptions(int64_t max_index, optional[shared_ptr[CDataType]] output_type) + int64_t max_index + optional[shared_ptr[CDataType]] output_type + + cdef cppclass CScatterOptions \ + "arrow::compute::ScatterOptions"(CFunctionOptions): + CScatterOptions(int64_t max_index) + int64_t max_index + + cdef cppclass CStrptimeOptions \ + "arrow::compute::StrptimeOptions"(CFunctionOptions): + CStrptimeOptions(c_string format, TimeUnit unit, c_bool raise_error) + c_string format + TimeUnit unit + c_bool raise_error + + cdef cppclass CStrftimeOptions \ + "arrow::compute::StrftimeOptions"(CFunctionOptions): + CStrftimeOptions(c_string format, c_string locale) + c_string format + c_string locale + + cdef cppclass CDayOfWeekOptions \ + "arrow::compute::DayOfWeekOptions"(CFunctionOptions): + CDayOfWeekOptions(c_bool count_from_zero, uint32_t week_start) + c_bool count_from_zero + uint32_t week_start + + cdef enum CAssumeTimezoneAmbiguous \ + "arrow::compute::AssumeTimezoneOptions::Ambiguous": + CAssumeTimezoneAmbiguous_AMBIGUOUS_RAISE \ + "arrow::compute::AssumeTimezoneOptions::AMBIGUOUS_RAISE" + CAssumeTimezoneAmbiguous_AMBIGUOUS_EARLIEST \ + "arrow::compute::AssumeTimezoneOptions::AMBIGUOUS_EARLIEST" + CAssumeTimezoneAmbiguous_AMBIGUOUS_LATEST \ + "arrow::compute::AssumeTimezoneOptions::AMBIGUOUS_LATEST" + + cdef enum CAssumeTimezoneNonexistent \ + "arrow::compute::AssumeTimezoneOptions::Nonexistent": + CAssumeTimezoneNonexistent_NONEXISTENT_RAISE \ + "arrow::compute::AssumeTimezoneOptions::NONEXISTENT_RAISE" + CAssumeTimezoneNonexistent_NONEXISTENT_EARLIEST \ + "arrow::compute::AssumeTimezoneOptions::NONEXISTENT_EARLIEST" + CAssumeTimezoneNonexistent_NONEXISTENT_LATEST \ + "arrow::compute::AssumeTimezoneOptions::NONEXISTENT_LATEST" + + cdef cppclass CAssumeTimezoneOptions \ + "arrow::compute::AssumeTimezoneOptions"(CFunctionOptions): + CAssumeTimezoneOptions(c_string timezone, + CAssumeTimezoneAmbiguous ambiguous, + CAssumeTimezoneNonexistent nonexistent) + c_string timezone + CAssumeTimezoneAmbiguous ambiguous + CAssumeTimezoneNonexistent nonexistent + + cdef cppclass CWeekOptions \ + "arrow::compute::WeekOptions"(CFunctionOptions): + CWeekOptions(c_bool week_starts_monday, c_bool count_from_zero, + c_bool first_week_is_fully_in_year) + c_bool week_starts_monday + c_bool count_from_zero + c_bool first_week_is_fully_in_year + + cdef cppclass CNullOptions \ + "arrow::compute::NullOptions"(CFunctionOptions): + CNullOptions(c_bool nan_is_null) + c_bool nan_is_null + + cdef cppclass CVarianceOptions \ + "arrow::compute::VarianceOptions"(CFunctionOptions): + CVarianceOptions(int ddof, c_bool skip_nulls, uint32_t min_count) + int ddof + c_bool skip_nulls + uint32_t min_count + + cdef cppclass CSkewOptions \ + "arrow::compute::SkewOptions"(CFunctionOptions): + CSkewOptions(c_bool skip_nulls, c_bool biased, uint32_t min_count) + c_bool skip_nulls + c_bool biased + uint32_t min_count + + cdef cppclass CScalarAggregateOptions \ + "arrow::compute::ScalarAggregateOptions"(CFunctionOptions): + CScalarAggregateOptions(c_bool skip_nulls, uint32_t min_count) + c_bool skip_nulls + uint32_t min_count + + cdef enum CCountMode "arrow::compute::CountOptions::CountMode": + CCountMode_ONLY_VALID "arrow::compute::CountOptions::ONLY_VALID" + CCountMode_ONLY_NULL "arrow::compute::CountOptions::ONLY_NULL" + CCountMode_ALL "arrow::compute::CountOptions::ALL" + + cdef cppclass CCountOptions \ + "arrow::compute::CountOptions"(CFunctionOptions): + CCountOptions(CCountMode mode) + CCountMode mode + + cdef cppclass CModeOptions \ + "arrow::compute::ModeOptions"(CFunctionOptions): + CModeOptions(int64_t n, c_bool skip_nulls, uint32_t min_count) + int64_t n + c_bool skip_nulls + uint32_t min_count + + cdef cppclass CIndexOptions \ + "arrow::compute::IndexOptions"(CFunctionOptions): + CIndexOptions(shared_ptr[CScalar] value) + shared_ptr[CScalar] value + + cdef cppclass CAggregate "arrow::compute::Aggregate": + c_string function + shared_ptr[CFunctionOptions] options + vector[CFieldRef] target + c_string name + + cdef enum CMapLookupOccurrence \ + "arrow::compute::MapLookupOptions::Occurrence": + CMapLookupOccurrence_ALL "arrow::compute::MapLookupOptions::ALL" + CMapLookupOccurrence_FIRST "arrow::compute::MapLookupOptions::FIRST" + CMapLookupOccurrence_LAST "arrow::compute::MapLookupOptions::LAST" + + cdef cppclass CMapLookupOptions \ + "arrow::compute::MapLookupOptions"(CFunctionOptions): + CMapLookupOptions(shared_ptr[CScalar] query_key, + CMapLookupOccurrence occurrence) + CMapLookupOccurrence occurrence + shared_ptr[CScalar] query_key + + cdef cppclass CMakeStructOptions \ + "arrow::compute::MakeStructOptions"(CFunctionOptions): + CMakeStructOptions(vector[c_string] n, + vector[c_bool] r, + vector[shared_ptr[const CKeyValueMetadata]] m) + CMakeStructOptions(vector[c_string] n) + vector[c_string] field_names + vector[c_bool] field_nullability + vector[shared_ptr[const CKeyValueMetadata]] field_metadata + + cdef cppclass CStructFieldOptions \ + "arrow::compute::StructFieldOptions"(CFunctionOptions): + CStructFieldOptions(vector[int] indices) + CStructFieldOptions(CFieldRef field_ref) + vector[int] indices + CFieldRef field_ref + + ctypedef enum CSortOrder" arrow::compute::SortOrder": + CSortOrder_Ascending \ + "arrow::compute::SortOrder::Ascending" + CSortOrder_Descending \ + "arrow::compute::SortOrder::Descending" + + ctypedef enum CNullPlacement" arrow::compute::NullPlacement": + CNullPlacement_AtStart \ + "arrow::compute::NullPlacement::AtStart" + CNullPlacement_AtEnd \ + "arrow::compute::NullPlacement::AtEnd" + + cdef cppclass CPartitionNthOptions \ + "arrow::compute::PartitionNthOptions"(CFunctionOptions): + CPartitionNthOptions(int64_t pivot, CNullPlacement) + int64_t pivot + CNullPlacement null_placement + + cdef cppclass CWinsorizeOptions \ + "arrow::compute::WinsorizeOptions"(CFunctionOptions): + CWinsorizeOptions(double lower_limit, double upper_limit) + + cdef cppclass CCumulativeOptions \ + "arrow::compute::CumulativeOptions"(CFunctionOptions): + CCumulativeOptions(c_bool skip_nulls) + CCumulativeOptions(shared_ptr[CScalar] start, c_bool skip_nulls) + optional[shared_ptr[CScalar]] start + c_bool skip_nulls + + cdef cppclass CPairwiseOptions \ + "arrow::compute::PairwiseOptions"(CFunctionOptions): + CPairwiseOptions(int64_t period) + int64_t period + + cdef cppclass CListFlattenOptions\ + "arrow::compute::ListFlattenOptions"(CFunctionOptions): + CListFlattenOptions(c_bool recursive) + c_bool recursive + + cdef cppclass CArraySortOptions \ + "arrow::compute::ArraySortOptions"(CFunctionOptions): + CArraySortOptions(CSortOrder, CNullPlacement) + CSortOrder order + CNullPlacement null_placement + + cdef cppclass CSortKey" arrow::compute::SortKey": + CSortKey(CFieldRef target, CSortOrder order) + CFieldRef target + CSortOrder order + + cdef cppclass COrdering" arrow::compute::Ordering": + COrdering(vector[CSortKey] sort_keys, CNullPlacement null_placement) + + cdef cppclass CSortOptions \ + "arrow::compute::SortOptions"(CFunctionOptions): + CSortOptions(vector[CSortKey] sort_keys, CNullPlacement) + vector[CSortKey] sort_keys + CNullPlacement null_placement + + cdef cppclass CSelectKOptions \ + "arrow::compute::SelectKOptions"(CFunctionOptions): + CSelectKOptions(int64_t k, vector[CSortKey] sort_keys) + int64_t k + vector[CSortKey] sort_keys + + cdef enum CQuantileInterp \ + "arrow::compute::QuantileOptions::Interpolation": + CQuantileInterp_LINEAR "arrow::compute::QuantileOptions::LINEAR" + CQuantileInterp_LOWER "arrow::compute::QuantileOptions::LOWER" + CQuantileInterp_HIGHER "arrow::compute::QuantileOptions::HIGHER" + CQuantileInterp_NEAREST "arrow::compute::QuantileOptions::NEAREST" + CQuantileInterp_MIDPOINT "arrow::compute::QuantileOptions::MIDPOINT" + + cdef cppclass CQuantileOptions \ + "arrow::compute::QuantileOptions"(CFunctionOptions): + CQuantileOptions(vector[double] q, CQuantileInterp interpolation, + c_bool skip_nulls, uint32_t min_count) + vector[double] q + CQuantileInterp interpolation + c_bool skip_nulls + uint32_t min_count + + cdef cppclass CTDigestOptions \ + "arrow::compute::TDigestOptions"(CFunctionOptions): + CTDigestOptions(vector[double] q, + uint32_t delta, uint32_t buffer_size, + c_bool skip_nulls, uint32_t min_count) + vector[double] q + uint32_t delta + uint32_t buffer_size + c_bool skip_nulls + uint32_t min_count + + cdef enum CUtf8NormalizeForm \ + "arrow::compute::Utf8NormalizeOptions::Form": + CUtf8NormalizeForm_NFC "arrow::compute::Utf8NormalizeOptions::NFC" + CUtf8NormalizeForm_NFKC "arrow::compute::Utf8NormalizeOptions::NFKC" + CUtf8NormalizeForm_NFD "arrow::compute::Utf8NormalizeOptions::NFD" + CUtf8NormalizeForm_NFKD "arrow::compute::Utf8NormalizeOptions::NFKD" + + cdef cppclass CUtf8NormalizeOptions \ + "arrow::compute::Utf8NormalizeOptions"(CFunctionOptions): + CUtf8NormalizeOptions(CUtf8NormalizeForm form) + CUtf8NormalizeForm form + + cdef cppclass CSetLookupOptions \ + "arrow::compute::SetLookupOptions"(CFunctionOptions): + CSetLookupOptions(CDatum value_set, c_bool skip_nulls) + CDatum value_set + c_bool skip_nulls + + cdef cppclass CRandomOptions \ + "arrow::compute::RandomOptions"(CFunctionOptions): + CRandomOptions(CRandomOptions) + + @staticmethod + CRandomOptions FromSystemRandom() + + @staticmethod + CRandomOptions FromSeed(uint64_t seed) + + cdef enum CRankOptionsTiebreaker \ + "arrow::compute::RankOptions::Tiebreaker": + CRankOptionsTiebreaker_Min "arrow::compute::RankOptions::Min" + CRankOptionsTiebreaker_Max "arrow::compute::RankOptions::Max" + CRankOptionsTiebreaker_First "arrow::compute::RankOptions::First" + CRankOptionsTiebreaker_Dense "arrow::compute::RankOptions::Dense" + + cdef cppclass CRankOptions \ + "arrow::compute::RankOptions"(CFunctionOptions): + CRankOptions(vector[CSortKey] sort_keys, CNullPlacement, + CRankOptionsTiebreaker tiebreaker) + vector[CSortKey] sort_keys + CNullPlacement null_placement + CRankOptionsTiebreaker tiebreaker + + cdef cppclass CRankQuantileOptions \ + "arrow::compute::RankQuantileOptions"(CFunctionOptions): + CRankQuantileOptions(vector[CSortKey] sort_keys, CNullPlacement) + vector[CSortKey] sort_keys + CNullPlacement null_placement + + cdef enum PivotWiderUnexpectedKeyBehavior \ + "arrow::compute::PivotWiderOptions::UnexpectedKeyBehavior": + PivotWiderUnexpectedKeyBehavior_Ignore "arrow::compute::PivotWiderOptions::kIgnore" + PivotWiderUnexpectedKeyBehavior_Raise "arrow::compute::PivotWiderOptions::kRaise" + + cdef cppclass CPivotWiderOptions \ + "arrow::compute::PivotWiderOptions"(CFunctionOptions): + CPivotWiderOptions(vector[c_string] key_names, + PivotWiderUnexpectedKeyBehavior) + + cdef enum DatumType" arrow::Datum::type": + DatumType_NONE" arrow::Datum::NONE" + DatumType_SCALAR" arrow::Datum::SCALAR" + DatumType_ARRAY" arrow::Datum::ARRAY" + DatumType_CHUNKED_ARRAY" arrow::Datum::CHUNKED_ARRAY" + DatumType_RECORD_BATCH" arrow::Datum::RECORD_BATCH" + DatumType_TABLE" arrow::Datum::TABLE" + DatumType_COLLECTION" arrow::Datum::COLLECTION" + + cdef cppclass CDatum" arrow::Datum": + CDatum() + CDatum(const shared_ptr[CArray]& value) + CDatum(const shared_ptr[CChunkedArray]& value) + CDatum(const shared_ptr[CScalar]& value) + CDatum(const shared_ptr[CRecordBatch]& value) + CDatum(const shared_ptr[CTable]& value) + + DatumType kind() const + c_string ToString() const + + const shared_ptr[CArrayData]& array() const + const shared_ptr[CChunkedArray]& chunked_array() const + const shared_ptr[CRecordBatch]& record_batch() const + const shared_ptr[CTable]& table() const + const shared_ptr[CScalar]& scalar() const + + cdef c_string ToString(DatumType kind) + + +cdef extern from * namespace "arrow::compute": + # inlined from compute/function_internal.h to avoid exposing + # implementation details + """ + #include "arrow/compute/function.h" + namespace arrow { + namespace compute { + namespace internal { + Result> DeserializeFunctionOptions( + const Buffer& buffer); + } // namespace internal + } // namespace compute + } // namespace arrow + """ + CResult[unique_ptr[CFunctionOptions]] DeserializeFunctionOptions \ + " arrow::compute::internal::DeserializeFunctionOptions"( + const CBuffer& buffer) + + +cdef extern from * namespace "arrow::compute": + # inlined from expression_internal.h to avoid + # proliferation of #include + """ + #include + + #include "arrow/type.h" + #include "arrow/datum.h" + + namespace arrow { + namespace compute { + struct KnownFieldValues { + std::unordered_map map; + }; + } // namespace compute + } // namespace arrow + """ + cdef struct CKnownFieldValues "arrow::compute::KnownFieldValues": + unordered_map[CFieldRef, CDatum, CFieldRefHash] map + +cdef extern from "arrow/compute/expression.h" \ + namespace "arrow::compute" nogil: + + cdef cppclass CExpression "arrow::compute::Expression": + c_bool Equals(const CExpression& other) const + c_string ToString() const + CResult[CExpression] Bind(const CSchema&) + const CFieldRef* field_ref() const + + cdef CExpression CMakeScalarExpression \ + "arrow::compute::literal"(shared_ptr[CScalar] value) + + cdef CExpression CMakeFieldExpression \ + "arrow::compute::field_ref"(CFieldRef) + + cdef CExpression CMakeFieldExpressionByIndex \ + "arrow::compute::field_ref"(int idx) + + cdef CExpression CMakeCallExpression \ + "arrow::compute::call"(c_string function, + vector[CExpression] arguments, + shared_ptr[CFunctionOptions] options) + + cdef CResult[shared_ptr[CBuffer]] CSerializeExpression \ + "arrow::compute::Serialize"(const CExpression&) + + cdef CResult[CExpression] CDeserializeExpression \ + "arrow::compute::Deserialize"(shared_ptr[CBuffer]) + + cdef CResult[CKnownFieldValues] \ + CExtractKnownFieldValues "arrow::compute::ExtractKnownFieldValues"( + const CExpression& partition_expression) + + +cdef extern from "arrow/extension_type.h" namespace "arrow": + cdef cppclass CExtensionTypeRegistry" arrow::ExtensionTypeRegistry": + @staticmethod + shared_ptr[CExtensionTypeRegistry] GetGlobalRegistry() + + cdef cppclass CExtensionType" arrow::ExtensionType"(CDataType): + c_string extension_name() + shared_ptr[CDataType] storage_type() + int byte_width() + int bit_width() + + @staticmethod + shared_ptr[CArray] WrapArray(shared_ptr[CDataType] ext_type, + shared_ptr[CArray] storage) + + @staticmethod + shared_ptr[CChunkedArray] WrapArray(shared_ptr[CDataType] ext_type, + shared_ptr[CChunkedArray] storage) + + cdef cppclass CExtensionArray" arrow::ExtensionArray"(CArray): + CExtensionArray(shared_ptr[CDataType], shared_ptr[CArray] storage) + + shared_ptr[CArray] storage() + + +cdef extern from "arrow/extension/json.h" namespace "arrow::extension" nogil: + cdef cppclass CJsonType" arrow::extension::JsonExtensionType"(CExtensionType): + + @staticmethod + CResult[shared_ptr[CDataType]] Make(shared_ptr[CDataType]& storage_type) + + +cdef extern from "arrow/extension/uuid.h" namespace "arrow::extension" nogil: + cdef cppclass CUuidType" arrow::extension::UuidType"(CExtensionType): + + @staticmethod + CResult[shared_ptr[CDataType]] Make() + + cdef cppclass CUuidArray" arrow::extension::UuidArray"(CExtensionArray): + pass + + +cdef extern from "arrow/extension/fixed_shape_tensor.h" namespace "arrow::extension" nogil: + cdef cppclass CFixedShapeTensorType \ + " arrow::extension::FixedShapeTensorType"(CExtensionType): + + CResult[shared_ptr[CTensor]] MakeTensor(const shared_ptr[CExtensionScalar]& scalar) const + + @staticmethod + CResult[shared_ptr[CDataType]] Make(const shared_ptr[CDataType]& value_type, + const vector[int64_t]& shape, + const vector[int64_t]& permutation, + const vector[c_string]& dim_names) + + const shared_ptr[CDataType] value_type() + const vector[int64_t] shape() + const vector[int64_t] permutation() + const vector[c_string] dim_names() + + cdef cppclass CFixedShapeTensorArray \ + " arrow::extension::FixedShapeTensorArray"(CExtensionArray): + const CResult[shared_ptr[CTensor]] ToTensor() const + + +cdef extern from "arrow/extension/opaque.h" namespace "arrow::extension" nogil: + cdef cppclass COpaqueType \ + " arrow::extension::OpaqueType"(CExtensionType): + + c_string type_name() + c_string vendor_name() + + cdef cppclass COpaqueArray \ + " arrow::extension::OpaqueArray"(CExtensionArray): + pass + + +cdef extern from "arrow/extension/bool8.h" namespace "arrow::extension" nogil: + cdef cppclass CBool8Type" arrow::extension::Bool8Type"(CExtensionType): + + @staticmethod + CResult[shared_ptr[CDataType]] Make() + + cdef cppclass CBool8Array" arrow::extension::Bool8Array"(CExtensionArray): + pass + +cdef extern from "arrow/util/compression.h" namespace "arrow" nogil: + cdef enum CCompressionType" arrow::Compression::type": + CCompressionType_UNCOMPRESSED" arrow::Compression::UNCOMPRESSED" + CCompressionType_SNAPPY" arrow::Compression::SNAPPY" + CCompressionType_GZIP" arrow::Compression::GZIP" + CCompressionType_BROTLI" arrow::Compression::BROTLI" + CCompressionType_ZSTD" arrow::Compression::ZSTD" + CCompressionType_LZ4" arrow::Compression::LZ4" + CCompressionType_LZ4_FRAME" arrow::Compression::LZ4_FRAME" + CCompressionType_BZ2" arrow::Compression::BZ2" + + cdef cppclass CCodec" arrow::util::Codec": + @staticmethod + CResult[unique_ptr[CCodec]] Create(CCompressionType codec) + + @staticmethod + CResult[unique_ptr[CCodec]] CreateWithLevel" Create"( + CCompressionType codec, + int compression_level) + + @staticmethod + c_bool SupportsCompressionLevel(CCompressionType codec) + + @staticmethod + CResult[int] MinimumCompressionLevel(CCompressionType codec) + + @staticmethod + CResult[int] MaximumCompressionLevel(CCompressionType codec) + + @staticmethod + CResult[int] DefaultCompressionLevel(CCompressionType codec) + + @staticmethod + c_bool IsAvailable(CCompressionType codec) + + CResult[int64_t] Decompress(int64_t input_len, const uint8_t* input, + int64_t output_len, + uint8_t* output_buffer) + CResult[int64_t] Compress(int64_t input_len, const uint8_t* input, + int64_t output_buffer_len, + uint8_t* output_buffer) + c_string name() const + int compression_level() const + int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) + + +cdef extern from "arrow/util/io_util.h" namespace "arrow::internal" nogil: + int ErrnoFromStatus(CStatus status) + int WinErrorFromStatus(CStatus status) + int SignalFromStatus(CStatus status) + + CStatus SendSignal(int signum) + CStatus SendSignalToThread(int signum, uint64_t thread_id) + + +cdef extern from "arrow/util/iterator.h" namespace "arrow" nogil: + cdef cppclass CIterator" arrow::Iterator"[T]: + CResult[T] Next() + CStatus Visit[Visitor](Visitor&& visitor) + cppclass RangeIterator: + CResult[T] operator*() + RangeIterator& operator++() + c_bool operator!=(RangeIterator) const + RangeIterator begin() + RangeIterator end() + CIterator[T] MakeVectorIterator[T](vector[T] v) + + +cdef extern from "arrow/util/secure_string.h" namespace "arrow" nogil: + cdef cppclass CSecureString" arrow::util::SecureString": + CSecureString() + CSecureString(c_string s) + CSecureString(const CSecureString& s) + CSecureString(size_t n, char c) + cpp_string_view as_view() + + +cdef extern from "arrow/util/thread_pool.h" namespace "arrow" nogil: + int GetCpuThreadPoolCapacity() + CStatus SetCpuThreadPoolCapacity(int threads) + +cdef extern from "arrow/array/concatenate.h" namespace "arrow" nogil: + CResult[shared_ptr[CArray]] Concatenate( + const vector[shared_ptr[CArray]]& arrays, + CMemoryPool* pool) + +cdef extern from "arrow/c/abi.h": + cdef struct ArrowSchema: + void (*release)(ArrowSchema*) noexcept nogil + + cdef struct ArrowArray: + void (*release)(ArrowArray*) noexcept nogil + + cdef struct ArrowArrayStream: + void (*release)(ArrowArrayStream*) noexcept nogil + + ctypedef int32_t ArrowDeviceType + cdef ArrowDeviceType ARROW_DEVICE_CUDA + + cdef struct ArrowDeviceArray: + ArrowArray array + int64_t device_id + int32_t device_type + +cdef extern from "arrow/c/bridge.h" namespace "arrow" nogil: + CStatus ExportType(CDataType&, ArrowSchema* out) + CResult[shared_ptr[CDataType]] ImportType(ArrowSchema*) + + CStatus ExportField(CField&, ArrowSchema* out) + CResult[shared_ptr[CField]] ImportField(ArrowSchema*) + + CStatus ExportSchema(CSchema&, ArrowSchema* out) + CResult[shared_ptr[CSchema]] ImportSchema(ArrowSchema*) + + CStatus ExportArray(CArray&, ArrowArray* out) + CStatus ExportArray(CArray&, ArrowArray* out, ArrowSchema* out_schema) + CResult[shared_ptr[CArray]] ImportArray(ArrowArray*, + shared_ptr[CDataType]) + CResult[shared_ptr[CArray]] ImportArray(ArrowArray*, ArrowSchema*) + + CStatus ExportRecordBatch(CRecordBatch&, ArrowArray* out) + CStatus ExportRecordBatch(CRecordBatch&, ArrowArray* out, + ArrowSchema* out_schema) + CResult[shared_ptr[CRecordBatch]] ImportRecordBatch(ArrowArray*, + shared_ptr[CSchema]) + CResult[shared_ptr[CRecordBatch]] ImportRecordBatch(ArrowArray*, + ArrowSchema*) + + CStatus ExportRecordBatchReader(shared_ptr[CRecordBatchReader], + ArrowArrayStream*) + CResult[shared_ptr[CRecordBatchReader]] ImportRecordBatchReader( + ArrowArrayStream*) + + CStatus ExportChunkedArray(shared_ptr[CChunkedArray], ArrowArrayStream*) + CResult[shared_ptr[CChunkedArray]] ImportChunkedArray(ArrowArrayStream*) + + CStatus ExportDeviceArray(const CArray&, shared_ptr[CSyncEvent], + ArrowDeviceArray* out, ArrowSchema*) + CResult[shared_ptr[CArray]] ImportDeviceArray( + ArrowDeviceArray*, shared_ptr[CDataType]) + CResult[shared_ptr[CArray]] ImportDeviceArray( + ArrowDeviceArray*, ArrowSchema*) + + CStatus ExportDeviceRecordBatch(const CRecordBatch&, shared_ptr[CSyncEvent], + ArrowDeviceArray* out, ArrowSchema*) + CResult[shared_ptr[CRecordBatch]] ImportDeviceRecordBatch( + ArrowDeviceArray*, shared_ptr[CSchema]) + CResult[shared_ptr[CRecordBatch]] ImportDeviceRecordBatch( + ArrowDeviceArray*, ArrowSchema*) + + +cdef extern from "arrow/util/byte_size.h" namespace "arrow::util" nogil: + CResult[int64_t] ReferencedBufferSize(const CArray& array_data) + CResult[int64_t] ReferencedBufferSize(const CRecordBatch& record_batch) + CResult[int64_t] ReferencedBufferSize(const CChunkedArray& chunked_array) + CResult[int64_t] ReferencedBufferSize(const CTable& table) + int64_t TotalBufferSize(const CArray& array) + int64_t TotalBufferSize(const CChunkedArray& array) + int64_t TotalBufferSize(const CRecordBatch& record_batch) + int64_t TotalBufferSize(const CTable& table) + +ctypedef PyObject* CallbackUdf(object user_function, const CUdfContext& context, object inputs) + + +cdef extern from "arrow/api.h" namespace "arrow" nogil: + + cdef cppclass CRecordBatchIterator "arrow::RecordBatchIterator"( + CIterator[shared_ptr[CRecordBatch]]): + pass + + +cdef extern from "arrow/python/udf.h" namespace "arrow::py" nogil: + cdef cppclass CUdfContext" arrow::py::UdfContext": + CMemoryPool *pool + int64_t batch_length + + cdef cppclass CUdfOptions" arrow::py::UdfOptions": + c_string func_name + CArity arity + CFunctionDoc func_doc + vector[shared_ptr[CDataType]] input_types + shared_ptr[CDataType] output_type + + CStatus RegisterScalarFunction(PyObject* function, + function[CallbackUdf] wrapper, const CUdfOptions& options, + CFunctionRegistry* registry) + + CStatus RegisterTabularFunction(PyObject* function, + function[CallbackUdf] wrapper, const CUdfOptions& options, + CFunctionRegistry* registry) + + CStatus RegisterAggregateFunction(PyObject* function, + function[CallbackUdf] wrapper, const CUdfOptions& options, + CFunctionRegistry* registry) + + CStatus RegisterVectorFunction(PyObject* function, + function[CallbackUdf] wrapper, const CUdfOptions& options, + CFunctionRegistry* registry) + + CResult[shared_ptr[CRecordBatchReader]] CallTabularFunction( + const c_string& func_name, const vector[CDatum]& args, CFunctionRegistry* registry) + +cdef extern from "arrow/compute/cast.h" namespace "arrow::compute": + CResult[CDatum] Cast(const CDatum& value, const CCastOptions& options) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_acero.pxd b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_acero.pxd new file mode 100644 index 0000000000000000000000000000000000000000..2861b5427e3addd87369a805894cfa8497330a93 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_acero.pxd @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# distutils: language = c++ + +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport * + + +cdef extern from "arrow/acero/options.h" namespace "arrow::acero" nogil: + cdef enum CJoinType "arrow::acero::JoinType": + CJoinType_LEFT_SEMI "arrow::acero::JoinType::LEFT_SEMI" + CJoinType_RIGHT_SEMI "arrow::acero::JoinType::RIGHT_SEMI" + CJoinType_LEFT_ANTI "arrow::acero::JoinType::LEFT_ANTI" + CJoinType_RIGHT_ANTI "arrow::acero::JoinType::RIGHT_ANTI" + CJoinType_INNER "arrow::acero::JoinType::INNER" + CJoinType_LEFT_OUTER "arrow::acero::JoinType::LEFT_OUTER" + CJoinType_RIGHT_OUTER "arrow::acero::JoinType::RIGHT_OUTER" + CJoinType_FULL_OUTER "arrow::acero::JoinType::FULL_OUTER" + + cdef cppclass CExecNodeOptions "arrow::acero::ExecNodeOptions": + pass + + cdef cppclass CSourceNodeOptions "arrow::acero::SourceNodeOptions"(CExecNodeOptions): + pass + + cdef cppclass CTableSourceNodeOptions "arrow::acero::TableSourceNodeOptions"(CExecNodeOptions): + CTableSourceNodeOptions(shared_ptr[CTable] table) + CTableSourceNodeOptions(shared_ptr[CTable] table, int64_t max_batch_size) + + cdef cppclass CSinkNodeOptions "arrow::acero::SinkNodeOptions"(CExecNodeOptions): + pass + + cdef cppclass CFilterNodeOptions "arrow::acero::FilterNodeOptions"(CExecNodeOptions): + CFilterNodeOptions(CExpression) + + cdef cppclass CProjectNodeOptions "arrow::acero::ProjectNodeOptions"(CExecNodeOptions): + CProjectNodeOptions(vector[CExpression] expressions) + CProjectNodeOptions(vector[CExpression] expressions, + vector[c_string] names) + + cdef cppclass CAggregateNodeOptions "arrow::acero::AggregateNodeOptions"(CExecNodeOptions): + CAggregateNodeOptions(vector[CAggregate] aggregates, vector[CFieldRef] names) + + cdef cppclass COrderByNodeOptions "arrow::acero::OrderByNodeOptions"(CExecNodeOptions): + COrderByNodeOptions(COrdering ordering) + + cdef cppclass CHashJoinNodeOptions "arrow::acero::HashJoinNodeOptions"(CExecNodeOptions): + CHashJoinNodeOptions(CJoinType, vector[CFieldRef] in_left_keys, + vector[CFieldRef] in_right_keys) + CHashJoinNodeOptions(CJoinType, vector[CFieldRef] in_left_keys, + vector[CFieldRef] in_right_keys, + CExpression filter, + c_string output_suffix_for_left, + c_string output_suffix_for_right) + CHashJoinNodeOptions(CJoinType join_type, + vector[CFieldRef] left_keys, + vector[CFieldRef] right_keys, + vector[CFieldRef] left_output, + vector[CFieldRef] right_output, + CExpression filter, + c_string output_suffix_for_left, + c_string output_suffix_for_right) + + cdef struct CAsofJoinKeys "arrow::acero::AsofJoinNodeOptions::Keys": + CFieldRef on_key + vector[CFieldRef] by_key + + cdef cppclass CAsofJoinNodeOptions "arrow::acero::AsofJoinNodeOptions"(CExecNodeOptions): + CAsofJoinNodeOptions(vector[CAsofJoinKeys] keys, int64_t tolerance) + + +cdef extern from "arrow/acero/exec_plan.h" namespace "arrow::acero" nogil: + cdef cppclass CDeclaration "arrow::acero::Declaration": + cppclass Input: + Input(CExecNode*) + Input(CDeclaration) + + c_string label + vector[Input] inputs + + CDeclaration() + CDeclaration(c_string factory_name, CExecNodeOptions options) + CDeclaration(c_string factory_name, vector[Input] inputs, shared_ptr[CExecNodeOptions] options) + + @staticmethod + CDeclaration Sequence(vector[CDeclaration] decls) + + cdef cppclass CExecNode "arrow::acero::ExecNode": + const vector[CExecNode*]& inputs() const + const shared_ptr[CSchema]& output_schema() const + + CResult[shared_ptr[CTable]] DeclarationToTable( + CDeclaration declaration, c_bool use_threads + ) + CResult[shared_ptr[CTable]] DeclarationToTable( + CDeclaration declaration, c_bool use_threads, + CMemoryPool* memory_pool, CFunctionRegistry* function_registry + ) + CResult[unique_ptr[CRecordBatchReader]] DeclarationToReader( + CDeclaration declaration, c_bool use_threads + ) + + CResult[c_string] DeclarationToString(const CDeclaration& declaration) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_cuda.pxd b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_cuda.pxd new file mode 100644 index 0000000000000000000000000000000000000000..172c9eb7c454c4a73c3fa4be17d643b22cb95ea0 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_cuda.pxd @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# distutils: language = c++ + +from pyarrow.includes.libarrow cimport * + +cdef extern from "arrow/gpu/cuda_api.h" namespace "arrow::cuda" nogil: + + cdef cppclass CCudaDeviceManager" arrow::cuda::CudaDeviceManager": + @staticmethod + CResult[CCudaDeviceManager*] Instance() + CResult[shared_ptr[CCudaContext]] GetContext(int gpu_number) + CResult[shared_ptr[CCudaContext]] GetSharedContext(int gpu_number, + void* handle) + CStatus AllocateHost(int device_number, int64_t nbytes, + shared_ptr[CCudaHostBuffer]* buffer) + int num_devices() const + + cdef cppclass CCudaContext" arrow::cuda::CudaContext": + CResult[shared_ptr[CCudaBuffer]] Allocate(int64_t nbytes) + CResult[shared_ptr[CCudaBuffer]] View(uint8_t* data, int64_t nbytes) + CResult[shared_ptr[CCudaBuffer]] OpenIpcBuffer( + const CCudaIpcMemHandle& ipc_handle) + CStatus Synchronize() + int64_t bytes_allocated() const + const void* handle() const + int device_number() const + CResult[uintptr_t] GetDeviceAddress(uintptr_t addr) + shared_ptr[CDevice] device() const + shared_ptr[CMemoryManager] memory_manager() const + + cdef cppclass CCudaIpcMemHandle" arrow::cuda::CudaIpcMemHandle": + @staticmethod + CResult[shared_ptr[CCudaIpcMemHandle]] FromBuffer( + const void* opaque_handle) + CResult[shared_ptr[CBuffer]] Serialize(CMemoryPool* pool) const + + cdef cppclass CCudaBuffer" arrow::cuda::CudaBuffer"(CBuffer): + CCudaBuffer(uint8_t* data, int64_t size, + const shared_ptr[CCudaContext]& context, + c_bool own_data=false, c_bool is_ipc=false) + CCudaBuffer(const shared_ptr[CCudaBuffer]& parent, + const int64_t offset, const int64_t size) + + @staticmethod + CResult[shared_ptr[CCudaBuffer]] FromBuffer(shared_ptr[CBuffer] buf) + + CStatus CopyToHost(const int64_t position, const int64_t nbytes, + void* out) const + CStatus CopyFromHost(const int64_t position, const void* data, + int64_t nbytes) + CStatus CopyFromDevice(const int64_t position, const void* data, + int64_t nbytes) + CStatus CopyFromAnotherDevice(const shared_ptr[CCudaContext]& src_ctx, + const int64_t position, const void* data, + int64_t nbytes) + CResult[shared_ptr[CCudaIpcMemHandle]] ExportForIpc() + shared_ptr[CCudaContext] context() const + + cdef cppclass \ + CCudaHostBuffer" arrow::cuda::CudaHostBuffer"(CMutableBuffer): + pass + + cdef cppclass \ + CCudaBufferReader" arrow::cuda::CudaBufferReader"(CBufferReader): + CCudaBufferReader(const shared_ptr[CBuffer]& buffer) + CResult[int64_t] Read(int64_t nbytes, void* buffer) + CResult[shared_ptr[CBuffer]] Read(int64_t nbytes) + + cdef cppclass \ + CCudaBufferWriter" arrow::cuda::CudaBufferWriter"(WritableFile): + CCudaBufferWriter(const shared_ptr[CCudaBuffer]& buffer) + CStatus Close() + CStatus Write(const void* data, int64_t nbytes) + CStatus WriteAt(int64_t position, const void* data, int64_t nbytes) + CStatus SetBufferSize(const int64_t buffer_size) + int64_t buffer_size() + int64_t num_bytes_buffered() const + + CResult[shared_ptr[CCudaHostBuffer]] AllocateCudaHostBuffer( + int device_number, const int64_t size) + + # Cuda prefix is added to avoid picking up arrow::cuda functions + # from arrow namespace. + CResult[shared_ptr[CCudaBuffer]] \ + CudaSerializeRecordBatch" arrow::cuda::SerializeRecordBatch"\ + (const CRecordBatch& batch, + CCudaContext* ctx) + CResult[shared_ptr[CRecordBatch]] \ + CudaReadRecordBatch" arrow::cuda::ReadRecordBatch"\ + (const shared_ptr[CSchema]& schema, + CDictionaryMemo* dictionary_memo, + const shared_ptr[CCudaBuffer]& buffer, + CMemoryPool* pool) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_dataset.pxd b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_dataset.pxd new file mode 100644 index 0000000000000000000000000000000000000000..efa07bba3cb8adae8fc1db19a3f3c35feef2c344 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_dataset.pxd @@ -0,0 +1,423 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# distutils: language = c++ + +from libcpp.unordered_map cimport unordered_map +from libcpp cimport bool as c_bool + +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport * +from pyarrow.includes.libarrow_acero cimport * +from pyarrow.includes.libarrow_fs cimport * + + +cdef extern from "arrow/dataset/plan.h" namespace "arrow::dataset::internal" nogil: + + cdef void Initialize() + + +ctypedef CStatus cb_writer_finish_internal(CFileWriter*) +ctypedef void cb_writer_finish(dict, CFileWriter*) + +cdef extern from "arrow/dataset/api.h" namespace "arrow::dataset" nogil: + + cdef enum ExistingDataBehavior" arrow::dataset::ExistingDataBehavior": + ExistingDataBehavior_DELETE_MATCHING" \ + arrow::dataset::ExistingDataBehavior::kDeleteMatchingPartitions" + ExistingDataBehavior_OVERWRITE_OR_IGNORE" \ + arrow::dataset::ExistingDataBehavior::kOverwriteOrIgnore" + ExistingDataBehavior_ERROR" \ + arrow::dataset::ExistingDataBehavior::kError" + + cdef cppclass CScanOptions "arrow::dataset::ScanOptions": + shared_ptr[CSchema] dataset_schema + shared_ptr[CSchema] projected_schema + c_bool use_threads + c_bool cache_metadata + CExpression filter + + cdef cppclass CScanNodeOptions "arrow::dataset::ScanNodeOptions"(CExecNodeOptions): + CScanNodeOptions(shared_ptr[CDataset] dataset, shared_ptr[CScanOptions] scan_options, bint require_sequenced_output, bint implicit_ordering) + + shared_ptr[CScanOptions] scan_options + + cdef cppclass CFragmentScanOptions "arrow::dataset::FragmentScanOptions": + c_string type_name() const + + ctypedef CIterator[shared_ptr[CScanTask]] CScanTaskIterator \ + "arrow::dataset::ScanTaskIterator" + + cdef cppclass CScanTask" arrow::dataset::ScanTask": + CResult[CRecordBatchIterator] Execute() + + cdef cppclass CFragment "arrow::dataset::Fragment": + CResult[shared_ptr[CSchema]] ReadPhysicalSchema() + CResult[CScanTaskIterator] Scan(shared_ptr[CScanOptions] options) + c_bool splittable() const + c_string type_name() const + const CExpression& partition_expression() const + + ctypedef vector[shared_ptr[CFragment]] CFragmentVector \ + "arrow::dataset::FragmentVector" + + ctypedef CIterator[shared_ptr[CFragment]] CFragmentIterator \ + "arrow::dataset::FragmentIterator" + + cdef cppclass CInMemoryFragment "arrow::dataset::InMemoryFragment"( + CFragment): + CInMemoryFragment(vector[shared_ptr[CRecordBatch]] record_batches, + CExpression partition_expression) + + cdef cppclass CTaggedRecordBatch "arrow::dataset::TaggedRecordBatch": + shared_ptr[CRecordBatch] record_batch + shared_ptr[CFragment] fragment + + ctypedef CIterator[CTaggedRecordBatch] CTaggedRecordBatchIterator \ + "arrow::dataset::TaggedRecordBatchIterator" + + cdef cppclass CScanner "arrow::dataset::Scanner": + CScanner(shared_ptr[CDataset], shared_ptr[CScanOptions]) + CScanner(shared_ptr[CFragment], shared_ptr[CScanOptions]) + CResult[CScanTaskIterator] Scan() + CResult[CTaggedRecordBatchIterator] ScanBatches() + CResult[shared_ptr[CTable]] ToTable() + CResult[shared_ptr[CTable]] TakeRows(const CArray& indices) + CResult[shared_ptr[CTable]] Head(int64_t num_rows) + CResult[int64_t] CountRows() + CResult[CFragmentIterator] GetFragments() + CResult[shared_ptr[CRecordBatchReader]] ToRecordBatchReader() + const shared_ptr[CScanOptions]& options() + + cdef cppclass CScannerBuilder "arrow::dataset::ScannerBuilder": + CScannerBuilder(shared_ptr[CDataset], + shared_ptr[CScanOptions] scan_options) + CScannerBuilder(shared_ptr[CSchema], shared_ptr[CFragment], + shared_ptr[CScanOptions] scan_options) + + @staticmethod + shared_ptr[CScannerBuilder] FromRecordBatchReader( + shared_ptr[CRecordBatchReader] reader) + CStatus ProjectColumns "Project"(const vector[c_string]& columns) + CStatus Project(vector[CExpression]& exprs, vector[c_string]& columns) + CStatus Filter(CExpression filter) + CStatus UseThreads(c_bool use_threads) + CStatus CacheMetadata(c_bool cache_metadata) + CStatus Pool(CMemoryPool* pool) + CStatus BatchSize(int64_t batch_size) + CStatus BatchReadahead(int32_t batch_readahead) + CStatus FragmentReadahead(int32_t fragment_readahead) + CStatus FragmentScanOptions( + shared_ptr[CFragmentScanOptions] fragment_scan_options) + CResult[shared_ptr[CScanOptions]] GetScanOptions() + CResult[shared_ptr[CScanner]] Finish() + shared_ptr[CSchema] schema() const + + ctypedef vector[shared_ptr[CDataset]] CDatasetVector \ + "arrow::dataset::DatasetVector" + + cdef cppclass CDataset "arrow::dataset::Dataset": + const shared_ptr[CSchema] & schema() + CResult[CFragmentIterator] GetFragments() + CResult[CFragmentIterator] GetFragments(CExpression predicate) + const CExpression & partition_expression() + c_string type_name() + + CResult[shared_ptr[CDataset]] ReplaceSchema(shared_ptr[CSchema]) + + CResult[shared_ptr[CScannerBuilder]] NewScan() + + cdef cppclass CInMemoryDataset "arrow::dataset::InMemoryDataset"( + CDataset): + CInMemoryDataset(shared_ptr[CRecordBatchReader]) + CInMemoryDataset(shared_ptr[CTable]) + + cdef cppclass CUnionDataset "arrow::dataset::UnionDataset"( + CDataset): + @staticmethod + CResult[shared_ptr[CUnionDataset]] Make(shared_ptr[CSchema] schema, + CDatasetVector children) + + const CDatasetVector& children() const + + cdef cppclass CInspectOptions "arrow::dataset::InspectOptions": + int fragments + CField.CMergeOptions field_merge_options + + cdef cppclass CFinishOptions "arrow::dataset::FinishOptions": + shared_ptr[CSchema] schema + CInspectOptions inspect_options + c_bool validate_fragments + + cdef cppclass CDatasetFactory "arrow::dataset::DatasetFactory": + CResult[vector[shared_ptr[CSchema]]] InspectSchemas(CInspectOptions) + CResult[shared_ptr[CSchema]] Inspect(CInspectOptions) + CResult[shared_ptr[CDataset]] FinishWithSchema "Finish"( + const shared_ptr[CSchema]& schema) + CResult[shared_ptr[CDataset]] Finish() + const CExpression& root_partition() + CStatus SetRootPartition(CExpression partition) + + cdef cppclass CUnionDatasetFactory "arrow::dataset::UnionDatasetFactory": + @staticmethod + CResult[shared_ptr[CDatasetFactory]] Make( + vector[shared_ptr[CDatasetFactory]] factories) + + cdef cppclass CFileSource "arrow::dataset::FileSource": + const c_string& path() const + const shared_ptr[CFileSystem]& filesystem() const + const shared_ptr[CBuffer]& buffer() const + const int64_t size() const + CResult[shared_ptr[CRandomAccessFile]] Open() const + # HACK: Cython can't handle all the overloads so don't declare them. + # This means invalid construction of CFileSource won't be caught in + # the C++ generation phase (though it will still be caught when + # the generated C++ is compiled). + CFileSource(...) + + cdef cppclass CFileWriteOptions \ + "arrow::dataset::FileWriteOptions": + const shared_ptr[CFileFormat]& format() const + c_string type_name() const + + cdef cppclass CFileWriter \ + "arrow::dataset::FileWriter": + const shared_ptr[CFileFormat]& format() const + const shared_ptr[CSchema]& schema() const + const shared_ptr[CFileWriteOptions]& options() const + const CFileLocator& destination() const + CResult[int64_t] GetBytesWritten() + + cdef cppclass CFileFormat "arrow::dataset::FileFormat": + shared_ptr[CFragmentScanOptions] default_fragment_scan_options + c_string type_name() const + CResult[shared_ptr[CSchema]] Inspect(const CFileSource&) const + CResult[shared_ptr[CFileFragment]] MakeFragment( + CFileSource source, + CExpression partition_expression, + shared_ptr[CSchema] physical_schema) + shared_ptr[CFileWriteOptions] DefaultWriteOptions() + + cdef cppclass CFileFragment "arrow::dataset::FileFragment"( + CFragment): + const CFileSource& source() const + const shared_ptr[CFileFormat]& format() const + + cdef cppclass CFileSystemDatasetWriteOptions \ + "arrow::dataset::FileSystemDatasetWriteOptions": + shared_ptr[CFileWriteOptions] file_write_options + shared_ptr[CFileSystem] filesystem + c_string base_dir + shared_ptr[CPartitioning] partitioning + c_bool preserve_order + int max_partitions + c_string basename_template + function[cb_writer_finish_internal] writer_pre_finish + function[cb_writer_finish_internal] writer_post_finish + ExistingDataBehavior existing_data_behavior + c_bool create_dir + uint32_t max_open_files + uint64_t max_rows_per_file + uint64_t min_rows_per_group + uint64_t max_rows_per_group + + cdef cppclass CFileSystemDataset \ + "arrow::dataset::FileSystemDataset"(CDataset): + @staticmethod + CResult[shared_ptr[CDataset]] Make( + shared_ptr[CSchema] schema, + CExpression source_partition, + shared_ptr[CFileFormat] format, + shared_ptr[CFileSystem] filesystem, + vector[shared_ptr[CFileFragment]] fragments) + + @staticmethod + CStatus Write( + const CFileSystemDatasetWriteOptions& write_options, + shared_ptr[CScanner] scanner) + + c_string type() + vector[c_string] files() + const shared_ptr[CFileFormat]& format() const + const shared_ptr[CFileSystem]& filesystem() const + const shared_ptr[CPartitioning]& partitioning() const + + cdef cppclass CIpcFileWriteOptions \ + "arrow::dataset::IpcFileWriteOptions"(CFileWriteOptions): + shared_ptr[CIpcWriteOptions] options + + cdef cppclass CIpcFileFormat "arrow::dataset::IpcFileFormat"( + CFileFormat): + pass + + cdef cppclass COrcFileFormat "arrow::dataset::OrcFileFormat"( + CFileFormat): + pass + + cdef cppclass CCsvFileWriteOptions \ + "arrow::dataset::CsvFileWriteOptions"(CFileWriteOptions): + shared_ptr[CCSVWriteOptions] write_options + CMemoryPool* pool + + cdef cppclass CCsvFileFormat "arrow::dataset::CsvFileFormat"( + CFileFormat): + CCSVParseOptions parse_options + + cdef cppclass CCsvFragmentScanOptions \ + "arrow::dataset::CsvFragmentScanOptions"(CFragmentScanOptions): + CCSVConvertOptions convert_options + CCSVReadOptions read_options + function[StreamWrapFunc] stream_transform_func + + cdef cppclass CJsonFileFormat "arrow::dataset::JsonFileFormat"(CFileFormat): + pass + + cdef cppclass CJsonFragmentScanOptions "arrow::dataset::JsonFragmentScanOptions"(CFragmentScanOptions): + CJSONParseOptions parse_options + CJSONReadOptions read_options + + cdef struct CPartitionPathFormat "arrow::dataset::PartitionPathFormat": + c_string directory + c_string filename + + cdef cppclass CPartitioning "arrow::dataset::Partitioning": + c_string type_name() const + CResult[CExpression] Parse(const c_string & path) const + CResult[CPartitionPathFormat] Format(const CExpression & expr) const + const shared_ptr[CSchema] & schema() + c_bool Equals(const CPartitioning& other) const + + cdef cppclass CSegmentEncoding" arrow::dataset::SegmentEncoding": + bint operator==(CSegmentEncoding) + + CSegmentEncoding CSegmentEncoding_None\ + " arrow::dataset::SegmentEncoding::None" + CSegmentEncoding CSegmentEncoding_Uri\ + " arrow::dataset::SegmentEncoding::Uri" + + cdef cppclass CKeyValuePartitioningOptions \ + "arrow::dataset::KeyValuePartitioningOptions": + CSegmentEncoding segment_encoding + + cdef cppclass CHivePartitioningOptions \ + "arrow::dataset::HivePartitioningOptions": + CSegmentEncoding segment_encoding + c_string null_fallback + + cdef cppclass CPartitioningFactoryOptions \ + "arrow::dataset::PartitioningFactoryOptions": + c_bool infer_dictionary + shared_ptr[CSchema] schema + CSegmentEncoding segment_encoding + + cdef cppclass CHivePartitioningFactoryOptions \ + "arrow::dataset::HivePartitioningFactoryOptions": + c_bool infer_dictionary + c_string null_fallback + shared_ptr[CSchema] schema + CSegmentEncoding segment_encoding + + cdef cppclass CPartitioningFactory "arrow::dataset::PartitioningFactory": + c_string type_name() const + + cdef cppclass CKeyValuePartitioning \ + "arrow::dataset::KeyValuePartitioning"(CPartitioning): + CKeyValuePartitioning(shared_ptr[CSchema] schema, + vector[shared_ptr[CArray]] dictionaries, + CKeyValuePartitioningOptions options) + + vector[shared_ptr[CArray]] dictionaries() const + CSegmentEncoding segment_encoding() + + cdef cppclass CDirectoryPartitioning \ + "arrow::dataset::DirectoryPartitioning"(CPartitioning): + CDirectoryPartitioning(shared_ptr[CSchema] schema, + vector[shared_ptr[CArray]] dictionaries) + + @staticmethod + shared_ptr[CPartitioningFactory] MakeFactory( + vector[c_string] field_names, CPartitioningFactoryOptions) + + vector[shared_ptr[CArray]] dictionaries() const + + cdef cppclass CHivePartitioning \ + "arrow::dataset::HivePartitioning"(CPartitioning): + CHivePartitioning(shared_ptr[CSchema] schema, + vector[shared_ptr[CArray]] dictionaries, + CHivePartitioningOptions options) + + @staticmethod + shared_ptr[CPartitioningFactory] MakeFactory( + CHivePartitioningFactoryOptions) + + vector[shared_ptr[CArray]] dictionaries() const + c_string null_fallback() const + + cdef cppclass CFilenamePartitioning \ + "arrow::dataset::FilenamePartitioning"(CPartitioning): + CFilenamePartitioning(shared_ptr[CSchema] schema, + vector[shared_ptr[CArray]] dictionaries) + + @staticmethod + shared_ptr[CPartitioningFactory] MakeFactory( + vector[c_string] field_names, CPartitioningFactoryOptions) + + vector[shared_ptr[CArray]] dictionaries() const + + cdef cppclass CPartitioningOrFactory \ + "arrow::dataset::PartitioningOrFactory": + CPartitioningOrFactory(shared_ptr[CPartitioning]) + CPartitioningOrFactory(shared_ptr[CPartitioningFactory]) + CPartitioningOrFactory & operator = (shared_ptr[CPartitioning]) + CPartitioningOrFactory & operator = ( + shared_ptr[CPartitioningFactory]) + shared_ptr[CPartitioning] partitioning() const + shared_ptr[CPartitioningFactory] factory() const + + cdef cppclass CFileSystemFactoryOptions \ + "arrow::dataset::FileSystemFactoryOptions": + CPartitioningOrFactory partitioning + c_string partition_base_dir + c_bool exclude_invalid_files + vector[c_string] selector_ignore_prefixes + + cdef cppclass CFileSystemDatasetFactory \ + "arrow::dataset::FileSystemDatasetFactory"( + CDatasetFactory): + @staticmethod + CResult[shared_ptr[CDatasetFactory]] MakeFromPaths "Make"( + shared_ptr[CFileSystem] filesystem, + vector[c_string] paths, + shared_ptr[CFileFormat] format, + CFileSystemFactoryOptions options + ) + + @staticmethod + CResult[shared_ptr[CDatasetFactory]] MakeFromSelector "Make"( + shared_ptr[CFileSystem] filesystem, + CFileSelector, + shared_ptr[CFileFormat] format, + CFileSystemFactoryOptions options + ) + + @staticmethod + CResult[shared_ptr[CDatasetFactory]] MakeFromFileInfos "Make"( + shared_ptr[CFileSystem] filesystem, + vector[CFileInfo] files, + shared_ptr[CFileFormat] format, + CFileSystemFactoryOptions options + ) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_dataset_parquet.pxd b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_dataset_parquet.pxd new file mode 100644 index 0000000000000000000000000000000000000000..e6f6f9ed4bf979993956b42a508fb57c53539c2f --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_dataset_parquet.pxd @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# distutils: language = c++ + +from pyarrow.includes.libarrow_dataset cimport * +from pyarrow.includes.libparquet_encryption cimport * + +from pyarrow._parquet cimport * + + +cdef extern from "arrow/dataset/parquet_encryption_config.h" namespace "arrow::dataset" nogil: + cdef cppclass CParquetEncryptionConfig "arrow::dataset::ParquetEncryptionConfig": + shared_ptr[CCryptoFactory] crypto_factory + shared_ptr[CKmsConnectionConfig] kms_connection_config + shared_ptr[CEncryptionConfiguration] encryption_config + + cdef cppclass CParquetDecryptionConfig "arrow::dataset::ParquetDecryptionConfig": + shared_ptr[CCryptoFactory] crypto_factory + shared_ptr[CKmsConnectionConfig] kms_connection_config + shared_ptr[CDecryptionConfiguration] decryption_config + + +cdef extern from "arrow/dataset/api.h" namespace "arrow::dataset" nogil: + + cdef cppclass CParquetFileWriter \ + "arrow::dataset::ParquetFileWriter"(CFileWriter): + const shared_ptr[FileWriter]& parquet_writer() const + + cdef cppclass CParquetFileWriteOptions \ + "arrow::dataset::ParquetFileWriteOptions"(CFileWriteOptions): + shared_ptr[WriterProperties] writer_properties + shared_ptr[ArrowWriterProperties] arrow_writer_properties + shared_ptr[CParquetEncryptionConfig] parquet_encryption_config + + cdef cppclass CParquetFileFragment "arrow::dataset::ParquetFileFragment"( + CFileFragment): + const vector[int]& row_groups() const + shared_ptr[CFileMetaData] metadata() const + CResult[vector[shared_ptr[CFragment]]] SplitByRowGroup( + CExpression predicate) + CResult[shared_ptr[CFragment]] SubsetWithFilter "Subset"( + CExpression predicate) + CResult[shared_ptr[CFragment]] SubsetWithIds "Subset"( + vector[int] row_group_ids) + CStatus EnsureCompleteMetadata() + + cdef cppclass CParquetFileFormatReaderOptions \ + "arrow::dataset::ParquetFileFormat::ReaderOptions": + unordered_set[c_string] dict_columns + TimeUnit coerce_int96_timestamp_unit + Type binary_type + Type list_type + + cdef cppclass CParquetFileFormat "arrow::dataset::ParquetFileFormat"( + CFileFormat): + CParquetFileFormatReaderOptions reader_options + CResult[shared_ptr[CFileFragment]] MakeFragment( + CFileSource source, + CExpression partition_expression, + shared_ptr[CSchema] physical_schema, + vector[int] row_groups) + + cdef cppclass CParquetFragmentScanOptions \ + "arrow::dataset::ParquetFragmentScanOptions"(CFragmentScanOptions): + shared_ptr[CReaderProperties] reader_properties + shared_ptr[ArrowReaderProperties] arrow_reader_properties + shared_ptr[CParquetDecryptionConfig] parquet_decryption_config + + cdef cppclass CParquetFactoryOptions \ + "arrow::dataset::ParquetFactoryOptions": + CPartitioningOrFactory partitioning + c_string partition_base_dir + c_bool validate_column_chunk_paths + + cdef cppclass CParquetDatasetFactory \ + "arrow::dataset::ParquetDatasetFactory"(CDatasetFactory): + @staticmethod + CResult[shared_ptr[CDatasetFactory]] MakeFromMetaDataPath "Make"( + const c_string& metadata_path, + shared_ptr[CFileSystem] filesystem, + shared_ptr[CParquetFileFormat] format, + CParquetFactoryOptions options + ) + + @staticmethod + CResult[shared_ptr[CDatasetFactory]] MakeFromMetaDataSource "Make"( + const CFileSource& metadata_path, + const c_string& base_path, + shared_ptr[CFileSystem] filesystem, + shared_ptr[CParquetFileFormat] format, + CParquetFactoryOptions options + ) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_feather.pxd b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_feather.pxd new file mode 100644 index 0000000000000000000000000000000000000000..fb80e0e78d9073360df673d2257706a4d324d498 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_feather.pxd @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# distutils: language = c++ + +from pyarrow.includes.libarrow cimport (CCompressionType, CStatus, CTable, + COutputStream, CResult, shared_ptr, + vector, CRandomAccessFile, CSchema, + c_string, CIpcReadOptions) + + +cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: + int kFeatherV1Version" arrow::ipc::feather::kFeatherV1Version" + int kFeatherV2Version" arrow::ipc::feather::kFeatherV2Version" + + cdef cppclass CFeatherProperties" arrow::ipc::feather::WriteProperties": + int version + int chunksize + CCompressionType compression + int compression_level + + CStatus WriteFeather" arrow::ipc::feather::WriteTable" \ + (const CTable& table, COutputStream* out, + CFeatherProperties properties) + + cdef cppclass CFeatherReader" arrow::ipc::feather::Reader": + @staticmethod + CResult[shared_ptr[CFeatherReader]] Open( + const shared_ptr[CRandomAccessFile]& file, + const CIpcReadOptions& options) + int version() + shared_ptr[CSchema] schema() + + CStatus Read(shared_ptr[CTable]* out) + CStatus Read(const vector[int] indices, shared_ptr[CTable]* out) + CStatus Read(const vector[c_string] names, shared_ptr[CTable]* out) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_flight.pxd b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_flight.pxd new file mode 100644 index 0000000000000000000000000000000000000000..97d56ad83f28f0a73feb1fa7354304ae7632da90 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_flight.pxd @@ -0,0 +1,621 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# distutils: language = c++ + +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport * +from pyarrow.includes.libarrow_python cimport CTimePoint + +from libcpp.map cimport multimap + + +cdef extern from "arrow/flight/api.h" namespace "arrow" nogil: + cdef char* CTracingServerMiddlewareName\ + " arrow::flight::TracingServerMiddleware::kMiddlewareName" + + cdef cppclass CActionType" arrow::flight::ActionType": + c_string type + c_string description + bint operator==(CActionType) + CResult[c_string] SerializeToString() + + @staticmethod + CResult[CActionType] Deserialize(const c_string& serialized) + + cdef cppclass CAction" arrow::flight::Action": + c_string type + shared_ptr[CBuffer] body + bint operator==(CAction) + CResult[c_string] SerializeToString() + c_string ToString() + + @staticmethod + CResult[CAction] Deserialize(const c_string& serialized) + + cdef cppclass CFlightResult" arrow::flight::Result": + CFlightResult() + CFlightResult(CFlightResult) + shared_ptr[CBuffer] body + bint operator==(CFlightResult) + CResult[c_string] SerializeToString() + c_string ToString() + + @staticmethod + CResult[CFlightResult] Deserialize(const c_string& serialized) + + cdef cppclass CBasicAuth" arrow::flight::BasicAuth": + CBasicAuth() + CBasicAuth(CBuffer) + CBasicAuth(CBasicAuth) + c_string username + c_string password + bint operator==(CBasicAuth) + CResult[c_string] SerializeToString() + c_string ToString() + + @staticmethod + CResult[CBasicAuth] Deserialize(const c_string& serialized) + + cdef cppclass CResultStream" arrow::flight::ResultStream": + CResult[unique_ptr[CFlightResult]] Next() + + cdef cppclass CDescriptorType \ + " arrow::flight::FlightDescriptor::DescriptorType": + bint operator==(CDescriptorType) + + CDescriptorType CDescriptorTypeUnknown\ + " arrow::flight::FlightDescriptor::UNKNOWN" + CDescriptorType CDescriptorTypePath\ + " arrow::flight::FlightDescriptor::PATH" + CDescriptorType CDescriptorTypeCmd\ + " arrow::flight::FlightDescriptor::CMD" + + cdef cppclass CFlightDescriptor" arrow::flight::FlightDescriptor": + CDescriptorType type + c_string cmd + vector[c_string] path + bint operator==(CFlightDescriptor) + CResult[c_string] SerializeToString() + c_string ToString() + + @staticmethod + CResult[CFlightDescriptor] Deserialize(const c_string& serialized) + + cdef cppclass CTicket" arrow::flight::Ticket": + CTicket() + c_string ticket + bint operator==(CTicket) + CResult[c_string] SerializeToString() + c_string ToString() + + @staticmethod + CResult[CTicket] Deserialize(const c_string& serialized) + + cdef cppclass CCriteria" arrow::flight::Criteria": + CCriteria() + c_string expression + bint operator==(CCriteria) + CResult[c_string] SerializeToString() + + @staticmethod + CResult[CCriteria] Deserialize(const c_string& serialized) + + cdef cppclass CLocation" arrow::flight::Location": + CLocation() + c_string ToString() + c_bool Equals(const CLocation& other) + + @staticmethod + CResult[CLocation] Parse(const c_string& uri_string) + + @staticmethod + CResult[CLocation] ForGrpcTcp(const c_string& host, int port) + + @staticmethod + CResult[CLocation] ForGrpcTls(const c_string& host, int port) + + @staticmethod + CResult[CLocation] ForGrpcUnix(const c_string& path) + + cdef cppclass CFlightEndpoint" arrow::flight::FlightEndpoint": + CFlightEndpoint() + + CTicket ticket + vector[CLocation] locations + optional[CTimePoint] expiration_time + c_string app_metadata + + bint operator==(CFlightEndpoint) + CResult[c_string] SerializeToString() + c_string ToString() + + @staticmethod + CResult[CFlightEndpoint] Deserialize(const c_string& serialized) + + cdef cppclass CFlightInfo" arrow::flight::FlightInfo": + CFlightInfo(CFlightInfo info) + int64_t total_records() + int64_t total_bytes() + c_bool ordered() + c_string app_metadata() + CResult[shared_ptr[CSchema]] GetSchema(CDictionaryMemo* memo) + CFlightDescriptor& descriptor() + const vector[CFlightEndpoint]& endpoints() + CResult[c_string] SerializeToString() + c_string ToString() + bint operator==(CFlightInfo) + + @staticmethod + CResult[unique_ptr[CFlightInfo]] Deserialize( + const c_string& serialized) + + cdef cppclass CSchemaResult" arrow::flight::SchemaResult": + CSchemaResult() + CSchemaResult(CSchemaResult result) + CResult[shared_ptr[CSchema]] GetSchema(CDictionaryMemo* memo) + bint operator==(CSchemaResult) + CResult[c_string] SerializeToString() + c_string ToString() + + @staticmethod + CResult[CSchemaResult] Deserialize(const c_string& serialized) + + cdef cppclass CFlightListing" arrow::flight::FlightListing": + CResult[unique_ptr[CFlightInfo]] Next() + + cdef cppclass CSimpleFlightListing" arrow::flight::SimpleFlightListing": + # This doesn't work with Cython >= 3 + # CSimpleFlightListing(vector[CFlightInfo]&& info) + CSimpleFlightListing(const vector[CFlightInfo]& info) + + cdef cppclass CFlightPayload" arrow::flight::FlightPayload": + shared_ptr[CBuffer] descriptor + shared_ptr[CBuffer] app_metadata + CIpcPayload ipc_message + + cdef cppclass CFlightDataStream" arrow::flight::FlightDataStream": + shared_ptr[CSchema] schema() + CResult[CFlightPayload] Next() + + cdef cppclass CFlightStreamChunk" arrow::flight::FlightStreamChunk": + CFlightStreamChunk() + shared_ptr[CRecordBatch] data + shared_ptr[CBuffer] app_metadata + + cdef cppclass CMetadataRecordBatchReader \ + " arrow::flight::MetadataRecordBatchReader": + CResult[shared_ptr[CSchema]] GetSchema() + CResult[CFlightStreamChunk] Next() + CResult[shared_ptr[CTable]] ToTable() + CIpcReadStats stats() const + + CResult[shared_ptr[CRecordBatchReader]] MakeRecordBatchReader\ + " arrow::flight::MakeRecordBatchReader"( + shared_ptr[CMetadataRecordBatchReader]) + + cdef cppclass CMetadataRecordBatchWriter \ + " arrow::flight::MetadataRecordBatchWriter"(CRecordBatchWriter): + CStatus Begin(shared_ptr[CSchema] schema, + const CIpcWriteOptions& options) + CStatus WriteMetadata(shared_ptr[CBuffer] app_metadata) + CStatus WriteWithMetadata(const CRecordBatch& batch, + shared_ptr[CBuffer] app_metadata) + + cdef cppclass CFlightStreamReader \ + " arrow::flight::FlightStreamReader"(CMetadataRecordBatchReader): + void Cancel() + CResult[shared_ptr[CTable]] ToTableWithStopToken" ToTable"\ + (const CStopToken& stop_token) + + cdef cppclass CFlightMessageReader \ + " arrow::flight::FlightMessageReader"(CMetadataRecordBatchReader): + CFlightDescriptor& descriptor() + + cdef cppclass CFlightMessageWriter \ + " arrow::flight::FlightMessageWriter"(CMetadataRecordBatchWriter): + pass + + cdef cppclass CFlightStreamWriter \ + " arrow::flight::FlightStreamWriter"(CMetadataRecordBatchWriter): + CStatus DoneWriting() + + cdef cppclass CRecordBatchStream \ + " arrow::flight::RecordBatchStream"(CFlightDataStream): + CRecordBatchStream(shared_ptr[CRecordBatchReader]& reader, + const CIpcWriteOptions& options) + + cdef cppclass CFlightMetadataReader" arrow::flight::FlightMetadataReader": + CStatus ReadMetadata(shared_ptr[CBuffer]* out) + + cdef cppclass CFlightMetadataWriter" arrow::flight::FlightMetadataWriter": + CStatus WriteMetadata(const CBuffer& message) + + cdef cppclass CServerAuthReader" arrow::flight::ServerAuthReader": + CStatus Read(c_string* token) + + cdef cppclass CServerAuthSender" arrow::flight::ServerAuthSender": + CStatus Write(c_string& token) + + cdef cppclass CClientAuthReader" arrow::flight::ClientAuthReader": + CStatus Read(c_string* token) + + cdef cppclass CClientAuthSender" arrow::flight::ClientAuthSender": + CStatus Write(c_string& token) + + cdef cppclass CServerAuthHandler" arrow::flight::ServerAuthHandler": + pass + + cdef cppclass CClientAuthHandler" arrow::flight::ClientAuthHandler": + pass + + cdef cppclass CServerCallContext" arrow::flight::ServerCallContext": + c_string& peer_identity() + c_string& peer() + c_bool is_cancelled() + void AddHeader(const c_string& key, const c_string& value) + void AddTrailer(const c_string& key, const c_string& value) + CServerMiddleware* GetMiddleware(const c_string& key) + + cdef cppclass CTimeoutDuration" arrow::flight::TimeoutDuration": + CTimeoutDuration(double) + double count() + + cdef cppclass CFlightCallOptions" arrow::flight::FlightCallOptions": + CFlightCallOptions() + CTimeoutDuration timeout + CIpcWriteOptions write_options + CIpcReadOptions read_options + vector[pair[c_string, c_string]] headers + CStopToken stop_token + + cdef cppclass CCertKeyPair" arrow::flight::CertKeyPair": + CCertKeyPair() + c_string pem_cert + c_string pem_key + + cdef cppclass CFlightMethod" arrow::flight::FlightMethod": + bint operator==(CFlightMethod) + + CFlightMethod CFlightMethodInvalid\ + " arrow::flight::FlightMethod::Invalid" + CFlightMethod CFlightMethodHandshake\ + " arrow::flight::FlightMethod::Handshake" + CFlightMethod CFlightMethodListFlights\ + " arrow::flight::FlightMethod::ListFlights" + CFlightMethod CFlightMethodGetFlightInfo\ + " arrow::flight::FlightMethod::GetFlightInfo" + CFlightMethod CFlightMethodGetSchema\ + " arrow::flight::FlightMethod::GetSchema" + CFlightMethod CFlightMethodDoGet\ + " arrow::flight::FlightMethod::DoGet" + CFlightMethod CFlightMethodDoPut\ + " arrow::flight::FlightMethod::DoPut" + CFlightMethod CFlightMethodDoAction\ + " arrow::flight::FlightMethod::DoAction" + CFlightMethod CFlightMethodListActions\ + " arrow::flight::FlightMethod::ListActions" + CFlightMethod CFlightMethodDoExchange\ + " arrow::flight::FlightMethod::DoExchange" + + cdef cppclass CCallInfo" arrow::flight::CallInfo": + CFlightMethod method + + ctypedef multimap[cpp_string_view, cpp_string_view] CCallHeaders\ + " arrow::flight::CallHeaders" + + cdef cppclass CAddCallHeaders" arrow::flight::AddCallHeaders": + void AddHeader(const c_string& key, const c_string& value) + + cdef cppclass CServerMiddleware" arrow::flight::ServerMiddleware": + c_string name() + + cdef cppclass CServerMiddlewareFactory\ + " arrow::flight::ServerMiddlewareFactory": + pass + + cdef cppclass CClientMiddleware" arrow::flight::ClientMiddleware": + pass + + cdef cppclass CClientMiddlewareFactory\ + " arrow::flight::ClientMiddlewareFactory": + pass + + cpdef cppclass CTracingServerMiddlewareTraceKey\ + " arrow::flight::TracingServerMiddleware::TraceKey": + CTracingServerMiddlewareTraceKey() + c_string key + c_string value + + cdef cppclass CTracingServerMiddleware\ + " arrow::flight::TracingServerMiddleware"(CServerMiddleware): + vector[CTracingServerMiddlewareTraceKey] GetTraceContext() + + cdef shared_ptr[CServerMiddlewareFactory] \ + MakeTracingServerMiddlewareFactory\ + " arrow::flight::MakeTracingServerMiddlewareFactory"() + + cdef cppclass CFlightServerOptions" arrow::flight::FlightServerOptions": + CFlightServerOptions(const CLocation& location) + CLocation location + unique_ptr[CServerAuthHandler] auth_handler + vector[CCertKeyPair] tls_certificates + c_bool verify_client + c_string root_certificates + vector[pair[c_string, shared_ptr[CServerMiddlewareFactory]]] middleware + + cdef cppclass CFlightClientOptions" arrow::flight::FlightClientOptions": + c_string tls_root_certs + c_string cert_chain + c_string private_key + c_string override_hostname + vector[shared_ptr[CClientMiddlewareFactory]] middleware + int64_t write_size_limit_bytes + vector[pair[c_string, CIntStringVariant]] generic_options + c_bool disable_server_verification + + @staticmethod + CFlightClientOptions Defaults() + + cdef cppclass CDoPutResult" arrow::flight::FlightClient::DoPutResult": + unique_ptr[CFlightStreamWriter] writer + unique_ptr[CFlightMetadataReader] reader + + cdef cppclass CDoExchangeResult" arrow::flight::FlightClient::DoExchangeResult": + unique_ptr[CFlightStreamWriter] writer + unique_ptr[CFlightStreamReader] reader + + cdef cppclass CFlightClient" arrow::flight::FlightClient": + @staticmethod + CResult[unique_ptr[CFlightClient]] Connect(const CLocation& location, + const CFlightClientOptions& options) + + c_bool supports_async() + CStatus CheckAsyncSupport() + + CStatus Authenticate(CFlightCallOptions& options, + unique_ptr[CClientAuthHandler] auth_handler) + + CResult[pair[c_string, c_string]] AuthenticateBasicToken( + CFlightCallOptions& options, + const c_string& username, + const c_string& password) + + CResult[unique_ptr[CResultStream]] DoAction(CFlightCallOptions& options, CAction& action) + CResult[vector[CActionType]] ListActions(CFlightCallOptions& options) + + CResult[unique_ptr[CFlightListing]] ListFlights(CFlightCallOptions& options, CCriteria criteria) + CResult[unique_ptr[CFlightInfo]] GetFlightInfo(CFlightCallOptions& options, + CFlightDescriptor& descriptor) + CFuture[CFlightInfo] GetFlightInfoAsync(CFlightCallOptions& options, + CFlightDescriptor& descriptor) + CResult[unique_ptr[CSchemaResult]] GetSchema(CFlightCallOptions& options, + CFlightDescriptor& descriptor) + CResult[unique_ptr[CFlightStreamReader]] DoGet(CFlightCallOptions& options, CTicket& ticket) + CResult[CDoPutResult] DoPut(CFlightCallOptions& options, + CFlightDescriptor& descriptor, + shared_ptr[CSchema]& schema) + CResult[CDoExchangeResult] DoExchange(CFlightCallOptions& options, + CFlightDescriptor& descriptor) + CStatus Close() + + cdef cppclass CFlightStatusCode" arrow::flight::FlightStatusCode": + bint operator==(CFlightStatusCode) + + CFlightStatusCode CFlightStatusInternal \ + " arrow::flight::FlightStatusCode::Internal" + CFlightStatusCode CFlightStatusTimedOut \ + " arrow::flight::FlightStatusCode::TimedOut" + CFlightStatusCode CFlightStatusCancelled \ + " arrow::flight::FlightStatusCode::Cancelled" + CFlightStatusCode CFlightStatusUnauthenticated \ + " arrow::flight::FlightStatusCode::Unauthenticated" + CFlightStatusCode CFlightStatusUnauthorized \ + " arrow::flight::FlightStatusCode::Unauthorized" + CFlightStatusCode CFlightStatusUnavailable \ + " arrow::flight::FlightStatusCode::Unavailable" + CFlightStatusCode CFlightStatusFailed \ + " arrow::flight::FlightStatusCode::Failed" + + cdef cppclass FlightStatusDetail" arrow::flight::FlightStatusDetail": + CFlightStatusCode code() + c_string extra_info() + + @staticmethod + shared_ptr[FlightStatusDetail] UnwrapStatus(const CStatus& status) + + cdef cppclass FlightWriteSizeStatusDetail\ + " arrow::flight::FlightWriteSizeStatusDetail": + int64_t limit() + int64_t actual() + + @staticmethod + shared_ptr[FlightWriteSizeStatusDetail] UnwrapStatus( + const CStatus& status) + + cdef CStatus MakeFlightError" arrow::flight::MakeFlightError" \ + (CFlightStatusCode code, const c_string& message) + + cdef CStatus MakeFlightError" arrow::flight::MakeFlightError" \ + (CFlightStatusCode code, + const c_string& message, + const c_string& extra_info) + +# Callbacks for implementing Flight servers +# Use typedef to emulate syntax for std::function +ctypedef CStatus cb_list_flights(object, const CServerCallContext&, + const CCriteria*, + unique_ptr[CFlightListing]*) +ctypedef CStatus cb_get_flight_info(object, const CServerCallContext&, + const CFlightDescriptor&, + unique_ptr[CFlightInfo]*) +ctypedef CStatus cb_get_schema(object, const CServerCallContext&, + const CFlightDescriptor&, + unique_ptr[CSchemaResult]*) +ctypedef CStatus cb_do_put(object, const CServerCallContext&, + unique_ptr[CFlightMessageReader], + unique_ptr[CFlightMetadataWriter]) +ctypedef CStatus cb_do_get(object, const CServerCallContext&, + const CTicket&, + unique_ptr[CFlightDataStream]*) +ctypedef CStatus cb_do_exchange(object, const CServerCallContext&, + unique_ptr[CFlightMessageReader], + unique_ptr[CFlightMessageWriter]) +ctypedef CStatus cb_do_action(object, const CServerCallContext&, + const CAction&, + unique_ptr[CResultStream]*) +ctypedef CStatus cb_list_actions(object, const CServerCallContext&, + vector[CActionType]*) +ctypedef CStatus cb_result_next(object, unique_ptr[CFlightResult]*) +ctypedef CStatus cb_data_stream_next(object, CFlightPayload*) +ctypedef CStatus cb_server_authenticate(object, CServerAuthSender*, + CServerAuthReader*) +ctypedef CStatus cb_is_valid(object, const c_string&, c_string*) +ctypedef CStatus cb_client_authenticate(object, CClientAuthSender*, + CClientAuthReader*) +ctypedef CStatus cb_get_token(object, c_string*) + +ctypedef CStatus cb_middleware_sending_headers(object, CAddCallHeaders*) +ctypedef CStatus cb_middleware_call_completed(object, const CStatus&) +ctypedef CStatus cb_client_middleware_received_headers( + object, const CCallHeaders&) +ctypedef CStatus cb_server_middleware_start_call( + object, + const CCallInfo&, + const CCallHeaders&, + shared_ptr[CServerMiddleware]*) +ctypedef CStatus cb_client_middleware_start_call( + object, + const CCallInfo&, + unique_ptr[CClientMiddleware]*) + +cdef extern from "arrow/python/flight.h" namespace "arrow::py::flight" nogil: + cdef char* CPyServerMiddlewareName\ + " arrow::py::flight::kPyServerMiddlewareName" + + cdef cppclass PyFlightServerVtable: + PyFlightServerVtable() + function[cb_list_flights] list_flights + function[cb_get_flight_info] get_flight_info + function[cb_get_schema] get_schema + function[cb_do_put] do_put + function[cb_do_get] do_get + function[cb_do_exchange] do_exchange + function[cb_do_action] do_action + function[cb_list_actions] list_actions + + cdef cppclass PyServerAuthHandlerVtable: + PyServerAuthHandlerVtable() + function[cb_server_authenticate] authenticate + function[cb_is_valid] is_valid + + cdef cppclass PyClientAuthHandlerVtable: + PyClientAuthHandlerVtable() + function[cb_client_authenticate] authenticate + function[cb_get_token] get_token + + cdef cppclass PyFlightServer: + PyFlightServer(object server, PyFlightServerVtable vtable) + + CStatus Init(CFlightServerOptions& options) + int port() + CStatus ServeWithSignals() except * + CStatus Shutdown() + CStatus Wait() + + cdef cppclass PyServerAuthHandler\ + " arrow::py::flight::PyServerAuthHandler"(CServerAuthHandler): + PyServerAuthHandler(object handler, PyServerAuthHandlerVtable vtable) + + cdef cppclass PyClientAuthHandler\ + " arrow::py::flight::PyClientAuthHandler"(CClientAuthHandler): + PyClientAuthHandler(object handler, PyClientAuthHandlerVtable vtable) + + cdef cppclass CPyFlightResultStream\ + " arrow::py::flight::PyFlightResultStream"(CResultStream): + CPyFlightResultStream(object generator, + function[cb_result_next] callback) + + cdef cppclass CPyFlightDataStream\ + " arrow::py::flight::PyFlightDataStream"(CFlightDataStream): + CPyFlightDataStream(object data_source, + unique_ptr[CFlightDataStream] stream) + + cdef cppclass CPyGeneratorFlightDataStream\ + " arrow::py::flight::PyGeneratorFlightDataStream"\ + (CFlightDataStream): + CPyGeneratorFlightDataStream(object generator, + shared_ptr[CSchema] schema, + function[cb_data_stream_next] callback, + const CIpcWriteOptions& options) + + cdef cppclass PyServerMiddlewareVtable\ + " arrow::py::flight::PyServerMiddleware::Vtable": + PyServerMiddlewareVtable() + function[cb_middleware_sending_headers] sending_headers + function[cb_middleware_call_completed] call_completed + + cdef cppclass PyClientMiddlewareVtable\ + " arrow::py::flight::PyClientMiddleware::Vtable": + PyClientMiddlewareVtable() + function[cb_middleware_sending_headers] sending_headers + function[cb_client_middleware_received_headers] received_headers + function[cb_middleware_call_completed] call_completed + + cdef cppclass CPyServerMiddleware\ + " arrow::py::flight::PyServerMiddleware"(CServerMiddleware): + CPyServerMiddleware(object middleware, PyServerMiddlewareVtable vtable) + void* py_object() + + cdef cppclass CPyServerMiddlewareFactory\ + " arrow::py::flight::PyServerMiddlewareFactory"\ + (CServerMiddlewareFactory): + CPyServerMiddlewareFactory( + object factory, + function[cb_server_middleware_start_call] start_call) + + cdef cppclass CPyClientMiddleware\ + " arrow::py::flight::PyClientMiddleware"(CClientMiddleware): + CPyClientMiddleware(object middleware, PyClientMiddlewareVtable vtable) + + cdef cppclass CPyClientMiddlewareFactory\ + " arrow::py::flight::PyClientMiddlewareFactory"\ + (CClientMiddlewareFactory): + CPyClientMiddlewareFactory( + object factory, + function[cb_client_middleware_start_call] start_call) + + cdef CStatus CreateFlightInfo" arrow::py::flight::CreateFlightInfo"( + shared_ptr[CSchema] schema, + CFlightDescriptor& descriptor, + vector[CFlightEndpoint] endpoints, + int64_t total_records, + int64_t total_bytes, + c_bool ordered, + const c_string& app_metadata, + unique_ptr[CFlightInfo]* out) + + cdef CStatus CreateSchemaResult" arrow::py::flight::CreateSchemaResult"( + shared_ptr[CSchema] schema, + unique_ptr[CSchemaResult]* out) + + +cdef extern from "" namespace "std" nogil: + cdef cppclass CIntStringVariant" std::variant": + CIntStringVariant() + CIntStringVariant(int) + CIntStringVariant(c_string) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_fs.pxd b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_fs.pxd new file mode 100644 index 0000000000000000000000000000000000000000..d448f2f66cfd2324fd18acb7a1f3d81ac8b2218c --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_fs.pxd @@ -0,0 +1,369 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# distutils: language = c++ + +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport * +from pyarrow.includes.libarrow_python cimport CTimePoint + +cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil: + + ctypedef enum CFileType "arrow::fs::FileType": + CFileType_NotFound "arrow::fs::FileType::NotFound" + CFileType_Unknown "arrow::fs::FileType::Unknown" + CFileType_File "arrow::fs::FileType::File" + CFileType_Directory "arrow::fs::FileType::Directory" + + cdef cppclass CFileInfo "arrow::fs::FileInfo": + CFileInfo() + CFileInfo(CFileInfo) + CFileInfo& operator=(CFileInfo) + CFileInfo(const CFileInfo&) + CFileInfo& operator=(const CFileInfo&) + + CFileType type() + void set_type(CFileType type) + c_string path() + void set_path(const c_string& path) + c_string base_name() + int64_t size() + void set_size(int64_t size) + c_string extension() + CTimePoint mtime() + void set_mtime(CTimePoint mtime) + + cdef cppclass CFileSelector "arrow::fs::FileSelector": + CFileSelector() + c_string base_dir + c_bool allow_not_found + c_bool recursive + + cdef cppclass CFileLocator "arrow::fs::FileLocator": + shared_ptr[CFileSystem] filesystem + c_string path + + cdef cppclass CFileSystem "arrow::fs::FileSystem": + shared_ptr[CFileSystem] shared_from_this() + c_string type_name() const + CResult[c_string] NormalizePath(c_string path) + CResult[c_string] MakeUri(c_string path) + CResult[CFileInfo] GetFileInfo(const c_string& path) + CResult[vector[CFileInfo]] GetFileInfo( + const vector[c_string]& paths) + CResult[vector[CFileInfo]] GetFileInfo(const CFileSelector& select) + CStatus CreateDir(const c_string& path, c_bool recursive) + CStatus DeleteDir(const c_string& path) + CStatus DeleteDirContents(const c_string& path, c_bool missing_dir_ok) + CStatus DeleteRootDirContents() + CStatus DeleteFile(const c_string& path) + CStatus DeleteFiles(const vector[c_string]& paths) + CStatus Move(const c_string& src, const c_string& dest) + CStatus CopyFile(const c_string& src, const c_string& dest) + CResult[shared_ptr[CInputStream]] OpenInputStream( + const c_string& path) + CResult[shared_ptr[CRandomAccessFile]] OpenInputFile( + const c_string& path) + CResult[shared_ptr[COutputStream]] OpenOutputStream( + const c_string& path, const shared_ptr[const CKeyValueMetadata]&) + CResult[shared_ptr[COutputStream]] OpenAppendStream( + const c_string& path, const shared_ptr[const CKeyValueMetadata]&) + c_bool Equals(const CFileSystem& other) + c_bool Equals(shared_ptr[CFileSystem] other) + + CResult[shared_ptr[CFileSystem]] CFileSystemFromUri \ + "arrow::fs::FileSystemFromUri"(const c_string& uri) + CResult[shared_ptr[CFileSystem]] CFileSystemFromUri \ + "arrow::fs::FileSystemFromUri"(const c_string& uri, c_string* out_path) + CResult[shared_ptr[CFileSystem]] CFileSystemFromUriOrPath \ + "arrow::fs::FileSystemFromUriOrPath"(const c_string& uri, + c_string* out_path) + + cdef cppclass CFileSystemGlobalOptions \ + "arrow::fs::FileSystemGlobalOptions": + c_string tls_ca_file_path + c_string tls_ca_dir_path + + CStatus CFileSystemsInitialize "arrow::fs::Initialize" \ + (const CFileSystemGlobalOptions& options) + + cdef cppclass CSubTreeFileSystem \ + "arrow::fs::SubTreeFileSystem"(CFileSystem): + CSubTreeFileSystem(const c_string& base_path, + shared_ptr[CFileSystem] base_fs) + c_string base_path() + shared_ptr[CFileSystem] base_fs() + + ctypedef enum CS3LogLevel "arrow::fs::S3LogLevel": + CS3LogLevel_Off "arrow::fs::S3LogLevel::Off" + CS3LogLevel_Fatal "arrow::fs::S3LogLevel::Fatal" + CS3LogLevel_Error "arrow::fs::S3LogLevel::Error" + CS3LogLevel_Warn "arrow::fs::S3LogLevel::Warn" + CS3LogLevel_Info "arrow::fs::S3LogLevel::Info" + CS3LogLevel_Debug "arrow::fs::S3LogLevel::Debug" + CS3LogLevel_Trace "arrow::fs::S3LogLevel::Trace" + + cdef struct CS3GlobalOptions "arrow::fs::S3GlobalOptions": + CS3LogLevel log_level + int num_event_loop_threads + + cdef cppclass CS3ProxyOptions "arrow::fs::S3ProxyOptions": + c_string scheme + c_string host + int port + c_string username + c_string password + c_bool Equals(const CS3ProxyOptions& other) + + @staticmethod + CResult[CS3ProxyOptions] FromUriString "FromUri"( + const c_string& uri_string) + + ctypedef enum CS3CredentialsKind "arrow::fs::S3CredentialsKind": + CS3CredentialsKind_Anonymous "arrow::fs::S3CredentialsKind::Anonymous" + CS3CredentialsKind_Default "arrow::fs::S3CredentialsKind::Default" + CS3CredentialsKind_Explicit "arrow::fs::S3CredentialsKind::Explicit" + CS3CredentialsKind_Role "arrow::fs::S3CredentialsKind::Role" + CS3CredentialsKind_WebIdentity \ + "arrow::fs::S3CredentialsKind::WebIdentity" + + cdef cppclass CS3RetryStrategy "arrow::fs::S3RetryStrategy": + @staticmethod + shared_ptr[CS3RetryStrategy] GetAwsDefaultRetryStrategy(int64_t max_attempts) + + @staticmethod + shared_ptr[CS3RetryStrategy] GetAwsStandardRetryStrategy(int64_t max_attempts) + + cdef cppclass CS3Options "arrow::fs::S3Options": + c_string region + double connect_timeout + double request_timeout + c_string endpoint_override + c_string scheme + c_bool background_writes + c_bool allow_delayed_open + c_bool allow_bucket_creation + c_bool allow_bucket_deletion + c_bool check_directory_existence_before_creation + c_bool force_virtual_addressing + c_string tls_ca_file_path + shared_ptr[const CKeyValueMetadata] default_metadata + c_string role_arn + c_string session_name + c_string external_id + int load_frequency + CS3ProxyOptions proxy_options + CS3CredentialsKind credentials_kind + shared_ptr[CS3RetryStrategy] retry_strategy + void ConfigureDefaultCredentials() + void ConfigureAccessKey(const c_string& access_key, + const c_string& secret_key, + const c_string& session_token) + c_string GetAccessKey() + c_string GetSecretKey() + c_string GetSessionToken() + c_bool Equals(const CS3Options& other) + + @staticmethod + CS3Options Defaults() + + @staticmethod + CS3Options Anonymous() + + @staticmethod + CS3Options FromAccessKey(const c_string& access_key, + const c_string& secret_key, + const c_string& session_token) + + @staticmethod + CS3Options FromAssumeRole(const c_string& role_arn, + const c_string& session_name, + const c_string& external_id, + const int load_frequency) + + cdef cppclass CS3FileSystem "arrow::fs::S3FileSystem"(CFileSystem): + @staticmethod + CResult[shared_ptr[CS3FileSystem]] Make(const CS3Options& options) + CS3Options options() + c_string region() + + cdef CStatus CInitializeS3 "arrow::fs::InitializeS3"( + const CS3GlobalOptions& options) + cdef CStatus CEnsureS3Initialized "arrow::fs::EnsureS3Initialized"() + cdef CStatus CFinalizeS3 "arrow::fs::FinalizeS3"() + cdef CStatus CEnsureS3Finalized "arrow::fs::EnsureS3Finalized"() + + cdef CResult[c_string] ResolveS3BucketRegion(const c_string& bucket) + + cdef cppclass CGcsCredentials "arrow::fs::GcsCredentials": + c_bool anonymous() + CTimePoint expiration() + c_string access_token() + c_string target_service_account() + + cdef cppclass CGcsOptions "arrow::fs::GcsOptions": + CGcsCredentials credentials + c_string endpoint_override + c_string scheme + c_string default_bucket_location + optional[c_string] project_id + optional[double] retry_limit_seconds + shared_ptr[const CKeyValueMetadata] default_metadata + c_bool Equals(const CS3Options& other) + + @staticmethod + CGcsOptions Defaults() + + @staticmethod + CGcsOptions Anonymous() + + @staticmethod + CGcsOptions FromAccessToken(const c_string& access_token, + CTimePoint expiration) + + @staticmethod + CGcsOptions FromImpersonatedServiceAccount(const CGcsCredentials& base_credentials, + c_string& target_service_account) + + cdef cppclass CGcsFileSystem "arrow::fs::GcsFileSystem": + @staticmethod + CResult[shared_ptr[CGcsFileSystem]] Make(const CGcsOptions& options) + CGcsOptions options() + + cdef cppclass CAzureOptions "arrow::fs::AzureOptions": + c_string account_name + c_string blob_storage_authority + c_string dfs_storage_authority + c_string blob_storage_scheme + c_string dfs_storage_scheme + + c_bool Equals(const CAzureOptions& other) + CStatus ConfigureDefaultCredential() + CStatus ConfigureAccountKeyCredential(c_string account_key) + CStatus ConfigureSASCredential(c_string sas_token) + CStatus ConfigureManagedIdentityCredential(c_string client_id) + CStatus ConfigureClientSecretCredential(c_string tenant_id, + c_string client_id, + c_string client_secret) + c_string SasToken() + c_string AccountKey() + c_string TenantId() + c_string ClientId() + c_string ClientSecret() + + cdef cppclass CAzureFileSystem "arrow::fs::AzureFileSystem": + @staticmethod + CResult[shared_ptr[CAzureFileSystem]] Make(const CAzureOptions& options) + CAzureOptions options() + + cdef cppclass CHdfsOptions "arrow::fs::HdfsOptions": + HdfsConnectionConfig connection_config + int32_t buffer_size + int16_t replication + int64_t default_block_size + + @staticmethod + CResult[CHdfsOptions] FromUriString "FromUri"( + const c_string& uri_string) + void ConfigureEndPoint(c_string host, int port) + void ConfigureDriver(c_bool use_hdfs3) + void ConfigureReplication(int16_t replication) + void ConfigureUser(c_string user_name) + void ConfigureBufferSize(int32_t buffer_size) + void ConfigureBlockSize(int64_t default_block_size) + void ConfigureKerberosTicketCachePath(c_string path) + void ConfigureExtraConf(c_string key, c_string value) + + cdef cppclass CHadoopFileSystem "arrow::fs::HadoopFileSystem"(CFileSystem): + @staticmethod + CResult[shared_ptr[CHadoopFileSystem]] Make( + const CHdfsOptions& options) + CHdfsOptions options() + + cdef cppclass CMockFileSystem "arrow::fs::internal::MockFileSystem"( + CFileSystem): + CMockFileSystem(CTimePoint current_time) + + CStatus CCopyFiles "arrow::fs::CopyFiles"( + const vector[CFileLocator]& sources, + const vector[CFileLocator]& destinations, + const CIOContext& io_context, + int64_t chunk_size, c_bool use_threads) + CStatus CCopyFilesWithSelector "arrow::fs::CopyFiles"( + const shared_ptr[CFileSystem]& source_fs, + const CFileSelector& source_sel, + const shared_ptr[CFileSystem]& destination_fs, + const c_string& destination_base_dir, + const CIOContext& io_context, + int64_t chunk_size, c_bool use_threads) + + +# Callbacks for implementing Python filesystems +# Use typedef to emulate syntax for std::function +ctypedef void CallbackGetTypeName(object, c_string*) +ctypedef c_bool CallbackEquals(object, const CFileSystem&) + +ctypedef void CallbackGetFileInfo(object, const c_string&, CFileInfo*) +ctypedef void CallbackGetFileInfoVector(object, const vector[c_string]&, + vector[CFileInfo]*) +ctypedef void CallbackGetFileInfoSelector(object, const CFileSelector&, + vector[CFileInfo]*) +ctypedef void CallbackCreateDir(object, const c_string&, c_bool) +ctypedef void CallbackDeleteDir(object, const c_string&) +ctypedef void CallbackDeleteDirContents(object, const c_string&, c_bool) +ctypedef void CallbackDeleteRootDirContents(object) +ctypedef void CallbackDeleteFile(object, const c_string&) +ctypedef void CallbackMove(object, const c_string&, const c_string&) +ctypedef void CallbackCopyFile(object, const c_string&, const c_string&) + +ctypedef void CallbackOpenInputStream(object, const c_string&, + shared_ptr[CInputStream]*) +ctypedef void CallbackOpenInputFile(object, const c_string&, + shared_ptr[CRandomAccessFile]*) +ctypedef void CallbackOpenOutputStream( + object, const c_string&, const shared_ptr[const CKeyValueMetadata]&, + shared_ptr[COutputStream]*) +ctypedef void CallbackNormalizePath(object, const c_string&, c_string*) + +cdef extern from "arrow/python/filesystem.h" namespace "arrow::py::fs" nogil: + + cdef cppclass CPyFileSystemVtable "arrow::py::fs::PyFileSystemVtable": + PyFileSystemVtable() + function[CallbackGetTypeName] get_type_name + function[CallbackEquals] equals + function[CallbackGetFileInfo] get_file_info + function[CallbackGetFileInfoVector] get_file_info_vector + function[CallbackGetFileInfoSelector] get_file_info_selector + function[CallbackCreateDir] create_dir + function[CallbackDeleteDir] delete_dir + function[CallbackDeleteDirContents] delete_dir_contents + function[CallbackDeleteRootDirContents] delete_root_dir_contents + function[CallbackDeleteFile] delete_file + function[CallbackMove] move + function[CallbackCopyFile] copy_file + function[CallbackOpenInputStream] open_input_stream + function[CallbackOpenInputFile] open_input_file + function[CallbackOpenOutputStream] open_output_stream + function[CallbackOpenOutputStream] open_append_stream + function[CallbackNormalizePath] normalize_path + + cdef cppclass CPyFileSystem "arrow::py::fs::PyFileSystem": + @staticmethod + shared_ptr[CPyFileSystem] Make(object handler, + CPyFileSystemVtable vtable) + + PyObject* handler() diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_python.pxd b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_python.pxd new file mode 100644 index 0000000000000000000000000000000000000000..e48fe9842f5d5aadf8ede474ad7f70f03dc7974d --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_python.pxd @@ -0,0 +1,297 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# distutils: language = c++ + +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport * + + +ctypedef CInvalidRowResult PyInvalidRowCallback(object, + const CCSVInvalidRow&) + + +cdef extern from "arrow/python/csv.h" namespace "arrow::py::csv": + + function[CInvalidRowHandler] MakeInvalidRowHandler( + function[PyInvalidRowCallback], object handler) + + +cdef extern from "arrow/python/api.h" namespace "arrow::py": + # Requires GIL + CResult[shared_ptr[CDataType]] InferArrowType( + object obj, object mask, c_bool pandas_null_sentinels) + + +cdef extern from "arrow/python/api.h" namespace "arrow::py::internal": + object NewMonthDayNanoTupleType() + CResult[PyObject*] MonthDayNanoIntervalArrayToPyList( + const CMonthDayNanoIntervalArray& array) + CResult[PyObject*] MonthDayNanoIntervalScalarToPyObject( + const CMonthDayNanoIntervalScalar& scalar) + + +cdef extern from "arrow/python/arrow_to_pandas.h" namespace "arrow::py::MapConversionType": + cdef enum MapConversionType "arrow::py::MapConversionType": + DEFAULT, + LOSSY, + STRICT_ + + +cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil: + shared_ptr[CDataType] GetPrimitiveType(Type type) + + object PyFloat_FromHalf(uint16_t value) + + cdef cppclass PyConversionOptions: + PyConversionOptions() + + shared_ptr[CDataType] type + int64_t size + CMemoryPool* pool + c_bool from_pandas + c_bool ignore_timezone + c_bool strict + + # TODO Some functions below are not actually "nogil" + + CResult[shared_ptr[CChunkedArray]] ConvertPySequence( + object obj, object mask, const PyConversionOptions& options, + CMemoryPool* pool) + + CResult[shared_ptr[CArray]] Arange(int64_t start, int64_t stop, + int64_t step, CMemoryPool* pool) + + CResult[shared_ptr[CDataType]] NumPyDtypeToArrow(object dtype) + + CStatus NdarrayToArrow(CMemoryPool* pool, object ao, object mo, + c_bool from_pandas, + const shared_ptr[CDataType]& type, + shared_ptr[CChunkedArray]* out) + + CStatus NdarrayToArrow(CMemoryPool* pool, object ao, object mo, + c_bool from_pandas, + const shared_ptr[CDataType]& type, + const CCastOptions& cast_options, + shared_ptr[CChunkedArray]* out) + + CStatus NdarrayToTensor(CMemoryPool* pool, object ao, + const vector[c_string]& dim_names, + shared_ptr[CTensor]* out) + + CStatus TensorToNdarray(const shared_ptr[CTensor]& tensor, object base, + PyObject** out) + + CStatus SparseCOOTensorToNdarray( + const shared_ptr[CSparseCOOTensor]& sparse_tensor, object base, + PyObject** out_data, PyObject** out_coords) + + CStatus SparseCSRMatrixToNdarray( + const shared_ptr[CSparseCSRMatrix]& sparse_tensor, object base, + PyObject** out_data, PyObject** out_indptr, PyObject** out_indices) + + CStatus SparseCSCMatrixToNdarray( + const shared_ptr[CSparseCSCMatrix]& sparse_tensor, object base, + PyObject** out_data, PyObject** out_indptr, PyObject** out_indices) + + CStatus SparseCSFTensorToNdarray( + const shared_ptr[CSparseCSFTensor]& sparse_tensor, object base, + PyObject** out_data, PyObject** out_indptr, PyObject** out_indices) + + CStatus NdarraysToSparseCOOTensor(CMemoryPool* pool, object data_ao, + object coords_ao, + const vector[int64_t]& shape, + const vector[c_string]& dim_names, + shared_ptr[CSparseCOOTensor]* out) + + CStatus NdarraysToSparseCSRMatrix(CMemoryPool* pool, object data_ao, + object indptr_ao, object indices_ao, + const vector[int64_t]& shape, + const vector[c_string]& dim_names, + shared_ptr[CSparseCSRMatrix]* out) + + CStatus NdarraysToSparseCSCMatrix(CMemoryPool* pool, object data_ao, + object indptr_ao, object indices_ao, + const vector[int64_t]& shape, + const vector[c_string]& dim_names, + shared_ptr[CSparseCSCMatrix]* out) + + CStatus NdarraysToSparseCSFTensor(CMemoryPool* pool, object data_ao, + object indptr_ao, object indices_ao, + const vector[int64_t]& shape, + const vector[int64_t]& axis_order, + const vector[c_string]& dim_names, + shared_ptr[CSparseCSFTensor]* out) + + CStatus TensorToSparseCOOTensor(shared_ptr[CTensor], + shared_ptr[CSparseCOOTensor]* out) + + CStatus TensorToSparseCSRMatrix(shared_ptr[CTensor], + shared_ptr[CSparseCSRMatrix]* out) + + CStatus TensorToSparseCSCMatrix(shared_ptr[CTensor], + shared_ptr[CSparseCSCMatrix]* out) + + CStatus TensorToSparseCSFTensor(shared_ptr[CTensor], + shared_ptr[CSparseCSFTensor]* out) + + CStatus ConvertArrayToPandas(const PandasOptions& options, + shared_ptr[CArray] arr, + object py_ref, PyObject** out) + + CStatus ConvertChunkedArrayToPandas(const PandasOptions& options, + shared_ptr[CChunkedArray] arr, + object py_ref, PyObject** out) + + CStatus ConvertTableToPandas(const PandasOptions& options, + shared_ptr[CTable] table, + PyObject** out) + + void c_set_default_memory_pool \ + " arrow::py::set_default_memory_pool"(CMemoryPool* pool)\ + + CMemoryPool* c_get_memory_pool \ + " arrow::py::get_memory_pool"() + + cdef cppclass PyBuffer(CBuffer): + @staticmethod + CResult[shared_ptr[CBuffer]] FromPyObject(object obj) + + cdef cppclass PyForeignBuffer(CBuffer): + @staticmethod + CStatus Make(const uint8_t* data, int64_t size, object base, + shared_ptr[CBuffer]* out) + + cdef cppclass PyReadableFile(CRandomAccessFile): + PyReadableFile(object fo) + + cdef cppclass PyOutputStream(COutputStream): + PyOutputStream(object fo) + + cdef cppclass PandasOptions: + CMemoryPool* pool + c_bool strings_to_categorical + c_bool zero_copy_only + c_bool integer_object_nulls + c_bool date_as_object + c_bool timestamp_as_object + c_bool use_threads + c_bool coerce_temporal_nanoseconds + c_bool ignore_timezone + c_bool deduplicate_objects + c_bool safe_cast + c_bool split_blocks + c_bool self_destruct + MapConversionType maps_as_pydicts + c_bool decode_dictionaries + shared_ptr[const unordered_set[c_string]] categorical_columns + shared_ptr[const unordered_set[c_string]] extension_columns + c_bool to_numpy + + +cdef extern from "arrow/python/api.h" namespace "arrow::py::internal" nogil: + cdef cppclass CTimePoint "arrow::py::internal::TimePoint": + pass + + CTimePoint PyDateTime_to_TimePoint(PyDateTime_DateTime* pydatetime) + int64_t TimePoint_to_ns(CTimePoint val) + CTimePoint TimePoint_from_s(double val) + CTimePoint TimePoint_from_ns(int64_t val) + + CResult[c_string] TzinfoToString(PyObject* pytzinfo) + CResult[PyObject*] StringToTzinfo(c_string) + + +cdef extern from "arrow/python/numpy_init.h" namespace "arrow::py": + int arrow_init_numpy() except -1 + + +cdef extern from "arrow/python/pyarrow.h" namespace "arrow::py": + int import_pyarrow() except -1 + + +cdef extern from "arrow/python/common.h" namespace "arrow::py": + c_bool IsPyError(const CStatus& status) + void RestorePyError(const CStatus& status) except * + + +cdef extern from "arrow/python/common.h" namespace "arrow::py" nogil: + cdef cppclass SharedPtrNoGIL[T](shared_ptr[T]): + # This looks like the only way to satisfy both Cython 2 and Cython 3 + SharedPtrNoGIL& operator=(...) + cdef cppclass UniquePtrNoGIL[T, DELETER=*](unique_ptr[T, DELETER]): + UniquePtrNoGIL& operator=(...) + + +cdef extern from "arrow/python/inference.h" namespace "arrow::py": + c_bool IsPyBool(object o) + c_bool IsPyInt(object o) + c_bool IsPyFloat(object o) + + +cdef extern from "arrow/python/ipc.h" namespace "arrow::py": + cdef cppclass CPyRecordBatchReader" arrow::py::PyRecordBatchReader" \ + (CRecordBatchReader): + @staticmethod + CResult[shared_ptr[CRecordBatchReader]] Make(shared_ptr[CSchema], + object) + + +cdef extern from "arrow/python/ipc.h" namespace "arrow::py" nogil: + cdef cppclass CCastingRecordBatchReader" arrow::py::CastingRecordBatchReader" \ + (CRecordBatchReader): + @staticmethod + CResult[shared_ptr[CRecordBatchReader]] Make(shared_ptr[CRecordBatchReader], + shared_ptr[CSchema]) + + +cdef extern from "arrow/python/extension_type.h" namespace "arrow::py": + cdef cppclass CPyExtensionType \ + " arrow::py::PyExtensionType"(CExtensionType): + @staticmethod + CStatus FromClass(const shared_ptr[CDataType] storage_type, + const c_string extension_name, object typ, + shared_ptr[CExtensionType]* out) + + @staticmethod + CStatus FromInstance(shared_ptr[CDataType] storage_type, + object inst, shared_ptr[CExtensionType]* out) + + object GetInstance() + CStatus SetInstance(object) + + c_string PyExtensionName() + CStatus RegisterPyExtensionType(shared_ptr[CDataType]) + CStatus UnregisterPyExtensionType(c_string type_name) + + +cdef extern from "arrow/python/benchmark.h" namespace "arrow::py::benchmark": + void Benchmark_PandasObjectIsNull(object lst) except * + + +cdef extern from "arrow/python/gdb.h" namespace "arrow::gdb" nogil: + void GdbTestSession "arrow::gdb::TestSession"() + +cdef extern from "arrow/python/config.h" namespace "arrow::py": + cdef cppclass CBuildInfo "arrow::py::BuildInfo": + c_string build_type + + const CBuildInfo& GetBuildInfo "arrow::py::GetBuildInfo"() + +cdef extern from "arrow/python/config.h" namespace "arrow::py::internal": + c_bool IsOpenTelemetryEnabled() + c_bool IsThreadingEnabled() diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_substrait.pxd b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_substrait.pxd new file mode 100644 index 0000000000000000000000000000000000000000..be9034127df94bc2e519ed91fcd97dc7f44d455e --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libarrow_substrait.pxd @@ -0,0 +1,100 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# distutils: language = c++ + +from libcpp.vector cimport vector as std_vector + +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport * +from pyarrow.includes.libarrow_acero cimport * + +ctypedef CResult[CDeclaration] CNamedTableProvider(const std_vector[c_string]&, const CSchema&) + +cdef extern from "arrow/engine/substrait/options.h" namespace "arrow::engine" nogil: + cdef enum ConversionStrictness \ + "arrow::engine::ConversionStrictness": + EXACT_ROUNDTRIP \ + "arrow::engine::ConversionStrictness::EXACT_ROUNDTRIP" + PRESERVE_STRUCTURE \ + "arrow::engine::ConversionStrictness::PRESERVE_STRUCTURE" + BEST_EFFORT \ + "arrow::engine::ConversionStrictness::BEST_EFFORT" + + cdef cppclass CConversionOptions \ + "arrow::engine::ConversionOptions": + CConversionOptions() + ConversionStrictness strictness + function[CNamedTableProvider] named_table_provider + c_bool allow_arrow_extensions + +cdef extern from "arrow/engine/substrait/extension_set.h" \ + namespace "arrow::engine" nogil: + + cdef struct CSubstraitId "arrow::engine::Id": + cpp_string_view uri + cpp_string_view name + + cdef struct CExtensionSetTypeRecord "arrow::engine::ExtensionSet::TypeRecord": + CSubstraitId id + shared_ptr[CDataType] type + + cdef cppclass CExtensionSet "arrow::engine::ExtensionSet": + CExtensionSet() + unordered_map[uint32_t, cpp_string_view]& uris() + CResult[uint32_t] EncodeType(const CDataType&) + CResult[CExtensionSetTypeRecord] DecodeType(uint32_t) + + cdef cppclass ExtensionIdRegistry: + std_vector[c_string] GetSupportedSubstraitFunctions() + + ExtensionIdRegistry* default_extension_id_registry() + +cdef extern from "arrow/engine/substrait/relation.h" namespace "arrow::engine" nogil: + + cdef cppclass CNamedExpression "arrow::engine::NamedExpression": + CExpression expression + c_string name + + cdef cppclass CBoundExpressions "arrow::engine::BoundExpressions": + std_vector[CNamedExpression] named_expressions + shared_ptr[CSchema] schema + +cdef extern from "arrow/engine/substrait/serde.h" namespace "arrow::engine" nogil: + + CResult[shared_ptr[CBuffer]] SerializeExpressions( + const CBoundExpressions& bound_expressions, const CConversionOptions& conversion_options) + + CResult[CBoundExpressions] DeserializeExpressions( + const CBuffer& serialized_expressions) + + CResult[shared_ptr[CBuffer]] SerializeSchema( + const CSchema &schema, CExtensionSet* extension_set, + const CConversionOptions& conversion_options) + + CResult[shared_ptr[CSchema]] DeserializeSchema( + const CBuffer& serialized_schema, const CExtensionSet& extension_set, + const CConversionOptions& conversion_options) + + +cdef extern from "arrow/engine/substrait/util.h" namespace "arrow::engine" nogil: + CResult[shared_ptr[CRecordBatchReader]] ExecuteSerializedPlan( + const CBuffer& substrait_buffer, const ExtensionIdRegistry* registry, + CFunctionRegistry* func_registry, const CConversionOptions& conversion_options, + c_bool use_threads) + + CResult[shared_ptr[CBuffer]] SerializeJsonPlan(const c_string& substrait_json) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libgandiva.pxd b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libgandiva.pxd new file mode 100644 index 0000000000000000000000000000000000000000..b57a708109cb0f989020b5f89891e07341647a3e --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libgandiva.pxd @@ -0,0 +1,298 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# distutils: language = c++ + +from libcpp.string cimport string as c_string +from libcpp.unordered_set cimport unordered_set as c_unordered_set +from libc.stdint cimport int64_t, int32_t, uint8_t, uintptr_t + +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport * + +cdef extern from "gandiva/node.h" namespace "gandiva" nogil: + + cdef cppclass CNode" gandiva::Node": + c_string ToString() + shared_ptr[CDataType] return_type() + + cdef cppclass CGandivaExpression" gandiva::Expression": + c_string ToString() + shared_ptr[CNode] root() + shared_ptr[CField] result() + + ctypedef vector[shared_ptr[CNode]] CNodeVector" gandiva::NodeVector" + + ctypedef vector[shared_ptr[CGandivaExpression]] \ + CExpressionVector" gandiva::ExpressionVector" + +cdef extern from "gandiva/selection_vector.h" namespace "gandiva" nogil: + + cdef cppclass CSelectionVector" gandiva::SelectionVector": + + shared_ptr[CArray] ToArray() + + enum CSelectionVector_Mode" gandiva::SelectionVector::Mode": + CSelectionVector_Mode_NONE" gandiva::SelectionVector::Mode::MODE_NONE" + CSelectionVector_Mode_UINT16" \ + gandiva::SelectionVector::Mode::MODE_UINT16" + CSelectionVector_Mode_UINT32" \ + gandiva::SelectionVector::Mode::MODE_UINT32" + CSelectionVector_Mode_UINT64" \ + gandiva::SelectionVector::Mode::MODE_UINT64" + + cdef CStatus SelectionVector_MakeInt16\ + "gandiva::SelectionVector::MakeInt16"( + int64_t max_slots, CMemoryPool* pool, + shared_ptr[CSelectionVector]* selection_vector) + + cdef CStatus SelectionVector_MakeInt32\ + "gandiva::SelectionVector::MakeInt32"( + int64_t max_slots, CMemoryPool* pool, + shared_ptr[CSelectionVector]* selection_vector) + + cdef CStatus SelectionVector_MakeInt64\ + "gandiva::SelectionVector::MakeInt64"( + int64_t max_slots, CMemoryPool* pool, + shared_ptr[CSelectionVector]* selection_vector) + +cdef inline CSelectionVector_Mode _ensure_selection_mode(str name) except *: + uppercase = name.upper() + if uppercase == 'NONE': + return CSelectionVector_Mode_NONE + elif uppercase == 'UINT16': + return CSelectionVector_Mode_UINT16 + elif uppercase == 'UINT32': + return CSelectionVector_Mode_UINT32 + elif uppercase == 'UINT64': + return CSelectionVector_Mode_UINT64 + else: + raise ValueError(f'Invalid value for Selection Mode: {name!r}') + +cdef inline str _selection_mode_name(CSelectionVector_Mode ctype): + if ctype == CSelectionVector_Mode_NONE: + return 'NONE' + elif ctype == CSelectionVector_Mode_UINT16: + return 'UINT16' + elif ctype == CSelectionVector_Mode_UINT32: + return 'UINT32' + elif ctype == CSelectionVector_Mode_UINT64: + return 'UINT64' + else: + raise RuntimeError('Unexpected CSelectionVector_Mode value') + +cdef extern from "gandiva/condition.h" namespace "gandiva" nogil: + + cdef cppclass CCondition" gandiva::Condition": + c_string ToString() + shared_ptr[CNode] root() + shared_ptr[CField] result() + +cdef extern from "gandiva/arrow.h" namespace "gandiva" nogil: + + ctypedef vector[shared_ptr[CArray]] CArrayVector" gandiva::ArrayVector" + + +cdef extern from "gandiva/tree_expr_builder.h" namespace "gandiva" nogil: + + cdef shared_ptr[CNode] TreeExprBuilder_MakeBoolLiteral \ + "gandiva::TreeExprBuilder::MakeLiteral"(c_bool value) + + cdef shared_ptr[CNode] TreeExprBuilder_MakeUInt8Literal \ + "gandiva::TreeExprBuilder::MakeLiteral"(uint8_t value) + + cdef shared_ptr[CNode] TreeExprBuilder_MakeUInt16Literal \ + "gandiva::TreeExprBuilder::MakeLiteral"(uint16_t value) + + cdef shared_ptr[CNode] TreeExprBuilder_MakeUInt32Literal \ + "gandiva::TreeExprBuilder::MakeLiteral"(uint32_t value) + + cdef shared_ptr[CNode] TreeExprBuilder_MakeUInt64Literal \ + "gandiva::TreeExprBuilder::MakeLiteral"(uint64_t value) + + cdef shared_ptr[CNode] TreeExprBuilder_MakeInt8Literal \ + "gandiva::TreeExprBuilder::MakeLiteral"(int8_t value) + + cdef shared_ptr[CNode] TreeExprBuilder_MakeInt16Literal \ + "gandiva::TreeExprBuilder::MakeLiteral"(int16_t value) + + cdef shared_ptr[CNode] TreeExprBuilder_MakeInt32Literal \ + "gandiva::TreeExprBuilder::MakeLiteral"(int32_t value) + + cdef shared_ptr[CNode] TreeExprBuilder_MakeInt64Literal \ + "gandiva::TreeExprBuilder::MakeLiteral"(int64_t value) + + cdef shared_ptr[CNode] TreeExprBuilder_MakeFloatLiteral \ + "gandiva::TreeExprBuilder::MakeLiteral"(float value) + + cdef shared_ptr[CNode] TreeExprBuilder_MakeDoubleLiteral \ + "gandiva::TreeExprBuilder::MakeLiteral"(double value) + + cdef shared_ptr[CNode] TreeExprBuilder_MakeStringLiteral \ + "gandiva::TreeExprBuilder::MakeStringLiteral"(const c_string& value) + + cdef shared_ptr[CNode] TreeExprBuilder_MakeBinaryLiteral \ + "gandiva::TreeExprBuilder::MakeBinaryLiteral"(const c_string& value) + + cdef shared_ptr[CGandivaExpression] TreeExprBuilder_MakeExpression\ + "gandiva::TreeExprBuilder::MakeExpression"( + shared_ptr[CNode] root_node, shared_ptr[CField] result_field) + + cdef shared_ptr[CNode] TreeExprBuilder_MakeFunction \ + "gandiva::TreeExprBuilder::MakeFunction"( + const c_string& name, const CNodeVector& children, + shared_ptr[CDataType] return_type) + + cdef shared_ptr[CNode] TreeExprBuilder_MakeField \ + "gandiva::TreeExprBuilder::MakeField"(shared_ptr[CField] field) + + cdef shared_ptr[CNode] TreeExprBuilder_MakeIf \ + "gandiva::TreeExprBuilder::MakeIf"( + shared_ptr[CNode] condition, shared_ptr[CNode] this_node, + shared_ptr[CNode] else_node, shared_ptr[CDataType] return_type) + + cdef shared_ptr[CNode] TreeExprBuilder_MakeAnd \ + "gandiva::TreeExprBuilder::MakeAnd"(const CNodeVector& children) + + cdef shared_ptr[CNode] TreeExprBuilder_MakeOr \ + "gandiva::TreeExprBuilder::MakeOr"(const CNodeVector& children) + + cdef shared_ptr[CCondition] TreeExprBuilder_MakeCondition \ + "gandiva::TreeExprBuilder::MakeCondition"( + shared_ptr[CNode] condition) + + cdef shared_ptr[CNode] TreeExprBuilder_MakeInExpressionInt32 \ + "gandiva::TreeExprBuilder::MakeInExpressionInt32"( + shared_ptr[CNode] node, const c_unordered_set[int32_t]& values) + + cdef shared_ptr[CNode] TreeExprBuilder_MakeInExpressionInt64 \ + "gandiva::TreeExprBuilder::MakeInExpressionInt64"( + shared_ptr[CNode] node, const c_unordered_set[int64_t]& values) + + cdef shared_ptr[CNode] TreeExprBuilder_MakeInExpressionTime32 \ + "gandiva::TreeExprBuilder::MakeInExpressionTime32"( + shared_ptr[CNode] node, const c_unordered_set[int32_t]& values) + + cdef shared_ptr[CNode] TreeExprBuilder_MakeInExpressionTime64 \ + "gandiva::TreeExprBuilder::MakeInExpressionTime64"( + shared_ptr[CNode] node, const c_unordered_set[int64_t]& values) + + cdef shared_ptr[CNode] TreeExprBuilder_MakeInExpressionDate32 \ + "gandiva::TreeExprBuilder::MakeInExpressionDate32"( + shared_ptr[CNode] node, const c_unordered_set[int32_t]& values) + + cdef shared_ptr[CNode] TreeExprBuilder_MakeInExpressionDate64 \ + "gandiva::TreeExprBuilder::MakeInExpressionDate64"( + shared_ptr[CNode] node, const c_unordered_set[int64_t]& values) + + cdef shared_ptr[CNode] TreeExprBuilder_MakeInExpressionTimeStamp \ + "gandiva::TreeExprBuilder::MakeInExpressionTimeStamp"( + shared_ptr[CNode] node, const c_unordered_set[int64_t]& values) + + cdef shared_ptr[CNode] TreeExprBuilder_MakeInExpressionString \ + "gandiva::TreeExprBuilder::MakeInExpressionString"( + shared_ptr[CNode] node, const c_unordered_set[c_string]& values) + + cdef shared_ptr[CNode] TreeExprBuilder_MakeInExpressionBinary \ + "gandiva::TreeExprBuilder::MakeInExpressionBinary"( + shared_ptr[CNode] node, const c_unordered_set[c_string]& values) + +cdef extern from "gandiva/projector.h" namespace "gandiva" nogil: + + cdef cppclass CProjector" gandiva::Projector": + + CStatus Evaluate( + const CRecordBatch& batch, CMemoryPool* pool, + const CArrayVector* output) + + CStatus Evaluate( + const CRecordBatch& batch, + const CSelectionVector* selection, + CMemoryPool* pool, + const CArrayVector* output) + + c_string DumpIR() + + cdef CStatus Projector_Make \ + "gandiva::Projector::Make"( + shared_ptr[CSchema] schema, const CExpressionVector& children, + shared_ptr[CProjector]* projector) + + cdef CStatus Projector_Make \ + "gandiva::Projector::Make"( + shared_ptr[CSchema] schema, const CExpressionVector& children, + CSelectionVector_Mode mode, + shared_ptr[CConfiguration] configuration, + shared_ptr[CProjector]* projector) + +cdef extern from "gandiva/filter.h" namespace "gandiva" nogil: + + cdef cppclass CFilter" gandiva::Filter": + + CStatus Evaluate( + const CRecordBatch& batch, + shared_ptr[CSelectionVector] out_selection) + + c_string DumpIR() + + cdef CStatus Filter_Make \ + "gandiva::Filter::Make"( + shared_ptr[CSchema] schema, shared_ptr[CCondition] condition, + shared_ptr[CConfiguration] configuration, + shared_ptr[CFilter]* filter) + +cdef extern from "gandiva/function_signature.h" namespace "gandiva" nogil: + + cdef cppclass CFunctionSignature" gandiva::FunctionSignature": + + CFunctionSignature(const c_string& base_name, + vector[shared_ptr[CDataType]] param_types, + shared_ptr[CDataType] ret_type) + + shared_ptr[CDataType] ret_type() const + + const c_string& base_name() const + + vector[shared_ptr[CDataType]] param_types() const + + c_string ToString() const + +cdef extern from "gandiva/expression_registry.h" namespace "gandiva" nogil: + + cdef vector[shared_ptr[CFunctionSignature]] \ + GetRegisteredFunctionSignatures() + +cdef extern from "gandiva/configuration.h" namespace "gandiva" nogil: + + cdef cppclass CConfiguration" gandiva::Configuration": + + CConfiguration() + + CConfiguration(bint optimize, bint dump_ir) + + void set_optimize(bint optimize) + + void set_dump_ir(bint dump_ir) + + cdef cppclass CConfigurationBuilder \ + " gandiva::ConfigurationBuilder": + @staticmethod + shared_ptr[CConfiguration] DefaultConfiguration() + + CConfigurationBuilder() + + shared_ptr[CConfiguration] build() diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libparquet.pxd b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libparquet.pxd new file mode 100644 index 0000000000000000000000000000000000000000..3d54c090a8c94e4779dc4a73319b4ed40f1caecb --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libparquet.pxd @@ -0,0 +1,638 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# distutils: language = c++ + +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport (Type, CChunkedArray, CScalar, CSchema, + CStatus, CTable, CMemoryPool, CBuffer, + CKeyValueMetadata, CRandomAccessFile, + COutputStream, CCacheOptions, + TimeUnit, CRecordBatchReader) + + +cdef extern from "parquet/api/schema.h" namespace "parquet::schema" nogil: + cdef cppclass Node: + pass + + cdef cppclass GroupNode(Node): + pass + + cdef cppclass PrimitiveNode(Node): + pass + + cdef cppclass ColumnPath: + c_string ToDotString() + vector[c_string] ToDotVector() + + +cdef extern from "parquet/api/schema.h" namespace "parquet" nogil: + enum ParquetType" parquet::Type::type": + ParquetType_BOOLEAN" parquet::Type::BOOLEAN" + ParquetType_INT32" parquet::Type::INT32" + ParquetType_INT64" parquet::Type::INT64" + ParquetType_INT96" parquet::Type::INT96" + ParquetType_FLOAT" parquet::Type::FLOAT" + ParquetType_DOUBLE" parquet::Type::DOUBLE" + ParquetType_BYTE_ARRAY" parquet::Type::BYTE_ARRAY" + ParquetType_FIXED_LEN_BYTE_ARRAY" parquet::Type::FIXED_LEN_BYTE_ARRAY" + + enum ParquetLogicalTypeId" parquet::LogicalType::Type::type": + ParquetLogicalType_UNDEFINED" parquet::LogicalType::Type::UNDEFINED" + ParquetLogicalType_STRING" parquet::LogicalType::Type::STRING" + ParquetLogicalType_MAP" parquet::LogicalType::Type::MAP" + ParquetLogicalType_LIST" parquet::LogicalType::Type::LIST" + ParquetLogicalType_ENUM" parquet::LogicalType::Type::ENUM" + ParquetLogicalType_DECIMAL" parquet::LogicalType::Type::DECIMAL" + ParquetLogicalType_DATE" parquet::LogicalType::Type::DATE" + ParquetLogicalType_TIME" parquet::LogicalType::Type::TIME" + ParquetLogicalType_TIMESTAMP" parquet::LogicalType::Type::TIMESTAMP" + ParquetLogicalType_INT" parquet::LogicalType::Type::INT" + ParquetLogicalType_FLOAT16" parquet::LogicalType::Type::FLOAT16" + ParquetLogicalType_JSON" parquet::LogicalType::Type::JSON" + ParquetLogicalType_BSON" parquet::LogicalType::Type::BSON" + ParquetLogicalType_UUID" parquet::LogicalType::Type::UUID" + ParquetLogicalType_GEOMETRY" parquet::LogicalType::Type::GEOMETRY" + ParquetLogicalType_GEOGRAPHY" parquet::LogicalType::Type::GEOGRAPHY" + ParquetLogicalType_NONE" parquet::LogicalType::Type::NONE" + + enum ParquetTimeUnit" parquet::LogicalType::TimeUnit::unit": + ParquetTimeUnit_UNKNOWN" parquet::LogicalType::TimeUnit::UNKNOWN" + ParquetTimeUnit_MILLIS" parquet::LogicalType::TimeUnit::MILLIS" + ParquetTimeUnit_MICROS" parquet::LogicalType::TimeUnit::MICROS" + ParquetTimeUnit_NANOS" parquet::LogicalType::TimeUnit::NANOS" + + enum ParquetEdgeInterpolationAlgorithm" parquet::LogicalType::EdgeInterpolationAlgorithm": + ParquetEdgeInterpolationAlgorithm_UNKNOWN" parquet::LogicalType::EdgeInterpolationAlgorithm::UNKNOWN" + ParquetEdgeInterpolationAlgorithm_SPHERICAL" parquet::LogicalType::EdgeInterpolationAlgorithm::SPHERICAL" + ParquetEdgeInterpolationAlgorithm_VINCENTY" parquet::LogicalType::EdgeInterpolationAlgorithm::VINCENTY" + ParquetEdgeInterpolationAlgorithm_THOMAS" parquet::LogicalType::EdgeInterpolationAlgorithm::THOMAS" + ParquetEdgeInterpolationAlgorithm_ANDOYER" parquet::LogicalType::EdgeInterpolationAlgorithm::ANDOYER" + ParquetEdgeInterpolationAlgorithm_KARNEY" parquet::LogicalType::EdgeInterpolationAlgorithm::KARNEY" + + enum ParquetConvertedType" parquet::ConvertedType::type": + ParquetConvertedType_NONE" parquet::ConvertedType::NONE" + ParquetConvertedType_UTF8" parquet::ConvertedType::UTF8" + ParquetConvertedType_MAP" parquet::ConvertedType::MAP" + ParquetConvertedType_MAP_KEY_VALUE \ + " parquet::ConvertedType::MAP_KEY_VALUE" + ParquetConvertedType_LIST" parquet::ConvertedType::LIST" + ParquetConvertedType_ENUM" parquet::ConvertedType::ENUM" + ParquetConvertedType_DECIMAL" parquet::ConvertedType::DECIMAL" + ParquetConvertedType_DATE" parquet::ConvertedType::DATE" + ParquetConvertedType_TIME_MILLIS" parquet::ConvertedType::TIME_MILLIS" + ParquetConvertedType_TIME_MICROS" parquet::ConvertedType::TIME_MICROS" + ParquetConvertedType_TIMESTAMP_MILLIS \ + " parquet::ConvertedType::TIMESTAMP_MILLIS" + ParquetConvertedType_TIMESTAMP_MICROS \ + " parquet::ConvertedType::TIMESTAMP_MICROS" + ParquetConvertedType_UINT_8" parquet::ConvertedType::UINT_8" + ParquetConvertedType_UINT_16" parquet::ConvertedType::UINT_16" + ParquetConvertedType_UINT_32" parquet::ConvertedType::UINT_32" + ParquetConvertedType_UINT_64" parquet::ConvertedType::UINT_64" + ParquetConvertedType_INT_8" parquet::ConvertedType::INT_8" + ParquetConvertedType_INT_16" parquet::ConvertedType::INT_16" + ParquetConvertedType_INT_32" parquet::ConvertedType::INT_32" + ParquetConvertedType_INT_64" parquet::ConvertedType::INT_64" + ParquetConvertedType_JSON" parquet::ConvertedType::JSON" + ParquetConvertedType_BSON" parquet::ConvertedType::BSON" + ParquetConvertedType_INTERVAL" parquet::ConvertedType::INTERVAL" + + enum ParquetRepetition" parquet::Repetition::type": + ParquetRepetition_REQUIRED" parquet::REPETITION::REQUIRED" + ParquetRepetition_OPTIONAL" parquet::REPETITION::OPTIONAL" + ParquetRepetition_REPEATED" parquet::REPETITION::REPEATED" + + enum ParquetEncoding" parquet::Encoding::type": + ParquetEncoding_PLAIN" parquet::Encoding::PLAIN" + ParquetEncoding_PLAIN_DICTIONARY" parquet::Encoding::PLAIN_DICTIONARY" + ParquetEncoding_RLE" parquet::Encoding::RLE" + ParquetEncoding_BIT_PACKED" parquet::Encoding::BIT_PACKED" + ParquetEncoding_DELTA_BINARY_PACKED \ + " parquet::Encoding::DELTA_BINARY_PACKED" + ParquetEncoding_DELTA_LENGTH_BYTE_ARRAY \ + " parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY" + ParquetEncoding_DELTA_BYTE_ARRAY" parquet::Encoding::DELTA_BYTE_ARRAY" + ParquetEncoding_RLE_DICTIONARY" parquet::Encoding::RLE_DICTIONARY" + ParquetEncoding_BYTE_STREAM_SPLIT \ + " parquet::Encoding::BYTE_STREAM_SPLIT" + + enum ParquetCompression" parquet::Compression::type": + ParquetCompression_UNCOMPRESSED" parquet::Compression::UNCOMPRESSED" + ParquetCompression_SNAPPY" parquet::Compression::SNAPPY" + ParquetCompression_GZIP" parquet::Compression::GZIP" + ParquetCompression_LZO" parquet::Compression::LZO" + ParquetCompression_BROTLI" parquet::Compression::BROTLI" + ParquetCompression_LZ4" parquet::Compression::LZ4" + ParquetCompression_ZSTD" parquet::Compression::ZSTD" + + enum ParquetVersion" parquet::ParquetVersion::type": + ParquetVersion_V1" parquet::ParquetVersion::PARQUET_1_0" + ParquetVersion_V2_4" parquet::ParquetVersion::PARQUET_2_4" + ParquetVersion_V2_6" parquet::ParquetVersion::PARQUET_2_6" + + enum ParquetSortOrder" parquet::SortOrder::type": + ParquetSortOrder_SIGNED" parquet::SortOrder::SIGNED" + ParquetSortOrder_UNSIGNED" parquet::SortOrder::UNSIGNED" + ParquetSortOrder_UNKNOWN" parquet::SortOrder::UNKNOWN" + + cdef cppclass CParquetLogicalType" parquet::LogicalType": + c_string ToString() const + c_string ToJSON() const + ParquetLogicalTypeId type() const + + cdef cppclass CParquetDecimalType \ + " parquet::DecimalLogicalType"(CParquetLogicalType): + int32_t precision() const + int32_t scale() const + + cdef cppclass CParquetIntType \ + " parquet::IntLogicalType"(CParquetLogicalType): + int bit_width() const + c_bool is_signed() const + + cdef cppclass CParquetTimeType \ + " parquet::TimeLogicalType"(CParquetLogicalType): + c_bool is_adjusted_to_utc() const + ParquetTimeUnit time_unit() const + + cdef cppclass CParquetTimestampType \ + " parquet::TimestampLogicalType"(CParquetLogicalType): + c_bool is_adjusted_to_utc() const + ParquetTimeUnit time_unit() const + + cdef cppclass CParquetGeometryType \ + " parquet::GeometryLogicalType"(CParquetLogicalType): + c_string crs() const + + cdef cppclass CParquetGeographyType \ + " parquet::GeographyLogicalType"(CParquetLogicalType): + c_string crs() const + ParquetEdgeInterpolationAlgorithm algorithm() const + + cdef cppclass ColumnDescriptor" parquet::ColumnDescriptor": + c_bool Equals(const ColumnDescriptor& other) + + shared_ptr[ColumnPath] path() + int16_t max_definition_level() + int16_t max_repetition_level() + + ParquetType physical_type() + const shared_ptr[const CParquetLogicalType]& logical_type() + ParquetConvertedType converted_type() + const c_string& name() + int type_length() + int type_precision() + int type_scale() + + cdef cppclass SchemaDescriptor: + const ColumnDescriptor* Column(int i) + shared_ptr[Node] schema() + GroupNode* group() + c_bool Equals(const SchemaDescriptor& other) + c_string ToString() + int num_columns() + + cdef c_string FormatStatValue(ParquetType parquet_type, c_string val) + + enum ParquetCipher" parquet::ParquetCipher::type": + ParquetCipher_AES_GCM_V1" parquet::ParquetCipher::AES_GCM_V1" + ParquetCipher_AES_GCM_CTR_V1" parquet::ParquetCipher::AES_GCM_CTR_V1" + + struct AadMetadata: + c_string aad_prefix + c_string aad_file_unique + c_bool supply_aad_prefix + + struct EncryptionAlgorithm: + ParquetCipher algorithm + AadMetadata aad + + +# Specific array<> types needed for GeoStatistics +cdef extern from "" namespace "std" nogil: + cdef cppclass double_array4 "std::array": + double_array4() except + + double& operator[](size_t) + + cdef cppclass bool_array4 "std::array": + bool_array4() except + + c_bool& operator[](size_t) + + +cdef extern from "parquet/geospatial/statistics.h" namespace "parquet" nogil: + + cdef cppclass CParquetGeoStatistics" parquet::geospatial::GeoStatistics": + c_bool is_valid() const + + double_array4 lower_bound() const + double_array4 upper_bound() const + bool_array4 dimension_valid() const + bool_array4 dimension_empty() const + + optional[vector[int32_t]] geometry_types() const + + c_string ToString() const + + +cdef extern from "parquet/api/reader.h" namespace "parquet" nogil: + cdef cppclass ColumnReader: + pass + + cdef cppclass BoolReader(ColumnReader): + pass + + cdef cppclass Int32Reader(ColumnReader): + pass + + cdef cppclass Int64Reader(ColumnReader): + pass + + cdef cppclass Int96Reader(ColumnReader): + pass + + cdef cppclass FloatReader(ColumnReader): + pass + + cdef cppclass DoubleReader(ColumnReader): + pass + + cdef cppclass ByteArrayReader(ColumnReader): + pass + + cdef cppclass RowGroupReader: + pass + + cdef cppclass CEncodedStatistics" parquet::EncodedStatistics": + const c_string& max() const + const c_string& min() const + int64_t null_count + int64_t distinct_count + bint has_min + bint has_max + bint has_null_count + bint has_distinct_count + + cdef cppclass ParquetByteArray" parquet::ByteArray": + uint32_t len + const uint8_t* ptr + + cdef cppclass ParquetFLBA" parquet::FLBA": + const uint8_t* ptr + + cdef cppclass CStatistics" parquet::Statistics": + int64_t null_count() const + int64_t distinct_count() const + int64_t num_values() const + bint HasMinMax() + bint HasNullCount() + bint HasDistinctCount() + c_bool Equals(const CStatistics&) const + void Reset() + c_string EncodeMin() + c_string EncodeMax() + CEncodedStatistics Encode() + void SetComparator() + ParquetType physical_type() const + const ColumnDescriptor* descr() const + + cdef cppclass CBoolStatistics" parquet::BoolStatistics"(CStatistics): + c_bool min() + c_bool max() + + cdef cppclass CInt32Statistics" parquet::Int32Statistics"(CStatistics): + int32_t min() + int32_t max() + + cdef cppclass CInt64Statistics" parquet::Int64Statistics"(CStatistics): + int64_t min() + int64_t max() + + cdef cppclass CFloatStatistics" parquet::FloatStatistics"(CStatistics): + float min() + float max() + + cdef cppclass CDoubleStatistics" parquet::DoubleStatistics"(CStatistics): + double min() + double max() + + cdef cppclass CByteArrayStatistics \ + " parquet::ByteArrayStatistics"(CStatistics): + ParquetByteArray min() + ParquetByteArray max() + + cdef cppclass CFLBAStatistics" parquet::FLBAStatistics"(CStatistics): + ParquetFLBA min() + ParquetFLBA max() + + cdef cppclass CColumnCryptoMetaData" parquet::ColumnCryptoMetaData": + shared_ptr[ColumnPath] path_in_schema() const + c_bool encrypted_with_footer_key() const + const c_string& key_metadata() const + + cdef cppclass ParquetIndexLocation" parquet::IndexLocation": + int64_t offset + int32_t length + + cdef cppclass CColumnChunkMetaData" parquet::ColumnChunkMetaData": + int64_t file_offset() const + const c_string& file_path() const + + c_bool is_metadata_set() const + ParquetType type() const + int64_t num_values() const + shared_ptr[ColumnPath] path_in_schema() const + bint is_stats_set() const + shared_ptr[CStatistics] statistics() const + c_bool is_geo_stats_set() const + shared_ptr[CParquetGeoStatistics] geo_statistics() const + ParquetCompression compression() const + const vector[ParquetEncoding]& encodings() const + c_bool Equals(const CColumnChunkMetaData&) const + + int64_t has_dictionary_page() const + int64_t dictionary_page_offset() const + int64_t data_page_offset() const + int64_t index_page_offset() const + int64_t total_compressed_size() const + int64_t total_uncompressed_size() const + unique_ptr[CColumnCryptoMetaData] crypto_metadata() const + optional[ParquetIndexLocation] GetColumnIndexLocation() const + optional[ParquetIndexLocation] GetOffsetIndexLocation() const + shared_ptr[const CKeyValueMetadata] key_value_metadata() const + + struct CSortingColumn" parquet::SortingColumn": + int column_idx + c_bool descending + c_bool nulls_first + + cdef cppclass CRowGroupMetaData" parquet::RowGroupMetaData": + c_bool Equals(const CRowGroupMetaData&) const + int num_columns() const + int64_t num_rows() const + int64_t total_byte_size() const + vector[CSortingColumn] sorting_columns() const + unique_ptr[CColumnChunkMetaData] ColumnChunk(int i) const + + cdef cppclass CFileMetaData" parquet::FileMetaData": + c_bool Equals(const CFileMetaData&) const + uint32_t size() + int num_columns() + int64_t num_rows() + int num_row_groups() + ParquetVersion version() + const c_string created_by() + int num_schema_elements() + + void set_file_path(const c_string& path) + void AppendRowGroups(const CFileMetaData& other) except + + + unique_ptr[CRowGroupMetaData] RowGroup(int i) + const SchemaDescriptor* schema() + shared_ptr[const CKeyValueMetadata] key_value_metadata() const + void WriteTo(COutputStream* dst) const + + inline c_bool is_encryption_algorithm_set() const + inline EncryptionAlgorithm encryption_algorithm() const + inline const c_string& footer_signing_key_metadata() const + + cdef shared_ptr[CFileMetaData] CFileMetaData_Make \ + " parquet::FileMetaData::Make"(const void* serialized_metadata, + uint32_t* metadata_len) + + cdef cppclass CReaderProperties" parquet::ReaderProperties": + c_bool is_buffered_stream_enabled() const + void enable_buffered_stream() + void disable_buffered_stream() + + void set_buffer_size(int64_t buf_size) + int64_t buffer_size() const + + void set_thrift_string_size_limit(int32_t size) + int32_t thrift_string_size_limit() const + + void set_thrift_container_size_limit(int32_t size) + int32_t thrift_container_size_limit() const + + void file_decryption_properties(shared_ptr[CFileDecryptionProperties] + decryption) + shared_ptr[CFileDecryptionProperties] file_decryption_properties() \ + const + + c_bool page_checksum_verification() const + void set_page_checksum_verification(c_bool check_crc) + + CReaderProperties default_reader_properties() + + cdef cppclass ArrowReaderProperties: + ArrowReaderProperties() + void set_binary_type(Type binary_type) + Type binary_type() + void set_list_type(Type list_type) + Type list_type() + void set_read_dictionary(int column_index, c_bool read_dict) + c_bool read_dictionary(int column_index) + void set_batch_size(int64_t batch_size) + int64_t batch_size() + void set_pre_buffer(c_bool pre_buffer) + c_bool pre_buffer() const + void set_cache_options(CCacheOptions options) + CCacheOptions cache_options() const + void set_coerce_int96_timestamp_unit(TimeUnit unit) + TimeUnit coerce_int96_timestamp_unit() const + void set_arrow_extensions_enabled(c_bool extensions_enabled) + c_bool get_arrow_extensions_enabled() const + + ArrowReaderProperties default_arrow_reader_properties() + + cdef cppclass ParquetFileReader: + shared_ptr[CFileMetaData] metadata() + + +cdef extern from "parquet/api/writer.h" namespace "parquet" nogil: + cdef cppclass BloomFilterOptions: + int32_t ndv + double fpp + + cdef cppclass CdcOptions: + int64_t min_chunk_size + int64_t max_chunk_size + int norm_level + + cdef cppclass WriterProperties: + cppclass Builder: + Builder* data_page_version(ParquetDataPageVersion version) + Builder* version(ParquetVersion version) + Builder* compression(ParquetCompression codec) + Builder* compression(const c_string& path, + ParquetCompression codec) + Builder* compression_level(int compression_level) + Builder* compression_level(const c_string& path, + int compression_level) + Builder* encryption( + shared_ptr[CFileEncryptionProperties] + file_encryption_properties) + Builder* disable_dictionary() + Builder* enable_dictionary() + Builder* enable_dictionary(const c_string& path) + Builder* set_sorting_columns(vector[CSortingColumn] sorting_columns) + Builder* disable_statistics() + Builder* enable_statistics() + Builder* enable_statistics(const c_string& path) + Builder* enable_store_decimal_as_integer() + Builder* disable_store_decimal_as_integer() + Builder* data_pagesize(int64_t size) + Builder* max_rows_per_page(int64_t max_rows) + Builder* encoding(ParquetEncoding encoding) + Builder* encoding(const c_string& path, + ParquetEncoding encoding) + Builder* max_row_group_length(int64_t size) + Builder* write_batch_size(int64_t batch_size) + Builder* dictionary_pagesize_limit(int64_t dictionary_pagesize_limit) + Builder* enable_write_page_index() + Builder* disable_write_page_index() + Builder* enable_page_checksum() + Builder* disable_page_checksum() + Builder* enable_content_defined_chunking() + Builder* disable_content_defined_chunking() + Builder* content_defined_chunking_options(CdcOptions options) + Builder* disable_bloom_filter(const c_string& path) + Builder* enable_bloom_filter(const c_string& path, + BloomFilterOptions bloom_filter_options) + shared_ptr[WriterProperties] build() + + cdef cppclass ArrowWriterProperties: + cppclass Builder: + Builder() + Builder* disable_deprecated_int96_timestamps() + Builder* enable_deprecated_int96_timestamps() + Builder* coerce_timestamps(TimeUnit unit) + Builder* allow_truncated_timestamps() + Builder* disallow_truncated_timestamps() + Builder* store_schema() + Builder* enable_compliant_nested_types() + Builder* disable_compliant_nested_types() + Builder* set_engine_version(ArrowWriterEngineVersion version) + Builder* set_time_adjusted_to_utc(c_bool adjusted) + shared_ptr[ArrowWriterProperties] build() + c_bool support_deprecated_int96_timestamps() + +cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil: + cdef cppclass FileReader: + FileReader(CMemoryPool* pool, unique_ptr[ParquetFileReader] reader) + + CStatus GetSchema(shared_ptr[CSchema]* out) + + CStatus ReadColumn(int i, shared_ptr[CChunkedArray]* out) + CStatus ReadSchemaField(int i, shared_ptr[CChunkedArray]* out) + + int num_row_groups() + CResult[shared_ptr[CTable]] ReadRowGroup(int i) + CResult[shared_ptr[CTable]] ReadRowGroup(int i, + const vector[int]& column_indices) + + CResult[shared_ptr[CTable]] ReadRowGroups(const vector[int]& row_groups) + CResult[shared_ptr[CTable]] ReadRowGroups(const vector[int]& row_groups, + const vector[int]& column_indices) + + CResult[unique_ptr[CRecordBatchReader]] GetRecordBatchReader(const vector[int]& row_group_indices, + const vector[int]& column_indices) + CResult[unique_ptr[CRecordBatchReader]] GetRecordBatchReader(const vector[int]& row_group_indices) + + CResult[shared_ptr[CTable]] ReadTable() + CResult[shared_ptr[CTable]] ReadTable(const vector[int]& column_indices) + + CStatus ScanContents(vector[int] columns, int32_t column_batch_size, + int64_t* num_rows) + + const ParquetFileReader* parquet_reader() + + void set_use_threads(c_bool use_threads) + + void set_batch_size(int64_t batch_size) + + cdef cppclass FileReaderBuilder: + FileReaderBuilder() + CStatus Open(const shared_ptr[CRandomAccessFile]& file, + const CReaderProperties& properties, + const shared_ptr[CFileMetaData]& metadata) + + ParquetFileReader* raw_reader() + FileReaderBuilder* memory_pool(CMemoryPool*) + FileReaderBuilder* properties(const ArrowReaderProperties&) + CStatus Build(unique_ptr[FileReader]* out) + + CStatus FromParquetSchema( + const SchemaDescriptor* parquet_schema, + const ArrowReaderProperties& properties, + const shared_ptr[const CKeyValueMetadata]& key_value_metadata, + shared_ptr[CSchema]* out) + + CStatus StatisticsAsScalars(const CStatistics& Statistics, + shared_ptr[CScalar]* min, + shared_ptr[CScalar]* max) + +cdef extern from "parquet/arrow/schema.h" namespace "parquet::arrow" nogil: + + CStatus ToParquetSchema( + const CSchema* arrow_schema, + const WriterProperties& properties, + const ArrowWriterProperties& arrow_properties, + shared_ptr[SchemaDescriptor]* out) + + +cdef extern from "parquet/properties.h" namespace "parquet" nogil: + cdef enum ArrowWriterEngineVersion: + V1 "parquet::ArrowWriterProperties::V1", + V2 "parquet::ArrowWriterProperties::V2" + + cdef cppclass ParquetDataPageVersion: + pass + + cdef ParquetDataPageVersion ParquetDataPageVersion_V1 \ + " parquet::ParquetDataPageVersion::V1" + cdef ParquetDataPageVersion ParquetDataPageVersion_V2 \ + " parquet::ParquetDataPageVersion::V2" + +cdef extern from "parquet/arrow/writer.h" namespace "parquet::arrow" nogil: + cdef cppclass FileWriter: + + @staticmethod + CResult[unique_ptr[FileWriter]] Open(const CSchema& schema, CMemoryPool* pool, + const shared_ptr[COutputStream]& sink, + const shared_ptr[WriterProperties]& properties, + const shared_ptr[ArrowWriterProperties]& arrow_properties) + + CStatus WriteTable(const CTable& table, int64_t chunk_size) + CStatus NewRowGroup() + CStatus Close() + CStatus AddKeyValueMetadata(const shared_ptr[const CKeyValueMetadata]& key_value_metadata) + + const shared_ptr[CFileMetaData] metadata() const + + CStatus WriteMetaDataFile( + const CFileMetaData& file_metadata, + const COutputStream* sink) + +cdef extern from "parquet/encryption/encryption.h" namespace "parquet" nogil: + cdef cppclass CFileDecryptionProperties\ + " parquet::FileDecryptionProperties": + pass + + cdef cppclass CFileEncryptionProperties\ + " parquet::FileEncryptionProperties": + pass diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libparquet_encryption.pxd b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libparquet_encryption.pxd new file mode 100644 index 0000000000000000000000000000000000000000..eead4b7c048ae8426f49e0b99c8548e6ad7b6891 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/includes/libparquet_encryption.pxd @@ -0,0 +1,187 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# distutils: language = c++ + +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport CSecureString +from pyarrow.includes.libarrow_fs cimport CFileSystem +from pyarrow._parquet cimport (ParquetCipher, + CFileEncryptionProperties, + CFileDecryptionProperties, + ParquetCipher_AES_GCM_V1, + ParquetCipher_AES_GCM_CTR_V1) + + +cdef extern from "parquet/encryption/kms_client.h" \ + namespace "parquet::encryption" nogil: + cdef cppclass CKmsClient" parquet::encryption::KmsClient": + c_string WrapKey(const CSecureString& key, + const c_string& master_key_identifier) except + + CSecureString UnwrapKey(const c_string& wrapped_key, + const c_string& master_key_identifier) except + + + cdef cppclass CKeyAccessToken" parquet::encryption::KeyAccessToken": + CKeyAccessToken(const c_string value) + void Refresh(const c_string& new_value) + const c_string& value() const + + cdef cppclass CKmsConnectionConfig \ + " parquet::encryption::KmsConnectionConfig": + CKmsConnectionConfig() + c_string kms_instance_id + c_string kms_instance_url + shared_ptr[CKeyAccessToken] refreshable_key_access_token + unordered_map[c_string, c_string] custom_kms_conf + +# Callbacks for implementing Python kms clients +# Use typedef to emulate syntax for std::function +ctypedef void CallbackWrapKey( + object, const CSecureString&, const c_string&, c_string*) +ctypedef void CallbackUnwrapKey( + object, const c_string&, const c_string&, CSecureString*) + +cdef extern from "parquet/encryption/kms_client_factory.h" \ + namespace "parquet::encryption" nogil: + cdef cppclass CKmsClientFactory" parquet::encryption::KmsClientFactory": + shared_ptr[CKmsClient] CreateKmsClient( + const CKmsConnectionConfig& kms_connection_config) except + + +# Callbacks for implementing Python kms client factories +# Use typedef to emulate syntax for std::function +ctypedef void CallbackCreateKmsClient( + object, + const CKmsConnectionConfig&, shared_ptr[CKmsClient]*) + +cdef extern from "parquet/encryption/crypto_factory.h" \ + namespace "parquet::encryption" nogil: + cdef cppclass CEncryptionConfiguration\ + " parquet::encryption::EncryptionConfiguration": + CEncryptionConfiguration(const c_string& footer_key) except + + c_string footer_key + c_string column_keys + c_bool uniform_encryption + ParquetCipher encryption_algorithm + c_bool plaintext_footer + c_bool double_wrapping + double cache_lifetime_seconds + c_bool internal_key_material + int32_t data_key_length_bits + + cdef cppclass CDecryptionConfiguration\ + " parquet::encryption::DecryptionConfiguration": + CDecryptionConfiguration() except + + double cache_lifetime_seconds + + cdef cppclass CCryptoFactory" parquet::encryption::CryptoFactory": + void RegisterKmsClientFactory( + shared_ptr[CKmsClientFactory] kms_client_factory) except + + shared_ptr[CFileEncryptionProperties] GetFileEncryptionProperties( + const CKmsConnectionConfig& kms_connection_config, + const CEncryptionConfiguration& encryption_config, + const c_string parquet_file_path, + const shared_ptr[CFileSystem] file_system) except +* + shared_ptr[CFileDecryptionProperties] GetFileDecryptionProperties( + const CKmsConnectionConfig& kms_connection_config, + const CDecryptionConfiguration& decryption_config, + const c_string parquet_file_path, + const shared_ptr[CFileSystem] file_system) except +* + void RemoveCacheEntriesForToken(const c_string& access_token) except + + void RemoveCacheEntriesForAllTokens() except + + void RotateMasterKeys(const CKmsConnectionConfig& kms_connection_config, + const c_string parquet_file_path, + const shared_ptr[CFileSystem] file_system, + c_bool double_wrapping, + double cache_lifetime_seconds) + +cdef extern from "parquet/encryption/file_key_material_store.h" \ + namespace "parquet::encryption" nogil: + cdef cppclass CFileKeyMaterialStore\ + "parquet::encryption::FileKeyMaterialStore": + @staticmethod + c_string GetKeyMaterial(c_string key_id_in_file) except + + vector[c_string] GetKeyIDSet() except + + +cdef extern from "parquet/encryption/file_system_key_material_store.h" \ + namespace "parquet::encryption" nogil: + cdef cppclass CFileSystemKeyMaterialStore\ + "parquet::encryption::FileSystemKeyMaterialStore": + + @staticmethod + shared_ptr[CFileSystemKeyMaterialStore] Make(c_string parquet_file_path, + shared_ptr[CFileSystem] file_system, + c_bool use_tmp_prefix) except + + + c_string GetKeyMaterial(c_string key_id_in_file) except + + + vector[c_string] GetKeyIDSet() except + + +cdef extern from "parquet/encryption/key_material.h" \ + namespace "parquet::encryption" nogil: + cdef cppclass CKeyMaterial "parquet::encryption::KeyMaterial": + @staticmethod + CKeyMaterial Parse(const c_string& key_material_string) + c_bool is_footer_key() + c_bool is_double_wrapped() + const c_string& master_key_id() + const c_string& wrapped_dek() + const c_string& kek_id() + const c_string& wrapped_kek() + const c_string& kms_instance_id() + const c_string& kms_instance_url() + +cdef extern from "arrow/python/parquet_encryption.h" \ + namespace "arrow::py::parquet::encryption" nogil: + cdef cppclass CPyKmsClientVtable \ + " arrow::py::parquet::encryption::PyKmsClientVtable": + CPyKmsClientVtable() + function[CallbackWrapKey] wrap_key + function[CallbackUnwrapKey] unwrap_key + + cdef cppclass CPyKmsClient\ + " arrow::py::parquet::encryption::PyKmsClient"(CKmsClient): + CPyKmsClient(object handler, CPyKmsClientVtable vtable) + + cdef cppclass CPyKmsClientFactoryVtable\ + " arrow::py::parquet::encryption::PyKmsClientFactoryVtable": + CPyKmsClientFactoryVtable() + function[CallbackCreateKmsClient] create_kms_client + + cdef cppclass CPyKmsClientFactory\ + " arrow::py::parquet::encryption::PyKmsClientFactory"( + CKmsClientFactory): + CPyKmsClientFactory(object handler, CPyKmsClientFactoryVtable vtable) + + cdef cppclass CPyCryptoFactory\ + " arrow::py::parquet::encryption::PyCryptoFactory"(CCryptoFactory): + CResult[shared_ptr[CFileEncryptionProperties]] \ + SafeGetFileEncryptionProperties( + const CKmsConnectionConfig& kms_connection_config, + const CEncryptionConfiguration& encryption_config, + const c_string parquet_file_path, + const shared_ptr[CFileSystem] filesystem) + CResult[shared_ptr[CFileDecryptionProperties]] \ + SafeGetFileDecryptionProperties( + const CKmsConnectionConfig& kms_connection_config, + const CDecryptionConfiguration& decryption_config, + const c_string parquet_file_path, + const shared_ptr[CFileSystem] filesystem) + CStatus SafeRotateMasterKeys(const CKmsConnectionConfig& kms_connection_config, + const c_string parquet_file_path, + const shared_ptr[CFileSystem] filesystem, + c_bool double_wrapping, + double cache_lifetime_seconds) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/interchange/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/interchange/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..020dfaea8d7e19d38fee4424ba2e08380f65d7df --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/interchange/__init__.py @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# flake8: noqa + +from .from_dataframe import from_dataframe diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/interchange/buffer.py b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/interchange/buffer.py new file mode 100644 index 0000000000000000000000000000000000000000..1aa3c5cb8a4c28538e23b8290f120e885a1a0164 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/interchange/buffer.py @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations +import enum + +import pyarrow as pa + + +class DlpackDeviceType(enum.IntEnum): + """Integer enum for device type codes matching DLPack.""" + + CPU = 1 + CUDA = 2 + CPU_PINNED = 3 + OPENCL = 4 + VULKAN = 7 + METAL = 8 + VPI = 9 + ROCM = 10 + + +class _PyArrowBuffer: + """ + Data in the buffer is guaranteed to be contiguous in memory. + + Note that there is no dtype attribute present, a buffer can be thought of + as simply a block of memory. However, if the column that the buffer is + attached to has a dtype that's supported by DLPack and ``__dlpack__`` is + implemented, then that dtype information will be contained in the return + value from ``__dlpack__``. + + This distinction is useful to support both data exchange via DLPack on a + buffer and (b) dtypes like variable-length strings which do not have a + fixed number of bytes per element. + """ + + def __init__(self, x: pa.Buffer, allow_copy: bool = True) -> None: + """ + Handle PyArrow Buffers. + """ + self._x = x + + @property + def bufsize(self) -> int: + """ + Buffer size in bytes. + """ + return self._x.size + + @property + def ptr(self) -> int: + """ + Pointer to start of the buffer as an integer. + """ + return self._x.address + + def __dlpack__(self): + """ + Produce DLPack capsule (see array API standard). + + Raises: + - TypeError : if the buffer contains unsupported dtypes. + - NotImplementedError : if DLPack support is not implemented + + Useful to have to connect to array libraries. Support optional because + it's not completely trivial to implement for a Python-only library. + """ + raise NotImplementedError("__dlpack__") + + def __dlpack_device__(self) -> tuple[DlpackDeviceType, int | None]: + """ + Device type and device ID for where the data in the buffer resides. + Uses device type codes matching DLPack. + Note: must be implemented even if ``__dlpack__`` is not. + """ + if self._x.is_cpu: + return (DlpackDeviceType.CPU, None) + else: + raise NotImplementedError("__dlpack_device__") + + def __repr__(self) -> str: + return ( + "PyArrowBuffer(" + + str( + { + "bufsize": self.bufsize, + "ptr": self.ptr, + "device": self.__dlpack_device__()[0].name, + } + ) + + ")" + ) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/interchange/column.py b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/interchange/column.py new file mode 100644 index 0000000000000000000000000000000000000000..82042aca3697ddc96d0bf12a964b8c640d169de9 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/interchange/column.py @@ -0,0 +1,529 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import enum +from typing import ( + Any, + Dict, + Iterable, + Optional, + Tuple, +) + +import sys +if sys.version_info >= (3, 8): + from typing import TypedDict +else: + from typing_extensions import TypedDict + +import pyarrow as pa +import pyarrow.compute as pc +from pyarrow.interchange.buffer import _PyArrowBuffer + + +class DtypeKind(enum.IntEnum): + """ + Integer enum for data types. + + Attributes + ---------- + INT : int + Matches to signed integer data type. + UINT : int + Matches to unsigned integer data type. + FLOAT : int + Matches to floating point data type. + BOOL : int + Matches to boolean data type. + STRING : int + Matches to string data type (UTF-8 encoded). + DATETIME : int + Matches to datetime data type. + CATEGORICAL : int + Matches to categorical data type. + """ + + INT = 0 + UINT = 1 + FLOAT = 2 + BOOL = 20 + STRING = 21 # UTF-8 + DATETIME = 22 + CATEGORICAL = 23 + + +Dtype = Tuple[DtypeKind, int, str, str] # see Column.dtype + + +_PYARROW_KINDS = { + pa.int8(): (DtypeKind.INT, "c"), + pa.int16(): (DtypeKind.INT, "s"), + pa.int32(): (DtypeKind.INT, "i"), + pa.int64(): (DtypeKind.INT, "l"), + pa.uint8(): (DtypeKind.UINT, "C"), + pa.uint16(): (DtypeKind.UINT, "S"), + pa.uint32(): (DtypeKind.UINT, "I"), + pa.uint64(): (DtypeKind.UINT, "L"), + pa.float16(): (DtypeKind.FLOAT, "e"), + pa.float32(): (DtypeKind.FLOAT, "f"), + pa.float64(): (DtypeKind.FLOAT, "g"), + pa.bool_(): (DtypeKind.BOOL, "b"), + pa.string(): (DtypeKind.STRING, "u"), + pa.large_string(): (DtypeKind.STRING, "U"), +} + + +class ColumnNullType(enum.IntEnum): + """ + Integer enum for null type representation. + + Attributes + ---------- + NON_NULLABLE : int + Non-nullable column. + USE_NAN : int + Use explicit float NaN value. + USE_SENTINEL : int + Sentinel value besides NaN. + USE_BITMASK : int + The bit is set/unset representing a null on a certain position. + USE_BYTEMASK : int + The byte is set/unset representing a null on a certain position. + """ + + NON_NULLABLE = 0 + USE_NAN = 1 + USE_SENTINEL = 2 + USE_BITMASK = 3 + USE_BYTEMASK = 4 + + +class ColumnBuffers(TypedDict): + # first element is a buffer containing the column data; + # second element is the data buffer's associated dtype + data: Tuple[_PyArrowBuffer, Dtype] + + # first element is a buffer containing mask values indicating missing data; + # second element is the mask value buffer's associated dtype. + # None if the null representation is not a bit or byte mask + validity: Optional[Tuple[_PyArrowBuffer, Dtype]] + + # first element is a buffer containing the offset values for + # variable-size binary data (e.g., variable-length strings); + # second element is the offsets buffer's associated dtype. + # None if the data buffer does not have an associated offsets buffer + offsets: Optional[Tuple[_PyArrowBuffer, Dtype]] + + +class CategoricalDescription(TypedDict): + # whether the ordering of dictionary indices is semantically meaningful + is_ordered: bool + # whether a dictionary-style mapping of categorical values to other objects + # exists + is_dictionary: bool + # Python-level only (e.g. ``{int: str}``). + # None if not a dictionary-style categorical. + categories: Optional[_PyArrowColumn] + + +class Endianness: + """Enum indicating the byte-order of a data-type.""" + + LITTLE = "<" + BIG = ">" + NATIVE = "=" + NA = "|" + + +class NoBufferPresent(Exception): + """Exception to signal that there is no requested buffer.""" + + +class _PyArrowColumn: + """ + A column object, with only the methods and properties required by the + interchange protocol defined. + + A column can contain one or more chunks. Each chunk can contain up to three + buffers - a data buffer, a mask buffer (depending on null representation), + and an offsets buffer (if variable-size binary; e.g., variable-length + strings). + + TBD: Arrow has a separate "null" dtype, and has no separate mask concept. + Instead, it seems to use "children" for both columns with a bit mask, + and for nested dtypes. Unclear whether this is elegant or confusing. + This design requires checking the null representation explicitly. + + The Arrow design requires checking: + 1. the ARROW_FLAG_NULLABLE (for sentinel values) + 2. if a column has two children, combined with one of those children + having a null dtype. + + Making the mask concept explicit seems useful. One null dtype would + not be enough to cover both bit and byte masks, so that would mean + even more checking if we did it the Arrow way. + + TBD: there's also the "chunk" concept here, which is implicit in Arrow as + multiple buffers per array (= column here). Semantically it may make + sense to have both: chunks were meant for example for lazy evaluation + of data which doesn't fit in memory, while multiple buffers per column + could also come from doing a selection operation on a single + contiguous buffer. + + Given these concepts, one would expect chunks to be all of the same + size (say a 10,000 row dataframe could have 10 chunks of 1,000 rows), + while multiple buffers could have data-dependent lengths. Not an issue + in pandas if one column is backed by a single NumPy array, but in + Arrow it seems possible. + Are multiple chunks *and* multiple buffers per column necessary for + the purposes of this interchange protocol, or must producers either + reuse the chunk concept for this or copy the data? + + Note: this Column object can only be produced by ``__dataframe__``, so + doesn't need its own version or ``__column__`` protocol. + """ + + def __init__( + self, column: pa.Array | pa.ChunkedArray, allow_copy: bool = True + ) -> None: + """ + Handles PyArrow Arrays and ChunkedArrays. + """ + # Store the column as a private attribute + if isinstance(column, pa.ChunkedArray): + if column.num_chunks == 1: + column = column.chunk(0) + else: + if not allow_copy: + raise RuntimeError( + "Chunks will be combined and a copy is required which " + "is forbidden by allow_copy=False" + ) + column = column.combine_chunks() + + self._allow_copy = allow_copy + + if pa.types.is_boolean(column.type): + if not allow_copy: + raise RuntimeError( + "Boolean column will be casted to uint8 and a copy " + "is required which is forbidden by allow_copy=False" + ) + self._dtype = self._dtype_from_arrowdtype(column.type, 8) + self._col = pc.cast(column, pa.uint8()) + else: + self._col = column + dtype = self._col.type + try: + bit_width = dtype.bit_width + except ValueError: + # in case of a variable-length strings, considered as array + # of bytes (8 bits) + bit_width = 8 + self._dtype = self._dtype_from_arrowdtype(dtype, bit_width) + + def size(self) -> int: + """ + Size of the column, in elements. + + Corresponds to DataFrame.num_rows() if column is a single chunk; + equal to size of this current chunk otherwise. + + Is a method rather than a property because it may cause a (potentially + expensive) computation for some dataframe implementations. + """ + return len(self._col) + + @property + def offset(self) -> int: + """ + Offset of first element. + + May be > 0 if using chunks; for example for a column with N chunks of + equal size M (only the last chunk may be shorter), + ``offset = n * M``, ``n = 0 .. N-1``. + """ + return self._col.offset + + @property + def dtype(self) -> Tuple[DtypeKind, int, str, str]: + """ + Dtype description as a tuple ``(kind, bit-width, format string, + endianness)``. + + Bit-width : the number of bits as an integer + Format string : data type description format string in Apache Arrow C + Data Interface format. + Endianness : current only native endianness (``=``) is supported + + Notes: + - Kind specifiers are aligned with DLPack where possible (hence the + jump to 20, leave enough room for future extension) + - Masks must be specified as boolean with either bit width 1 (for + bit masks) or 8 (for byte masks). + - Dtype width in bits was preferred over bytes + - Endianness isn't too useful, but included now in case in the + future we need to support non-native endianness + - Went with Apache Arrow format strings over NumPy format strings + because they're more complete from a dataframe perspective + - Format strings are mostly useful for datetime specification, and + for categoricals. + - For categoricals, the format string describes the type of the + categorical in the data buffer. In case of a separate encoding of + the categorical (e.g. an integer to string mapping), this can + be derived from ``self.describe_categorical``. + - Data types not included: complex, Arrow-style null, binary, + decimal, and nested (list, struct, map, union) dtypes. + """ + return self._dtype + + def _dtype_from_arrowdtype( + self, dtype: pa.DataType, bit_width: int + ) -> Tuple[DtypeKind, int, str, str]: + """ + See `self.dtype` for details. + """ + # Note: 'c' (complex) not handled yet (not in array spec v1). + # 'b', 'B' (bytes), 'S', 'a', (old-style string) 'V' (void) + # not handled datetime and timedelta both map to datetime + # (is timedelta handled?) + + if pa.types.is_timestamp(dtype): + kind = DtypeKind.DATETIME + ts = dtype.unit[0] + tz = dtype.tz if dtype.tz else "" + f_string = f"ts{ts}:{tz}" + return kind, bit_width, f_string, Endianness.NATIVE + elif pa.types.is_dictionary(dtype): + kind = DtypeKind.CATEGORICAL + arr = self._col + indices_dtype = arr.indices.type + _, f_string = _PYARROW_KINDS.get(indices_dtype) + return kind, bit_width, f_string, Endianness.NATIVE + else: + kind, f_string = _PYARROW_KINDS.get(dtype, (None, None)) + if kind is None: + raise ValueError( + f"Data type {dtype} not supported by interchange protocol") + + return kind, bit_width, f_string, Endianness.NATIVE + + @property + def describe_categorical(self) -> CategoricalDescription: + """ + If the dtype is categorical, there are two options: + - There are only values in the data buffer. + - There is a separate non-categorical Column encoding categorical + values. + + Raises TypeError if the dtype is not categorical + + Returns the dictionary with description on how to interpret the + data buffer: + - "is_ordered" : bool, whether the ordering of dictionary indices + is semantically meaningful. + - "is_dictionary" : bool, whether a mapping of + categorical values to other objects exists + - "categories" : Column representing the (implicit) mapping of + indices to category values (e.g. an array of + cat1, cat2, ...). None if not a dictionary-style + categorical. + + TBD: are there any other in-memory representations that are needed? + """ + arr = self._col + if not pa.types.is_dictionary(arr.type): + raise TypeError( + "describe_categorical only works on a column with " + "categorical dtype!" + ) + + return { + "is_ordered": self._col.type.ordered, + "is_dictionary": True, + "categories": _PyArrowColumn(arr.dictionary), + } + + @property + def describe_null(self) -> Tuple[ColumnNullType, Any]: + """ + Return the missing value (or "null") representation the column dtype + uses, as a tuple ``(kind, value)``. + + Value : if kind is "sentinel value", the actual value. If kind is a bit + mask or a byte mask, the value (0 or 1) indicating a missing value. + None otherwise. + """ + # In case of no missing values, we need to set ColumnNullType to + # non nullable as in the current __dataframe__ protocol bit/byte masks + # cannot be None + if self.null_count == 0: + return ColumnNullType.NON_NULLABLE, None + else: + return ColumnNullType.USE_BITMASK, 0 + + @property + def null_count(self) -> int: + """ + Number of null elements, if known. + + Note: Arrow uses -1 to indicate "unknown", but None seems cleaner. + """ + arrow_null_count = self._col.null_count + n = arrow_null_count if arrow_null_count != -1 else None + return n + + @property + def metadata(self) -> Dict[str, Any]: + """ + The metadata for the column. See `DataFrame.metadata` for more details. + """ + pass + + def num_chunks(self) -> int: + """ + Return the number of chunks the column consists of. + """ + return 1 + + def get_chunks( + self, n_chunks: Optional[int] = None + ) -> Iterable[_PyArrowColumn]: + """ + Return an iterator yielding the chunks. + + See `DataFrame.get_chunks` for details on ``n_chunks``. + """ + if n_chunks and n_chunks > 1: + chunk_size = self.size() // n_chunks + if self.size() % n_chunks != 0: + chunk_size += 1 + + array = self._col + i = 0 + for start in range(0, chunk_size * n_chunks, chunk_size): + yield _PyArrowColumn( + array.slice(start, chunk_size), self._allow_copy + ) + i += 1 + else: + yield self + + def get_buffers(self) -> ColumnBuffers: + """ + Return a dictionary containing the underlying buffers. + + The returned dictionary has the following contents: + + - "data": a two-element tuple whose first element is a buffer + containing the data and whose second element is the data + buffer's associated dtype. + - "validity": a two-element tuple whose first element is a buffer + containing mask values indicating missing data and + whose second element is the mask value buffer's + associated dtype. None if the null representation is + not a bit or byte mask. + - "offsets": a two-element tuple whose first element is a buffer + containing the offset values for variable-size binary + data (e.g., variable-length strings) and whose second + element is the offsets buffer's associated dtype. None + if the data buffer does not have an associated offsets + buffer. + """ + buffers: ColumnBuffers = { + "data": self._get_data_buffer(), + "validity": None, + "offsets": None, + } + + try: + buffers["validity"] = self._get_validity_buffer() + except NoBufferPresent: + pass + + try: + buffers["offsets"] = self._get_offsets_buffer() + except NoBufferPresent: + pass + + return buffers + + def _get_data_buffer( + self, + ) -> Tuple[_PyArrowBuffer, Any]: # Any is for self.dtype tuple + """ + Return the buffer containing the data and the buffer's + associated dtype. + """ + array = self._col + dtype = self.dtype + + # In case of dictionary arrays, use indices + # to define a buffer, codes are transferred through + # describe_categorical() + if pa.types.is_dictionary(array.type): + array = array.indices + dtype = _PyArrowColumn(array).dtype + + n = len(array.buffers()) + if n == 2: + return _PyArrowBuffer(array.buffers()[1]), dtype + elif n == 3: + return _PyArrowBuffer(array.buffers()[2]), dtype + + def _get_validity_buffer(self) -> Tuple[_PyArrowBuffer, Any]: + """ + Return the buffer containing the mask values indicating missing data + and the buffer's associated dtype. + Raises NoBufferPresent if null representation is not a bit or byte + mask. + """ + # Define the dtype of the returned buffer + dtype = (DtypeKind.BOOL, 1, "b", Endianness.NATIVE) + array = self._col + buff = array.buffers()[0] + if buff: + return _PyArrowBuffer(buff), dtype + else: + raise NoBufferPresent( + "There are no missing values so " + "does not have a separate mask") + + def _get_offsets_buffer(self) -> Tuple[_PyArrowBuffer, Any]: + """ + Return the buffer containing the offset values for variable-size binary + data (e.g., variable-length strings) and the buffer's associated dtype. + Raises NoBufferPresent if the data buffer does not have an associated + offsets buffer. + """ + array = self._col + n = len(array.buffers()) + if n == 2: + raise NoBufferPresent( + "This column has a fixed-length dtype so " + "it does not have an offsets buffer" + ) + elif n == 3: + # Define the dtype of the returned buffer + dtype = self._col.type + if pa.types.is_large_string(dtype): + dtype = (DtypeKind.INT, 64, "l", Endianness.NATIVE) + else: + dtype = (DtypeKind.INT, 32, "i", Endianness.NATIVE) + return _PyArrowBuffer(array.buffers()[1]), dtype diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/interchange/dataframe.py b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/interchange/dataframe.py new file mode 100644 index 0000000000000000000000000000000000000000..df9dbbc3a76ca4d00fcbf48d90009803d8204d69 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/interchange/dataframe.py @@ -0,0 +1,217 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations +from typing import ( + Any, + Iterable, + Optional, + Sequence, +) + +import pyarrow as pa + +from pyarrow.interchange.column import _PyArrowColumn + + +class _PyArrowDataFrame: + """ + A data frame class, with only the methods required by the interchange + protocol defined. + + A "data frame" represents an ordered collection of named columns. + A column's "name" must be a unique string. + Columns may be accessed by name or by position. + + This could be a public data frame class, or an object with the methods and + attributes defined on this DataFrame class could be returned from the + ``__dataframe__`` method of a public data frame class in a library adhering + to the dataframe interchange protocol specification. + """ + + def __init__( + self, df: pa.Table | pa.RecordBatch, + nan_as_null: bool = False, + allow_copy: bool = True + ) -> None: + """ + Constructor - an instance of this (private) class is returned from + `pa.Table.__dataframe__` or `pa.RecordBatch.__dataframe__`. + """ + self._df = df + # ``nan_as_null`` is a keyword intended for the consumer to tell the + # producer to overwrite null values in the data with ``NaN`` (or + # ``NaT``). + if nan_as_null is True: + raise RuntimeError( + "nan_as_null=True currently has no effect, " + "use the default nan_as_null=False" + ) + self._nan_as_null = nan_as_null + self._allow_copy = allow_copy + + def __dataframe__( + self, nan_as_null: bool = False, allow_copy: bool = True + ) -> _PyArrowDataFrame: + """ + Construct a new exchange object, potentially changing the parameters. + ``nan_as_null`` is a keyword intended for the consumer to tell the + producer to overwrite null values in the data with ``NaN``. + It is intended for cases where the consumer does not support the bit + mask or byte mask that is the producer's native representation. + ``allow_copy`` is a keyword that defines whether or not the library is + allowed to make a copy of the data. For example, copying data would be + necessary if a library supports strided buffers, given that this + protocol specifies contiguous buffers. + """ + return _PyArrowDataFrame(self._df, nan_as_null, allow_copy) + + @property + def metadata(self) -> dict[str, Any]: + """ + The metadata for the data frame, as a dictionary with string keys. The + contents of `metadata` may be anything, they are meant for a library + to store information that it needs to, e.g., roundtrip losslessly or + for two implementations to share data that is not (yet) part of the + interchange protocol specification. For avoiding collisions with other + entries, please add name the keys with the name of the library + followed by a period and the desired name, e.g, ``pandas.indexcol``. + """ + # The metadata for the data frame, as a dictionary with string keys. + # Add schema metadata here (pandas metadata or custom metadata) + if self._df.schema.metadata: + schema_metadata = {"pyarrow." + k.decode('utf8'): v.decode('utf8') + for k, v in self._df.schema.metadata.items()} + return schema_metadata + else: + return {} + + def num_columns(self) -> int: + """ + Return the number of columns in the DataFrame. + """ + return self._df.num_columns + + def num_rows(self) -> int: + """ + Return the number of rows in the DataFrame, if available. + """ + return self._df.num_rows + + def num_chunks(self) -> int: + """ + Return the number of chunks the DataFrame consists of. + """ + if isinstance(self._df, pa.RecordBatch): + return 1 + else: + # pyarrow.Table can have columns with different number + # of chunks so we take the number of chunks that + # .to_batches() returns as it takes the min chunk size + # of all the columns (to_batches is a zero copy method) + batches = self._df.to_batches() + return len(batches) + + def column_names(self) -> Iterable[str]: + """ + Return an iterator yielding the column names. + """ + return self._df.schema.names + + def get_column(self, i: int) -> _PyArrowColumn: + """ + Return the column at the indicated position. + """ + return _PyArrowColumn(self._df.column(i), + allow_copy=self._allow_copy) + + def get_column_by_name(self, name: str) -> _PyArrowColumn: + """ + Return the column whose name is the indicated name. + """ + return _PyArrowColumn(self._df.column(name), + allow_copy=self._allow_copy) + + def get_columns(self) -> Iterable[_PyArrowColumn]: + """ + Return an iterator yielding the columns. + """ + return [ + _PyArrowColumn(col, allow_copy=self._allow_copy) + for col in self._df.columns + ] + + def select_columns(self, indices: Sequence[int]) -> _PyArrowDataFrame: + """ + Create a new DataFrame by selecting a subset of columns by index. + """ + return _PyArrowDataFrame( + self._df.select(list(indices)), self._nan_as_null, self._allow_copy + ) + + def select_columns_by_name( + self, names: Sequence[str] + ) -> _PyArrowDataFrame: + """ + Create a new DataFrame by selecting a subset of columns by name. + """ + return _PyArrowDataFrame( + self._df.select(list(names)), self._nan_as_null, self._allow_copy + ) + + def get_chunks( + self, n_chunks: Optional[int] = None + ) -> Iterable[_PyArrowDataFrame]: + """ + Return an iterator yielding the chunks. + + By default (None), yields the chunks that the data is stored as by the + producer. If given, ``n_chunks`` must be a multiple of + ``self.num_chunks()``, meaning the producer must subdivide each chunk + before yielding it. + + Note that the producer must ensure that all columns are chunked the + same way. + """ + # Subdivide chunks + if n_chunks and n_chunks > 1: + chunk_size = self.num_rows() // n_chunks + if self.num_rows() % n_chunks != 0: + chunk_size += 1 + if isinstance(self._df, pa.Table): + batches = self._df.to_batches(max_chunksize=chunk_size) + else: + batches = [] + for start in range(0, chunk_size * n_chunks, chunk_size): + batches.append(self._df.slice(start, chunk_size)) + # In case when the size of the chunk is such that the resulting + # list is one less chunk then n_chunks -> append an empty chunk + if len(batches) == n_chunks - 1: + batches.append(pa.record_batch([[]], schema=self._df.schema)) + # yields the chunks that the data is stored as + else: + if isinstance(self._df, pa.Table): + batches = self._df.to_batches() + else: + batches = [self._df] + + # Create an iterator of RecordBatches + iterator = [_PyArrowDataFrame(batch, + self._nan_as_null, + self._allow_copy) + for batch in batches] + return iterator diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/interchange/from_dataframe.py b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/interchange/from_dataframe.py new file mode 100644 index 0000000000000000000000000000000000000000..3048095d99aba7050520bc772c0d5c6a23f95730 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/interchange/from_dataframe.py @@ -0,0 +1,589 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from typing import ( + Any, + Tuple, +) + +from pyarrow.interchange.column import ( + DtypeKind, + ColumnBuffers, + ColumnNullType, +) + +import pyarrow as pa +import re + +import pyarrow.compute as pc +from pyarrow.interchange.column import Dtype + + +# A typing protocol could be added later to let Mypy validate code using +# `from_dataframe` better. +DataFrameObject = Any +ColumnObject = Any +BufferObject = Any + + +_PYARROW_DTYPES: dict[DtypeKind, dict[int, Any]] = { + DtypeKind.INT: {8: pa.int8(), + 16: pa.int16(), + 32: pa.int32(), + 64: pa.int64()}, + DtypeKind.UINT: {8: pa.uint8(), + 16: pa.uint16(), + 32: pa.uint32(), + 64: pa.uint64()}, + DtypeKind.FLOAT: {16: pa.float16(), + 32: pa.float32(), + 64: pa.float64()}, + DtypeKind.BOOL: {1: pa.bool_(), + 8: pa.uint8()}, + DtypeKind.STRING: {8: pa.string()}, +} + + +def from_dataframe(df: DataFrameObject, allow_copy=True) -> pa.Table: + """ + Build a ``pa.Table`` from any DataFrame supporting the interchange protocol. + + Parameters + ---------- + df : DataFrameObject + Object supporting the interchange protocol, i.e. `__dataframe__` + method. + allow_copy : bool, default: True + Whether to allow copying the memory to perform the conversion + (if false then zero-copy approach is requested). + + Returns + ------- + pa.Table + """ + if isinstance(df, pa.Table): + return df + elif isinstance(df, pa.RecordBatch): + return pa.Table.from_batches([df]) + + if not hasattr(df, "__dataframe__"): + raise ValueError("`df` does not support __dataframe__") + + return _from_dataframe(df.__dataframe__(allow_copy=allow_copy), + allow_copy=allow_copy) + + +def _from_dataframe(df: DataFrameObject, allow_copy=True): + """ + Build a ``pa.Table`` from the DataFrame interchange object. + + Parameters + ---------- + df : DataFrameObject + Object supporting the interchange protocol, i.e. `__dataframe__` + method. + allow_copy : bool, default: True + Whether to allow copying the memory to perform the conversion + (if false then zero-copy approach is requested). + + Returns + ------- + pa.Table + """ + batches = [] + for chunk in df.get_chunks(): + batch = protocol_df_chunk_to_pyarrow(chunk, allow_copy) + batches.append(batch) + + if not batches: + batch = protocol_df_chunk_to_pyarrow(df) + batches.append(batch) + + return pa.Table.from_batches(batches) + + +def protocol_df_chunk_to_pyarrow( + df: DataFrameObject, + allow_copy: bool = True +) -> pa.RecordBatch: + """ + Convert interchange protocol chunk to ``pa.RecordBatch``. + + Parameters + ---------- + df : DataFrameObject + Object supporting the interchange protocol, i.e. `__dataframe__` + method. + allow_copy : bool, default: True + Whether to allow copying the memory to perform the conversion + (if false then zero-copy approach is requested). + + Returns + ------- + pa.RecordBatch + """ + # We need a dict of columns here, with each column being a pa.Array + columns: dict[str, pa.Array] = {} + for name in df.column_names(): + if not isinstance(name, str): + raise ValueError(f"Column {name} is not a string") + if name in columns: + raise ValueError(f"Column {name} is not unique") + col = df.get_column_by_name(name) + dtype = col.dtype[0] + if dtype in ( + DtypeKind.INT, + DtypeKind.UINT, + DtypeKind.FLOAT, + DtypeKind.STRING, + DtypeKind.DATETIME, + ): + columns[name] = column_to_array(col, allow_copy) + elif dtype == DtypeKind.BOOL: + columns[name] = bool_column_to_array(col, allow_copy) + elif dtype == DtypeKind.CATEGORICAL: + columns[name] = categorical_column_to_dictionary(col, allow_copy) + else: + raise NotImplementedError(f"Data type {dtype} not handled yet") + + return pa.RecordBatch.from_pydict(columns) + + +def column_to_array( + col: ColumnObject, + allow_copy: bool = True, +) -> pa.Array: + """ + Convert a column holding one of the primitive dtypes to a PyArrow array. + A primitive type is one of: int, uint, float, bool (1 bit). + + Parameters + ---------- + col : ColumnObject + allow_copy : bool, default: True + Whether to allow copying the memory to perform the conversion + (if false then zero-copy approach is requested). + + Returns + ------- + pa.Array + """ + buffers = col.get_buffers() + data_type = col.dtype + data = buffers_to_array(buffers, data_type, + col.size(), + col.describe_null, + col.offset, + allow_copy) + return data + + +def bool_column_to_array( + col: ColumnObject, + allow_copy: bool = True, +) -> pa.Array: + """ + Convert a column holding boolean dtype to a PyArrow array. + + Parameters + ---------- + col : ColumnObject + allow_copy : bool, default: True + Whether to allow copying the memory to perform the conversion + (if false then zero-copy approach is requested). + + Returns + ------- + pa.Array + """ + buffers = col.get_buffers() + size = buffers["data"][1][1] + + # If booleans are byte-packed a copy to bit-packed will be made + if size == 8 and not allow_copy: + raise RuntimeError( + "Boolean column will be casted from uint8 and a copy " + "is required which is forbidden by allow_copy=False" + ) + + data_type = col.dtype + data = buffers_to_array(buffers, data_type, + col.size(), + col.describe_null, + col.offset) + if size == 8: + data = pc.cast(data, pa.bool_()) + + return data + + +def categorical_column_to_dictionary( + col: ColumnObject, + allow_copy: bool = True, +) -> pa.DictionaryArray: + """ + Convert a column holding categorical data to a pa.DictionaryArray. + + Parameters + ---------- + col : ColumnObject + allow_copy : bool, default: True + Whether to allow copying the memory to perform the conversion + (if false then zero-copy approach is requested). + + Returns + ------- + pa.DictionaryArray + """ + if not allow_copy: + raise RuntimeError( + "Categorical column will be casted from uint8 and a copy " + "is required which is forbidden by allow_copy=False" + ) + + categorical = col.describe_categorical + + if not categorical["is_dictionary"]: + raise NotImplementedError( + "Non-dictionary categoricals not supported yet") + + # We need to first convert the dictionary column + cat_column = categorical["categories"] + dictionary = column_to_array(cat_column) + # Then we need to convert the indices + # Here we need to use the buffer data type! + buffers = col.get_buffers() + _, data_type = buffers["data"] + indices = buffers_to_array(buffers, data_type, + col.size(), + col.describe_null, + col.offset) + + # Constructing a pa.DictionaryArray + dict_array = pa.DictionaryArray.from_arrays(indices, dictionary) + + return dict_array + + +def parse_datetime_format_str(format_str): + """Parse datetime `format_str` to interpret the `data`.""" + + # timestamp 'ts{unit}:tz' + timestamp_meta = re.match(r"ts([smun]):(.*)", format_str) + if timestamp_meta: + unit, tz = timestamp_meta.group(1), timestamp_meta.group(2) + if unit != "s": + # the format string describes only a first letter of the unit, so + # add one extra letter to convert the unit to numpy-style: + # 'm' -> 'ms', 'u' -> 'us', 'n' -> 'ns' + unit += "s" + + return unit, tz + + raise NotImplementedError(f"DateTime kind is not supported: {format_str}") + + +def map_date_type(data_type): + """Map column date type to pyarrow date type. """ + kind, bit_width, f_string, _ = data_type + + if kind == DtypeKind.DATETIME: + unit, tz = parse_datetime_format_str(f_string) + return pa.timestamp(unit, tz=tz) + else: + pa_dtype = _PYARROW_DTYPES.get(kind, {}).get(bit_width, None) + + # Error if dtype is not supported + if pa_dtype: + return pa_dtype + else: + raise NotImplementedError( + f"Conversion for {data_type} is not yet supported.") + + +def buffers_to_array( + buffers: ColumnBuffers, + data_type: Tuple[DtypeKind, int, str, str], + length: int, + describe_null: ColumnNullType, + offset: int = 0, + allow_copy: bool = True, +) -> pa.Array: + """ + Build a PyArrow array from the passed buffer. + + Parameters + ---------- + buffer : ColumnBuffers + Dictionary containing tuples of underlying buffers and + their associated dtype. + data_type : Tuple[DtypeKind, int, str, str], + Dtype description of the column as a tuple ``(kind, bit-width, format string, + endianness)``. + length : int + The number of values in the array. + describe_null: ColumnNullType + Null representation the column dtype uses, + as a tuple ``(kind, value)`` + offset : int, default: 0 + Number of elements to offset from the start of the buffer. + allow_copy : bool, default: True + Whether to allow copying the memory to perform the conversion + (if false then zero-copy approach is requested). + + Returns + ------- + pa.Array + + Notes + ----- + The returned array doesn't own the memory. The caller of this function + is responsible for keeping the memory owner object alive as long as + the returned PyArrow array is being used. + """ + data_buff, _ = buffers["data"] + try: + validity_buff, validity_dtype = buffers["validity"] + except TypeError: + validity_buff = None + try: + offset_buff, offset_dtype = buffers["offsets"] + except TypeError: + offset_buff = None + + # Construct a pyarrow Buffer + data_pa_buffer = pa.foreign_buffer(data_buff.ptr, data_buff.bufsize, + base=data_buff) + + # Construct a validity pyarrow Buffer, if applicable + if validity_buff: + validity_pa_buff = validity_buffer_from_mask(validity_buff, + validity_dtype, + describe_null, + length, + offset, + allow_copy) + else: + validity_pa_buff = validity_buffer_nan_sentinel(data_pa_buffer, + data_type, + describe_null, + length, + offset, + allow_copy) + + # Construct a pyarrow Array from buffers + data_dtype = map_date_type(data_type) + + if offset_buff: + _, offset_bit_width, _, _ = offset_dtype + # If an offset buffer exists, construct an offset pyarrow Buffer + # and add it to the construction of an array + offset_pa_buffer = pa.foreign_buffer(offset_buff.ptr, + offset_buff.bufsize, + base=offset_buff) + + if data_type[2] == 'U': + string_type = pa.large_string() + else: + if offset_bit_width == 64: + string_type = pa.large_string() + else: + string_type = pa.string() + array = pa.Array.from_buffers( + string_type, + length, + [validity_pa_buff, offset_pa_buffer, data_pa_buffer], + offset=offset, + ) + else: + array = pa.Array.from_buffers( + data_dtype, + length, + [validity_pa_buff, data_pa_buffer], + offset=offset, + ) + + return array + + +def validity_buffer_from_mask( + validity_buff: BufferObject, + validity_dtype: Dtype, + describe_null: ColumnNullType, + length: int, + offset: int = 0, + allow_copy: bool = True, +) -> pa.Buffer: + """ + Build a PyArrow buffer from the passed mask buffer. + + Parameters + ---------- + validity_buff : BufferObject + Tuple of underlying validity buffer and associated dtype. + validity_dtype : Dtype + Dtype description as a tuple ``(kind, bit-width, format string, + endianness)``. + describe_null : ColumnNullType + Null representation the column dtype uses, + as a tuple ``(kind, value)`` + length : int + The number of values in the array. + offset : int, default: 0 + Number of elements to offset from the start of the buffer. + allow_copy : bool, default: True + Whether to allow copying the memory to perform the conversion + (if false then zero-copy approach is requested). + + Returns + ------- + pa.Buffer + """ + null_kind, sentinel_val = describe_null + validity_kind, _, _, _ = validity_dtype + assert validity_kind == DtypeKind.BOOL + + if null_kind == ColumnNullType.NON_NULLABLE: + # Sliced array can have a NON_NULLABLE ColumnNullType due + # to no missing values in that slice of an array though the bitmask + # exists and validity_buff must be set to None in this case + return None + + elif null_kind == ColumnNullType.USE_BYTEMASK or ( + null_kind == ColumnNullType.USE_BITMASK and sentinel_val == 1 + ): + buff = pa.foreign_buffer(validity_buff.ptr, + validity_buff.bufsize, + base=validity_buff) + + if null_kind == ColumnNullType.USE_BYTEMASK: + if not allow_copy: + raise RuntimeError( + "To create a bitmask a copy of the data is " + "required which is forbidden by allow_copy=False" + ) + mask = pa.Array.from_buffers(pa.int8(), length, + [None, buff], + offset=offset) + mask_bool = pc.cast(mask, pa.bool_()) + else: + mask_bool = pa.Array.from_buffers(pa.bool_(), length, + [None, buff], + offset=offset) + + if sentinel_val == 1: + mask_bool = pc.invert(mask_bool) + + return mask_bool.buffers()[1] + + elif null_kind == ColumnNullType.USE_BITMASK and sentinel_val == 0: + return pa.foreign_buffer(validity_buff.ptr, + validity_buff.bufsize, + base=validity_buff) + else: + raise NotImplementedError( + f"{describe_null} null representation is not yet supported.") + + +def validity_buffer_nan_sentinel( + data_pa_buffer: BufferObject, + data_type: Dtype, + describe_null: ColumnNullType, + length: int, + offset: int = 0, + allow_copy: bool = True, +) -> pa.Buffer: + """ + Build a PyArrow buffer from NaN or sentinel values. + + Parameters + ---------- + data_pa_buffer : pa.Buffer + PyArrow buffer for the column data. + data_type : Dtype + Dtype description as a tuple ``(kind, bit-width, format string, + endianness)``. + describe_null : ColumnNullType + Null representation the column dtype uses, + as a tuple ``(kind, value)`` + length : int + The number of values in the array. + offset : int, default: 0 + Number of elements to offset from the start of the buffer. + allow_copy : bool, default: True + Whether to allow copying the memory to perform the conversion + (if false then zero-copy approach is requested). + + Returns + ------- + pa.Buffer + """ + kind, bit_width, _, _ = data_type + data_dtype = map_date_type(data_type) + null_kind, sentinel_val = describe_null + + # Check for float NaN values + if null_kind == ColumnNullType.USE_NAN: + if not allow_copy: + raise RuntimeError( + "To create a bitmask a copy of the data is " + "required which is forbidden by allow_copy=False" + ) + + if kind == DtypeKind.FLOAT and bit_width == 16: + # 'pyarrow.compute.is_nan' kernel not yet implemented + # for float16 + raise NotImplementedError( + f"{data_type} with {null_kind} is not yet supported.") + else: + pyarrow_data = pa.Array.from_buffers( + data_dtype, + length, + [None, data_pa_buffer], + offset=offset, + ) + mask = pc.is_nan(pyarrow_data) + mask = pc.invert(mask) + return mask.buffers()[1] + + # Check for sentinel values + elif null_kind == ColumnNullType.USE_SENTINEL: + if not allow_copy: + raise RuntimeError( + "To create a bitmask a copy of the data is " + "required which is forbidden by allow_copy=False" + ) + + if kind == DtypeKind.DATETIME: + sentinel_dtype = pa.int64() + else: + sentinel_dtype = data_dtype + pyarrow_data = pa.Array.from_buffers(sentinel_dtype, + length, + [None, data_pa_buffer], + offset=offset) + sentinel_arr = pc.equal(pyarrow_data, sentinel_val) + mask_bool = pc.invert(sentinel_arr) + return mask_bool.buffers()[1] + + elif null_kind == ColumnNullType.NON_NULLABLE: + pass + else: + raise NotImplementedError( + f"{describe_null} null representation is not yet supported.") diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/io.pxi b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/io.pxi new file mode 100644 index 0000000000000000000000000000000000000000..6a9f32ecc4115d21c51a9cb68ebcd6701e628471 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/io.pxi @@ -0,0 +1,2912 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Cython wrappers for IO interfaces defined in arrow::io and messaging in +# arrow::ipc + +from libc.stdlib cimport malloc, free + +from cpython.bytearray cimport PyByteArray_FromStringAndSize + +import codecs +import pickle +import re +import sys +import threading +import time +import warnings +from io import BufferedIOBase, IOBase, TextIOBase, UnsupportedOperation +from queue import Queue, Empty as QueueEmpty + +from pyarrow.lib cimport check_status, HaveLibHdfs +from pyarrow.util import _is_path_like, _stringify_path + + +# 64K +DEFAULT_BUFFER_SIZE = 2 ** 16 + + +cdef extern from "Python.h": + # To let us get a PyObject* and avoid Cython auto-ref-counting + PyObject* PyBytes_FromStringAndSizeNative" PyBytes_FromStringAndSize"( + char *v, Py_ssize_t len) except NULL + + +def have_libhdfs(): + """ + Return true if HDFS (HadoopFileSystem) library is set up correctly. + """ + try: + with nogil: + check_status(HaveLibHdfs()) + return True + except Exception: + return False + + +def io_thread_count(): + """ + Return the number of threads to use for I/O operations. + + Many operations, such as scanning a dataset, will implicitly make + use of this pool. The number of threads is set to a fixed value at + startup. It can be modified at runtime by calling + :func:`set_io_thread_count()`. + + See Also + -------- + set_io_thread_count : Modify the size of this pool. + cpu_count : The analogous function for the CPU thread pool. + """ + return GetIOThreadPoolCapacity() + + +def set_io_thread_count(int count): + """ + Set the number of threads to use for I/O operations. + + Many operations, such as scanning a dataset, will implicitly make + use of this pool. + + Parameters + ---------- + count : int + The max number of threads that may be used for I/O. + Must be positive. + + See Also + -------- + io_thread_count : Get the size of this pool. + set_cpu_count : The analogous function for the CPU thread pool. + """ + if count < 1: + raise ValueError("IO thread count must be strictly positive") + check_status(SetIOThreadPoolCapacity(count)) + + +cdef class NativeFile(_Weakrefable): + """ + The base class for all Arrow streams. + + Streams are either readable, writable, or both. + They optionally support seeking. + + While this class exposes methods to read or write data from Python, the + primary intent of using a Arrow stream is to pass it to other Arrow + facilities that will make use of it, such as Arrow IPC routines. + + Be aware that there are subtle differences with regular Python files, + e.g. destroying a writable Arrow stream without closing it explicitly + will not flush any pending data. + """ + + # Default chunk size for chunked reads. + # Use a large enough value for networked filesystems. + _default_chunk_size = 256 * 1024 + + def __cinit__(self): + self.own_file = False + self.is_readable = False + self.is_writable = False + self.is_seekable = False + self._is_appending = False + + def __dealloc__(self): + if self.own_file: + self.close() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, tb): + self.close() + + def __repr__(self): + name = f"pyarrow.{self.__class__.__name__}" + return (f"<{name} " + f"closed={self.closed} " + f"own_file={self.own_file} " + f"is_seekable={self.is_seekable} " + f"is_writable={self.is_writable} " + f"is_readable={self.is_readable}>") + + @property + def mode(self): + """ + The file mode. Currently instances of NativeFile may support: + + * rb: binary read + * wb: binary write + * rb+: binary read and write + * ab: binary append + """ + # Emulate built-in file modes + if self.is_readable and self.is_writable: + return 'rb+' + elif self.is_readable: + return 'rb' + elif self.is_writable and self._is_appending: + return 'ab' + elif self.is_writable: + return 'wb' + else: + raise ValueError('File object is malformed, has no mode') + + def readable(self): + self._assert_open() + return self.is_readable + + def writable(self): + self._assert_open() + return self.is_writable + + def seekable(self): + self._assert_open() + return self.is_seekable + + def isatty(self): + self._assert_open() + return False + + def fileno(self): + """ + NOT IMPLEMENTED + """ + raise UnsupportedOperation() + + @property + def closed(self): + if self.is_readable: + return self.input_stream.get().closed() + elif self.is_writable: + return self.output_stream.get().closed() + else: + return True + + def close(self): + if not self.closed: + with nogil: + if self.is_readable: + check_status(self.input_stream.get().Close()) + else: + check_status(self.output_stream.get().Close()) + + cdef set_random_access_file(self, shared_ptr[CRandomAccessFile] handle): + self.input_stream = handle + self.random_access = handle + self.is_seekable = True + + cdef set_input_stream(self, shared_ptr[CInputStream] handle): + self.input_stream = handle + self.random_access.reset() + self.is_seekable = False + + cdef set_output_stream(self, shared_ptr[COutputStream] handle): + self.output_stream = handle + + cdef shared_ptr[CRandomAccessFile] get_random_access_file(self) except *: + self._assert_readable() + self._assert_seekable() + return self.random_access + + cdef shared_ptr[CInputStream] get_input_stream(self) except *: + self._assert_readable() + return self.input_stream + + cdef shared_ptr[COutputStream] get_output_stream(self) except *: + self._assert_writable() + return self.output_stream + + def _assert_open(self): + if self.closed: + raise ValueError("I/O operation on closed file") + + def _assert_readable(self): + self._assert_open() + if not self.is_readable: + # XXX UnsupportedOperation + raise IOError("only valid on readable files") + + def _assert_writable(self): + self._assert_open() + if not self.is_writable: + raise IOError("only valid on writable files") + + def _assert_seekable(self): + self._assert_open() + if not self.is_seekable: + raise IOError("only valid on seekable files") + + def size(self): + """ + Return file size + """ + cdef int64_t size + + handle = self.get_random_access_file() + with nogil: + size = GetResultValue(handle.get().GetSize()) + + return size + + def metadata(self): + """ + Return file metadata + """ + cdef: + shared_ptr[const CKeyValueMetadata] c_metadata + + handle = self.get_input_stream() + with nogil: + c_metadata = GetResultValue(handle.get().ReadMetadata()) + + metadata = {} + if c_metadata.get() != nullptr: + for i in range(c_metadata.get().size()): + metadata[frombytes(c_metadata.get().key(i))] = \ + c_metadata.get().value(i) + return metadata + + def tell(self): + """ + Return current stream position + """ + cdef int64_t position + + if self.is_readable: + rd_handle = self.get_random_access_file() + with nogil: + position = GetResultValue(rd_handle.get().Tell()) + else: + wr_handle = self.get_output_stream() + with nogil: + position = GetResultValue(wr_handle.get().Tell()) + + return position + + def seek(self, int64_t position, int whence=0): + """ + Change current file stream position + + Parameters + ---------- + position : int + Byte offset, interpreted relative to value of whence argument + whence : int, default 0 + Point of reference for seek offset + + Notes + ----- + Values of whence: + * 0 -- start of stream (the default); offset should be zero or positive + * 1 -- current stream position; offset may be negative + * 2 -- end of stream; offset is usually negative + + Returns + ------- + int + The new absolute stream position. + """ + cdef int64_t offset + handle = self.get_random_access_file() + + with nogil: + if whence == 0: + offset = position + elif whence == 1: + offset = GetResultValue(handle.get().Tell()) + offset = offset + position + elif whence == 2: + offset = GetResultValue(handle.get().GetSize()) + offset = offset + position + else: + with gil: + raise ValueError(f"Invalid value of whence: {whence}") + check_status(handle.get().Seek(offset)) + + return self.tell() + + def flush(self): + """ + Flush the stream, if applicable. + + An error is raised if stream is not writable. + """ + self._assert_open() + # For IOBase compatibility, flush() on an input stream is a no-op + if self.is_writable: + handle = self.get_output_stream() + with nogil: + check_status(handle.get().Flush()) + + def write(self, data): + """ + Write data to the file. + + Parameters + ---------- + data : bytes-like object or exporter of buffer protocol + + Returns + ------- + int + nbytes: number of bytes written + """ + self._assert_writable() + handle = self.get_output_stream() + + cdef shared_ptr[CBuffer] buf = as_c_buffer(data) + + with nogil: + check_status(handle.get().WriteBuffer(buf)) + return buf.get().size() + + def read(self, nbytes=None): + """ + Read and return up to n bytes. + + If *nbytes* is None, then the entire remaining file contents are read. + + Parameters + ---------- + nbytes : int, default None + + Returns + ------- + data : bytes + """ + cdef: + int64_t c_nbytes + int64_t bytes_read = 0 + PyObject* obj + + if nbytes is None: + if not self.is_seekable: + # Cannot get file size => read chunkwise + bs = self._default_chunk_size + chunks = [] + while True: + chunk = self.read(bs) + if not chunk: + break + chunks.append(chunk) + return b"".join(chunks) + + c_nbytes = self.size() - self.tell() + else: + c_nbytes = nbytes + + handle = self.get_input_stream() + + # Allocate empty write space + obj = PyBytes_FromStringAndSizeNative(NULL, c_nbytes) + + cdef uint8_t* buf = cp.PyBytes_AS_STRING( obj) + with nogil: + bytes_read = GetResultValue(handle.get().Read(c_nbytes, buf)) + + if bytes_read < c_nbytes: + cp._PyBytes_Resize(&obj, bytes_read) + + return PyObject_to_object(obj) + + def get_stream(self, file_offset, nbytes): + """ + Return an input stream that reads a file segment independent of the + state of the file. + + Allows reading portions of a random access file as an input stream + without interfering with each other. + + Parameters + ---------- + file_offset : int + nbytes : int + + Returns + ------- + stream : NativeFile + """ + cdef: + shared_ptr[CInputStream] data + int64_t c_file_offset + int64_t c_nbytes + + c_file_offset = file_offset + c_nbytes = nbytes + + handle = self.get_random_access_file() + + data = GetResultValue( + CRandomAccessFile.GetStream(handle, c_file_offset, c_nbytes)) + + stream = NativeFile() + stream.set_input_stream(data) + stream.is_readable = True + + return stream + + def read_at(self, nbytes, offset): + """ + Read indicated number of bytes at offset from the file + + Parameters + ---------- + nbytes : int + offset : int + + Returns + ------- + data : bytes + """ + cdef: + int64_t c_nbytes + int64_t c_offset + int64_t bytes_read = 0 + PyObject* obj + + c_nbytes = nbytes + + c_offset = offset + + handle = self.get_random_access_file() + + # Allocate empty write space + obj = PyBytes_FromStringAndSizeNative(NULL, c_nbytes) + + cdef uint8_t* buf = cp.PyBytes_AS_STRING( obj) + with nogil: + bytes_read = GetResultValue(handle.get(). + ReadAt(c_offset, c_nbytes, buf)) + + if bytes_read < c_nbytes: + cp._PyBytes_Resize(&obj, bytes_read) + + return PyObject_to_object(obj) + + def read1(self, nbytes=None): + """Read and return up to n bytes. + + Unlike read(), if *nbytes* is None then a chunk is read, not the + entire file. + + Parameters + ---------- + nbytes : int, default None + The maximum number of bytes to read. + + Returns + ------- + data : bytes + """ + if nbytes is None: + # The expectation when passing `nbytes=None` is not to read the + # entire file but to issue a single underlying read call up to + # a reasonable size (the use case being to read a bufferable + # amount of bytes, such as with io.TextIOWrapper). + nbytes = self._default_chunk_size + return self.read(nbytes) + + def readall(self): + return self.read() + + def readinto(self, b): + """ + Read into the supplied buffer + + Parameters + ---------- + b : buffer-like object + A writable buffer object (such as a bytearray). + + Returns + ------- + written : int + number of bytes written + """ + + cdef: + int64_t bytes_read + uint8_t* buf + Buffer py_buf + int64_t buf_len + + handle = self.get_input_stream() + + py_buf = py_buffer(b) + buf_len = py_buf.size + buf = py_buf.buffer.get().mutable_data() + + with nogil: + bytes_read = GetResultValue(handle.get().Read(buf_len, buf)) + + return bytes_read + + def readline(self, size=None): + """NOT IMPLEMENTED. Read and return a line of bytes from the file. + + If size is specified, read at most size bytes. + + Line terminator is always b"\\n". + + Parameters + ---------- + size : int + maximum number of bytes read + """ + raise UnsupportedOperation() + + def readlines(self, hint=None): + """NOT IMPLEMENTED. Read lines of the file + + Parameters + ---------- + hint : int + maximum number of bytes read until we stop + """ + raise UnsupportedOperation() + + def __iter__(self): + self._assert_readable() + return self + + def __next__(self): + line = self.readline() + if not line: + raise StopIteration + return line + + def read_buffer(self, nbytes=None): + """ + Read from buffer. + + Parameters + ---------- + nbytes : int, optional + maximum number of bytes read + """ + cdef: + int64_t c_nbytes + int64_t bytes_read = 0 + shared_ptr[CBuffer] output + + handle = self.get_input_stream() + + if nbytes is None: + if not self.is_seekable: + # Cannot get file size => read chunkwise + return py_buffer(self.read()) + c_nbytes = self.size() - self.tell() + else: + c_nbytes = nbytes + + with nogil: + output = GetResultValue(handle.get().ReadBuffer(c_nbytes)) + + return pyarrow_wrap_buffer(output) + + def truncate(self): + """ + NOT IMPLEMENTED + """ + raise UnsupportedOperation() + + def writelines(self, lines): + """ + Write lines to the file. + + Parameters + ---------- + lines : iterable + Iterable of bytes-like objects or exporters of buffer protocol + """ + self._assert_writable() + + for line in lines: + self.write(line) + + def download(self, stream_or_path, buffer_size=None): + """ + Read this file completely to a local path or destination stream. + + This method first seeks to the beginning of the file. + + Parameters + ---------- + stream_or_path : str or file-like object + If a string, a local file path to write to; otherwise, + should be a writable stream. + buffer_size : int, optional + The buffer size to use for data transfers. + """ + cdef: + int64_t bytes_read = 0 + uint8_t* buf + + if not is_threading_enabled(): + return self._download_nothreads(stream_or_path, buffer_size) + + handle = self.get_input_stream() + + buffer_size = buffer_size or DEFAULT_BUFFER_SIZE + + write_queue = Queue(50) + + if not hasattr(stream_or_path, 'read'): + stream = open(stream_or_path, 'wb') + + def cleanup(): + stream.close() + else: + stream = stream_or_path + + def cleanup(): + pass + + done = False + exc_info = None + + def bg_write(): + try: + while not done or write_queue.qsize() > 0: + try: + buf = write_queue.get(timeout=0.01) + except QueueEmpty: + continue + stream.write(buf) + except Exception as e: + exc_info = sys.exc_info() + finally: + cleanup() + + self.seek(0) + + writer_thread = threading.Thread(target=bg_write) + + # This isn't ideal -- PyBytes_FromStringAndSize copies the data from + # the passed buffer, so it's hard for us to avoid doubling the memory + buf = malloc(buffer_size) + if buf == NULL: + raise MemoryError(f"Failed to allocate {buffer_size} bytes") + + writer_thread.start() + + cdef int64_t total_bytes = 0 + cdef int32_t c_buffer_size = buffer_size + + try: + while True: + with nogil: + bytes_read = GetResultValue( + handle.get().Read(c_buffer_size, buf)) + + total_bytes += bytes_read + + # EOF + if bytes_read == 0: + break + + pybuf = cp.PyBytes_FromStringAndSize(buf, + bytes_read) + + if writer_thread.is_alive(): + while write_queue.full(): + time.sleep(0.01) + else: + break + + write_queue.put_nowait(pybuf) + finally: + free(buf) + done = True + + writer_thread.join() + if exc_info is not None: + raise exc_info[0], exc_info[1], exc_info[2] + + def _download_nothreads(self, stream_or_path, buffer_size=None): + """ + Internal method to do a download without separate threads, queues etc. + Called by download above if is_threading_enabled() == False + """ + cdef: + int64_t bytes_read = 0 + uint8_t* buf + + handle = self.get_input_stream() + + buffer_size = buffer_size or DEFAULT_BUFFER_SIZE + + if not hasattr(stream_or_path, 'read'): + stream = open(stream_or_path, 'wb') + + def cleanup(): + stream.close() + else: + stream = stream_or_path + + def cleanup(): + pass + + self.seek(0) + + # This isn't ideal -- PyBytes_FromStringAndSize copies the data from + # the passed buffer, so it's hard for us to avoid doubling the memory + buf = malloc(buffer_size) + if buf == NULL: + raise MemoryError(f"Failed to allocate {buffer_size} bytes") + + cdef int64_t total_bytes = 0 + cdef int32_t c_buffer_size = buffer_size + + try: + while True: + with nogil: + bytes_read = GetResultValue( + handle.get().Read(c_buffer_size, buf)) + + total_bytes += bytes_read + + # EOF + if bytes_read == 0: + break + + pybuf = cp.PyBytes_FromStringAndSize(buf, + bytes_read) + + # no background thread - write on main thread + stream.write(pybuf) + finally: + free(buf) + cleanup() + + def upload(self, stream, buffer_size=None): + """ + Write from a source stream to this file. + + Parameters + ---------- + stream : file-like object + Source stream to pipe to this file. + buffer_size : int, optional + The buffer size to use for data transfers. + """ + if not is_threading_enabled(): + return self._upload_nothreads(stream, buffer_size) + + write_queue = Queue(50) + self._assert_writable() + + buffer_size = buffer_size or DEFAULT_BUFFER_SIZE + + done = False + exc_info = None + + def bg_write(): + try: + while not done or write_queue.qsize() > 0: + try: + buf = write_queue.get(timeout=0.01) + except QueueEmpty: + continue + + self.write(buf) + + except Exception as e: + exc_info = sys.exc_info() + + writer_thread = threading.Thread(target=bg_write) + writer_thread.start() + + try: + while True: + buf = stream.read(buffer_size) + if not buf: + break + + if writer_thread.is_alive(): + while write_queue.full(): + time.sleep(0.01) + else: + break + + write_queue.put_nowait(buf) + finally: + done = True + + writer_thread.join() + if exc_info is not None: + raise exc_info[0], exc_info[1], exc_info[2] + + def _upload_nothreads(self, stream, buffer_size=None): + """ + Internal method to do an upload without separate threads, queues etc. + Called by upload above if is_threading_enabled() == False + """ + self._assert_writable() + + buffer_size = buffer_size or DEFAULT_BUFFER_SIZE + + while True: + buf = stream.read(buffer_size) + if not buf: + break + + # no threading - just write + self.write(buf) + + +BufferedIOBase.register(NativeFile) + +# ---------------------------------------------------------------------- +# Python file-like objects + + +cdef class PythonFile(NativeFile): + """ + A stream backed by a Python file object. + + This class allows using Python file objects with arbitrary Arrow + functions, including functions written in another language than Python. + + As a downside, there is a non-zero redirection cost in translating + Arrow stream calls to Python method calls. Furthermore, Python's + Global Interpreter Lock may limit parallelism in some situations. + + Examples + -------- + >>> import io + >>> import pyarrow as pa + >>> pa.PythonFile(io.BytesIO()) + + + Create a stream for writing: + + >>> buf = io.BytesIO() + >>> f = pa.PythonFile(buf, mode = 'w') + >>> f.writable() + True + >>> f.write(b'PythonFile') + 10 + >>> buf.getvalue() + b'PythonFile' + >>> f.close() + >>> f + + + Create a stream for reading: + + >>> buf = io.BytesIO(b'PythonFile') + >>> f = pa.PythonFile(buf, mode = 'r') + >>> f.mode + 'rb' + >>> f.read() + b'PythonFile' + >>> f + + >>> f.close() + >>> f + + """ + cdef: + object handle + + def __cinit__(self, handle, mode=None): + self.handle = handle + + if mode is None: + try: + inferred_mode = handle.mode + except AttributeError: + # Not all file-like objects have a mode attribute + # (e.g. BytesIO) + try: + inferred_mode = 'w' if handle.writable() else 'r' + except AttributeError: + raise ValueError("could not infer open mode for file-like " + "object %r, please pass it explicitly" + % (handle,)) + else: + inferred_mode = mode + + if inferred_mode.startswith('w'): + kind = 'w' + elif inferred_mode.startswith('r'): + kind = 'r' + else: + raise ValueError(f'Invalid file mode: {mode}') + + # If mode was given, check it matches the given file + if mode is not None: + if isinstance(handle, IOBase): + # Python 3 IO object + if kind == 'r': + if not handle.readable(): + raise TypeError("readable file expected") + else: + if not handle.writable(): + raise TypeError("writable file expected") + # (other duck-typed file-like objects are possible) + + # If possible, check the file is a binary file + if isinstance(handle, TextIOBase): + raise TypeError("binary file expected, got text file") + + if kind == 'r': + self.set_random_access_file( + shared_ptr[CRandomAccessFile](new PyReadableFile(handle))) + self.is_readable = True + else: + self.set_output_stream( + shared_ptr[COutputStream](new PyOutputStream(handle))) + self.is_writable = True + + def truncate(self, pos=None): + """ + Parameters + ---------- + pos : int, optional + """ + self.handle.truncate(pos) + + def readline(self, size=None): + """ + Read and return a line of bytes from the file. + + If size is specified, read at most size bytes. + + Parameters + ---------- + size : int + Maximum number of bytes read + """ + return self.handle.readline(size) + + def readlines(self, hint=None): + """ + Read lines of the file. + + Parameters + ---------- + hint : int + Maximum number of bytes read until we stop + """ + return self.handle.readlines(hint) + + +cdef class MemoryMappedFile(NativeFile): + """ + A stream that represents a memory-mapped file. + + Supports 'r', 'r+', 'w' modes. + + Examples + -------- + Create a new file with memory map: + + >>> import pyarrow as pa + >>> mmap = pa.create_memory_map('example_mmap.dat', 10) + >>> mmap + + >>> mmap.close() + + Open an existing file with memory map: + + >>> with pa.memory_map('example_mmap.dat') as mmap: + ... mmap + ... + + """ + cdef: + shared_ptr[CMemoryMappedFile] handle + object path + + @staticmethod + def create(path, size): + """ + Create a MemoryMappedFile + + Parameters + ---------- + path : str + Where to create the file. + size : int + Size of the memory mapped file. + """ + cdef: + shared_ptr[CMemoryMappedFile] handle + c_string c_path = encode_file_path(path) + int64_t c_size = size + + with nogil: + handle = GetResultValue(CMemoryMappedFile.Create(c_path, c_size)) + + cdef MemoryMappedFile result = MemoryMappedFile() + result.path = path + result.is_readable = True + result.is_writable = True + result.set_output_stream( handle) + result.set_random_access_file( handle) + result.handle = handle + + return result + + def _open(self, path, mode='r'): + self.path = path + + cdef: + FileMode c_mode + shared_ptr[CMemoryMappedFile] handle + c_string c_path = encode_file_path(path) + + if mode in ('r', 'rb'): + c_mode = FileMode_READ + self.is_readable = True + elif mode in ('w', 'wb'): + c_mode = FileMode_WRITE + self.is_writable = True + elif mode in ('r+', 'r+b', 'rb+'): + c_mode = FileMode_READWRITE + self.is_readable = True + self.is_writable = True + else: + raise ValueError(f'Invalid file mode: {mode}') + + with nogil: + handle = GetResultValue(CMemoryMappedFile.Open(c_path, c_mode)) + + self.set_output_stream( handle) + self.set_random_access_file( handle) + self.handle = handle + + def resize(self, new_size): + """ + Resize the map and underlying file. + + Parameters + ---------- + new_size : new size in bytes + """ + check_status(self.handle.get().Resize(new_size)) + + def fileno(self): + self._assert_open() + return self.handle.get().file_descriptor() + + +def memory_map(path, mode='r'): + """ + Open memory map at file path. Size of the memory map cannot change. + + Parameters + ---------- + path : str + mode : {'r', 'r+', 'w'}, default 'r' + Whether the file is opened for reading ('r'), writing ('w') + or both ('r+'). + + Returns + ------- + mmap : MemoryMappedFile + + Examples + -------- + Reading from a memory map without any memory allocation or copying: + + >>> import pyarrow as pa + >>> with pa.output_stream('example_mmap.txt') as stream: + ... stream.write(b'Constructing a buffer referencing the mapped memory') + ... + 51 + >>> with pa.memory_map('example_mmap.txt') as mmap: + ... mmap.read_at(6,45) + ... + b'memory' + """ + _check_is_file(path) + + cdef MemoryMappedFile mmap = MemoryMappedFile() + mmap._open(path, mode) + return mmap + + +cdef _check_is_file(path): + if os.path.isdir(path): + raise IOError(f"Expected file path, but {path} is a directory") + + +def create_memory_map(path, size): + """ + Create a file of the given size and memory-map it. + + Parameters + ---------- + path : str + The file path to create, on the local filesystem. + size : int + The file size to create. + + Returns + ------- + mmap : MemoryMappedFile + + Examples + -------- + Create a file with a memory map: + + >>> import pyarrow as pa + >>> with pa.create_memory_map('example_mmap_create.dat', 27) as mmap: + ... mmap.write(b'Create a memory-mapped file') + ... mmap.read_at(10, 9) + ... + 27 + b'memory-map' + """ + return MemoryMappedFile.create(path, size) + + +cdef class OSFile(NativeFile): + """ + A stream backed by a regular file descriptor. + + Examples + -------- + Create a new file to write to: + + >>> import pyarrow as pa + >>> with pa.OSFile('example_osfile.arrow', mode='w') as f: + ... f.writable() + ... f.write(b'OSFile') + ... f.seekable() + ... + True + 6 + False + + Open the file to read: + + >>> with pa.OSFile('example_osfile.arrow', mode='r') as f: + ... f.mode + ... f.read() + ... + 'rb' + b'OSFile' + + Open the file to append: + + >>> with pa.OSFile('example_osfile.arrow', mode='ab') as f: + ... f.mode + ... f.write(b' is super!') + ... + 'ab' + 10 + >>> with pa.OSFile('example_osfile.arrow') as f: + ... f.read() + ... + b'OSFile is super!' + + Inspect created OSFile: + + >>> pa.OSFile('example_osfile.arrow') + + """ + cdef: + object path + + def __cinit__(self, path, mode='r', MemoryPool memory_pool=None): + _check_is_file(path) + self.path = path + + cdef: + FileMode c_mode + shared_ptr[Readable] handle + c_string c_path = encode_file_path(path) + + if mode in ('r', 'rb'): + self._open_readable(c_path, maybe_unbox_memory_pool(memory_pool)) + elif mode in ('w', 'wb'): + self._open_writable(c_path) + elif mode in ('a', 'ab'): + self._open_writable(c_path, append=True) + else: + raise ValueError(f'Invalid file mode: {mode}') + + cdef _open_readable(self, c_string path, CMemoryPool* pool): + cdef shared_ptr[ReadableFile] handle + + with nogil: + handle = GetResultValue(ReadableFile.Open(path, pool)) + + self.is_readable = True + self.set_random_access_file( handle) + + cdef _open_writable(self, c_string path, c_bool append=False): + with nogil: + self.output_stream = GetResultValue( + FileOutputStream.OpenWithAppend(path, append) + ) + self.is_writable = True + self._is_appending = append + + def fileno(self): + self._assert_open() + return self.handle.file_descriptor() + + +cdef class FixedSizeBufferWriter(NativeFile): + """ + A stream writing to a Arrow buffer. + + Examples + -------- + Create a stream to write to ``pyarrow.Buffer``: + + >>> import pyarrow as pa + >>> buf = pa.allocate_buffer(5) + >>> with pa.output_stream(buf) as stream: + ... stream.write(b'abcde') + ... stream + ... + 5 + + + Inspect the buffer: + + >>> buf.to_pybytes() + b'abcde' + >>> buf + + """ + + def __cinit__(self, Buffer buffer): + self.output_stream.reset(new CFixedSizeBufferWriter(buffer.buffer)) + self.is_writable = True + + def set_memcopy_threads(self, int num_threads): + """ + Parameters + ---------- + num_threads : int + """ + cdef CFixedSizeBufferWriter* writer = \ + self.output_stream.get() + writer.set_memcopy_threads(num_threads) + + def set_memcopy_blocksize(self, int64_t blocksize): + """ + Parameters + ---------- + blocksize : int64 + """ + cdef CFixedSizeBufferWriter* writer = \ + self.output_stream.get() + writer.set_memcopy_blocksize(blocksize) + + def set_memcopy_threshold(self, int64_t threshold): + """ + Parameters + ---------- + threshold : int64 + """ + cdef CFixedSizeBufferWriter* writer = \ + self.output_stream.get() + writer.set_memcopy_threshold(threshold) + + +# ---------------------------------------------------------------------- +# Arrow buffers + + +cdef class Buffer(_Weakrefable): + """ + The base class for all Arrow buffers. + + A buffer represents a contiguous memory area. Many buffers will own + their memory, though not all of them do. + """ + + def __cinit__(self): + pass + + def __init__(self): + raise TypeError("Do not call Buffer's constructor directly, use " + "`pyarrow.py_buffer` function instead.") + + cdef void init(self, const shared_ptr[CBuffer]& buffer): + self.buffer = buffer + self.shape[0] = self.size + self.strides[0] = (1) + + def __len__(self): + return self.size + + def __repr__(self): + name = f"pyarrow.{self.__class__.__name__}" + return (f"<{name} " + f"address={hex(self.address)} " + f"size={self.size} " + f"is_cpu={self.is_cpu} " + f"is_mutable={self.is_mutable}>") + + def _assert_cpu(self): + if not self.is_cpu: + raise NotImplementedError("Implemented only for data on CPU device") + + @property + def size(self): + """ + The buffer size in bytes. + """ + return self.buffer.get().size() + + @property + def address(self): + """ + The buffer's address, as an integer. + + The returned address may point to CPU or device memory. + Use `is_cpu()` to disambiguate. + """ + return self.buffer.get().address() + + def hex(self): + """ + Compute hexadecimal representation of the buffer. + + Returns + ------- + : bytes + """ + self._assert_cpu() + return self.buffer.get().ToHexString() + + @property + def is_mutable(self): + """ + Whether the buffer is mutable. + """ + return self.buffer.get().is_mutable() + + @property + def is_cpu(self): + """ + Whether the buffer is CPU-accessible. + """ + return self.buffer.get().is_cpu() + + @property + def device(self): + """ + The device where the buffer resides. + + Returns + ------- + Device + """ + return Device.wrap(self.buffer.get().device()) + + @property + def memory_manager(self): + """ + The memory manager associated with the buffer. + + Returns + ------- + MemoryManager + """ + return MemoryManager.wrap(self.buffer.get().memory_manager()) + + @property + def device_type(self): + """ + The device type where the buffer resides. + + Returns + ------- + DeviceAllocationType + """ + return _wrap_device_allocation_type(self.buffer.get().device_type()) + + @property + def parent(self): + cdef shared_ptr[CBuffer] parent_buf = self.buffer.get().parent() + + if parent_buf.get() == NULL: + return None + else: + return pyarrow_wrap_buffer(parent_buf) + + def __getitem__(self, key): + if isinstance(key, slice): + if (key.step or 1) != 1: + raise IndexError('only slices with step 1 supported') + return _normalize_slice(self, key) + + return self.getitem(_normalize_index(key, self.size)) + + cdef getitem(self, int64_t i): + self._assert_cpu() + return self.buffer.get().data()[i] + + def slice(self, offset=0, length=None): + """ + Slice this buffer. Memory is not copied. + + You can also use the Python slice notation ``buffer[start:stop]``. + + Parameters + ---------- + offset : int, default 0 + Offset from start of buffer to slice. + length : int, default None + Length of slice (default is until end of Buffer starting from + offset). + + Returns + ------- + sliced : Buffer + A logical view over this buffer. + """ + cdef shared_ptr[CBuffer] result + + if offset < 0: + raise IndexError('Offset must be non-negative') + + if length is None: + result = GetResultValue(SliceBufferSafe(self.buffer, offset)) + else: + result = GetResultValue(SliceBufferSafe(self.buffer, offset, + length)) + return pyarrow_wrap_buffer(result) + + def equals(self, Buffer other): + """ + Determine if two buffers contain exactly the same data. + + Parameters + ---------- + other : Buffer + + Returns + ------- + are_equal : bool + True if buffer contents and size are equal + """ + if self.device != other.device: + raise ValueError( + "Device on which the data resides differs between buffers: " + f"{self.device.type_name} and {other.device.type_name}." + ) + if not self.is_cpu: + if self.address != other.address: + raise NotImplementedError( + "Implemented only for data on CPU device or data with equal " + "addresses" + ) + + cdef c_bool result = False + with nogil: + result = self.buffer.get().Equals(deref(other.buffer.get())) + return result + + def __eq__(self, other): + if isinstance(other, Buffer): + return self.equals(other) + else: + return self.equals(py_buffer(other)) + + def __reduce_ex__(self, protocol): + self._assert_cpu() + + if protocol >= 5: + bufobj = pickle.PickleBuffer(self) + elif self.buffer.get().is_mutable(): + # Need to pass a bytearray to recreate a mutable buffer when + # unpickling. + bufobj = PyByteArray_FromStringAndSize( + self.buffer.get().data(), + self.buffer.get().size()) + else: + bufobj = self.to_pybytes() + return py_buffer, (bufobj,) + + def to_pybytes(self): + """ + Return this buffer as a Python bytes object. Memory is copied. + """ + self._assert_cpu() + + return cp.PyBytes_FromStringAndSize( + self.buffer.get().data(), + self.buffer.get().size()) + + def __getbuffer__(self, cp.Py_buffer* buffer, int flags): + self._assert_cpu() + + if self.buffer.get().is_mutable(): + buffer.readonly = 0 + else: + if flags & cp.PyBUF_WRITABLE: + raise BufferError("Writable buffer requested but Arrow " + "buffer was not mutable") + buffer.readonly = 1 + buffer.buf = self.buffer.get().data() + buffer.len = self.size + if buffer.buf == NULL: + # ARROW-16048: Ensure we don't export a NULL address. + assert buffer.len == 0 + buffer.buf = cp.PyBytes_AS_STRING(b"") + buffer.format = 'b' + buffer.internal = NULL + buffer.itemsize = 1 + buffer.ndim = 1 + buffer.obj = self + buffer.shape = self.shape + buffer.strides = self.strides + buffer.suboffsets = NULL + + +cdef class ResizableBuffer(Buffer): + """ + A base class for buffers that can be resized. + """ + + cdef void init_rz(self, const shared_ptr[CResizableBuffer]& buffer): + self.init( buffer) + + def resize(self, int64_t new_size, shrink_to_fit=False): + """ + Resize buffer to indicated size. + + Parameters + ---------- + new_size : int + New size of buffer (padding may be added internally). + shrink_to_fit : bool, default False + If this is true, the buffer is shrunk when new_size is less + than the current size. + If this is false, the buffer is never shrunk. + """ + cdef c_bool c_shrink_to_fit = shrink_to_fit + with nogil: + check_status(( self.buffer.get()) + .Resize(new_size, c_shrink_to_fit)) + + +cdef shared_ptr[CResizableBuffer] _allocate_buffer(CMemoryPool* pool) except *: + with nogil: + return to_shared(GetResultValue(AllocateResizableBuffer(0, pool))) + + +def allocate_buffer(int64_t size, MemoryPool memory_pool=None, + resizable=False): + """ + Allocate a mutable buffer. + + Parameters + ---------- + size : int + Number of bytes to allocate (plus internal padding) + memory_pool : MemoryPool, optional + The pool to allocate memory from. + If not given, the default memory pool is used. + resizable : bool, default False + If true, the returned buffer is resizable. + + Returns + ------- + buffer : Buffer or ResizableBuffer + """ + cdef: + CMemoryPool* cpool = maybe_unbox_memory_pool(memory_pool) + shared_ptr[CResizableBuffer] c_rz_buffer + shared_ptr[CBuffer] c_buffer + + if resizable: + with nogil: + c_rz_buffer = to_shared(GetResultValue( + AllocateResizableBuffer(size, cpool))) + return pyarrow_wrap_resizable_buffer(c_rz_buffer) + else: + with nogil: + c_buffer = to_shared(GetResultValue(AllocateBuffer(size, cpool))) + return pyarrow_wrap_buffer(c_buffer) + + +cdef class BufferOutputStream(NativeFile): + """ + An output stream that writes to a resizable buffer. + + The buffer is produced as a result when ``getvalue()`` is called. + + Examples + -------- + Create an output stream, write data to it and finalize it with + ``getvalue()``: + + >>> import pyarrow as pa + >>> f = pa.BufferOutputStream() + >>> f.write(b'pyarrow.Buffer') + 14 + >>> f.closed + False + >>> f.getvalue() + + >>> f.closed + True + """ + + cdef: + shared_ptr[CResizableBuffer] buffer + + def __cinit__(self, MemoryPool memory_pool=None): + self.buffer = _allocate_buffer(maybe_unbox_memory_pool(memory_pool)) + self.output_stream.reset(new CBufferOutputStream( + self.buffer)) + self.is_writable = True + + def getvalue(self): + """ + Finalize output stream and return result as pyarrow.Buffer. + + Returns + ------- + value : Buffer + """ + with nogil: + check_status(self.output_stream.get().Close()) + return pyarrow_wrap_buffer( self.buffer) + + +cdef class MockOutputStream(NativeFile): + + def __cinit__(self): + self.output_stream.reset(new CMockOutputStream()) + self.is_writable = True + + def size(self): + handle = self.output_stream.get() + return handle.GetExtentBytesWritten() + + +cdef class BufferReader(NativeFile): + """ + Zero-copy reader from objects convertible to Arrow buffer. + + Parameters + ---------- + obj : Python bytes or pyarrow.Buffer + + Examples + -------- + Create an Arrow input stream and inspect it: + + >>> import pyarrow as pa + >>> data = b'reader data' + >>> buf = memoryview(data) + >>> with pa.input_stream(buf) as stream: + ... stream.size() + ... stream.read(6) + ... stream.seek(7) + ... stream.read(15) + ... + 11 + b'reader' + 7 + b'data' + """ + cdef: + Buffer buffer + + # XXX Needed to make numpydoc happy + def __init__(self, obj): + pass + + def __cinit__(self, object obj): + self.buffer = as_buffer(obj) + self.set_random_access_file(shared_ptr[CRandomAccessFile]( + new CBufferReader(self.buffer.buffer))) + self.is_readable = True + + +cdef class CompressedInputStream(NativeFile): + """ + An input stream wrapper which decompresses data on the fly. + + Parameters + ---------- + stream : string, path, pyarrow.NativeFile, or file-like object + Input stream object to wrap with the compression. + compression : str + The compression type ("bz2", "brotli", "gzip", "lz4" or "zstd"). + + Examples + -------- + Create an output stream which compresses the data: + + >>> import pyarrow as pa + >>> data = b"Compressed stream" + >>> raw = pa.BufferOutputStream() + >>> with pa.CompressedOutputStream(raw, "gzip") as compressed: + ... compressed.write(data) + ... + 17 + + Create an input stream with decompression referencing the + buffer with compressed data: + + >>> cdata = raw.getvalue() + >>> with pa.input_stream(cdata, compression="gzip") as compressed: + ... compressed.read() + ... + b'Compressed stream' + + which actually translates to the use of ``BufferReader``and + ``CompressedInputStream``: + + >>> raw = pa.BufferReader(cdata) + >>> with pa.CompressedInputStream(raw, "gzip") as compressed: + ... compressed.read() + ... + b'Compressed stream' + """ + + def __init__(self, object stream, str compression not None): + cdef: + NativeFile nf + Codec codec = Codec(compression) + shared_ptr[CInputStream] c_reader + shared_ptr[CCompressedInputStream] compressed_stream + nf = get_native_file(stream, False) + c_reader = nf.get_input_stream() + compressed_stream = GetResultValue( + CCompressedInputStream.Make(codec.unwrap(), c_reader) + ) + self.set_input_stream( compressed_stream) + self.is_readable = True + + +cdef class CompressedOutputStream(NativeFile): + """ + An output stream wrapper which compresses data on the fly. + + Parameters + ---------- + stream : string, path, pyarrow.NativeFile, or file-like object + Input stream object to wrap with the compression. + compression : str + The compression type ("bz2", "brotli", "gzip", "lz4" or "zstd"). + + Examples + -------- + Create an output stream which compresses the data: + + >>> import pyarrow as pa + >>> data = b"Compressed stream" + >>> raw = pa.BufferOutputStream() + >>> with pa.CompressedOutputStream(raw, "gzip") as compressed: + ... compressed.write(data) + ... + 17 + """ + + def __init__(self, object stream, str compression not None): + cdef: + Codec codec = Codec(compression) + shared_ptr[COutputStream] c_writer + shared_ptr[CCompressedOutputStream] compressed_stream + get_writer(stream, &c_writer) + compressed_stream = GetResultValue( + CCompressedOutputStream.Make(codec.unwrap(), c_writer) + ) + self.set_output_stream( compressed_stream) + self.is_writable = True + + +ctypedef CBufferedInputStream* _CBufferedInputStreamPtr +ctypedef CBufferedOutputStream* _CBufferedOutputStreamPtr +ctypedef CRandomAccessFile* _RandomAccessFilePtr + + +cdef class BufferedInputStream(NativeFile): + """ + An input stream that performs buffered reads from + an unbuffered input stream, which can mitigate the overhead + of many small reads in some cases. + + Parameters + ---------- + stream : NativeFile + The input stream to wrap with the buffer + buffer_size : int + Size of the temporary read buffer. + memory_pool : MemoryPool + The memory pool used to allocate the buffer. + """ + + def __init__(self, NativeFile stream, int buffer_size, + MemoryPool memory_pool=None): + cdef shared_ptr[CBufferedInputStream] buffered_stream + + if buffer_size <= 0: + raise ValueError('Buffer size must be larger than zero') + buffered_stream = GetResultValue(CBufferedInputStream.Create( + buffer_size, maybe_unbox_memory_pool(memory_pool), + stream.get_input_stream())) + + self.set_input_stream( buffered_stream) + self.is_readable = True + + def detach(self): + """ + Release the raw InputStream. + Further operations on this stream are invalid. + + Returns + ------- + raw : NativeFile + The underlying raw input stream + """ + cdef: + shared_ptr[CInputStream] c_raw + _CBufferedInputStreamPtr buffered + NativeFile raw + + buffered = dynamic_cast[_CBufferedInputStreamPtr]( + self.input_stream.get()) + assert buffered != nullptr + + with nogil: + c_raw = GetResultValue(buffered.Detach()) + + raw = NativeFile() + raw.is_readable = True + # Find out whether the raw stream is a RandomAccessFile + # or a mere InputStream. This helps us support seek() etc. + # selectively. + if dynamic_cast[_RandomAccessFilePtr](c_raw.get()) != nullptr: + raw.set_random_access_file( + static_pointer_cast[CRandomAccessFile, CInputStream](c_raw)) + else: + raw.set_input_stream(c_raw) + return raw + + +cdef class BufferedOutputStream(NativeFile): + """ + An output stream that performs buffered reads from + an unbuffered output stream, which can mitigate the overhead + of many small writes in some cases. + + Parameters + ---------- + stream : NativeFile + The writable output stream to wrap with the buffer + buffer_size : int + Size of the buffer that should be added. + memory_pool : MemoryPool + The memory pool used to allocate the buffer. + """ + + def __init__(self, NativeFile stream, int buffer_size, + MemoryPool memory_pool=None): + cdef shared_ptr[CBufferedOutputStream] buffered_stream + + if buffer_size <= 0: + raise ValueError('Buffer size must be larger than zero') + buffered_stream = GetResultValue(CBufferedOutputStream.Create( + buffer_size, maybe_unbox_memory_pool(memory_pool), + stream.get_output_stream())) + + self.set_output_stream( buffered_stream) + self.is_writable = True + + def detach(self): + """ + Flush any buffered writes and release the raw OutputStream. + Further operations on this stream are invalid. + + Returns + ------- + raw : NativeFile + The underlying raw output stream. + """ + cdef: + shared_ptr[COutputStream] c_raw + _CBufferedOutputStreamPtr buffered + NativeFile raw + + buffered = dynamic_cast[_CBufferedOutputStreamPtr]( + self.output_stream.get()) + assert buffered != nullptr + + with nogil: + c_raw = GetResultValue(buffered.Detach()) + + raw = NativeFile() + raw.is_writable = True + raw.set_output_stream(c_raw) + return raw + + +cdef void _cb_transform(transform_func, const shared_ptr[CBuffer]& src, + shared_ptr[CBuffer]* dest) except *: + py_dest = transform_func(pyarrow_wrap_buffer(src)) + dest[0] = pyarrow_unwrap_buffer(py_buffer(py_dest)) + + +cdef class TransformInputStream(NativeFile): + """ + Transform an input stream. + + Parameters + ---------- + stream : NativeFile + The stream to transform. + transform_func : callable + The transformation to apply. + """ + + def __init__(self, NativeFile stream, transform_func): + self.set_input_stream(TransformInputStream.make_native( + stream.get_input_stream(), transform_func)) + self.is_readable = True + + @staticmethod + cdef shared_ptr[CInputStream] make_native( + shared_ptr[CInputStream] stream, transform_func) except *: + cdef: + shared_ptr[CInputStream] transform_stream + CTransformInputStreamVTable vtable + + vtable.transform = _cb_transform + return MakeTransformInputStream(stream, move(vtable), + transform_func) + + +class Transcoder: + + def __init__(self, decoder, encoder): + self._decoder = decoder + self._encoder = encoder + + def __call__(self, buf): + final = len(buf) == 0 + return self._encoder.encode(self._decoder.decode(buf, final), final) + + +cdef shared_ptr[function[StreamWrapFunc]] make_streamwrap_func( + src_encoding, dest_encoding) except *: + """ + Create a function that will add a transcoding transformation to a stream. + Data from that stream will be decoded according to ``src_encoding`` and + then re-encoded according to ``dest_encoding``. + The created function can be used to wrap streams. + + Parameters + ---------- + src_encoding : str + The codec to use when reading data. + dest_encoding : str + The codec to use for emitted data. + """ + cdef: + shared_ptr[function[StreamWrapFunc]] empty_func + CTransformInputStreamVTable vtable + + vtable.transform = _cb_transform + src_codec = codecs.lookup(src_encoding) + dest_codec = codecs.lookup(dest_encoding) + return MakeStreamTransformFunc(move(vtable), + Transcoder(src_codec.incrementaldecoder(), + dest_codec.incrementalencoder())) + + +def transcoding_input_stream(stream, src_encoding, dest_encoding): + """ + Add a transcoding transformation to the stream. + Incoming data will be decoded according to ``src_encoding`` and + then re-encoded according to ``dest_encoding``. + + Parameters + ---------- + stream : NativeFile + The stream to which the transformation should be applied. + src_encoding : str + The codec to use when reading data. + dest_encoding : str + The codec to use for emitted data. + """ + src_codec = codecs.lookup(src_encoding) + dest_codec = codecs.lookup(dest_encoding) + if src_codec.name == dest_codec.name: + # Avoid losing performance on no-op transcoding + # (encoding errors won't be detected) + return stream + return TransformInputStream(stream, + Transcoder(src_codec.incrementaldecoder(), + dest_codec.incrementalencoder())) + + +cdef shared_ptr[CInputStream] native_transcoding_input_stream( + shared_ptr[CInputStream] stream, src_encoding, + dest_encoding) except *: + src_codec = codecs.lookup(src_encoding) + dest_codec = codecs.lookup(dest_encoding) + if src_codec.name == dest_codec.name: + # Avoid losing performance on no-op transcoding + # (encoding errors won't be detected) + return stream + return TransformInputStream.make_native( + stream, Transcoder(src_codec.incrementaldecoder(), + dest_codec.incrementalencoder())) + + +def py_buffer(object obj): + """ + Construct an Arrow buffer from a Python bytes-like or buffer-like object + + Parameters + ---------- + obj : object + the object from which the buffer should be constructed. + """ + cdef shared_ptr[CBuffer] buf + buf = GetResultValue(PyBuffer.FromPyObject(obj)) + return pyarrow_wrap_buffer(buf) + + +def foreign_buffer(address, size, base=None): + """ + Construct an Arrow buffer with the given *address* and *size*. + + The buffer will be optionally backed by the Python *base* object, if given. + The *base* object will be kept alive as long as this buffer is alive, + including across language boundaries (for example if the buffer is + referenced by C++ code). + + Parameters + ---------- + address : int + The starting address of the buffer. The address can + refer to both device or host memory but it must be + accessible from device after mapping it with + `get_device_address` method. + size : int + The size of device buffer in bytes. + base : {None, object} + Object that owns the referenced memory. + """ + cdef: + uintptr_t c_addr = address + int64_t c_size = size + shared_ptr[CBuffer] buf + + check_status(PyForeignBuffer.Make( c_addr, c_size, + base, &buf)) + return pyarrow_wrap_buffer(buf) + + +def as_buffer(object o): + if isinstance(o, Buffer): + return o + return py_buffer(o) + + +cdef shared_ptr[CBuffer] as_c_buffer(object o) except *: + cdef shared_ptr[CBuffer] buf + if isinstance(o, Buffer): + buf = ( o).buffer + if buf == nullptr: + raise ValueError("got null buffer") + else: + buf = GetResultValue(PyBuffer.FromPyObject(o)) + return buf + + +cdef NativeFile get_native_file(object source, c_bool use_memory_map): + try: + source_path = _stringify_path(source) + except TypeError: + if isinstance(source, Buffer): + source = BufferReader(source) + elif not isinstance(source, NativeFile) and hasattr(source, 'read'): + # Optimistically hope this is file-like + source = PythonFile(source, mode='r') + else: + if use_memory_map: + source = memory_map(source_path, mode='r') + else: + source = OSFile(source_path, mode='r') + + return source + + +cdef get_reader(object source, c_bool use_memory_map, + shared_ptr[CRandomAccessFile]* reader): + cdef NativeFile nf + + nf = get_native_file(source, use_memory_map) + reader[0] = nf.get_random_access_file() + + +cdef get_input_stream(object source, c_bool use_memory_map, + shared_ptr[CInputStream]* out): + """ + Like get_reader(), but can automatically decompress, and returns + an InputStream. + """ + cdef: + NativeFile nf + Codec codec + shared_ptr[CInputStream] input_stream + + try: + codec = Codec.detect(source) + except TypeError: + codec = None + + nf = get_native_file(source, use_memory_map) + input_stream = nf.get_input_stream() + + # codec is None if compression can't be detected + if codec is not None: + input_stream = GetResultValue( + CCompressedInputStream.Make(codec.unwrap(), input_stream) + ) + + out[0] = input_stream + + +cdef get_writer(object source, shared_ptr[COutputStream]* writer): + cdef NativeFile nf + + try: + source_path = _stringify_path(source) + except TypeError: + if not isinstance(source, NativeFile) and hasattr(source, 'write'): + # Optimistically hope this is file-like + source = PythonFile(source, mode='w') + else: + source = OSFile(source_path, mode='w') + + if isinstance(source, NativeFile): + nf = source + writer[0] = nf.get_output_stream() + else: + raise TypeError(f'Unable to write to object of type: {type(source)}') + + +# --------------------------------------------------------------------- + + +def _detect_compression(path): + if isinstance(path, str): + if path.endswith('.bz2'): + return 'bz2' + elif path.endswith('.gz'): + return 'gzip' + elif path.endswith('.lz4'): + return 'lz4' + elif path.endswith('.zst'): + return 'zstd' + + +cdef CCompressionType _ensure_compression(str name) except *: + uppercase = name.upper() + if uppercase == 'BZ2': + return CCompressionType_BZ2 + elif uppercase == 'GZIP': + return CCompressionType_GZIP + elif uppercase == 'BROTLI': + return CCompressionType_BROTLI + elif uppercase == 'LZ4' or uppercase == 'LZ4_FRAME': + return CCompressionType_LZ4_FRAME + elif uppercase == 'LZ4_RAW': + return CCompressionType_LZ4 + elif uppercase == 'SNAPPY': + return CCompressionType_SNAPPY + elif uppercase == 'ZSTD': + return CCompressionType_ZSTD + else: + raise ValueError(f'Invalid value for compression: {name!r}') + + +cdef class CacheOptions(_Weakrefable): + """ + Cache options for a pre-buffered fragment scan. + + Parameters + ---------- + hole_size_limit : int, default 8KiB + The maximum distance in bytes between two consecutive ranges; beyond + this value, ranges are not combined. + range_size_limit : int, default 32MiB + The maximum size in bytes of a combined range; if combining two + consecutive ranges would produce a range of a size greater than this, + they are not combined + lazy : bool, default True + lazy = false: request all byte ranges when PreBuffer or WillNeed is called. + lazy = True, prefetch_limit = 0: request merged byte ranges only after the reader + needs them. + lazy = True, prefetch_limit = k: prefetch up to k merged byte ranges ahead of the + range that is currently being read. + prefetch_limit : int, default 0 + The maximum number of ranges to be prefetched. This is only used for + lazy cache to asynchronously read some ranges after reading the target + range. + """ + + def __init__(self, *, hole_size_limit=None, range_size_limit=None, lazy=None, prefetch_limit=None): + self.wrapped = CCacheOptions.LazyDefaults() + if hole_size_limit is not None: + self.hole_size_limit = hole_size_limit + if range_size_limit is not None: + self.range_size_limit = range_size_limit + if lazy is not None: + self.lazy = lazy + if prefetch_limit is not None: + self.prefetch_limit = prefetch_limit + + cdef void init(self, CCacheOptions options): + self.wrapped = options + + cdef inline CCacheOptions unwrap(self): + return self.wrapped + + @staticmethod + cdef wrap(CCacheOptions options): + self = CacheOptions() + self.init(options) + return self + + @property + def hole_size_limit(self): + return self.wrapped.hole_size_limit + + @hole_size_limit.setter + def hole_size_limit(self, hole_size_limit): + self.wrapped.hole_size_limit = hole_size_limit + + @property + def range_size_limit(self): + return self.wrapped.range_size_limit + + @range_size_limit.setter + def range_size_limit(self, range_size_limit): + self.wrapped.range_size_limit = range_size_limit + + @property + def lazy(self): + return self.wrapped.lazy + + @lazy.setter + def lazy(self, lazy): + self.wrapped.lazy = lazy + + @property + def prefetch_limit(self): + return self.wrapped.prefetch_limit + + @prefetch_limit.setter + def prefetch_limit(self, prefetch_limit): + self.wrapped.prefetch_limit = prefetch_limit + + def __eq__(self, CacheOptions other): + try: + return self.unwrap().Equals(other.unwrap()) + except TypeError: + return False + + @staticmethod + def from_network_metrics(time_to_first_byte_millis, transfer_bandwidth_mib_per_sec, + ideal_bandwidth_utilization_frac=0.9, max_ideal_request_size_mib=64): + """ + Create suitable CacheOptions based on provided network metrics. + + Typically this will be used with object storage solutions like Amazon S3, + Google Cloud Storage and Azure Blob Storage. + + Parameters + ---------- + time_to_first_byte_millis : int + Seek-time or Time-To-First-Byte (TTFB) in milliseconds, also called call + setup latency of a new read request. The value is a positive integer. + transfer_bandwidth_mib_per_sec : int + Data transfer Bandwidth (BW) in MiB/sec (per connection). The value is a positive + integer. + ideal_bandwidth_utilization_frac : int, default 0.9 + Transfer bandwidth utilization fraction (per connection) to maximize the net + data load. The value is a positive float less than 1. + max_ideal_request_size_mib : int, default 64 + The maximum single data request size (in MiB) to maximize the net data load. + + Returns + ------- + CacheOptions + """ + return CacheOptions.wrap(CCacheOptions.MakeFromNetworkMetrics( + time_to_first_byte_millis, transfer_bandwidth_mib_per_sec, + ideal_bandwidth_utilization_frac, max_ideal_request_size_mib)) + + @staticmethod + def _reconstruct(kwargs): + # __reduce__ doesn't allow passing named arguments directly to the + # reconstructor, hence this wrapper. + return CacheOptions(**kwargs) + + def __reduce__(self): + kwargs = dict( + hole_size_limit=self.hole_size_limit, + range_size_limit=self.range_size_limit, + lazy=self.lazy, + prefetch_limit=self.prefetch_limit, + ) + return CacheOptions._reconstruct, (kwargs,) + + +cdef class Codec(_Weakrefable): + """ + Compression codec. + + Parameters + ---------- + compression : str + Type of compression codec to initialize, valid values are: 'gzip', + 'bz2', 'brotli', 'lz4' (or 'lz4_frame'), 'lz4_raw', 'zstd' and + 'snappy'. + compression_level : int, None + Optional parameter specifying how aggressively to compress. The + possible ranges and effect of this parameter depend on the specific + codec chosen. Higher values compress more but typically use more + resources (CPU/RAM). Some codecs support negative values. + + gzip + The compression_level maps to the memlevel parameter of + deflateInit2. Higher levels use more RAM but are faster + and should have higher compression ratios. + + bz2 + The compression level maps to the blockSize100k parameter of + the BZ2_bzCompressInit function. Higher levels use more RAM + but are faster and should have higher compression ratios. + + brotli + The compression level maps to the BROTLI_PARAM_QUALITY + parameter. Higher values are slower and should have higher + compression ratios. + + lz4/lz4_frame/lz4_raw + The compression level parameter is not supported and must + be None + + zstd + The compression level maps to the compressionLevel parameter + of ZSTD_initCStream. Negative values are supported. Higher + values are slower and should have higher compression ratios. + + snappy + The compression level parameter is not supported and must + be None + + + Raises + ------ + ValueError + If invalid compression value is passed. + + Examples + -------- + >>> import pyarrow as pa + >>> pa.Codec.is_available('gzip') + True + >>> codec = pa.Codec('gzip') + >>> codec.name + 'gzip' + >>> codec.compression_level + 9 + """ + + def __init__(self, str compression not None, compression_level=None): + cdef CCompressionType typ = _ensure_compression(compression) + if compression_level is not None: + self.wrapped = shared_ptr[CCodec](move(GetResultValue( + CCodec.CreateWithLevel(typ, compression_level)))) + else: + self.wrapped = shared_ptr[CCodec](move(GetResultValue( + CCodec.Create(typ)))) + + cdef inline CCodec* unwrap(self) nogil: + return self.wrapped.get() + + @staticmethod + def detect(path): + """ + Detect and instantiate compression codec based on file extension. + + Parameters + ---------- + path : str, path-like + File-path to detect compression from. + + Raises + ------ + TypeError + If the passed value is not path-like. + ValueError + If the compression can't be detected from the path. + + Returns + ------- + Codec + """ + return Codec(_detect_compression(_stringify_path(path))) + + @staticmethod + def is_available(str compression not None): + """ + Returns whether the compression support has been built and enabled. + + Parameters + ---------- + compression : str + Type of compression codec, + refer to Codec docstring for a list of supported ones. + + Returns + ------- + bool + """ + cdef CCompressionType typ = _ensure_compression(compression) + return CCodec.IsAvailable(typ) + + @staticmethod + def supports_compression_level(str compression not None): + """ + Returns true if the compression level parameter is supported + for the given codec. + + Parameters + ---------- + compression : str + Type of compression codec, + refer to Codec docstring for a list of supported ones. + """ + cdef CCompressionType typ = _ensure_compression(compression) + return CCodec.SupportsCompressionLevel(typ) + + @staticmethod + def default_compression_level(str compression not None): + """ + Returns the compression level that Arrow will use for the codec if + None is specified. + + Parameters + ---------- + compression : str + Type of compression codec, + refer to Codec docstring for a list of supported ones. + """ + cdef CCompressionType typ = _ensure_compression(compression) + return GetResultValue(CCodec.DefaultCompressionLevel(typ)) + + @staticmethod + def minimum_compression_level(str compression not None): + """ + Returns the smallest valid value for the compression level + + Parameters + ---------- + compression : str + Type of compression codec, + refer to Codec docstring for a list of supported ones. + """ + cdef CCompressionType typ = _ensure_compression(compression) + return GetResultValue(CCodec.MinimumCompressionLevel(typ)) + + @staticmethod + def maximum_compression_level(str compression not None): + """ + Returns the largest valid value for the compression level + + Parameters + ---------- + compression : str + Type of compression codec, + refer to Codec docstring for a list of supported ones. + """ + cdef CCompressionType typ = _ensure_compression(compression) + return GetResultValue(CCodec.MaximumCompressionLevel(typ)) + + @property + def name(self): + """Returns the name of the codec""" + return frombytes(self.unwrap().name()) + + @property + def compression_level(self): + """Returns the compression level parameter of the codec""" + if self.name == 'snappy': + return None + return self.unwrap().compression_level() + + def compress(self, object buf, asbytes=False, memory_pool=None): + """ + Compress data from buffer-like object. + + Parameters + ---------- + buf : pyarrow.Buffer, bytes, or other object supporting buffer protocol + asbytes : bool, default False + Return result as Python bytes object, otherwise Buffer + memory_pool : MemoryPool, default None + Memory pool to use for buffer allocations, if any + + Returns + ------- + compressed : pyarrow.Buffer or bytes (if asbytes=True) + """ + cdef: + shared_ptr[CBuffer] owned_buf + CBuffer* c_buf + PyObject* pyobj + ResizableBuffer out_buf + int64_t max_output_size + int64_t output_length + uint8_t* output_buffer = NULL + + owned_buf = as_c_buffer(buf) + c_buf = owned_buf.get() + + max_output_size = self.wrapped.get().MaxCompressedLen( + c_buf.size(), c_buf.data() + ) + + if asbytes: + pyobj = PyBytes_FromStringAndSizeNative(NULL, max_output_size) + output_buffer = cp.PyBytes_AS_STRING( pyobj) + else: + out_buf = allocate_buffer( + max_output_size, memory_pool=memory_pool, resizable=True + ) + output_buffer = out_buf.buffer.get().mutable_data() + + with nogil: + output_length = GetResultValue( + self.unwrap().Compress( + c_buf.size(), + c_buf.data(), + max_output_size, + output_buffer + ) + ) + + if asbytes: + cp._PyBytes_Resize(&pyobj, output_length) + return PyObject_to_object(pyobj) + else: + out_buf.resize(output_length) + return out_buf + + def decompress(self, object buf, decompressed_size=None, asbytes=False, + memory_pool=None): + """ + Decompress data from buffer-like object. + + Parameters + ---------- + buf : pyarrow.Buffer, bytes, or memoryview-compatible object + decompressed_size : int, default None + Size of the decompressed result + asbytes : boolean, default False + Return result as Python bytes object, otherwise Buffer + memory_pool : MemoryPool, default None + Memory pool to use for buffer allocations, if any. + + Returns + ------- + uncompressed : pyarrow.Buffer or bytes (if asbytes=True) + """ + cdef: + shared_ptr[CBuffer] owned_buf + CBuffer* c_buf + Buffer out_buf + int64_t output_size + uint8_t* output_buffer = NULL + + owned_buf = as_c_buffer(buf) + c_buf = owned_buf.get() + + if decompressed_size is None: + raise ValueError( + "Must pass decompressed_size" + ) + + output_size = decompressed_size + + if asbytes: + pybuf = cp.PyBytes_FromStringAndSize(NULL, output_size) + output_buffer = cp.PyBytes_AS_STRING(pybuf) + else: + out_buf = allocate_buffer(output_size, memory_pool=memory_pool) + output_buffer = out_buf.buffer.get().mutable_data() + + with nogil: + GetResultValue( + self.unwrap().Decompress( + c_buf.size(), + c_buf.data(), + output_size, + output_buffer + ) + ) + + return pybuf if asbytes else out_buf + + def __repr__(self): + name = f"pyarrow.{self.__class__.__name__}" + return (f"<{name} " + f"name={self.name} " + f"compression_level={self.compression_level}>") + + +def compress(object buf, codec='lz4', asbytes=False, memory_pool=None): + """ + Compress data from buffer-like object. + + Parameters + ---------- + buf : pyarrow.Buffer, bytes, or other object supporting buffer protocol + codec : str, default 'lz4' + Compression codec. + Supported types: {'brotli, 'gzip', 'lz4', 'lz4_raw', 'snappy', 'zstd'} + asbytes : bool, default False + Return result as Python bytes object, otherwise Buffer. + memory_pool : MemoryPool, default None + Memory pool to use for buffer allocations, if any. + + Returns + ------- + compressed : pyarrow.Buffer or bytes (if asbytes=True) + """ + cdef Codec coder = Codec(codec) + return coder.compress(buf, asbytes=asbytes, memory_pool=memory_pool) + + +def decompress(object buf, decompressed_size=None, codec='lz4', + asbytes=False, memory_pool=None): + """ + Decompress data from buffer-like object. + + Parameters + ---------- + buf : pyarrow.Buffer, bytes, or memoryview-compatible object + Input object to decompress data from. + decompressed_size : int, default None + Size of the decompressed result + codec : str, default 'lz4' + Compression codec. + Supported types: {'brotli, 'gzip', 'lz4', 'lz4_raw', 'snappy', 'zstd'} + asbytes : bool, default False + Return result as Python bytes object, otherwise Buffer. + memory_pool : MemoryPool, default None + Memory pool to use for buffer allocations, if any. + + Returns + ------- + uncompressed : pyarrow.Buffer or bytes (if asbytes=True) + """ + cdef Codec decoder = Codec(codec) + return decoder.decompress(buf, asbytes=asbytes, memory_pool=memory_pool, + decompressed_size=decompressed_size) + + +def input_stream(source, compression='detect', buffer_size=None): + """ + Create an Arrow input stream. + + Parameters + ---------- + source : str, Path, buffer, or file-like object + The source to open for reading. + compression : str optional, default 'detect' + The compression algorithm to use for on-the-fly decompression. + If "detect" and source is a file path, then compression will be + chosen based on the file extension. + If None, no compression will be applied. + Otherwise, a well-known algorithm name must be supplied (e.g. "gzip"). + buffer_size : int, default None + If None or 0, no buffering will happen. Otherwise the size of the + temporary read buffer. + + Examples + -------- + Create a readable BufferReader (NativeFile) from a Buffer or a memoryview object: + + >>> import pyarrow as pa + >>> buf = memoryview(b"some data") + >>> with pa.input_stream(buf) as stream: + ... stream.read(4) + ... + b'some' + + Create a readable OSFile (NativeFile) from a string or file path: + + >>> import gzip + >>> with gzip.open('example.gz', 'wb') as f: + ... f.write(b'some data') + ... + 9 + >>> with pa.input_stream('example.gz') as stream: + ... stream.read() + ... + b'some data' + + Create a readable PythonFile (NativeFile) from a a Python file object: + + >>> with open('example.txt', mode='w') as f: + ... f.write('some text') + ... + 9 + >>> with pa.input_stream('example.txt') as stream: + ... stream.read(6) + ... + b'some t' + """ + cdef NativeFile stream + + try: + source_path = _stringify_path(source) + except TypeError: + source_path = None + + if isinstance(source, NativeFile): + stream = source + elif source_path is not None: + stream = OSFile(source_path, 'r') + elif isinstance(source, (Buffer, memoryview)): + stream = BufferReader(as_buffer(source)) + elif (hasattr(source, 'read') and + hasattr(source, 'close') and + hasattr(source, 'closed')): + stream = PythonFile(source, 'r') + else: + raise TypeError( + f"pa.input_stream() called with instance of '{source.__class__}'") + + if compression == 'detect': + # detect for OSFile too + compression = _detect_compression(source_path) + + if buffer_size is not None and buffer_size != 0: + stream = BufferedInputStream(stream, buffer_size) + + if compression is not None: + stream = CompressedInputStream(stream, compression) + + return stream + + +def output_stream(source, compression='detect', buffer_size=None): + """ + Create an Arrow output stream. + + Parameters + ---------- + source : str, Path, buffer, file-like object + The source to open for writing. + compression : str optional, default 'detect' + The compression algorithm to use for on-the-fly compression. + If "detect" and source is a file path, then compression will be + chosen based on the file extension. + If None, no compression will be applied. + Otherwise, a well-known algorithm name must be supplied (e.g. "gzip"). + buffer_size : int, default None + If None or 0, no buffering will happen. Otherwise the size of the + temporary write buffer. + + Examples + -------- + Create a writable NativeFile from a pyarrow Buffer: + + >>> import pyarrow as pa + >>> data = b"buffer data" + >>> empty_obj = bytearray(11) + >>> buf = pa.py_buffer(empty_obj) + >>> with pa.output_stream(buf) as stream: + ... stream.write(data) + ... + 11 + >>> with pa.input_stream(buf) as stream: + ... stream.read(6) + ... + b'buffer' + + or from a memoryview object: + + >>> buf = memoryview(empty_obj) + >>> with pa.output_stream(buf) as stream: + ... stream.write(data) + ... + 11 + >>> with pa.input_stream(buf) as stream: + ... stream.read() + ... + b'buffer data' + + Create a writable NativeFile from a string or file path: + + >>> with pa.output_stream('example_second.txt') as stream: + ... stream.write(b'Write some data') + ... + 15 + >>> with pa.input_stream('example_second.txt') as stream: + ... stream.read() + ... + b'Write some data' + """ + cdef NativeFile stream + + try: + source_path = _stringify_path(source) + except TypeError: + source_path = None + + if isinstance(source, NativeFile): + stream = source + elif source_path is not None: + stream = OSFile(source_path, 'w') + elif isinstance(source, (Buffer, memoryview)): + stream = FixedSizeBufferWriter(as_buffer(source)) + elif (hasattr(source, 'write') and + hasattr(source, 'close') and + hasattr(source, 'closed')): + stream = PythonFile(source, 'w') + else: + raise TypeError( + f"pa.output_stream() called with instance of '{source.__class__}'") + + if compression == 'detect': + compression = _detect_compression(source_path) + + if buffer_size is not None and buffer_size != 0: + stream = BufferedOutputStream(stream, buffer_size) + + if compression is not None: + stream = CompressedOutputStream(stream, compression) + + return stream diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/ipc.pxi b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/ipc.pxi new file mode 100644 index 0000000000000000000000000000000000000000..e7b5e45c602050eaea063dfc3ac04eac0a7b54c1 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/ipc.pxi @@ -0,0 +1,1504 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from cpython.pycapsule cimport PyCapsule_CheckExact, PyCapsule_GetPointer, PyCapsule_New + +from collections import namedtuple +import warnings +from cython import sizeof + +cpdef enum MetadataVersion: + V1 = CMetadataVersion_V1 + V2 = CMetadataVersion_V2 + V3 = CMetadataVersion_V3 + V4 = CMetadataVersion_V4 + V5 = CMetadataVersion_V5 + + +cdef object _wrap_metadata_version(CMetadataVersion version): + return MetadataVersion( version) + + +cdef CMetadataVersion _unwrap_metadata_version( + MetadataVersion version) except *: + if version == MetadataVersion.V1: + return CMetadataVersion_V1 + elif version == MetadataVersion.V2: + return CMetadataVersion_V2 + elif version == MetadataVersion.V3: + return CMetadataVersion_V3 + elif version == MetadataVersion.V4: + return CMetadataVersion_V4 + elif version == MetadataVersion.V5: + return CMetadataVersion_V5 + raise ValueError("Not a metadata version: " + repr(version)) + + +cpdef enum Alignment: + Any = CAlignment_Any + DataTypeSpecific = CAlignment_DataTypeSpecific + At64Byte = CAlignment_64Byte + + +cdef object _wrap_alignment(CAlignment alignment): + return Alignment( alignment) + + +cdef CAlignment _unwrap_alignment(Alignment alignment) except *: + if alignment == Alignment.Any: + return CAlignment_Any + elif alignment == Alignment.DataTypeSpecific: + return CAlignment_DataTypeSpecific + elif alignment == Alignment.At64Byte: + return CAlignment_64Byte + raise ValueError("Not an alignment: " + repr(alignment)) + + +_WriteStats = namedtuple( + 'WriteStats', + ('num_messages', 'num_record_batches', 'num_dictionary_batches', + 'num_dictionary_deltas', 'num_replaced_dictionaries')) + + +class WriteStats(_WriteStats): + """IPC write statistics + + Parameters + ---------- + num_messages : int + Number of messages. + num_record_batches : int + Number of record batches. + num_dictionary_batches : int + Number of dictionary batches. + num_dictionary_deltas : int + Delta of dictionaries. + num_replaced_dictionaries : int + Number of replaced dictionaries. + """ + __slots__ = () + + +@staticmethod +cdef _wrap_write_stats(CIpcWriteStats c): + return WriteStats(c.num_messages, c.num_record_batches, + c.num_dictionary_batches, c.num_dictionary_deltas, + c.num_replaced_dictionaries) + + +_ReadStats = namedtuple( + 'ReadStats', + ('num_messages', 'num_record_batches', 'num_dictionary_batches', + 'num_dictionary_deltas', 'num_replaced_dictionaries')) + + +class ReadStats(_ReadStats): + """IPC read statistics + + Parameters + ---------- + num_messages : int + Number of messages. + num_record_batches : int + Number of record batches. + num_dictionary_batches : int + Number of dictionary batches. + num_dictionary_deltas : int + Delta of dictionaries. + num_replaced_dictionaries : int + Number of replaced dictionaries. + """ + __slots__ = () + + +cdef _wrap_read_stats(CIpcReadStats c): + return ReadStats(c.num_messages, c.num_record_batches, + c.num_dictionary_batches, c.num_dictionary_deltas, + c.num_replaced_dictionaries) + + +cdef class IpcReadOptions(_Weakrefable): + """ + Serialization options for reading IPC format. + + Parameters + ---------- + ensure_native_endian : bool, default True + Whether to convert incoming data to platform-native endianness. + ensure_alignment : Alignment, default Alignment.Any + Data is copied to aligned memory locations if mis-aligned. + Some use cases might require data to have a specific alignment, for example, + for the data buffer of an int32 array to be aligned on a 4-byte boundary. + use_threads : bool + Whether to use the global CPU thread pool to parallelize any + computational tasks like decompression + included_fields : list + If empty (the default), return all deserialized fields. + If non-empty, the values are the indices of fields to read on + the top-level schema + """ + __slots__ = () + + # cdef block is in lib.pxd + + def __init__(self, *, bint ensure_native_endian=True, + Alignment ensure_alignment=Alignment.Any, + bint use_threads=True, list included_fields=None): + self.c_options = CIpcReadOptions.Defaults() + self.ensure_native_endian = ensure_native_endian + self.ensure_alignment = ensure_alignment + self.use_threads = use_threads + if included_fields is not None: + self.included_fields = included_fields + + @property + def ensure_native_endian(self): + return self.c_options.ensure_native_endian + + @ensure_native_endian.setter + def ensure_native_endian(self, bint value): + self.c_options.ensure_native_endian = value + + @property + def ensure_alignment(self): + return _wrap_alignment(self.c_options.ensure_alignment) + + @ensure_alignment.setter + def ensure_alignment(self, Alignment value): + self.c_options.ensure_alignment = _unwrap_alignment(value) + + @property + def use_threads(self): + return self.c_options.use_threads + + @use_threads.setter + def use_threads(self, bint value): + self.c_options.use_threads = value + + @property + def included_fields(self): + return self.c_options.included_fields + + @included_fields.setter + def included_fields(self, list value not None): + self.c_options.included_fields = value + + def __repr__(self): + alignment = Alignment(self.ensure_alignment).name + + return (f"") + + +cdef IpcReadOptions wrap_ipc_read_options(CIpcReadOptions c): + """Get Python's IpcReadOptions from C++'s IpcReadOptions + """ + + return IpcReadOptions( + ensure_native_endian=c.ensure_native_endian, + ensure_alignment=c.ensure_alignment, + use_threads=c.use_threads, + included_fields=c.included_fields, + ) + + +cdef object _get_compression_from_codec(shared_ptr[CCodec] codec): + if codec == nullptr: + return None + else: + return frombytes(codec.get().name()) + + +cdef class IpcWriteOptions(_Weakrefable): + """ + Serialization options for the IPC format. + + Parameters + ---------- + metadata_version : MetadataVersion, default MetadataVersion.V5 + The metadata version to write. V5 is the current and latest, + V4 is the pre-1.0 metadata version (with incompatible Union layout). + allow_64bit : bool, default False + If true, allow field lengths that don't fit in a signed 32-bit int. + use_legacy_format : bool, default False + Whether to use the pre-Arrow 0.15 IPC format. + compression : str, Codec, or None + compression codec to use for record batch buffers. + If None then batch buffers will be uncompressed. + Must be "lz4", "zstd" or None. + To specify a compression_level use `pyarrow.Codec` + use_threads : bool + Whether to use the global CPU thread pool to parallelize any + computational tasks like compression. + emit_dictionary_deltas : bool + Whether to emit dictionary deltas. Default is false for maximum + stream compatibility. + unify_dictionaries : bool + If true then calls to write_table will attempt to unify dictionaries + across all batches in the table. This can help avoid the need for + replacement dictionaries (which the file format does not support) + but requires computing the unified dictionary and then remapping + the indices arrays. + + This parameter is ignored when writing to the IPC stream format as + the IPC stream format can support replacement dictionaries. + """ + __slots__ = () + + # cdef block is in lib.pxd + + def __init__(self, *, metadata_version=MetadataVersion.V5, + bint allow_64bit=False, use_legacy_format=False, + compression=None, bint use_threads=True, + bint emit_dictionary_deltas=False, + bint unify_dictionaries=False): + self.c_options = CIpcWriteOptions.Defaults() + self.allow_64bit = allow_64bit + self.use_legacy_format = use_legacy_format + self.metadata_version = metadata_version + if compression is not None: + self.compression = compression + self.use_threads = use_threads + self.emit_dictionary_deltas = emit_dictionary_deltas + self.unify_dictionaries = unify_dictionaries + + @property + def allow_64bit(self): + return self.c_options.allow_64bit + + @allow_64bit.setter + def allow_64bit(self, bint value): + self.c_options.allow_64bit = value + + @property + def use_legacy_format(self): + return self.c_options.write_legacy_ipc_format + + @use_legacy_format.setter + def use_legacy_format(self, bint value): + self.c_options.write_legacy_ipc_format = value + + @property + def metadata_version(self): + return _wrap_metadata_version(self.c_options.metadata_version) + + @metadata_version.setter + def metadata_version(self, value): + self.c_options.metadata_version = _unwrap_metadata_version(value) + + @property + def compression(self): + return _get_compression_from_codec(self.c_options.codec) + + @compression.setter + def compression(self, value): + if value is None: + self.c_options.codec.reset() + elif isinstance(value, str): + codec_type = _ensure_compression(value) + if codec_type != CCompressionType_ZSTD and codec_type != CCompressionType_LZ4_FRAME: + raise ValueError("Compression type must be lz4, zstd or None") + self.c_options.codec = shared_ptr[CCodec](GetResultValue( + CCodec.Create(codec_type)).release()) + elif isinstance(value, Codec): + if value.name != "lz4" and value.name != "zstd": + raise ValueError("Compression type must be lz4, zstd or None") + self.c_options.codec = (value).wrapped + else: + raise TypeError( + "Property `compression` must be None, str, or pyarrow.Codec") + + @property + def use_threads(self): + return self.c_options.use_threads + + @use_threads.setter + def use_threads(self, bint value): + self.c_options.use_threads = value + + @property + def emit_dictionary_deltas(self): + return self.c_options.emit_dictionary_deltas + + @emit_dictionary_deltas.setter + def emit_dictionary_deltas(self, bint value): + self.c_options.emit_dictionary_deltas = value + + @property + def unify_dictionaries(self): + return self.c_options.unify_dictionaries + + @unify_dictionaries.setter + def unify_dictionaries(self, bint value): + self.c_options.unify_dictionaries = value + + def __repr__(self): + compression_repr = f"compression=\"{self.compression}\" " \ + if self.compression is not None else "" + + metadata_version = MetadataVersion(self.metadata_version).name + + return (f"") + + +cdef IpcWriteOptions wrap_ipc_write_options(CIpcWriteOptions c): + """Get Python's IpcWriteOptions from C++'s IpcWriteOptions + """ + + return IpcWriteOptions( + metadata_version=c.metadata_version, + allow_64bit=c.allow_64bit, + use_legacy_format=c.write_legacy_ipc_format, + compression=_get_compression_from_codec(c.codec), + use_threads=c.use_threads, + emit_dictionary_deltas=c.emit_dictionary_deltas, + unify_dictionaries=c.unify_dictionaries, + ) + + +cdef class Message(_Weakrefable): + """ + Container for an Arrow IPC message with metadata and optional body + """ + + def __cinit__(self): + pass + + def __init__(self): + raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, use " + "`pyarrow.ipc.read_message` function instead.") + + @property + def type(self): + return frombytes(FormatMessageType(self.message.get().type())) + + @property + def metadata(self): + return pyarrow_wrap_buffer(self.message.get().metadata()) + + @property + def metadata_version(self): + return _wrap_metadata_version(self.message.get().metadata_version()) + + @property + def body(self): + cdef shared_ptr[CBuffer] body = self.message.get().body() + if body.get() == NULL: + return None + else: + return pyarrow_wrap_buffer(body) + + def equals(self, Message other): + """ + Returns True if the message contents (metadata and body) are identical + + Parameters + ---------- + other : Message + + Returns + ------- + are_equal : bool + """ + cdef c_bool result + with nogil: + result = self.message.get().Equals(deref(other.message.get())) + return result + + def serialize_to(self, NativeFile sink, alignment=8, memory_pool=None): + """ + Write message to generic OutputStream + + Parameters + ---------- + sink : NativeFile + alignment : int, default 8 + Byte alignment for metadata and body + memory_pool : MemoryPool, default None + Uses default memory pool if not specified + """ + cdef: + int64_t output_length = 0 + COutputStream* out + CIpcWriteOptions options + + options.alignment = alignment + out = sink.get_output_stream().get() + with nogil: + check_status(self.message.get() + .SerializeTo(out, options, &output_length)) + + def serialize(self, alignment=8, memory_pool=None): + """ + Write message as encapsulated IPC message + + Parameters + ---------- + alignment : int, default 8 + Byte alignment for metadata and body + memory_pool : MemoryPool, default None + Uses default memory pool if not specified + + Returns + ------- + serialized : Buffer + """ + stream = BufferOutputStream(memory_pool) + self.serialize_to(stream, alignment=alignment, memory_pool=memory_pool) + return stream.getvalue() + + def __repr__(self): + if self.message == nullptr: + return """pyarrow.Message(uninitialized)""" + + metadata_len = self.metadata.size + body = self.body + body_len = 0 if body is None else body.size + + return f"""pyarrow.Message +type: {self.type} +metadata length: {metadata_len} +body length: {body_len}""" + + +cdef class MessageReader(_Weakrefable): + """ + Interface for reading Message objects from some source (like an + InputStream) + """ + cdef: + unique_ptr[CMessageReader] reader + + def __cinit__(self): + pass + + def __init__(self): + raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, use " + "`pyarrow.ipc.MessageReader.open_stream` function " + "instead.") + + @staticmethod + def open_stream(source): + """ + Open stream from source, if you want to use memory map use + MemoryMappedFile as source. + + Parameters + ---------- + source : bytes/buffer-like, pyarrow.NativeFile, or file-like Python object + A readable source, like an InputStream + """ + cdef: + MessageReader result = MessageReader.__new__(MessageReader) + shared_ptr[CInputStream] in_stream + unique_ptr[CMessageReader] reader + + _get_input_stream(source, &in_stream) + with nogil: + reader = CMessageReader.Open(in_stream) + result.reader.reset(reader.release()) + + return result + + def __iter__(self): + return self + + def __next__(self): + return self.read_next_message() + + def read_next_message(self): + """ + Read next Message from the stream. + + Raises + ------ + StopIteration + At end of stream + """ + cdef Message result = Message.__new__(Message) + + with nogil: + result.message = move(GetResultValue(self.reader.get() + .ReadNextMessage())) + + if result.message.get() == NULL: + raise StopIteration + + return result + +# ---------------------------------------------------------------------- +# File and stream readers and writers + +cdef class _CRecordBatchWriter(_Weakrefable): + """The base RecordBatchWriter wrapper. + + Provides common implementations of convenience methods. Should not + be instantiated directly by user code. + """ + + # cdef block is in lib.pxd + + def write(self, table_or_batch): + """ + Write RecordBatch or Table to stream. + + Parameters + ---------- + table_or_batch : {RecordBatch, Table} + """ + if isinstance(table_or_batch, RecordBatch): + self.write_batch(table_or_batch) + elif isinstance(table_or_batch, Table): + self.write_table(table_or_batch) + else: + raise ValueError(type(table_or_batch)) + + def write_batch(self, RecordBatch batch, custom_metadata=None): + """ + Write RecordBatch to stream. + + Parameters + ---------- + batch : RecordBatch + custom_metadata : mapping or KeyValueMetadata + Keys and values must be string-like / coercible to bytes + """ + metadata = ensure_metadata(custom_metadata, allow_none=True) + c_meta = pyarrow_unwrap_metadata(metadata) + + with nogil: + check_status(self.writer.get() + .WriteRecordBatch(deref(batch.batch), c_meta)) + + def write_table(self, Table table, max_chunksize=None): + """ + Write Table to stream in (contiguous) RecordBatch objects. + + Parameters + ---------- + table : Table + max_chunksize : int, default None + Maximum number of rows for RecordBatch chunks. Individual chunks may + be smaller depending on the chunk layout of individual columns. + """ + cdef: + # max_chunksize must be > 0 to have any impact + int64_t c_max_chunksize = -1 + + if max_chunksize is not None: + c_max_chunksize = max_chunksize + + with nogil: + check_status(self.writer.get().WriteTable(table.table[0], + c_max_chunksize)) + + def close(self): + """ + Close stream and write end-of-stream 0 marker. + """ + with nogil: + check_status(self.writer.get().Close()) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + @property + def stats(self): + """ + Current IPC write statistics. + """ + if not self.writer: + raise ValueError("Operation on closed writer") + return _wrap_write_stats(self.writer.get().stats()) + + +cdef class _RecordBatchStreamWriter(_CRecordBatchWriter): + cdef: + CIpcWriteOptions options + bint closed + + def __cinit__(self): + pass + + def __dealloc__(self): + pass + + @property + def _use_legacy_format(self): + # For testing (see test_ipc.py) + return self.options.write_legacy_ipc_format + + @property + def _metadata_version(self): + # For testing (see test_ipc.py) + return _wrap_metadata_version(self.options.metadata_version) + + def _open(self, sink, Schema schema not None, + IpcWriteOptions options=IpcWriteOptions()): + cdef: + shared_ptr[COutputStream] c_sink + + self.options = options.c_options + get_writer(sink, &c_sink) + with nogil: + self.writer = GetResultValue( + MakeStreamWriter(c_sink, schema.sp_schema, + self.options)) + + +cdef _get_input_stream(object source, shared_ptr[CInputStream]* out): + try: + source = as_buffer(source) + except TypeError: + # Non-buffer-like + pass + + get_input_stream(source, True, out) + + +class _ReadPandasMixin: + + def read_pandas(self, **options): + """ + Read contents of stream to a pandas.DataFrame. + + Read all record batches as a pyarrow.Table then convert it to a + pandas.DataFrame using Table.to_pandas. + + Parameters + ---------- + **options + Arguments to forward to :meth:`Table.to_pandas`. + + Returns + ------- + df : pandas.DataFrame + """ + table = self.read_all() + return table.to_pandas(**options) + + +cdef class RecordBatchReader(_Weakrefable): + """Base class for reading stream of record batches. + + Record batch readers function as iterators of record batches that also + provide the schema (without the need to get any batches). + + Warnings + -------- + Do not call this class's constructor directly, use one of the + ``RecordBatchReader.from_*`` functions instead. + + Notes + ----- + To import and export using the Arrow C stream interface, use the + ``_import_from_c`` and ``_export_to_c`` methods. However, keep in mind this + interface is intended for expert users. + + Examples + -------- + >>> import pyarrow as pa + >>> schema = pa.schema([('x', pa.int64())]) + >>> def iter_record_batches(): + ... for i in range(2): + ... yield pa.RecordBatch.from_arrays([pa.array([1, 2, 3])], schema=schema) + >>> reader = pa.RecordBatchReader.from_batches(schema, iter_record_batches()) + >>> print(reader.schema) + x: int64 + >>> for batch in reader: + ... print(batch) + pyarrow.RecordBatch + x: int64 + ---- + x: [1,2,3] + pyarrow.RecordBatch + x: int64 + ---- + x: [1,2,3] + """ + + # cdef block is in lib.pxd + + def __init__(self): + raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, " + "use one of the RecordBatchReader.from_* functions instead.") + + def __iter__(self): + return self + + def __next__(self): + return self.read_next_batch() + + @property + def schema(self): + """ + Shared schema of the record batches in the stream. + + Returns + ------- + Schema + """ + cdef shared_ptr[CSchema] c_schema + + with nogil: + c_schema = self.reader.get().schema() + + return pyarrow_wrap_schema(c_schema) + + def read_next_batch(self): + """ + Read next RecordBatch from the stream. + + Raises + ------ + StopIteration: + At end of stream. + + Returns + ------- + RecordBatch + """ + cdef shared_ptr[CRecordBatch] batch + + with nogil: + check_status(self.reader.get().ReadNext(&batch)) + + if batch.get() == NULL: + raise StopIteration + + return pyarrow_wrap_batch(batch) + + def read_next_batch_with_custom_metadata(self): + """ + Read next RecordBatch from the stream along with its custom metadata. + + Raises + ------ + StopIteration: + At end of stream. + + Returns + ------- + batch : RecordBatch + custom_metadata : KeyValueMetadata + """ + cdef: + CRecordBatchWithMetadata batch_with_metadata + + with nogil: + batch_with_metadata = GetResultValue(self.reader.get().ReadNext()) + + if batch_with_metadata.batch.get() == NULL: + raise StopIteration + + return _wrap_record_batch_with_metadata(batch_with_metadata) + + def iter_batches_with_custom_metadata(self): + """ + Iterate over record batches from the stream along with their custom + metadata. + + Yields + ------ + RecordBatchWithMetadata + """ + while True: + try: + yield self.read_next_batch_with_custom_metadata() + except StopIteration: + return + + def read_all(self): + """ + Read all record batches as a pyarrow.Table. + + Returns + ------- + Table + """ + cdef shared_ptr[CTable] table + with nogil: + check_status(self.reader.get().ToTable().Value(&table)) + return pyarrow_wrap_table(table) + + read_pandas = _ReadPandasMixin.read_pandas + + def close(self): + """ + Release any resources associated with the reader. + """ + with nogil: + check_status(self.reader.get().Close()) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def cast(self, target_schema): + """ + Wrap this reader with one that casts each batch lazily as it is pulled. + Currently only a safe cast to target_schema is implemented. + + Parameters + ---------- + target_schema : Schema + Schema to cast to, the names and order of fields must match. + + Returns + ------- + RecordBatchReader + """ + cdef: + shared_ptr[CSchema] c_schema + shared_ptr[CRecordBatchReader] c_reader + RecordBatchReader out + + if self.schema.names != target_schema.names: + raise ValueError("Target schema's field names are not matching " + f"the table's field names: {self.schema.names}, " + f"{target_schema.names}") + + c_schema = pyarrow_unwrap_schema(target_schema) + c_reader = GetResultValue(CCastingRecordBatchReader.Make( + self.reader, c_schema)) + + out = RecordBatchReader.__new__(RecordBatchReader) + out.reader = c_reader + return out + + def _export_to_c(self, out_ptr): + """ + Export to a C ArrowArrayStream struct, given its pointer. + + Parameters + ---------- + out_ptr: int + The raw pointer to a C ArrowArrayStream struct. + + Be careful: if you don't pass the ArrowArrayStream struct to a + consumer, array memory will leak. This is a low-level function + intended for expert users. + """ + cdef: + void* c_ptr = _as_c_pointer(out_ptr) + with nogil: + check_status(ExportRecordBatchReader( + self.reader, c_ptr)) + + @staticmethod + def _import_from_c(in_ptr): + """ + Import RecordBatchReader from a C ArrowArrayStream struct, + given its pointer. + + Parameters + ---------- + in_ptr: int + The raw pointer to a C ArrowArrayStream struct. + + This is a low-level function intended for expert users. + """ + cdef: + void* c_ptr = _as_c_pointer(in_ptr) + shared_ptr[CRecordBatchReader] c_reader + RecordBatchReader self + + with nogil: + c_reader = GetResultValue(ImportRecordBatchReader( + c_ptr)) + + self = RecordBatchReader.__new__(RecordBatchReader) + self.reader = c_reader + return self + + def __arrow_c_stream__(self, requested_schema=None): + """ + Export to a C ArrowArrayStream PyCapsule. + + Parameters + ---------- + requested_schema : PyCapsule, default None + The schema to which the stream should be casted, passed as a + PyCapsule containing a C ArrowSchema representation of the + requested schema. + + Returns + ------- + PyCapsule + A capsule containing a C ArrowArrayStream struct. + """ + cdef: + ArrowArrayStream* c_stream + + if requested_schema is not None: + out_schema = Schema._import_from_c_capsule(requested_schema) + if self.schema != out_schema: + return self.cast(out_schema).__arrow_c_stream__() + + stream_capsule = alloc_c_stream(&c_stream) + + with nogil: + check_status(ExportRecordBatchReader(self.reader, c_stream)) + + return stream_capsule + + @staticmethod + def _import_from_c_capsule(stream): + """ + Import RecordBatchReader from a C ArrowArrayStream PyCapsule. + + Parameters + ---------- + stream: PyCapsule + A capsule containing a C ArrowArrayStream PyCapsule. + + Returns + ------- + RecordBatchReader + """ + cdef: + ArrowArrayStream* c_stream + shared_ptr[CRecordBatchReader] c_reader + RecordBatchReader self + + c_stream = PyCapsule_GetPointer( + stream, 'arrow_array_stream' + ) + + with nogil: + c_reader = GetResultValue(ImportRecordBatchReader(c_stream)) + + self = RecordBatchReader.__new__(RecordBatchReader) + self.reader = c_reader + return self + + @staticmethod + def from_stream(data, schema=None): + """ + Create RecordBatchReader from a Arrow-compatible stream object. + + This accepts objects implementing the Arrow PyCapsule Protocol for + streams, i.e. objects that have a ``__arrow_c_stream__`` method. + + Parameters + ---------- + data : Arrow-compatible stream object + Any object that implements the Arrow PyCapsule Protocol for + streams. + schema : Schema, default None + The schema to which the stream should be casted, if supported + by the stream object. + + Returns + ------- + RecordBatchReader + """ + + if not hasattr(data, "__arrow_c_stream__"): + raise TypeError( + "Expected an object implementing the Arrow PyCapsule Protocol for " + "streams (i.e. having a `__arrow_c_stream__` method), " + f"got {type(data)!r}." + ) + + if schema is not None: + if not hasattr(schema, "__arrow_c_schema__"): + raise TypeError( + "Expected an object implementing the Arrow PyCapsule Protocol for " + "schema (i.e. having a `__arrow_c_schema__` method), " + f"got {type(schema)!r}." + ) + requested = schema.__arrow_c_schema__() + else: + requested = None + + capsule = data.__arrow_c_stream__(requested) + return RecordBatchReader._import_from_c_capsule(capsule) + + @staticmethod + def from_batches(Schema schema not None, batches): + """ + Create RecordBatchReader from an iterable of batches. + + Parameters + ---------- + schema : Schema + The shared schema of the record batches + batches : Iterable[RecordBatch] + The batches that this reader will return. + + Returns + ------- + reader : RecordBatchReader + """ + cdef: + shared_ptr[CSchema] c_schema + shared_ptr[CRecordBatchReader] c_reader + RecordBatchReader self + + c_schema = pyarrow_unwrap_schema(schema) + c_reader = GetResultValue(CPyRecordBatchReader.Make( + c_schema, batches)) + + self = RecordBatchReader.__new__(RecordBatchReader) + self.reader = c_reader + return self + + +cdef class _RecordBatchStreamReader(RecordBatchReader): + cdef: + shared_ptr[CInputStream] in_stream + CIpcReadOptions options + CRecordBatchStreamReader* stream_reader + + def __cinit__(self): + pass + + def _open(self, source, IpcReadOptions options=IpcReadOptions(), + MemoryPool memory_pool=None): + self.options = options.c_options + self.options.memory_pool = maybe_unbox_memory_pool(memory_pool) + _get_input_stream(source, &self.in_stream) + with nogil: + self.reader = GetResultValue(CRecordBatchStreamReader.Open( + self.in_stream, self.options)) + self.stream_reader = self.reader.get() + + @property + def stats(self): + """ + Current IPC read statistics. + """ + if not self.reader: + raise ValueError("Operation on closed reader") + return _wrap_read_stats(self.stream_reader.stats()) + + +cdef class _RecordBatchFileWriter(_RecordBatchStreamWriter): + + def _open(self, sink, Schema schema not None, + IpcWriteOptions options=IpcWriteOptions(), + metadata=None): + cdef: + shared_ptr[COutputStream] c_sink + shared_ptr[const CKeyValueMetadata] c_meta + + self.options = options.c_options + get_writer(sink, &c_sink) + + metadata = ensure_metadata(metadata, allow_none=True) + c_meta = pyarrow_unwrap_metadata(metadata) + + with nogil: + self.writer = GetResultValue( + MakeFileWriter(c_sink, schema.sp_schema, self.options, c_meta)) + +_RecordBatchWithMetadata = namedtuple( + 'RecordBatchWithMetadata', + ('batch', 'custom_metadata')) + + +class RecordBatchWithMetadata(_RecordBatchWithMetadata): + """RecordBatch with its custom metadata + + Parameters + ---------- + batch : RecordBatch + custom_metadata : KeyValueMetadata + """ + __slots__ = () + + +@staticmethod +cdef _wrap_record_batch_with_metadata(CRecordBatchWithMetadata c): + return RecordBatchWithMetadata(pyarrow_wrap_batch(c.batch), + pyarrow_wrap_metadata(c.custom_metadata)) + + +cdef class _RecordBatchFileReader(_Weakrefable): + cdef: + SharedPtrNoGIL[CRecordBatchFileReader] reader + shared_ptr[CRandomAccessFile] file + CIpcReadOptions options + + cdef readonly: + Schema schema + + def __cinit__(self): + pass + + def _open(self, source, footer_offset=None, + IpcReadOptions options=IpcReadOptions(), + MemoryPool memory_pool=None): + self.options = options.c_options + self.options.memory_pool = maybe_unbox_memory_pool(memory_pool) + try: + source = as_buffer(source) + except TypeError: + pass + + get_reader(source, False, &self.file) + + cdef int64_t offset = 0 + if footer_offset is not None: + offset = footer_offset + + with nogil: + if offset != 0: + self.reader = GetResultValue( + CRecordBatchFileReader.Open2(self.file.get(), offset, + self.options)) + + else: + self.reader = GetResultValue( + CRecordBatchFileReader.Open(self.file.get(), + self.options)) + + self.schema = pyarrow_wrap_schema(self.reader.get().schema()) + + @property + def num_record_batches(self): + """ + The number of record batches in the IPC file. + """ + return self.reader.get().num_record_batches() + + def get_batch(self, int i): + """ + Read the record batch with the given index. + + Parameters + ---------- + i : int + The index of the record batch in the IPC file. + + Returns + ------- + batch : RecordBatch + """ + cdef shared_ptr[CRecordBatch] batch + + if i < 0 or i >= self.num_record_batches: + raise ValueError(f'Batch number {i} out of range') + + with nogil: + batch = GetResultValue(self.reader.get().ReadRecordBatch(i)) + + return pyarrow_wrap_batch(batch) + + # TODO(wesm): ARROW-503: Function was renamed. Remove after a period of + # time has passed + get_record_batch = get_batch + + def get_batch_with_custom_metadata(self, int i): + """ + Read the record batch with the given index along with + its custom metadata + + Parameters + ---------- + i : int + The index of the record batch in the IPC file. + + Returns + ------- + batch : RecordBatch + custom_metadata : KeyValueMetadata + """ + cdef: + CRecordBatchWithMetadata batch_with_metadata + + if i < 0 or i >= self.num_record_batches: + raise ValueError(f'Batch number {i} out of range') + + with nogil: + batch_with_metadata = GetResultValue( + self.reader.get().ReadRecordBatchWithCustomMetadata(i)) + + return _wrap_record_batch_with_metadata(batch_with_metadata) + + def read_all(self): + """ + Read all record batches as a pyarrow.Table + """ + cdef: + vector[shared_ptr[CRecordBatch]] batches + shared_ptr[CTable] table + int i, nbatches + + nbatches = self.num_record_batches + + batches.resize(nbatches) + with nogil: + for i in range(nbatches): + batches[i] = GetResultValue(self.reader.get() + .ReadRecordBatch(i)) + table = GetResultValue( + CTable.FromRecordBatches(self.schema.sp_schema, move(batches))) + + return pyarrow_wrap_table(table) + + read_pandas = _ReadPandasMixin.read_pandas + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + pass + + @property + def stats(self): + """ + Current IPC read statistics. + """ + if not self.reader: + raise ValueError("Operation on closed reader") + return _wrap_read_stats(self.reader.get().stats()) + + @property + def metadata(self): + """ + File-level custom metadata as dict, where both keys and values are byte-like. + This kind of metadata can be written via ``ipc.new_file(..., metadata=...)``. + """ + wrapped = pyarrow_wrap_metadata(self.reader.get().metadata()) + return wrapped.to_dict() if wrapped is not None else None + + +def get_tensor_size(Tensor tensor): + """ + Return total size of serialized Tensor including metadata and padding. + + Parameters + ---------- + tensor : Tensor + The tensor for which we want to known the size. + """ + cdef int64_t size + with nogil: + check_status(GetTensorSize(deref(tensor.tp), &size)) + return size + + +def get_record_batch_size(RecordBatch batch): + """ + Return total size of serialized RecordBatch including metadata and padding. + + Parameters + ---------- + batch : RecordBatch + The recordbatch for which we want to know the size. + """ + cdef int64_t size + with nogil: + check_status(GetRecordBatchSize(deref(batch.batch), &size)) + return size + + +def write_tensor(Tensor tensor, NativeFile dest): + """ + Write pyarrow.Tensor to pyarrow.NativeFile object its current position. + + Parameters + ---------- + tensor : pyarrow.Tensor + dest : pyarrow.NativeFile + + Returns + ------- + bytes_written : int + Total number of bytes written to the file + """ + cdef: + int32_t metadata_length + int64_t body_length + + handle = dest.get_output_stream() + + with nogil: + check_status( + WriteTensor(deref(tensor.tp), handle.get(), + &metadata_length, &body_length)) + + return metadata_length + body_length + + +cdef NativeFile as_native_file(source): + if not isinstance(source, NativeFile): + if hasattr(source, 'read'): + source = PythonFile(source) + else: + source = BufferReader(source) + + if not isinstance(source, NativeFile): + raise ValueError( + f'Unable to read message from object with type: {type(source)}') + return source + + +def read_tensor(source): + """Read pyarrow.Tensor from pyarrow.NativeFile object from current + position. If the file source supports zero copy (e.g. a memory map), then + this operation does not allocate any memory. This function not assume that + the stream is aligned + + Parameters + ---------- + source : pyarrow.NativeFile + + Returns + ------- + tensor : Tensor + + """ + cdef: + shared_ptr[CTensor] sp_tensor + CInputStream* c_stream + NativeFile nf = as_native_file(source) + + c_stream = nf.get_input_stream().get() + with nogil: + sp_tensor = GetResultValue(ReadTensor(c_stream)) + return pyarrow_wrap_tensor(sp_tensor) + + +def read_message(source): + """ + Read length-prefixed message from file or buffer-like object + + Parameters + ---------- + source : pyarrow.NativeFile, file-like object, or buffer-like object + + Returns + ------- + message : Message + """ + cdef: + Message result = Message.__new__(Message) + CInputStream* c_stream + + cdef NativeFile nf = as_native_file(source) + c_stream = nf.get_input_stream().get() + + with nogil: + result.message = move( + GetResultValue(ReadMessage(c_stream, c_default_memory_pool()))) + + if result.message == nullptr: + raise EOFError("End of Arrow stream") + + return result + + +def read_schema(obj, DictionaryMemo dictionary_memo=None): + """ + Read Schema from message or buffer + + Parameters + ---------- + obj : buffer or Message + dictionary_memo : DictionaryMemo, optional + Needed to be able to reconstruct dictionary-encoded fields + with read_record_batch + + Returns + ------- + schema : Schema + """ + cdef: + shared_ptr[CSchema] result + shared_ptr[CRandomAccessFile] cpp_file + Message message + CDictionaryMemo temp_memo + CDictionaryMemo* arg_dict_memo + + if dictionary_memo is not None: + arg_dict_memo = dictionary_memo.memo + else: + arg_dict_memo = &temp_memo + + if isinstance(obj, Message): + message = obj + with nogil: + result = GetResultValue(ReadSchema( + deref(message.message.get()), arg_dict_memo)) + else: + get_reader(obj, False, &cpp_file) + with nogil: + result = GetResultValue(ReadSchema(cpp_file.get(), arg_dict_memo)) + + return pyarrow_wrap_schema(result) + + +def read_record_batch(obj, Schema schema, + DictionaryMemo dictionary_memo=None): + """ + Read RecordBatch from message, given a known schema. If reading data from a + complete IPC stream, use ipc.open_stream instead + + Parameters + ---------- + obj : Message or Buffer-like + schema : Schema + dictionary_memo : DictionaryMemo, optional + If message contains dictionaries, must pass a populated + DictionaryMemo + + Returns + ------- + batch : RecordBatch + """ + cdef: + shared_ptr[CRecordBatch] result + Message message + CDictionaryMemo temp_memo + CDictionaryMemo* arg_dict_memo + + if isinstance(obj, Message): + message = obj + else: + message = read_message(obj) + + if dictionary_memo is not None: + arg_dict_memo = dictionary_memo.memo + else: + arg_dict_memo = &temp_memo + + with nogil: + result = GetResultValue( + ReadRecordBatch(deref(message.message.get()), + schema.sp_schema, + arg_dict_memo, + CIpcReadOptions.Defaults())) + + return pyarrow_wrap_batch(result) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/ipc.py b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/ipc.py new file mode 100644 index 0000000000000000000000000000000000000000..4d3f046fad33b3661109e8e90d73a97b454c9b7b --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/ipc.py @@ -0,0 +1,280 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Arrow file and stream reader/writer classes, and other messaging tools + +import os + +import pyarrow as pa + +from pyarrow.lib import (IpcReadOptions, IpcWriteOptions, ReadStats, WriteStats, # noqa + Message, MessageReader, + RecordBatchReader, _ReadPandasMixin, + MetadataVersion, Alignment, + read_message, read_record_batch, read_schema, + read_tensor, write_tensor, + get_record_batch_size, get_tensor_size) +import pyarrow.lib as lib + + +class RecordBatchStreamReader(lib._RecordBatchStreamReader): + """ + Reader for the Arrow streaming binary format. + + Parameters + ---------- + source : bytes/buffer-like, pyarrow.NativeFile, or file-like Python object + Either an in-memory buffer, or a readable file object. + If you want to use memory map use MemoryMappedFile as source. + options : pyarrow.ipc.IpcReadOptions + Options for IPC deserialization. + If None, default values will be used. + memory_pool : MemoryPool, default None + If None, default memory pool is used. + """ + + def __init__(self, source, *, options=None, memory_pool=None): + options = _ensure_default_ipc_read_options(options) + self._open(source, options=options, memory_pool=memory_pool) + + +_ipc_writer_class_doc = """\ +Parameters +---------- +sink : str, pyarrow.NativeFile, or file-like Python object + Either a file path, or a writable file object. +schema : pyarrow.Schema + The Arrow schema for data to be written to the file. +options : pyarrow.ipc.IpcWriteOptions + Options for IPC serialization. + + If None, default values will be used: the legacy format will not + be used unless overridden by setting the environment variable + ARROW_PRE_0_15_IPC_FORMAT=1, and the V5 metadata version will be + used unless overridden by setting the environment variable + ARROW_PRE_1_0_METADATA_VERSION=1.""" + + +_ipc_file_writer_class_doc = ( + _ipc_writer_class_doc + + "\n" + + """\ +metadata : dict | pyarrow.KeyValueMetadata, optional + Key/value pairs (both must be bytes-like) that will be stored + in the file footer and are retrievable via + pyarrow.ipc.open_file(...).metadata.""" +) + + +class RecordBatchStreamWriter(lib._RecordBatchStreamWriter): + __doc__ = f"""Writer for the Arrow streaming binary format + +{_ipc_writer_class_doc}""" + + def __init__(self, sink, schema, *, options=None): + options = _get_legacy_format_default(options) + self._open(sink, schema, options=options) + + +class RecordBatchFileReader(lib._RecordBatchFileReader): + """ + Class for reading Arrow record batch data from the Arrow binary file format + + Parameters + ---------- + source : bytes/buffer-like, pyarrow.NativeFile, or file-like Python object + Either an in-memory buffer, or a readable file object. + If you want to use memory map use MemoryMappedFile as source. + footer_offset : int, default None + If the file is embedded in some larger file, this is the byte offset to + the very end of the file data + options : pyarrow.ipc.IpcReadOptions + Options for IPC serialization. + If None, default values will be used. + memory_pool : MemoryPool, default None + If None, default memory pool is used. + """ + + def __init__(self, source, footer_offset=None, *, options=None, + memory_pool=None): + options = _ensure_default_ipc_read_options(options) + self._open(source, footer_offset=footer_offset, + options=options, memory_pool=memory_pool) + + +class RecordBatchFileWriter(lib._RecordBatchFileWriter): + + __doc__ = f"""Writer to create the Arrow binary file format + +{_ipc_file_writer_class_doc}""" + + def __init__(self, sink, schema, *, options=None, metadata=None): + options = _get_legacy_format_default(options) + self._open(sink, schema, options=options, metadata=metadata) + + +def _get_legacy_format_default(options): + if options: + if not isinstance(options, IpcWriteOptions): + raise TypeError(f"expected IpcWriteOptions, got {type(options)}") + return options + + metadata_version = MetadataVersion.V5 + use_legacy_format = \ + bool(int(os.environ.get('ARROW_PRE_0_15_IPC_FORMAT', '0'))) + if bool(int(os.environ.get('ARROW_PRE_1_0_METADATA_VERSION', '0'))): + metadata_version = MetadataVersion.V4 + return IpcWriteOptions(use_legacy_format=use_legacy_format, + metadata_version=metadata_version) + + +def _ensure_default_ipc_read_options(options): + if options and not isinstance(options, IpcReadOptions): + raise TypeError(f"expected IpcReadOptions, got {type(options)}") + return options or IpcReadOptions() + + +def new_stream(sink, schema, *, options=None): + return RecordBatchStreamWriter(sink, schema, + options=options) + + +new_stream.__doc__ = f"""\ +Create an Arrow columnar IPC stream writer instance + +{_ipc_writer_class_doc} + +Returns +------- +writer : RecordBatchStreamWriter + A writer for the given sink +""" + + +def open_stream(source, *, options=None, memory_pool=None): + """ + Create reader for Arrow streaming format. + + Parameters + ---------- + source : bytes/buffer-like, pyarrow.NativeFile, or file-like Python object + Either an in-memory buffer, or a readable file object. + options : pyarrow.ipc.IpcReadOptions + Options for IPC serialization. + If None, default values will be used. + memory_pool : MemoryPool, default None + If None, default memory pool is used. + + Returns + ------- + reader : RecordBatchStreamReader + A reader for the given source + """ + return RecordBatchStreamReader(source, options=options, + memory_pool=memory_pool) + + +def new_file(sink, schema, *, options=None, metadata=None): + return RecordBatchFileWriter(sink, schema, options=options, metadata=metadata) + + +new_file.__doc__ = f"""\ +Create an Arrow columnar IPC file writer instance + +{_ipc_file_writer_class_doc} + +Returns +------- +writer : RecordBatchFileWriter + A writer for the given sink +""" + + +def open_file(source, footer_offset=None, *, options=None, memory_pool=None): + """ + Create reader for Arrow file format. + + Parameters + ---------- + source : bytes/buffer-like, pyarrow.NativeFile, or file-like Python object + Either an in-memory buffer, or a readable file object. + footer_offset : int, default None + If the file is embedded in some larger file, this is the byte offset to + the very end of the file data. + options : pyarrow.ipc.IpcReadOptions + Options for IPC serialization. + If None, default values will be used. + memory_pool : MemoryPool, default None + If None, default memory pool is used. + + Returns + ------- + reader : RecordBatchFileReader + A reader for the given source + """ + return RecordBatchFileReader( + source, footer_offset=footer_offset, + options=options, memory_pool=memory_pool) + + +def serialize_pandas(df, *, nthreads=None, preserve_index=None): + """ + Serialize a pandas DataFrame into a buffer protocol compatible object. + + Parameters + ---------- + df : pandas.DataFrame + nthreads : int, default None + Number of threads to use for conversion to Arrow, default all CPUs. + preserve_index : bool, default None + The default of None will store the index as a column, except for + RangeIndex which is stored as metadata only. If True, always + preserve the pandas index data as a column. If False, no index + information is saved and the result will have a default RangeIndex. + + Returns + ------- + buf : buffer + An object compatible with the buffer protocol. + """ + batch = pa.RecordBatch.from_pandas(df, nthreads=nthreads, + preserve_index=preserve_index) + sink = pa.BufferOutputStream() + with pa.RecordBatchStreamWriter(sink, batch.schema) as writer: + writer.write_batch(batch) + return sink.getvalue() + + +def deserialize_pandas(buf, *, use_threads=True): + """Deserialize a buffer protocol compatible object into a pandas DataFrame. + + Parameters + ---------- + buf : buffer + An object compatible with the buffer protocol. + use_threads : bool, default True + Whether to parallelize the conversion using multiple threads. + + Returns + ------- + df : pandas.DataFrame + The buffer deserialized as pandas DataFrame + """ + buffer_reader = pa.BufferReader(buf) + with pa.RecordBatchStreamReader(buffer_reader) as reader: + table = reader.read_all() + return table.to_pandas(use_threads=use_threads) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/json.py b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/json.py new file mode 100644 index 0000000000000000000000000000000000000000..e06e46a1c57cb4cf0b03d35542bd2afaa5b1ce44 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/json.py @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +from pyarrow._json import ReadOptions, ParseOptions, read_json, open_json # noqa diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/jvm.py b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/jvm.py new file mode 100644 index 0000000000000000000000000000000000000000..a7fe3656a4104c9a5287aaca0f49ccf24b9ff38d --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/jvm.py @@ -0,0 +1,335 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Functions to interact with Arrow memory allocated by Arrow Java. + +These functions convert the objects holding the metadata, the actual +data is not copied at all. + +This will only work with a JVM running in the same process such as provided +through jpype. Modules that talk to a remote JVM like py4j will not work as the +memory addresses reported by them are not reachable in the python process. +""" + +import pyarrow as pa + + +class _JvmBufferNanny: + """ + An object that keeps a org.apache.arrow.memory.ArrowBuf's underlying + memory alive. + """ + ref_manager = None + + def __init__(self, jvm_buf): + ref_manager = jvm_buf.getReferenceManager() + # Will raise a java.lang.IllegalArgumentException if the buffer + # is already freed. It seems that exception cannot easily be + # caught... + ref_manager.retain() + self.ref_manager = ref_manager + + def __del__(self): + if self.ref_manager is not None: + self.ref_manager.release() + + +def jvm_buffer(jvm_buf): + """ + Construct an Arrow buffer from org.apache.arrow.memory.ArrowBuf + + Parameters + ---------- + + jvm_buf: org.apache.arrow.memory.ArrowBuf + Arrow Buffer representation on the JVM. + + Returns + ------- + pyarrow.Buffer + Python Buffer that references the JVM memory. + """ + nanny = _JvmBufferNanny(jvm_buf) + address = jvm_buf.memoryAddress() + size = jvm_buf.capacity() + return pa.foreign_buffer(address, size, base=nanny) + + +def _from_jvm_int_type(jvm_type): + """ + Convert a JVM int type to its Python equivalent. + + Parameters + ---------- + jvm_type : org.apache.arrow.vector.types.pojo.ArrowType$Int + + Returns + ------- + typ : pyarrow.DataType + """ + + bit_width = jvm_type.getBitWidth() + if jvm_type.getIsSigned(): + if bit_width == 8: + return pa.int8() + elif bit_width == 16: + return pa.int16() + elif bit_width == 32: + return pa.int32() + elif bit_width == 64: + return pa.int64() + else: + if bit_width == 8: + return pa.uint8() + elif bit_width == 16: + return pa.uint16() + elif bit_width == 32: + return pa.uint32() + elif bit_width == 64: + return pa.uint64() + + +def _from_jvm_float_type(jvm_type): + """ + Convert a JVM float type to its Python equivalent. + + Parameters + ---------- + jvm_type: org.apache.arrow.vector.types.pojo.ArrowType$FloatingPoint + + Returns + ------- + typ: pyarrow.DataType + """ + precision = jvm_type.getPrecision().toString() + if precision == 'HALF': + return pa.float16() + elif precision == 'SINGLE': + return pa.float32() + elif precision == 'DOUBLE': + return pa.float64() + + +def _from_jvm_time_type(jvm_type): + """ + Convert a JVM time type to its Python equivalent. + + Parameters + ---------- + jvm_type: org.apache.arrow.vector.types.pojo.ArrowType$Time + + Returns + ------- + typ: pyarrow.DataType + """ + time_unit = jvm_type.getUnit().toString() + if time_unit == 'SECOND': + assert jvm_type.getBitWidth() == 32 + return pa.time32('s') + elif time_unit == 'MILLISECOND': + assert jvm_type.getBitWidth() == 32 + return pa.time32('ms') + elif time_unit == 'MICROSECOND': + assert jvm_type.getBitWidth() == 64 + return pa.time64('us') + elif time_unit == 'NANOSECOND': + assert jvm_type.getBitWidth() == 64 + return pa.time64('ns') + + +def _from_jvm_timestamp_type(jvm_type): + """ + Convert a JVM timestamp type to its Python equivalent. + + Parameters + ---------- + jvm_type: org.apache.arrow.vector.types.pojo.ArrowType$Timestamp + + Returns + ------- + typ: pyarrow.DataType + """ + time_unit = jvm_type.getUnit().toString() + timezone = jvm_type.getTimezone() + if timezone is not None: + timezone = str(timezone) + if time_unit == 'SECOND': + return pa.timestamp('s', tz=timezone) + elif time_unit == 'MILLISECOND': + return pa.timestamp('ms', tz=timezone) + elif time_unit == 'MICROSECOND': + return pa.timestamp('us', tz=timezone) + elif time_unit == 'NANOSECOND': + return pa.timestamp('ns', tz=timezone) + + +def _from_jvm_date_type(jvm_type): + """ + Convert a JVM date type to its Python equivalent + + Parameters + ---------- + jvm_type: org.apache.arrow.vector.types.pojo.ArrowType$Date + + Returns + ------- + typ: pyarrow.DataType + """ + day_unit = jvm_type.getUnit().toString() + if day_unit == 'DAY': + return pa.date32() + elif day_unit == 'MILLISECOND': + return pa.date64() + + +def field(jvm_field): + """ + Construct a Field from a org.apache.arrow.vector.types.pojo.Field + instance. + + Parameters + ---------- + jvm_field: org.apache.arrow.vector.types.pojo.Field + + Returns + ------- + pyarrow.Field + """ + name = str(jvm_field.getName()) + jvm_type = jvm_field.getType() + + typ = None + if not jvm_type.isComplex(): + type_str = jvm_type.getTypeID().toString() + if type_str == 'Null': + typ = pa.null() + elif type_str == 'Int': + typ = _from_jvm_int_type(jvm_type) + elif type_str == 'FloatingPoint': + typ = _from_jvm_float_type(jvm_type) + elif type_str == 'Utf8': + typ = pa.string() + elif type_str == 'Binary': + typ = pa.binary() + elif type_str == 'FixedSizeBinary': + typ = pa.binary(jvm_type.getByteWidth()) + elif type_str == 'Bool': + typ = pa.bool_() + elif type_str == 'Time': + typ = _from_jvm_time_type(jvm_type) + elif type_str == 'Timestamp': + typ = _from_jvm_timestamp_type(jvm_type) + elif type_str == 'Date': + typ = _from_jvm_date_type(jvm_type) + elif type_str == 'Decimal': + typ = pa.decimal128(jvm_type.getPrecision(), jvm_type.getScale()) + else: + raise NotImplementedError( + f"Unsupported JVM type: {type_str}") + else: + # TODO: The following JVM types are not implemented: + # Struct, List, FixedSizeList, Union, Dictionary + raise NotImplementedError( + "JVM field conversion only implemented for primitive types.") + + nullable = jvm_field.isNullable() + jvm_metadata = jvm_field.getMetadata() + if jvm_metadata.isEmpty(): + metadata = None + else: + metadata = {str(entry.getKey()): str(entry.getValue()) + for entry in jvm_metadata.entrySet()} + return pa.field(name, typ, nullable, metadata) + + +def schema(jvm_schema): + """ + Construct a Schema from a org.apache.arrow.vector.types.pojo.Schema + instance. + + Parameters + ---------- + jvm_schema: org.apache.arrow.vector.types.pojo.Schema + + Returns + ------- + pyarrow.Schema + """ + fields = jvm_schema.getFields() + fields = [field(f) for f in fields] + jvm_metadata = jvm_schema.getCustomMetadata() + if jvm_metadata.isEmpty(): + metadata = None + else: + metadata = {str(entry.getKey()): str(entry.getValue()) + for entry in jvm_metadata.entrySet()} + return pa.schema(fields, metadata) + + +def array(jvm_array): + """ + Construct an (Python) Array from its JVM equivalent. + + Parameters + ---------- + jvm_array : org.apache.arrow.vector.ValueVector + + Returns + ------- + array : Array + """ + if jvm_array.getField().getType().isComplex(): + minor_type_str = jvm_array.getMinorType().toString() + raise NotImplementedError( + f"Cannot convert JVM Arrow array of type {minor_type_str}, " + "complex types not yet implemented.") + dtype = field(jvm_array.getField()).type + buffers = [jvm_buffer(buf) + for buf in list(jvm_array.getBuffers(False))] + + # If JVM has an empty Vector, buffer list will be empty so create manually + if len(buffers) == 0: + return pa.array([], type=dtype) + + length = jvm_array.getValueCount() + null_count = jvm_array.getNullCount() + return pa.Array.from_buffers(dtype, length, buffers, null_count) + + +def record_batch(jvm_vector_schema_root): + """ + Construct a (Python) RecordBatch from a JVM VectorSchemaRoot + + Parameters + ---------- + jvm_vector_schema_root : org.apache.arrow.vector.VectorSchemaRoot + + Returns + ------- + record_batch: pyarrow.RecordBatch + """ + pa_schema = schema(jvm_vector_schema_root.getSchema()) + + arrays = [] + for name in pa_schema.names: + arrays.append(array(jvm_vector_schema_root.getVector(name))) + + return pa.RecordBatch.from_arrays( + arrays, + pa_schema.names, + metadata=pa_schema.metadata + ) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/lib.cp312-win_amd64.pyd b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/lib.cp312-win_amd64.pyd new file mode 100644 index 0000000000000000000000000000000000000000..609f0a9fe6871fe349afedcbb463a837ea45b9c2 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/lib.cp312-win_amd64.pyd @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ddd73a5f53645d245a17bb76edb89e639c2af043934aa4c484d9f29cfffc257 +size 3610624 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/lib.pxd b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/lib.pxd new file mode 100644 index 0000000000000000000000000000000000000000..9a9c87b81ff2406c46054170ea9abe773f1f2d90 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/lib.pxd @@ -0,0 +1,753 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# cython: language_level = 3 + +from cpython cimport PyObject +from cpython.slice cimport PySlice_Check + +from libcpp cimport nullptr, bool as c_bool +from libcpp.cast cimport dynamic_cast +from libcpp.memory cimport static_pointer_cast, dynamic_pointer_cast +from libcpp.utility cimport move + +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport * +from pyarrow.includes.libarrow_python cimport * + + +cdef int check_status(const CStatus& status) except -1 nogil +cdef object convert_status(const CStatus& status) + + +cdef class _Weakrefable: + cdef object __weakref__ + + +cdef class IpcWriteOptions(_Weakrefable): + cdef: + CIpcWriteOptions c_options + + +cdef IpcWriteOptions wrap_ipc_write_options(CIpcWriteOptions c) + + +cdef class IpcReadOptions(_Weakrefable): + cdef: + CIpcReadOptions c_options + + +cdef IpcReadOptions wrap_ipc_read_options(CIpcReadOptions c) + + +cdef _wrap_read_stats(CIpcReadStats c) + + +cdef class Message(_Weakrefable): + cdef: + unique_ptr[CMessage] message + + +cdef class MemoryPool(_Weakrefable): + cdef: + CMemoryPool* pool + + cdef void init(self, CMemoryPool* pool) + + +cdef CMemoryPool* maybe_unbox_memory_pool(MemoryPool memory_pool) + + +cdef object box_memory_pool(CMemoryPool* pool) + + +cdef class DataType(_Weakrefable): + cdef: + shared_ptr[CDataType] sp_type + CDataType* type + bytes pep3118_format + + cdef void init(self, const shared_ptr[CDataType]& type) except * + cpdef Field field(self, i) + + +cdef class ListType(DataType): + cdef: + const CListType* list_type + + +cdef class LargeListType(DataType): + cdef: + const CLargeListType* list_type + + +cdef class ListViewType(DataType): + cdef: + const CListViewType* list_view_type + + +cdef class LargeListViewType(DataType): + cdef: + const CLargeListViewType* list_view_type + + +cdef class MapType(DataType): + cdef: + const CMapType* map_type + + +cdef class FixedSizeListType(DataType): + cdef: + const CFixedSizeListType* list_type + + +cdef class StructType(DataType): + cdef: + const CStructType* struct_type + + cdef Field field_by_name(self, name) + + +cdef class DictionaryMemo(_Weakrefable): + cdef: + # Even though the CDictionaryMemo instance is private, we allocate + # it on the heap so as to avoid C++ ABI issues with Python wheels. + shared_ptr[CDictionaryMemo] sp_memo + CDictionaryMemo* memo + + +cdef class DictionaryType(DataType): + cdef: + const CDictionaryType* dict_type + + +cdef class TimestampType(DataType): + cdef: + const CTimestampType* ts_type + + +cdef class Time32Type(DataType): + cdef: + const CTime32Type* time_type + + +cdef class Time64Type(DataType): + cdef: + const CTime64Type* time_type + + +cdef class DurationType(DataType): + cdef: + const CDurationType* duration_type + + +cdef class FixedSizeBinaryType(DataType): + cdef: + const CFixedSizeBinaryType* fixed_size_binary_type + + +cdef class Decimal32Type(FixedSizeBinaryType): + cdef: + const CDecimal32Type* decimal32_type + + +cdef class Decimal64Type(FixedSizeBinaryType): + cdef: + const CDecimal64Type* decimal64_type + + +cdef class Decimal128Type(FixedSizeBinaryType): + cdef: + const CDecimal128Type* decimal128_type + + +cdef class Decimal256Type(FixedSizeBinaryType): + cdef: + const CDecimal256Type* decimal256_type + + +cdef class RunEndEncodedType(DataType): + cdef: + const CRunEndEncodedType* run_end_encoded_type + + +cdef class BaseExtensionType(DataType): + cdef: + const CExtensionType* ext_type + + +cdef class ExtensionType(BaseExtensionType): + cdef: + const CPyExtensionType* cpy_ext_type + + +cdef class FixedShapeTensorType(BaseExtensionType): + cdef: + const CFixedShapeTensorType* tensor_ext_type + +cdef class Bool8Type(BaseExtensionType): + cdef: + const CBool8Type* bool8_ext_type + +cdef class OpaqueType(BaseExtensionType): + cdef: + const COpaqueType* opaque_ext_type + +cdef class UuidType(BaseExtensionType): + cdef: + const CUuidType* uuid_ext_type + +cdef class JsonType(BaseExtensionType): + cdef: + const CJsonType* json_ext_type + + +cdef class _Metadata(_Weakrefable): + # required because KeyValueMetadata also extends collections.abc.Mapping + # and the first parent class must be an extension type + pass + + +cdef class KeyValueMetadata(_Metadata): + cdef: + shared_ptr[const CKeyValueMetadata] wrapped + const CKeyValueMetadata* metadata + + cdef void init(self, const shared_ptr[const CKeyValueMetadata]& wrapped) + + @staticmethod + cdef wrap(const shared_ptr[const CKeyValueMetadata]& sp) + cdef inline shared_ptr[const CKeyValueMetadata] unwrap(self) nogil + + +cdef class Field(_Weakrefable): + cdef: + shared_ptr[CField] sp_field + CField* field + + cdef readonly: + DataType type + + cdef void init(self, const shared_ptr[CField]& field) + + +cdef class Schema(_Weakrefable): + cdef: + shared_ptr[CSchema] sp_schema + CSchema* schema + + cdef void init(self, const vector[shared_ptr[CField]]& fields) + cdef void init_schema(self, const shared_ptr[CSchema]& schema) + + +cdef class Scalar(_Weakrefable): + cdef: + shared_ptr[CScalar] wrapped + + cdef void init(self, const shared_ptr[CScalar]& wrapped) + + @staticmethod + cdef wrap(const shared_ptr[CScalar]& wrapped) + + cdef inline shared_ptr[CScalar] unwrap(self) nogil + + +cdef class ArrayStatistics(_Weakrefable): + cdef: + shared_ptr[CArrayStatistics] sp_statistics + + cdef void init(self, const shared_ptr[CArrayStatistics]& sp_statistics) except * + cdef _get_value(self, const optional[CArrayStatisticsValueType]& optional_value) + + +cdef class _PandasConvertible(_Weakrefable): + pass + + +cdef class Array(_PandasConvertible): + cdef: + shared_ptr[CArray] sp_array + CArray* ap + + cdef readonly: + DataType type + # To allow Table to propagate metadata to pandas.Series + object _name + + cdef void init(self, const shared_ptr[CArray]& sp_array) except * + cdef getitem(self, int64_t i) + cdef int64_t length(self) + cdef void _assert_cpu(self) except * + + +cdef class Tensor(_Weakrefable): + cdef: + shared_ptr[CTensor] sp_tensor + CTensor* tp + + cdef readonly: + DataType type + bytes _ssize_t_shape + bytes _ssize_t_strides + + cdef void init(self, const shared_ptr[CTensor]& sp_tensor) + + +cdef class SparseCSRMatrix(_Weakrefable): + cdef: + shared_ptr[CSparseCSRMatrix] sp_sparse_tensor + CSparseCSRMatrix* stp + + cdef readonly: + DataType type + + cdef void init(self, const shared_ptr[CSparseCSRMatrix]& sp_sparse_tensor) + + +cdef class SparseCSCMatrix(_Weakrefable): + cdef: + shared_ptr[CSparseCSCMatrix] sp_sparse_tensor + CSparseCSCMatrix* stp + + cdef readonly: + DataType type + + cdef void init(self, const shared_ptr[CSparseCSCMatrix]& sp_sparse_tensor) + + +cdef class SparseCOOTensor(_Weakrefable): + cdef: + shared_ptr[CSparseCOOTensor] sp_sparse_tensor + CSparseCOOTensor* stp + + cdef readonly: + DataType type + + cdef void init(self, const shared_ptr[CSparseCOOTensor]& sp_sparse_tensor) + + +cdef class SparseCSFTensor(_Weakrefable): + cdef: + shared_ptr[CSparseCSFTensor] sp_sparse_tensor + CSparseCSFTensor* stp + + cdef readonly: + DataType type + + cdef void init(self, const shared_ptr[CSparseCSFTensor]& sp_sparse_tensor) + + +cdef class NullArray(Array): + pass + + +cdef class BooleanArray(Array): + pass + + +cdef class NumericArray(Array): + pass + + +cdef class IntegerArray(NumericArray): + pass + + +cdef class FloatingPointArray(NumericArray): + pass + + +cdef class Int8Array(IntegerArray): + pass + + +cdef class UInt8Array(IntegerArray): + pass + + +cdef class Int16Array(IntegerArray): + pass + + +cdef class UInt16Array(IntegerArray): + pass + + +cdef class Int32Array(IntegerArray): + pass + + +cdef class UInt32Array(IntegerArray): + pass + + +cdef class Int64Array(IntegerArray): + pass + + +cdef class UInt64Array(IntegerArray): + pass + + +cdef class HalfFloatArray(FloatingPointArray): + pass + + +cdef class FloatArray(FloatingPointArray): + pass + + +cdef class DoubleArray(FloatingPointArray): + pass + + +cdef class FixedSizeBinaryArray(Array): + pass + + +cdef class Decimal32Array(FixedSizeBinaryArray): + pass + + +cdef class Decimal64Array(FixedSizeBinaryArray): + pass + + +cdef class Decimal128Array(FixedSizeBinaryArray): + pass + + +cdef class Decimal256Array(FixedSizeBinaryArray): + pass + + +cdef class StructArray(Array): + pass + + +cdef class BaseListArray(Array): + pass + + +cdef class ListArray(BaseListArray): + pass + + +cdef class LargeListArray(BaseListArray): + pass + + +cdef class ListViewArray(BaseListArray): + pass + + +cdef class LargeListViewArray(BaseListArray): + pass + + +cdef class MapArray(ListArray): + pass + + +cdef class FixedSizeListArray(BaseListArray): + pass + + +cdef class UnionArray(Array): + pass + + +cdef class StringArray(Array): + pass + + +cdef class BinaryArray(Array): + pass + + +cdef class StringViewArray(Array): + pass + + +cdef class BinaryViewArray(Array): + pass + + +cdef class DictionaryArray(Array): + cdef: + object _indices, _dictionary + + +cdef class ExtensionArray(Array): + pass + + +cdef class MonthDayNanoIntervalArray(Array): + pass + + +cdef wrap_array_output(PyObject* output) +cdef wrap_datum(const CDatum& datum) + + +cdef class ChunkedArray(_PandasConvertible): + cdef: + shared_ptr[CChunkedArray] sp_chunked_array + CChunkedArray* chunked_array + c_bool _is_cpu + c_bool _init_is_cpu + + cdef readonly: + # To allow Table to propagate metadata to pandas.Series + object _name + + cdef void init(self, const shared_ptr[CChunkedArray]& chunked_array) + cdef getitem(self, int64_t i) + + +cdef class _Tabular(_PandasConvertible): + cdef void _assert_cpu(self) except * + + +cdef class Table(_Tabular): + cdef: + shared_ptr[CTable] sp_table + CTable* table + c_bool _is_cpu + c_bool _init_is_cpu + + cdef void init(self, const shared_ptr[CTable]& table) + + +cdef class RecordBatch(_Tabular): + cdef: + shared_ptr[CRecordBatch] sp_batch + CRecordBatch* batch + Schema _schema + + cdef void init(self, const shared_ptr[CRecordBatch]& table) + + +cdef class Device(_Weakrefable): + cdef: + shared_ptr[CDevice] device + + cdef void init(self, const shared_ptr[CDevice]& device) + + @staticmethod + cdef wrap(const shared_ptr[CDevice]& device) + + cdef inline shared_ptr[CDevice] unwrap(self) nogil + + +cdef class MemoryManager(_Weakrefable): + cdef: + shared_ptr[CMemoryManager] memory_manager + + cdef void init(self, const shared_ptr[CMemoryManager]& memory_manager) + + @staticmethod + cdef wrap(const shared_ptr[CMemoryManager]& mm) + + cdef inline shared_ptr[CMemoryManager] unwrap(self) nogil + + +cdef class Buffer(_Weakrefable): + cdef: + shared_ptr[CBuffer] buffer + Py_ssize_t shape[1] + Py_ssize_t strides[1] + + cdef void init(self, const shared_ptr[CBuffer]& buffer) + cdef getitem(self, int64_t i) + + +cdef class ResizableBuffer(Buffer): + + cdef void init_rz(self, const shared_ptr[CResizableBuffer]& buffer) + + +cdef class NativeFile(_Weakrefable): + cdef: + shared_ptr[CInputStream] input_stream + shared_ptr[CRandomAccessFile] random_access + shared_ptr[COutputStream] output_stream + bint is_readable + bint is_writable + bint is_seekable + bint _is_appending + bint own_file + + # By implementing these "virtual" functions (all functions in Cython + # extension classes are technically virtual in the C++ sense) we can expose + # the arrow::io abstract file interfaces to other components throughout the + # suite of Arrow C++ libraries + cdef set_random_access_file(self, shared_ptr[CRandomAccessFile] handle) + cdef set_input_stream(self, shared_ptr[CInputStream] handle) + cdef set_output_stream(self, shared_ptr[COutputStream] handle) + + cdef shared_ptr[CRandomAccessFile] get_random_access_file(self) except * + cdef shared_ptr[CInputStream] get_input_stream(self) except * + cdef shared_ptr[COutputStream] get_output_stream(self) except * + + +cdef class BufferedInputStream(NativeFile): + pass + + +cdef class BufferedOutputStream(NativeFile): + pass + + +cdef class CompressedInputStream(NativeFile): + pass + + +cdef class CompressedOutputStream(NativeFile): + pass + + +cdef class _CRecordBatchWriter(_Weakrefable): + cdef: + SharedPtrNoGIL[CRecordBatchWriter] writer + + +cdef class RecordBatchReader(_Weakrefable): + cdef: + SharedPtrNoGIL[CRecordBatchReader] reader + + +cdef class CacheOptions(_Weakrefable): + cdef: + CCacheOptions wrapped + + cdef void init(self, CCacheOptions options) + + cdef inline CCacheOptions unwrap(self) + + @staticmethod + cdef wrap(const CCacheOptions options) + + +cdef class Codec(_Weakrefable): + cdef: + shared_ptr[CCodec] wrapped + + cdef inline CCodec* unwrap(self) nogil + + +# This class is only used internally for now +cdef class StopToken: + cdef: + CStopToken stop_token + + cdef void init(self, CStopToken stop_token) + + +cdef get_input_stream(object source, c_bool use_memory_map, + shared_ptr[CInputStream]* reader) +cdef get_reader(object source, c_bool use_memory_map, + shared_ptr[CRandomAccessFile]* reader) +cdef get_writer(object source, shared_ptr[COutputStream]* writer) +cdef NativeFile get_native_file(object source, c_bool use_memory_map) + +cdef shared_ptr[CInputStream] native_transcoding_input_stream( + shared_ptr[CInputStream] stream, src_encoding, + dest_encoding) except * + +cdef shared_ptr[function[StreamWrapFunc]] make_streamwrap_func( + src_encoding, dest_encoding) except * + +# Default is allow_none=False +cpdef DataType ensure_type(object type, bint allow_none=*) + +cdef DataType primitive_type(Type type) + +cdef timeunit_to_string(TimeUnit unit) +cdef TimeUnit string_to_timeunit(unit) except * + +# Exceptions may be raised when converting dict values, so need to +# check exception state on return +cdef shared_ptr[const CKeyValueMetadata] pyarrow_unwrap_metadata( + object meta) except * +cdef object pyarrow_wrap_metadata( + const shared_ptr[const CKeyValueMetadata]& meta) + +cdef CField.CMergeOptions _parse_field_merge_options(str promote_options) except * + +# +# Public Cython API for 3rd party code +# +# If you add functions to this list, please also update +# `cpp/src/arrow/python/pyarrow.{h, cc}` +# + +# Wrapping C++ -> Python + +cdef public object pyarrow_wrap_buffer(const shared_ptr[CBuffer]& buf) +cdef public object pyarrow_wrap_resizable_buffer( + const shared_ptr[CResizableBuffer]& buf) + +cdef public object pyarrow_wrap_data_type(const shared_ptr[CDataType]& type) +cdef public object pyarrow_wrap_field(const shared_ptr[CField]& field) +cdef public object pyarrow_wrap_schema(const shared_ptr[CSchema]& type) + +cdef public object pyarrow_wrap_scalar(const shared_ptr[CScalar]& sp_scalar) + +cdef public object pyarrow_wrap_array(const shared_ptr[CArray]& sp_array) +cdef public object pyarrow_wrap_chunked_array( + const shared_ptr[CChunkedArray]& sp_array) + +cdef public object pyarrow_wrap_sparse_coo_tensor( + const shared_ptr[CSparseCOOTensor]& sp_sparse_tensor) +cdef public object pyarrow_wrap_sparse_csc_matrix( + const shared_ptr[CSparseCSCMatrix]& sp_sparse_tensor) +cdef public object pyarrow_wrap_sparse_csf_tensor( + const shared_ptr[CSparseCSFTensor]& sp_sparse_tensor) +cdef public object pyarrow_wrap_sparse_csr_matrix( + const shared_ptr[CSparseCSRMatrix]& sp_sparse_tensor) +cdef public object pyarrow_wrap_tensor(const shared_ptr[CTensor]& sp_tensor) + +cdef public object pyarrow_wrap_batch(const shared_ptr[CRecordBatch]& cbatch) +cdef public object pyarrow_wrap_table(const shared_ptr[CTable]& ctable) + +# Unwrapping Python -> C++ + +cdef public shared_ptr[CBuffer] pyarrow_unwrap_buffer(object buffer) + +cdef public shared_ptr[CDataType] pyarrow_unwrap_data_type(object data_type) +cdef public shared_ptr[CField] pyarrow_unwrap_field(object field) +cdef public shared_ptr[CSchema] pyarrow_unwrap_schema(object schema) + +cdef public shared_ptr[CScalar] pyarrow_unwrap_scalar(object scalar) + +cdef public shared_ptr[CArray] pyarrow_unwrap_array(object array) +cdef public shared_ptr[CChunkedArray] pyarrow_unwrap_chunked_array( + object array) + +cdef public shared_ptr[CSparseCOOTensor] pyarrow_unwrap_sparse_coo_tensor( + object sparse_tensor) +cdef public shared_ptr[CSparseCSCMatrix] pyarrow_unwrap_sparse_csc_matrix( + object sparse_tensor) +cdef public shared_ptr[CSparseCSFTensor] pyarrow_unwrap_sparse_csf_tensor( + object sparse_tensor) +cdef public shared_ptr[CSparseCSRMatrix] pyarrow_unwrap_sparse_csr_matrix( + object sparse_tensor) +cdef public shared_ptr[CTensor] pyarrow_unwrap_tensor(object tensor) + +cdef public shared_ptr[CRecordBatch] pyarrow_unwrap_batch(object batch) +cdef public shared_ptr[CTable] pyarrow_unwrap_table(object table) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/lib.pyx b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/lib.pyx new file mode 100644 index 0000000000000000000000000000000000000000..8fcd8c59e7248423e42123d8cb879a81b92c1eba --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/lib.pyx @@ -0,0 +1,246 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# cython: profile = False +# cython: nonecheck = True +# distutils: language = c++ + +import datetime +import decimal as _pydecimal +try: + import numpy as np +except ImportError: + np = None +import os +import sys + +from cython.operator cimport dereference as deref +from pyarrow.includes.libarrow cimport * +from pyarrow.includes.libarrow_python cimport * +from pyarrow.includes.common cimport PyObject_to_object +cimport pyarrow.includes.libarrow_python as libarrow_python +cimport cpython as cp + + +# Initialize NumPy C API only if numpy was able to be imported +if np is not None: + arrow_init_numpy() + +# Initialize PyArrow C++ API +# (used from some of our C++ code, see e.g. ARROW-5260) +import_pyarrow() + + +MonthDayNano = NewMonthDayNanoTupleType() + + +def cpu_count(): + """ + Return the number of threads to use in parallel operations. + + The number of threads is determined at startup by inspecting the + ``OMP_NUM_THREADS`` and ``OMP_THREAD_LIMIT`` environment variables. + If neither is present, it will default to the number of hardware threads + on the system. It can be modified at runtime by calling + :func:`set_cpu_count()`. + + See Also + -------- + set_cpu_count : Modify the size of this pool. + io_thread_count : The analogous function for the I/O thread pool. + """ + return GetCpuThreadPoolCapacity() + + +def set_cpu_count(int count): + """ + Set the number of threads to use in parallel operations. + + Parameters + ---------- + count : int + The number of concurrent threads that should be used. + + See Also + -------- + cpu_count : Get the size of this pool. + set_io_thread_count : The analogous function for the I/O thread pool. + """ + if count < 1: + raise ValueError("CPU count must be strictly positive") + check_status(SetCpuThreadPoolCapacity(count)) + + +def is_opentelemetry_enabled() -> bool: + """ + Returns True if OpenTelemetry is enabled in libarrow. + """ + return libarrow_python.IsOpenTelemetryEnabled() + + +def is_threading_enabled() -> bool: + """ + Returns True if threading is enabled in libarrow. + + If it isn't enabled, then python shouldn't create any + threads either, because we're probably on a system where + threading doesn't work (e.g. Emscripten). + """ + return libarrow_python.IsThreadingEnabled() + + +Type_NA = _Type_NA +Type_BOOL = _Type_BOOL +Type_UINT8 = _Type_UINT8 +Type_INT8 = _Type_INT8 +Type_UINT16 = _Type_UINT16 +Type_INT16 = _Type_INT16 +Type_UINT32 = _Type_UINT32 +Type_INT32 = _Type_INT32 +Type_UINT64 = _Type_UINT64 +Type_INT64 = _Type_INT64 +Type_HALF_FLOAT = _Type_HALF_FLOAT +Type_FLOAT = _Type_FLOAT +Type_DOUBLE = _Type_DOUBLE +Type_DECIMAL32 = _Type_DECIMAL32 +Type_DECIMAL64 = _Type_DECIMAL64 +Type_DECIMAL128 = _Type_DECIMAL128 +Type_DECIMAL256 = _Type_DECIMAL256 +Type_DATE32 = _Type_DATE32 +Type_DATE64 = _Type_DATE64 +Type_TIMESTAMP = _Type_TIMESTAMP +Type_TIME32 = _Type_TIME32 +Type_TIME64 = _Type_TIME64 +Type_DURATION = _Type_DURATION +Type_INTERVAL_MONTH_DAY_NANO = _Type_INTERVAL_MONTH_DAY_NANO +Type_BINARY = _Type_BINARY +Type_STRING = _Type_STRING +Type_LARGE_BINARY = _Type_LARGE_BINARY +Type_LARGE_STRING = _Type_LARGE_STRING +Type_FIXED_SIZE_BINARY = _Type_FIXED_SIZE_BINARY +Type_BINARY_VIEW = _Type_BINARY_VIEW +Type_STRING_VIEW = _Type_STRING_VIEW +Type_LIST = _Type_LIST +Type_LARGE_LIST = _Type_LARGE_LIST +Type_LIST_VIEW = _Type_LIST_VIEW +Type_LARGE_LIST_VIEW = _Type_LARGE_LIST_VIEW +Type_MAP = _Type_MAP +Type_FIXED_SIZE_LIST = _Type_FIXED_SIZE_LIST +Type_STRUCT = _Type_STRUCT +Type_SPARSE_UNION = _Type_SPARSE_UNION +Type_DENSE_UNION = _Type_DENSE_UNION +Type_DICTIONARY = _Type_DICTIONARY +Type_RUN_END_ENCODED = _Type_RUN_END_ENCODED +Type_INTERVAL_MONTHS = _Type_INTERVAL_MONTHS +Type_INTERVAL_DAY_TIME = _Type_INTERVAL_DAY_TIME + +UnionMode_SPARSE = _UnionMode_SPARSE +UnionMode_DENSE = _UnionMode_DENSE + +__pc = None +__pac = None +__cuda_loaded = None + + +def _pc(): + global __pc + if __pc is None: + import pyarrow.compute as pc + __pc = pc + return __pc + + +def _pac(): + global __pac + if __pac is None: + import pyarrow.acero as pac + __pac = pac + return __pac + + +def _ensure_cuda_loaded(): + # Try importing the cuda module to ensure libarrow_cuda gets loaded + # to register the CUDA device for the C Data Interface import + global __cuda_loaded + if __cuda_loaded is None: + try: + import pyarrow.cuda # no-cython-lint + __cuda_loaded = True + except ImportError as exc: + __cuda_loaded = str(exc) + + if __cuda_loaded is not True: + raise ImportError( + "Trying to import data on a CUDA device, but PyArrow is not built with " + f"CUDA support.\n(importing 'pyarrow.cuda' resulted in \"{__cuda_loaded}\")." + ) + + +def _gdb_test_session(): + GdbTestSession() + + +# Assorted compatibility helpers +include "compat.pxi" + +# Exception types and Status handling +include "error.pxi" + +# Configuration information +include "config.pxi" + +# pandas API shim +include "pandas-shim.pxi" + +# Memory pools and allocation +include "memory.pxi" + +# Device type and memory manager +include "device.pxi" + +# DataType, Field, Schema +include "types.pxi" + +# Array scalar values +include "scalar.pxi" + +# Array types +include "array.pxi" + +# Builders +include "builder.pxi" + +# Column, Table, Record Batch +include "table.pxi" + +# Tensors +include "tensor.pxi" + +# DLPack +include "_dlpack.pxi" + +# File IO +include "io.pxi" + +# IPC / Messaging +include "ipc.pxi" + +# Micro-benchmark routines +include "benchmark.pxi" + +# Public API +include "public-api.pxi" diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/memory.pxi b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/memory.pxi new file mode 100644 index 0000000000000000000000000000000000000000..f44fc8acdce27d6ad020be834e36fa6ee83b50bb --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/memory.pxi @@ -0,0 +1,296 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# cython: profile=False +# distutils: language = c++ +# cython: embedsignature = True + + +cdef class MemoryPool(_Weakrefable): + """ + Base class for memory allocation. + + Besides tracking its number of allocated bytes, a memory pool also + takes care of the required 64-byte alignment for Arrow data. + """ + + def __init__(self): + raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, " + "use pyarrow.*_memory_pool instead.") + + cdef void init(self, CMemoryPool* pool): + self.pool = pool + + def release_unused(self): + """ + Attempt to return to the OS any memory being held onto by the pool. + + This function should not be called except potentially for + benchmarking or debugging as it could be expensive and detrimental to + performance. + + This is best effort and may not have any effect on some memory pools + or in some situations (e.g. fragmentation). + """ + cdef CMemoryPool* pool = c_get_memory_pool() + with nogil: + pool.ReleaseUnused() + + def bytes_allocated(self): + """ + Return the number of bytes that are currently allocated from this + memory pool. + """ + return self.pool.bytes_allocated() + + def total_bytes_allocated(self): + """ + Return the total number of bytes that have been allocated from this + memory pool. + """ + return self.pool.total_bytes_allocated() + + def max_memory(self): + """ + Return the peak memory allocation in this memory pool. + This can be an approximate number in multi-threaded applications. + + None is returned if the pool implementation doesn't know how to + compute this number. + """ + ret = self.pool.max_memory() + return ret if ret >= 0 else None + + def num_allocations(self): + """ + Return the number of allocations or reallocations that were made + using this memory pool. + """ + return self.pool.num_allocations() + + def print_stats(self): + """ + Print statistics about this memory pool. + + The output format is implementation-specific. Not all memory pools + implement this method. + """ + with nogil: + self.pool.PrintStats() + + @property + def backend_name(self): + """ + The name of the backend used by this MemoryPool (e.g. "jemalloc"). + """ + return frombytes(self.pool.backend_name()) + + def __repr__(self): + name = f"pyarrow.{self.__class__.__name__}" + return (f"<{name} " + f"backend_name={self.backend_name} " + f"bytes_allocated={self.bytes_allocated()} " + f"max_memory={self.max_memory()}>") + + +cdef CMemoryPool* maybe_unbox_memory_pool(MemoryPool memory_pool): + if memory_pool is None: + return c_get_memory_pool() + else: + return memory_pool.pool + + +cdef api object box_memory_pool(CMemoryPool *c_pool): + cdef MemoryPool pool = MemoryPool.__new__(MemoryPool) + pool.init(c_pool) + return pool + + +cdef class LoggingMemoryPool(MemoryPool): + cdef: + unique_ptr[CLoggingMemoryPool] logging_pool + + def __init__(self): + raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, " + "use pyarrow.logging_memory_pool instead.") + + +cdef class ProxyMemoryPool(MemoryPool): + """ + Memory pool implementation that tracks the number of bytes and + maximum memory allocated through its direct calls, while redirecting + to another memory pool. + """ + cdef: + unique_ptr[CProxyMemoryPool] proxy_pool + + def __init__(self): + raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, " + "use pyarrow.proxy_memory_pool instead.") + + +def default_memory_pool(): + """ + Return the process-global memory pool. + + Examples + -------- + >>> default_memory_pool() + + """ + cdef: + MemoryPool pool = MemoryPool.__new__(MemoryPool) + pool.init(c_get_memory_pool()) + return pool + + +def proxy_memory_pool(MemoryPool parent): + """ + Create and return a MemoryPool instance that redirects to the + *parent*, but with separate allocation statistics. + + Parameters + ---------- + parent : MemoryPool + The real memory pool that should be used for allocations. + """ + cdef ProxyMemoryPool out = ProxyMemoryPool.__new__(ProxyMemoryPool) + out.proxy_pool.reset(new CProxyMemoryPool(parent.pool)) + out.init(out.proxy_pool.get()) + return out + + +def logging_memory_pool(MemoryPool parent): + """ + Create and return a MemoryPool instance that redirects to the + *parent*, but also dumps allocation logs on stderr. + + Parameters + ---------- + parent : MemoryPool + The real memory pool that should be used for allocations. + """ + cdef LoggingMemoryPool out = LoggingMemoryPool.__new__( + LoggingMemoryPool, parent) + out.logging_pool.reset(new CLoggingMemoryPool(parent.pool)) + out.init(out.logging_pool.get()) + return out + + +def system_memory_pool(): + """ + Return a memory pool based on the C malloc heap. + """ + cdef: + MemoryPool pool = MemoryPool.__new__(MemoryPool) + pool.init(c_system_memory_pool()) + return pool + + +def jemalloc_memory_pool(): + """ + Return a memory pool based on the jemalloc heap. + + NotImplementedError is raised if jemalloc support is not enabled. + """ + cdef: + CMemoryPool* c_pool + MemoryPool pool = MemoryPool.__new__(MemoryPool) + check_status(c_jemalloc_memory_pool(&c_pool)) + pool.init(c_pool) + return pool + + +def mimalloc_memory_pool(): + """ + Return a memory pool based on the mimalloc heap. + + NotImplementedError is raised if mimalloc support is not enabled. + """ + cdef: + CMemoryPool* c_pool + MemoryPool pool = MemoryPool.__new__(MemoryPool) + check_status(c_mimalloc_memory_pool(&c_pool)) + pool.init(c_pool) + return pool + + +def set_memory_pool(MemoryPool pool): + """ + Set the default memory pool. + + Parameters + ---------- + pool : MemoryPool + The memory pool that should be used by default. + """ + c_set_default_memory_pool(pool.pool) + + +cdef MemoryPool _default_memory_pool = default_memory_pool() +cdef LoggingMemoryPool _logging_memory_pool = logging_memory_pool( + _default_memory_pool) + + +def log_memory_allocations(enable=True): + """ + Enable or disable memory allocator logging for debugging purposes + + Parameters + ---------- + enable : bool, default True + Pass False to disable logging + """ + if enable: + set_memory_pool(_logging_memory_pool) + else: + set_memory_pool(_default_memory_pool) + + +def total_allocated_bytes(): + """ + Return the currently allocated bytes from the default memory pool. + Other memory pools may not be accounted for. + """ + cdef CMemoryPool* pool = c_get_memory_pool() + return pool.bytes_allocated() + + +def jemalloc_set_decay_ms(decay_ms): + """ + Set arenas.dirty_decay_ms and arenas.muzzy_decay_ms to indicated number of + milliseconds. A value of 0 (the default) results in dirty / muzzy memory + pages being released right away to the OS, while a higher value will result + in a time-based decay. See the jemalloc docs for more information + + It's best to set this at the start of your application. + + Parameters + ---------- + decay_ms : int + Number of milliseconds to set for jemalloc decay conf parameters. Note + that this change will only affect future memory arenas + """ + check_status(c_jemalloc_set_decay_ms(decay_ms)) + + +def supported_memory_backends(): + """ + Return a list of available memory pool backends + """ + cdef vector[c_string] backends = c_supported_memory_backends() + return [backend.decode() for backend in backends] diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/orc.py b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/orc.py new file mode 100644 index 0000000000000000000000000000000000000000..ad6edacde565000226837eb2135572865a7cb660 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/orc.py @@ -0,0 +1,384 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +from numbers import Integral +import warnings + +from pyarrow.lib import Table +import pyarrow._orc as _orc +from pyarrow.fs import _resolve_filesystem_and_path + + +class ORCFile: + """ + Reader interface for a single ORC file + + Parameters + ---------- + source : str or pyarrow.NativeFile + Readable source. For passing Python file objects or byte buffers, + see pyarrow.io.PythonFileInterface or pyarrow.io.BufferReader. + """ + + def __init__(self, source): + self.reader = _orc.ORCReader() + self.reader.open(source) + + @property + def metadata(self): + """The file metadata, as an arrow KeyValueMetadata""" + return self.reader.metadata() + + @property + def schema(self): + """The file schema, as an arrow schema""" + return self.reader.schema() + + @property + def nrows(self): + """The number of rows in the file""" + return self.reader.nrows() + + @property + def nstripes(self): + """The number of stripes in the file""" + return self.reader.nstripes() + + @property + def file_version(self): + """Format version of the ORC file, must be 0.11 or 0.12""" + return self.reader.file_version() + + @property + def software_version(self): + """Software instance and version that wrote this file""" + return self.reader.software_version() + + @property + def compression(self): + """Compression codec of the file""" + return self.reader.compression() + + @property + def compression_size(self): + """Number of bytes to buffer for the compression codec in the file""" + return self.reader.compression_size() + + @property + def writer(self): + """Name of the writer that wrote this file. + If the writer is unknown then its Writer ID + (a number) is returned""" + return self.reader.writer() + + @property + def writer_version(self): + """Version of the writer""" + return self.reader.writer_version() + + @property + def row_index_stride(self): + """Number of rows per an entry in the row index or 0 + if there is no row index""" + return self.reader.row_index_stride() + + @property + def nstripe_statistics(self): + """Number of stripe statistics""" + return self.reader.nstripe_statistics() + + @property + def content_length(self): + """Length of the data stripes in the file in bytes""" + return self.reader.content_length() + + @property + def stripe_statistics_length(self): + """The number of compressed bytes in the file stripe statistics""" + return self.reader.stripe_statistics_length() + + @property + def file_footer_length(self): + """The number of compressed bytes in the file footer""" + return self.reader.file_footer_length() + + @property + def file_postscript_length(self): + """The number of bytes in the file postscript""" + return self.reader.file_postscript_length() + + @property + def file_length(self): + """The number of bytes in the file""" + return self.reader.file_length() + + def _select_names(self, columns=None): + if columns is None: + return None + + schema = self.schema + names = [] + for col in columns: + if isinstance(col, Integral): + col = int(col) + if 0 <= col < len(schema): + col = schema[col].name + names.append(col) + else: + raise ValueError("Column indices must be in 0 <= ind < %d," + " got %d" % (len(schema), col)) + else: + return columns + + return names + + def read_stripe(self, n, columns=None): + """Read a single stripe from the file. + + Parameters + ---------- + n : int + The stripe index + columns : list + If not None, only these columns will be read from the stripe. A + column name may be a prefix of a nested field, e.g. 'a' will select + 'a.b', 'a.c', and 'a.d.e' + + Returns + ------- + pyarrow.RecordBatch + Content of the stripe as a RecordBatch. + """ + columns = self._select_names(columns) + return self.reader.read_stripe(n, columns=columns) + + def read(self, columns=None): + """Read the whole file. + + Parameters + ---------- + columns : list + If not None, only these columns will be read from the file. A + column name may be a prefix of a nested field, e.g. 'a' will select + 'a.b', 'a.c', and 'a.d.e'. Output always follows the + ordering of the file and not the `columns` list. + + Returns + ------- + pyarrow.Table + Content of the file as a Table. + """ + columns = self._select_names(columns) + return self.reader.read(columns=columns) + + +_orc_writer_args_docs = """file_version : {"0.11", "0.12"}, default "0.12" + Determine which ORC file version to use. + `Hive 0.11 / ORC v0 `_ + is the older version + while `Hive 0.12 / ORC v1 `_ + is the newer one. +batch_size : int, default 1024 + Number of rows the ORC writer writes at a time. +stripe_size : int, default 64 * 1024 * 1024 + Size of each ORC stripe in bytes. +compression : string, default 'uncompressed' + The compression codec. + Valid values: {'UNCOMPRESSED', 'SNAPPY', 'ZLIB', 'LZ4', 'ZSTD'} + Note that LZ0 is currently not supported. +compression_block_size : int, default 64 * 1024 + Size of each compression block in bytes. +compression_strategy : string, default 'speed' + The compression strategy i.e. speed vs size reduction. + Valid values: {'SPEED', 'COMPRESSION'} +row_index_stride : int, default 10000 + The row index stride i.e. the number of rows per + an entry in the row index. +padding_tolerance : double, default 0.0 + The padding tolerance. +dictionary_key_size_threshold : double, default 0.0 + The dictionary key size threshold. 0 to disable dictionary encoding. + 1 to always enable dictionary encoding. +bloom_filter_columns : None, set-like or list-like, default None + Columns that use the bloom filter. +bloom_filter_fpp : double, default 0.05 + Upper limit of the false-positive rate of the bloom filter. +""" + + +class ORCWriter: + __doc__ = f""" +Writer interface for a single ORC file + +Parameters +---------- +where : str or pyarrow.io.NativeFile + Writable target. For passing Python file objects or byte buffers, + see pyarrow.io.PythonFileInterface, pyarrow.io.BufferOutputStream + or pyarrow.io.FixedSizeBufferWriter. +{_orc_writer_args_docs} +""" + + is_open = False + + def __init__(self, where, *, + file_version='0.12', + batch_size=1024, + stripe_size=64 * 1024 * 1024, + compression='uncompressed', + compression_block_size=65536, + compression_strategy='speed', + row_index_stride=10000, + padding_tolerance=0.0, + dictionary_key_size_threshold=0.0, + bloom_filter_columns=None, + bloom_filter_fpp=0.05, + ): + self.writer = _orc.ORCWriter() + self.writer.open( + where, + file_version=file_version, + batch_size=batch_size, + stripe_size=stripe_size, + compression=compression, + compression_block_size=compression_block_size, + compression_strategy=compression_strategy, + row_index_stride=row_index_stride, + padding_tolerance=padding_tolerance, + dictionary_key_size_threshold=dictionary_key_size_threshold, + bloom_filter_columns=bloom_filter_columns, + bloom_filter_fpp=bloom_filter_fpp + ) + self.is_open = True + + def __del__(self): + self.close() + + def __enter__(self): + return self + + def __exit__(self, *args, **kwargs): + self.close() + + def write(self, table): + """ + Write the table into an ORC file. The schema of the table must + be equal to the schema used when opening the ORC file. + + Parameters + ---------- + table : pyarrow.Table + The table to be written into the ORC file + """ + assert self.is_open + self.writer.write(table) + + def close(self): + """ + Close the ORC file + """ + if self.is_open: + self.writer.close() + self.is_open = False + + +def read_table(source, columns=None, filesystem=None): + filesystem, path = _resolve_filesystem_and_path(source, filesystem) + if filesystem is not None: + source = filesystem.open_input_file(path) + + if columns is not None and len(columns) == 0: + result = ORCFile(source).read().select(columns) + else: + result = ORCFile(source).read(columns=columns) + + return result + + +read_table.__doc__ = """ +Read a Table from an ORC file. + +Parameters +---------- +source : str, pyarrow.NativeFile, or file-like object + If a string passed, can be a single file name. For file-like objects, + only read a single file. Use pyarrow.BufferReader to read a file + contained in a bytes or buffer-like object. +columns : list + If not None, only these columns will be read from the file. A column + name may be a prefix of a nested field, e.g. 'a' will select 'a.b', + 'a.c', and 'a.d.e'. Output always follows the ordering of the file and + not the `columns` list. If empty, no columns will be read. Note + that the table will still have the correct num_rows set despite having + no columns. +filesystem : FileSystem, default None + If nothing passed, will be inferred based on path. + Path will try to be found in the local on-disk filesystem otherwise + it will be parsed as an URI to determine the filesystem. +""" + + +def write_table(table, where, *, + file_version='0.12', + batch_size=1024, + stripe_size=64 * 1024 * 1024, + compression='uncompressed', + compression_block_size=65536, + compression_strategy='speed', + row_index_stride=10000, + padding_tolerance=0.0, + dictionary_key_size_threshold=0.0, + bloom_filter_columns=None, + bloom_filter_fpp=0.05): + if isinstance(where, Table): + warnings.warn( + "The order of the arguments has changed. Pass as " + "'write_table(table, where)' instead. The old order will raise " + "an error in the future.", FutureWarning, stacklevel=2 + ) + table, where = where, table + with ORCWriter( + where, + file_version=file_version, + batch_size=batch_size, + stripe_size=stripe_size, + compression=compression, + compression_block_size=compression_block_size, + compression_strategy=compression_strategy, + row_index_stride=row_index_stride, + padding_tolerance=padding_tolerance, + dictionary_key_size_threshold=dictionary_key_size_threshold, + bloom_filter_columns=bloom_filter_columns, + bloom_filter_fpp=bloom_filter_fpp + ) as writer: + writer.write(table) + + +write_table.__doc__ = f""" +Write a table into an ORC file. + +Parameters +---------- +table : pyarrow.lib.Table + The table to be written into the ORC file +where : str or pyarrow.io.NativeFile + Writable target. For passing Python file objects or byte buffers, + see pyarrow.io.PythonFileInterface, pyarrow.io.BufferOutputStream + or pyarrow.io.FixedSizeBufferWriter. +{_orc_writer_args_docs} +""" diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/pandas-shim.pxi b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/pandas-shim.pxi new file mode 100644 index 0000000000000000000000000000000000000000..a76b580515e08ccfe175c146c22ff75267161e2a --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/pandas-shim.pxi @@ -0,0 +1,283 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# pandas lazy-loading API shim that reduces API call and import overhead + +import warnings +from threading import Lock + + +cdef class _PandasAPIShim(object): + """ + Lazy pandas importer that isolates usages of pandas APIs and avoids + importing pandas until it's actually needed + """ + cdef: + bint _tried_importing_pandas + bint _have_pandas + + cdef readonly: + object _loose_version, _version + object _pd, _types_api, _compat_module + object _data_frame, _index, _series, _categorical_type + object _datetimetz_type, _extension_array, _extension_dtype + object _array_like_types, _is_extension_array_dtype, _lock + bint has_sparse + bint _pd024 + bint _is_v1, _is_ge_v21, _is_ge_v23, _is_ge_v3, _is_ge_v3_strict + + def __init__(self): + self._lock = Lock() + self._tried_importing_pandas = False + self._have_pandas = 0 + + cdef _import_pandas(self, bint raise_): + try: + import pandas as pd + import pyarrow.pandas_compat as pdcompat + except ImportError: + self._have_pandas = False + if raise_: + raise + else: + return + + from pyarrow.vendored.version import Version + + self._pd = pd + self._version = pd.__version__ + self._loose_version = Version(pd.__version__) + self._is_v1 = False + + if self._loose_version < Version('1.0.0'): + self._have_pandas = False + if raise_: + raise ImportError( + f"pyarrow requires pandas 1.0.0 or above, pandas {self._version} is " + "installed" + ) + else: + warnings.warn( + f"pyarrow requires pandas 1.0.0 or above, pandas {self._version} is " + "installed. Therefore, pandas-specific integration is not " + "used.", + stacklevel=2 + ) + return + + self._is_v1 = self._loose_version < Version('2.0.0') + self._is_ge_v21 = self._loose_version >= Version('2.1.0') + self._is_ge_v23 = self._loose_version >= Version('2.3.0.dev0') + self._is_ge_v3 = self._loose_version >= Version('3.0.0.dev0') + self._is_ge_v3_strict = self._loose_version >= Version('3.0.0') + + self._compat_module = pdcompat + self._data_frame = pd.DataFrame + self._index = pd.Index + self._categorical_type = pd.Categorical + self._series = pd.Series + self._extension_array = pd.api.extensions.ExtensionArray + self._array_like_types = ( + self._series, self._index, self._categorical_type, + self._extension_array) + self._extension_dtype = pd.api.extensions.ExtensionDtype + self._is_extension_array_dtype = ( + pd.api.types.is_extension_array_dtype) + self._types_api = pd.api.types + self._datetimetz_type = pd.api.types.DatetimeTZDtype + self._have_pandas = True + self.has_sparse = False + + cdef inline _check_import(self, bint raise_=True): + if not self._tried_importing_pandas: + with self._lock: + if not self._tried_importing_pandas: + try: + self._import_pandas(raise_) + finally: + self._tried_importing_pandas = True + return + + if not self._have_pandas and raise_: + self._import_pandas(raise_) + + def series(self, *args, **kwargs): + self._check_import() + return self._series(*args, **kwargs) + + def data_frame(self, *args, **kwargs): + self._check_import() + return self._data_frame(*args, **kwargs) + + cdef inline bint _have_pandas_internal(self): + if not self._tried_importing_pandas: + self._check_import(raise_=False) + return self._have_pandas + + @property + def have_pandas(self): + return self._have_pandas_internal() + + @property + def compat(self): + self._check_import() + return self._compat_module + + @property + def pd(self): + self._check_import() + return self._pd + + cpdef infer_dtype(self, obj): + self._check_import() + try: + return self._types_api.infer_dtype(obj, skipna=False) + except AttributeError: + return self._pd.lib.infer_dtype(obj) + + cpdef pandas_dtype(self, dtype): + self._check_import() + try: + return self._types_api.pandas_dtype(dtype) + except AttributeError: + return None + + @property + def loose_version(self): + self._check_import() + return self._loose_version + + @property + def version(self): + self._check_import() + return self._version + + def is_v1(self): + self._check_import() + return self._is_v1 + + def is_ge_v21(self): + self._check_import() + return self._is_ge_v21 + + def is_ge_v23(self): + self._check_import() + return self._is_ge_v23 + + def is_ge_v3(self): + self._check_import() + return self._is_ge_v3 + + def is_ge_v3_strict(self): + self._check_import() + return self._is_ge_v3_strict + + def uses_string_dtype(self): + if self.is_ge_v3_strict(): + return True + try: + if self.is_ge_v23() and self.pd.options.future.infer_string: + return True + except: + pass + return False + + @property + def categorical_type(self): + self._check_import() + return self._categorical_type + + @property + def datetimetz_type(self): + self._check_import() + return self._datetimetz_type + + @property + def extension_dtype(self): + self._check_import() + return self._extension_dtype + + cpdef is_array_like(self, obj): + self._check_import() + return isinstance(obj, self._array_like_types) + + cpdef is_categorical(self, obj): + if self._have_pandas_internal(): + return isinstance(obj, self._categorical_type) + else: + return False + + cpdef is_datetimetz(self, obj): + if self._have_pandas_internal(): + return isinstance(obj, self._datetimetz_type) + else: + return False + + cpdef is_extension_array_dtype(self, obj): + self._check_import() + if self._is_extension_array_dtype: + return self._is_extension_array_dtype(obj) + else: + return False + + cpdef is_sparse(self, obj): + if self._have_pandas_internal(): + return isinstance(obj.dtype, self.pd.SparseDtype) + else: + return False + + cpdef is_data_frame(self, obj): + if self._have_pandas_internal(): + return isinstance(obj, self._data_frame) + else: + return False + + cpdef is_series(self, obj): + if self._have_pandas_internal(): + return isinstance(obj, self._series) + else: + return False + + cpdef is_index(self, obj): + if self._have_pandas_internal(): + return isinstance(obj, self._index) + else: + return False + + cpdef get_values(self, obj): + """ + Get the underlying array values of a pandas Series or Index in the + format (np.ndarray or pandas ExtensionArray) as we need them. + + Assumes obj is a pandas Series or Index. + """ + self._check_import() + if isinstance(obj.dtype, (self.pd.api.types.IntervalDtype, + self.pd.api.types.PeriodDtype)): + return obj.array + return obj.values + + def get_rangeindex_attribute(self, level, name): + # public start/stop/step attributes added in pandas 0.25.0 + self._check_import() + if hasattr(level, name): + return getattr(level, name) + return getattr(level, '_' + name) + + +cdef _PandasAPIShim pandas_api = _PandasAPIShim() +_pandas_api = pandas_api diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/pandas_compat.py b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/pandas_compat.py new file mode 100644 index 0000000000000000000000000000000000000000..8480818e26df46d3b85fd1b1548f14f55653b9c6 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/pandas_compat.py @@ -0,0 +1,1295 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import ast +from collections.abc import Sequence +from concurrent import futures +# import threading submodule upfront to avoid partially initialized +# module bug (ARROW-11983) +import concurrent.futures.thread # noqa +from copy import deepcopy +import decimal +from itertools import zip_longest +import json +import operator +import re +import warnings + +try: + import numpy as np +except ImportError: + np = None +import pyarrow as pa +from pyarrow.lib import _pandas_api, frombytes, is_threading_enabled # noqa + + +_logical_type_map = {} +_numpy_logical_type_map = {} +_pandas_logical_type_map = {} + + +def get_logical_type_map(): + global _logical_type_map + + if not _logical_type_map: + _logical_type_map.update({ + pa.lib.Type_NA: 'empty', + pa.lib.Type_BOOL: 'bool', + pa.lib.Type_INT8: 'int8', + pa.lib.Type_INT16: 'int16', + pa.lib.Type_INT32: 'int32', + pa.lib.Type_INT64: 'int64', + pa.lib.Type_UINT8: 'uint8', + pa.lib.Type_UINT16: 'uint16', + pa.lib.Type_UINT32: 'uint32', + pa.lib.Type_UINT64: 'uint64', + pa.lib.Type_HALF_FLOAT: 'float16', + pa.lib.Type_FLOAT: 'float32', + pa.lib.Type_DOUBLE: 'float64', + pa.lib.Type_DATE32: 'date', + pa.lib.Type_DATE64: 'date', + pa.lib.Type_TIME32: 'time', + pa.lib.Type_TIME64: 'time', + pa.lib.Type_BINARY: 'bytes', + pa.lib.Type_FIXED_SIZE_BINARY: 'bytes', + pa.lib.Type_STRING: 'unicode', + }) + return _logical_type_map + + +def get_logical_type(arrow_type): + logical_type_map = get_logical_type_map() + + try: + return logical_type_map[arrow_type.id] + except KeyError: + if isinstance(arrow_type, pa.lib.DictionaryType): + return 'categorical' + elif isinstance(arrow_type, pa.lib.ListType): + return f'list[{get_logical_type(arrow_type.value_type)}]' + elif isinstance(arrow_type, pa.lib.TimestampType): + return 'datetimetz' if arrow_type.tz is not None else 'datetime' + elif pa.types.is_decimal(arrow_type): + return 'decimal' + return 'object' + + +def get_numpy_logical_type_map(): + global _numpy_logical_type_map + if not _numpy_logical_type_map: + _numpy_logical_type_map.update({ + np.bool_: 'bool', + np.int8: 'int8', + np.int16: 'int16', + np.int32: 'int32', + np.int64: 'int64', + np.uint8: 'uint8', + np.uint16: 'uint16', + np.uint32: 'uint32', + np.uint64: 'uint64', + np.float32: 'float32', + np.float64: 'float64', + 'datetime64[D]': 'date', + np.str_: 'string', + np.bytes_: 'bytes', + }) + return _numpy_logical_type_map + + +def get_logical_type_from_numpy(pandas_collection): + numpy_logical_type_map = get_numpy_logical_type_map() + try: + return numpy_logical_type_map[pandas_collection.dtype.type] + except KeyError: + if hasattr(pandas_collection.dtype, 'tz'): + return 'datetimetz' + # See https://github.com/pandas-dev/pandas/issues/24739 (infer_dtype will + # result in "datetime64" without unit, while pandas astype requires a unit) + if str(pandas_collection.dtype).startswith('datetime64'): + return str(pandas_collection.dtype) + result = _pandas_api.infer_dtype(pandas_collection) + if result == 'string': + return 'unicode' + return result + + +def get_extension_dtype_info(column): + dtype = column.dtype + if str(dtype) == 'category': + cats = getattr(column, 'cat', column) + assert cats is not None + metadata = { + 'num_categories': len(cats.categories), + 'ordered': cats.ordered, + } + physical_dtype = str(cats.codes.dtype) + elif hasattr(dtype, 'tz'): + metadata = {'timezone': pa.lib.tzinfo_to_string(dtype.tz)} + physical_dtype = f'datetime64[{dtype.unit}]' + else: + metadata = None + physical_dtype = str(dtype) + return physical_dtype, metadata + + +def get_column_metadata(column, name, arrow_type, field_name): + """Construct the metadata for a given column + + Parameters + ---------- + column : pandas.Series or pandas.Index + name : str + arrow_type : pyarrow.DataType + field_name : str + Equivalent to `name` when `column` is a `Series`, otherwise if `column` + is a pandas Index then `field_name` will not be the same as `name`. + This is the name of the field in the arrow Table's schema. + + Returns + ------- + dict + """ + logical_type = get_logical_type(arrow_type) + + string_dtype, extra_metadata = get_extension_dtype_info(column) + if logical_type == 'decimal': + extra_metadata = { + 'precision': arrow_type.precision, + 'scale': arrow_type.scale, + } + string_dtype = 'object' + + if ( + name is not None + and not (isinstance(name, float) and np.isnan(name)) + and not isinstance(name, str) + ): + raise TypeError( + f"Column name must be a string. Got column {name} of type " + f"{type(name).__name__}" + ) + + assert isinstance(field_name, str), str(type(field_name)) + return { + 'name': name, + 'field_name': field_name, + 'pandas_type': logical_type, + 'numpy_type': string_dtype, + 'metadata': extra_metadata, + } + + +def construct_metadata(columns_to_convert, df, column_names, index_levels, + index_descriptors, preserve_index, types, + column_field_names=None): + """Returns a dictionary containing enough metadata to reconstruct a pandas + DataFrame as an Arrow Table, including index columns. + + Parameters + ---------- + columns_to_convert : list[pd.Series] + df : pandas.DataFrame + column_names : list[str | None] + column_field_names: list[str] + index_levels : List[pd.Index] + index_descriptors : List[Dict] + preserve_index : bool + types : List[pyarrow.DataType] + + Returns + ------- + dict + """ + if column_field_names is None: + # backwards compatibility for external projects that are using + # `construct_metadata` such as cudf + # see https://github.com/apache/arrow/pull/44963#discussion_r1875771953 + column_field_names = [str(name) for name in column_names] + + serialized_index_levels = [ + (level, descriptor) + for level, descriptor in zip(index_levels, index_descriptors) + if not isinstance(descriptor, dict) + ] + + num_serialized_index_levels = len(serialized_index_levels) + + # Use ntypes instead of Python shorthand notation [:-len(x)] as [:-0] + # behaves differently to what we want. + ntypes = len(types) + df_types = types[:ntypes - num_serialized_index_levels] + index_types = types[ntypes - num_serialized_index_levels:] + + column_metadata = [] + for col, name, field_name, arrow_type in zip(columns_to_convert, column_names, + column_field_names, df_types): + metadata = get_column_metadata(col, name=name, + arrow_type=arrow_type, + field_name=field_name) + column_metadata.append(metadata) + + index_column_metadata = [] + if preserve_index is not False: + non_str_index_names = [] + for (level, descriptor), arrow_type in zip( + serialized_index_levels, index_types + ): + if level.name is not None and not isinstance(level.name, str): + non_str_index_names.append(level.name) + + metadata = get_column_metadata( + level, + name=_column_name_to_strings(level.name), + arrow_type=arrow_type, + field_name=descriptor, + ) + index_column_metadata.append(metadata) + + if len(non_str_index_names) > 0: + warnings.warn( + f"The DataFrame has non-str index name `{non_str_index_names}`" + " which will be converted to string" + " and not roundtrip correctly.", + UserWarning, stacklevel=4) + + column_indexes = [] + + levels = getattr(df.columns, 'levels', [df.columns]) + names = getattr(df.columns, 'names', [df.columns.name]) + for level, name in zip(levels, names): + metadata = _get_simple_index_descriptor(level, name) + column_indexes.append(metadata) + else: + index_descriptors = index_column_metadata = column_indexes = [] + + attributes = df.attrs if hasattr(df, "attrs") else {} + + try: + json.dumps(attributes) + except Exception as e: + attributes = {} + warnings.warn( + f"Could not serialize pd.DataFrame.attrs: {e}," + f" defaulting to empty attributes.", + UserWarning, stacklevel=4) + + return { + b'pandas': json.dumps({ + 'index_columns': index_descriptors, + 'column_indexes': column_indexes, + 'columns': column_metadata + index_column_metadata, + 'attributes': attributes, + 'creator': { + 'library': 'pyarrow', + 'version': pa.__version__ + }, + 'pandas_version': _pandas_api.version + }).encode('utf8') + } + + +def _get_simple_index_descriptor(level, name): + string_dtype, extra_metadata = get_extension_dtype_info(level) + pandas_type = get_logical_type_from_numpy(level) + if 'mixed' in pandas_type: + warnings.warn( + "The DataFrame has column names of mixed type. They will be " + "converted to strings and not roundtrip correctly.", + UserWarning, stacklevel=4) + if pandas_type == 'unicode': + assert not extra_metadata + extra_metadata = {'encoding': 'UTF-8'} + return { + 'name': name, + 'field_name': name, + 'pandas_type': pandas_type, + 'numpy_type': string_dtype, + 'metadata': extra_metadata, + } + + +def _column_name_to_strings(name): + """Convert a column name (or level) to either a string or a recursive + collection of strings. + + Parameters + ---------- + name : str or tuple + + Returns + ------- + value : str or tuple + + Examples + -------- + >>> name = 'foo' + >>> _column_name_to_strings(name) + 'foo' + >>> name = ('foo', 'bar') + >>> _column_name_to_strings(name) + "('foo', 'bar')" + >>> import pandas as pd + >>> name = (1, pd.Timestamp('2017-02-01 00:00:00')) + >>> _column_name_to_strings(name) + "('1', '2017-02-01 00:00:00')" + """ + if isinstance(name, str): + return name + elif isinstance(name, bytes): + # XXX: should we assume that bytes in Python 3 are UTF-8? + return name.decode('utf8') + elif isinstance(name, tuple): + return str(tuple(map(_column_name_to_strings, name))) + elif isinstance(name, Sequence): + raise TypeError("Unsupported type for MultiIndex level") + elif name is None or (isinstance(name, float) and np.isnan(name)): + return name + return str(name) + + +def _index_level_name(index, i, column_names): + """Return the name of an index level or a default name if `index.name` is + None or is already a column name. + + Parameters + ---------- + index : pandas.Index + i : int + + Returns + ------- + name : str + """ + if index.name is not None and index.name not in column_names: + return _column_name_to_strings(index.name) + else: + return f'__index_level_{i:d}__' + + +def _get_columns_to_convert(df, schema, preserve_index, columns): + columns = _resolve_columns_of_interest(df, schema, columns) + + if not df.columns.is_unique: + raise ValueError( + f'Duplicate column names found: {list(df.columns)}' + ) + + if schema is not None: + return _get_columns_to_convert_given_schema(df, schema, preserve_index) + + column_names = [] + column_field_names = [] + + index_levels = ( + _get_index_level_values(df.index) if preserve_index is not False + else [] + ) + + columns_to_convert = [] + convert_fields = [] + + for name in columns: + col = df[name] + name = _column_name_to_strings(name) + + if _pandas_api.is_sparse(col): + raise TypeError( + f"Sparse pandas data (column {name}) not supported.") + + columns_to_convert.append(col) + convert_fields.append(None) + column_names.append(name) + column_field_names.append(str(name)) + + index_descriptors = [] + index_column_names = [] + for i, index_level in enumerate(index_levels): + name = _index_level_name(index_level, i, column_names) + if (isinstance(index_level, _pandas_api.pd.RangeIndex) and + preserve_index is None): + descr = _get_range_index_descriptor(index_level) + else: + columns_to_convert.append(index_level) + convert_fields.append(None) + descr = name + index_column_names.append(name) + index_descriptors.append(descr) + + all_names = column_field_names + index_column_names + + # all_names : all of the columns in the resulting table including the data + # columns and serialized index columns + # column_names : the names of the data columns + # index_column_names : the names of the serialized index columns + # index_descriptors : descriptions of each index to be used for + # reconstruction + # index_levels : the extracted index level values + # columns_to_convert : assembled raw data (both data columns and indexes) + # to be converted to Arrow format + # columns_fields : specified column to use for coercion / casting + # during serialization, if a Schema was provided + return (all_names, column_names, column_field_names, index_column_names, + index_descriptors, index_levels, columns_to_convert, convert_fields) + + +def _get_columns_to_convert_given_schema(df, schema, preserve_index): + """ + Specialized version of _get_columns_to_convert in case a Schema is + specified. + In that case, the Schema is used as the single point of truth for the + table structure (types, which columns are included, order of columns, ...). + """ + column_names = [] + columns_to_convert = [] + convert_fields = [] + index_descriptors = [] + index_column_names = [] + index_levels = [] + + for name in schema.names: + try: + col = df[name] + is_index = False + except KeyError: + try: + col = _get_index_level(df, name) + except (KeyError, IndexError): + # name not found as index level + raise KeyError( + f"name '{name}' present in the specified schema is not found " + "in the columns or index") + if preserve_index is False: + raise ValueError( + f"name '{name}' present in the specified schema corresponds " + "to the index, but 'preserve_index=False' was " + "specified") + elif (preserve_index is None and + isinstance(col, _pandas_api.pd.RangeIndex)): + raise ValueError( + f"name '{name}' is present in the schema, but it is a " + "RangeIndex which will not be converted as a column " + "in the Table, but saved as metadata-only not in " + "columns. Specify 'preserve_index=True' to force it " + "being added as a column, or remove it from the " + "specified schema") + is_index = True + + if _pandas_api.is_sparse(col): + raise TypeError( + f"Sparse pandas data (column {name}) not supported.") + + field = schema.field(name) + columns_to_convert.append(col) + convert_fields.append(field) + column_names.append(name) + + if is_index: + index_column_names.append(name) + index_descriptors.append(name) + index_levels.append(col) + + all_names = column_names + index_column_names + + return (all_names, column_names, column_names, index_column_names, + index_descriptors, index_levels, columns_to_convert, convert_fields) + + +def _get_index_level(df, name): + """ + Get the index level of a DataFrame given 'name' (column name in an arrow + Schema). + """ + key = name + if name not in df.index.names and _is_generated_index_name(name): + # we know we have an autogenerated name => extract number and get + # the index level positionally + key = int(name[len("__index_level_"):-2]) + return df.index.get_level_values(key) + + +def _level_name(name): + # preserve type when default serializable, otherwise str it + try: + json.dumps(name) + return name + except TypeError: + return str(name) + + +def _get_range_index_descriptor(level): + # public start/stop/step attributes added in pandas 0.25.0 + return { + 'kind': 'range', + 'name': _level_name(level.name), + 'start': _pandas_api.get_rangeindex_attribute(level, 'start'), + 'stop': _pandas_api.get_rangeindex_attribute(level, 'stop'), + 'step': _pandas_api.get_rangeindex_attribute(level, 'step') + } + + +def _get_index_level_values(index): + n = len(getattr(index, 'levels', [index])) + return [index.get_level_values(i) for i in range(n)] + + +def _resolve_columns_of_interest(df, schema, columns): + if schema is not None and columns is not None: + raise ValueError('Schema and columns arguments are mutually ' + 'exclusive, pass only one of them') + elif schema is not None: + columns = schema.names + elif columns is not None: + columns = [c for c in columns if c in df.columns] + else: + columns = df.columns + + return columns + + +def dataframe_to_types(df, preserve_index, columns=None): + (all_names, + column_names, + column_field_names, + _, + index_descriptors, + index_columns, + columns_to_convert, + _) = _get_columns_to_convert(df, None, preserve_index, columns) + + types = [] + # If pandas knows type, skip conversion + for c in columns_to_convert: + values = c.values + if _pandas_api.is_categorical(values): + type_ = pa.array(c, from_pandas=True).type + elif _pandas_api.is_extension_array_dtype(values): + empty = c.head(0) if isinstance( + c, _pandas_api.pd.Series) else c[:0] + type_ = pa.array(empty, from_pandas=True).type + else: + values, type_ = get_datetimetz_type(values, c.dtype, None) + type_ = pa.lib._ndarray_to_arrow_type(values, type_) + if type_ is None: + type_ = pa.array(c, from_pandas=True).type + types.append(type_) + + metadata = construct_metadata( + columns_to_convert, df, column_names, index_columns, index_descriptors, + preserve_index, types, column_field_names=column_field_names + ) + + return all_names, types, metadata + + +def dataframe_to_arrays(df, schema, preserve_index, nthreads=1, columns=None, + safe=True): + (all_names, + column_names, + column_field_names, + index_column_names, + index_descriptors, + index_columns, + columns_to_convert, + convert_fields) = _get_columns_to_convert(df, schema, preserve_index, + columns) + + # NOTE(wesm): If nthreads=None, then we use a heuristic to decide whether + # using a thread pool is worth it. Currently the heuristic is whether the + # nrows > 100 * ncols and ncols > 1. + if nthreads is None: + nrows, ncols = len(df), len(df.columns) + if nrows > ncols * 100 and ncols > 1: + nthreads = pa.cpu_count() + else: + nthreads = 1 + # if we don't have threading in libarrow, don't use threading here either + if not is_threading_enabled(): + nthreads = 1 + + def convert_column(col, field): + if field is None: + field_nullable = True + type_ = None + else: + field_nullable = field.nullable + type_ = field.type + + try: + result = pa.array(col, type=type_, from_pandas=True, safe=safe) + except (pa.ArrowInvalid, + pa.ArrowNotImplementedError, + pa.ArrowTypeError) as e: + e.args += ( + f"Conversion failed for column {col.name} with type {col.dtype}",) + raise e + if not field_nullable and result.null_count > 0: + raise ValueError(f"Field {field} was non-nullable but pandas column " + f"had {result.null_count} null values") + return result + + def _can_definitely_zero_copy(arr): + return (isinstance(arr, np.ndarray) and + arr.flags.contiguous and + issubclass(arr.dtype.type, np.integer)) + + if nthreads == 1: + arrays = [convert_column(c, f) + for c, f in zip(columns_to_convert, convert_fields)] + else: + arrays = [] + with futures.ThreadPoolExecutor(nthreads) as executor: + for c, f in zip(columns_to_convert, convert_fields): + if _can_definitely_zero_copy(c.values): + arrays.append(convert_column(c, f)) + else: + arrays.append(executor.submit(convert_column, c, f)) + + for i, maybe_fut in enumerate(arrays): + if isinstance(maybe_fut, futures.Future): + arrays[i] = maybe_fut.result() + + types = [x.type for x in arrays] + + if schema is None: + fields = [] + for name, type_ in zip(all_names, types): + fields.append(pa.field(name, type_)) + schema = pa.schema(fields) + + pandas_metadata = construct_metadata( + columns_to_convert, df, column_names, index_columns, index_descriptors, + preserve_index, types, column_field_names=column_field_names + ) + metadata = deepcopy(schema.metadata) if schema.metadata else dict() + metadata.update(pandas_metadata) + schema = schema.with_metadata(metadata) + + # If dataframe is empty but with RangeIndex -> + # remember the length of the indexes + n_rows = None + if len(arrays) == 0: + try: + kind = index_descriptors[0]["kind"] + if kind == "range": + start = index_descriptors[0]["start"] + stop = index_descriptors[0]["stop"] + step = index_descriptors[0]["step"] + n_rows = len(range(start, stop, step)) + except IndexError: + pass + + return arrays, schema, n_rows + + +def get_datetimetz_type(values, dtype, type_): + if values.dtype.type != np.datetime64: + return values, type_ + + if _pandas_api.is_datetimetz(dtype) and type_ is None: + # If no user type passed, construct a tz-aware timestamp type + tz = dtype.tz + unit = dtype.unit + type_ = pa.timestamp(unit, tz) + elif type_ is None: + # Trust the NumPy dtype + type_ = pa.from_numpy_dtype(values.dtype) + + return values, type_ + +# ---------------------------------------------------------------------- +# Converting pyarrow.Table efficiently to pandas.DataFrame + + +def _reconstruct_block(item, columns=None, extension_columns=None, return_block=True): + """ + Construct a pandas Block from the `item` dictionary coming from pyarrow's + serialization or returned by arrow::python::ConvertTableToPandas. + + This function takes care of converting dictionary types to pandas + categorical, Timestamp-with-timezones to the proper pandas Block, and + conversion to pandas ExtensionBlock + + Parameters + ---------- + item : dict + For basic types, this is a dictionary in the form of + {'block': np.ndarray of values, 'placement': pandas block placement}. + Additional keys are present for other types (dictionary, timezone, + object). + columns : + Column names of the table being constructed, used for extension types + extension_columns : dict + Dictionary of {column_name: pandas_dtype} that includes all columns + and corresponding dtypes that will be converted to a pandas + ExtensionBlock. + + Returns + ------- + pandas Block + + """ + import pandas.core.internals as _int + + block_arr = item.get('block', None) + placement = item['placement'] + if 'dictionary' in item: + arr = _pandas_api.categorical_type.from_codes( + block_arr, categories=item['dictionary'], + ordered=item['ordered']) + elif 'timezone' in item: + unit, _ = np.datetime_data(block_arr.dtype) + dtype = make_datetimetz(unit, item['timezone']) + if _pandas_api.is_ge_v21(): + arr = _pandas_api.pd.array( + block_arr.view("int64"), dtype=dtype, copy=False + ) + else: + arr = block_arr + if return_block: + block = _int.make_block(block_arr, placement=placement, + klass=_int.DatetimeTZBlock, + dtype=dtype) + return block + elif 'py_array' in item: + # create ExtensionBlock + arr = item['py_array'] + assert len(placement) == 1 + name = columns[placement[0]] + pandas_dtype = extension_columns[name] + if not hasattr(pandas_dtype, '__from_arrow__'): + raise ValueError("This column does not support to be converted " + "to a pandas ExtensionArray") + arr = pandas_dtype.__from_arrow__(arr) + else: + arr = block_arr + + if return_block: + return _int.make_block(arr, placement=placement) + else: + return arr, placement + + +def make_datetimetz(unit, tz): + if _pandas_api.is_v1(): + unit = 'ns' # ARROW-3789: Coerce date/timestamp types to datetime64[ns] + tz = pa.lib.string_to_tzinfo(tz) + return _pandas_api.datetimetz_type(unit, tz=tz) + + +def table_to_dataframe( + options, table, categories=None, ignore_metadata=False, types_mapper=None +): + all_columns = [] + column_indexes = [] + attributes = {} + pandas_metadata = table.schema.pandas_metadata + + if not ignore_metadata and pandas_metadata is not None: + all_columns = pandas_metadata['columns'] + column_indexes = pandas_metadata.get('column_indexes', []) + attributes = pandas_metadata.get('attributes', {}) + index_descriptors = pandas_metadata['index_columns'] + table = _add_any_metadata(table, pandas_metadata) + table, index = _reconstruct_index(table, index_descriptors, + all_columns, types_mapper) + ext_columns_dtypes = _get_extension_dtypes( + table, all_columns, types_mapper, options, categories) + else: + index = _pandas_api.pd.RangeIndex(table.num_rows) + ext_columns_dtypes = _get_extension_dtypes( + table, [], types_mapper, options, categories + ) + + _check_data_column_metadata_consistency(all_columns) + columns = _deserialize_column_index(table, all_columns, column_indexes) + + column_names = table.column_names + result = pa.lib.table_to_blocks(options, table, categories, + list(ext_columns_dtypes.keys())) + if _pandas_api.is_ge_v3(): + from pandas.api.internals import create_dataframe_from_blocks + + blocks = [ + _reconstruct_block( + item, column_names, ext_columns_dtypes, return_block=False) + for item in result + ] + df = create_dataframe_from_blocks(blocks, index=index, columns=columns) + df.attrs = attributes + + return df + else: + from pandas.core.internals import BlockManager + from pandas import DataFrame + + blocks = [ + _reconstruct_block(item, column_names, ext_columns_dtypes) + for item in result + ] + axes = [columns, index] + mgr = BlockManager(blocks, axes) + if _pandas_api.is_ge_v21(): + df = DataFrame._from_mgr(mgr, mgr.axes) + else: + df = DataFrame(mgr) + + df.attrs = attributes + + return df + + +# Set of the string repr of all numpy dtypes that can be stored in a pandas +# dataframe (complex not included since not supported by Arrow) +_pandas_supported_numpy_types = { + "int8", "int16", "int32", "int64", + "uint8", "uint16", "uint32", "uint64", + "float16", "float32", "float64", + "object", "bool" +} + + +def _get_extension_dtypes(table, columns_metadata, types_mapper, options, categories): + """ + Based on the stored column pandas metadata and the extension types + in the arrow schema, infer which columns should be converted to a + pandas extension dtype. + + The 'numpy_type' field in the column metadata stores the string + representation of the original pandas dtype (and, despite its name, + not the 'pandas_type' field). + Based on this string representation, a pandas/numpy dtype is constructed + and then we can check if this dtype supports conversion from arrow. + + """ + strings_to_categorical = options["strings_to_categorical"] + categories = categories or [] + + ext_columns = {} + + # older pandas version that does not yet support extension dtypes + if _pandas_api.extension_dtype is None: + return ext_columns + + # use the specified mapping of built-in arrow types to pandas dtypes + if types_mapper: + for field in table.schema: + typ = field.type + pandas_dtype = types_mapper(typ) + if pandas_dtype is not None: + ext_columns[field.name] = pandas_dtype + + # infer from extension type in the schema + for field in table.schema: + typ = field.type + if field.name not in ext_columns and isinstance(typ, pa.BaseExtensionType): + try: + pandas_dtype = typ.to_pandas_dtype() + except NotImplementedError: + pass + else: + ext_columns[field.name] = pandas_dtype + + # infer the extension columns from the pandas metadata + for col_meta in columns_metadata: + try: + name = col_meta['field_name'] + except KeyError: + name = col_meta['name'] + dtype = col_meta['numpy_type'] + + if name not in ext_columns and dtype not in _pandas_supported_numpy_types: + # pandas_dtype is expensive, so avoid doing this for types + # that are certainly numpy dtypes + pandas_dtype = _pandas_api.pandas_dtype(dtype) + if isinstance(pandas_dtype, _pandas_api.extension_dtype): + if isinstance(pandas_dtype, _pandas_api.pd.StringDtype): + # when the metadata indicate to use the string dtype, + # ignore this in case: + # - it is specified to convert strings / this column to categorical + # - the column itself is dictionary encoded and would otherwise be + # converted to categorical + if strings_to_categorical or name in categories: + continue + try: + if pa.types.is_dictionary(table.schema.field(name).type): + continue + except KeyError: + pass + if hasattr(pandas_dtype, "__from_arrow__"): + ext_columns[name] = pandas_dtype + + # for pandas 3.0+, use pandas' new default string dtype + if _pandas_api.uses_string_dtype() and not strings_to_categorical: + for field in table.schema: + if field.name not in ext_columns and ( + pa.types.is_string(field.type) + or pa.types.is_large_string(field.type) + or pa.types.is_string_view(field.type) + ) and field.name not in categories: + ext_columns[field.name] = _pandas_api.pd.StringDtype(na_value=np.nan) + + return ext_columns + + +def _check_data_column_metadata_consistency(all_columns): + # It can never be the case in a released version of pyarrow that + # c['name'] is None *and* 'field_name' is not a key in the column metadata, + # because the change to allow c['name'] to be None and the change to add + # 'field_name' are in the same release (0.8.0) + assert all( + (c['name'] is None and 'field_name' in c) or c['name'] is not None + for c in all_columns + ) + + +def _deserialize_column_index(block_table, all_columns, column_indexes): + if all_columns: + columns_name_dict = { + c.get('field_name', _column_name_to_strings(c['name'])): c['name'] + for c in all_columns + } + columns_values = [ + columns_name_dict.get(name, name) for name in block_table.column_names + ] + else: + columns_values = block_table.column_names + + # Construct the base index + if len(column_indexes) > 1: + # If we're passed multiple column indexes then evaluate with + # ast.literal_eval, since the column index values show up as a list of + # tuples + columns = _pandas_api.pd.MultiIndex.from_tuples( + list(map(ast.literal_eval, columns_values)), + names=[col_index['name'] for col_index in column_indexes], + ) + else: + columns = _pandas_api.pd.Index( + columns_values, name=column_indexes[0]["name"] if column_indexes else None + ) + + # if we're reconstructing the index + if len(column_indexes) > 0: + columns = _reconstruct_columns_from_metadata(columns, column_indexes) + + return columns + + +def _reconstruct_index(table, index_descriptors, all_columns, types_mapper=None): + # 0. 'field_name' is the name of the column in the arrow Table + # 1. 'name' is the user-facing name of the column, that is, it came from + # pandas + # 2. 'field_name' and 'name' differ for index columns + # 3. We fall back on c['name'] for backwards compatibility + field_name_to_metadata = { + c.get('field_name', c['name']): c + for c in all_columns + } + + # Build up a list of index columns and names while removing those columns + # from the original table + index_arrays = [] + index_names = [] + result_table = table + for descr in index_descriptors: + if isinstance(descr, str): + result_table, index_level, index_name = _extract_index_level( + table, result_table, descr, field_name_to_metadata, types_mapper) + if index_level is None: + # ARROW-1883: the serialized index column was not found + continue + elif descr['kind'] == 'range': + index_name = descr['name'] + index_level = _pandas_api.pd.RangeIndex(descr['start'], + descr['stop'], + step=descr['step'], + name=index_name) + if len(index_level) != len(table): + # Possibly the result of munged metadata + continue + else: + raise ValueError(f"Unrecognized index kind: {descr['kind']}") + index_arrays.append(index_level) + index_names.append(index_name) + + pd = _pandas_api.pd + + # Reconstruct the row index + if len(index_arrays) > 1: + index = pd.MultiIndex.from_arrays(index_arrays, names=index_names) + elif len(index_arrays) == 1: + index = index_arrays[0] + if not isinstance(index, pd.Index): + # Box anything that wasn't boxed above + index = pd.Index(index, name=index_names[0]) + else: + index = pd.RangeIndex(table.num_rows) + + return result_table, index + + +def _extract_index_level(table, result_table, field_name, + field_name_to_metadata, types_mapper=None): + logical_name = field_name_to_metadata[field_name]['name'] + index_name = _backwards_compatible_index_name(field_name, logical_name) + i = table.schema.get_field_index(field_name) + + if i == -1: + # The serialized index column was removed by the user + return result_table, None, None + + col = table.column(i) + index_level = col.to_pandas(types_mapper=types_mapper) + index_level.name = None + result_table = result_table.remove_column( + result_table.schema.get_field_index(field_name) + ) + return result_table, index_level, index_name + + +def _backwards_compatible_index_name(raw_name, logical_name): + """Compute the name of an index column that is compatible with older + versions of :mod:`pyarrow`. + + Parameters + ---------- + raw_name : str + logical_name : str + + Returns + ------- + result : str + + Notes + ----- + * Part of :func:`~pyarrow.pandas_compat.table_to_blockmanager` + """ + # Part of table_to_blockmanager + if raw_name == logical_name and _is_generated_index_name(raw_name): + return None + else: + return logical_name + + +def _is_generated_index_name(name): + pattern = r'^__index_level_\d+__$' + return re.match(pattern, name) is not None + + +def get_pandas_logical_type_map(): + global _pandas_logical_type_map + + if not _pandas_logical_type_map: + _pandas_logical_type_map.update({ + 'date': 'datetime64[D]', + 'datetime': 'datetime64[ns]', + 'datetimetz': 'datetime64[ns]', + 'unicode': 'str', + 'bytes': np.bytes_, + 'string': 'str', + 'integer': np.int64, + 'floating': np.float64, + 'decimal': np.object_, + 'empty': np.object_, + }) + return _pandas_logical_type_map + + +def _pandas_type_to_numpy_type(pandas_type): + """Get the numpy dtype that corresponds to a pandas type. + + Parameters + ---------- + pandas_type : str + The result of a call to pandas.lib.infer_dtype. + + Returns + ------- + dtype : np.dtype + The dtype that corresponds to `pandas_type`. + """ + pandas_logical_type_map = get_pandas_logical_type_map() + try: + return pandas_logical_type_map[pandas_type] + except KeyError: + if 'mixed' in pandas_type: + # catching 'mixed', 'mixed-integer' and 'mixed-integer-float' + return np.object_ + return np.dtype(pandas_type) + + +def _reconstruct_columns_from_metadata(columns, column_indexes): + """Construct a pandas MultiIndex from `columns` and column index metadata + in `column_indexes`. + + Parameters + ---------- + columns : List[pd.Index] + The columns coming from a pyarrow.Table + column_indexes : List[Dict[str, str]] + The column index metadata deserialized from the JSON schema metadata + in a :class:`~pyarrow.Table`. + + Returns + ------- + result : MultiIndex + The index reconstructed using `column_indexes` metadata with levels of + the correct type. + + Notes + ----- + * Part of :func:`~pyarrow.pandas_compat.table_to_blockmanager` + """ + pd = _pandas_api.pd + # Get levels and labels, and provide sane defaults if the index has a + # single level to avoid if/else spaghetti. + levels = getattr(columns, 'levels', None) or [columns] + labels = getattr(columns, 'codes', None) or [None] + + # Convert each level to the dtype provided in the metadata + levels_dtypes = [ + (level, col_index.get('pandas_type', str(level.dtype)), + col_index.get('numpy_type', None)) + for level, col_index in zip_longest( + levels, column_indexes, fillvalue={} + ) + ] + + new_levels = [] + encoder = operator.methodcaller('encode', 'UTF-8') + + for level, pandas_dtype, numpy_dtype in levels_dtypes: + dtype = _pandas_type_to_numpy_type(pandas_dtype) + # Since our metadata is UTF-8 encoded, Python turns things that were + # bytes into unicode strings when json.loads-ing them. We need to + # convert them back to bytes to preserve metadata. + if dtype == np.bytes_: + level = level.map(encoder) + # ARROW-13756: if index is timezone aware DataTimeIndex + elif pandas_dtype == "datetimetz": + tz = pa.lib.string_to_tzinfo( + column_indexes[0]['metadata']['timezone']) + level = pd.to_datetime(level, utc=True).tz_convert(tz) + if _pandas_api.is_ge_v3(): + # with pandas 3+, to_datetime returns a unit depending on the string + # data, so we restore it to the original unit from the metadata + level = level.as_unit(np.datetime_data(numpy_dtype)[0]) + # GH-41503: if the column index was decimal, restore to decimal + elif pandas_dtype == "decimal": + level = _pandas_api.pd.Index([decimal.Decimal(i) for i in level]) + elif ( + level.dtype == "str" and numpy_dtype == "object" + and ("mixed" in pandas_dtype or pandas_dtype in ["unicode", "string"]) + ): + # the metadata indicate that the original dataframe used object dtype, + # but ignore this and keep string dtype if: + # - the original columns used mixed types -> we don't attempt to faithfully + # roundtrip in this case, but keep the column names as strings + # - the original columns were inferred to be strings but stored in object + # dtype -> we don't restore the object dtype because all metadata + # generated using pandas < 3 will have this case by default, and + # for pandas >= 3 we want to use the default string dtype for .columns + new_levels.append(level) + continue + elif level.dtype != dtype: + level = level.astype(dtype) + # ARROW-9096: if original DataFrame was upcast we keep that + if level.dtype != numpy_dtype and pandas_dtype != "datetimetz": + level = level.astype(numpy_dtype) + + new_levels.append(level) + + if len(new_levels) > 1: + return pd.MultiIndex(new_levels, labels, names=columns.names) + else: + return pd.Index(new_levels[0], dtype=new_levels[0].dtype, name=columns.name) + + +def _add_any_metadata(table, pandas_metadata): + modified_columns = {} + modified_fields = {} + + schema = table.schema + + index_columns = pandas_metadata['index_columns'] + # only take index columns into account if they are an actual table column + index_columns = [idx_col for idx_col in index_columns + if isinstance(idx_col, str)] + n_index_levels = len(index_columns) + n_columns = len(pandas_metadata['columns']) - n_index_levels + + # Add time zones + for i, col_meta in enumerate(pandas_metadata['columns']): + + raw_name = col_meta.get('field_name') + if not raw_name: + # deal with metadata written with arrow < 0.8 or fastparquet + raw_name = col_meta['name'] + if i >= n_columns: + # index columns + raw_name = index_columns[i - n_columns] + if raw_name is None: + raw_name = 'None' + + idx = schema.get_field_index(raw_name) + if idx != -1: + if col_meta['pandas_type'] == 'datetimetz': + col = table[idx] + if not isinstance(col.type, pa.lib.TimestampType): + continue + metadata = col_meta['metadata'] + if not metadata: + continue + metadata_tz = metadata.get('timezone') + if metadata_tz and metadata_tz != col.type.tz: + converted = col.to_pandas() + tz_aware_type = pa.timestamp('ns', tz=metadata_tz) + with_metadata = pa.Array.from_pandas(converted, + type=tz_aware_type) + + modified_fields[idx] = pa.field(schema[idx].name, + tz_aware_type) + modified_columns[idx] = with_metadata + + if len(modified_columns) > 0: + columns = [] + fields = [] + for i in range(len(table.schema)): + if i in modified_columns: + columns.append(modified_columns[i]) + fields.append(modified_fields[i]) + else: + columns.append(table[i]) + fields.append(table.schema[i]) + return pa.Table.from_arrays(columns, schema=pa.schema(fields)) + else: + return table + + +# ---------------------------------------------------------------------- +# Helper functions used in lib + + +def make_tz_aware(series, tz): + """ + Make a datetime64 Series timezone-aware for the given tz + """ + tz = pa.lib.string_to_tzinfo(tz) + series = (series.dt.tz_localize('utc') + .dt.tz_convert(tz)) + return series diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/parquet.dll b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/parquet.dll new file mode 100644 index 0000000000000000000000000000000000000000..2ee482353751440aa7a2ec52fbb19c018d4ce001 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/parquet.dll @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5ee209147e56d6dd6be8c6a99b2040261750dbc56801bf9857c80f8c176d8c3 +size 6891520 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/parquet.lib b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/parquet.lib new file mode 100644 index 0000000000000000000000000000000000000000..c72a57322594729febb5619c8e0eac347ac23ce2 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/parquet.lib @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:237f58cad63e959a82be9fdd55c815abcd8965b510799f6273305467f9f432b6 +size 863334 diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/parquet/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/parquet/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f6c10cef44400e329fb0e457dc9c63afd54db6ea --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/parquet/__init__.py @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# flake8: noqa + +from .core import * diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/parquet/core.py b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/parquet/core.py new file mode 100644 index 0000000000000000000000000000000000000000..0c9d7ec09ff24f7ab83038770fefa78c485b9604 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/parquet/core.py @@ -0,0 +1,2489 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +from collections import defaultdict +from contextlib import nullcontext +from functools import reduce + +import inspect +import json +import os +import re +import operator + +import pyarrow as pa + +try: + import pyarrow._parquet as _parquet +except ImportError as exc: + raise ImportError( + "The pyarrow installation is not built with support " + f"for the Parquet file format ({str(exc)})" + ) from None + +from pyarrow._parquet import (ParquetReader, Statistics, # noqa + FileMetaData, RowGroupMetaData, + ColumnChunkMetaData, + ParquetSchema, ColumnSchema, + ParquetLogicalType, + FileEncryptionProperties, + FileDecryptionProperties, + SortingColumn) +from pyarrow.fs import (LocalFileSystem, FileType, _resolve_filesystem_and_path, + _ensure_filesystem) +from pyarrow.util import guid, _is_path_like, _stringify_path, _deprecate_api + + +def _check_contains_null(val): + if isinstance(val, bytes): + for byte in val: + if isinstance(byte, bytes): + compare_to = chr(0) + else: + compare_to = 0 + if byte == compare_to: + return True + elif isinstance(val, str): + return '\x00' in val + return False + + +def _check_filters(filters, check_null_strings=True): + """ + Check if filters are well-formed. + """ + if filters is not None: + if len(filters) == 0 or any(len(f) == 0 for f in filters): + raise ValueError("Malformed filters") + if isinstance(filters[0][0], str): + # We have encountered the situation where we have one nesting level + # too few: + # We have [(,,), ..] instead of [[(,,), ..]] + filters = [filters] + if check_null_strings: + for conjunction in filters: + for col, op, val in conjunction: + if ( + isinstance(val, list) and + all(_check_contains_null(v) for v in val) or + _check_contains_null(val) + ): + raise NotImplementedError( + "Null-terminated binary strings are not supported " + "as filter values." + ) + return filters + + +_DNF_filter_doc = """Predicates are expressed using an ``Expression`` or using + the disjunctive normal form (DNF), like ``[[('x', '=', 0), ...], ...]``. + DNF allows arbitrary boolean logical combinations of single column predicates. + The innermost tuples each describe a single column predicate. The list of inner + predicates is interpreted as a conjunction (AND), forming a more selective and + multiple column predicate. Finally, the most outer list combines these filters + as a disjunction (OR). + + Predicates may also be passed as List[Tuple]. This form is interpreted + as a single conjunction. To express OR in predicates, one must + use the (preferred) List[List[Tuple]] notation. + + Each tuple has format: (``key``, ``op``, ``value``) and compares the + ``key`` with the ``value``. + The supported ``op`` are: ``=`` or ``==``, ``!=``, ``<``, ``>``, ``<=``, + ``>=``, ``in`` and ``not in``. If the ``op`` is ``in`` or ``not in``, the + ``value`` must be a collection such as a ``list``, a ``set`` or a + ``tuple``. + + Examples: + + Using the ``Expression`` API: + + .. code-block:: python + + import pyarrow.compute as pc + pc.field('x') = 0 + pc.field('y').isin(['a', 'b', 'c']) + ~pc.field('y').isin({'a', 'b'}) + + Using the DNF format: + + .. code-block:: python + + ('x', '=', 0) + ('y', 'in', ['a', 'b', 'c']) + ('z', 'not in', {'a','b'}) + + """ + + +def filters_to_expression(filters): + """ + Check if filters are well-formed and convert to an ``Expression``. + + Parameters + ---------- + filters : List[Tuple] or List[List[Tuple]] + + Notes + ----- + See internal ``pyarrow._DNF_filter_doc`` attribute for more details. + + Examples + -------- + + >>> filters_to_expression([('foo', '==', 'bar')]) + + + Returns + ------- + pyarrow.compute.Expression + An Expression representing the filters + """ + import pyarrow.dataset as ds + + if isinstance(filters, ds.Expression): + return filters + + filters = _check_filters(filters, check_null_strings=False) + + def convert_single_predicate(col, op, val): + field = ds.field(col) + + if op == "=" or op == "==": + return field == val + elif op == "!=": + return field != val + elif op == '<': + return field < val + elif op == '>': + return field > val + elif op == '<=': + return field <= val + elif op == '>=': + return field >= val + elif op == 'in': + return field.isin(val) + elif op == 'not in': + return ~field.isin(val) + else: + raise ValueError(f'"{col}" is not a valid operator in predicates.') + + disjunction_members = [] + + for conjunction in filters: + conjunction_members = [ + convert_single_predicate(col, op, val) + for col, op, val in conjunction + ] + + disjunction_members.append(reduce(operator.and_, conjunction_members)) + + return reduce(operator.or_, disjunction_members) + + +_filters_to_expression = _deprecate_api( + "_filters_to_expression", "filters_to_expression", + filters_to_expression, "10.0.0", DeprecationWarning) + + +# ---------------------------------------------------------------------- +# Reading a single Parquet file + + +class ParquetFile: + """ + Reader interface for a single Parquet file. + + Parameters + ---------- + source : str, pathlib.Path, pyarrow.NativeFile, or file-like object + Readable source. For passing bytes or buffer-like file containing a + Parquet file, use pyarrow.BufferReader. + metadata : FileMetaData, default None + Use existing metadata object, rather than reading from file. + common_metadata : FileMetaData, default None + Will be used in reads for pandas schema metadata if not found in the + main file's metadata, no other uses at the moment. + read_dictionary : list + List of column names to read directly as DictionaryArray. + binary_type : pyarrow.DataType, default None + If given, Parquet binary columns will be read as this datatype. + This setting is ignored if a serialized Arrow schema is found in + the Parquet metadata. + list_type : subclass of pyarrow.DataType, default None + If given, non-MAP repeated columns will be read as an instance of + this datatype (either pyarrow.ListType or pyarrow.LargeListType). + This setting is ignored if a serialized Arrow schema is found in + the Parquet metadata. + memory_map : bool, default False + If the source is a file path, use a memory map to read file, which can + improve performance in some environments. + buffer_size : int, default 0 + If positive, perform read buffering when deserializing individual + column chunks. Otherwise IO calls are unbuffered. + pre_buffer : bool, default False + Coalesce and issue file reads in parallel to improve performance on + high-latency filesystems (e.g. S3). If True, Arrow will use a + background I/O thread pool. + coerce_int96_timestamp_unit : str, default None + Cast timestamps that are stored in INT96 format to a particular + resolution (e.g. 'ms'). Setting to None is equivalent to 'ns' + and therefore INT96 timestamps will be inferred as timestamps + in nanoseconds. + decryption_properties : FileDecryptionProperties, default None + File decryption properties for Parquet Modular Encryption. + thrift_string_size_limit : int, default None + If not None, override the maximum total string size allocated + when decoding Thrift structures. The default limit should be + sufficient for most Parquet files. + thrift_container_size_limit : int, default None + If not None, override the maximum total size of containers allocated + when decoding Thrift structures. The default limit should be + sufficient for most Parquet files. + filesystem : FileSystem, default None + If nothing passed, will be inferred based on path. + Path will try to be found in the local on-disk filesystem otherwise + it will be parsed as an URI to determine the filesystem. + page_checksum_verification : bool, default False + If True, verify the checksum for each page read from the file. + arrow_extensions_enabled : bool, default True + If True, read Parquet logical types as Arrow extension types where possible, + (e.g., read JSON as the canonical `arrow.json` extension type or UUID as + the canonical `arrow.uuid` extension type). + + Examples + -------- + + Generate an example PyArrow Table and write it to Parquet file: + + >>> import pyarrow as pa + >>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], + ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", + ... "Brittle stars", "Centipede"]}) + + >>> import pyarrow.parquet as pq + >>> pq.write_table(table, 'example.parquet') + + Create a ``ParquetFile`` object from the Parquet file: + + >>> parquet_file = pq.ParquetFile('example.parquet') + + Read the data: + + >>> parquet_file.read() + pyarrow.Table + n_legs: int64 + animal: string + ---- + n_legs: [[2,2,4,4,5,100]] + animal: [["Flamingo","Parrot","Dog","Horse","Brittle stars","Centipede"]] + + Create a ParquetFile object with "animal" column as DictionaryArray: + + >>> parquet_file = pq.ParquetFile('example.parquet', + ... read_dictionary=["animal"]) + >>> parquet_file.read() + pyarrow.Table + n_legs: int64 + animal: dictionary + ---- + n_legs: [[2,2,4,4,5,100]] + animal: [ -- dictionary: + ["Flamingo","Parrot",...,"Brittle stars","Centipede"] -- indices: + [0,1,2,3,4,5]] + """ + + def __init__(self, source, *, metadata=None, common_metadata=None, + read_dictionary=None, binary_type=None, list_type=None, + memory_map=False, buffer_size=0, pre_buffer=False, + coerce_int96_timestamp_unit=None, + decryption_properties=None, thrift_string_size_limit=None, + thrift_container_size_limit=None, filesystem=None, + page_checksum_verification=False, arrow_extensions_enabled=True): + + self._close_source = getattr(source, 'closed', True) + + filesystem, source = _resolve_filesystem_and_path( + source, filesystem, memory_map=memory_map) + if filesystem is not None: + source = filesystem.open_input_file(source) + self._close_source = True # We opened it here, ensure we close it. + + self.reader = ParquetReader() + self.reader.open( + source, use_memory_map=memory_map, + buffer_size=buffer_size, pre_buffer=pre_buffer, + read_dictionary=read_dictionary, metadata=metadata, + binary_type=binary_type, list_type=list_type, + coerce_int96_timestamp_unit=coerce_int96_timestamp_unit, + decryption_properties=decryption_properties, + thrift_string_size_limit=thrift_string_size_limit, + thrift_container_size_limit=thrift_container_size_limit, + page_checksum_verification=page_checksum_verification, + arrow_extensions_enabled=arrow_extensions_enabled, + ) + self.common_metadata = common_metadata + self._nested_paths_by_prefix = self._build_nested_paths() + + def __enter__(self): + return self + + def __exit__(self, *args, **kwargs): + self.close() + + def _build_nested_paths(self): + paths = self.reader.column_paths + + result = defaultdict(list) + + for i, path in enumerate(paths): + key = path[0] + rest = path[1:] + while True: + result[key].append(i) + + if not rest: + break + + key = '.'.join((key, rest[0])) + rest = rest[1:] + + return result + + @property + def metadata(self): + """ + Return the Parquet metadata. + """ + return self.reader.metadata + + @property + def schema(self): + """ + Return the Parquet schema, unconverted to Arrow types + """ + return self.metadata.schema + + @property + def schema_arrow(self): + """ + Return the inferred Arrow schema, converted from the whole Parquet + file's schema + + Examples + -------- + Generate an example Parquet file: + + >>> import pyarrow as pa + >>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], + ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", + ... "Brittle stars", "Centipede"]}) + >>> import pyarrow.parquet as pq + >>> pq.write_table(table, 'example.parquet') + >>> parquet_file = pq.ParquetFile('example.parquet') + + Read the Arrow schema: + + >>> parquet_file.schema_arrow + n_legs: int64 + animal: string + """ + return self.reader.schema_arrow + + @property + def num_row_groups(self): + """ + Return the number of row groups of the Parquet file. + + Examples + -------- + >>> import pyarrow as pa + >>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], + ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", + ... "Brittle stars", "Centipede"]}) + >>> import pyarrow.parquet as pq + >>> pq.write_table(table, 'example.parquet') + >>> parquet_file = pq.ParquetFile('example.parquet') + + >>> parquet_file.num_row_groups + 1 + """ + return self.reader.num_row_groups + + def close(self, force: bool = False): + if self._close_source or force: + self.reader.close() + + @property + def closed(self) -> bool: + return self.reader.closed + + def read_row_group(self, i, columns=None, use_threads=True, + use_pandas_metadata=False): + """ + Read a single row group from a Parquet file. + + Parameters + ---------- + i : int + Index of the individual row group that we want to read. + columns : list + If not None, only these columns will be read from the row group. A + column name may be a prefix of a nested field, e.g. 'a' will select + 'a.b', 'a.c', and 'a.d.e'. + use_threads : bool, default True + Perform multi-threaded column reads. + use_pandas_metadata : bool, default False + If True and file has custom pandas schema metadata, ensure that + index columns are also loaded. + + Returns + ------- + pyarrow.table.Table + Content of the row group as a table (of columns) + + Examples + -------- + >>> import pyarrow as pa + >>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], + ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", + ... "Brittle stars", "Centipede"]}) + >>> import pyarrow.parquet as pq + >>> pq.write_table(table, 'example.parquet') + >>> parquet_file = pq.ParquetFile('example.parquet') + + >>> parquet_file.read_row_group(0) + pyarrow.Table + n_legs: int64 + animal: string + ---- + n_legs: [[2,2,4,4,5,100]] + animal: [["Flamingo","Parrot",...,"Brittle stars","Centipede"]] + """ + column_indices = self._get_column_indices( + columns, use_pandas_metadata=use_pandas_metadata) + return self.reader.read_row_group(i, column_indices=column_indices, + use_threads=use_threads) + + def read_row_groups(self, row_groups, columns=None, use_threads=True, + use_pandas_metadata=False): + """ + Read a multiple row groups from a Parquet file. + + Parameters + ---------- + row_groups : list + Only these row groups will be read from the file. + columns : list + If not None, only these columns will be read from the row group. A + column name may be a prefix of a nested field, e.g. 'a' will select + 'a.b', 'a.c', and 'a.d.e'. + use_threads : bool, default True + Perform multi-threaded column reads. + use_pandas_metadata : bool, default False + If True and file has custom pandas schema metadata, ensure that + index columns are also loaded. + + Returns + ------- + pyarrow.table.Table + Content of the row groups as a table (of columns). + + Examples + -------- + >>> import pyarrow as pa + >>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], + ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", + ... "Brittle stars", "Centipede"]}) + >>> import pyarrow.parquet as pq + >>> pq.write_table(table, 'example.parquet') + >>> parquet_file = pq.ParquetFile('example.parquet') + + >>> parquet_file.read_row_groups([0,0]) + pyarrow.Table + n_legs: int64 + animal: string + ---- + n_legs: [[2,2,4,4,5,...,2,4,4,5,100]] + animal: [["Flamingo","Parrot","Dog",...,"Brittle stars","Centipede"]] + """ + column_indices = self._get_column_indices( + columns, use_pandas_metadata=use_pandas_metadata) + return self.reader.read_row_groups(row_groups, + column_indices=column_indices, + use_threads=use_threads) + + def iter_batches(self, batch_size=65536, row_groups=None, columns=None, + use_threads=True, use_pandas_metadata=False): + """ + Read streaming batches from a Parquet file. + + Parameters + ---------- + batch_size : int, default 64K + Maximum number of records to yield per batch. Batches may be + smaller if there aren't enough rows in the file. + row_groups : list + Only these row groups will be read from the file. + columns : list + If not None, only these columns will be read from the file. A + column name may be a prefix of a nested field, e.g. 'a' will select + 'a.b', 'a.c', and 'a.d.e'. + use_threads : boolean, default True + Perform multi-threaded column reads. + use_pandas_metadata : boolean, default False + If True and file has custom pandas schema metadata, ensure that + index columns are also loaded. + + Yields + ------ + pyarrow.RecordBatch + Contents of each batch as a record batch + + Examples + -------- + Generate an example Parquet file: + + >>> import pyarrow as pa + >>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], + ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", + ... "Brittle stars", "Centipede"]}) + >>> import pyarrow.parquet as pq + >>> pq.write_table(table, 'example.parquet') + >>> parquet_file = pq.ParquetFile('example.parquet') + >>> for i in parquet_file.iter_batches(): + ... print("RecordBatch") + ... print(i.to_pandas()) + ... + RecordBatch + n_legs animal + 0 2 Flamingo + 1 2 Parrot + 2 4 Dog + 3 4 Horse + 4 5 Brittle stars + 5 100 Centipede + """ + if batch_size <= 0: + raise ValueError("batch_size must be greater than zero") + + if row_groups is None: + row_groups = range(0, self.metadata.num_row_groups) + column_indices = self._get_column_indices( + columns, use_pandas_metadata=use_pandas_metadata) + + batches = self.reader.iter_batches(batch_size, + row_groups=row_groups, + column_indices=column_indices, + use_threads=use_threads) + return batches + + def read(self, columns=None, use_threads=True, use_pandas_metadata=False): + """ + Read a Table from Parquet format. + + Parameters + ---------- + columns : list + If not None, only these columns will be read from the file. A + column name may be a prefix of a nested field, e.g. 'a' will select + 'a.b', 'a.c', and 'a.d.e'. + use_threads : bool, default True + Perform multi-threaded column reads. + use_pandas_metadata : bool, default False + If True and file has custom pandas schema metadata, ensure that + index columns are also loaded. + + Returns + ------- + pyarrow.table.Table + Content of the file as a table (of columns). + + Examples + -------- + Generate an example Parquet file: + + >>> import pyarrow as pa + >>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], + ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", + ... "Brittle stars", "Centipede"]}) + >>> import pyarrow.parquet as pq + >>> pq.write_table(table, 'example.parquet') + >>> parquet_file = pq.ParquetFile('example.parquet') + + Read a Table: + + >>> parquet_file.read(columns=["animal"]) + pyarrow.Table + animal: string + ---- + animal: [["Flamingo","Parrot",...,"Brittle stars","Centipede"]] + """ + column_indices = self._get_column_indices( + columns, use_pandas_metadata=use_pandas_metadata) + return self.reader.read_all(column_indices=column_indices, + use_threads=use_threads) + + def scan_contents(self, columns=None, batch_size=65536): + """ + Read contents of file for the given columns and batch size. + + Notes + ----- + This function's primary purpose is benchmarking. + The scan is executed on a single thread. + + Parameters + ---------- + columns : list of integers, default None + Select columns to read, if None scan all columns. + batch_size : int, default 64K + Number of rows to read at a time internally. + + Returns + ------- + num_rows : int + Number of rows in file + + Examples + -------- + >>> import pyarrow as pa + >>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], + ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", + ... "Brittle stars", "Centipede"]}) + >>> import pyarrow.parquet as pq + >>> pq.write_table(table, 'example.parquet') + >>> parquet_file = pq.ParquetFile('example.parquet') + + >>> parquet_file.scan_contents() + 6 + """ + column_indices = self._get_column_indices(columns) + return self.reader.scan_contents(column_indices, + batch_size=batch_size) + + def _get_column_indices(self, column_names, use_pandas_metadata=False): + if column_names is None: + return None + + indices = [] + + for name in column_names: + if name in self._nested_paths_by_prefix: + indices.extend(self._nested_paths_by_prefix[name]) + + if use_pandas_metadata: + file_keyvalues = self.metadata.metadata + common_keyvalues = (self.common_metadata.metadata + if self.common_metadata is not None + else None) + + if file_keyvalues and b'pandas' in file_keyvalues: + index_columns = _get_pandas_index_columns(file_keyvalues) + elif common_keyvalues and b'pandas' in common_keyvalues: + index_columns = _get_pandas_index_columns(common_keyvalues) + else: + index_columns = [] + + if indices is not None and index_columns: + indices += [self.reader.column_name_idx(descr) + for descr in index_columns + if not isinstance(descr, dict)] + + return indices + + +_SPARK_DISALLOWED_CHARS = re.compile('[ ,;{}()\n\t=]') + + +def _sanitized_spark_field_name(name): + return _SPARK_DISALLOWED_CHARS.sub('_', name) + + +def _sanitize_schema(schema, flavor): + if 'spark' in flavor: + sanitized_fields = [] + + schema_changed = False + + for field in schema: + name = field.name + sanitized_name = _sanitized_spark_field_name(name) + + if sanitized_name != name: + schema_changed = True + sanitized_field = pa.field(sanitized_name, field.type, + field.nullable, field.metadata) + sanitized_fields.append(sanitized_field) + else: + sanitized_fields.append(field) + + new_schema = pa.schema(sanitized_fields, metadata=schema.metadata) + return new_schema, schema_changed + else: + return schema, False + + +def _sanitize_table(table, new_schema, flavor): + # TODO: This will not handle prohibited characters in nested field names + if 'spark' in flavor: + column_data = [table[i] for i in range(table.num_columns)] + return pa.Table.from_arrays(column_data, schema=new_schema) + else: + return table + + +_parquet_writer_arg_docs = """version : {"1.0", "2.4", "2.6"}, default "2.6" + Determine which Parquet logical types are available for use, whether the + reduced set from the Parquet 1.x.x format or the expanded logical types + added in later format versions. + Files written with version='2.4' or '2.6' may not be readable in all + Parquet implementations, so version='1.0' is likely the choice that + maximizes file compatibility. + UINT32 and some logical types are only available with version '2.4'. + Nanosecond timestamps are only available with version '2.6'. + Other features such as compression algorithms or the new serialized + data page format must be enabled separately (see 'compression' and + 'data_page_version'). +use_dictionary : bool or list, default True + Specify if we should use dictionary encoding in general or only for + some columns. + When encoding the column, if the dictionary size is too large, the + column will fallback to ``PLAIN`` encoding. Specially, ``BOOLEAN`` type + doesn't support dictionary encoding. +compression : str or dict, default 'snappy' + Specify the compression codec, either on a general basis or per-column. + Valid values: {'NONE', 'SNAPPY', 'GZIP', 'BROTLI', 'LZ4', 'LZ4_RAW', 'ZSTD'}. + 'LZ4_RAW' is accepted as an alias for 'LZ4' (both use the LZ4_RAW + codec as defined in the Parquet specification). +write_statistics : bool or list, default True + Specify if we should write statistics in general (default is True) or only + for some columns. +use_deprecated_int96_timestamps : bool, default None + Write timestamps to INT96 Parquet format. Defaults to False unless enabled + by flavor argument. This take priority over the coerce_timestamps option. +coerce_timestamps : str, default None + Cast timestamps to a particular resolution. If omitted, defaults are chosen + depending on `version`. For ``version='1.0'`` and ``version='2.4'``, + nanoseconds are cast to microseconds ('us'), while for + ``version='2.6'`` (the default), they are written natively without loss + of resolution. Seconds are always cast to milliseconds ('ms') by default, + as Parquet does not have any temporal type with seconds resolution. + If the casting results in loss of data, it will raise an exception + unless ``allow_truncated_timestamps=True`` is given. + Valid values: {None, 'ms', 'us'} +allow_truncated_timestamps : bool, default False + Allow loss of data when coercing timestamps to a particular + resolution. E.g. if microsecond or nanosecond data is lost when coercing to + 'ms', do not raise an exception. Passing ``allow_truncated_timestamp=True`` + will NOT result in the truncation exception being ignored unless + ``coerce_timestamps`` is not None. +data_page_size : int, default None + Set a target threshold for the approximate encoded size of data + pages within a column chunk (in bytes). If None, use the default data page + size of 1MByte. +max_rows_per_page : int, default None + Maximum number of rows per page within a column chunk. + If None, use the default of 20000. + Smaller values reduce memory usage during reads but increase metadata overhead. +flavor : {'spark'}, default None + Sanitize schema or set other compatibility options to work with + various target systems. +filesystem : FileSystem, default None + If nothing passed, will be inferred from `where` if path-like, else + `where` is already a file-like object so no filesystem is needed. +compression_level : int or dict, default None + Specify the compression level for a codec, either on a general basis or + per-column. If None is passed, arrow selects the compression level for + the compression codec in use. The compression level has a different + meaning for each codec, so you have to read the documentation of the + codec you are using. + An exception is thrown if the compression codec does not allow specifying + a compression level. +use_byte_stream_split : bool or list, default False + Specify if the byte_stream_split encoding should be used in general or + only for some columns. If both dictionary and byte_stream_stream are + enabled, then dictionary is preferred. + The byte_stream_split encoding is valid for integer, floating-point + and fixed-size binary data types (including decimals); it should be + combined with a compression codec so as to achieve size reduction. +column_encoding : string or dict, default None + Specify the encoding scheme on a per column basis. + Can only be used when ``use_dictionary`` is set to False, and + cannot be used in combination with ``use_byte_stream_split``. + Currently supported values: {'PLAIN', 'BYTE_STREAM_SPLIT', + 'DELTA_BINARY_PACKED', 'DELTA_LENGTH_BYTE_ARRAY', 'DELTA_BYTE_ARRAY'}. + Certain encodings are only compatible with certain data types. + Please refer to the encodings section of `Reading and writing Parquet + files `_. +data_page_version : {"1.0", "2.0"}, default "1.0" + The serialized Parquet data page format version to write, defaults to + 1.0. This does not impact the file schema logical types and Arrow to + Parquet type casting behavior; for that use the "version" option. +use_compliant_nested_type : bool, default True + Whether to write compliant Parquet nested type (lists) as defined + `here `_, defaults to ``True``. + For ``use_compliant_nested_type=True``, this will write into a list + with 3-level structure where the middle level, named ``list``, + is a repeated group with a single field named ``element``:: + + group (LIST) { + repeated group list { + element; + } + } + + For ``use_compliant_nested_type=False``, this will also write into a list + with 3-level structure, where the name of the single field of the middle + level ``list`` is taken from the element name for nested columns in Arrow, + which defaults to ``item``:: + + group (LIST) { + repeated group list { + item; + } + } + +encryption_properties : FileEncryptionProperties, default None + File encryption properties for Parquet Modular Encryption. + If None, no encryption will be done. + The encryption properties can be created using: + ``CryptoFactory.file_encryption_properties()``. +write_batch_size : int, default None + Number of values to write to a page at a time. If None, use the default of + 1024. ``write_batch_size`` is complementary to ``data_page_size``. If pages + are exceeding the ``data_page_size`` due to large column values, lowering + the batch size can help keep page sizes closer to the intended size. +dictionary_pagesize_limit : int, default None + Specify the dictionary page size limit per row group. If None, use the + default 1MB. +store_schema : bool, default True + By default, the Arrow schema is serialized and stored in the Parquet + file metadata (in the "ARROW:schema" key). When reading the file, + if this key is available, it will be used to more faithfully recreate + the original Arrow data. For example, for tz-aware timestamp columns + it will restore the timezone (Parquet only stores the UTC values without + timezone), or columns with duration type will be restored from the int64 + Parquet column. +write_page_index : bool, default False + Whether to write a page index in general for all columns. + Writing statistics to the page index disables the old method of writing + statistics to each data page header. The page index makes statistics-based + filtering more efficient than the page header, as it gathers all the + statistics for a Parquet file in a single place, avoiding scattered I/O. + Note that the page index is not yet used on the read size by PyArrow. +write_page_checksum : bool, default False + Whether to write page checksums in general for all columns. + Page checksums enable detection of data corruption, which might occur during + transmission or in the storage. +sorting_columns : Sequence of SortingColumn, default None + Specify the sort order of the data being written. The writer does not sort + the data nor does it verify that the data is sorted. The sort order is + written to the row group metadata, which can then be used by readers. +store_decimal_as_integer : bool, default False + Allow decimals with 1 <= precision <= 18 to be stored as integers. + In Parquet, DECIMAL can be stored in any of the following physical types: + + - int32: for 1 <= precision <= 9. + - int64: for 10 <= precision <= 18. + - fixed_len_byte_array: precision is limited by the array size. + Length n can store <= floor(log_10(2^(8*n - 1) - 1)) base-10 digits. + - binary: precision is unlimited. The minimum number of bytes to store the + unscaled value is used. + + By default, this is DISABLED and all decimal types annotate fixed_len_byte_array. + When enabled, the writer will use the following physical types to store decimals: + + - int32: for 1 <= precision <= 9. + - int64: for 10 <= precision <= 18. + - fixed_len_byte_array: for precision > 18. + + As a consequence, decimal columns stored in integer types are more compact. +use_content_defined_chunking : bool or dict, default False + Optimize parquet files for content addressable storage (CAS) systems by writing + data pages according to content-defined chunk boundaries. This allows for more + efficient deduplication of data across files, hence more efficient network + transfers and storage. The chunking is based on a rolling hash algorithm that + identifies chunk boundaries based on the actual content of the data. + + Note that it is an experimental feature and the API may change in the future. + + If set to ``True``, a default configuration is used with `min_chunk_size=256 KiB` + and `max_chunk_size=1024 KiB`. The chunk size distribution approximates a normal + distribution between `min_chunk_size` and `max_chunk_size` (sizes are accounted + before any Parquet encodings). + + A `dict` can be passed to adjust the chunker parameters with the following keys: + + - `min_chunk_size`: minimum chunk size in bytes, default 256 KiB + The rolling hash will not be updated until this size is reached for each chunk. + Note that all data sent through the hash function is counted towards the chunk + size, including definition and repetition levels if present. + - `max_chunk_size`: maximum chunk size in bytes, default is 1024 KiB + The chunker will create a new chunk whenever the chunk size exceeds this value. + Note that the parquet writer has a related `data_pagesize` property that controls + the maximum size of a parquet data page after encoding. While setting + `data_page_size` to a smaller value than `max_chunk_size` doesn't affect the + chunking effectiveness, it results in more small parquet data pages. + - `norm_level`: normalization level to center the chunk size around the average + size more aggressively, default 0 + Increasing the normalization level increases the probability of finding a chunk, + improving the deduplication ratio, but also increasing the number of small chunks + resulting in many small parquet data pages. The default value provides a good + balance between deduplication ratio and fragmentation. Use norm_level=1 or + norm_level=2 to reach a higher deduplication ratio at the expense of + fragmentation. + +write_time_adjusted_to_utc : bool, default False + Set the value of isAdjustedTOUTC when writing a TIME column. + If True, this tells the Parquet reader that the TIME columns + are expressed in reference to midnight in the UTC timezone. + If False (the default), the TIME columns are assumed to be expressed + in reference to midnight in an unknown, presumably local, timezone. +bloom_filter_options : dict, default None + Create Bloom filters for the columns specified by the provided `dict`. + + Bloom filters can be configured with two parameters: number of distinct values + (NDV), and false-positive probability (FPP). + + Bloom filters are most effective for high-cardinality columns. A good default + is to set NDV equal to the number of rows. Lower values reduce disk usage but + may not be worthwhile for very small NDVs. Increasing NDV (without increasing FPP) + increases disk and memory usage. + + Lower FPP values require more disk and memory space. For a fixed NDV, the + space requirement grows roughly proportional to log(1/FPP). Recommended + values are 0.1, 0.05, or 0.01. Very small values are counterproductive as + the bitset may exceed the size of the actual data. Set NDV appropriately + to minimize space usage. + + The keys of the `dict` are column paths. For each path, the value can be either: + + - A dictionary, with keys `ndv` and `fpp`. The value for `ndv` must be a positive + integer. If the 'ndv' key is not present, the default value of `1048576` will be + used. The value for `fpp` must be a float between 0.0 and 1.0. If the `fpp` key + is not present, the default value of `0.05` will be used. + - A boolean, with ``True`` indicating that a Bloom filter should be produced with + the above mentioned default values of `ndv=1048576` and `fpp=0.05`. This is + equivalent to passing an empty dict. +""" + +_parquet_writer_example_doc = """\ +Generate an example PyArrow Table and RecordBatch: + +>>> import pyarrow as pa +>>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], +... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", +... "Brittle stars", "Centipede"]}) +>>> batch = pa.record_batch([[2, 2, 4, 4, 5, 100], +... ["Flamingo", "Parrot", "Dog", "Horse", +... "Brittle stars", "Centipede"]], +... names=['n_legs', 'animal']) + +create a ParquetWriter object: + +>>> import pyarrow.parquet as pq +>>> writer = pq.ParquetWriter('example.parquet', table.schema) + +and write the Table into the Parquet file: + +>>> writer.write_table(table) +>>> writer.close() + +>>> pq.read_table('example.parquet').to_pandas() + n_legs animal +0 2 Flamingo +1 2 Parrot +2 4 Dog +3 4 Horse +4 5 Brittle stars +5 100 Centipede + +create a ParquetWriter object for the RecordBatch: + +>>> writer2 = pq.ParquetWriter('example2.parquet', batch.schema) + +and write the RecordBatch into the Parquet file: + +>>> writer2.write_batch(batch) +>>> writer2.close() + +>>> pq.read_table('example2.parquet').to_pandas() + n_legs animal +0 2 Flamingo +1 2 Parrot +2 4 Dog +3 4 Horse +4 5 Brittle stars +5 100 Centipede +""" + + +class ParquetWriter: + + __doc__ = f""" +Class for incrementally building a Parquet file for Arrow tables. + +Parameters +---------- +where : path or file-like object +schema : pyarrow.Schema +{_parquet_writer_arg_docs} +writer_engine_version : unused +**options : dict + If options contains a key `metadata_collector` then the + corresponding value is assumed to be a list (or any object with + `.append` method) that will be filled with the file metadata instance + of the written file. + +Examples +-------- +{_parquet_writer_example_doc} +""" + + def __init__(self, where, schema, filesystem=None, + flavor=None, + version='2.6', + use_dictionary=True, + compression='snappy', + write_statistics=True, + use_deprecated_int96_timestamps=None, + compression_level=None, + use_byte_stream_split=False, + column_encoding=None, + writer_engine_version=None, + data_page_version='1.0', + use_compliant_nested_type=True, + encryption_properties=None, + write_batch_size=None, + dictionary_pagesize_limit=None, + store_schema=True, + write_page_index=False, + write_page_checksum=False, + sorting_columns=None, + store_decimal_as_integer=False, + write_time_adjusted_to_utc=False, + max_rows_per_page=None, + **options): + if use_deprecated_int96_timestamps is None: + # Use int96 timestamps for Spark + if flavor is not None and 'spark' in flavor: + use_deprecated_int96_timestamps = True + else: + use_deprecated_int96_timestamps = False + + self.flavor = flavor + if flavor is not None: + schema, self.schema_changed = _sanitize_schema(schema, flavor) + else: + self.schema_changed = False + + self.schema = schema + self.where = where + + # If we open a file using a filesystem, store file handle so we can be + # sure to close it when `self.close` is called. + self.file_handle = None + + filesystem, path = _resolve_filesystem_and_path(where, filesystem) + if filesystem is not None: + # ARROW-10480: do not auto-detect compression. While + # a filename like foo.parquet.gz is nonconforming, it + # shouldn't implicitly apply compression. + sink = self.file_handle = filesystem.open_output_stream( + path, compression=None) + else: + sink = where + self._metadata_collector = options.pop('metadata_collector', None) + engine_version = 'V2' + self.writer = _parquet.ParquetWriter( + sink, schema, + version=version, + compression=compression, + use_dictionary=use_dictionary, + write_statistics=write_statistics, + use_deprecated_int96_timestamps=use_deprecated_int96_timestamps, + compression_level=compression_level, + use_byte_stream_split=use_byte_stream_split, + column_encoding=column_encoding, + writer_engine_version=engine_version, + data_page_version=data_page_version, + use_compliant_nested_type=use_compliant_nested_type, + encryption_properties=encryption_properties, + write_batch_size=write_batch_size, + dictionary_pagesize_limit=dictionary_pagesize_limit, + store_schema=store_schema, + write_page_index=write_page_index, + write_page_checksum=write_page_checksum, + sorting_columns=sorting_columns, + store_decimal_as_integer=store_decimal_as_integer, + write_time_adjusted_to_utc=write_time_adjusted_to_utc, + max_rows_per_page=max_rows_per_page, + **options) + self.is_open = True + + def __del__(self): + if getattr(self, 'is_open', False): + self.close() + + def __enter__(self): + return self + + def __exit__(self, *args, **kwargs): + self.close() + # return false since we want to propagate exceptions + return False + + def write(self, table_or_batch, row_group_size=None): + """ + Write RecordBatch or Table to the Parquet file. + + Parameters + ---------- + table_or_batch : {RecordBatch, Table} + row_group_size : int, default None + Maximum number of rows in each written row group. If None, the row + group size will be the minimum of the number of rows in the + Table/RecordBatch and 1024 * 1024. + """ + if isinstance(table_or_batch, pa.RecordBatch): + self.write_batch(table_or_batch, row_group_size) + elif isinstance(table_or_batch, pa.Table): + self.write_table(table_or_batch, row_group_size) + else: + raise TypeError(type(table_or_batch)) + + def write_batch(self, batch, row_group_size=None): + """ + Write RecordBatch to the Parquet file. + + Parameters + ---------- + batch : RecordBatch + row_group_size : int, default None + Maximum number of rows in written row group. If None, the + row group size will be the minimum of the RecordBatch + size (in rows) and 1024 * 1024. If set larger than 64 * 1024 * 1024 + then 64 * 1024 * 1024 will be used instead. + """ + table = pa.Table.from_batches([batch], batch.schema) + self.write_table(table, row_group_size) + + def write_table(self, table, row_group_size=None): + """ + Write Table to the Parquet file. + + Parameters + ---------- + table : Table + row_group_size : int, default None + Maximum number of rows in each written row group. If None, + the row group size will be the minimum of the Table size (in rows) + and 1024 * 1024. If set larger than 64 * 1024 * 1024 then + 64 * 1024 * 1024 will be used instead. + + """ + if self.schema_changed: + table = _sanitize_table(table, self.schema, self.flavor) + assert self.is_open + + if not table.schema.equals(self.schema, check_metadata=False): + msg = ( + "Table schema does not match schema used to create file: \n" + f"table:\n{table.schema!s} vs. \nfile:\n{self.schema!s}" + ) + raise ValueError(msg) + + self.writer.write_table(table, row_group_size=row_group_size) + + def close(self): + """ + Close the connection to the Parquet file. + """ + if self.is_open: + self.writer.close() + self.is_open = False + if self._metadata_collector is not None: + self._metadata_collector.append(self.writer.metadata) + if self.file_handle is not None: + self.file_handle.close() + + def add_key_value_metadata(self, key_value_metadata): + """ + Add key-value metadata to the file. + This will overwrite any existing metadata with the same key. + + Parameters + ---------- + key_value_metadata : dict + Keys and values must be string-like / coercible to bytes. + """ + assert self.is_open + self.writer.add_key_value_metadata(key_value_metadata) + + +def _get_pandas_index_columns(keyvalues): + return (json.loads(keyvalues[b'pandas'].decode('utf8')) + ['index_columns']) + + +EXCLUDED_PARQUET_PATHS = {'_SUCCESS'} + + +_read_docstring_common = """\ +read_dictionary : list, default None + List of names or column paths (for nested types) to read directly + as DictionaryArray. Only supported for BYTE_ARRAY storage. To read + a flat column as dictionary-encoded pass the column name. For + nested types, you must pass the full column "path", which could be + something like level1.level2.list.item. Refer to the Parquet + file's schema to obtain the paths. +binary_type : pyarrow.DataType, default None + If given, Parquet binary columns will be read as this datatype. + This setting is ignored if a serialized Arrow schema is found in + the Parquet metadata. +list_type : subclass of pyarrow.DataType, default None + If given, non-MAP repeated columns will be read as an instance of + this datatype (either pyarrow.ListType or pyarrow.LargeListType). + This setting is ignored if a serialized Arrow schema is found in + the Parquet metadata. +memory_map : bool, default False + If the source is a file path, use a memory map to read file, which can + improve performance in some environments. +buffer_size : int, default 0 + If positive, perform read buffering when deserializing individual + column chunks. Otherwise IO calls are unbuffered. +partitioning : pyarrow.dataset.Partitioning or str or list of str, \ +default "hive" + The partitioning scheme for a partitioned dataset. The default of "hive" + assumes directory names with key=value pairs like "/year=2009/month=11". + In addition, a scheme like "/2009/11" is also supported, in which case + you need to specify the field names or a full schema. See the + ``pyarrow.dataset.partitioning()`` function for more details.""" + + +_parquet_dataset_example = """\ +Generate an example PyArrow Table and write it to a partitioned dataset: + +>>> import pyarrow as pa +>>> table = pa.table({'year': [2020, 2022, 2021, 2022, 2019, 2021], +... 'n_legs': [2, 2, 4, 4, 5, 100], +... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", +... "Brittle stars", "Centipede"]}) +>>> import pyarrow.parquet as pq +>>> pq.write_to_dataset(table, root_path='dataset_v2', +... partition_cols=['year']) + +create a ParquetDataset object from the dataset source: + +>>> dataset = pq.ParquetDataset('dataset_v2/') + +and read the data: + +>>> dataset.read().to_pandas() + n_legs animal year +0 5 Brittle stars 2019 +1 2 Flamingo 2020 +2 4 Dog 2021 +3 100 Centipede 2021 +4 2 Parrot 2022 +5 4 Horse 2022 + +create a ParquetDataset object with filter: + +>>> dataset = pq.ParquetDataset('dataset_v2/', +... filters=[('n_legs','=',4)]) +>>> dataset.read().to_pandas() + n_legs animal year +0 4 Dog 2021 +1 4 Horse 2022 +""" + + +class ParquetDataset: + __doc__ = f""" +Encapsulates details of reading a complete Parquet dataset possibly +consisting of multiple files and partitions in subdirectories. + +Parameters +---------- +path_or_paths : str or List[str] + A directory name, single file name, or list of file names. +filesystem : FileSystem, default None + If nothing passed, will be inferred based on path. + Path will try to be found in the local on-disk filesystem otherwise + it will be parsed as an URI to determine the filesystem. +schema : pyarrow.parquet.Schema + Optionally provide the Schema for the Dataset, in which case it will + not be inferred from the source. +filters : pyarrow.compute.Expression or List[Tuple] or List[List[Tuple]], default None + Rows which do not match the filter predicate will be removed from scanned + data. Partition keys embedded in a nested directory structure will be + exploited to avoid loading files at all if they contain no matching rows. + Within-file level filtering and different partitioning schemes are supported. + + {_DNF_filter_doc} +{_read_docstring_common} +ignore_prefixes : list, optional + Files matching any of these prefixes will be ignored by the + discovery process. + This is matched to the basename of a path. + By default this is ['.', '_']. + Note that discovery happens only if a directory is passed as source. +pre_buffer : bool, default True + Coalesce and issue file reads in parallel to improve performance on + high-latency filesystems (e.g. S3, GCS). If True, Arrow will use a + background I/O thread pool. If using a filesystem layer that itself + performs readahead (e.g. fsspec's S3FS), disable readahead for best + results. Set to False if you want to prioritize minimal memory usage + over maximum speed. +coerce_int96_timestamp_unit : str, default None + Cast timestamps that are stored in INT96 format to a particular resolution + (e.g. 'ms'). Setting to None is equivalent to 'ns' and therefore INT96 + timestamps will be inferred as timestamps in nanoseconds. +decryption_properties : FileDecryptionProperties or None + File-level decryption properties. + The decryption properties can be created using + ``CryptoFactory.file_decryption_properties()``. +thrift_string_size_limit : int, default None + If not None, override the maximum total string size allocated + when decoding Thrift structures. The default limit should be + sufficient for most Parquet files. +thrift_container_size_limit : int, default None + If not None, override the maximum total size of containers allocated + when decoding Thrift structures. The default limit should be + sufficient for most Parquet files. +page_checksum_verification : bool, default False + If True, verify the page checksum for each page read from the file. +arrow_extensions_enabled : bool, default True + If True, read Parquet logical types as Arrow extension types where possible, + (e.g., read JSON as the canonical `arrow.json` extension type or UUID as + the canonical `arrow.uuid` extension type). + +Examples +-------- +{_parquet_dataset_example} +""" + + def __init__(self, path_or_paths, filesystem=None, schema=None, *, filters=None, + read_dictionary=None, binary_type=None, list_type=None, + memory_map=False, buffer_size=None, partitioning="hive", + ignore_prefixes=None, + pre_buffer=True, coerce_int96_timestamp_unit=None, + decryption_properties=None, thrift_string_size_limit=None, + thrift_container_size_limit=None, + page_checksum_verification=False, + arrow_extensions_enabled=True): + import pyarrow.dataset as ds + + # map format arguments + read_options = { + "pre_buffer": pre_buffer, + "coerce_int96_timestamp_unit": coerce_int96_timestamp_unit, + "thrift_string_size_limit": thrift_string_size_limit, + "thrift_container_size_limit": thrift_container_size_limit, + "page_checksum_verification": page_checksum_verification, + "arrow_extensions_enabled": arrow_extensions_enabled, + "binary_type": binary_type, + "list_type": list_type, + } + if buffer_size: + read_options.update(use_buffered_stream=True, + buffer_size=buffer_size) + if read_dictionary is not None: + read_options.update(dictionary_columns=read_dictionary) + + if decryption_properties is not None: + read_options.update(decryption_properties=decryption_properties) + + self._filter_expression = None + if filters is not None: + self._filter_expression = filters_to_expression(filters) + + # map old filesystems to new one + if filesystem is not None: + filesystem = _ensure_filesystem( + filesystem, use_mmap=memory_map) + elif filesystem is None and memory_map: + # if memory_map is specified, assume local file system (string + # path can in principle be URI for any filesystem) + filesystem = LocalFileSystem(use_mmap=memory_map) + + # This needs to be checked after _ensure_filesystem, because that + # handles the case of an fsspec LocalFileSystem + if ( + hasattr(path_or_paths, "__fspath__") and + filesystem is not None and + not isinstance(filesystem, LocalFileSystem) + ): + raise TypeError( + "Path-like objects with __fspath__ must only be used with " + f"local file systems, not {type(filesystem)}" + ) + + # check for single fragment dataset or dataset directory + single_file = None + self._base_dir = None + if not isinstance(path_or_paths, list): + if _is_path_like(path_or_paths): + filesystem, path_or_paths = _resolve_filesystem_and_path( + path_or_paths, filesystem, memory_map=memory_map + ) + finfo = filesystem.get_file_info(path_or_paths) + if finfo.type == FileType.Directory: + self._base_dir = path_or_paths + else: + single_file = path_or_paths + + parquet_format = ds.ParquetFileFormat(**read_options) + + if single_file is not None: + fragment = parquet_format.make_fragment(single_file, filesystem) + + self._dataset = ds.FileSystemDataset( + [fragment], schema=schema or fragment.physical_schema, + format=parquet_format, + filesystem=fragment.filesystem + ) + return + + # check partitioning to enable dictionary encoding + if partitioning == "hive": + partitioning = ds.HivePartitioning.discover( + infer_dictionary=True) + + self._dataset = ds.dataset(path_or_paths, filesystem=filesystem, + schema=schema, format=parquet_format, + partitioning=partitioning, + ignore_prefixes=ignore_prefixes) + + def equals(self, other): + if not isinstance(other, ParquetDataset): + raise TypeError('`other` must be an instance of ParquetDataset') + + return (self.schema == other.schema and + self._dataset.format == other._dataset.format and + self.filesystem == other.filesystem and + # self.fragments == other.fragments and + self.files == other.files) + + def __eq__(self, other): + try: + return self.equals(other) + except TypeError: + return NotImplemented + + @property + def schema(self): + """ + Schema of the Dataset. + + Examples + -------- + Generate an example dataset: + + >>> import pyarrow as pa + >>> table = pa.table({'year': [2020, 2022, 2021, 2022, 2019, 2021], + ... 'n_legs': [2, 2, 4, 4, 5, 100], + ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", + ... "Brittle stars", "Centipede"]}) + >>> import pyarrow.parquet as pq + >>> pq.write_to_dataset(table, root_path='dataset_v2_schema', + ... partition_cols=['year']) + >>> dataset = pq.ParquetDataset('dataset_v2_schema/') + + Read the schema: + + >>> dataset.schema + n_legs: int64 + animal: string + year: dictionary + """ + return self._dataset.schema + + def read(self, columns=None, use_threads=True, use_pandas_metadata=False): + """ + Read (multiple) Parquet files as a single pyarrow.Table. + + Parameters + ---------- + columns : List[str] + Names of columns to read from the dataset. The partition fields + are not automatically included. + use_threads : bool, default True + Perform multi-threaded column reads. + use_pandas_metadata : bool, default False + If True and file has custom pandas schema metadata, ensure that + index columns are also loaded. + + Returns + ------- + pyarrow.Table + Content of the file as a table (of columns). + + Examples + -------- + Generate an example dataset: + + >>> import pyarrow as pa + >>> table = pa.table({'year': [2020, 2022, 2021, 2022, 2019, 2021], + ... 'n_legs': [2, 2, 4, 4, 5, 100], + ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", + ... "Brittle stars", "Centipede"]}) + >>> import pyarrow.parquet as pq + >>> pq.write_to_dataset(table, root_path='dataset_v2_read', + ... partition_cols=['year']) + >>> dataset = pq.ParquetDataset('dataset_v2_read/') + + Read the dataset: + + >>> dataset.read(columns=["n_legs"]) + pyarrow.Table + n_legs: int64 + ---- + n_legs: [[5],[2],[4,100],[2,4]] + """ + # if use_pandas_metadata, we need to include index columns in the + # column selection, to be able to restore those in the pandas DataFrame + metadata = self.schema.metadata or {} + + if use_pandas_metadata: + # if the dataset schema metadata itself doesn't have pandas + # then try to get this from common file (for backwards compat) + if b"pandas" not in metadata: + common_metadata = self._get_common_pandas_metadata() + if common_metadata: + metadata = common_metadata + + if columns is not None and use_pandas_metadata: + if metadata and b'pandas' in metadata: + # RangeIndex can be represented as dict instead of column name + index_columns = [ + col for col in _get_pandas_index_columns(metadata) + if not isinstance(col, dict) + ] + columns = ( + list(columns) + list(set(index_columns) - set(columns)) + ) + + table = self._dataset.to_table( + columns=columns, filter=self._filter_expression, + use_threads=use_threads + ) + + # if use_pandas_metadata, restore the pandas metadata (which gets + # lost if doing a specific `columns` selection in to_table) + if use_pandas_metadata: + if metadata and b"pandas" in metadata: + new_metadata = table.schema.metadata or {} + new_metadata.update({b"pandas": metadata[b"pandas"]}) + table = table.replace_schema_metadata(new_metadata) + + return table + + def _get_common_pandas_metadata(self): + + if not self._base_dir: + return None + + metadata = None + for name in ["_common_metadata", "_metadata"]: + metadata_path = os.path.join(str(self._base_dir), name) + finfo = self.filesystem.get_file_info(metadata_path) + if finfo.is_file: + pq_meta = read_metadata( + metadata_path, filesystem=self.filesystem) + metadata = pq_meta.metadata + if metadata and b'pandas' in metadata: + break + + return metadata + + def read_pandas(self, **kwargs): + """ + Read dataset including pandas metadata, if any. Other arguments passed + through to :func:`read`, see docstring for further details. + + Parameters + ---------- + **kwargs : optional + Additional options for :func:`read` + + Examples + -------- + Generate an example parquet file: + + >>> import pyarrow as pa + >>> import pandas as pd + >>> df = pd.DataFrame({'year': [2020, 2022, 2021, 2022, 2019, 2021], + ... 'n_legs': [2, 2, 4, 4, 5, 100], + ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", + ... "Brittle stars", "Centipede"]}) + >>> table = pa.Table.from_pandas(df) + >>> import pyarrow.parquet as pq + >>> pq.write_table(table, 'table_V2.parquet') + >>> dataset = pq.ParquetDataset('table_V2.parquet') + + Read the dataset with pandas metadata: + + >>> dataset.read_pandas(columns=["n_legs"]) + pyarrow.Table + n_legs: int64 + ---- + n_legs: [[2,2,4,4,5,100]] + + >>> dataset.read_pandas(columns=["n_legs"]).schema.pandas_metadata + {'index_columns': [{'kind': 'range', 'name': None, 'start': 0, ...} + """ + return self.read(use_pandas_metadata=True, **kwargs) + + @property + def fragments(self): + """ + A list of the Dataset source fragments or pieces with absolute + file paths. + + Examples + -------- + Generate an example dataset: + + >>> import pyarrow as pa + >>> table = pa.table({'year': [2020, 2022, 2021, 2022, 2019, 2021], + ... 'n_legs': [2, 2, 4, 4, 5, 100], + ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", + ... "Brittle stars", "Centipede"]}) + >>> import pyarrow.parquet as pq + >>> pq.write_to_dataset(table, root_path='dataset_v2_fragments', + ... partition_cols=['year']) + >>> dataset = pq.ParquetDataset('dataset_v2_fragments/') + + List the fragments: + + >>> dataset.fragments + [>> import pyarrow as pa + >>> table = pa.table({'year': [2020, 2022, 2021, 2022, 2019, 2021], + ... 'n_legs': [2, 2, 4, 4, 5, 100], + ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", + ... "Brittle stars", "Centipede"]}) + >>> import pyarrow.parquet as pq + >>> pq.write_to_dataset(table, root_path='dataset_v2_files', + ... partition_cols=['year']) + >>> dataset = pq.ParquetDataset('dataset_v2_files/') + + List the files: + + >>> dataset.files + ['dataset_v2_files/year=2019/...-0.parquet', ... + """ + return self._dataset.files + + @property + def filesystem(self): + """ + The filesystem type of the Dataset source. + """ + return self._dataset.filesystem + + @property + def partitioning(self): + """ + The partitioning of the Dataset source, if discovered. + """ + return self._dataset.partitioning + + +_read_table_docstring = """ +{0} + +Parameters +---------- +source : str, list of str, pyarrow.NativeFile, or file-like object + If a string is passed, can be a single file name or directory name. If a + list of strings is passed, should be file names. For file-like objects, + only read a single file. Use pyarrow.BufferReader to read a file contained + in a bytes or buffer-like object. +columns : list + If not None, only these columns will be read from the file. A column + name may be a prefix of a nested field, e.g. 'a' will select 'a.b', + 'a.c', and 'a.d.e'. If empty, no columns will be read. Note + that the table will still have the correct num_rows set despite having + no columns. +use_threads : bool, default True + Perform multi-threaded column reads. +schema : Schema, optional + Optionally provide the Schema for the parquet dataset, in which case it + will not be inferred from the source. +{1} +filesystem : FileSystem, default None + If nothing passed, will be inferred based on path. + Path will try to be found in the local on-disk filesystem otherwise + it will be parsed as an URI to determine the filesystem. +filters : pyarrow.compute.Expression or List[Tuple] or List[List[Tuple]], default None + Rows which do not match the filter predicate will be removed from scanned + data. Partition keys embedded in a nested directory structure will be + exploited to avoid loading files at all if they contain no matching rows. + Within-file level filtering and different partitioning schemes are supported. + + {3} +ignore_prefixes : list, optional + Files matching any of these prefixes will be ignored by the + discovery process. + This is matched to the basename of a path. + By default this is ['.', '_']. + Note that discovery happens only if a directory is passed as source. +pre_buffer : bool, default True + Coalesce and issue file reads in parallel to improve performance on + high-latency filesystems (e.g. S3). If True, Arrow will use a + background I/O thread pool. If using a filesystem layer that itself + performs readahead (e.g. fsspec's S3FS), disable readahead for best + results. +coerce_int96_timestamp_unit : str, default None + Cast timestamps that are stored in INT96 format to a particular + resolution (e.g. 'ms'). Setting to None is equivalent to 'ns' + and therefore INT96 timestamps will be inferred as timestamps + in nanoseconds. +decryption_properties : FileDecryptionProperties or None + File-level decryption properties. + The decryption properties can be created using + ``CryptoFactory.file_decryption_properties()``. +thrift_string_size_limit : int, default None + If not None, override the maximum total string size allocated + when decoding Thrift structures. The default limit should be + sufficient for most Parquet files. +thrift_container_size_limit : int, default None + If not None, override the maximum total size of containers allocated + when decoding Thrift structures. The default limit should be + sufficient for most Parquet files. +page_checksum_verification : bool, default False + If True, verify the checksum for each page read from the file. +arrow_extensions_enabled : bool, default True + If True, read Parquet logical types as Arrow extension types where possible, + (e.g., read JSON as the canonical `arrow.json` extension type or UUID as + the canonical `arrow.uuid` extension type). + +Returns +------- +{2} + +{4} +""" + +_read_table_example = """\ + +Examples +-------- + +Generate an example PyArrow Table and write it to a partitioned dataset: + +>>> import pyarrow as pa +>>> table = pa.table({'year': [2020, 2022, 2021, 2022, 2019, 2021], +... 'n_legs': [2, 2, 4, 4, 5, 100], +... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", +... "Brittle stars", "Centipede"]}) +>>> import pyarrow.parquet as pq +>>> pq.write_to_dataset(table, root_path='dataset_name_2', +... partition_cols=['year']) + +Read the data: + +>>> pq.read_table('dataset_name_2').to_pandas() + n_legs animal year +0 5 Brittle stars 2019 +1 2 Flamingo 2020 +2 4 Dog 2021 +3 100 Centipede 2021 +4 2 Parrot 2022 +5 4 Horse 2022 + + +Read only a subset of columns: + +>>> pq.read_table('dataset_name_2', columns=["n_legs", "animal"]) +pyarrow.Table +n_legs: int64 +animal: string +---- +n_legs: [[5],[2],[4,100],[2,4]] +animal: [["Brittle stars"],["Flamingo"],["Dog","Centipede"],["Parrot","Horse"]] + +Read a subset of columns and read one column as DictionaryArray: + +>>> pq.read_table('dataset_name_2', columns=["n_legs", "animal"], +... read_dictionary=["animal"]) +pyarrow.Table +n_legs: int64 +animal: dictionary +---- +n_legs: [[5],[2],[4,100],[2,4]] +animal: [ -- dictionary: +["Brittle stars"] -- indices: +[0], -- dictionary: +["Flamingo"] -- indices: +[0], -- dictionary: +["Dog","Centipede"] -- indices: +[0,1], -- dictionary: +["Parrot","Horse"] -- indices: +[0,1]] + +Read the table with filter: + +>>> pq.read_table('dataset_name_2', columns=["n_legs", "animal"], +... filters=[('n_legs','<',4)]).to_pandas() + n_legs animal +0 2 Flamingo +1 2 Parrot + +Read data from a single Parquet file: + +>>> pq.write_table(table, 'example.parquet') +>>> pq.read_table('dataset_name_2').to_pandas() + n_legs animal year +0 5 Brittle stars 2019 +1 2 Flamingo 2020 +2 4 Dog 2021 +3 100 Centipede 2021 +4 2 Parrot 2022 +5 4 Horse 2022 +""" + + +def read_table(source, *, columns=None, use_threads=True, + schema=None, use_pandas_metadata=False, read_dictionary=None, + binary_type=None, list_type=None, memory_map=False, buffer_size=0, + partitioning="hive", filesystem=None, filters=None, + ignore_prefixes=None, pre_buffer=True, + coerce_int96_timestamp_unit=None, + decryption_properties=None, thrift_string_size_limit=None, + thrift_container_size_limit=None, + page_checksum_verification=False, + arrow_extensions_enabled=True): + + try: + dataset = ParquetDataset( + source, + schema=schema, + filesystem=filesystem, + partitioning=partitioning, + memory_map=memory_map, + read_dictionary=read_dictionary, + binary_type=binary_type, + list_type=list_type, + buffer_size=buffer_size, + filters=filters, + ignore_prefixes=ignore_prefixes, + pre_buffer=pre_buffer, + coerce_int96_timestamp_unit=coerce_int96_timestamp_unit, + decryption_properties=decryption_properties, + thrift_string_size_limit=thrift_string_size_limit, + thrift_container_size_limit=thrift_container_size_limit, + page_checksum_verification=page_checksum_verification, + arrow_extensions_enabled=arrow_extensions_enabled, + ) + except ImportError: + # fall back on ParquetFile for simple cases when pyarrow.dataset + # module is not available + if filters is not None: + raise ValueError( + "the 'filters' keyword is not supported when the " + "pyarrow.dataset module is not available" + ) + if partitioning != "hive": + raise ValueError( + "the 'partitioning' keyword is not supported when the " + "pyarrow.dataset module is not available" + ) + if schema is not None: + raise ValueError( + "the 'schema' argument is not supported when the " + "pyarrow.dataset module is not available" + ) + if isinstance(source, list): + raise ValueError( + "the 'source' argument cannot be a list of files " + "when the pyarrow.dataset module is not available" + ) + + filesystem, path = _resolve_filesystem_and_path(source, filesystem) + if filesystem is not None: + if not filesystem.get_file_info(path).is_file: + raise ValueError( + "the 'source' argument should be " + "an existing parquet file and not a directory " + "when the pyarrow.dataset module is not available" + ) + + source = filesystem.open_input_file(path) + + dataset = ParquetFile( + source, read_dictionary=read_dictionary, + binary_type=binary_type, + list_type=list_type, + memory_map=memory_map, buffer_size=buffer_size, + pre_buffer=pre_buffer, + coerce_int96_timestamp_unit=coerce_int96_timestamp_unit, + decryption_properties=decryption_properties, + thrift_string_size_limit=thrift_string_size_limit, + thrift_container_size_limit=thrift_container_size_limit, + page_checksum_verification=page_checksum_verification, + ) + + return dataset.read(columns=columns, use_threads=use_threads, + use_pandas_metadata=use_pandas_metadata) + + +read_table.__doc__ = _read_table_docstring.format( + """Read a Table from Parquet format""", + "\n".join(("""use_pandas_metadata : bool, default False + If True and file has custom pandas schema metadata, ensure that + index columns are also loaded.""", _read_docstring_common)), + """pyarrow.Table + Content of the file as a table (of columns)""", + _DNF_filter_doc, _read_table_example) + + +def read_pandas(source, columns=None, **kwargs): + return read_table( + source, columns=columns, use_pandas_metadata=True, **kwargs + ) + + +read_pandas.__doc__ = _read_table_docstring.format( + 'Read a Table from Parquet format, also reading DataFrame\n' + 'index values if known in the file metadata', + "\n".join((_read_docstring_common, + """**kwargs + additional options for :func:`read_table`""")), + """pyarrow.Table + Content of the file as a Table of Columns, including DataFrame + indexes as columns""", + _DNF_filter_doc, "") + + +def write_table(table, where, row_group_size=None, version='2.6', + use_dictionary=True, compression='snappy', + write_statistics=True, + use_deprecated_int96_timestamps=None, + coerce_timestamps=None, + allow_truncated_timestamps=False, + data_page_size=None, flavor=None, + filesystem=None, + compression_level=None, + use_byte_stream_split=False, + column_encoding=None, + data_page_version='1.0', + use_compliant_nested_type=True, + encryption_properties=None, + write_batch_size=None, + dictionary_pagesize_limit=None, + store_schema=True, + write_page_index=False, + write_page_checksum=False, + sorting_columns=None, + store_decimal_as_integer=False, + write_time_adjusted_to_utc=False, + max_rows_per_page=None, + bloom_filter_options=None, + **kwargs): + # Implementor's note: when adding keywords here / updating defaults, also + # update it in write_to_dataset and _dataset_parquet.pyx ParquetFileWriteOptions + row_group_size = kwargs.pop('chunk_size', row_group_size) + use_int96 = use_deprecated_int96_timestamps + try: + with ParquetWriter( + where, table.schema, + filesystem=filesystem, + version=version, + flavor=flavor, + use_dictionary=use_dictionary, + write_statistics=write_statistics, + coerce_timestamps=coerce_timestamps, + data_page_size=data_page_size, + allow_truncated_timestamps=allow_truncated_timestamps, + compression=compression, + use_deprecated_int96_timestamps=use_int96, + compression_level=compression_level, + use_byte_stream_split=use_byte_stream_split, + column_encoding=column_encoding, + data_page_version=data_page_version, + use_compliant_nested_type=use_compliant_nested_type, + encryption_properties=encryption_properties, + write_batch_size=write_batch_size, + dictionary_pagesize_limit=dictionary_pagesize_limit, + store_schema=store_schema, + write_page_index=write_page_index, + write_page_checksum=write_page_checksum, + sorting_columns=sorting_columns, + store_decimal_as_integer=store_decimal_as_integer, + write_time_adjusted_to_utc=write_time_adjusted_to_utc, + max_rows_per_page=max_rows_per_page, + bloom_filter_options=bloom_filter_options, + **kwargs) as writer: + writer.write_table(table, row_group_size=row_group_size) + except Exception: + if _is_path_like(where): + try: + os.remove(_stringify_path(where)) + except os.error: + pass + raise + + +_write_table_example = """\ +Generate an example PyArrow Table: + +>>> import pyarrow as pa +>>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], +... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", +... "Brittle stars", "Centipede"]}) + +and write the Table into Parquet file: + +>>> import pyarrow.parquet as pq +>>> pq.write_table(table, 'example.parquet') + +Defining row group size for the Parquet file: + +>>> pq.write_table(table, 'example.parquet', row_group_size=3) + +Defining row group compression (default is Snappy): + +>>> pq.write_table(table, 'example.parquet', compression='none') + +Defining row group compression and encoding per-column: + +>>> pq.write_table(table, 'example.parquet', +... compression={'n_legs': 'snappy', 'animal': 'gzip'}, +... use_dictionary=['n_legs', 'animal']) + +Defining column encoding per-column: + +>>> pq.write_table(table, 'example.parquet', +... column_encoding={'animal':'PLAIN'}, +... use_dictionary=False) +""" + +write_table.__doc__ = f""" +Write a Table to Parquet format. + +Parameters +---------- +table : pyarrow.Table +where : string or pyarrow.NativeFile +row_group_size : int, default None + Maximum number of rows in each written row group. If None, the + row group size will be the minimum of the Table size (in rows) + and 1024 * 1024. If set larger than 64 * 1024 * 1024 then + 64 * 1024 * 1024 will be used instead. +{_parquet_writer_arg_docs} +**kwargs : optional + Additional options for ParquetWriter + +Examples +-------- +{_write_table_example} +""" + + +def write_to_dataset(table, root_path, partition_cols=None, + filesystem=None, schema=None, partitioning=None, + basename_template=None, use_threads=None, + file_visitor=None, existing_data_behavior=None, + **kwargs): + """Wrapper around dataset.write_dataset for writing a Table to + Parquet format by partitions. + For each combination of partition columns and values, + a subdirectories are created in the following + manner: + + root_dir/ + group1=value1 + group2=value1 + .parquet + group2=value2 + .parquet + group1=valueN + group2=value1 + .parquet + group2=valueN + .parquet + + Parameters + ---------- + table : pyarrow.Table + root_path : str, pathlib.Path + The root directory of the dataset. + partition_cols : list, + Column names by which to partition the dataset. + Columns are partitioned in the order they are given. + filesystem : FileSystem, default None + If nothing passed, will be inferred based on path. + Path will try to be found in the local on-disk filesystem otherwise + it will be parsed as an URI to determine the filesystem. + schema : Schema, optional + This Schema of the dataset. + partitioning : Partitioning or list[str], optional + The partitioning scheme specified with the + ``pyarrow.dataset.partitioning()`` function or a list of field names. + When providing a list of field names, you can use + ``partitioning_flavor`` to drive which partitioning type should be + used. + basename_template : str, optional + A template string used to generate basenames of written data files. + The token '{i}' will be replaced with an automatically incremented + integer. If not specified, it defaults to "guid-{i}.parquet". + use_threads : bool, default True + Write files in parallel. If enabled, then maximum parallelism will be + used determined by the number of available CPU cores. + file_visitor : function + If set, this function will be called with a WrittenFile instance + for each file created during the call. This object will have both + a path attribute and a metadata attribute. + + The path attribute will be a string containing the path to + the created file. + + The metadata attribute will be the parquet metadata of the file. + This metadata will have the file path attribute set and can be used + to build a _metadata file. The metadata attribute will be None if + the format is not parquet. + + Example visitor which simple collects the filenames created:: + + visited_paths = [] + + def file_visitor(written_file): + visited_paths.append(written_file.path) + + existing_data_behavior : 'overwrite_or_ignore' | 'error' | \ +'delete_matching' + Controls how the dataset will handle data that already exists in + the destination. The default behaviour is 'overwrite_or_ignore'. + + 'overwrite_or_ignore' will ignore any existing data and will + overwrite files with the same name as an output file. Other + existing files will be ignored. This behavior, in combination + with a unique basename_template for each write, will allow for + an append workflow. + + 'error' will raise an error if any data exists in the destination. + + 'delete_matching' is useful when you are writing a partitioned + dataset. The first time each partition directory is encountered + the entire directory will be deleted. This allows you to overwrite + old partitions completely. + **kwargs : dict, + Used as additional kwargs for :func:`pyarrow.dataset.write_dataset` + function for matching kwargs, and remainder to + :func:`pyarrow.dataset.ParquetFileFormat.make_write_options`. + See the docstring of :func:`write_table` and + :func:`pyarrow.dataset.write_dataset` for the available options. + Using `metadata_collector` in kwargs allows one to collect the + file metadata instances of dataset pieces. The file paths in the + ColumnChunkMetaData will be set relative to `root_path`. + + Examples + -------- + Generate an example PyArrow Table: + + >>> import pyarrow as pa + >>> table = pa.table({'year': [2020, 2022, 2021, 2022, 2019, 2021], + ... 'n_legs': [2, 2, 4, 4, 5, 100], + ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", + ... "Brittle stars", "Centipede"]}) + + and write it to a partitioned dataset: + + >>> import pyarrow.parquet as pq + >>> pq.write_to_dataset(table, root_path='dataset_name_3', + ... partition_cols=['year']) + >>> pq.ParquetDataset('dataset_name_3').files + ['dataset_name_3/year=2019/...-0.parquet', ... + + Write a single Parquet file into the root folder: + + >>> pq.write_to_dataset(table, root_path='dataset_name_4') + >>> pq.ParquetDataset('dataset_name_4/').files + ['dataset_name_4/...-0.parquet'] + """ + metadata_collector = kwargs.pop('metadata_collector', None) + + # Check for conflicting keywords + msg_confl = ( + "The '{1}' argument is not supported. " + "Use only '{0}' instead." + ) + if partition_cols is not None and partitioning is not None: + raise ValueError(msg_confl.format("partitioning", + "partition_cols")) + + if metadata_collector is not None and file_visitor is not None: + raise ValueError(msg_confl.format("file_visitor", + "metadata_collector")) + + import pyarrow.dataset as ds + + # extract write_dataset specific options + # reset assumed to go to make_write_options + write_dataset_kwargs = dict() + for key in inspect.signature(ds.write_dataset).parameters: + if key in kwargs: + write_dataset_kwargs[key] = kwargs.pop(key) + write_dataset_kwargs['max_rows_per_group'] = kwargs.pop( + 'row_group_size', kwargs.pop("chunk_size", None) + ) + + if metadata_collector is not None: + def file_visitor(written_file): + metadata_collector.append(written_file.metadata) + + # map format arguments + parquet_format = ds.ParquetFileFormat() + write_options = parquet_format.make_write_options(**kwargs) + + # map old filesystems to new one + if filesystem is not None: + filesystem = _ensure_filesystem(filesystem) + + if partition_cols: + part_schema = table.select(partition_cols).schema + partitioning = ds.partitioning(part_schema, flavor="hive") + + if basename_template is None: + basename_template = guid() + '-{i}.parquet' + + if existing_data_behavior is None: + existing_data_behavior = 'overwrite_or_ignore' + + ds.write_dataset( + table, root_path, filesystem=filesystem, + format=parquet_format, file_options=write_options, schema=schema, + partitioning=partitioning, use_threads=use_threads, + file_visitor=file_visitor, + basename_template=basename_template, + existing_data_behavior=existing_data_behavior, + **write_dataset_kwargs) + return + + +def write_metadata(schema, where, metadata_collector=None, filesystem=None, + **kwargs): + """ + Write metadata-only Parquet file from schema. This can be used with + `write_to_dataset` to generate `_common_metadata` and `_metadata` sidecar + files. + + Parameters + ---------- + schema : pyarrow.Schema + where : string or pyarrow.NativeFile + metadata_collector : list + where to collect metadata information. + filesystem : FileSystem, default None + If nothing passed, will be inferred from `where` if path-like, else + `where` is already a file-like object so no filesystem is needed. + **kwargs : dict, + Additional kwargs for ParquetWriter class. See docstring for + `ParquetWriter` for more information. + + Examples + -------- + Generate example data: + + >>> import pyarrow as pa + >>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], + ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", + ... "Brittle stars", "Centipede"]}) + + Write a dataset and collect metadata information. + + >>> metadata_collector = [] + >>> import pyarrow.parquet as pq + >>> pq.write_to_dataset( + ... table, 'dataset_metadata', + ... metadata_collector=metadata_collector) + + Write the `_common_metadata` parquet file without row groups statistics. + + >>> pq.write_metadata( + ... table.schema, 'dataset_metadata/_common_metadata') + + Write the `_metadata` parquet file with row groups statistics. + + >>> pq.write_metadata( + ... table.schema, 'dataset_metadata/_metadata', + ... metadata_collector=metadata_collector) + """ + filesystem, where = _resolve_filesystem_and_path(where, filesystem) + + if hasattr(where, "seek"): # file-like + cursor_position = where.tell() + + writer = ParquetWriter(where, schema, filesystem, **kwargs) + writer.close() + + if metadata_collector is not None: + # ParquetWriter doesn't expose the metadata until it's written. Write + # it and read it again. + metadata = read_metadata(where, filesystem=filesystem) + if hasattr(where, "seek"): + where.seek(cursor_position) # file-like, set cursor back. + + for m in metadata_collector: + metadata.append_row_groups(m) + if filesystem is not None: + with filesystem.open_output_stream(where) as f: + metadata.write_metadata_file(f) + else: + metadata.write_metadata_file(where) + + +def read_metadata(where, memory_map=False, decryption_properties=None, + filesystem=None): + """ + Read FileMetaData from footer of a single Parquet file. + + Parameters + ---------- + where : str (file path) or file-like object + memory_map : bool, default False + Create memory map when the source is a file path. + decryption_properties : FileDecryptionProperties, default None + Decryption properties for reading encrypted Parquet files. + filesystem : FileSystem, default None + If nothing passed, will be inferred based on path. + Path will try to be found in the local on-disk filesystem otherwise + it will be parsed as an URI to determine the filesystem. + + Returns + ------- + metadata : FileMetaData + The metadata of the Parquet file + + Examples + -------- + >>> import pyarrow as pa + >>> import pyarrow.parquet as pq + >>> table = pa.table({'n_legs': [4, 5, 100], + ... 'animal': ["Dog", "Brittle stars", "Centipede"]}) + >>> pq.write_table(table, 'example.parquet') + + >>> pq.read_metadata('example.parquet') + + created_by: parquet-cpp-arrow version ... + num_columns: 2 + num_rows: 3 + num_row_groups: 1 + format_version: 2.6 + serialized_size: ... + """ + filesystem, where = _resolve_filesystem_and_path(where, filesystem) + file_ctx = nullcontext() + if filesystem is not None: + file_ctx = where = filesystem.open_input_file(where) + + with file_ctx: + file = ParquetFile(where, memory_map=memory_map, + decryption_properties=decryption_properties) + return file.metadata + + +def read_schema(where, memory_map=False, decryption_properties=None, + filesystem=None): + """ + Read effective Arrow schema from Parquet file metadata. + + Parameters + ---------- + where : str (file path) or file-like object + memory_map : bool, default False + Create memory map when the source is a file path. + decryption_properties : FileDecryptionProperties, default None + Decryption properties for reading encrypted Parquet files. + filesystem : FileSystem, default None + If nothing passed, will be inferred based on path. + Path will try to be found in the local on-disk filesystem otherwise + it will be parsed as an URI to determine the filesystem. + + Returns + ------- + schema : pyarrow.Schema + The schema of the Parquet file + + Examples + -------- + >>> import pyarrow as pa + >>> import pyarrow.parquet as pq + >>> table = pa.table({'n_legs': [4, 5, 100], + ... 'animal': ["Dog", "Brittle stars", "Centipede"]}) + >>> pq.write_table(table, 'example.parquet') + + >>> pq.read_schema('example.parquet') + n_legs: int64 + animal: string + """ + filesystem, where = _resolve_filesystem_and_path(where, filesystem) + file_ctx = nullcontext() + if filesystem is not None: + file_ctx = where = filesystem.open_input_file(where) + + with file_ctx: + file = ParquetFile( + where, memory_map=memory_map, + decryption_properties=decryption_properties) + return file.schema.to_arrow_schema() + + +__all__ = ( + "ColumnChunkMetaData", + "ColumnSchema", + "FileDecryptionProperties", + "FileEncryptionProperties", + "FileMetaData", + "ParquetDataset", + "ParquetFile", + "ParquetLogicalType", + "ParquetReader", + "ParquetSchema", + "ParquetWriter", + "RowGroupMetaData", + "SortingColumn", + "Statistics", + "read_metadata", + "read_pandas", + "read_schema", + "read_table", + "write_metadata", + "write_table", + "write_to_dataset", + "_filters_to_expression", + "filters_to_expression", +) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/parquet/encryption.py b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/parquet/encryption.py new file mode 100644 index 0000000000000000000000000000000000000000..ebbdda0f05dd47120e6de69658c0d3622b4d3154 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/parquet/encryption.py @@ -0,0 +1,23 @@ +# pylint: disable=unused-wildcard-import, unused-import + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from pyarrow._parquet_encryption import (CryptoFactory, # noqa + EncryptionConfiguration, + DecryptionConfiguration, + KmsConnectionConfig, + KmsClient) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/public-api.pxi b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/public-api.pxi new file mode 100644 index 0000000000000000000000000000000000000000..72686ebb9206229f9c8cd8477991f56f7cf209ff --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/public-api.pxi @@ -0,0 +1,443 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from libcpp.memory cimport shared_ptr +from pyarrow.includes.libarrow cimport (CArray, CDataType, CField, + CRecordBatch, CSchema, + CTable, CTensor, + CSparseCOOTensor, CSparseCSRMatrix, + CSparseCSCMatrix, CSparseCSFTensor) + +# You cannot assign something to a dereferenced pointer in Cython thus these +# methods don't use Status to indicate a successful operation. + + +cdef api bint pyarrow_is_buffer(object buffer): + return isinstance(buffer, Buffer) + + +cdef api shared_ptr[CBuffer] pyarrow_unwrap_buffer(object buffer): + cdef Buffer buf + if pyarrow_is_buffer(buffer): + buf = (buffer) + return buf.buffer + + return shared_ptr[CBuffer]() + + +cdef api object pyarrow_wrap_buffer(const shared_ptr[CBuffer]& buf): + cdef Buffer result = Buffer.__new__(Buffer) + result.init(buf) + return result + + +cdef api object pyarrow_wrap_resizable_buffer( + const shared_ptr[CResizableBuffer]& buf): + cdef ResizableBuffer result = ResizableBuffer.__new__(ResizableBuffer) + result.init_rz(buf) + return result + + +cdef api bint pyarrow_is_data_type(object type_): + return isinstance(type_, DataType) + + +cdef api shared_ptr[CDataType] pyarrow_unwrap_data_type( + object data_type): + cdef DataType type_ + if pyarrow_is_data_type(data_type): + type_ = (data_type) + return type_.sp_type + + return shared_ptr[CDataType]() + + +# Workaround for Cython parsing bug +# https://github.com/cython/cython/issues/2143 +ctypedef const CPyExtensionType* _CPyExtensionTypePtr + + +cdef api object pyarrow_wrap_data_type( + const shared_ptr[CDataType]& type): + cdef: + const CExtensionType* ext_type + const CPyExtensionType* cpy_ext_type + DataType out + + if type.get() == NULL: + return None + + if type.get().id() == _Type_DICTIONARY: + out = DictionaryType.__new__(DictionaryType) + elif type.get().id() == _Type_LIST: + out = ListType.__new__(ListType) + elif type.get().id() == _Type_LARGE_LIST: + out = LargeListType.__new__(LargeListType) + elif type.get().id() == _Type_LIST_VIEW: + out = ListViewType.__new__(ListViewType) + elif type.get().id() == _Type_LARGE_LIST_VIEW: + out = LargeListViewType.__new__(LargeListViewType) + elif type.get().id() == _Type_MAP: + out = MapType.__new__(MapType) + elif type.get().id() == _Type_FIXED_SIZE_LIST: + out = FixedSizeListType.__new__(FixedSizeListType) + elif type.get().id() == _Type_STRUCT: + out = StructType.__new__(StructType) + elif type.get().id() == _Type_SPARSE_UNION: + out = SparseUnionType.__new__(SparseUnionType) + elif type.get().id() == _Type_DENSE_UNION: + out = DenseUnionType.__new__(DenseUnionType) + elif type.get().id() == _Type_TIME32: + out = Time32Type.__new__(Time32Type) + elif type.get().id() == _Type_TIME64: + out = Time64Type.__new__(Time64Type) + elif type.get().id() == _Type_TIMESTAMP: + out = TimestampType.__new__(TimestampType) + elif type.get().id() == _Type_DURATION: + out = DurationType.__new__(DurationType) + elif type.get().id() == _Type_FIXED_SIZE_BINARY: + out = FixedSizeBinaryType.__new__(FixedSizeBinaryType) + elif type.get().id() == _Type_DECIMAL32: + out = Decimal32Type.__new__(Decimal32Type) + elif type.get().id() == _Type_DECIMAL64: + out = Decimal64Type.__new__(Decimal64Type) + elif type.get().id() == _Type_DECIMAL128: + out = Decimal128Type.__new__(Decimal128Type) + elif type.get().id() == _Type_DECIMAL256: + out = Decimal256Type.__new__(Decimal256Type) + elif type.get().id() == _Type_RUN_END_ENCODED: + out = RunEndEncodedType.__new__(RunEndEncodedType) + elif type.get().id() == _Type_EXTENSION: + ext_type = type.get() + cpy_ext_type = dynamic_cast[_CPyExtensionTypePtr](ext_type) + extension_name = ext_type.extension_name() + if cpy_ext_type != nullptr: + return cpy_ext_type.GetInstance() + elif extension_name == b"arrow.bool8": + out = Bool8Type.__new__(Bool8Type) + elif extension_name == b"arrow.fixed_shape_tensor": + out = FixedShapeTensorType.__new__(FixedShapeTensorType) + elif extension_name == b"arrow.opaque": + out = OpaqueType.__new__(OpaqueType) + elif extension_name == b"arrow.uuid": + out = UuidType.__new__(UuidType) + elif extension_name == b"arrow.json": + out = JsonType.__new__(JsonType) + else: + out = BaseExtensionType.__new__(BaseExtensionType) + else: + out = DataType.__new__(DataType) + + out.init(type) + return out + + +cdef object pyarrow_wrap_metadata( + const shared_ptr[const CKeyValueMetadata]& meta): + if meta.get() == nullptr: + return None + else: + return KeyValueMetadata.wrap(meta) + + +cdef api bint pyarrow_is_metadata(object metadata): + return isinstance(metadata, KeyValueMetadata) + + +cdef shared_ptr[const CKeyValueMetadata] pyarrow_unwrap_metadata(object meta): + cdef shared_ptr[const CKeyValueMetadata] c_meta + if pyarrow_is_metadata(meta): + c_meta = (meta).unwrap() + return c_meta + + +cdef api bint pyarrow_is_field(object field): + return isinstance(field, Field) + + +cdef api shared_ptr[CField] pyarrow_unwrap_field(object field): + cdef Field field_ + if pyarrow_is_field(field): + field_ = (field) + return field_.sp_field + + return shared_ptr[CField]() + + +cdef api object pyarrow_wrap_field(const shared_ptr[CField]& field): + if field.get() == NULL: + return None + cdef Field out = Field.__new__(Field) + out.init(field) + return out + + +cdef api bint pyarrow_is_schema(object schema): + return isinstance(schema, Schema) + + +cdef api shared_ptr[CSchema] pyarrow_unwrap_schema(object schema): + cdef Schema sch + if pyarrow_is_schema(schema): + sch = (schema) + return sch.sp_schema + + return shared_ptr[CSchema]() + + +cdef api object pyarrow_wrap_schema(const shared_ptr[CSchema]& schema): + cdef Schema out = Schema.__new__(Schema) + out.init_schema(schema) + return out + + +cdef api bint pyarrow_is_array(object array): + return isinstance(array, Array) + + +cdef api shared_ptr[CArray] pyarrow_unwrap_array(object array): + cdef Array arr + if pyarrow_is_array(array): + arr = (array) + return arr.sp_array + + return shared_ptr[CArray]() + + +cdef api object pyarrow_wrap_array(const shared_ptr[CArray]& sp_array): + if sp_array.get() == NULL: + raise ValueError('Array was NULL') + + klass = get_array_class_from_type(sp_array.get().type()) + + cdef Array arr = klass.__new__(klass) + arr.init(sp_array) + return arr + + +cdef api bint pyarrow_is_chunked_array(object array): + return isinstance(array, ChunkedArray) + + +cdef api shared_ptr[CChunkedArray] pyarrow_unwrap_chunked_array(object array): + cdef ChunkedArray arr + if pyarrow_is_chunked_array(array): + arr = (array) + return arr.sp_chunked_array + + return shared_ptr[CChunkedArray]() + + +cdef api object pyarrow_wrap_chunked_array( + const shared_ptr[CChunkedArray]& sp_array): + if sp_array.get() == NULL: + raise ValueError('ChunkedArray was NULL') + + cdef CDataType* data_type = sp_array.get().type().get() + + if data_type == NULL: + raise ValueError('ChunkedArray data type was NULL') + + cdef ChunkedArray arr = ChunkedArray.__new__(ChunkedArray) + arr.init(sp_array) + return arr + + +cdef api bint pyarrow_is_scalar(object value): + return isinstance(value, Scalar) + + +cdef api shared_ptr[CScalar] pyarrow_unwrap_scalar(object scalar): + if pyarrow_is_scalar(scalar): + return ( scalar).unwrap() + return shared_ptr[CScalar]() + + +cdef api object pyarrow_wrap_scalar(const shared_ptr[CScalar]& sp_scalar): + if sp_scalar.get() == NULL: + raise ValueError('Scalar was NULL') + + cdef CDataType* data_type = sp_scalar.get().type.get() + + if data_type == NULL: + raise ValueError('Scalar data type was NULL') + + if data_type.id() == _Type_NA: + return _NULL + + if data_type.id() not in _scalar_classes: + raise ValueError('Scalar type not supported') + + klass = get_scalar_class_from_type(sp_scalar.get().type) + + cdef Scalar scalar = klass.__new__(klass) + scalar.init(sp_scalar) + return scalar + + +cdef api bint pyarrow_is_tensor(object tensor): + return isinstance(tensor, Tensor) + + +cdef api shared_ptr[CTensor] pyarrow_unwrap_tensor(object tensor): + cdef Tensor ten + if pyarrow_is_tensor(tensor): + ten = (tensor) + return ten.sp_tensor + + return shared_ptr[CTensor]() + + +cdef api object pyarrow_wrap_tensor( + const shared_ptr[CTensor]& sp_tensor): + if sp_tensor.get() == NULL: + raise ValueError('Tensor was NULL') + + cdef Tensor tensor = Tensor.__new__(Tensor) + tensor.init(sp_tensor) + return tensor + + +cdef api bint pyarrow_is_sparse_coo_tensor(object sparse_tensor): + return isinstance(sparse_tensor, SparseCOOTensor) + +cdef api shared_ptr[CSparseCOOTensor] pyarrow_unwrap_sparse_coo_tensor( + object sparse_tensor): + cdef SparseCOOTensor sten + if pyarrow_is_sparse_coo_tensor(sparse_tensor): + sten = (sparse_tensor) + return sten.sp_sparse_tensor + + return shared_ptr[CSparseCOOTensor]() + +cdef api object pyarrow_wrap_sparse_coo_tensor( + const shared_ptr[CSparseCOOTensor]& sp_sparse_tensor): + if sp_sparse_tensor.get() == NULL: + raise ValueError('SparseCOOTensor was NULL') + + cdef SparseCOOTensor sparse_tensor = SparseCOOTensor.__new__( + SparseCOOTensor) + sparse_tensor.init(sp_sparse_tensor) + return sparse_tensor + + +cdef api bint pyarrow_is_sparse_csr_matrix(object sparse_tensor): + return isinstance(sparse_tensor, SparseCSRMatrix) + +cdef api shared_ptr[CSparseCSRMatrix] pyarrow_unwrap_sparse_csr_matrix( + object sparse_tensor): + cdef SparseCSRMatrix sten + if pyarrow_is_sparse_csr_matrix(sparse_tensor): + sten = (sparse_tensor) + return sten.sp_sparse_tensor + + return shared_ptr[CSparseCSRMatrix]() + +cdef api object pyarrow_wrap_sparse_csr_matrix( + const shared_ptr[CSparseCSRMatrix]& sp_sparse_tensor): + if sp_sparse_tensor.get() == NULL: + raise ValueError('SparseCSRMatrix was NULL') + + cdef SparseCSRMatrix sparse_tensor = SparseCSRMatrix.__new__( + SparseCSRMatrix) + sparse_tensor.init(sp_sparse_tensor) + return sparse_tensor + + +cdef api bint pyarrow_is_sparse_csc_matrix(object sparse_tensor): + return isinstance(sparse_tensor, SparseCSCMatrix) + +cdef api shared_ptr[CSparseCSCMatrix] pyarrow_unwrap_sparse_csc_matrix( + object sparse_tensor): + cdef SparseCSCMatrix sten + if pyarrow_is_sparse_csc_matrix(sparse_tensor): + sten = (sparse_tensor) + return sten.sp_sparse_tensor + + return shared_ptr[CSparseCSCMatrix]() + +cdef api object pyarrow_wrap_sparse_csc_matrix( + const shared_ptr[CSparseCSCMatrix]& sp_sparse_tensor): + if sp_sparse_tensor.get() == NULL: + raise ValueError('SparseCSCMatrix was NULL') + + cdef SparseCSCMatrix sparse_tensor = SparseCSCMatrix.__new__( + SparseCSCMatrix) + sparse_tensor.init(sp_sparse_tensor) + return sparse_tensor + + +cdef api bint pyarrow_is_sparse_csf_tensor(object sparse_tensor): + return isinstance(sparse_tensor, SparseCSFTensor) + +cdef api shared_ptr[CSparseCSFTensor] pyarrow_unwrap_sparse_csf_tensor( + object sparse_tensor): + cdef SparseCSFTensor sten + if pyarrow_is_sparse_csf_tensor(sparse_tensor): + sten = (sparse_tensor) + return sten.sp_sparse_tensor + + return shared_ptr[CSparseCSFTensor]() + +cdef api object pyarrow_wrap_sparse_csf_tensor( + const shared_ptr[CSparseCSFTensor]& sp_sparse_tensor): + if sp_sparse_tensor.get() == NULL: + raise ValueError('SparseCSFTensor was NULL') + + cdef SparseCSFTensor sparse_tensor = SparseCSFTensor.__new__( + SparseCSFTensor) + sparse_tensor.init(sp_sparse_tensor) + return sparse_tensor + + +cdef api bint pyarrow_is_table(object table): + return isinstance(table, Table) + + +cdef api shared_ptr[CTable] pyarrow_unwrap_table(object table): + cdef Table tab + if pyarrow_is_table(table): + tab = (table) + return tab.sp_table + + return shared_ptr[CTable]() + + +cdef api object pyarrow_wrap_table(const shared_ptr[CTable]& ctable): + cdef Table table = Table.__new__(Table) + table.init(ctable) + return table + + +cdef api bint pyarrow_is_batch(object batch): + return isinstance(batch, RecordBatch) + + +cdef api shared_ptr[CRecordBatch] pyarrow_unwrap_batch(object batch): + cdef RecordBatch bat + if pyarrow_is_batch(batch): + bat = (batch) + return bat.sp_batch + + return shared_ptr[CRecordBatch]() + + +cdef api object pyarrow_wrap_batch( + const shared_ptr[CRecordBatch]& cbatch): + cdef RecordBatch batch = RecordBatch.__new__(RecordBatch) + batch.init(cbatch) + return batch diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/py.typed b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e0533d99236c221d78ae010813e267296c94eb6a --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/py.typed @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/scalar.pxi b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/scalar.pxi new file mode 100644 index 0000000000000000000000000000000000000000..943966095ba621109dbe2ad5cbb23a69d0e8995a --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/scalar.pxi @@ -0,0 +1,1748 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import collections +import warnings +from uuid import UUID +from collections.abc import Sequence, Mapping + + +cdef class Scalar(_Weakrefable): + """ + The base class for scalars. + + Notes + ----- + This class supports Python's standard operators + for element-wise operations, i.e. arithmetic (`+`, `-`, `/`, `%`, `**`), + bitwise (`&`, `|`, `^`, `>>`, `<<`) and others. + They can be used directly instead of calling underlying + `pyarrow.compute` functions explicitly. + + Examples + -------- + >>> import pyarrow as pa + >>> pa.scalar(42) + pa.scalar(17) + + + >>> pa.scalar(6) ** 3 + + + >>> arr = pa.array([1, 2, 3], type=pa.int8()) + >>> val = pa.scalar(42) + >>> val - arr + + [ + 41, + 40, + 39 + ] + """ + + def __init__(self): + raise TypeError(f"Do not call {self.__class__.__name__}'s constructor directly, " + "use pa.scalar() instead.") + + cdef void init(self, const shared_ptr[CScalar]& wrapped): + self.wrapped = wrapped + + @staticmethod + cdef wrap(const shared_ptr[CScalar]& wrapped): + cdef: + Scalar self + Type type_id = wrapped.get().type.get().id() + shared_ptr[CDataType] sp_data_type = wrapped.get().type + + if type_id == _Type_NA: + return _NULL + + if type_id not in _scalar_classes: + raise NotImplementedError( + "Wrapping scalar of type " + frombytes(sp_data_type.get().ToString())) + + typ = get_scalar_class_from_type(sp_data_type) + self = typ.__new__(typ) + self.init(wrapped) + + return self + + cdef inline shared_ptr[CScalar] unwrap(self) nogil: + return self.wrapped + + @property + def type(self): + """ + Data type of the Scalar object. + """ + return pyarrow_wrap_data_type(self.wrapped.get().type) + + @property + def is_valid(self): + """ + Holds a valid (non-null) value. + """ + return self.wrapped.get().is_valid + + def cast(self, object target_type=None, safe=None, options=None, memory_pool=None): + """ + Cast scalar value to another data type. + + See :func:`pyarrow.compute.cast` for usage. + + Parameters + ---------- + target_type : DataType, default None + Type to cast scalar to. + safe : boolean, default True + Whether to check for conversion errors such as overflow. + options : CastOptions, default None + Additional checks pass by CastOptions + memory_pool : MemoryPool, optional + memory pool to use for allocations during function execution. + + Returns + ------- + scalar : A Scalar of the given target data type. + """ + return _pc().cast(self, target_type, safe=safe, + options=options, memory_pool=memory_pool) + + def validate(self, *, full=False): + """ + Perform validation checks. An exception is raised if validation fails. + + By default only cheap validation checks are run. Pass `full=True` + for thorough validation checks (potentially O(n)). + + Parameters + ---------- + full : bool, default False + If True, run expensive checks, otherwise cheap checks only. + + Raises + ------ + ArrowInvalid + """ + if full: + with nogil: + check_status(self.wrapped.get().ValidateFull()) + else: + with nogil: + check_status(self.wrapped.get().Validate()) + + def __repr__(self): + return f'' + + def __str__(self): + return str(self.as_py()) + + def equals(self, Scalar other not None): + """ + Parameters + ---------- + other : pyarrow.Scalar + + Returns + ------- + bool + """ + return self.wrapped.get().Equals(other.unwrap().get()[0]) + + def __eq__(self, other): + try: + return self.equals(other) + except TypeError: + return NotImplemented + + def __hash__(self): + cdef CScalarHash hasher + return hasher(self.wrapped) + + def __reduce__(self): + return scalar, (self.as_py(), self.type) + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python representation. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + The default behavior (`None`), is to convert Arrow Map arrays to + Python association lists (list-of-tuples) in the same order as the + Arrow Map, as in [(key1, value1), (key2, value2), ...]. + + If 'lossy' or 'strict', convert Arrow Map arrays to native Python dicts. + + If 'lossy', whenever duplicate keys are detected, a warning will be printed. + The last seen value of a duplicate key will be in the Python dictionary. + If 'strict', this instead results in an exception being raised when detected. + """ + raise NotImplementedError() + + def __abs__(self): + return _pc().call_function('abs_checked', [self]) + + def __add__(self, object other): + return _pc().call_function('add_checked', [self, other]) + + def __truediv__(self, object other): + return _pc().call_function('divide_checked', [self, other]) + + def __mul__(self, object other): + return _pc().call_function('multiply_checked', [self, other]) + + def __neg__(self): + return _pc().call_function('negate_checked', [self]) + + def __pow__(self, object other): + return _pc().call_function('power_checked', [self, other]) + + def __sub__(self, object other): + return _pc().call_function('subtract_checked', [self, other]) + + def __and__(self, object other): + return _pc().call_function('bit_wise_and', [self, other]) + + def __or__(self, object other): + return _pc().call_function('bit_wise_or', [self, other]) + + def __xor__(self, object other): + return _pc().call_function('bit_wise_xor', [self, other]) + + def __lshift__(self, object other): + return _pc().call_function('shift_left_checked', [self, other]) + + def __rshift__(self, object other): + return _pc().call_function('shift_right_checked', [self, other]) + + +_NULL = NA = None + + +cdef class NullScalar(Scalar): + """ + Concrete class for null scalars. + """ + + def __cinit__(self): + global NA + if NA is not None: + raise RuntimeError('Cannot create multiple NullScalar instances') + self.init(shared_ptr[CScalar](new CNullScalar())) + + def __init__(self): + pass + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python None. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + return None + + +_NULL = NA = NullScalar() + + +cdef class BooleanScalar(Scalar): + """ + Concrete class for boolean scalars. + """ + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python bool. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + cdef CBooleanScalar* sp = self.wrapped.get() + return sp.value if sp.is_valid else None + + def __bool__(self): + return self.as_py() or False + +cdef class UInt8Scalar(Scalar): + """ + Concrete class for uint8 scalars. + """ + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python int. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + cdef CUInt8Scalar* sp = self.wrapped.get() + return sp.value if sp.is_valid else None + + def __index__(self): + return self.as_py() + + +cdef class Int8Scalar(Scalar): + """ + Concrete class for int8 scalars. + """ + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python int. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + cdef CInt8Scalar* sp = self.wrapped.get() + return sp.value if sp.is_valid else None + + def __index__(self): + return self.as_py() + + +cdef class UInt16Scalar(Scalar): + """ + Concrete class for uint16 scalars. + """ + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python int. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + cdef CUInt16Scalar* sp = self.wrapped.get() + return sp.value if sp.is_valid else None + + def __index__(self): + return self.as_py() + + +cdef class Int16Scalar(Scalar): + """ + Concrete class for int16 scalars. + """ + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python int. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + cdef CInt16Scalar* sp = self.wrapped.get() + return sp.value if sp.is_valid else None + + def __index__(self): + return self.as_py() + + +cdef class UInt32Scalar(Scalar): + """ + Concrete class for uint32 scalars. + """ + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python int. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + cdef CUInt32Scalar* sp = self.wrapped.get() + return sp.value if sp.is_valid else None + + def __index__(self): + return self.as_py() + + +cdef class Int32Scalar(Scalar): + """ + Concrete class for int32 scalars. + """ + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python int. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + cdef CInt32Scalar* sp = self.wrapped.get() + return sp.value if sp.is_valid else None + + def __index__(self): + return self.as_py() + + +cdef class UInt64Scalar(Scalar): + """ + Concrete class for uint64 scalars. + """ + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python int. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + cdef CUInt64Scalar* sp = self.wrapped.get() + return sp.value if sp.is_valid else None + + def __index__(self): + return self.as_py() + + +cdef class Int64Scalar(Scalar): + """ + Concrete class for int64 scalars. + """ + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python int. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + cdef CInt64Scalar* sp = self.wrapped.get() + return sp.value if sp.is_valid else None + + def __index__(self): + return self.as_py() + + +cdef class HalfFloatScalar(Scalar): + """ + Concrete class for float scalars. + """ + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python float. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + cdef CHalfFloatScalar* sp = self.wrapped.get() + return PyFloat_FromHalf(sp.value) if sp.is_valid else None + + def __float__(self): + return self.as_py() + + def __int__(self): + return int(self.as_py()) + + +cdef class FloatScalar(Scalar): + """ + Concrete class for float scalars. + """ + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python float. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + cdef CFloatScalar* sp = self.wrapped.get() + return sp.value if sp.is_valid else None + + def __float__(self): + return self.as_py() + + def __int__(self): + return int(float(self)) + + +cdef class DoubleScalar(Scalar): + """ + Concrete class for double scalars. + """ + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python float. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + cdef CDoubleScalar* sp = self.wrapped.get() + return sp.value if sp.is_valid else None + + def __float__(self): + return self.as_py() + + def __int__(self): + return int(float(self)) + + +cdef class Decimal32Scalar(Scalar): + """ + Concrete class for decimal32 scalars. + """ + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python Decimal. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + cdef: + CDecimal32Scalar* sp = self.wrapped.get() + CDecimal32Type* dtype = sp.type.get() + if sp.is_valid: + return _pydecimal.Decimal( + frombytes(sp.value.ToString(dtype.scale())) + ) + else: + return None + + +cdef class Decimal64Scalar(Scalar): + """ + Concrete class for decimal64 scalars. + """ + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python Decimal. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + cdef: + CDecimal64Scalar* sp = self.wrapped.get() + CDecimal64Type* dtype = sp.type.get() + if sp.is_valid: + return _pydecimal.Decimal( + frombytes(sp.value.ToString(dtype.scale())) + ) + else: + return None + + +cdef class Decimal128Scalar(Scalar): + """ + Concrete class for decimal128 scalars. + """ + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python Decimal. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + cdef: + CDecimal128Scalar* sp = self.wrapped.get() + CDecimal128Type* dtype = sp.type.get() + if sp.is_valid: + return _pydecimal.Decimal( + frombytes(sp.value.ToString(dtype.scale())) + ) + else: + return None + + +cdef class Decimal256Scalar(Scalar): + """ + Concrete class for decimal256 scalars. + """ + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python Decimal. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + cdef: + CDecimal256Scalar* sp = self.wrapped.get() + CDecimal256Type* dtype = sp.type.get() + if sp.is_valid: + return _pydecimal.Decimal( + frombytes(sp.value.ToString(dtype.scale())) + ) + else: + return None + + +cdef class Date32Scalar(Scalar): + """ + Concrete class for date32 scalars. + """ + + @property + def value(self): + cdef CDate32Scalar* sp = self.wrapped.get() + return sp.value if sp.is_valid else None + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python datetime.datetime instance. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + cdef CDate32Scalar* sp = self.wrapped.get() + + if sp.is_valid: + # shift to seconds since epoch + return ( + datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value) + ) + else: + return None + + +cdef class Date64Scalar(Scalar): + """ + Concrete class for date64 scalars. + """ + + @property + def value(self): + cdef CDate64Scalar* sp = self.wrapped.get() + return sp.value if sp.is_valid else None + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python datetime.datetime instance. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + cdef CDate64Scalar* sp = self.wrapped.get() + + if sp.is_valid: + return ( + datetime.date(1970, 1, 1) + + datetime.timedelta(days=sp.value / 86400000) + ) + else: + return None + + +def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None): + if unit == TimeUnit_SECOND: + delta = datetime.timedelta(seconds=value) + elif unit == TimeUnit_MILLI: + delta = datetime.timedelta(milliseconds=value) + elif unit == TimeUnit_MICRO: + delta = datetime.timedelta(microseconds=value) + else: + # TimeUnit_NANO: prefer pandas timestamps if available + if _pandas_api.have_pandas: + return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns') + # otherwise safely truncate to microsecond resolution datetime + if value % 1000 != 0: + raise ValueError( + f"Nanosecond resolution temporal type {value} is not safely " + "convertible to microseconds to convert to datetime.datetime. " + "Install pandas to return as Timestamp with nanosecond " + "support or access the .value attribute." + ) + delta = datetime.timedelta(microseconds=value // 1000) + + dt = datetime.datetime(1970, 1, 1) + delta + # adjust timezone if set to the datatype + if tzinfo is not None: + dt = dt.replace(tzinfo=datetime.timezone.utc).astimezone(tzinfo) + + return dt + + +cdef class Time32Scalar(Scalar): + """ + Concrete class for time32 scalars. + """ + + @property + def value(self): + cdef CTime32Scalar* sp = self.wrapped.get() + return sp.value if sp.is_valid else None + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python datetime.timedelta instance. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + cdef: + CTime32Scalar* sp = self.wrapped.get() + CTime32Type* dtype = sp.type.get() + + if sp.is_valid: + return _datetime_from_int(sp.value, unit=dtype.unit()).time() + else: + return None + + +cdef class Time64Scalar(Scalar): + """ + Concrete class for time64 scalars. + """ + + @property + def value(self): + cdef CTime64Scalar* sp = self.wrapped.get() + return sp.value if sp.is_valid else None + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python datetime.timedelta instance. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + cdef: + CTime64Scalar* sp = self.wrapped.get() + CTime64Type* dtype = sp.type.get() + + if sp.is_valid: + return _datetime_from_int(sp.value, unit=dtype.unit()).time() + else: + return None + + +cdef class TimestampScalar(Scalar): + """ + Concrete class for timestamp scalars. + """ + + @property + def value(self): + cdef CTimestampScalar* sp = self.wrapped.get() + return sp.value if sp.is_valid else None + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Pandas Timestamp instance (if units are + nanoseconds and pandas is available), otherwise as a Python + datetime.datetime instance. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + cdef: + CTimestampScalar* sp = self.wrapped.get() + CTimestampType* dtype = sp.type.get() + + if not sp.is_valid: + return None + + if not dtype.timezone().empty(): + tzinfo = string_to_tzinfo(frombytes(dtype.timezone())) + else: + tzinfo = None + + return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo) + + def __repr__(self): + """ + Return the representation of TimestampScalar using `strftime` to avoid + original repr datetime values being out of range. + """ + cdef: + CTimestampScalar* sp = self.wrapped.get() + CTimestampType* dtype = sp.type.get() + + if not dtype.timezone().empty(): + type_format = str(_pc().strftime(self, format="%Y-%m-%dT%H:%M:%S%z")) + else: + type_format = str(_pc().strftime(self)) + return f'' + + +cdef class DurationScalar(Scalar): + """ + Concrete class for duration scalars. + """ + + @property + def value(self): + cdef CDurationScalar* sp = self.wrapped.get() + return sp.value if sp.is_valid else None + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Pandas Timedelta instance (if units are + nanoseconds and pandas is available), otherwise as a Python + datetime.timedelta instance. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + cdef: + CDurationScalar* sp = self.wrapped.get() + CDurationType* dtype = sp.type.get() + TimeUnit unit = dtype.unit() + + if not sp.is_valid: + return None + + if unit == TimeUnit_SECOND: + return datetime.timedelta(seconds=sp.value) + elif unit == TimeUnit_MILLI: + return datetime.timedelta(milliseconds=sp.value) + elif unit == TimeUnit_MICRO: + return datetime.timedelta(microseconds=sp.value) + else: + # TimeUnit_NANO: prefer pandas timestamps if available + if _pandas_api.have_pandas: + return _pandas_api.pd.Timedelta(sp.value, unit='ns') + # otherwise safely truncate to microsecond resolution timedelta + if sp.value % 1000 != 0: + raise ValueError( + f"Nanosecond duration {sp.value} is not safely convertible to " + "microseconds to convert to datetime.timedelta. Install " + "pandas to return as Timedelta with nanosecond support or " + "access the .value attribute." + ) + return datetime.timedelta(microseconds=sp.value // 1000) + + +cdef class MonthDayNanoIntervalScalar(Scalar): + """ + Concrete class for month, day, nanosecond interval scalars. + """ + + @property + def value(self): + """ + Same as self.as_py() + """ + return self.as_py() + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a pyarrow.MonthDayNano. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + cdef: + PyObject* val + CMonthDayNanoIntervalScalar* scalar + scalar = self.wrapped.get() + val = GetResultValue(MonthDayNanoIntervalScalarToPyObject( + deref(scalar))) + return PyObject_to_object(val) + + +cdef class BinaryScalar(Scalar): + """ + Concrete class for binary-like scalars. + """ + + def as_buffer(self): + """ + Return a view over this value as a Buffer object. + """ + cdef CBaseBinaryScalar* sp = self.wrapped.get() + return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python bytes. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + buffer = self.as_buffer() + return None if buffer is None else buffer.to_pybytes() + + def __bytes__(self): + return self.as_py() + + def __getbuffer__(self, cp.Py_buffer* buffer, int flags): + buf = self.as_buffer() + if buf is None: + raise ValueError("Cannot export buffer from null Arrow Scalar") + cp.PyObject_GetBuffer(buf, buffer, flags) + + +cdef class LargeBinaryScalar(BinaryScalar): + pass + + +cdef class FixedSizeBinaryScalar(BinaryScalar): + pass + + +cdef class StringScalar(BinaryScalar): + """ + Concrete class for string-like (utf8) scalars. + """ + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python string. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + buffer = self.as_buffer() + return None if buffer is None else str(buffer, 'utf8') + + +cdef class LargeStringScalar(StringScalar): + pass + + +cdef class BinaryViewScalar(BinaryScalar): + pass + + +cdef class StringViewScalar(StringScalar): + pass + + +cdef class ListScalar(Scalar, Sequence): + """ + Concrete class for list-like scalars. + """ + + @property + def values(self): + cdef CBaseListScalar* sp = self.wrapped.get() + if sp.is_valid: + return pyarrow_wrap_array(sp.value) + else: + return None + + def __len__(self): + """ + Return the number of values. + """ + return len(self.values) + + def __getitem__(self, i): + """ + Return the value at the given index. + """ + return self.values[_normalize_index(i, len(self))] + + def __iter__(self): + """ + Iterate over this element's values. + """ + return iter(self.values) + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python list. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + The default behavior (`None`), is to convert Arrow Map arrays to + Python association lists (list-of-tuples) in the same order as the + Arrow Map, as in [(key1, value1), (key2, value2), ...]. + + If 'lossy' or 'strict', convert Arrow Map arrays to native Python dicts. + + If 'lossy', whenever duplicate keys are detected, a warning will be printed. + The last seen value of a duplicate key will be in the Python dictionary. + If 'strict', this instead results in an exception being raised when detected. + """ + arr = self.values + return None if arr is None else arr.to_pylist(maps_as_pydicts=maps_as_pydicts) + + +cdef class FixedSizeListScalar(ListScalar): + pass + + +cdef class LargeListScalar(ListScalar): + pass + + +cdef class ListViewScalar(ListScalar): + pass + + +cdef class LargeListViewScalar(ListScalar): + pass + + +cdef class StructScalar(Scalar, Mapping): + """ + Concrete class for struct scalars. + """ + + def __len__(self): + cdef CStructScalar* sp = self.wrapped.get() + return sp.value.size() + + def __iter__(self): + cdef: + CStructScalar* sp = self.wrapped.get() + CStructType* dtype = sp.type.get() + vector[shared_ptr[CField]] fields = dtype.fields() + + for i in range(dtype.num_fields()): + yield frombytes(fields[i].get().name()) + + def items(self): + return ((key, self[i]) for i, key in enumerate(self)) + + def __contains__(self, key): + return key in list(self) + + def __getitem__(self, key): + """ + Return the child value for the given field. + + Parameters + ---------- + index : Union[int, str] + Index / position or name of the field. + + Returns + ------- + result : Scalar + """ + cdef: + CFieldRef ref + CStructScalar* sp = self.wrapped.get() + + if isinstance(key, (bytes, str)): + ref = CFieldRef( tobytes(key)) + elif isinstance(key, int): + ref = CFieldRef( key) + else: + raise TypeError('Expected integer or string index') + + try: + return Scalar.wrap(GetResultValue(sp.field(ref))) + except ArrowInvalid as exc: + if isinstance(key, int): + raise IndexError(key) from exc + else: + raise KeyError(key) from exc + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python dict. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + The default behavior (`None`), is to convert Arrow Map arrays to + Python association lists (list-of-tuples) in the same order as the + Arrow Map, as in [(key1, value1), (key2, value2), ...]. + + If 'lossy' or 'strict', convert Arrow Map arrays to native Python dicts. + + If 'lossy', whenever duplicate keys are detected, a warning will be printed. + The last seen value of a duplicate key will be in the Python dictionary. + If 'strict', this instead results in an exception being raised when detected. + """ + if self.is_valid: + try: + return {k: self[k].as_py(maps_as_pydicts=maps_as_pydicts) for k in self.keys()} + except KeyError: + raise ValueError( + "Converting to Python dictionary is not supported when " + "duplicate field names are present") + else: + return None + + def _as_py_tuple(self): + # a version that returns a tuple instead of dict to support repr/str + # with the presence of duplicate field names + if self.is_valid: + return [(key, self[i].as_py()) for i, key in enumerate(self)] + else: + return None + + def __repr__(self): + return f'' + + def __str__(self): + return str(self._as_py_tuple()) + + +cdef class MapScalar(ListScalar, Mapping): + """ + Concrete class for map scalars. + """ + + def __getitem__(self, i): + """ + Return the value at the given index or key. + """ + + arr = self.values + if arr is None: + raise IndexError(i) if isinstance(i, int) else KeyError(i) + + key_field = self.type.key_field.name + item_field = self.type.item_field.name + + if isinstance(i, (bytes, str)): + try: + key_index = list(self.keys()).index(i) + except ValueError: + raise KeyError(i) + + dct = arr[_normalize_index(key_index, len(arr))] + return dct[item_field] + + dct = arr[_normalize_index(i, len(arr))] + return (dct[key_field], dct[item_field]) + + def __iter__(self): + """ + Iterate over this element's values. + """ + arr = self.values + if arr is None: + return + for k, v in zip(arr.field(self.type.key_field.name), arr.field(self.type.item_field.name)): + yield (k.as_py(), v.as_py()) + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this value as a Python list or dict, depending on 'maps_as_pydicts'. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + The default behavior (`None`), is to convert Arrow Map arrays to + Python association lists (list-of-tuples) in the same order as the + Arrow Map, as in [(key1, value1), (key2, value2), ...]. + + If 'lossy' or 'strict', convert Arrow Map arrays to native Python dicts. + + If 'lossy', whenever duplicate keys are detected, a warning will be printed. + The last seen value of a duplicate key will be in the Python dictionary. + If 'strict', this instead results in an exception being raised when detected. + """ + if maps_as_pydicts not in (None, "lossy", "strict"): + raise ValueError( + "Invalid value for 'maps_as_pydicts': " + + "valid values are 'lossy', 'strict' or `None` (default). " + + f"Received {maps_as_pydicts!r}." + ) + if not self.is_valid: + return None + if not maps_as_pydicts: + return list(self) + result_dict = {} + if self.values is None: + return result_dict + + for key, value in zip(self.keys(), self.values.field(self.type.item_field.name)): + if key in result_dict: + if maps_as_pydicts == "strict": + raise KeyError( + "Converting to Python dictionary is not supported in strict mode " + f"when duplicate keys are present (duplicate key was '{key}')." + ) + else: + warnings.warn( + f"Encountered key '{key}' which was already encountered.") + result_dict[key] = value.as_py(maps_as_pydicts=maps_as_pydicts) + return result_dict + + def keys(self): + """ + Return the keys of the map as a list. + """ + arr = self.values + if arr is None: + return [] + key_field = self.type.key_field.name + return [k.as_py() for k in arr.field(key_field)] + + +cdef class DictionaryScalar(Scalar): + """ + Concrete class for dictionary-encoded scalars. + """ + + @staticmethod + def _reconstruct(type, is_valid, index, dictionary): + cdef: + CDictionaryScalarIndexAndDictionary value + shared_ptr[CDictionaryScalar] wrapped + DataType type_ + Scalar index_ + Array dictionary_ + + type_ = ensure_type(type, allow_none=False) + if not isinstance(type_, DictionaryType): + raise TypeError('Must pass a DictionaryType instance') + + if isinstance(index, Scalar): + if not index.type.equals(type.index_type): + raise TypeError("The Scalar value passed as index must have " + "identical type to the dictionary type's " + "index_type") + index_ = index + else: + index_ = scalar(index, type=type_.index_type) + + if isinstance(dictionary, Array): + if not dictionary.type.equals(type.value_type): + raise TypeError("The Array passed as dictionary must have " + "identical type to the dictionary type's " + "value_type") + dictionary_ = dictionary + else: + dictionary_ = array(dictionary, type=type_.value_type) + + value.index = pyarrow_unwrap_scalar(index_) + value.dictionary = pyarrow_unwrap_array(dictionary_) + + wrapped = make_shared[CDictionaryScalar]( + value, pyarrow_unwrap_data_type(type_), (is_valid) + ) + return Scalar.wrap( wrapped) + + def __reduce__(self): + return DictionaryScalar._reconstruct, ( + self.type, self.is_valid, self.index, self.dictionary + ) + + @property + def index(self): + """ + Return this value's underlying index as a scalar. + """ + cdef CDictionaryScalar* sp = self.wrapped.get() + return Scalar.wrap(sp.value.index) + + @property + def value(self): + """ + Return the encoded value as a scalar. + """ + cdef CDictionaryScalar* sp = self.wrapped.get() + return Scalar.wrap(GetResultValue(sp.GetEncodedValue())) + + @property + def dictionary(self): + cdef CDictionaryScalar* sp = self.wrapped.get() + return pyarrow_wrap_array(sp.value.dictionary) + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this encoded value as a Python object. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + The default behavior (`None`), is to convert Arrow Map arrays to + Python association lists (list-of-tuples) in the same order as the + Arrow Map, as in [(key1, value1), (key2, value2), ...]. + + If 'lossy' or 'strict', convert Arrow Map arrays to native Python dicts. + + If 'lossy', whenever duplicate keys are detected, a warning will be printed. + The last seen value of a duplicate key will be in the Python dictionary. + If 'strict', this instead results in an exception being raised when detected. + """ + return self.value.as_py(maps_as_pydicts=maps_as_pydicts) if self.is_valid else None + + +cdef class RunEndEncodedScalar(Scalar): + """ + Concrete class for RunEndEncoded scalars. + """ + @property + def value(self): + """ + Return underlying value as a scalar. + """ + cdef CRunEndEncodedScalar* sp = self.wrapped.get() + return Scalar.wrap(sp.value) + + def as_py(self, *, maps_as_pydicts=None): + """ + Return underlying value as a Python object. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + The default behavior (`None`), is to convert Arrow Map arrays to + Python association lists (list-of-tuples) in the same order as the + Arrow Map, as in [(key1, value1), (key2, value2), ...]. + + If 'lossy' or 'strict', convert Arrow Map arrays to native Python dicts. + + If 'lossy', whenever duplicate keys are detected, a warning will be printed. + The last seen value of a duplicate key will be in the Python dictionary. + If 'strict', this instead results in an exception being raised when detected. + """ + return self.value.as_py(maps_as_pydicts=maps_as_pydicts) + + +cdef class UnionScalar(Scalar): + """ + Concrete class for Union scalars. + """ + + @property + def value(self): + """ + Return underlying value as a scalar. + """ + cdef CSparseUnionScalar* sp + cdef CDenseUnionScalar* dp + if self.type.id == _Type_SPARSE_UNION: + sp = self.wrapped.get() + return Scalar.wrap(sp.value[sp.child_id]) if sp.is_valid else None + else: + dp = self.wrapped.get() + return Scalar.wrap(dp.value) if dp.is_valid else None + + def as_py(self, *, maps_as_pydicts=None): + """ + Return underlying value as a Python object. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + The default behavior (`None`), is to convert Arrow Map arrays to + Python association lists (list-of-tuples) in the same order as the + Arrow Map, as in [(key1, value1), (key2, value2), ...]. + + If 'lossy' or 'strict', convert Arrow Map arrays to native Python dicts. + + If 'lossy', whenever duplicate keys are detected, a warning will be printed. + The last seen value of a duplicate key will be in the Python dictionary. + If 'strict', this instead results in an exception being raised when detected. + """ + value = self.value + return None if value is None else value.as_py(maps_as_pydicts=maps_as_pydicts) + + @property + def type_code(self): + """ + Return the union type code for this scalar. + """ + cdef CUnionScalar* sp = self.wrapped.get() + return sp.type_code + + +cdef class ExtensionScalar(Scalar): + """ + Concrete class for Extension scalars. + """ + + @property + def value(self): + """ + Return storage value as a scalar. + """ + cdef CExtensionScalar* sp = self.wrapped.get() + return Scalar.wrap(sp.value) if sp.is_valid else None + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this scalar as a Python object. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + The default behavior (`None`), is to convert Arrow Map arrays to + Python association lists (list-of-tuples) in the same order as the + Arrow Map, as in [(key1, value1), (key2, value2), ...]. + + If 'lossy' or 'strict', convert Arrow Map arrays to native Python dicts. + + If 'lossy', whenever duplicate keys are detected, a warning will be printed. + The last seen value of a duplicate key will be in the Python dictionary. + If 'strict', this instead results in an exception being raised when detected. + """ + return None if self.value is None else self.value.as_py(maps_as_pydicts=maps_as_pydicts) + + @staticmethod + def from_storage(BaseExtensionType typ, value): + """ + Construct ExtensionScalar from type and storage value. + + Parameters + ---------- + typ : DataType + The extension type for the result scalar. + value : object + The storage value for the result scalar. + + Returns + ------- + ext_scalar : ExtensionScalar + """ + cdef: + shared_ptr[CExtensionScalar] sp_scalar + shared_ptr[CScalar] sp_storage + CExtensionScalar* ext_scalar + + if value is None: + storage = None + elif isinstance(value, Scalar): + if value.type != typ.storage_type: + raise TypeError(f"Incompatible storage type {value.type} " + f"for extension type {typ}") + storage = value + else: + storage = scalar(value, typ.storage_type) + + cdef c_bool is_valid = storage is not None and storage.is_valid + if is_valid: + sp_storage = pyarrow_unwrap_scalar(storage) + else: + sp_storage = MakeNullScalar(( typ.storage_type).sp_type) + sp_scalar = make_shared[CExtensionScalar](sp_storage, typ.sp_type, + is_valid) + with nogil: + check_status(sp_scalar.get().Validate()) + return pyarrow_wrap_scalar( sp_scalar) + + +class JsonScalar(ExtensionScalar): + """ + Concrete class for JSON extension scalar. + """ + + +class UuidScalar(ExtensionScalar): + """ + Concrete class for Uuid extension scalar. + """ + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this scalar as a Python UUID. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + return None if self.value is None else UUID(bytes=self.value.as_py()) + + +cdef class FixedShapeTensorScalar(ExtensionScalar): + """ + Concrete class for fixed shape tensor extension scalar. + """ + + def to_numpy(self): + """ + Convert fixed shape tensor scalar to a numpy.ndarray. + + The resulting ndarray's shape matches the permuted shape of the + fixed shape tensor scalar. + The conversion is zero-copy. + + Returns + ------- + numpy.ndarray + """ + return self.to_tensor().to_numpy() + + def to_tensor(self): + """ + Convert fixed shape tensor extension scalar to a pyarrow.Tensor, using shape + and strides derived from corresponding FixedShapeTensorType. + + The conversion is zero-copy. + + Returns + ------- + pyarrow.Tensor + Tensor represented stored in FixedShapeTensorScalar. + """ + cdef: + CFixedShapeTensorType* c_type = static_pointer_cast[CFixedShapeTensorType, CDataType]( + self.wrapped.get().type).get() + shared_ptr[CExtensionScalar] scalar = static_pointer_cast[CExtensionScalar, CScalar](self.wrapped) + shared_ptr[CTensor] ctensor + + with nogil: + ctensor = GetResultValue(c_type.MakeTensor(scalar)) + return pyarrow_wrap_tensor(ctensor) + + +cdef class OpaqueScalar(ExtensionScalar): + """ + Concrete class for opaque extension scalar. + """ + + +cdef class Bool8Scalar(ExtensionScalar): + """ + Concrete class for bool8 extension scalar. + """ + + def as_py(self, *, maps_as_pydicts=None): + """ + Return this scalar as a Python object. + + Parameters + ---------- + maps_as_pydicts : str, optional, default `None` + Valid values are `None`, 'lossy', or 'strict'. + This parameter is ignored for non-nested Scalars. + """ + py_val = super().as_py() + return None if py_val is None else py_val != 0 + +cdef dict _scalar_classes = { + _Type_BOOL: BooleanScalar, + _Type_UINT8: UInt8Scalar, + _Type_UINT16: UInt16Scalar, + _Type_UINT32: UInt32Scalar, + _Type_UINT64: UInt64Scalar, + _Type_INT8: Int8Scalar, + _Type_INT16: Int16Scalar, + _Type_INT32: Int32Scalar, + _Type_INT64: Int64Scalar, + _Type_HALF_FLOAT: HalfFloatScalar, + _Type_FLOAT: FloatScalar, + _Type_DOUBLE: DoubleScalar, + _Type_DECIMAL32: Decimal32Scalar, + _Type_DECIMAL64: Decimal64Scalar, + _Type_DECIMAL128: Decimal128Scalar, + _Type_DECIMAL256: Decimal256Scalar, + _Type_DATE32: Date32Scalar, + _Type_DATE64: Date64Scalar, + _Type_TIME32: Time32Scalar, + _Type_TIME64: Time64Scalar, + _Type_TIMESTAMP: TimestampScalar, + _Type_DURATION: DurationScalar, + _Type_BINARY: BinaryScalar, + _Type_LARGE_BINARY: LargeBinaryScalar, + _Type_FIXED_SIZE_BINARY: FixedSizeBinaryScalar, + _Type_BINARY_VIEW: BinaryViewScalar, + _Type_STRING: StringScalar, + _Type_LARGE_STRING: LargeStringScalar, + _Type_STRING_VIEW: StringViewScalar, + _Type_LIST: ListScalar, + _Type_LARGE_LIST: LargeListScalar, + _Type_FIXED_SIZE_LIST: FixedSizeListScalar, + _Type_LIST_VIEW: ListViewScalar, + _Type_LARGE_LIST_VIEW: LargeListViewScalar, + _Type_STRUCT: StructScalar, + _Type_MAP: MapScalar, + _Type_DICTIONARY: DictionaryScalar, + _Type_RUN_END_ENCODED: RunEndEncodedScalar, + _Type_SPARSE_UNION: UnionScalar, + _Type_DENSE_UNION: UnionScalar, + _Type_INTERVAL_MONTH_DAY_NANO: MonthDayNanoIntervalScalar, + _Type_EXTENSION: ExtensionScalar, +} + + +cdef object get_scalar_class_from_type( + const shared_ptr[CDataType]& sp_data_type): + cdef CDataType* data_type = sp_data_type.get() + if data_type == NULL: + raise ValueError('Scalar data type was NULL') + + if data_type.id() == _Type_EXTENSION: + py_ext_data_type = pyarrow_wrap_data_type(sp_data_type) + return py_ext_data_type.__arrow_ext_scalar_class__() + else: + return _scalar_classes[data_type.id()] + + +def scalar(value, type=None, *, from_pandas=None, MemoryPool memory_pool=None): + """ + Create a pyarrow.Scalar instance from a Python object. + + Parameters + ---------- + value : Any + Python object coercible to arrow's type system. + type : pyarrow.DataType + Explicit type to attempt to coerce to, otherwise will be inferred from + the value. + from_pandas : bool, default None + Use pandas's semantics for inferring nulls from values in + ndarray-like data. Defaults to False if not passed explicitly by user, + or True if a pandas object is passed in. + memory_pool : pyarrow.MemoryPool, optional + If not passed, will allocate memory from the currently-set default + memory pool. + + Returns + ------- + scalar : pyarrow.Scalar + + Examples + -------- + >>> import pyarrow as pa + + >>> pa.scalar(42) + + + >>> pa.scalar("string") + + + >>> pa.scalar([1, 2]) + + + >>> pa.scalar([1, 2], type=pa.list_(pa.int16())) + + """ + cdef: + DataType ty + PyConversionOptions options + shared_ptr[CScalar] scalar + shared_ptr[CArray] array + shared_ptr[CChunkedArray] chunked + bint is_pandas_object = False + CMemoryPool* pool + + type = ensure_type(type, allow_none=True) + pool = maybe_unbox_memory_pool(memory_pool) + + extension_type = None + if type is not None and type.id == _Type_EXTENSION: + extension_type = type + type = type.storage_type + + if _is_array_like(value): + value = get_values(value, &is_pandas_object) + + options.size = 1 + + if type is not None: + ty = ensure_type(type) + options.type = ty.sp_type + + if from_pandas is None: + options.from_pandas = is_pandas_object + else: + options.from_pandas = from_pandas + + value = [value] + with nogil: + chunked = GetResultValue(ConvertPySequence(value, None, options, pool)) + + # get the first chunk + assert chunked.get().num_chunks() == 1 + array = chunked.get().chunk(0) + + # retrieve the scalar from the first position + scalar = GetResultValue(array.get().GetScalar(0)) + result = Scalar.wrap(scalar) + + if extension_type is not None: + result = ExtensionScalar.from_storage(extension_type, result) + return result diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/CMakeLists.txt b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..493cea916f8ecbfa021a8ef41b6ae89652f56a9f --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/CMakeLists.txt @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +arrow_install_all_headers("arrow/python") +add_subdirectory(vendored) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/api.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/api.h new file mode 100644 index 0000000000000000000000000000000000000000..2b4263943a2d701202ad14b249118902096eeba6 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/api.h @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "arrow/python/arrow_to_pandas.h" +#include "arrow/python/common.h" +#include "arrow/python/datetime.h" +#include "arrow/python/helpers.h" +#include "arrow/python/inference.h" +#include "arrow/python/io.h" +#include "arrow/python/numpy_convert.h" +#include "arrow/python/numpy_to_arrow.h" +#include "arrow/python/python_to_arrow.h" +#include "arrow/python/util.h" diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/arrow_to_pandas.cc b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/arrow_to_pandas.cc new file mode 100644 index 0000000000000000000000000000000000000000..3edbbde0421e427a970fca1e8b685cd76bd7c58c --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/arrow_to_pandas.cc @@ -0,0 +1,2659 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Functions for pandas conversion via NumPy + +#include "arrow/python/arrow_to_pandas.h" +#include "arrow/python/numpy_interop.h" // IWYU pragma: expand + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/array.h" +#include "arrow/buffer.h" +#include "arrow/datum.h" +#include "arrow/status.h" +#include "arrow/table.h" +#include "arrow/type.h" +#include "arrow/type_traits.h" +#include "arrow/util/checked_cast.h" +#include "arrow/util/hashing.h" +#include "arrow/util/int_util.h" +#include "arrow/util/logging.h" +#include "arrow/util/macros.h" +#include "arrow/util/parallel.h" +#include "arrow/visit_type_inline.h" + +#include "arrow/compute/api.h" + +#include "arrow/python/arrow_to_python_internal.h" +#include "arrow/python/common.h" +#include "arrow/python/datetime.h" +#include "arrow/python/decimal.h" +#include "arrow/python/helpers.h" +#include "arrow/python/numpy_convert.h" +#include "arrow/python/numpy_internal.h" +#include "arrow/python/pyarrow.h" +#include "arrow/python/python_to_arrow.h" +#include "arrow/python/type_traits.h" + +namespace arrow { + +class MemoryPool; + +using internal::checked_cast; +using internal::CheckIndexBounds; +using internal::OptionalParallelFor; + +namespace py { +namespace { + +// Fix options for conversion of an inner (child) array. +PandasOptions MakeInnerOptions(PandasOptions options) { + // Make sure conversion of inner dictionary arrays always returns an array, + // not a dict {'indices': array, 'dictionary': array, 'ordered': bool} + options.decode_dictionaries = true; + options.categorical_columns.reset(); + options.strings_to_categorical = false; + + // In ARROW-7723, we found as a result of ARROW-3789 that second + // through microsecond resolution tz-aware timestamps were being promoted to + // use the DATETIME_NANO_TZ conversion path, yielding a datetime64[ns] NumPy + // array in this function. PyArray_GETITEM returns datetime.datetime for + // units second through microsecond but PyLong for nanosecond (because + // datetime.datetime does not support nanoseconds). + // We force the object conversion to preserve the value of the timezone. + // Nanoseconds are returned as integers. + options.coerce_temporal_nanoseconds = false; + + return options; +} + +// ---------------------------------------------------------------------- +// PyCapsule code for setting ndarray base to reference C++ object + +struct ArrayCapsule { + std::shared_ptr array; +}; + +struct BufferCapsule { + std::shared_ptr buffer; +}; + +void ArrayCapsule_Destructor(PyObject* capsule) { + delete reinterpret_cast(PyCapsule_GetPointer(capsule, "arrow::Array")); +} + +void BufferCapsule_Destructor(PyObject* capsule) { + delete reinterpret_cast(PyCapsule_GetPointer(capsule, "arrow::Buffer")); +} + +// ---------------------------------------------------------------------- +// pandas 0.x DataFrame conversion internals + +using internal::arrow_traits; +using internal::npy_traits; + +template +struct WrapBytes {}; + +template <> +struct WrapBytes { + static inline PyObject* Wrap(const char* data, int64_t length) { + return PyUnicode_FromStringAndSize(data, length); + } +}; + +template <> +struct WrapBytes { + static inline PyObject* Wrap(const char* data, int64_t length) { + return PyUnicode_FromStringAndSize(data, length); + } +}; + +template <> +struct WrapBytes { + static inline PyObject* Wrap(const char* data, int64_t length) { + return PyUnicode_FromStringAndSize(data, length); + } +}; + +template <> +struct WrapBytes { + static inline PyObject* Wrap(const char* data, int64_t length) { + return PyBytes_FromStringAndSize(data, length); + } +}; + +template <> +struct WrapBytes { + static inline PyObject* Wrap(const char* data, int64_t length) { + return PyBytes_FromStringAndSize(data, length); + } +}; + +template <> +struct WrapBytes { + static inline PyObject* Wrap(const char* data, int64_t length) { + return PyBytes_FromStringAndSize(data, length); + } +}; + +template <> +struct WrapBytes { + static inline PyObject* Wrap(const char* data, int64_t length) { + return PyBytes_FromStringAndSize(data, length); + } +}; + +static inline bool ListTypeSupported(const DataType& type) { + switch (type.id()) { + case Type::BOOL: + case Type::UINT8: + case Type::INT8: + case Type::UINT16: + case Type::INT16: + case Type::UINT32: + case Type::INT32: + case Type::INT64: + case Type::UINT64: + case Type::HALF_FLOAT: + case Type::FLOAT: + case Type::DOUBLE: + case Type::DECIMAL128: + case Type::DECIMAL256: + case Type::BINARY: + case Type::LARGE_BINARY: + case Type::STRING: + case Type::LARGE_STRING: + case Type::DATE32: + case Type::DATE64: + case Type::STRUCT: + case Type::MAP: + case Type::TIME32: + case Type::TIME64: + case Type::TIMESTAMP: + case Type::DURATION: + case Type::DICTIONARY: + case Type::INTERVAL_MONTH_DAY_NANO: + case Type::NA: // empty list + // The above types are all supported. + return true; + case Type::FIXED_SIZE_LIST: + case Type::LIST: + case Type::LARGE_LIST: + case Type::LIST_VIEW: + case Type::LARGE_LIST_VIEW: { + const auto& list_type = checked_cast(type); + return ListTypeSupported(*list_type.value_type()); + } + case Type::EXTENSION: { + const auto& ext = checked_cast(*type.GetSharedPtr()); + return ListTypeSupported(*(ext.storage_type())); + } + default: + break; + } + return false; +} + +Status CapsulizeArray(const std::shared_ptr& arr, PyObject** out) { + auto capsule = new ArrayCapsule{{arr}}; + *out = PyCapsule_New(reinterpret_cast(capsule), "arrow::Array", + &ArrayCapsule_Destructor); + if (*out == nullptr) { + delete capsule; + RETURN_IF_PYERROR(); + } + return Status::OK(); +} + +Status CapsulizeBuffer(const std::shared_ptr& buffer, PyObject** out) { + auto capsule = new BufferCapsule{{buffer}}; + *out = PyCapsule_New(reinterpret_cast(capsule), "arrow::Buffer", + &BufferCapsule_Destructor); + if (*out == nullptr) { + delete capsule; + RETURN_IF_PYERROR(); + } + return Status::OK(); +} + +Status SetNdarrayBase(PyArrayObject* arr, PyObject* base) { + if (PyArray_SetBaseObject(arr, base) == -1) { + // Error occurred, trust that SetBaseObject sets the error state + Py_XDECREF(base); + RETURN_IF_PYERROR(); + } + return Status::OK(); +} + +Status SetBufferBase(PyArrayObject* arr, const std::shared_ptr& buffer) { + PyObject* base; + RETURN_NOT_OK(CapsulizeBuffer(buffer, &base)); + return SetNdarrayBase(arr, base); +} + +inline void set_numpy_metadata(int type, const DataType* datatype, PyArray_Descr* out) { + auto metadata = + reinterpret_cast(PyDataType_C_METADATA(out)); + if (type == NPY_DATETIME) { + if (datatype->id() == Type::TIMESTAMP) { + const auto& timestamp_type = checked_cast(*datatype); + metadata->meta.base = internal::NumPyFrequency(timestamp_type.unit()); + } else { + ARROW_DCHECK(false) + << "NPY_DATETIME views only supported for Arrow TIMESTAMP types"; + } + } else if (type == NPY_TIMEDELTA) { + ARROW_DCHECK_EQ(datatype->id(), Type::DURATION); + const auto& duration_type = checked_cast(*datatype); + metadata->meta.base = internal::NumPyFrequency(duration_type.unit()); + } +} + +Status PyArray_NewFromPool(int nd, npy_intp* dims, PyArray_Descr* descr, MemoryPool* pool, + PyObject** out) { + // ARROW-6570: Allocate memory from MemoryPool for a couple reasons + // + // * Track allocations + // * Get better performance through custom allocators + int64_t total_size = PyDataType_ELSIZE(descr); + for (int i = 0; i < nd; ++i) { + total_size *= dims[i]; + } + + ARROW_ASSIGN_OR_RAISE(auto buffer, AllocateBuffer(total_size, pool)); + *out = PyArray_NewFromDescr(&PyArray_Type, descr, nd, dims, + /*strides=*/nullptr, + /*data=*/buffer->mutable_data(), + /*flags=*/NPY_ARRAY_CARRAY | NPY_ARRAY_WRITEABLE, + /*obj=*/nullptr); + if (*out == nullptr) { + RETURN_IF_PYERROR(); + // Trust that error set if NULL returned + } + return SetBufferBase(reinterpret_cast(*out), std::move(buffer)); +} + +template +inline const T* GetPrimitiveValues(const Array& arr) { + if (arr.length() == 0) { + return nullptr; + } + const int elsize = arr.type()->byte_width(); + const auto& prim_arr = checked_cast(arr); + return reinterpret_cast(prim_arr.values()->data() + arr.offset() * elsize); +} + +Status MakeNumPyView(std::shared_ptr arr, PyObject* py_ref, int npy_type, int ndim, + npy_intp* dims, PyObject** out) { + PyAcquireGIL lock; + + PyArray_Descr* descr = internal::GetSafeNumPyDtype(npy_type); + set_numpy_metadata(npy_type, arr->type().get(), descr); + PyObject* result = PyArray_NewFromDescr( + &PyArray_Type, descr, ndim, dims, /*strides=*/nullptr, + const_cast(GetPrimitiveValues(*arr)), /*flags=*/0, nullptr); + PyArrayObject* np_arr = reinterpret_cast(result); + if (np_arr == nullptr) { + // Error occurred, trust that error set + return Status::OK(); + } + + PyObject* base; + if (py_ref == nullptr) { + // Capsule will be owned by the ndarray, no incref necessary. See + // ARROW-1973 + RETURN_NOT_OK(CapsulizeArray(arr, &base)); + } else { + Py_INCREF(py_ref); + base = py_ref; + } + RETURN_NOT_OK(SetNdarrayBase(np_arr, base)); + + // Do not allow Arrow data to be mutated + PyArray_CLEARFLAGS(np_arr, NPY_ARRAY_WRITEABLE); + *out = result; + return Status::OK(); +} + +class PandasWriter { + public: + enum type { + OBJECT, + UINT8, + INT8, + UINT16, + INT16, + UINT32, + INT32, + UINT64, + INT64, + HALF_FLOAT, + FLOAT, + DOUBLE, + BOOL, + DATETIME_DAY, + DATETIME_SECOND, + DATETIME_MILLI, + DATETIME_MICRO, + DATETIME_NANO, + DATETIME_SECOND_TZ, + DATETIME_MILLI_TZ, + DATETIME_MICRO_TZ, + DATETIME_NANO_TZ, + TIMEDELTA_SECOND, + TIMEDELTA_MILLI, + TIMEDELTA_MICRO, + TIMEDELTA_NANO, + CATEGORICAL, + EXTENSION + }; + + PandasWriter(const PandasOptions& options, int64_t num_rows, int num_columns) + : options_(options), num_rows_(num_rows), num_columns_(num_columns) { + PyAcquireGIL lock; + internal::InitPandasStaticData(); + } + virtual ~PandasWriter() {} + + void SetBlockData(PyObject* arr) { + block_arr_.reset(arr); + block_data_ = + reinterpret_cast(PyArray_DATA(reinterpret_cast(arr))); + } + + /// \brief Either copy or wrap single array to create pandas-compatible array + /// for Series or DataFrame. num_columns_ can only be 1. Will try to zero + /// copy if possible (or error if not possible and zero_copy_only=True) + virtual Status TransferSingle(std::shared_ptr data, PyObject* py_ref) = 0; + + /// \brief Copy ChunkedArray into a multi-column block + virtual Status CopyInto(std::shared_ptr data, int64_t rel_placement) = 0; + + Status EnsurePlacementAllocated() { + std::lock_guard guard(allocation_lock_); + if (placement_data_ != nullptr) { + return Status::OK(); + } + PyAcquireGIL lock; + npy_intp placement_dims[1] = {num_columns_}; + PyObject* placement_arr = PyArray_SimpleNew(1, placement_dims, NPY_INT64); + RETURN_IF_PYERROR(); + placement_arr_.reset(placement_arr); + placement_data_ = reinterpret_cast( + PyArray_DATA(reinterpret_cast(placement_arr))); + return Status::OK(); + } + + Status EnsureAllocated() { + std::lock_guard guard(allocation_lock_); + if (block_data_ != nullptr) { + return Status::OK(); + } + RETURN_NOT_OK(Allocate()); + return Status::OK(); + } + + virtual bool CanZeroCopy(const ChunkedArray& data) const { return false; } + + virtual Status Write(std::shared_ptr data, int64_t abs_placement, + int64_t rel_placement) { + RETURN_NOT_OK(EnsurePlacementAllocated()); + if (num_columns_ == 1 && options_.allow_zero_copy_blocks) { + RETURN_NOT_OK(TransferSingle(data, /*py_ref=*/nullptr)); + } else { + RETURN_NOT_OK( + CheckNoZeroCopy("Cannot do zero copy conversion into " + "multi-column DataFrame block")); + RETURN_NOT_OK(EnsureAllocated()); + RETURN_NOT_OK(CopyInto(data, rel_placement)); + } + placement_data_[rel_placement] = abs_placement; + return Status::OK(); + } + + virtual Status GetDataFrameResult(PyObject** out) { + PyObject* result = PyDict_New(); + RETURN_IF_PYERROR(); + + PyObject* block; + RETURN_NOT_OK(GetResultBlock(&block)); + + PyDict_SetItemString(result, "block", block); + PyDict_SetItemString(result, "placement", placement_arr_.obj()); + + RETURN_NOT_OK(AddResultMetadata(result)); + *out = result; + return Status::OK(); + } + + // Caller steals the reference to this object + virtual Status GetSeriesResult(PyObject** out) { + RETURN_NOT_OK(MakeBlock1D()); + // Caller owns the object now + *out = block_arr_.detach(); + return Status::OK(); + } + + protected: + virtual Status AddResultMetadata(PyObject* result) { return Status::OK(); } + + Status MakeBlock1D() { + // For Series or for certain DataFrame block types, we need to shape to a + // 1D array when there is only one column + PyAcquireGIL lock; + + ARROW_DCHECK_EQ(1, num_columns_); + + npy_intp new_dims[1] = {static_cast(num_rows_)}; + PyArray_Dims dims; + dims.ptr = new_dims; + dims.len = 1; + + PyObject* reshaped = PyArray_Newshape( + reinterpret_cast(block_arr_.obj()), &dims, NPY_ANYORDER); + RETURN_IF_PYERROR(); + + // ARROW-8801: Here a PyArrayObject is created that is not being managed by + // any OwnedRef object. This object is then put in the resulting object + // with PyDict_SetItemString, which increments the reference count, so a + // memory leak ensues. There are several ways to fix the memory leak but a + // simple one is to put the reshaped 1D block array in this OwnedRefNoGIL + // so it will be correctly decref'd when this class is destructed. + block_arr_.reset(reshaped); + return Status::OK(); + } + + virtual Status GetResultBlock(PyObject** out) { + *out = block_arr_.obj(); + return Status::OK(); + } + + Status CheckNoZeroCopy(const std::string& message) { + if (options_.zero_copy_only) { + return Status::Invalid(message); + } + return Status::OK(); + } + + Status CheckNotZeroCopyOnly(const ChunkedArray& data) { + if (options_.zero_copy_only) { + return Status::Invalid("Needed to copy ", data.num_chunks(), " chunks with ", + data.null_count(), " nulls, but zero_copy_only was True"); + } + return Status::OK(); + } + + virtual Status Allocate() { + return Status::NotImplemented("Override Allocate in subclasses"); + } + + Status AllocateNDArray(int npy_type, int ndim = 2) { + PyAcquireGIL lock; + + PyObject* block_arr = nullptr; + npy_intp block_dims[2] = {0, 0}; + + if (ndim == 2) { + block_dims[0] = num_columns_; + block_dims[1] = num_rows_; + } else { + block_dims[0] = num_rows_; + } + PyArray_Descr* descr = internal::GetSafeNumPyDtype(npy_type); + if (PyDataType_REFCHK(descr)) { + // ARROW-6876: if the array has refcounted items, let Numpy + // own the array memory so as to decref elements on array destruction + block_arr = PyArray_SimpleNewFromDescr(ndim, block_dims, descr); + RETURN_IF_PYERROR(); + } else { + RETURN_NOT_OK( + PyArray_NewFromPool(ndim, block_dims, descr, options_.pool, &block_arr)); + } + + SetBlockData(block_arr); + return Status::OK(); + } + + void SetDatetimeUnit(NPY_DATETIMEUNIT unit) { + PyAcquireGIL lock; + auto date_dtype = + reinterpret_cast(PyDataType_C_METADATA( + PyArray_DESCR(reinterpret_cast(block_arr_.obj())))); + date_dtype->meta.base = unit; + } + + PandasOptions options_; + + std::mutex allocation_lock_; + + int64_t num_rows_; + int num_columns_; + + OwnedRefNoGIL block_arr_; + uint8_t* block_data_ = nullptr; + + // ndarray + OwnedRefNoGIL placement_arr_; + int64_t* placement_data_ = nullptr; + + private: + ARROW_DISALLOW_COPY_AND_ASSIGN(PandasWriter); +}; + +template +inline void ConvertIntegerWithNulls(const PandasOptions& options, + const ChunkedArray& data, OutType* out_values) { + for (int c = 0; c < data.num_chunks(); c++) { + const auto& arr = *data.chunk(c); + const InType* in_values = GetPrimitiveValues(arr); + // Upcast to double, set NaN as appropriate + + for (int i = 0; i < arr.length(); ++i) { + *out_values++ = + arr.IsNull(i) ? static_cast(NAN) : static_cast(in_values[i]); + } + } +} + +template +inline void ConvertIntegerNoNullsSameType(const PandasOptions& options, + const ChunkedArray& data, T* out_values) { + for (int c = 0; c < data.num_chunks(); c++) { + const auto& arr = *data.chunk(c); + if (arr.length() > 0) { + const T* in_values = GetPrimitiveValues(arr); + memcpy(out_values, in_values, sizeof(T) * arr.length()); + out_values += arr.length(); + } + } +} + +template +inline void ConvertIntegerNoNullsCast(const PandasOptions& options, + const ChunkedArray& data, OutType* out_values) { + for (int c = 0; c < data.num_chunks(); c++) { + const auto& arr = *data.chunk(c); + const InType* in_values = GetPrimitiveValues(arr); + for (int64_t i = 0; i < arr.length(); ++i) { + *out_values = in_values[i]; + } + } +} + +template +struct MemoizationTraits { + using Scalar = typename T::c_type; +}; + +template +struct MemoizationTraits> { + // For binary, we memoize string_view as a scalar value to avoid having to + // unnecessarily copy the memory into the memo table data structure + using Scalar = std::string_view; +}; + +// Generic Array -> PyObject** converter that handles object deduplication, if +// requested +template +inline Status ConvertAsPyObjects(const PandasOptions& options, const ChunkedArray& data, + WrapFunction&& wrap_func, PyObject** out_values) { + using ArrayType = typename TypeTraits::ArrayType; + using Scalar = typename MemoizationTraits::Scalar; + + auto convert_chunks = [&](auto&& wrap_func) -> Status { + for (int c = 0; c < data.num_chunks(); c++) { + const auto& arr = arrow::internal::checked_cast(*data.chunk(c)); + RETURN_NOT_OK(internal::WriteArrayObjects(arr, wrap_func, out_values)); + out_values += arr.length(); + } + return Status::OK(); + }; + + if (options.deduplicate_objects) { + // GH-40316: only allocate a memo table if deduplication is enabled. + ::arrow::internal::ScalarMemoTable memo_table(options.pool); + std::vector unique_values; + int32_t memo_size = 0; + + auto WrapMemoized = [&](const Scalar& value, PyObject** out_values) { + int32_t memo_index; + RETURN_NOT_OK(memo_table.GetOrInsert(value, &memo_index)); + if (memo_index == memo_size) { + // New entry + RETURN_NOT_OK(wrap_func(value, out_values)); + unique_values.push_back(*out_values); + ++memo_size; + } else { + // Duplicate entry + Py_INCREF(unique_values[memo_index]); + *out_values = unique_values[memo_index]; + } + return Status::OK(); + }; + return convert_chunks(std::move(WrapMemoized)); + } else { + return convert_chunks(std::forward(wrap_func)); + } +} + +Status ConvertStruct(PandasOptions options, const ChunkedArray& data, + PyObject** out_values) { + if (data.num_chunks() == 0) { + return Status::OK(); + } + // ChunkedArray has at least one chunk + auto arr = checked_cast(data.chunk(0).get()); + // Use it to cache the struct type and number of fields for all chunks + int32_t num_fields = arr->num_fields(); + auto array_type = arr->type(); + std::vector fields_data(num_fields * data.num_chunks()); + OwnedRef dict_item; + + // See notes in MakeInnerOptions. + options = MakeInnerOptions(std::move(options)); + // Don't blindly convert because timestamps in lists are handled differently. + options.timestamp_as_object = true; + + for (int c = 0; c < data.num_chunks(); c++) { + auto fields_data_offset = c * num_fields; + auto arr = checked_cast(data.chunk(c).get()); + // Convert the struct arrays first + for (int32_t i = 0; i < num_fields; i++) { + auto field = arr->field(static_cast(i)); + // In case the field is an extension array, use .storage() to convert to Pandas + if (field->type()->id() == Type::EXTENSION) { + const ExtensionArray& arr_ext = checked_cast(*field); + field = arr_ext.storage(); + } + RETURN_NOT_OK(ConvertArrayToPandas(options, field, nullptr, + fields_data[i + fields_data_offset].ref())); + ARROW_DCHECK(PyArray_Check(fields_data[i + fields_data_offset].obj())); + } + + // Construct a dictionary for each row + const bool has_nulls = data.null_count() > 0; + for (int64_t i = 0; i < arr->length(); ++i) { + if (has_nulls && arr->IsNull(i)) { + Py_INCREF(Py_None); + *out_values = Py_None; + } else { + // Build the new dict object for the row + dict_item.reset(PyDict_New()); + RETURN_IF_PYERROR(); + for (int32_t field_idx = 0; field_idx < num_fields; ++field_idx) { + OwnedRef field_value; + auto name = array_type->field(static_cast(field_idx))->name(); + if (!arr->field(static_cast(field_idx))->IsNull(i)) { + // Value exists in child array, obtain it + auto array = reinterpret_cast( + fields_data[field_idx + fields_data_offset].obj()); + auto ptr = reinterpret_cast(PyArray_GETPTR1(array, i)); + field_value.reset(PyArray_GETITEM(array, ptr)); + RETURN_IF_PYERROR(); + } else { + // Translate the Null to a None + Py_INCREF(Py_None); + field_value.reset(Py_None); + } + // PyDict_SetItemString increments reference count + auto setitem_result = + PyDict_SetItemString(dict_item.obj(), name.c_str(), field_value.obj()); + RETURN_IF_PYERROR(); + ARROW_DCHECK_EQ(setitem_result, 0); + } + *out_values = dict_item.obj(); + // Grant ownership to the resulting array + Py_INCREF(*out_values); + } + ++out_values; + } + } + return Status::OK(); +} + +Status DecodeDictionaries(MemoryPool* pool, const std::shared_ptr& dense_type, + ArrayVector* arrays) { + compute::ExecContext ctx(pool); + compute::CastOptions options; + for (size_t i = 0; i < arrays->size(); ++i) { + ARROW_ASSIGN_OR_RAISE((*arrays)[i], + compute::Cast(*(*arrays)[i], dense_type, options, &ctx)); + } + return Status::OK(); +} + +Status DecodeDictionaries(MemoryPool* pool, const std::shared_ptr& dense_type, + std::shared_ptr* array) { + auto chunks = (*array)->chunks(); + RETURN_NOT_OK(DecodeDictionaries(pool, dense_type, &chunks)); + *array = std::make_shared(std::move(chunks), dense_type); + return Status::OK(); +} + +template +enable_if_list_like ConvertListsLike(PandasOptions options, + const ChunkedArray& data, + PyObject** out_values) { + using ListArrayT = typename TypeTraits::ArrayType; + // Get column of underlying value arrays + ArrayVector value_arrays; + for (int c = 0; c < data.num_chunks(); c++) { + const auto& arr = checked_cast(*data.chunk(c)); + // values() does not account for offsets, so we need to slice into it. + // We can't use Flatten(), because it removes the values behind a null list + // value, and that makes the offsets into original list values and our + // flattened_values array different. + std::shared_ptr flattened_values = arr.values()->Slice( + arr.value_offset(0), arr.value_offset(arr.length()) - arr.value_offset(0)); + if (arr.value_type()->id() == Type::EXTENSION) { + const auto& arr_ext = checked_cast(*flattened_values); + value_arrays.emplace_back(arr_ext.storage()); + } else { + value_arrays.emplace_back(flattened_values); + } + } + + using ListArrayType = typename ListArrayT::TypeClass; + const auto& list_type = checked_cast(*data.type()); + auto value_type = list_type.value_type(); + if (value_type->id() == Type::EXTENSION) { + value_type = checked_cast(*value_type).storage_type(); + } + + auto flat_column = std::make_shared(value_arrays, value_type); + + options = MakeInnerOptions(std::move(options)); + + OwnedRefNoGIL owned_numpy_array; + RETURN_NOT_OK(ConvertChunkedArrayToPandas(options, flat_column, nullptr, + owned_numpy_array.ref())); + PyObject* numpy_array = owned_numpy_array.obj(); + ARROW_DCHECK(PyArray_Check(numpy_array)); + + int64_t chunk_offset = 0; + for (int c = 0; c < data.num_chunks(); c++) { + const auto& arr = checked_cast(*data.chunk(c)); + const bool has_nulls = data.null_count() > 0; + for (int64_t i = 0; i < arr.length(); ++i) { + if (has_nulls && arr.IsNull(i)) { + Py_INCREF(Py_None); + *out_values = Py_None; + } else { + // Need to subtract value_offset(0) since the original chunk might be a slice + // into another array. + OwnedRef start(PyLong_FromLongLong(arr.value_offset(i) + chunk_offset - + arr.value_offset(0))); + OwnedRef end(PyLong_FromLongLong(arr.value_offset(i + 1) + chunk_offset - + arr.value_offset(0))); + OwnedRef slice(PySlice_New(start.obj(), end.obj(), nullptr)); + + if (ARROW_PREDICT_FALSE(slice.obj() == nullptr)) { + // Fall out of loop, will return from RETURN_IF_PYERROR + break; + } + *out_values = PyObject_GetItem(numpy_array, slice.obj()); + + if (*out_values == nullptr) { + // Fall out of loop, will return from RETURN_IF_PYERROR + break; + } + } + ++out_values; + } + RETURN_IF_PYERROR(); + + chunk_offset += arr.value_offset(arr.length()) - arr.value_offset(0); + } + + return Status::OK(); +} + +// TODO GH-40579: optimize ListView conversion to avoid unnecessary copies +template +enable_if_list_view ConvertListsLike(PandasOptions options, + const ChunkedArray& data, + PyObject** out_values) { + using ListViewArrayType = typename TypeTraits::ArrayType; + using NonViewType = + std::conditional_t; + using NonViewClass = typename TypeTraits::ArrayType; + ArrayVector list_arrays; + for (int c = 0; c < data.num_chunks(); c++) { + const auto& arr = checked_cast(*data.chunk(c)); + ARROW_ASSIGN_OR_RAISE(auto non_view_array, + NonViewClass::FromListView(arr, options.pool)); + list_arrays.emplace_back(non_view_array); + } + auto chunked_array = std::make_shared(list_arrays); + return ConvertListsLike(options, *chunked_array, out_values); +} + +template +Status ConvertMapHelper(F1 resetRow, F2 addPairToRow, F3 stealRow, + const ChunkedArray& data, PyArrayObject* py_keys, + PyArrayObject* py_items, + // needed for null checks in items + const std::vector> item_arrays, + PyObject** out_values) { + OwnedRef key_value; + OwnedRef item_value; + + int64_t chunk_offset = 0; + for (int c = 0; c < data.num_chunks(); ++c) { + const auto& arr = checked_cast(*data.chunk(c)); + const bool has_nulls = data.null_count() > 0; + + // Make a list of key/item pairs for each row in array + for (int64_t i = 0; i < arr.length(); ++i) { + if (has_nulls && arr.IsNull(i)) { + Py_INCREF(Py_None); + *out_values = Py_None; + } else { + int64_t entry_offset = arr.value_offset(i); + int64_t num_pairs = arr.value_offset(i + 1) - entry_offset; + + // Build the new list object for the row of Python pairs + RETURN_NOT_OK(resetRow(num_pairs)); + + // Add each key/item pair in the row + for (int64_t j = 0; j < num_pairs; ++j) { + // Get key value, key is non-nullable for a valid row + auto ptr_key = reinterpret_cast( + PyArray_GETPTR1(py_keys, chunk_offset + entry_offset + j)); + key_value.reset(PyArray_GETITEM(py_keys, ptr_key)); + RETURN_IF_PYERROR(); + + if (item_arrays[c]->IsNull(entry_offset + j)) { + // Translate the Null to a None + Py_INCREF(Py_None); + item_value.reset(Py_None); + } else { + // Get valid value from item array + auto ptr_item = reinterpret_cast( + PyArray_GETPTR1(py_items, chunk_offset + entry_offset + j)); + item_value.reset(PyArray_GETITEM(py_items, ptr_item)); + RETURN_IF_PYERROR(); + } + + // Add the key/item pair to the row + RETURN_NOT_OK(addPairToRow(j, key_value, item_value)); + } + + // Pass ownership to the resulting array + *out_values = stealRow(); + } + ++out_values; + } + RETURN_IF_PYERROR(); + + chunk_offset += arr.values()->length(); + } + + return Status::OK(); +} + +// A more helpful error message around TypeErrors that may stem from unhashable keys +Status CheckMapAsPydictsTypeError() { + if (ARROW_PREDICT_TRUE(!PyErr_Occurred())) { + return Status::OK(); + } + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + // Modify the error string directly, so it is re-raised + // with our additional info. + // + // There are not many interesting things happening when this + // is hit. This is intended to only be called directly after + // PyDict_SetItem, where a finite set of errors could occur. + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + std::string message; + RETURN_NOT_OK(internal::PyObject_StdStringStr(value, &message)); + message += + ". If keys are not hashable, then you must use the option " + "[maps_as_pydicts=None (default)]"; + + // resets the error + PyErr_SetString(PyExc_TypeError, message.c_str()); + } + return ConvertPyError(); +} + +Status CheckForDuplicateKeys(bool error_on_duplicate_keys, Py_ssize_t total_dict_len, + Py_ssize_t total_raw_len) { + if (total_dict_len < total_raw_len) { + const char* message = + "[maps_as_pydicts] " + "After conversion of Arrow maps to pydicts, " + "detected data loss due to duplicate keys. " + "Original input length is [%lld], total converted pydict length is [%lld]."; + std::array buf; + std::snprintf(buf.data(), buf.size(), message, total_raw_len, total_dict_len); + + if (error_on_duplicate_keys) { + return Status::UnknownError(buf.data()); + } else { + ARROW_LOG(WARNING) << buf.data(); + } + } + return Status::OK(); +} + +Status ConvertMap(PandasOptions options, const ChunkedArray& data, + PyObject** out_values) { + // Get columns of underlying key/item arrays + std::vector> key_arrays; + std::vector> item_arrays; + for (int c = 0; c < data.num_chunks(); ++c) { + const auto& map_arr = checked_cast(*data.chunk(c)); + key_arrays.emplace_back(map_arr.keys()); + item_arrays.emplace_back(map_arr.items()); + } + + const auto& map_type = checked_cast(*data.type()); + auto key_type = map_type.key_type(); + auto item_type = map_type.item_type(); + + // ARROW-6899: Convert dictionary-encoded children to dense instead of + // failing below. A more efficient conversion than this could be done later + if (key_type->id() == Type::DICTIONARY) { + auto dense_type = checked_cast(*key_type).value_type(); + RETURN_NOT_OK(DecodeDictionaries(options.pool, dense_type, &key_arrays)); + key_type = dense_type; + } + if (item_type->id() == Type::DICTIONARY) { + auto dense_type = checked_cast(*item_type).value_type(); + RETURN_NOT_OK(DecodeDictionaries(options.pool, dense_type, &item_arrays)); + item_type = dense_type; + } + + // See notes in MakeInnerOptions. + options = MakeInnerOptions(std::move(options)); + // Don't blindly convert because timestamps in lists are handled differently. + options.timestamp_as_object = true; + + auto flat_keys = std::make_shared(key_arrays, key_type); + auto flat_items = std::make_shared(item_arrays, item_type); + OwnedRefNoGIL owned_numpy_keys; + RETURN_NOT_OK( + ConvertChunkedArrayToPandas(options, flat_keys, nullptr, owned_numpy_keys.ref())); + OwnedRefNoGIL owned_numpy_items; + RETURN_NOT_OK( + ConvertChunkedArrayToPandas(options, flat_items, nullptr, owned_numpy_items.ref())); + PyArrayObject* py_keys = reinterpret_cast(owned_numpy_keys.obj()); + PyArrayObject* py_items = reinterpret_cast(owned_numpy_items.obj()); + + if (options.maps_as_pydicts == MapConversionType::DEFAULT) { + // The default behavior to express an Arrow MAP as a list of [(key, value), ...] pairs + OwnedRef list_item; + return ConvertMapHelper( + [&list_item](int64_t num_pairs) { + list_item.reset(PyList_New(num_pairs)); + return CheckPyError(); + }, + [&list_item](int64_t idx, OwnedRef& key_value, OwnedRef& item_value) { + PyList_SET_ITEM(list_item.obj(), idx, + PyTuple_Pack(2, key_value.obj(), item_value.obj())); + return CheckPyError(); + }, + [&list_item] { return list_item.detach(); }, data, py_keys, py_items, item_arrays, + out_values); + } else { + // Use a native pydict + OwnedRef dict_item; + Py_ssize_t total_dict_len{0}; + Py_ssize_t total_raw_len{0}; + + bool error_on_duplicate_keys; + if (options.maps_as_pydicts == MapConversionType::LOSSY) { + error_on_duplicate_keys = false; + } else if (options.maps_as_pydicts == MapConversionType::STRICT_) { + error_on_duplicate_keys = true; + } else { + auto val = std::underlying_type_t(options.maps_as_pydicts); + return Status::UnknownError("Received unknown option for maps_as_pydicts: " + + std::to_string(val)); + } + + auto status = ConvertMapHelper( + [&dict_item, &total_raw_len](int64_t num_pairs) { + total_raw_len += num_pairs; + dict_item.reset(PyDict_New()); + return CheckPyError(); + }, + [&dict_item]([[maybe_unused]] int64_t idx, OwnedRef& key_value, + OwnedRef& item_value) { + auto setitem_result = + PyDict_SetItem(dict_item.obj(), key_value.obj(), item_value.obj()); + ARROW_RETURN_NOT_OK(CheckMapAsPydictsTypeError()); + // returns -1 if there are internal errors around hashing/resizing + return setitem_result == 0 ? Status::OK() + : Status::UnknownError( + "[maps_as_pydicts] " + "Unexpected failure inserting Arrow (key, " + "value) pair into Python dict"); + }, + [&dict_item, &total_dict_len] { + total_dict_len += PyDict_Size(dict_item.obj()); + return dict_item.detach(); + }, + data, py_keys, py_items, item_arrays, out_values); + + ARROW_RETURN_NOT_OK(status); + // If there were no errors generating the pydicts, + // then check if we detected any data loss from duplicate keys. + return CheckForDuplicateKeys(error_on_duplicate_keys, total_dict_len, total_raw_len); + } +} + +template +inline void ConvertNumericNullable(const ChunkedArray& data, InType na_value, + OutType* out_values) { + for (int c = 0; c < data.num_chunks(); c++) { + const auto& arr = *data.chunk(c); + const InType* in_values = GetPrimitiveValues(arr); + + if (arr.null_count() > 0) { + for (int64_t i = 0; i < arr.length(); ++i) { + *out_values++ = arr.IsNull(i) ? na_value : in_values[i]; + } + } else { + memcpy(out_values, in_values, sizeof(InType) * arr.length()); + out_values += arr.length(); + } + } +} + +template +inline void ConvertNumericNullableCast(const ChunkedArray& data, InType na_value, + OutType* out_values) { + for (int c = 0; c < data.num_chunks(); c++) { + const auto& arr = *data.chunk(c); + const InType* in_values = GetPrimitiveValues(arr); + + for (int64_t i = 0; i < arr.length(); ++i) { + *out_values++ = arr.IsNull(i) ? static_cast(na_value) + : static_cast(in_values[i]); + } + } +} + +template +class TypedPandasWriter : public PandasWriter { + public: + using T = typename npy_traits::value_type; + + using PandasWriter::PandasWriter; + + Status TransferSingle(std::shared_ptr data, PyObject* py_ref) override { + if (CanZeroCopy(*data)) { + PyObject* wrapped; + npy_intp dims[2] = {static_cast(num_columns_), + static_cast(num_rows_)}; + RETURN_NOT_OK( + MakeNumPyView(data->chunk(0), py_ref, NPY_TYPE, /*ndim=*/2, dims, &wrapped)); + SetBlockData(wrapped); + return Status::OK(); + } else { + RETURN_NOT_OK(CheckNotZeroCopyOnly(*data)); + RETURN_NOT_OK(EnsureAllocated()); + return CopyInto(data, /*rel_placement=*/0); + } + } + + Status CheckTypeExact(const DataType& type, Type::type expected) { + if (type.id() != expected) { + return Status::NotImplemented("Cannot write Arrow data of type ", type.ToString(), + " to pandas block with NumPy type ", + GetNumPyTypeName(NPY_TYPE)); + } + return Status::OK(); + } + + T* GetBlockColumnStart(int64_t rel_placement) { + return reinterpret_cast(block_data_) + rel_placement * num_rows_; + } + + protected: + Status Allocate() override { return AllocateNDArray(NPY_TYPE); } +}; + +struct ObjectWriterVisitor { + const PandasOptions& options; + const ChunkedArray& data; + PyObject** out_values; + + Status Visit(const NullType& type) { + for (int c = 0; c < data.num_chunks(); c++) { + std::shared_ptr arr = data.chunk(c); + + for (int64_t i = 0; i < arr->length(); ++i) { + // All values are null + Py_INCREF(Py_None); + *out_values = Py_None; + ++out_values; + } + } + return Status::OK(); + } + + Status Visit(const BooleanType& type) { + for (int c = 0; c < data.num_chunks(); c++) { + const auto& arr = checked_cast(*data.chunk(c)); + + for (int64_t i = 0; i < arr.length(); ++i) { + if (arr.IsNull(i)) { + Py_INCREF(Py_None); + *out_values++ = Py_None; + } else if (arr.Value(i)) { + // True + Py_INCREF(Py_True); + *out_values++ = Py_True; + } else { + // False + Py_INCREF(Py_False); + *out_values++ = Py_False; + } + } + } + return Status::OK(); + } + + template + enable_if_integer Visit(const Type& type) { + using T = typename Type::c_type; + auto WrapValue = [](T value, PyObject** out) { + *out = std::is_signed::value ? PyLong_FromLongLong(value) + : PyLong_FromUnsignedLongLong(value); + RETURN_IF_PYERROR(); + return Status::OK(); + }; + return ConvertAsPyObjects(options, data, WrapValue, out_values); + } + + template + enable_if_t::value || is_binary_view_like_type::value || + is_fixed_size_binary_type::value, + Status> + Visit(const Type& type) { + auto WrapValue = [](const std::string_view& view, PyObject** out) { + *out = WrapBytes::Wrap(view.data(), view.length()); + if (*out == nullptr) { + PyErr_Clear(); + return Status::UnknownError("Wrapping ", view, " failed"); + } + return Status::OK(); + }; + return ConvertAsPyObjects(options, data, WrapValue, out_values); + } + + template + enable_if_date Visit(const Type& type) { + auto WrapValue = [](typename Type::c_type value, PyObject** out) { + RETURN_NOT_OK(internal::PyDate_from_int(value, Type::UNIT, out)); + RETURN_IF_PYERROR(); + return Status::OK(); + }; + return ConvertAsPyObjects(options, data, WrapValue, out_values); + } + + template + enable_if_time Visit(const Type& type) { + const TimeUnit::type unit = type.unit(); + auto WrapValue = [unit](typename Type::c_type value, PyObject** out) { + RETURN_NOT_OK(internal::PyTime_from_int(value, unit, out)); + RETURN_IF_PYERROR(); + return Status::OK(); + }; + return ConvertAsPyObjects(options, data, WrapValue, out_values); + } + + template + enable_if_timestamp Visit(const Type& type) { + const TimeUnit::type unit = type.unit(); + OwnedRef tzinfo; + + auto ConvertTimezoneNaive = [&](typename Type::c_type value, PyObject** out) { + RETURN_NOT_OK(internal::PyDateTime_from_int(value, unit, out)); + RETURN_IF_PYERROR(); + return Status::OK(); + }; + auto ConvertTimezoneAware = [&](typename Type::c_type value, PyObject** out) { + PyObject* naive_datetime; + RETURN_NOT_OK(ConvertTimezoneNaive(value, &naive_datetime)); + + // convert the timezone naive datetime object to timezone aware + // two step conversion of the datetime mimics Python's code: + // dt.replace(tzinfo=datetime.timezone.utc).astimezone(tzinfo) + // first step: replacing timezone with timezone.utc (replace method) + OwnedRef args(PyTuple_New(0)); + OwnedRef keywords(PyDict_New()); + PyDict_SetItemString(keywords.obj(), "tzinfo", PyDateTime_TimeZone_UTC); + OwnedRef naive_datetime_replace(PyObject_GetAttrString(naive_datetime, "replace")); + OwnedRef datetime_utc( + PyObject_Call(naive_datetime_replace.obj(), args.obj(), keywords.obj())); + // second step: adjust the datetime to tzinfo timezone (astimezone method) + *out = PyObject_CallMethod(datetime_utc.obj(), "astimezone", "O", tzinfo.obj()); + + // the timezone naive object is no longer required + Py_DECREF(naive_datetime); + RETURN_IF_PYERROR(); + + return Status::OK(); + }; + + if (!type.timezone().empty() && !options.ignore_timezone) { + // convert timezone aware + PyObject* tzobj; + ARROW_ASSIGN_OR_RAISE(tzobj, internal::StringToTzinfo(type.timezone())); + tzinfo.reset(tzobj); + RETURN_IF_PYERROR(); + RETURN_NOT_OK( + ConvertAsPyObjects(options, data, ConvertTimezoneAware, out_values)); + } else { + // convert timezone naive + RETURN_NOT_OK( + ConvertAsPyObjects(options, data, ConvertTimezoneNaive, out_values)); + } + + return Status::OK(); + } + + template + enable_if_t::value, Status> Visit( + const Type& type) { + OwnedRef args(PyTuple_New(0)); + OwnedRef kwargs(PyDict_New()); + RETURN_IF_PYERROR(); + auto to_date_offset = [&](const MonthDayNanoIntervalType::MonthDayNanos& interval, + PyObject** out) { + ARROW_DCHECK(internal::BorrowPandasDataOffsetType() != nullptr); + // DateOffset objects do not add nanoseconds component to pd.Timestamp. + // as of Pandas 1.3.3 + // (https://github.com/pandas-dev/pandas/issues/43892). + // So convert microseconds and remainder to preserve data + // but give users more expected results. + int64_t microseconds = interval.nanoseconds / 1000; + int64_t nanoseconds; + if (interval.nanoseconds >= 0) { + nanoseconds = interval.nanoseconds % 1000; + } else { + nanoseconds = -((-interval.nanoseconds) % 1000); + } + + PyDict_SetItemString(kwargs.obj(), "months", PyLong_FromLong(interval.months)); + PyDict_SetItemString(kwargs.obj(), "days", PyLong_FromLong(interval.days)); + PyDict_SetItemString(kwargs.obj(), "microseconds", + PyLong_FromLongLong(microseconds)); + PyDict_SetItemString(kwargs.obj(), "nanoseconds", PyLong_FromLongLong(nanoseconds)); + *out = + PyObject_Call(internal::BorrowPandasDataOffsetType(), args.obj(), kwargs.obj()); + RETURN_IF_PYERROR(); + return Status::OK(); + }; + return ConvertAsPyObjects(options, data, to_date_offset, + out_values); + } + + template + Status VisitDecimal(const DecimalT& type) { + OwnedRef decimal; + OwnedRef Decimal; + RETURN_NOT_OK(internal::ImportModule("decimal", &decimal)); + RETURN_NOT_OK(internal::ImportFromModule(decimal.obj(), "Decimal", &Decimal)); + PyObject* decimal_constructor = Decimal.obj(); + + for (int c = 0; c < data.num_chunks(); c++) { + const auto& arr = checked_cast(*data.chunk(c)); + + for (int64_t i = 0; i < arr.length(); ++i) { + if (arr.IsNull(i)) { + Py_INCREF(Py_None); + *out_values++ = Py_None; + } else { + *out_values++ = + internal::DecimalFromString(decimal_constructor, arr.FormatValue(i)); + RETURN_IF_PYERROR(); + } + } + } + + return Status::OK(); + } + + Status Visit(const Decimal32Type& type) { + return VisitDecimal(type); + } + + Status Visit(const Decimal64Type& type) { + return VisitDecimal(type); + } + + Status Visit(const Decimal128Type& type) { + return VisitDecimal(type); + } + + Status Visit(const Decimal256Type& type) { + return VisitDecimal(type); + } + + template + enable_if_t::value || is_list_view_type::value, Status> Visit( + const T& type) { + if (!ListTypeSupported(*type.value_type())) { + return Status::NotImplemented( + "Not implemented type for conversion from List to Pandas: ", + type.value_type()->ToString()); + } + return ConvertListsLike(options, data, out_values); + } + + Status Visit(const MapType& type) { return ConvertMap(options, data, out_values); } + + Status Visit(const StructType& type) { + return ConvertStruct(options, data, out_values); + } + + template + enable_if_t::value || + std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value || + (std::is_base_of::value && + !std::is_same::value) || + std::is_base_of::value, + Status> + Visit(const Type& type) { + return Status::NotImplemented("No implemented conversion to object dtype: ", + type.ToString()); + } +}; + +class ObjectWriter : public TypedPandasWriter { + public: + using TypedPandasWriter::TypedPandasWriter; + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + PyAcquireGIL lock; + ObjectWriterVisitor visitor{this->options_, *data, + this->GetBlockColumnStart(rel_placement)}; + return VisitTypeInline(*data->type(), &visitor); + } +}; + +static inline bool IsNonNullContiguous(const ChunkedArray& data) { + return data.num_chunks() == 1 && data.null_count() == 0; +} + +template +class IntWriter : public TypedPandasWriter { + public: + using ArrowType = typename npy_traits::TypeClass; + using TypedPandasWriter::TypedPandasWriter; + + bool CanZeroCopy(const ChunkedArray& data) const override { + return IsNonNullContiguous(data); + } + + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + RETURN_NOT_OK(this->CheckTypeExact(*data->type(), ArrowType::type_id)); + ConvertIntegerNoNullsSameType( + this->options_, *data, this->GetBlockColumnStart(rel_placement)); + return Status::OK(); + } +}; + +template +class FloatWriter : public TypedPandasWriter { + public: + using ArrowType = typename npy_traits::TypeClass; + using TypedPandasWriter::TypedPandasWriter; + using T = typename ArrowType::c_type; + + bool CanZeroCopy(const ChunkedArray& data) const override { + return IsNonNullContiguous(data) && data.type()->id() == ArrowType::type_id; + } + + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + Type::type in_type = data->type()->id(); + auto out_values = this->GetBlockColumnStart(rel_placement); + +#define INTEGER_CASE(IN_TYPE) \ + ConvertIntegerWithNulls(this->options_, *data, out_values); \ + break; + + switch (in_type) { + case Type::UINT8: + INTEGER_CASE(uint8_t); + case Type::INT8: + INTEGER_CASE(int8_t); + case Type::UINT16: + INTEGER_CASE(uint16_t); + case Type::INT16: + INTEGER_CASE(int16_t); + case Type::UINT32: + INTEGER_CASE(uint32_t); + case Type::INT32: + INTEGER_CASE(int32_t); + case Type::UINT64: + INTEGER_CASE(uint64_t); + case Type::INT64: + INTEGER_CASE(int64_t); + case Type::HALF_FLOAT: + ConvertNumericNullableCast(*data, npy_traits::na_sentinel, out_values); + case Type::FLOAT: + ConvertNumericNullableCast(*data, npy_traits::na_sentinel, out_values); + break; + case Type::DOUBLE: + ConvertNumericNullableCast(*data, npy_traits::na_sentinel, out_values); + break; + default: + return Status::NotImplemented("Cannot write Arrow data of type ", + data->type()->ToString(), + " to a Pandas floating point block"); + } + +#undef INTEGER_CASE + + return Status::OK(); + } +}; + +using UInt8Writer = IntWriter; +using Int8Writer = IntWriter; +using UInt16Writer = IntWriter; +using Int16Writer = IntWriter; +using UInt32Writer = IntWriter; +using Int32Writer = IntWriter; +using UInt64Writer = IntWriter; +using Int64Writer = IntWriter; +using Float16Writer = FloatWriter; +using Float32Writer = FloatWriter; +using Float64Writer = FloatWriter; + +class BoolWriter : public TypedPandasWriter { + public: + using TypedPandasWriter::TypedPandasWriter; + + Status TransferSingle(std::shared_ptr data, PyObject* py_ref) override { + RETURN_NOT_OK( + CheckNoZeroCopy("Zero copy conversions not possible with " + "boolean types")); + RETURN_NOT_OK(EnsureAllocated()); + return CopyInto(data, /*rel_placement=*/0); + } + + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + RETURN_NOT_OK(this->CheckTypeExact(*data->type(), Type::BOOL)); + auto out_values = this->GetBlockColumnStart(rel_placement); + for (int c = 0; c < data->num_chunks(); c++) { + const auto& arr = checked_cast(*data->chunk(c)); + for (int64_t i = 0; i < arr.length(); ++i) { + *out_values++ = static_cast(arr.Value(i)); + } + } + return Status::OK(); + } +}; + +// ---------------------------------------------------------------------- +// Date / timestamp types + +template +inline void ConvertDatetime(const ChunkedArray& data, int64_t* out_values) { + for (int c = 0; c < data.num_chunks(); c++) { + const auto& arr = *data.chunk(c); + const T* in_values = GetPrimitiveValues(arr); + + for (int64_t i = 0; i < arr.length(); ++i) { + *out_values++ = arr.IsNull(i) ? kPandasTimestampNull + : (static_cast(in_values[i]) * SHIFT); + } + } +} + +template +void ConvertDatesShift(const ChunkedArray& data, int64_t* out_values) { + for (int c = 0; c < data.num_chunks(); c++) { + const auto& arr = *data.chunk(c); + const T* in_values = GetPrimitiveValues(arr); + for (int64_t i = 0; i < arr.length(); ++i) { + *out_values++ = arr.IsNull(i) ? kPandasTimestampNull + : static_cast(in_values[i]) / SHIFT; + } + } +} + +class DatetimeDayWriter : public TypedPandasWriter { + public: + using TypedPandasWriter::TypedPandasWriter; + + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + int64_t* out_values = this->GetBlockColumnStart(rel_placement); + const auto& type = checked_cast(*data->type()); + switch (type.unit()) { + case DateUnit::DAY: + ConvertDatesShift(*data, out_values); + break; + case DateUnit::MILLI: + ConvertDatesShift(*data, out_values); + break; + } + return Status::OK(); + } + + protected: + Status Allocate() override { + RETURN_NOT_OK(this->AllocateNDArray(NPY_DATETIME)); + SetDatetimeUnit(NPY_FR_D); + return Status::OK(); + } +}; + +template +class DatetimeWriter : public TypedPandasWriter { + public: + using TypedPandasWriter::TypedPandasWriter; + + bool CanZeroCopy(const ChunkedArray& data) const override { + if (data.type()->id() == Type::TIMESTAMP) { + const auto& type = checked_cast(*data.type()); + return IsNonNullContiguous(data) && type.unit() == UNIT; + } else { + return false; + } + } + + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + const auto& ts_type = checked_cast(*data->type()); + ARROW_DCHECK_EQ(UNIT, ts_type.unit()) << "Should only call instances of this writer " + << "with arrays of the correct unit"; + ConvertNumericNullable(*data, kPandasTimestampNull, + this->GetBlockColumnStart(rel_placement)); + return Status::OK(); + } + + protected: + Status Allocate() override { + RETURN_NOT_OK(this->AllocateNDArray(NPY_DATETIME)); + SetDatetimeUnit(internal::NumPyFrequency(UNIT)); + return Status::OK(); + } +}; + +using DatetimeSecondWriter = DatetimeWriter; + +class DatetimeMilliWriter : public DatetimeWriter { + public: + using DatetimeWriter::DatetimeWriter; + + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + Type::type type = data->type()->id(); + int64_t* out_values = this->GetBlockColumnStart(rel_placement); + if (type == Type::DATE32) { + // Convert from days since epoch to datetime64[ms] + ConvertDatetime(*data, out_values); + } else if (type == Type::DATE64) { + ConvertNumericNullable(*data, kPandasTimestampNull, out_values); + } else { + const auto& ts_type = checked_cast(*data->type()); + ARROW_DCHECK_EQ(TimeUnit::MILLI, ts_type.unit()) + << "Should only call instances of this writer " + << "with arrays of the correct unit"; + ConvertNumericNullable(*data, kPandasTimestampNull, out_values); + } + return Status::OK(); + } +}; + +using DatetimeMicroWriter = DatetimeWriter; + +class DatetimeNanoWriter : public DatetimeWriter { + public: + using DatetimeWriter::DatetimeWriter; + + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + Type::type type = data->type()->id(); + int64_t* out_values = this->GetBlockColumnStart(rel_placement); + compute::ExecContext ctx(options_.pool); + compute::CastOptions options; + if (options_.safe_cast) { + options = compute::CastOptions::Safe(); + } else { + options = compute::CastOptions::Unsafe(); + } + Datum out; + auto target_type = timestamp(TimeUnit::NANO); + + if (type == Type::DATE32) { + // Convert from days since epoch to datetime64[ns] + ConvertDatetime(*data, out_values); + } else if (type == Type::DATE64) { + // Date64Type is millisecond timestamp stored as int64_t + // TODO(wesm): Do we want to make sure to zero out the milliseconds? + ConvertDatetime(*data, out_values); + } else if (type == Type::TIMESTAMP) { + const auto& ts_type = checked_cast(*data->type()); + + if (ts_type.unit() == TimeUnit::NANO) { + ConvertNumericNullable(*data, kPandasTimestampNull, out_values); + } else if (ts_type.unit() == TimeUnit::MICRO || ts_type.unit() == TimeUnit::MILLI || + ts_type.unit() == TimeUnit::SECOND) { + ARROW_ASSIGN_OR_RAISE(out, compute::Cast(data, target_type, options, &ctx)); + ConvertNumericNullable(*out.chunked_array(), kPandasTimestampNull, + out_values); + } else { + return Status::NotImplemented("Unsupported time unit"); + } + } else { + return Status::NotImplemented("Cannot write Arrow data of type ", + data->type()->ToString(), + " to a Pandas datetime block."); + } + return Status::OK(); + } +}; + +template +class DatetimeTZWriter : public BASE { + public: + DatetimeTZWriter(const PandasOptions& options, const std::string& timezone, + int64_t num_rows) + : BASE(options, num_rows, 1), timezone_(timezone) {} + + protected: + Status GetResultBlock(PyObject** out) override { + RETURN_NOT_OK(this->MakeBlock1D()); + *out = this->block_arr_.obj(); + return Status::OK(); + } + + Status AddResultMetadata(PyObject* result) override { + PyObject* py_tz = PyUnicode_FromStringAndSize( + timezone_.c_str(), static_cast(timezone_.size())); + RETURN_IF_PYERROR(); + PyDict_SetItemString(result, "timezone", py_tz); + Py_DECREF(py_tz); + return Status::OK(); + } + + private: + std::string timezone_; +}; + +using DatetimeSecondTZWriter = DatetimeTZWriter; +using DatetimeMilliTZWriter = DatetimeTZWriter; +using DatetimeMicroTZWriter = DatetimeTZWriter; +using DatetimeNanoTZWriter = DatetimeTZWriter; + +template +class TimedeltaWriter : public TypedPandasWriter { + public: + using TypedPandasWriter::TypedPandasWriter; + + Status AllocateTimedelta(int ndim) { + RETURN_NOT_OK(this->AllocateNDArray(NPY_TIMEDELTA, ndim)); + SetDatetimeUnit(internal::NumPyFrequency(UNIT)); + return Status::OK(); + } + + bool CanZeroCopy(const ChunkedArray& data) const override { + const auto& type = checked_cast(*data.type()); + return IsNonNullContiguous(data) && type.unit() == UNIT; + } + + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + const auto& type = checked_cast(*data->type()); + ARROW_DCHECK_EQ(UNIT, type.unit()) << "Should only call instances of this writer " + << "with arrays of the correct unit"; + ConvertNumericNullable(*data, kPandasTimestampNull, + this->GetBlockColumnStart(rel_placement)); + return Status::OK(); + } + + protected: + Status Allocate() override { return AllocateTimedelta(2); } +}; + +using TimedeltaSecondWriter = TimedeltaWriter; +using TimedeltaMilliWriter = TimedeltaWriter; +using TimedeltaMicroWriter = TimedeltaWriter; + +class TimedeltaNanoWriter : public TimedeltaWriter { + public: + using TimedeltaWriter::TimedeltaWriter; + + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + Type::type type = data->type()->id(); + int64_t* out_values = this->GetBlockColumnStart(rel_placement); + if (type == Type::DURATION) { + const auto& ts_type = checked_cast(*data->type()); + if (ts_type.unit() == TimeUnit::NANO) { + ConvertNumericNullable(*data, kPandasTimestampNull, out_values); + } else if (ts_type.unit() == TimeUnit::MICRO) { + ConvertDatetime(*data, out_values); + } else if (ts_type.unit() == TimeUnit::MILLI) { + ConvertDatetime(*data, out_values); + } else if (ts_type.unit() == TimeUnit::SECOND) { + ConvertDatetime(*data, out_values); + } else { + return Status::NotImplemented("Unsupported time unit"); + } + } else { + return Status::NotImplemented("Cannot write Arrow data of type ", + data->type()->ToString(), + " to a Pandas timedelta block."); + } + return Status::OK(); + } +}; + +Status MakeZeroLengthArray(const std::shared_ptr& type, + std::shared_ptr* out) { + std::unique_ptr builder; + RETURN_NOT_OK(MakeBuilder(default_memory_pool(), type, &builder)); + RETURN_NOT_OK(builder->Resize(0)); + return builder->Finish(out); +} + +bool NeedDictionaryUnification(const ChunkedArray& data) { + if (data.num_chunks() < 2) { + return false; + } + const auto& arr_first = checked_cast(*data.chunk(0)); + for (int c = 1; c < data.num_chunks(); c++) { + const auto& arr = checked_cast(*data.chunk(c)); + if (!(arr_first.dictionary()->Equals(arr.dictionary()))) { + return true; + } + } + return false; +} + +template +class CategoricalWriter + : public TypedPandasWriter::npy_type> { + public: + using TRAITS = arrow_traits; + using ArrayType = typename TypeTraits::ArrayType; + using T = typename TRAITS::T; + + explicit CategoricalWriter(const PandasOptions& options, int64_t num_rows) + : TypedPandasWriter(options, num_rows, 1), + ordered_(false), + needs_copy_(false) {} + + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + return Status::NotImplemented("categorical type"); + } + + Status TransferSingle(std::shared_ptr data, PyObject* py_ref) override { + const auto& dict_type = checked_cast(*data->type()); + std::shared_ptr dict; + if (data->num_chunks() == 0) { + // no dictionary values => create empty array + RETURN_NOT_OK(this->AllocateNDArray(TRAITS::npy_type, 1)); + RETURN_NOT_OK(MakeZeroLengthArray(dict_type.value_type(), &dict)); + } else { + ARROW_DCHECK_EQ(IndexType::type_id, dict_type.index_type()->id()); + RETURN_NOT_OK(WriteIndices(*data, &dict)); + } + + PyObject* pydict; + RETURN_NOT_OK(ConvertArrayToPandas(this->options_, dict, nullptr, &pydict)); + dictionary_.reset(pydict); + ordered_ = dict_type.ordered(); + return Status::OK(); + } + + Status Write(std::shared_ptr data, int64_t abs_placement, + int64_t rel_placement) override { + RETURN_NOT_OK(this->EnsurePlacementAllocated()); + RETURN_NOT_OK(TransferSingle(data, /*py_ref=*/nullptr)); + this->placement_data_[rel_placement] = abs_placement; + return Status::OK(); + } + + Status GetSeriesResult(PyObject** out) override { + PyAcquireGIL lock; + + PyObject* result = PyDict_New(); + RETURN_IF_PYERROR(); + + // Expected single array dictionary layout + PyDict_SetItemString(result, "indices", this->block_arr_.obj()); + RETURN_IF_PYERROR(); + RETURN_NOT_OK(AddResultMetadata(result)); + + *out = result; + return Status::OK(); + } + + protected: + Status AddResultMetadata(PyObject* result) override { + PyDict_SetItemString(result, "dictionary", dictionary_.obj()); + PyObject* py_ordered = ordered_ ? Py_True : Py_False; + Py_INCREF(py_ordered); + PyDict_SetItemString(result, "ordered", py_ordered); + return Status::OK(); + } + + Status WriteIndicesUniform(const ChunkedArray& data) { + // For unsigned types, upcast to signed since pandas uses -1 for nulls + // uint8 to int16, uint16 to int32, uint32 to int64, signed types unchanged + using OutputType = std::conditional_t< + std::is_same::value, int16_t, + std::conditional_t< + std::is_same::value, int32_t, + std::conditional_t::value, int64_t, T>>>; + const int npy_output_type = std::is_same::value ? NPY_INT16 + : std::is_same::value ? NPY_INT32 + : std::is_same::value + ? NPY_INT64 + : TRAITS::npy_type; + + RETURN_NOT_OK(this->AllocateNDArray(npy_output_type, 1)); + auto out_values = reinterpret_cast(this->block_data_); + + for (int c = 0; c < data.num_chunks(); c++) { + const auto& arr = checked_cast(*data.chunk(c)); + const auto& indices = checked_cast(*arr.indices()); + auto values = reinterpret_cast(indices.raw_values()); + + RETURN_NOT_OK(CheckIndexBounds(*indices.data(), arr.dictionary()->length())); + // Null is -1 in CategoricalBlock + for (int i = 0; i < arr.length(); ++i) { + if (indices.IsValid(i)) { + *out_values++ = static_cast(values[i]); + } else { + *out_values++ = -1; + } + } + } + return Status::OK(); + } + + Status WriteIndicesVarying(const ChunkedArray& data, std::shared_ptr* out_dict) { + // Yield int32 indices to allow for dictionary outgrowing the current index + // type + RETURN_NOT_OK(this->AllocateNDArray(NPY_INT32, 1)); + auto out_values = reinterpret_cast(this->block_data_); + + const auto& dict_type = checked_cast(*data.type()); + + ARROW_ASSIGN_OR_RAISE(auto unifier, DictionaryUnifier::Make(dict_type.value_type(), + this->options_.pool)); + for (int c = 0; c < data.num_chunks(); c++) { + const auto& arr = checked_cast(*data.chunk(c)); + const auto& indices = checked_cast(*arr.indices()); + auto values = reinterpret_cast(indices.raw_values()); + + std::shared_ptr transpose_buffer; + RETURN_NOT_OK(unifier->Unify(*arr.dictionary(), &transpose_buffer)); + + auto transpose = reinterpret_cast(transpose_buffer->data()); + int64_t dict_length = arr.dictionary()->length(); + + RETURN_NOT_OK(CheckIndexBounds(*indices.data(), dict_length)); + + // Null is -1 in CategoricalBlock + for (int i = 0; i < arr.length(); ++i) { + if (indices.IsValid(i)) { + *out_values++ = transpose[values[i]]; + } else { + *out_values++ = -1; + } + } + } + + std::shared_ptr unused_type; + return unifier->GetResult(&unused_type, out_dict); + } + + Status WriteIndices(const ChunkedArray& data, std::shared_ptr* out_dict) { + ARROW_DCHECK_GT(data.num_chunks(), 0); + + // Sniff the first chunk + const auto& arr_first = checked_cast(*data.chunk(0)); + const auto indices_first = std::static_pointer_cast(arr_first.indices()); + + // For unsigned types, we need to convert to signed for pandas compatibility + // even when there are no nulls, so we skip the fast path + const bool is_unsigned = std::is_unsigned::value; + + if (data.num_chunks() == 1 && indices_first->null_count() == 0 && !is_unsigned) { + RETURN_NOT_OK( + CheckIndexBounds(*indices_first->data(), arr_first.dictionary()->length())); + + PyObject* wrapped; + npy_intp dims[1] = {static_cast(this->num_rows_)}; + RETURN_NOT_OK(MakeNumPyView(indices_first, /*py_ref=*/nullptr, TRAITS::npy_type, + /*ndim=*/1, dims, &wrapped)); + this->SetBlockData(wrapped); + *out_dict = arr_first.dictionary(); + } else { + RETURN_NOT_OK(this->CheckNotZeroCopyOnly(data)); + if (NeedDictionaryUnification(data)) { + RETURN_NOT_OK(WriteIndicesVarying(data, out_dict)); + } else { + RETURN_NOT_OK(WriteIndicesUniform(data)); + *out_dict = arr_first.dictionary(); + } + } + return Status::OK(); + } + + OwnedRefNoGIL dictionary_; + bool ordered_; + bool needs_copy_; +}; + +class ExtensionWriter : public PandasWriter { + public: + using PandasWriter::PandasWriter; + + Status Allocate() override { + // no-op + return Status::OK(); + } + + Status TransferSingle(std::shared_ptr data, PyObject* py_ref) override { + PyAcquireGIL lock; + PyObject* py_array; + py_array = wrap_chunked_array(data); + py_array_.reset(py_array); + + return Status::OK(); + } + + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + return TransferSingle(data, nullptr); + } + + Status GetDataFrameResult(PyObject** out) override { + PyAcquireGIL lock; + PyObject* result = PyDict_New(); + RETURN_IF_PYERROR(); + + PyDict_SetItemString(result, "py_array", py_array_.obj()); + PyDict_SetItemString(result, "placement", placement_arr_.obj()); + *out = result; + return Status::OK(); + } + + Status GetSeriesResult(PyObject** out) override { + *out = py_array_.detach(); + return Status::OK(); + } + + protected: + OwnedRefNoGIL py_array_; +}; + +Status MakeWriter(const PandasOptions& options, PandasWriter::type writer_type, + const DataType& type, int64_t num_rows, int num_columns, + std::shared_ptr* writer) { +#define BLOCK_CASE(NAME, TYPE) \ + case PandasWriter::NAME: \ + *writer = std::make_shared(options, num_rows, num_columns); \ + break; + +#define CATEGORICAL_CASE(TYPE) \ + case TYPE::type_id: \ + *writer = std::make_shared>(options, num_rows); \ + break; + +#define TZ_CASE(NAME, TYPE) \ + case PandasWriter::NAME: { \ + const auto& ts_type = checked_cast(type); \ + *writer = std::make_shared(options, ts_type.timezone(), num_rows); \ + } break; + + switch (writer_type) { + case PandasWriter::CATEGORICAL: { + const auto& index_type = *checked_cast(type).index_type(); + switch (index_type.id()) { + CATEGORICAL_CASE(Int8Type); + CATEGORICAL_CASE(Int16Type); + CATEGORICAL_CASE(Int32Type); + CATEGORICAL_CASE(Int64Type); + CATEGORICAL_CASE(UInt8Type); + CATEGORICAL_CASE(UInt16Type); + CATEGORICAL_CASE(UInt32Type); + case Type::UINT64: + return Status::TypeError( + "Converting UInt64 dictionary indices to pandas is not supported."); + default: + // Unreachable + ARROW_DCHECK(false); + break; + } + } break; + case PandasWriter::EXTENSION: + *writer = std::make_shared(options, num_rows, num_columns); + break; + BLOCK_CASE(OBJECT, ObjectWriter); + BLOCK_CASE(UINT8, UInt8Writer); + BLOCK_CASE(INT8, Int8Writer); + BLOCK_CASE(UINT16, UInt16Writer); + BLOCK_CASE(INT16, Int16Writer); + BLOCK_CASE(UINT32, UInt32Writer); + BLOCK_CASE(INT32, Int32Writer); + BLOCK_CASE(UINT64, UInt64Writer); + BLOCK_CASE(INT64, Int64Writer); + BLOCK_CASE(HALF_FLOAT, Float16Writer); + BLOCK_CASE(FLOAT, Float32Writer); + BLOCK_CASE(DOUBLE, Float64Writer); + BLOCK_CASE(BOOL, BoolWriter); + BLOCK_CASE(DATETIME_DAY, DatetimeDayWriter); + BLOCK_CASE(DATETIME_SECOND, DatetimeSecondWriter); + BLOCK_CASE(DATETIME_MILLI, DatetimeMilliWriter); + BLOCK_CASE(DATETIME_MICRO, DatetimeMicroWriter); + BLOCK_CASE(DATETIME_NANO, DatetimeNanoWriter); + BLOCK_CASE(TIMEDELTA_SECOND, TimedeltaSecondWriter); + BLOCK_CASE(TIMEDELTA_MILLI, TimedeltaMilliWriter); + BLOCK_CASE(TIMEDELTA_MICRO, TimedeltaMicroWriter); + BLOCK_CASE(TIMEDELTA_NANO, TimedeltaNanoWriter); + TZ_CASE(DATETIME_SECOND_TZ, DatetimeSecondTZWriter); + TZ_CASE(DATETIME_MILLI_TZ, DatetimeMilliTZWriter); + TZ_CASE(DATETIME_MICRO_TZ, DatetimeMicroTZWriter); + TZ_CASE(DATETIME_NANO_TZ, DatetimeNanoTZWriter); + default: + return Status::NotImplemented("Unsupported block type"); + } + +#undef BLOCK_CASE +#undef CATEGORICAL_CASE + + return Status::OK(); +} + +static Status GetPandasWriterType(const ChunkedArray& data, const PandasOptions& options, + PandasWriter::type* output_type) { +#define INTEGER_CASE(NAME) \ + *output_type = \ + data.null_count() > 0 \ + ? options.integer_object_nulls ? PandasWriter::OBJECT : PandasWriter::DOUBLE \ + : PandasWriter::NAME; \ + break; + + switch (data.type()->id()) { + case Type::BOOL: + *output_type = data.null_count() > 0 ? PandasWriter::OBJECT : PandasWriter::BOOL; + break; + case Type::UINT8: + INTEGER_CASE(UINT8); + case Type::INT8: + INTEGER_CASE(INT8); + case Type::UINT16: + INTEGER_CASE(UINT16); + case Type::INT16: + INTEGER_CASE(INT16); + case Type::UINT32: + INTEGER_CASE(UINT32); + case Type::INT32: + INTEGER_CASE(INT32); + case Type::UINT64: + INTEGER_CASE(UINT64); + case Type::INT64: + INTEGER_CASE(INT64); + case Type::HALF_FLOAT: + *output_type = PandasWriter::HALF_FLOAT; + break; + case Type::FLOAT: + *output_type = PandasWriter::FLOAT; + break; + case Type::DOUBLE: + *output_type = PandasWriter::DOUBLE; + break; + case Type::STRING: // fall through + case Type::LARGE_STRING: // fall through + case Type::STRING_VIEW: // fall through + case Type::BINARY: // fall through + case Type::LARGE_BINARY: + case Type::BINARY_VIEW: + case Type::NA: // fall through + case Type::FIXED_SIZE_BINARY: // fall through + case Type::STRUCT: // fall through + case Type::TIME32: // fall through + case Type::TIME64: // fall through + case Type::DECIMAL32: // fall through + case Type::DECIMAL64: // fall through + case Type::DECIMAL128: // fall through + case Type::DECIMAL256: // fall through + case Type::INTERVAL_MONTH_DAY_NANO: // fall through + *output_type = PandasWriter::OBJECT; + break; + case Type::DATE32: + if (options.date_as_object) { + *output_type = PandasWriter::OBJECT; + } else if (options.coerce_temporal_nanoseconds) { + *output_type = PandasWriter::DATETIME_NANO; + } else if (options.to_numpy) { + // Numpy supports Day, but Pandas does not + *output_type = PandasWriter::DATETIME_DAY; + } else { + *output_type = PandasWriter::DATETIME_MILLI; + } + break; + case Type::DATE64: + if (options.date_as_object) { + *output_type = PandasWriter::OBJECT; + } else if (options.coerce_temporal_nanoseconds) { + *output_type = PandasWriter::DATETIME_NANO; + } else { + *output_type = PandasWriter::DATETIME_MILLI; + } + break; + case Type::TIMESTAMP: { + const auto& ts_type = checked_cast(*data.type()); + if (options.timestamp_as_object && ts_type.unit() != TimeUnit::NANO) { + // Nanoseconds are never out of bounds for pandas, so in that case + // we don't convert to object + *output_type = PandasWriter::OBJECT; + } else if (options.coerce_temporal_nanoseconds) { + if (!ts_type.timezone().empty()) { + *output_type = PandasWriter::DATETIME_NANO_TZ; + } else { + *output_type = PandasWriter::DATETIME_NANO; + } + } else { + if (!ts_type.timezone().empty()) { + switch (ts_type.unit()) { + case TimeUnit::SECOND: + *output_type = PandasWriter::DATETIME_SECOND_TZ; + break; + case TimeUnit::MILLI: + *output_type = PandasWriter::DATETIME_MILLI_TZ; + break; + case TimeUnit::MICRO: + *output_type = PandasWriter::DATETIME_MICRO_TZ; + break; + case TimeUnit::NANO: + *output_type = PandasWriter::DATETIME_NANO_TZ; + break; + } + } else { + switch (ts_type.unit()) { + case TimeUnit::SECOND: + *output_type = PandasWriter::DATETIME_SECOND; + break; + case TimeUnit::MILLI: + *output_type = PandasWriter::DATETIME_MILLI; + break; + case TimeUnit::MICRO: + *output_type = PandasWriter::DATETIME_MICRO; + break; + case TimeUnit::NANO: + *output_type = PandasWriter::DATETIME_NANO; + break; + } + } + } + } break; + case Type::DURATION: { + const auto& dur_type = checked_cast(*data.type()); + if (options.coerce_temporal_nanoseconds) { + *output_type = PandasWriter::TIMEDELTA_NANO; + } else { + switch (dur_type.unit()) { + case TimeUnit::SECOND: + *output_type = PandasWriter::TIMEDELTA_SECOND; + break; + case TimeUnit::MILLI: + *output_type = PandasWriter::TIMEDELTA_MILLI; + break; + case TimeUnit::MICRO: + *output_type = PandasWriter::TIMEDELTA_MICRO; + break; + case TimeUnit::NANO: + *output_type = PandasWriter::TIMEDELTA_NANO; + break; + } + } + } break; + case Type::FIXED_SIZE_LIST: + case Type::LIST: + case Type::LARGE_LIST: + case Type::LIST_VIEW: + case Type::LARGE_LIST_VIEW: + case Type::MAP: { + auto list_type = std::static_pointer_cast(data.type()); + if (!ListTypeSupported(*list_type->value_type())) { + return Status::NotImplemented("Not implemented type for Arrow list to pandas: ", + list_type->value_type()->ToString()); + } + *output_type = PandasWriter::OBJECT; + } break; + case Type::DICTIONARY: + *output_type = PandasWriter::CATEGORICAL; + break; + case Type::EXTENSION: + *output_type = PandasWriter::EXTENSION; + break; + default: + return Status::NotImplemented( + "No known equivalent Pandas block for Arrow data of type ", + data.type()->ToString(), " is known."); + } + return Status::OK(); +} + +// Construct the exact pandas "BlockManager" memory layout +// +// * For each column determine the correct output pandas type +// * Allocate 2D blocks (ncols x nrows) for each distinct data type in output +// * Allocate block placement arrays +// * Write Arrow columns out into each slice of memory; populate block +// * placement arrays as we go +class PandasBlockCreator { + public: + using WriterMap = std::unordered_map>; + + explicit PandasBlockCreator(const PandasOptions& options, FieldVector fields, + ChunkedArrayVector arrays) + : options_(options), fields_(std::move(fields)), arrays_(std::move(arrays)) { + num_columns_ = static_cast(arrays_.size()); + if (num_columns_ > 0) { + num_rows_ = arrays_[0]->length(); + } + column_block_placement_.resize(num_columns_); + } + virtual ~PandasBlockCreator() = default; + + virtual Status Convert(PyObject** out) = 0; + + Status AppendBlocks(const WriterMap& blocks, PyObject* list) { + for (const auto& it : blocks) { + PyObject* item; + RETURN_NOT_OK(it.second->GetDataFrameResult(&item)); + if (PyList_Append(list, item) < 0) { + RETURN_IF_PYERROR(); + } + + // ARROW-1017; PyList_Append increments object refcount + Py_DECREF(item); + } + return Status::OK(); + } + + protected: + PandasOptions options_; + + FieldVector fields_; + ChunkedArrayVector arrays_; + int num_columns_; + int64_t num_rows_; + + // column num -> relative placement within internal block + std::vector column_block_placement_; +}; + +// Helper function for extension chunked arrays +// Constructing a storage chunked array of an extension chunked array +std::shared_ptr GetStorageChunkedArray(std::shared_ptr arr) { + auto value_type = checked_cast(*arr->type()).storage_type(); + ArrayVector storage_arrays; + for (int c = 0; c < arr->num_chunks(); c++) { + const auto& arr_ext = checked_cast(*arr->chunk(c)); + storage_arrays.emplace_back(arr_ext.storage()); + } + return std::make_shared(std::move(storage_arrays), value_type); +}; + +// Helper function to decode RunEndEncodedArray +Result> GetDecodedChunkedArray( + std::shared_ptr arr) { + ARROW_ASSIGN_OR_RAISE(Datum decoded, compute::RunEndDecode(arr)); + ARROW_DCHECK(decoded.is_chunked_array()); + return decoded.chunked_array(); +}; + +class ConsolidatedBlockCreator : public PandasBlockCreator { + public: + using PandasBlockCreator::PandasBlockCreator; + + Status Convert(PyObject** out) override { + column_types_.resize(num_columns_); + RETURN_NOT_OK(CreateBlocks()); + RETURN_NOT_OK(WriteTableToBlocks()); + PyAcquireGIL lock; + + PyObject* result = PyList_New(0); + RETURN_IF_PYERROR(); + + RETURN_NOT_OK(AppendBlocks(blocks_, result)); + RETURN_NOT_OK(AppendBlocks(singleton_blocks_, result)); + + *out = result; + return Status::OK(); + } + + Status GetBlockType(int column_index, PandasWriter::type* out) { + if (options_.IsExtensionColumn(fields_[column_index]->name())) { + *out = PandasWriter::EXTENSION; + return Status::OK(); + } else { + // In case of an extension array default to the storage type + if (arrays_[column_index]->type()->id() == Type::EXTENSION) { + arrays_[column_index] = GetStorageChunkedArray(arrays_[column_index]); + } + // In case of a RunEndEncodedArray default to the values type + else if (arrays_[column_index]->type()->id() == Type::RUN_END_ENCODED) { + ARROW_ASSIGN_OR_RAISE(arrays_[column_index], + GetDecodedChunkedArray(arrays_[column_index])); + } + return GetPandasWriterType(*arrays_[column_index], options_, out); + } + } + + Status CreateBlocks() { + for (int i = 0; i < num_columns_; ++i) { + const DataType& type = *arrays_[i]->type(); + PandasWriter::type output_type; + RETURN_NOT_OK(GetBlockType(i, &output_type)); + + int block_placement = 0; + std::shared_ptr writer; + if (output_type == PandasWriter::CATEGORICAL || + output_type == PandasWriter::DATETIME_SECOND_TZ || + output_type == PandasWriter::DATETIME_MILLI_TZ || + output_type == PandasWriter::DATETIME_MICRO_TZ || + output_type == PandasWriter::DATETIME_NANO_TZ || + output_type == PandasWriter::EXTENSION) { + RETURN_NOT_OK(MakeWriter(options_, output_type, type, num_rows_, + /*num_columns=*/1, &writer)); + singleton_blocks_[i] = writer; + } else { + auto it = block_sizes_.find(output_type); + if (it != block_sizes_.end()) { + block_placement = it->second; + // Increment count + ++it->second; + } else { + // Add key to map + block_sizes_[output_type] = 1; + } + } + column_types_[i] = output_type; + column_block_placement_[i] = block_placement; + } + + // Create normal non-categorical blocks + for (const auto& it : this->block_sizes_) { + PandasWriter::type output_type = static_cast(it.first); + std::shared_ptr block; + RETURN_NOT_OK(MakeWriter(this->options_, output_type, /*unused*/ *null(), num_rows_, + it.second, &block)); + this->blocks_[output_type] = block; + } + return Status::OK(); + } + + Status GetWriter(int i, std::shared_ptr* block) { + PandasWriter::type output_type = this->column_types_[i]; + switch (output_type) { + case PandasWriter::CATEGORICAL: + case PandasWriter::DATETIME_SECOND_TZ: + case PandasWriter::DATETIME_MILLI_TZ: + case PandasWriter::DATETIME_MICRO_TZ: + case PandasWriter::DATETIME_NANO_TZ: + case PandasWriter::EXTENSION: { + auto it = this->singleton_blocks_.find(i); + if (it == this->singleton_blocks_.end()) { + return Status::KeyError("No block allocated"); + } + *block = it->second; + } break; + default: + auto it = this->blocks_.find(output_type); + if (it == this->blocks_.end()) { + return Status::KeyError("No block allocated"); + } + *block = it->second; + break; + } + return Status::OK(); + } + + Status WriteTableToBlocks() { + auto WriteColumn = [this](int i) { + std::shared_ptr block; + RETURN_NOT_OK(this->GetWriter(i, &block)); + // ARROW-3789 Use std::move on the array to permit self-destructing + return block->Write(std::move(arrays_[i]), i, this->column_block_placement_[i]); + }; + + return OptionalParallelFor(options_.use_threads, num_columns_, WriteColumn); + } + + private: + // column num -> block type id + std::vector column_types_; + + // block type -> type count + std::unordered_map block_sizes_; + std::unordered_map block_types_; + + // block type -> block + WriterMap blocks_; + + WriterMap singleton_blocks_; +}; + +/// \brief Create blocks for pandas.DataFrame block manager using one block per +/// column strategy. This permits some zero-copy optimizations as well as the +/// ability for the table to "self-destruct" if selected by the user. +class SplitBlockCreator : public PandasBlockCreator { + public: + using PandasBlockCreator::PandasBlockCreator; + + Status GetWriter(int i, std::shared_ptr* writer) { + PandasWriter::type output_type = PandasWriter::OBJECT; + const DataType& type = *arrays_[i]->type(); + if (options_.IsExtensionColumn(fields_[i]->name())) { + output_type = PandasWriter::EXTENSION; + } else { + // Null count needed to determine output type + RETURN_NOT_OK(GetPandasWriterType(*arrays_[i], options_, &output_type)); + } + return MakeWriter(this->options_, output_type, type, num_rows_, 1, writer); + } + + Status Convert(PyObject** out) override { + PyAcquireGIL lock; + + PyObject* result = PyList_New(0); + RETURN_IF_PYERROR(); + + for (int i = 0; i < num_columns_; ++i) { + std::shared_ptr writer; + RETURN_NOT_OK(GetWriter(i, &writer)); + // ARROW-3789 Use std::move on the array to permit self-destructing + RETURN_NOT_OK(writer->Write(std::move(arrays_[i]), i, /*rel_placement=*/0)); + + PyObject* item; + RETURN_NOT_OK(writer->GetDataFrameResult(&item)); + if (PyList_Append(result, item) < 0) { + RETURN_IF_PYERROR(); + } + // PyList_Append increments object refcount + Py_DECREF(item); + } + + *out = result; + return Status::OK(); + } + + private: + std::vector> writers_; +}; + +Status ConvertCategoricals(const PandasOptions& options, ChunkedArrayVector* arrays, + FieldVector* fields) { + std::vector columns_to_encode; + + // For Categorical conversions + auto EncodeColumn = [&](int j) { + int i = columns_to_encode[j]; + if (options.zero_copy_only) { + return Status::Invalid("Need to dictionary encode a column, but ", + "only zero-copy conversions allowed"); + } + compute::ExecContext ctx(options.pool); + ARROW_ASSIGN_OR_RAISE( + Datum out, DictionaryEncode((*arrays)[i], + compute::DictionaryEncodeOptions::Defaults(), &ctx)); + (*arrays)[i] = out.chunked_array(); + (*fields)[i] = (*fields)[i]->WithType((*arrays)[i]->type()); + return Status::OK(); + }; + + if (options.HasCategoricalColumns()) { + for (int i = 0; i < static_cast(arrays->size()); i++) { + if ((*arrays)[i]->type()->id() != Type::DICTIONARY && + options.IsCategoricalColumn((*fields)[i]->name())) { + columns_to_encode.push_back(i); + } + } + } + if (options.strings_to_categorical) { + for (int i = 0; i < static_cast(arrays->size()); i++) { + if (is_base_binary_like((*arrays)[i]->type()->id()) || + is_binary_view_like((*arrays)[i]->type()->id())) { + columns_to_encode.push_back(i); + } + } + } + return OptionalParallelFor(options.use_threads, + static_cast(columns_to_encode.size()), EncodeColumn); +} + +} // namespace + +Status ConvertArrayToPandas(const PandasOptions& options, std::shared_ptr arr, + PyObject* py_ref, PyObject** out) { + return ConvertChunkedArrayToPandas( + options, std::make_shared(std::move(arr)), py_ref, out); +} + +Status ConvertChunkedArrayToPandas(const PandasOptions& options, + std::shared_ptr arr, PyObject* py_ref, + PyObject** out) { + if (options.decode_dictionaries && arr->type()->id() == Type::DICTIONARY) { + // XXX we should return an error as below if options.zero_copy_only + // is true, but that would break compatibility with existing tests. + const auto& dense_type = + checked_cast(*arr->type()).value_type(); + RETURN_NOT_OK(DecodeDictionaries(options.pool, dense_type, &arr)); + ARROW_DCHECK_NE(arr->type()->id(), Type::DICTIONARY); + + // The original Python DictionaryArray won't own the memory anymore + // as we actually built a new array when we decoded the DictionaryArray + // thus let the final resulting numpy array own the memory through a Capsule + py_ref = nullptr; + } + + if (options.strings_to_categorical && (is_base_binary_like(arr->type()->id()) || + is_binary_view_like(arr->type()->id()))) { + if (options.zero_copy_only) { + return Status::Invalid("Need to dictionary encode a column, but ", + "only zero-copy conversions allowed"); + } + compute::ExecContext ctx(options.pool); + ARROW_ASSIGN_OR_RAISE( + Datum out, + DictionaryEncode(arr, compute::DictionaryEncodeOptions::Defaults(), &ctx)); + arr = out.chunked_array(); + } + + PandasOptions modified_options = options; + modified_options.strings_to_categorical = false; + + // ARROW-7596: We permit the hybrid Series/DataFrame code path to do zero copy + // optimizations that we do not allow in the default case when converting + // Table->DataFrame + modified_options.allow_zero_copy_blocks = true; + + // In case of an extension array default to the storage type + if (arr->type()->id() == Type::EXTENSION) { + arr = GetStorageChunkedArray(arr); + } + // In case of a RunEndEncodedArray decode the array + else if (arr->type()->id() == Type::RUN_END_ENCODED) { + if (options.zero_copy_only) { + return Status::Invalid("Need to dencode a RunEndEncodedArray, but ", + "only zero-copy conversions allowed"); + } + ARROW_ASSIGN_OR_RAISE(arr, GetDecodedChunkedArray(arr)); + + // Because we built a new array when we decoded the RunEndEncodedArray + // the final resulting numpy array should own the memory through a Capsule + py_ref = nullptr; + } + + PandasWriter::type output_type; + RETURN_NOT_OK(GetPandasWriterType(*arr, modified_options, &output_type)); + if (options.decode_dictionaries) { + ARROW_DCHECK_NE(output_type, PandasWriter::CATEGORICAL); + } + + std::shared_ptr writer; + RETURN_NOT_OK(MakeWriter(modified_options, output_type, *arr->type(), arr->length(), + /*num_columns=*/1, &writer)); + RETURN_NOT_OK(writer->TransferSingle(std::move(arr), py_ref)); + return writer->GetSeriesResult(out); +} + +Status ConvertTableToPandas(const PandasOptions& options, std::shared_ptr
table, + PyObject** out) { + ChunkedArrayVector arrays = table->columns(); + FieldVector fields = table->fields(); + + // ARROW-3789: allow "self-destructing" by releasing references to columns as + // we convert them to pandas + table = nullptr; + + RETURN_NOT_OK(ConvertCategoricals(options, &arrays, &fields)); + + PandasOptions modified_options = options; + modified_options.strings_to_categorical = false; + modified_options.categorical_columns.reset(); + + if (options.split_blocks) { + modified_options.allow_zero_copy_blocks = true; + SplitBlockCreator helper(modified_options, std::move(fields), std::move(arrays)); + return helper.Convert(out); + } else { + ConsolidatedBlockCreator helper(modified_options, std::move(fields), + std::move(arrays)); + return helper.Convert(out); + } +} + +} // namespace py +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/arrow_to_pandas.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/arrow_to_pandas.h new file mode 100644 index 0000000000000000000000000000000000000000..23e9d0c77ec3019c02c354e79edd539fe393dc60 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/arrow_to_pandas.h @@ -0,0 +1,165 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Functions for converting between pandas's NumPy-based data representation +// and Arrow data structures + +#pragma once + +#include "arrow/python/platform.h" + +#include +#include +#include + +#include "arrow/memory_pool.h" +#include "arrow/python/visibility.h" + +namespace arrow { + +class Array; +class ChunkedArray; +class Column; +class DataType; +class MemoryPool; +class Status; +class Table; + +namespace py { + +enum class MapConversionType { + DEFAULT, // convert arrow maps to assoc lists (list of kev-value tuples) in Pandas + LOSSY, // report warnings when lossiness is encountered due to duplicate keys + STRICT_, // raise a Python exception when lossiness is encountered due to duplicate + // keys +}; + +struct PandasOptions { + bool HasCategoricalColumns() const { + return categorical_columns && !categorical_columns->empty(); + } + + bool IsCategoricalColumn(const std::string& name) const { + return categorical_columns && categorical_columns->count(name); + } + + bool HasExtensionColumns() const { + return extension_columns && !extension_columns->empty(); + } + + bool IsExtensionColumn(const std::string& name) const { + return extension_columns && extension_columns->count(name); + } + + /// arrow::MemoryPool to use for memory allocations + MemoryPool* pool = default_memory_pool(); + + /// If true, we will convert all string columns to categoricals + bool strings_to_categorical = false; + bool zero_copy_only = false; + bool integer_object_nulls = false; + bool date_as_object = false; + bool timestamp_as_object = false; + bool use_threads = false; + + /// Coerce all date and timestamp to datetime64[ns] + bool coerce_temporal_nanoseconds = false; + + /// Used to maintain backwards compatibility for + /// timezone bugs (see ARROW-9528). Should be removed + /// after Arrow 2.0 release. + bool ignore_timezone = false; + + /// \brief If true, do not create duplicate PyObject versions of equal + /// objects. This only applies to immutable objects like strings or datetime + /// objects + bool deduplicate_objects = false; + + /// \brief For certain data types, a cast is needed in order to store the + /// data in a pandas DataFrame or Series (e.g. timestamps are always stored + /// as nanoseconds in pandas). This option controls whether it is a safe + /// cast or not. + bool safe_cast = true; + + /// \brief If true, create one block per column rather than consolidated + /// blocks (1 per data type). Do zero-copy wrapping when there are no + /// nulls. pandas currently will consolidate the blocks on its own, causing + /// increased memory use, so keep this in mind if you are working on a + /// memory-constrained situation. + bool split_blocks = false; + + /// \brief If true, allow non-writable zero-copy views to be created for + /// single column blocks. This option is also used to provide zero copy for + /// Series data + bool allow_zero_copy_blocks = false; + + /// \brief If true, attempt to deallocate buffers in passed Arrow object if + /// it is the only remaining shared_ptr copy of it. See ARROW-3789 for + /// original context for this feature. Only currently implemented for Table + /// conversions + bool self_destruct = false; + + /// \brief The default behavior (DEFAULT), is to convert Arrow Map arrays to + /// Python association lists (list-of-tuples) in the same order as the Arrow + /// Map, as in [(key1, value1), (key2, value2), ...] + /// If LOSSY or STRICT, convert Arrow Map arrays to native Python dicts. + /// This can change the ordering of (key, value) pairs, and will deduplicate + /// multiple keys, resulting in a possible loss of data. + /// If 'lossy', this key deduplication results in a warning printed + /// when detected. If 'strict', this instead results in an exception + /// being raised when detected. + MapConversionType maps_as_pydicts = MapConversionType::DEFAULT; + + // Used internally for nested arrays. + bool decode_dictionaries = false; + + // Columns that should be casted to categorical + // + // This is wrapped in a shared_ptr because this struct is copied internally for + // each column or nested field (see GH-47861). + std::shared_ptr> categorical_columns; + + // Columns that should be passed through to be converted to + // ExtensionArray/Block + std::shared_ptr> extension_columns; + + // Used internally to decipher between to_numpy() and to_pandas() when + // the expected output differs + bool to_numpy = false; +}; + +ARROW_PYTHON_EXPORT +Status ConvertArrayToPandas(const PandasOptions& options, std::shared_ptr arr, + PyObject* py_ref, PyObject** out); + +ARROW_PYTHON_EXPORT +Status ConvertChunkedArrayToPandas(const PandasOptions& options, + std::shared_ptr col, PyObject* py_ref, + PyObject** out); + +// Convert a whole table as efficiently as possible to a pandas.DataFrame. +// +// The returned Python object is a list of tuples consisting of the exact 2D +// BlockManager structure of the pandas.DataFrame used as of pandas 0.19.x. +// +// tuple item: (indices: ndarray[int32], block: ndarray[TYPE, ndim=2]) +ARROW_PYTHON_EXPORT +Status ConvertTableToPandas(const PandasOptions& options, std::shared_ptr
table, + PyObject** out); + +} // namespace py +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/arrow_to_python_internal.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/arrow_to_python_internal.h new file mode 100644 index 0000000000000000000000000000000000000000..2f9e8e7671763d9ebf7dc0b1007bf8c29e461267 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/arrow_to_python_internal.h @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "arrow/array.h" +#include "arrow/python/platform.h" + +namespace arrow { +namespace py { +namespace internal { +// TODO(ARROW-12976): See if we can refactor Pandas ObjectWriter logic +// to the .cc file and move this there as well if we can. + +// Converts array to a sequency of python objects. +template +inline Status WriteArrayObjects(const ArrayType& arr, WriteValue&& write_func, + Assigner out_values) { + // TODO(ARROW-12976): Use visitor here? + const bool has_nulls = arr.null_count() > 0; + for (int64_t i = 0; i < arr.length(); ++i) { + if (has_nulls && arr.IsNull(i)) { + Py_INCREF(Py_None); + *out_values = Py_None; + } else { + RETURN_NOT_OK(write_func(arr.GetView(i), out_values)); + } + ++out_values; + } + return Status::OK(); +} + +} // namespace internal +} // namespace py +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/async.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/async.h new file mode 100644 index 0000000000000000000000000000000000000000..601fcc8fc6873944832ae1675b59cb0dbb0316a6 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/async.h @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "arrow/python/common.h" +#include "arrow/status.h" +#include "arrow/util/future.h" + +namespace arrow::py { + +/// \brief Bind a Python callback to an arrow::Future. +/// +/// If the Future finishes successfully, py_wrapper is called with its +/// result value and should return a PyObject*. If py_wrapper is successful, +/// py_cb is called with its return value. +/// +/// If either the Future or py_wrapper fails, py_cb is called with the +/// associated Python exception. +/// +/// \param future The future to bind to. +/// \param py_cb The Python callback function. Will be passed the result of +/// py_wrapper, or a Python exception if the future failed or one was +/// raised by py_wrapper. +/// \param py_wrapper A function (likely defined in Cython) to convert the C++ +/// result of the future to a Python object. +template +void BindFuture(Future future, PyObject* py_cb, PyWrapper py_wrapper) { + Py_INCREF(py_cb); + OwnedRefNoGIL cb_ref(py_cb); + + auto future_cb = [cb_ref = std::move(cb_ref), + py_wrapper = std::move(py_wrapper)](Result result) { + SafeCallIntoPythonVoid([&]() { + OwnedRef py_value_or_exc{WrapResult(std::move(result), std::move(py_wrapper))}; + Py_XDECREF( + PyObject_CallFunctionObjArgs(cb_ref.obj(), py_value_or_exc.obj(), NULLPTR)); + ARROW_WARN_NOT_OK(CheckPyError(), "Internal error in async call"); + }); + }; + future.AddCallback(std::move(future_cb)); +} + +} // namespace arrow::py diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/benchmark.cc b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/benchmark.cc new file mode 100644 index 0000000000000000000000000000000000000000..1e2d3467dc426f431a6f89cd2fa458613cafdb47 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/benchmark.cc @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/python/benchmark.h" +#include "arrow/python/helpers.h" + +namespace arrow { +namespace py { +namespace benchmark { + +void Benchmark_PandasObjectIsNull(PyObject* list) { + if (!PyList_CheckExact(list)) { + PyErr_SetString(PyExc_TypeError, "expected a list"); + return; + } + Py_ssize_t i, n = PyList_GET_SIZE(list); + for (i = 0; i < n; i++) { + internal::PandasObjectIsNull(PyList_GET_ITEM(list, i)); + } +} + +} // namespace benchmark +} // namespace py +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/benchmark.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/benchmark.h new file mode 100644 index 0000000000000000000000000000000000000000..62a1f2d807bd911a8fc86edb2b691191b70bdc11 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/benchmark.h @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "arrow/python/platform.h" + +#include "arrow/python/visibility.h" + +namespace arrow { +namespace py { +namespace benchmark { + +// Micro-benchmark routines for use from ASV + +// Run PandasObjectIsNull() once over every object in *list* +ARROW_PYTHON_EXPORT +void Benchmark_PandasObjectIsNull(PyObject* list); + +} // namespace benchmark +} // namespace py +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/common.cc b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/common.cc new file mode 100644 index 0000000000000000000000000000000000000000..b0b6d454c06dc75954f011a2d144d20617183a25 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/common.cc @@ -0,0 +1,246 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/python/common.h" + +#include +#include +#include +#include + +#include "arrow/memory_pool.h" +#include "arrow/status.h" +#include "arrow/util/checked_cast.h" +#include "arrow/util/logging.h" + +#include "arrow/python/helpers.h" + +namespace arrow { + +using internal::checked_cast; + +namespace py { + +static std::mutex memory_pool_mutex; +static MemoryPool* default_python_pool = nullptr; + +void set_default_memory_pool(MemoryPool* pool) { + std::lock_guard guard(memory_pool_mutex); + default_python_pool = pool; +} + +MemoryPool* get_memory_pool() { + std::lock_guard guard(memory_pool_mutex); + if (default_python_pool) { + return default_python_pool; + } else { + return default_memory_pool(); + } +} + +// ---------------------------------------------------------------------- +// PythonErrorDetail + +namespace { + +const char kErrorDetailTypeId[] = "arrow::py::PythonErrorDetail"; + +// Try to match the Python exception type with an appropriate Status code +StatusCode MapPyError(PyObject* exc_type) { + StatusCode code; + + if (PyErr_GivenExceptionMatches(exc_type, PyExc_MemoryError)) { + code = StatusCode::OutOfMemory; + } else if (PyErr_GivenExceptionMatches(exc_type, PyExc_IndexError)) { + code = StatusCode::IndexError; + } else if (PyErr_GivenExceptionMatches(exc_type, PyExc_KeyError)) { + code = StatusCode::KeyError; + } else if (PyErr_GivenExceptionMatches(exc_type, PyExc_TypeError)) { + code = StatusCode::TypeError; + } else if (PyErr_GivenExceptionMatches(exc_type, PyExc_ValueError) || + PyErr_GivenExceptionMatches(exc_type, PyExc_OverflowError)) { + code = StatusCode::Invalid; + } else if (PyErr_GivenExceptionMatches(exc_type, PyExc_EnvironmentError)) { + code = StatusCode::IOError; + } else if (PyErr_GivenExceptionMatches(exc_type, PyExc_NotImplementedError)) { + code = StatusCode::NotImplemented; + } else { + code = StatusCode::UnknownError; + } + return code; +} + +// PythonErrorDetail indicates a Python exception was raised. +class PythonErrorDetail : public StatusDetail { + public: + const char* type_id() const override { return kErrorDetailTypeId; } + + std::string ToString() const override { + // This is simple enough not to need the GIL + Result result = FormatImpl(); + + if (result.ok()) { + return result.ValueOrDie(); + } else { + // Fallback to just the exception type + const auto ty = reinterpret_cast(exc_type_.obj()); + return std::string("Python exception: ") + ty->tp_name; + } + } + + void RestorePyError() const { + Py_INCREF(exc_type_.obj()); + Py_INCREF(exc_value_.obj()); + Py_INCREF(exc_traceback_.obj()); + PyErr_Restore(exc_type_.obj(), exc_value_.obj(), exc_traceback_.obj()); + } + + PyObject* exc_type() const { return exc_type_.obj(); } + + PyObject* exc_value() const { return exc_value_.obj(); } + + static std::shared_ptr FromPyError() { + PyObject* exc_type = nullptr; + PyObject* exc_value = nullptr; + PyObject* exc_traceback = nullptr; + + PyErr_Fetch(&exc_type, &exc_value, &exc_traceback); + PyErr_NormalizeException(&exc_type, &exc_value, &exc_traceback); + ARROW_CHECK(exc_type) + << "PythonErrorDetail::FromPyError called without a Python error set"; + ARROW_DCHECK(PyType_Check(exc_type)); + ARROW_DCHECK(exc_value); // Ensured by PyErr_NormalizeException, double-check + if (exc_traceback == nullptr) { + // Needed by PyErr_Restore() + Py_INCREF(Py_None); + exc_traceback = Py_None; + } + + std::shared_ptr detail(new PythonErrorDetail); + detail->exc_type_.reset(exc_type); + detail->exc_value_.reset(exc_value); + detail->exc_traceback_.reset(exc_traceback); + return detail; + } + + protected: + Result FormatImpl() const { + PyAcquireGIL lock; + + // Use traceback.format_exception() + OwnedRef traceback_module; + RETURN_NOT_OK(internal::ImportModule("traceback", &traceback_module)); + + OwnedRef fmt_exception; + RETURN_NOT_OK(internal::ImportFromModule(traceback_module.obj(), "format_exception", + &fmt_exception)); + + OwnedRef formatted; + formatted.reset(PyObject_CallFunctionObjArgs(fmt_exception.obj(), exc_type_.obj(), + exc_value_.obj(), exc_traceback_.obj(), + NULL)); + RETURN_IF_PYERROR(); + + std::stringstream ss; + ss << "Python exception: "; + Py_ssize_t num_lines = PySequence_Length(formatted.obj()); + RETURN_IF_PYERROR(); + + for (Py_ssize_t i = 0; i < num_lines; ++i) { + Py_ssize_t line_size; + + PyObject* line = PySequence_GetItem(formatted.obj(), i); + RETURN_IF_PYERROR(); + + const char* data = PyUnicode_AsUTF8AndSize(line, &line_size); + RETURN_IF_PYERROR(); + + ss << std::string_view(data, line_size); + } + return ss.str(); + } + + PythonErrorDetail() = default; + + OwnedRefNoGIL exc_type_, exc_value_, exc_traceback_; +}; + +} // namespace + +// ---------------------------------------------------------------------- +// Python exception <-> Status + +Status ConvertPyError(StatusCode code) { + auto detail = PythonErrorDetail::FromPyError(); + if (code == StatusCode::UnknownError) { + code = MapPyError(detail->exc_type()); + } + + std::string message; + RETURN_NOT_OK(internal::PyObject_StdStringStr(detail->exc_value(), &message)); + return Status(code, message, detail); +} + +bool IsPyError(const Status& status) { + if (status.ok()) { + return false; + } + auto detail = status.detail(); + bool result = detail != nullptr && detail->type_id() == kErrorDetailTypeId; + return result; +} + +void RestorePyError(const Status& status) { + ARROW_CHECK(IsPyError(status)); + const auto& detail = checked_cast(*status.detail()); + detail.RestorePyError(); +} + +// ---------------------------------------------------------------------- +// PyBuffer + +PyBuffer::PyBuffer() : Buffer(nullptr, 0) {} + +Status PyBuffer::Init(PyObject* obj) { + if (!PyObject_GetBuffer(obj, &py_buf_, PyBUF_ANY_CONTIGUOUS)) { + data_ = reinterpret_cast(py_buf_.buf); + ARROW_CHECK_NE(data_, nullptr) << "Null pointer in Py_buffer"; + size_ = py_buf_.len; + capacity_ = py_buf_.len; + is_mutable_ = !py_buf_.readonly; + return Status::OK(); + } else { + return ConvertPyError(StatusCode::Invalid); + } +} + +Result> PyBuffer::FromPyObject(PyObject* obj) { + PyBuffer* buf = new PyBuffer(); + std::shared_ptr res(buf); + RETURN_NOT_OK(buf->Init(obj)); + return res; +} + +PyBuffer::~PyBuffer() { + if (data_ != nullptr) { + PyAcquireGIL lock; + PyBuffer_Release(&py_buf_); + } +} + +} // namespace py +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/common.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/common.h new file mode 100644 index 0000000000000000000000000000000000000000..13df38ec1f91967ada18e1e28886fb0371a732f9 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/src/arrow/python/common.h @@ -0,0 +1,471 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "arrow/buffer.h" +#include "arrow/python/pyarrow.h" +#include "arrow/python/visibility.h" +#include "arrow/result.h" +#include "arrow/util/macros.h" + +namespace arrow { + +class MemoryPool; +template +class Result; + +namespace py { + +// Convert current Python error to a Status. The Python error state is cleared +// and can be restored with RestorePyError(). +ARROW_PYTHON_EXPORT Status ConvertPyError(StatusCode code = StatusCode::UnknownError); +// Query whether the given Status is a Python error (as wrapped by ConvertPyError()). +ARROW_PYTHON_EXPORT bool IsPyError(const Status& status); +// Restore a Python error wrapped in a Status. +ARROW_PYTHON_EXPORT void RestorePyError(const Status& status); + +// Catch a pending Python exception and return the corresponding Status. +// If no exception is pending, Status::OK() is returned. +inline Status CheckPyError(StatusCode code = StatusCode::UnknownError) { + if (ARROW_PREDICT_TRUE(!PyErr_Occurred())) { + return Status::OK(); + } else { + return ConvertPyError(code); + } +} + +#define RETURN_IF_PYERROR() ARROW_RETURN_NOT_OK(CheckPyError()) + +#define PY_RETURN_IF_ERROR(CODE) ARROW_RETURN_NOT_OK(CheckPyError(CODE)) + +// For Cython, as you can't define template C++ functions in Cython, only use them. +// This function can set a Python exception. It assumes that T has a (cheap) +// default constructor. +template +T GetResultValue(Result result) { + if (ARROW_PREDICT_TRUE(result.ok())) { + return *std::move(result); + } else { + int r = internal::check_status(result.status()); // takes the GIL + assert(r == -1); // should have errored out + ARROW_UNUSED(r); + return {}; + } +} + +/// \brief Wrap a Result and return the corresponding Python object. +/// +/// If the Result is successful, py_wrapper is called with its result value +/// and should return a PyObject*. If py_wrapper is successful (returns +/// a non-NULL value), its return value is returned. +/// +/// If either the Result or py_wrapper fails, the associated Python exception +/// is raised and NULL is returned. +// +/// \param result The Result whose value to wrap in a Python object. +/// \param py_wrapper A function (likely defined in Cython) to convert the C++ +/// value of the Result to a Python object. +/// \return A new Python reference, or NULL if an exception occurred +template +PyObject* WrapResult(Result result, PyWrapper&& py_wrapper) { + static_assert(std::is_same_v()))>, + "PyWrapper argument to WrapResult should return a PyObject* " + "when called with a T*"); + Status st = result.status(); + if (st.ok()) { + PyObject* py_value = py_wrapper(result.MoveValueUnsafe()); + st = CheckPyError(); + if (st.ok()) { + return py_value; + } + Py_XDECREF(py_value); // should be null, but who knows + } + // Status is an error, convert it to an exception. + return internal::convert_status(st); +} + +// A RAII-style helper that ensures the GIL is acquired inside a lexical block. +class ARROW_PYTHON_EXPORT PyAcquireGIL { + public: + PyAcquireGIL() : acquired_gil_(false) { acquire(); } + + ~PyAcquireGIL() { release(); } + + void acquire() { + if (!acquired_gil_) { + state_ = PyGILState_Ensure(); + acquired_gil_ = true; + } + } + + // idempotent + void release() { + if (acquired_gil_) { + PyGILState_Release(state_); + acquired_gil_ = false; + } + } + + private: + bool acquired_gil_; + PyGILState_STATE state_; + ARROW_DISALLOW_COPY_AND_ASSIGN(PyAcquireGIL); +}; + +// A RAII-style helper that releases the GIL until the end of a lexical block +class ARROW_PYTHON_EXPORT PyReleaseGIL { + public: + PyReleaseGIL() : ptr_(PyEval_SaveThread(), &unique_ptr_deleter) {} + + private: + static void unique_ptr_deleter(PyThreadState* state) { + if (state) { + PyEval_RestoreThread(state); + } + } + std::unique_ptr ptr_; +}; + +// A helper to call safely into the Python interpreter from arbitrary C++ code. +// The GIL is acquired, and the current thread's error status is preserved. +template +auto SafeCallIntoPython(Function&& func) -> decltype(func()) { + PyAcquireGIL lock; + PyObject* exc_type; + PyObject* exc_value; + PyObject* exc_traceback; + PyErr_Fetch(&exc_type, &exc_value, &exc_traceback); + auto maybe_status = std::forward(func)(); + // If the return Status is a "Python error", the current Python error status + // describes the error and shouldn't be clobbered. + if (!IsPyError(::arrow::ToStatus(maybe_status)) && exc_type != NULLPTR) { + PyErr_Restore(exc_type, exc_value, exc_traceback); + } + return maybe_status; +} + +template +auto SafeCallIntoPythonVoid(Function&& func) -> decltype(func()) { + PyAcquireGIL lock; + PyObject* exc_type; + PyObject* exc_value; + PyObject* exc_traceback; + PyErr_Fetch(&exc_type, &exc_value, &exc_traceback); + func(); + if (exc_type != NULLPTR) { + PyErr_Restore(exc_type, exc_value, exc_traceback); + } +} + +// A RAII primitive that DECREFs the underlying PyObject* when it +// goes out of scope. +class ARROW_PYTHON_EXPORT OwnedRef { + public: + OwnedRef() : obj_(NULLPTR) {} + OwnedRef(OwnedRef&& other) : OwnedRef(other.detach()) {} + explicit OwnedRef(PyObject* obj) : obj_(obj) {} + + OwnedRef& operator=(OwnedRef&& other) { + obj_ = other.detach(); + return *this; + } + + ~OwnedRef() { + // GH-38626: destructor may be called after the Python interpreter is finalized. + if (Py_IsInitialized()) { + reset(); + } + } + + void reset(PyObject* obj) { + Py_XDECREF(obj_); + obj_ = obj; + } + + void reset() { reset(NULLPTR); } + + PyObject* detach() { + PyObject* result = obj_; + obj_ = NULLPTR; + return result; + } + + PyObject* obj() const { return obj_; } + + PyObject** ref() { return &obj_; } + + operator bool() const { return obj_ != NULLPTR; } + + private: + ARROW_DISALLOW_COPY_AND_ASSIGN(OwnedRef); + + PyObject* obj_; +}; + +// Same as OwnedRef, but ensures the GIL is taken when it goes out of scope. +// This is for situations where the GIL is not always known to be held +// (e.g. if it is released in the middle of a function for performance reasons) +class ARROW_PYTHON_EXPORT OwnedRefNoGIL : public OwnedRef { + public: + OwnedRefNoGIL() : OwnedRef() {} + OwnedRefNoGIL(OwnedRefNoGIL&& other) : OwnedRef(other.detach()) {} + explicit OwnedRefNoGIL(PyObject* obj) : OwnedRef(obj) {} + + ~OwnedRefNoGIL() { + // GH-38626: destructor may be called after the Python interpreter is finalized. + if (Py_IsInitialized() && obj() != NULLPTR) { + PyAcquireGIL lock; + reset(); + } + } +}; + +template