Add loan-advisor-agent (part 3)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/api/writer.h +25 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/reader.h +424 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/schema.h +184 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/test_util.h +487 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/writer.h +176 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/benchmark_util.h +47 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/bloom_filter.h +392 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/bloom_filter_reader.h +68 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/bloom_filter_writer.h +105 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_page.h +181 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_reader.h +458 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_scanner.h +264 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_writer.h +317 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encoding.h +476 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/crypto_factory.h +152 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/encryption.h +434 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_key_material_store.h +57 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_key_unwrapper.h +96 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_key_wrapper.h +84 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_system_key_material_store.h +93 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_encryption_key.h +58 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_material.h +129 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_metadata.h +91 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_toolkit.h +106 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/kms_client.h +97 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/kms_client_factory.h +38 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/local_wrap_kms_client.h +95 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/test_encryption_util.h +136 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/test_in_memory_kms.h +101 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/two_level_cache_with_expiration.h +149 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/type_fwd.h +28 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/exception.h +175 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/file_reader.h +257 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/file_writer.h +260 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/geospatial/statistics.h +198 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/hasher.h +136 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/index_location.h +49 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_comparison.h +38 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_comparison_inc.h +61 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_conversion.h +216 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_conversion_inc.h +356 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/metadata.h +536 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/page_index.h +391 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/parquet_version.h +31 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/platform.h +116 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/printer.h +46 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/properties.h +1529 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/schema.h +494 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/size_statistics.h +101 -0
.gitattributes
CHANGED
|
@@ -537,3 +537,6 @@ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_python.dll filter=lfs d
|
|
| 537 |
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_python.lib filter=lfs diff=lfs merge=lfs -text
|
| 538 |
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_substrait.dll filter=lfs diff=lfs merge=lfs -text
|
| 539 |
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_substrait.lib filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
| 537 |
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_python.lib filter=lfs diff=lfs merge=lfs -text
|
| 538 |
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_substrait.dll filter=lfs diff=lfs merge=lfs -text
|
| 539 |
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_substrait.lib filter=lfs diff=lfs merge=lfs -text
|
| 540 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/lib.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 541 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/parquet.dll filter=lfs diff=lfs merge=lfs -text
|
| 542 |
+
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/parquet.lib filter=lfs diff=lfs merge=lfs -text
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/api/writer.h
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include "parquet/api/io.h"
|
| 21 |
+
#include "parquet/api/schema.h"
|
| 22 |
+
#include "parquet/column_writer.h"
|
| 23 |
+
#include "parquet/exception.h"
|
| 24 |
+
#include "parquet/file_writer.h"
|
| 25 |
+
#include "parquet/statistics.h"
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/reader.h
ADDED
|
@@ -0,0 +1,424 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <cstdint>
|
| 21 |
+
// N.B. we don't include async_generator.h as it's relatively heavy
|
| 22 |
+
#include <functional>
|
| 23 |
+
#include <memory>
|
| 24 |
+
#include <vector>
|
| 25 |
+
|
| 26 |
+
#include "parquet/file_reader.h"
|
| 27 |
+
#include "parquet/platform.h"
|
| 28 |
+
#include "parquet/properties.h"
|
| 29 |
+
|
| 30 |
+
namespace arrow {
|
| 31 |
+
|
| 32 |
+
class ChunkedArray;
|
| 33 |
+
class KeyValueMetadata;
|
| 34 |
+
class RecordBatchReader;
|
| 35 |
+
struct Scalar;
|
| 36 |
+
class Schema;
|
| 37 |
+
class Table;
|
| 38 |
+
class RecordBatch;
|
| 39 |
+
|
| 40 |
+
} // namespace arrow
|
| 41 |
+
|
| 42 |
+
namespace parquet {
|
| 43 |
+
|
| 44 |
+
class FileMetaData;
|
| 45 |
+
class SchemaDescriptor;
|
| 46 |
+
|
| 47 |
+
namespace arrow {
|
| 48 |
+
|
| 49 |
+
class ColumnChunkReader;
|
| 50 |
+
class ColumnReader;
|
| 51 |
+
struct SchemaManifest;
|
| 52 |
+
class RowGroupReader;
|
| 53 |
+
|
| 54 |
+
/// \brief Arrow read adapter class for deserializing Parquet files as Arrow row batches.
|
| 55 |
+
///
|
| 56 |
+
/// This interfaces caters for different use cases and thus provides different
|
| 57 |
+
/// interfaces. In its most simplistic form, we cater for a user that wants to
|
| 58 |
+
/// read the whole Parquet at once with the `FileReader::ReadTable` method.
|
| 59 |
+
///
|
| 60 |
+
/// More advanced users that also want to implement parallelism on top of each
|
| 61 |
+
/// single Parquet files should do this on the RowGroup level. For this, they can
|
| 62 |
+
/// call `FileReader::RowGroup(i)->ReadTable` to receive only the specified
|
| 63 |
+
/// RowGroup as a table.
|
| 64 |
+
///
|
| 65 |
+
/// In the most advanced situation, where a consumer wants to independently read
|
| 66 |
+
/// RowGroups in parallel and consume each column individually, they can call
|
| 67 |
+
/// `FileReader::RowGroup(i)->Column(j)->Read` and receive an `arrow::Column`
|
| 68 |
+
/// instance.
|
| 69 |
+
///
|
| 70 |
+
/// Finally, one can also get a stream of record batches using
|
| 71 |
+
/// `FileReader::GetRecordBatchReader()`. This can internally decode columns
|
| 72 |
+
/// in parallel if use_threads was enabled in the ArrowReaderProperties.
|
| 73 |
+
///
|
| 74 |
+
/// The parquet format supports an optional integer field_id which can be assigned
|
| 75 |
+
/// to a field. Arrow will convert these field IDs to a metadata key named
|
| 76 |
+
/// PARQUET:field_id on the appropriate field.
|
| 77 |
+
// TODO(wesm): nested data does not always make sense with this user
|
| 78 |
+
// interface unless you are only reading a single leaf node from a branch of
|
| 79 |
+
// a table. For example:
|
| 80 |
+
//
|
| 81 |
+
// repeated group data {
|
| 82 |
+
// optional group record {
|
| 83 |
+
// optional int32 val1;
|
| 84 |
+
// optional byte_array val2;
|
| 85 |
+
// optional bool val3;
|
| 86 |
+
// }
|
| 87 |
+
// optional int32 val4;
|
| 88 |
+
// }
|
| 89 |
+
//
|
| 90 |
+
// In the Parquet file, there are 4 leaf nodes:
|
| 91 |
+
//
|
| 92 |
+
// * data.record.val1
|
| 93 |
+
// * data.record.val2
|
| 94 |
+
// * data.record.val3
|
| 95 |
+
// * data.val4
|
| 96 |
+
//
|
| 97 |
+
// When materializing this data in an Arrow array, we would have:
|
| 98 |
+
//
|
| 99 |
+
// data: list<struct<
|
| 100 |
+
// record: struct<
|
| 101 |
+
// val1: int32,
|
| 102 |
+
// val2: string (= list<uint8>),
|
| 103 |
+
// val3: bool,
|
| 104 |
+
// >,
|
| 105 |
+
// val4: int32
|
| 106 |
+
// >>
|
| 107 |
+
//
|
| 108 |
+
// However, in the Parquet format, each leaf node has its own repetition and
|
| 109 |
+
// definition levels describing the structure of the intermediate nodes in
|
| 110 |
+
// this array structure. Thus, we will need to scan the leaf data for a group
|
| 111 |
+
// of leaf nodes part of the same type tree to create a single result Arrow
|
| 112 |
+
// nested array structure.
|
| 113 |
+
//
|
| 114 |
+
// This is additionally complicated "chunky" repeated fields or very large byte
|
| 115 |
+
// arrays
|
| 116 |
+
class PARQUET_EXPORT FileReader {
|
| 117 |
+
public:
|
| 118 |
+
/// Factory function to create a FileReader from a ParquetFileReader and properties
|
| 119 |
+
/// \deprecated Deprecated in 23.0.0. Use arrow::Result version instead.
|
| 120 |
+
ARROW_DEPRECATED("Deprecated in 23.0.0. Use arrow::Result version instead.")
|
| 121 |
+
static ::arrow::Status Make(::arrow::MemoryPool* pool,
|
| 122 |
+
std::unique_ptr<ParquetFileReader> reader,
|
| 123 |
+
const ArrowReaderProperties& properties,
|
| 124 |
+
std::unique_ptr<FileReader>* out);
|
| 125 |
+
|
| 126 |
+
/// Factory function to create a FileReader from a ParquetFileReader
|
| 127 |
+
/// \deprecated Deprecated in 23.0.0. Use arrow::Result version instead.
|
| 128 |
+
ARROW_DEPRECATED("Deprecated in 23.0.0. Use arrow::Result version instead.")
|
| 129 |
+
static ::arrow::Status Make(::arrow::MemoryPool* pool,
|
| 130 |
+
std::unique_ptr<ParquetFileReader> reader,
|
| 131 |
+
std::unique_ptr<FileReader>* out);
|
| 132 |
+
|
| 133 |
+
/// Factory function to create a FileReader from a ParquetFileReader and properties
|
| 134 |
+
static ::arrow::Result<std::unique_ptr<FileReader>> Make(
|
| 135 |
+
::arrow::MemoryPool* pool, std::unique_ptr<ParquetFileReader> reader,
|
| 136 |
+
const ArrowReaderProperties& properties);
|
| 137 |
+
|
| 138 |
+
/// Factory function to create a FileReader from a ParquetFileReader
|
| 139 |
+
static ::arrow::Result<std::unique_ptr<FileReader>> Make(
|
| 140 |
+
::arrow::MemoryPool* pool, std::unique_ptr<ParquetFileReader> reader);
|
| 141 |
+
|
| 142 |
+
// Since the distribution of columns amongst a Parquet file's row groups may
|
| 143 |
+
// be uneven (the number of values in each column chunk can be different), we
|
| 144 |
+
// provide a column-oriented read interface. The ColumnReader hides the
|
| 145 |
+
// details of paging through the file's row groups and yielding
|
| 146 |
+
// fully-materialized arrow::Array instances
|
| 147 |
+
//
|
| 148 |
+
// Returns error status if the column of interest is not flat.
|
| 149 |
+
// The indicated column index is relative to the schema
|
| 150 |
+
virtual ::arrow::Status GetColumn(int i, std::unique_ptr<ColumnReader>* out) = 0;
|
| 151 |
+
|
| 152 |
+
/// \brief Return arrow schema for all the columns.
|
| 153 |
+
virtual ::arrow::Status GetSchema(std::shared_ptr<::arrow::Schema>* out) = 0;
|
| 154 |
+
|
| 155 |
+
/// \brief Read column as a whole into a chunked array.
|
| 156 |
+
///
|
| 157 |
+
/// The index i refers the index of the top level schema field, which may
|
| 158 |
+
/// be nested or flat - e.g.
|
| 159 |
+
///
|
| 160 |
+
/// 0 foo.bar
|
| 161 |
+
/// foo.bar.baz
|
| 162 |
+
/// foo.qux
|
| 163 |
+
/// 1 foo2
|
| 164 |
+
/// 2 foo3
|
| 165 |
+
///
|
| 166 |
+
/// i=0 will read the entire foo struct, i=1 the foo2 primitive column etc
|
| 167 |
+
virtual ::arrow::Status ReadColumn(int i,
|
| 168 |
+
std::shared_ptr<::arrow::ChunkedArray>* out) = 0;
|
| 169 |
+
|
| 170 |
+
/// \brief Return a RecordBatchReader of all row groups and columns.
|
| 171 |
+
virtual ::arrow::Result<std::unique_ptr<::arrow::RecordBatchReader>>
|
| 172 |
+
GetRecordBatchReader() = 0;
|
| 173 |
+
|
| 174 |
+
/// \brief Return a RecordBatchReader of row groups selected from row_group_indices.
|
| 175 |
+
///
|
| 176 |
+
/// Note that the ordering in row_group_indices matters. FileReaders must outlive
|
| 177 |
+
/// their RecordBatchReaders.
|
| 178 |
+
///
|
| 179 |
+
/// \returns error Result if row_group_indices contains an invalid index
|
| 180 |
+
virtual ::arrow::Result<std::unique_ptr<::arrow::RecordBatchReader>>
|
| 181 |
+
GetRecordBatchReader(const std::vector<int>& row_group_indices) = 0;
|
| 182 |
+
|
| 183 |
+
/// \brief Return a RecordBatchReader of row groups selected from
|
| 184 |
+
/// row_group_indices, whose columns are selected by column_indices.
|
| 185 |
+
///
|
| 186 |
+
/// Note that the ordering in row_group_indices and column_indices
|
| 187 |
+
/// matter. FileReaders must outlive their RecordBatchReaders.
|
| 188 |
+
///
|
| 189 |
+
/// \returns error Result if either row_group_indices or column_indices
|
| 190 |
+
/// contains an invalid index
|
| 191 |
+
virtual ::arrow::Result<std::unique_ptr<::arrow::RecordBatchReader>>
|
| 192 |
+
GetRecordBatchReader(const std::vector<int>& row_group_indices,
|
| 193 |
+
const std::vector<int>& column_indices) = 0;
|
| 194 |
+
|
| 195 |
+
/// \brief Return a RecordBatchReader of row groups selected from
|
| 196 |
+
/// row_group_indices, whose columns are selected by column_indices.
|
| 197 |
+
///
|
| 198 |
+
/// Note that the ordering in row_group_indices and column_indices
|
| 199 |
+
/// matter. FileReaders must outlive their RecordBatchReaders.
|
| 200 |
+
///
|
| 201 |
+
/// \param row_group_indices which row groups to read (order determines read order).
|
| 202 |
+
/// \param column_indices which columns to read (order determines output schema).
|
| 203 |
+
/// \param[out] out record batch stream from parquet data.
|
| 204 |
+
///
|
| 205 |
+
/// \returns error Status if either row_group_indices or column_indices
|
| 206 |
+
/// contains an invalid index
|
| 207 |
+
/// \deprecated Deprecated in 21.0.0. Use arrow::Result version instead.
|
| 208 |
+
ARROW_DEPRECATED("Deprecated in 21.0.0. Use arrow::Result version instead.")
|
| 209 |
+
::arrow::Status GetRecordBatchReader(const std::vector<int>& row_group_indices,
|
| 210 |
+
const std::vector<int>& column_indices,
|
| 211 |
+
std::shared_ptr<::arrow::RecordBatchReader>* out);
|
| 212 |
+
|
| 213 |
+
/// \deprecated Deprecated in 21.0.0. Use arrow::Result version instead.
|
| 214 |
+
ARROW_DEPRECATED("Deprecated in 21.0.0. Use arrow::Result version instead.")
|
| 215 |
+
::arrow::Status GetRecordBatchReader(const std::vector<int>& row_group_indices,
|
| 216 |
+
std::shared_ptr<::arrow::RecordBatchReader>* out);
|
| 217 |
+
|
| 218 |
+
/// \deprecated Deprecated in 21.0.0. Use arrow::Result version instead.
|
| 219 |
+
ARROW_DEPRECATED("Deprecated in 21.0.0. Use arrow::Result version instead.")
|
| 220 |
+
::arrow::Status GetRecordBatchReader(std::shared_ptr<::arrow::RecordBatchReader>* out);
|
| 221 |
+
|
| 222 |
+
/// \brief Return a generator of record batches.
|
| 223 |
+
///
|
| 224 |
+
/// The FileReader must outlive the generator, so this requires that you pass in a
|
| 225 |
+
/// shared_ptr.
|
| 226 |
+
///
|
| 227 |
+
/// \returns error Result if either row_group_indices or column_indices contains an
|
| 228 |
+
/// invalid index
|
| 229 |
+
virtual ::arrow::Result<
|
| 230 |
+
std::function<::arrow::Future<std::shared_ptr<::arrow::RecordBatch>>()>>
|
| 231 |
+
GetRecordBatchGenerator(std::shared_ptr<FileReader> reader,
|
| 232 |
+
const std::vector<int> row_group_indices,
|
| 233 |
+
const std::vector<int> column_indices,
|
| 234 |
+
::arrow::internal::Executor* cpu_executor = NULLPTR,
|
| 235 |
+
int64_t rows_to_readahead = 0) = 0;
|
| 236 |
+
|
| 237 |
+
/// Read all columns into a Table
|
| 238 |
+
virtual ::arrow::Result<std::shared_ptr<::arrow::Table>> ReadTable() = 0;
|
| 239 |
+
|
| 240 |
+
/// \deprecated Deprecated in 24.0.0. Use arrow::Result version instead.
|
| 241 |
+
ARROW_DEPRECATED("Deprecated in 24.0.0. Use arrow::Result version instead.")
|
| 242 |
+
::arrow::Status ReadTable(std::shared_ptr<::arrow::Table>* out);
|
| 243 |
+
|
| 244 |
+
/// \brief Read the given columns into a Table
|
| 245 |
+
///
|
| 246 |
+
/// The indicated column indices are relative to the internal representation
|
| 247 |
+
/// of the parquet table. For instance :
|
| 248 |
+
/// 0 foo.bar
|
| 249 |
+
/// foo.bar.baz 0
|
| 250 |
+
/// foo.bar.baz2 1
|
| 251 |
+
/// foo.qux 2
|
| 252 |
+
/// 1 foo2 3
|
| 253 |
+
/// 2 foo3 4
|
| 254 |
+
///
|
| 255 |
+
/// i=0 will read foo.bar.baz, i=1 will read only foo.bar.baz2 and so on.
|
| 256 |
+
/// Only leaf fields have indices; foo itself doesn't have an index.
|
| 257 |
+
/// To get the index for a particular leaf field, one can use
|
| 258 |
+
/// manifest().schema_fields to get the top level fields, and then walk the
|
| 259 |
+
/// tree to identify the relevant leaf fields and access its column_index.
|
| 260 |
+
/// To get the total number of leaf fields, use FileMetadata.num_columns().
|
| 261 |
+
virtual ::arrow::Result<std::shared_ptr<::arrow::Table>> ReadTable(
|
| 262 |
+
const std::vector<int>& column_indices) = 0;
|
| 263 |
+
|
| 264 |
+
/// \deprecated Deprecated in 24.0.0. Use arrow::Result version instead.
|
| 265 |
+
ARROW_DEPRECATED("Deprecated in 24.0.0. Use arrow::Result version instead.")
|
| 266 |
+
::arrow::Status ReadTable(const std::vector<int>& column_indices,
|
| 267 |
+
std::shared_ptr<::arrow::Table>* out);
|
| 268 |
+
|
| 269 |
+
/// \brief Read the given row group columns into a Table
|
| 270 |
+
virtual ::arrow::Result<std::shared_ptr<::arrow::Table>> ReadRowGroup(
|
| 271 |
+
int i, const std::vector<int>& column_indices) = 0;
|
| 272 |
+
|
| 273 |
+
/// \brief Read the given row group into a Table
|
| 274 |
+
virtual ::arrow::Result<std::shared_ptr<::arrow::Table>> ReadRowGroup(int i) = 0;
|
| 275 |
+
|
| 276 |
+
/// \brief Read the given row groups columns into a Table
|
| 277 |
+
virtual ::arrow::Result<std::shared_ptr<::arrow::Table>> ReadRowGroups(
|
| 278 |
+
const std::vector<int>& row_groups, const std::vector<int>& column_indices) = 0;
|
| 279 |
+
|
| 280 |
+
/// \brief Read the given row groups into a Table
|
| 281 |
+
virtual ::arrow::Result<std::shared_ptr<::arrow::Table>> ReadRowGroups(
|
| 282 |
+
const std::vector<int>& row_groups) = 0;
|
| 283 |
+
|
| 284 |
+
/// \deprecated Deprecated in 24.0.0. Use arrow::Result version instead.
|
| 285 |
+
ARROW_DEPRECATED("Deprecated in 24.0.0. Use arrow::Result version instead.")
|
| 286 |
+
::arrow::Status ReadRowGroup(int i, const std::vector<int>& column_indices,
|
| 287 |
+
std::shared_ptr<::arrow::Table>* out);
|
| 288 |
+
|
| 289 |
+
/// \deprecated Deprecated in 24.0.0. Use arrow::Result version instead.
|
| 290 |
+
ARROW_DEPRECATED("Deprecated in 24.0.0. Use arrow::Result version instead.")
|
| 291 |
+
::arrow::Status ReadRowGroup(int i, std::shared_ptr<::arrow::Table>* out);
|
| 292 |
+
|
| 293 |
+
/// \deprecated Deprecated in 24.0.0. Use arrow::Result version instead.
|
| 294 |
+
ARROW_DEPRECATED("Deprecated in 24.0.0. Use arrow::Result version instead.")
|
| 295 |
+
::arrow::Status ReadRowGroups(const std::vector<int>& row_groups,
|
| 296 |
+
const std::vector<int>& column_indices,
|
| 297 |
+
std::shared_ptr<::arrow::Table>* out);
|
| 298 |
+
|
| 299 |
+
/// \deprecated Deprecated in 24.0.0. Use arrow::Result version instead.
|
| 300 |
+
ARROW_DEPRECATED("Deprecated in 24.0.0. Use arrow::Result version instead.")
|
| 301 |
+
::arrow::Status ReadRowGroups(const std::vector<int>& row_groups,
|
| 302 |
+
std::shared_ptr<::arrow::Table>* out);
|
| 303 |
+
|
| 304 |
+
/// \brief Scan file contents with one thread, return number of rows
|
| 305 |
+
virtual ::arrow::Status ScanContents(std::vector<int> columns,
|
| 306 |
+
const int32_t column_batch_size,
|
| 307 |
+
int64_t* num_rows) = 0;
|
| 308 |
+
|
| 309 |
+
/// \brief Return a reader for the RowGroup, this object must not outlive the
|
| 310 |
+
/// FileReader.
|
| 311 |
+
virtual std::shared_ptr<RowGroupReader> RowGroup(int row_group_index) = 0;
|
| 312 |
+
|
| 313 |
+
/// \brief The number of row groups in the file
|
| 314 |
+
virtual int num_row_groups() const = 0;
|
| 315 |
+
|
| 316 |
+
virtual ParquetFileReader* parquet_reader() const = 0;
|
| 317 |
+
|
| 318 |
+
/// Set whether to use multiple threads during reads of multiple columns.
|
| 319 |
+
/// By default only one thread is used.
|
| 320 |
+
virtual void set_use_threads(bool use_threads) = 0;
|
| 321 |
+
|
| 322 |
+
/// Set number of records to read per batch for the RecordBatchReader.
|
| 323 |
+
virtual void set_batch_size(int64_t batch_size) = 0;
|
| 324 |
+
|
| 325 |
+
virtual const ArrowReaderProperties& properties() const = 0;
|
| 326 |
+
|
| 327 |
+
virtual const SchemaManifest& manifest() const = 0;
|
| 328 |
+
|
| 329 |
+
virtual ~FileReader() = default;
|
| 330 |
+
};
|
| 331 |
+
|
| 332 |
+
class RowGroupReader {
|
| 333 |
+
public:
|
| 334 |
+
virtual ~RowGroupReader() = default;
|
| 335 |
+
virtual std::shared_ptr<ColumnChunkReader> Column(int column_index) = 0;
|
| 336 |
+
virtual ::arrow::Status ReadTable(const std::vector<int>& column_indices,
|
| 337 |
+
std::shared_ptr<::arrow::Table>* out) = 0;
|
| 338 |
+
virtual ::arrow::Status ReadTable(std::shared_ptr<::arrow::Table>* out) = 0;
|
| 339 |
+
|
| 340 |
+
private:
|
| 341 |
+
struct Iterator;
|
| 342 |
+
};
|
| 343 |
+
|
| 344 |
+
class ColumnChunkReader {
|
| 345 |
+
public:
|
| 346 |
+
virtual ~ColumnChunkReader() = default;
|
| 347 |
+
virtual ::arrow::Status Read(std::shared_ptr<::arrow::ChunkedArray>* out) = 0;
|
| 348 |
+
};
|
| 349 |
+
|
| 350 |
+
// At this point, the column reader is a stream iterator. It only knows how to
|
| 351 |
+
// read the next batch of values for a particular column from the file until it
|
| 352 |
+
// runs out.
|
| 353 |
+
//
|
| 354 |
+
// We also do not expose any internal Parquet details, such as row groups. This
|
| 355 |
+
// might change in the future.
|
| 356 |
+
class PARQUET_EXPORT ColumnReader {
|
| 357 |
+
public:
|
| 358 |
+
virtual ~ColumnReader() = default;
|
| 359 |
+
|
| 360 |
+
// Scan the next array of the indicated size. The actual size of the
|
| 361 |
+
// returned array may be less than the passed size depending how much data is
|
| 362 |
+
// available in the file.
|
| 363 |
+
//
|
| 364 |
+
// When all the data in the file has been exhausted, the result is set to
|
| 365 |
+
// nullptr.
|
| 366 |
+
//
|
| 367 |
+
// Returns Status::OK on a successful read, including if you have exhausted
|
| 368 |
+
// the data available in the file.
|
| 369 |
+
virtual ::arrow::Status NextBatch(int64_t batch_size,
|
| 370 |
+
std::shared_ptr<::arrow::ChunkedArray>* out) = 0;
|
| 371 |
+
};
|
| 372 |
+
|
| 373 |
+
/// \brief Experimental helper class for bindings (like Python) that struggle
|
| 374 |
+
/// either with std::move or C++ exceptions
|
| 375 |
+
class PARQUET_EXPORT FileReaderBuilder {
|
| 376 |
+
public:
|
| 377 |
+
FileReaderBuilder();
|
| 378 |
+
|
| 379 |
+
/// Create FileReaderBuilder from Arrow file and optional properties / metadata
|
| 380 |
+
::arrow::Status Open(std::shared_ptr<::arrow::io::RandomAccessFile> file,
|
| 381 |
+
const ReaderProperties& properties = default_reader_properties(),
|
| 382 |
+
std::shared_ptr<FileMetaData> metadata = NULLPTR);
|
| 383 |
+
|
| 384 |
+
/// Create FileReaderBuilder from file path and optional properties / metadata
|
| 385 |
+
::arrow::Status OpenFile(const std::string& path, bool memory_map = false,
|
| 386 |
+
const ReaderProperties& props = default_reader_properties(),
|
| 387 |
+
std::shared_ptr<FileMetaData> metadata = NULLPTR);
|
| 388 |
+
|
| 389 |
+
ParquetFileReader* raw_reader() { return raw_reader_.get(); }
|
| 390 |
+
|
| 391 |
+
/// Set Arrow MemoryPool for memory allocation
|
| 392 |
+
FileReaderBuilder* memory_pool(::arrow::MemoryPool* pool);
|
| 393 |
+
/// Set Arrow reader properties
|
| 394 |
+
FileReaderBuilder* properties(const ArrowReaderProperties& arg_properties);
|
| 395 |
+
/// Build FileReader instance
|
| 396 |
+
::arrow::Status Build(std::unique_ptr<FileReader>* out);
|
| 397 |
+
::arrow::Result<std::unique_ptr<FileReader>> Build();
|
| 398 |
+
|
| 399 |
+
private:
|
| 400 |
+
::arrow::MemoryPool* pool_;
|
| 401 |
+
ArrowReaderProperties properties_;
|
| 402 |
+
std::unique_ptr<ParquetFileReader> raw_reader_;
|
| 403 |
+
};
|
| 404 |
+
|
| 405 |
+
/// \defgroup parquet-arrow-reader-factories Factory functions for Parquet Arrow readers
|
| 406 |
+
///
|
| 407 |
+
/// @{
|
| 408 |
+
|
| 409 |
+
/// \brief Build FileReader from Arrow file and MemoryPool
|
| 410 |
+
///
|
| 411 |
+
/// Advanced settings are supported through the FileReaderBuilder class.
|
| 412 |
+
PARQUET_EXPORT
|
| 413 |
+
::arrow::Result<std::unique_ptr<FileReader>> OpenFile(
|
| 414 |
+
std::shared_ptr<::arrow::io::RandomAccessFile>, ::arrow::MemoryPool* allocator);
|
| 415 |
+
|
| 416 |
+
/// @}
|
| 417 |
+
|
| 418 |
+
PARQUET_EXPORT
|
| 419 |
+
::arrow::Status StatisticsAsScalars(const Statistics& Statistics,
|
| 420 |
+
std::shared_ptr<::arrow::Scalar>* min,
|
| 421 |
+
std::shared_ptr<::arrow::Scalar>* max);
|
| 422 |
+
|
| 423 |
+
} // namespace arrow
|
| 424 |
+
} // namespace parquet
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/schema.h
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <cassert>
|
| 21 |
+
#include <memory>
|
| 22 |
+
#include <unordered_map>
|
| 23 |
+
#include <unordered_set>
|
| 24 |
+
#include <vector>
|
| 25 |
+
|
| 26 |
+
#include "arrow/result.h"
|
| 27 |
+
#include "arrow/status.h"
|
| 28 |
+
#include "arrow/type.h"
|
| 29 |
+
#include "arrow/type_fwd.h"
|
| 30 |
+
|
| 31 |
+
#include "parquet/level_conversion.h"
|
| 32 |
+
#include "parquet/platform.h"
|
| 33 |
+
#include "parquet/schema.h"
|
| 34 |
+
|
| 35 |
+
namespace parquet {
|
| 36 |
+
|
| 37 |
+
class ArrowReaderProperties;
|
| 38 |
+
class ArrowWriterProperties;
|
| 39 |
+
class WriterProperties;
|
| 40 |
+
|
| 41 |
+
namespace arrow {
|
| 42 |
+
|
| 43 |
+
/// \defgroup arrow-to-parquet-schema-conversion Functions to convert an Arrow
|
| 44 |
+
/// schema into a Parquet schema.
|
| 45 |
+
///
|
| 46 |
+
/// @{
|
| 47 |
+
|
| 48 |
+
PARQUET_EXPORT
|
| 49 |
+
::arrow::Status FieldToNode(const std::shared_ptr<::arrow::Field>& field,
|
| 50 |
+
const WriterProperties& properties,
|
| 51 |
+
const ArrowWriterProperties& arrow_properties,
|
| 52 |
+
schema::NodePtr* out);
|
| 53 |
+
|
| 54 |
+
PARQUET_EXPORT
|
| 55 |
+
::arrow::Status ToParquetSchema(const ::arrow::Schema* arrow_schema,
|
| 56 |
+
const WriterProperties& properties,
|
| 57 |
+
const ArrowWriterProperties& arrow_properties,
|
| 58 |
+
std::shared_ptr<SchemaDescriptor>* out);
|
| 59 |
+
|
| 60 |
+
PARQUET_EXPORT
|
| 61 |
+
::arrow::Status ToParquetSchema(const ::arrow::Schema* arrow_schema,
|
| 62 |
+
const WriterProperties& properties,
|
| 63 |
+
std::shared_ptr<SchemaDescriptor>* out);
|
| 64 |
+
|
| 65 |
+
/// @}
|
| 66 |
+
|
| 67 |
+
/// \defgroup parquet-to-arrow-schema-conversion Functions to convert a Parquet
|
| 68 |
+
/// schema into an Arrow schema.
|
| 69 |
+
///
|
| 70 |
+
/// @{
|
| 71 |
+
|
| 72 |
+
PARQUET_EXPORT
|
| 73 |
+
::arrow::Status FromParquetSchema(
|
| 74 |
+
const SchemaDescriptor* parquet_schema, const ArrowReaderProperties& properties,
|
| 75 |
+
const std::shared_ptr<const ::arrow::KeyValueMetadata>& key_value_metadata,
|
| 76 |
+
std::shared_ptr<::arrow::Schema>* out);
|
| 77 |
+
|
| 78 |
+
PARQUET_EXPORT
|
| 79 |
+
::arrow::Status FromParquetSchema(const SchemaDescriptor* parquet_schema,
|
| 80 |
+
const ArrowReaderProperties& properties,
|
| 81 |
+
std::shared_ptr<::arrow::Schema>* out);
|
| 82 |
+
|
| 83 |
+
PARQUET_EXPORT
|
| 84 |
+
::arrow::Status FromParquetSchema(const SchemaDescriptor* parquet_schema,
|
| 85 |
+
std::shared_ptr<::arrow::Schema>* out);
|
| 86 |
+
|
| 87 |
+
/// @}
|
| 88 |
+
|
| 89 |
+
/// \brief Bridge between an arrow::Field and parquet column indices.
|
| 90 |
+
struct PARQUET_EXPORT SchemaField {
|
| 91 |
+
std::shared_ptr<::arrow::Field> field;
|
| 92 |
+
std::vector<SchemaField> children;
|
| 93 |
+
|
| 94 |
+
// Only set for leaf nodes
|
| 95 |
+
int column_index = -1;
|
| 96 |
+
|
| 97 |
+
parquet::internal::LevelInfo level_info;
|
| 98 |
+
|
| 99 |
+
bool is_leaf() const { return column_index != -1; }
|
| 100 |
+
};
|
| 101 |
+
|
| 102 |
+
/// \brief Bridge between a parquet Schema and an arrow Schema.
|
| 103 |
+
///
|
| 104 |
+
/// Expose parquet columns as a tree structure. Useful traverse and link
|
| 105 |
+
/// between arrow's Schema and parquet's Schema.
|
| 106 |
+
struct PARQUET_EXPORT SchemaManifest {
|
| 107 |
+
static ::arrow::Status Make(
|
| 108 |
+
const SchemaDescriptor* schema,
|
| 109 |
+
const std::shared_ptr<const ::arrow::KeyValueMetadata>& metadata,
|
| 110 |
+
const ArrowReaderProperties& properties, SchemaManifest* manifest);
|
| 111 |
+
|
| 112 |
+
const SchemaDescriptor* descr;
|
| 113 |
+
std::shared_ptr<::arrow::Schema> origin_schema;
|
| 114 |
+
std::shared_ptr<const ::arrow::KeyValueMetadata> schema_metadata;
|
| 115 |
+
std::vector<SchemaField> schema_fields;
|
| 116 |
+
|
| 117 |
+
std::unordered_map<int, const SchemaField*> column_index_to_field;
|
| 118 |
+
std::unordered_map<const SchemaField*, const SchemaField*> child_to_parent;
|
| 119 |
+
|
| 120 |
+
::arrow::Status GetColumnField(int column_index, const SchemaField** out) const {
|
| 121 |
+
auto it = column_index_to_field.find(column_index);
|
| 122 |
+
if (it == column_index_to_field.end()) {
|
| 123 |
+
return ::arrow::Status::KeyError("Column index ", column_index,
|
| 124 |
+
" not found in schema manifest, may be malformed");
|
| 125 |
+
}
|
| 126 |
+
*out = it->second;
|
| 127 |
+
return ::arrow::Status::OK();
|
| 128 |
+
}
|
| 129 |
+
|
| 130 |
+
const SchemaField* GetParent(const SchemaField* field) const {
|
| 131 |
+
// Returns nullptr also if not found
|
| 132 |
+
auto it = child_to_parent.find(field);
|
| 133 |
+
if (it == child_to_parent.end()) {
|
| 134 |
+
return NULLPTR;
|
| 135 |
+
}
|
| 136 |
+
return it->second;
|
| 137 |
+
}
|
| 138 |
+
|
| 139 |
+
/// Coalesce a list of field indices (relative to the equivalent arrow::Schema) which
|
| 140 |
+
/// correspond to the column root (first node below the parquet schema's root group) of
|
| 141 |
+
/// each leaf referenced in column_indices.
|
| 142 |
+
///
|
| 143 |
+
/// For example, for leaves `a.b.c`, `a.b.d.e`, and `i.j.k` (column_indices=[0,1,3])
|
| 144 |
+
/// the roots are `a` and `i` (return=[0,2]).
|
| 145 |
+
///
|
| 146 |
+
/// root
|
| 147 |
+
/// -- a <------
|
| 148 |
+
/// -- -- b | |
|
| 149 |
+
/// -- -- -- c |
|
| 150 |
+
/// -- -- -- d |
|
| 151 |
+
/// -- -- -- -- e
|
| 152 |
+
/// -- f
|
| 153 |
+
/// -- -- g
|
| 154 |
+
/// -- -- -- h
|
| 155 |
+
/// -- i <---
|
| 156 |
+
/// -- -- j |
|
| 157 |
+
/// -- -- -- k
|
| 158 |
+
::arrow::Result<std::vector<int>> GetFieldIndices(
|
| 159 |
+
const std::vector<int>& column_indices) const {
|
| 160 |
+
const schema::GroupNode* group = descr->group_node();
|
| 161 |
+
std::unordered_set<int> already_added;
|
| 162 |
+
|
| 163 |
+
std::vector<int> out;
|
| 164 |
+
for (int column_idx : column_indices) {
|
| 165 |
+
if (column_idx < 0 || column_idx >= descr->num_columns()) {
|
| 166 |
+
return ::arrow::Status::IndexError("Column index ", column_idx, " is not valid");
|
| 167 |
+
}
|
| 168 |
+
|
| 169 |
+
auto field_node = descr->GetColumnRoot(column_idx);
|
| 170 |
+
auto field_idx = group->FieldIndex(*field_node);
|
| 171 |
+
if (field_idx == -1) {
|
| 172 |
+
return ::arrow::Status::IndexError("Column index ", column_idx, " is not valid");
|
| 173 |
+
}
|
| 174 |
+
|
| 175 |
+
if (already_added.insert(field_idx).second) {
|
| 176 |
+
out.push_back(field_idx);
|
| 177 |
+
}
|
| 178 |
+
}
|
| 179 |
+
return out;
|
| 180 |
+
}
|
| 181 |
+
};
|
| 182 |
+
|
| 183 |
+
} // namespace arrow
|
| 184 |
+
} // namespace parquet
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/test_util.h
ADDED
|
@@ -0,0 +1,487 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <limits>
|
| 21 |
+
#include <memory>
|
| 22 |
+
#include <random>
|
| 23 |
+
#include <string>
|
| 24 |
+
#include <utility>
|
| 25 |
+
#include <vector>
|
| 26 |
+
|
| 27 |
+
#include "arrow/array.h"
|
| 28 |
+
#include "arrow/array/builder_binary.h"
|
| 29 |
+
#include "arrow/array/builder_decimal.h"
|
| 30 |
+
#include "arrow/array/builder_primitive.h"
|
| 31 |
+
#include "arrow/testing/gtest_util.h"
|
| 32 |
+
#include "arrow/testing/random.h"
|
| 33 |
+
#include "arrow/type_fwd.h"
|
| 34 |
+
#include "arrow/type_traits.h"
|
| 35 |
+
#include "arrow/util/decimal.h"
|
| 36 |
+
#include "arrow/util/float16.h"
|
| 37 |
+
#include "parquet/column_reader.h"
|
| 38 |
+
#include "parquet/test_util.h"
|
| 39 |
+
|
| 40 |
+
namespace parquet {
|
| 41 |
+
|
| 42 |
+
using internal::RecordReader;
|
| 43 |
+
|
| 44 |
+
namespace arrow {
|
| 45 |
+
|
| 46 |
+
using ::arrow::Array;
|
| 47 |
+
using ::arrow::ChunkedArray;
|
| 48 |
+
using ::arrow::Status;
|
| 49 |
+
|
| 50 |
+
template <typename T, int32_t PRECISION, typename = ::arrow::enable_if_decimal<T>>
|
| 51 |
+
struct DecimalWithPrecisionAndScale {
|
| 52 |
+
using type = T;
|
| 53 |
+
static_assert(PRECISION >= T::kMinPrecision && PRECISION <= T::kMaxPrecision,
|
| 54 |
+
"Invalid precision value");
|
| 55 |
+
static constexpr ::arrow::Type::type type_id = T::type_id;
|
| 56 |
+
static constexpr int32_t precision = PRECISION;
|
| 57 |
+
static constexpr int32_t scale = PRECISION - 1;
|
| 58 |
+
};
|
| 59 |
+
template <int32_t PRECISION>
|
| 60 |
+
using Decimal32WithPrecisionAndScale =
|
| 61 |
+
DecimalWithPrecisionAndScale<::arrow::Decimal32Type, PRECISION>;
|
| 62 |
+
template <int32_t PRECISION>
|
| 63 |
+
using Decimal64WithPrecisionAndScale =
|
| 64 |
+
DecimalWithPrecisionAndScale<::arrow::Decimal64Type, PRECISION>;
|
| 65 |
+
template <int32_t PRECISION>
|
| 66 |
+
using Decimal128WithPrecisionAndScale =
|
| 67 |
+
DecimalWithPrecisionAndScale<::arrow::Decimal128Type, PRECISION>;
|
| 68 |
+
template <int32_t PRECISION>
|
| 69 |
+
using Decimal256WithPrecisionAndScale =
|
| 70 |
+
DecimalWithPrecisionAndScale<::arrow::Decimal256Type, PRECISION>;
|
| 71 |
+
|
| 72 |
+
template <class ArrowType>
|
| 73 |
+
::arrow::enable_if_floating_point<ArrowType, Status> NonNullArray(
|
| 74 |
+
size_t size, std::shared_ptr<Array>* out) {
|
| 75 |
+
using c_type = typename ArrowType::c_type;
|
| 76 |
+
std::vector<c_type> values;
|
| 77 |
+
if constexpr (::arrow::is_half_float_type<ArrowType>::value) {
|
| 78 |
+
values.resize(size);
|
| 79 |
+
test::random_float16_numbers(static_cast<int>(size), 0, ::arrow::util::Float16(0.0f),
|
| 80 |
+
::arrow::util::Float16(1.0f), values.data());
|
| 81 |
+
} else {
|
| 82 |
+
::arrow::random_real(size, 0, static_cast<c_type>(0), static_cast<c_type>(1),
|
| 83 |
+
&values);
|
| 84 |
+
}
|
| 85 |
+
::arrow::NumericBuilder<ArrowType> builder;
|
| 86 |
+
RETURN_NOT_OK(builder.AppendValues(values.data(), values.size()));
|
| 87 |
+
return builder.Finish(out);
|
| 88 |
+
}
|
| 89 |
+
|
| 90 |
+
template <class ArrowType>
|
| 91 |
+
::arrow::enable_if_integer<ArrowType, Status> NonNullArray(size_t size,
|
| 92 |
+
std::shared_ptr<Array>* out) {
|
| 93 |
+
std::vector<typename ArrowType::c_type> values;
|
| 94 |
+
::arrow::randint(size, 0, 64, &values);
|
| 95 |
+
|
| 96 |
+
// Passing data type so this will work with TimestampType too
|
| 97 |
+
::arrow::NumericBuilder<ArrowType> builder(std::make_shared<ArrowType>(),
|
| 98 |
+
::arrow::default_memory_pool());
|
| 99 |
+
RETURN_NOT_OK(builder.AppendValues(values.data(), values.size()));
|
| 100 |
+
return builder.Finish(out);
|
| 101 |
+
}
|
| 102 |
+
|
| 103 |
+
template <class ArrowType>
|
| 104 |
+
::arrow::enable_if_date<ArrowType, Status> NonNullArray(size_t size,
|
| 105 |
+
std::shared_ptr<Array>* out) {
|
| 106 |
+
std::vector<typename ArrowType::c_type> values;
|
| 107 |
+
::arrow::randint(size, 0, 24, &values);
|
| 108 |
+
for (size_t i = 0; i < size; i++) {
|
| 109 |
+
values[i] *= 86400000;
|
| 110 |
+
}
|
| 111 |
+
|
| 112 |
+
// Passing data type so this will work with TimestampType too
|
| 113 |
+
::arrow::NumericBuilder<ArrowType> builder(std::make_shared<ArrowType>(),
|
| 114 |
+
::arrow::default_memory_pool());
|
| 115 |
+
RETURN_NOT_OK(builder.AppendValues(values.data(), values.size()));
|
| 116 |
+
return builder.Finish(out);
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
template <class ArrowType>
|
| 120 |
+
::arrow::enable_if_base_binary<ArrowType, Status> NonNullArray(
|
| 121 |
+
size_t size, std::shared_ptr<Array>* out) {
|
| 122 |
+
using BuilderType = typename ::arrow::TypeTraits<ArrowType>::BuilderType;
|
| 123 |
+
BuilderType builder;
|
| 124 |
+
for (size_t i = 0; i < size; i++) {
|
| 125 |
+
RETURN_NOT_OK(builder.Append("test-string"));
|
| 126 |
+
}
|
| 127 |
+
return builder.Finish(out);
|
| 128 |
+
}
|
| 129 |
+
|
| 130 |
+
template <typename ArrowType>
|
| 131 |
+
::arrow::enable_if_fixed_size_binary<ArrowType, Status> NonNullArray(
|
| 132 |
+
size_t size, std::shared_ptr<Array>* out) {
|
| 133 |
+
using BuilderType = typename ::arrow::TypeTraits<ArrowType>::BuilderType;
|
| 134 |
+
// set byte_width to the length of "fixed": 5
|
| 135 |
+
// todo: find a way to generate test data with more diversity.
|
| 136 |
+
BuilderType builder(::arrow::fixed_size_binary(5));
|
| 137 |
+
for (size_t i = 0; i < size; i++) {
|
| 138 |
+
RETURN_NOT_OK(builder.Append("fixed"));
|
| 139 |
+
}
|
| 140 |
+
return builder.Finish(out);
|
| 141 |
+
}
|
| 142 |
+
|
| 143 |
+
template <int32_t byte_width>
|
| 144 |
+
static void random_decimals(int64_t n, uint32_t seed, int32_t precision, uint8_t* out) {
|
| 145 |
+
auto gen = ::arrow::random::RandomArrayGenerator(seed);
|
| 146 |
+
std::shared_ptr<Array> decimals;
|
| 147 |
+
if constexpr (byte_width == 4) {
|
| 148 |
+
decimals = gen.Decimal32(::arrow::decimal32(precision, 0), n);
|
| 149 |
+
} else if constexpr (byte_width == 8) {
|
| 150 |
+
decimals = gen.Decimal64(::arrow::decimal64(precision, 0), n);
|
| 151 |
+
} else if constexpr (byte_width == 16) {
|
| 152 |
+
decimals = gen.Decimal128(::arrow::decimal128(precision, 0), n);
|
| 153 |
+
} else {
|
| 154 |
+
decimals = gen.Decimal256(::arrow::decimal256(precision, 0), n);
|
| 155 |
+
}
|
| 156 |
+
std::memcpy(out, decimals->data()->GetValues<uint8_t>(1, 0), byte_width * n);
|
| 157 |
+
}
|
| 158 |
+
|
| 159 |
+
template <typename ArrowType, int32_t precision = ArrowType::precision>
|
| 160 |
+
::arrow::enable_if_t<std::is_same_v<ArrowType, DecimalWithPrecisionAndScale<
|
| 161 |
+
typename ArrowType::type, precision>>,
|
| 162 |
+
Status>
|
| 163 |
+
NonNullArray(size_t size, std::shared_ptr<Array>* out) {
|
| 164 |
+
constexpr int32_t kDecimalPrecision = precision;
|
| 165 |
+
constexpr int32_t kDecimalScale = ArrowType::scale;
|
| 166 |
+
|
| 167 |
+
const auto type =
|
| 168 |
+
std::make_shared<typename ArrowType::type>(kDecimalPrecision, kDecimalScale);
|
| 169 |
+
const int32_t byte_width = type->byte_width();
|
| 170 |
+
|
| 171 |
+
constexpr int32_t seed = 0;
|
| 172 |
+
|
| 173 |
+
ARROW_ASSIGN_OR_RAISE(auto out_buf, ::arrow::AllocateBuffer(size * byte_width));
|
| 174 |
+
random_decimals<ArrowType::type::kByteWidth>(size, seed, kDecimalPrecision,
|
| 175 |
+
out_buf->mutable_data());
|
| 176 |
+
|
| 177 |
+
using Builder = typename ::arrow::TypeTraits<typename ArrowType::type>::BuilderType;
|
| 178 |
+
Builder builder(type);
|
| 179 |
+
RETURN_NOT_OK(builder.AppendValues(out_buf->data(), size));
|
| 180 |
+
return builder.Finish(out);
|
| 181 |
+
}
|
| 182 |
+
|
| 183 |
+
template <class ArrowType>
|
| 184 |
+
::arrow::enable_if_boolean<ArrowType, Status> NonNullArray(size_t size,
|
| 185 |
+
std::shared_ptr<Array>* out) {
|
| 186 |
+
std::vector<uint8_t> values;
|
| 187 |
+
::arrow::randint(size, 0, 1, &values);
|
| 188 |
+
::arrow::BooleanBuilder builder;
|
| 189 |
+
RETURN_NOT_OK(builder.AppendValues(values.data(), values.size()));
|
| 190 |
+
return builder.Finish(out);
|
| 191 |
+
}
|
| 192 |
+
|
| 193 |
+
// This helper function only supports (size/2) nulls.
|
| 194 |
+
template <typename ArrowType>
|
| 195 |
+
::arrow::enable_if_floating_point<ArrowType, Status> NullableArray(
|
| 196 |
+
size_t size, size_t num_nulls, uint32_t seed, std::shared_ptr<Array>* out) {
|
| 197 |
+
using c_type = typename ArrowType::c_type;
|
| 198 |
+
std::vector<c_type> values;
|
| 199 |
+
if constexpr (::arrow::is_half_float_type<ArrowType>::value) {
|
| 200 |
+
values.resize(size);
|
| 201 |
+
test::random_float16_numbers(static_cast<int>(size), 0, ::arrow::util::Float16(-1e4f),
|
| 202 |
+
::arrow::util::Float16(1e4f), values.data());
|
| 203 |
+
} else {
|
| 204 |
+
::arrow::random_real(size, seed, static_cast<c_type>(-1e10),
|
| 205 |
+
static_cast<c_type>(1e10), &values);
|
| 206 |
+
}
|
| 207 |
+
std::vector<uint8_t> valid_bytes(size, 1);
|
| 208 |
+
|
| 209 |
+
for (size_t i = 0; i < num_nulls; i++) {
|
| 210 |
+
valid_bytes[i * 2] = 0;
|
| 211 |
+
}
|
| 212 |
+
|
| 213 |
+
::arrow::NumericBuilder<ArrowType> builder;
|
| 214 |
+
if (values.size() > 0) {
|
| 215 |
+
RETURN_NOT_OK(builder.AppendValues(values.data(), values.size(), valid_bytes.data()));
|
| 216 |
+
}
|
| 217 |
+
return builder.Finish(out);
|
| 218 |
+
}
|
| 219 |
+
|
| 220 |
+
// This helper function only supports (size/2) nulls.
|
| 221 |
+
template <typename ArrowType>
|
| 222 |
+
::arrow::enable_if_integer<ArrowType, Status> NullableArray(size_t size, size_t num_nulls,
|
| 223 |
+
uint32_t seed,
|
| 224 |
+
std::shared_ptr<Array>* out) {
|
| 225 |
+
std::vector<typename ArrowType::c_type> values;
|
| 226 |
+
|
| 227 |
+
// Seed is random in Arrow right now
|
| 228 |
+
(void)seed;
|
| 229 |
+
::arrow::randint(size, 0, 64, &values);
|
| 230 |
+
std::vector<uint8_t> valid_bytes(size, 1);
|
| 231 |
+
|
| 232 |
+
for (size_t i = 0; i < num_nulls; i++) {
|
| 233 |
+
valid_bytes[i * 2] = 0;
|
| 234 |
+
}
|
| 235 |
+
|
| 236 |
+
// Passing data type so this will work with TimestampType too
|
| 237 |
+
::arrow::NumericBuilder<ArrowType> builder(std::make_shared<ArrowType>(),
|
| 238 |
+
::arrow::default_memory_pool());
|
| 239 |
+
RETURN_NOT_OK(builder.AppendValues(values.data(), values.size(), valid_bytes.data()));
|
| 240 |
+
return builder.Finish(out);
|
| 241 |
+
}
|
| 242 |
+
|
| 243 |
+
template <typename ArrowType>
|
| 244 |
+
::arrow::enable_if_date<ArrowType, Status> NullableArray(size_t size, size_t num_nulls,
|
| 245 |
+
uint32_t seed,
|
| 246 |
+
std::shared_ptr<Array>* out) {
|
| 247 |
+
std::vector<typename ArrowType::c_type> values;
|
| 248 |
+
|
| 249 |
+
// Seed is random in Arrow right now
|
| 250 |
+
(void)seed;
|
| 251 |
+
::arrow::randint(size, 0, 24, &values);
|
| 252 |
+
for (size_t i = 0; i < size; i++) {
|
| 253 |
+
values[i] *= 86400000;
|
| 254 |
+
}
|
| 255 |
+
std::vector<uint8_t> valid_bytes(size, 1);
|
| 256 |
+
|
| 257 |
+
for (size_t i = 0; i < num_nulls; i++) {
|
| 258 |
+
valid_bytes[i * 2] = 0;
|
| 259 |
+
}
|
| 260 |
+
|
| 261 |
+
// Passing data type so this will work with TimestampType too
|
| 262 |
+
::arrow::NumericBuilder<ArrowType> builder(std::make_shared<ArrowType>(),
|
| 263 |
+
::arrow::default_memory_pool());
|
| 264 |
+
RETURN_NOT_OK(builder.AppendValues(values.data(), values.size(), valid_bytes.data()));
|
| 265 |
+
return builder.Finish(out);
|
| 266 |
+
}
|
| 267 |
+
|
| 268 |
+
// This helper function only supports (size/2) nulls yet.
|
| 269 |
+
template <typename ArrowType>
|
| 270 |
+
::arrow::enable_if_base_binary<ArrowType, Status> NullableArray(
|
| 271 |
+
size_t size, size_t num_nulls, uint32_t seed, std::shared_ptr<::arrow::Array>* out) {
|
| 272 |
+
std::vector<uint8_t> valid_bytes(size, 1);
|
| 273 |
+
|
| 274 |
+
for (size_t i = 0; i < num_nulls; i++) {
|
| 275 |
+
valid_bytes[i * 2] = 0;
|
| 276 |
+
}
|
| 277 |
+
|
| 278 |
+
using BuilderType = typename ::arrow::TypeTraits<ArrowType>::BuilderType;
|
| 279 |
+
BuilderType builder;
|
| 280 |
+
|
| 281 |
+
const int kBufferSize = 10;
|
| 282 |
+
uint8_t buffer[kBufferSize];
|
| 283 |
+
for (size_t i = 0; i < size; i++) {
|
| 284 |
+
if (!valid_bytes[i]) {
|
| 285 |
+
RETURN_NOT_OK(builder.AppendNull());
|
| 286 |
+
} else {
|
| 287 |
+
::arrow::random_bytes(kBufferSize, seed + static_cast<uint32_t>(i), buffer);
|
| 288 |
+
if (ArrowType::is_utf8) {
|
| 289 |
+
// Trivially force data to be valid UTF8 by making it all ASCII
|
| 290 |
+
for (auto& byte : buffer) {
|
| 291 |
+
byte &= 0x7f;
|
| 292 |
+
}
|
| 293 |
+
}
|
| 294 |
+
RETURN_NOT_OK(builder.Append(buffer, kBufferSize));
|
| 295 |
+
}
|
| 296 |
+
}
|
| 297 |
+
return builder.Finish(out);
|
| 298 |
+
}
|
| 299 |
+
|
| 300 |
+
// This helper function only supports (size/2) nulls yet,
|
| 301 |
+
// same as NullableArray<String|Binary>(..)
|
| 302 |
+
template <typename ArrowType>
|
| 303 |
+
::arrow::enable_if_fixed_size_binary<ArrowType, Status> NullableArray(
|
| 304 |
+
size_t size, size_t num_nulls, uint32_t seed, std::shared_ptr<::arrow::Array>* out) {
|
| 305 |
+
std::vector<uint8_t> valid_bytes(size, 1);
|
| 306 |
+
|
| 307 |
+
for (size_t i = 0; i < num_nulls; i++) {
|
| 308 |
+
valid_bytes[i * 2] = 0;
|
| 309 |
+
}
|
| 310 |
+
|
| 311 |
+
using BuilderType = typename ::arrow::TypeTraits<ArrowType>::BuilderType;
|
| 312 |
+
const int byte_width = 10;
|
| 313 |
+
BuilderType builder(::arrow::fixed_size_binary(byte_width));
|
| 314 |
+
|
| 315 |
+
const int kBufferSize = byte_width;
|
| 316 |
+
uint8_t buffer[kBufferSize];
|
| 317 |
+
for (size_t i = 0; i < size; i++) {
|
| 318 |
+
if (!valid_bytes[i]) {
|
| 319 |
+
RETURN_NOT_OK(builder.AppendNull());
|
| 320 |
+
} else {
|
| 321 |
+
::arrow::random_bytes(kBufferSize, seed + static_cast<uint32_t>(i), buffer);
|
| 322 |
+
RETURN_NOT_OK(builder.Append(buffer));
|
| 323 |
+
}
|
| 324 |
+
}
|
| 325 |
+
return builder.Finish(out);
|
| 326 |
+
}
|
| 327 |
+
|
| 328 |
+
template <typename ArrowType, int32_t precision = ArrowType::precision>
|
| 329 |
+
::arrow::enable_if_t<std::is_same_v<ArrowType, DecimalWithPrecisionAndScale<
|
| 330 |
+
typename ArrowType::type, precision>>,
|
| 331 |
+
Status>
|
| 332 |
+
NullableArray(size_t size, size_t num_nulls, uint32_t seed,
|
| 333 |
+
std::shared_ptr<::arrow::Array>* out) {
|
| 334 |
+
std::vector<uint8_t> valid_bytes(size, '\1');
|
| 335 |
+
|
| 336 |
+
for (size_t i = 0; i < num_nulls; ++i) {
|
| 337 |
+
valid_bytes[i * 2] = '\0';
|
| 338 |
+
}
|
| 339 |
+
|
| 340 |
+
constexpr int32_t kDecimalPrecision = precision;
|
| 341 |
+
constexpr int32_t kDecimalScale = ArrowType::scale;
|
| 342 |
+
|
| 343 |
+
const auto type =
|
| 344 |
+
std::make_shared<typename ArrowType::type>(kDecimalPrecision, kDecimalScale);
|
| 345 |
+
const int32_t byte_width = type->byte_width();
|
| 346 |
+
|
| 347 |
+
ARROW_ASSIGN_OR_RAISE(auto out_buf, ::arrow::AllocateBuffer(size * byte_width));
|
| 348 |
+
random_decimals<ArrowType::type::kByteWidth>(size, seed, precision,
|
| 349 |
+
out_buf->mutable_data());
|
| 350 |
+
|
| 351 |
+
using Builder = typename ::arrow::TypeTraits<typename ArrowType::type>::BuilderType;
|
| 352 |
+
Builder builder(type);
|
| 353 |
+
RETURN_NOT_OK(builder.AppendValues(out_buf->data(), size, valid_bytes.data()));
|
| 354 |
+
return builder.Finish(out);
|
| 355 |
+
}
|
| 356 |
+
|
| 357 |
+
// This helper function only supports (size/2) nulls yet.
|
| 358 |
+
template <class ArrowType>
|
| 359 |
+
::arrow::enable_if_boolean<ArrowType, Status> NullableArray(size_t size, size_t num_nulls,
|
| 360 |
+
uint32_t seed,
|
| 361 |
+
std::shared_ptr<Array>* out) {
|
| 362 |
+
std::vector<uint8_t> values;
|
| 363 |
+
|
| 364 |
+
// Seed is random in Arrow right now
|
| 365 |
+
(void)seed;
|
| 366 |
+
|
| 367 |
+
::arrow::randint(size, 0, 1, &values);
|
| 368 |
+
std::vector<uint8_t> valid_bytes(size, 1);
|
| 369 |
+
|
| 370 |
+
for (size_t i = 0; i < num_nulls; i++) {
|
| 371 |
+
valid_bytes[i * 2] = 0;
|
| 372 |
+
}
|
| 373 |
+
|
| 374 |
+
::arrow::BooleanBuilder builder;
|
| 375 |
+
RETURN_NOT_OK(builder.AppendValues(values.data(), values.size(), valid_bytes.data()));
|
| 376 |
+
return builder.Finish(out);
|
| 377 |
+
}
|
| 378 |
+
|
| 379 |
+
/// Wrap an Array into a ListArray by splitting it up into size lists.
|
| 380 |
+
///
|
| 381 |
+
/// This helper function only supports (size/2) nulls.
|
| 382 |
+
Status MakeListArray(const std::shared_ptr<Array>& values, int64_t size,
|
| 383 |
+
int64_t null_count, const std::string& item_name,
|
| 384 |
+
bool nullable_values, std::shared_ptr<::arrow::ListArray>* out) {
|
| 385 |
+
// We always include an empty list
|
| 386 |
+
int64_t non_null_entries = size - null_count - 1;
|
| 387 |
+
int64_t length_per_entry = values->length() / non_null_entries;
|
| 388 |
+
|
| 389 |
+
auto offsets = AllocateBuffer();
|
| 390 |
+
RETURN_NOT_OK(offsets->Resize((size + 1) * sizeof(int32_t)));
|
| 391 |
+
int32_t* offsets_ptr = reinterpret_cast<int32_t*>(offsets->mutable_data());
|
| 392 |
+
|
| 393 |
+
auto null_bitmap = AllocateBuffer();
|
| 394 |
+
int64_t bitmap_size = ::arrow::bit_util::BytesForBits(size);
|
| 395 |
+
RETURN_NOT_OK(null_bitmap->Resize(bitmap_size));
|
| 396 |
+
uint8_t* null_bitmap_ptr = null_bitmap->mutable_data();
|
| 397 |
+
memset(null_bitmap_ptr, 0, bitmap_size);
|
| 398 |
+
|
| 399 |
+
int32_t current_offset = 0;
|
| 400 |
+
for (int64_t i = 0; i < size; i++) {
|
| 401 |
+
offsets_ptr[i] = current_offset;
|
| 402 |
+
if (!(((i % 2) == 0) && ((i / 2) < null_count))) {
|
| 403 |
+
// Non-null list (list with index 1 is always empty).
|
| 404 |
+
::arrow::bit_util::SetBit(null_bitmap_ptr, i);
|
| 405 |
+
if (i != 1) {
|
| 406 |
+
current_offset += static_cast<int32_t>(length_per_entry);
|
| 407 |
+
}
|
| 408 |
+
}
|
| 409 |
+
}
|
| 410 |
+
offsets_ptr[size] = static_cast<int32_t>(values->length());
|
| 411 |
+
|
| 412 |
+
auto value_field = ::arrow::field(item_name, values->type(), nullable_values);
|
| 413 |
+
*out = std::make_shared<::arrow::ListArray>(::arrow::list(value_field), size, offsets,
|
| 414 |
+
values, null_bitmap, null_count);
|
| 415 |
+
|
| 416 |
+
return Status::OK();
|
| 417 |
+
}
|
| 418 |
+
|
| 419 |
+
// Make an array containing only empty lists, with a null values array
|
| 420 |
+
Status MakeEmptyListsArray(int64_t size, std::shared_ptr<Array>* out_array) {
|
| 421 |
+
// Allocate an offsets buffer containing only zeroes
|
| 422 |
+
const int64_t offsets_nbytes = (size + 1) * sizeof(int32_t);
|
| 423 |
+
ARROW_ASSIGN_OR_RAISE(auto offsets_buffer, ::arrow::AllocateBuffer(offsets_nbytes));
|
| 424 |
+
memset(offsets_buffer->mutable_data(), 0, offsets_nbytes);
|
| 425 |
+
|
| 426 |
+
auto value_field =
|
| 427 |
+
::arrow::field("item", ::arrow::float64(), false /* nullable_values */);
|
| 428 |
+
auto list_type = ::arrow::list(value_field);
|
| 429 |
+
|
| 430 |
+
std::vector<std::shared_ptr<Buffer>> child_buffers = {nullptr /* null bitmap */,
|
| 431 |
+
nullptr /* values */};
|
| 432 |
+
auto child_data =
|
| 433 |
+
::arrow::ArrayData::Make(value_field->type(), 0, std::move(child_buffers));
|
| 434 |
+
|
| 435 |
+
std::vector<std::shared_ptr<Buffer>> buffers = {nullptr /* bitmap */,
|
| 436 |
+
std::move(offsets_buffer)};
|
| 437 |
+
auto array_data = ::arrow::ArrayData::Make(list_type, size, std::move(buffers));
|
| 438 |
+
array_data->child_data.push_back(child_data);
|
| 439 |
+
|
| 440 |
+
*out_array = ::arrow::MakeArray(array_data);
|
| 441 |
+
return Status::OK();
|
| 442 |
+
}
|
| 443 |
+
|
| 444 |
+
std::shared_ptr<::arrow::Table> MakeSimpleTable(
|
| 445 |
+
const std::shared_ptr<ChunkedArray>& values, bool nullable) {
|
| 446 |
+
auto schema = ::arrow::schema({::arrow::field("col", values->type(), nullable)});
|
| 447 |
+
return ::arrow::Table::Make(schema, {values});
|
| 448 |
+
}
|
| 449 |
+
|
| 450 |
+
std::shared_ptr<::arrow::Table> MakeSimpleTable(const std::shared_ptr<Array>& values,
|
| 451 |
+
bool nullable) {
|
| 452 |
+
auto carr = std::make_shared<::arrow::ChunkedArray>(values);
|
| 453 |
+
return MakeSimpleTable(carr, nullable);
|
| 454 |
+
}
|
| 455 |
+
|
| 456 |
+
template <typename T>
|
| 457 |
+
void ExpectArray(T* expected, Array* result) {
|
| 458 |
+
auto p_array = static_cast<::arrow::PrimitiveArray*>(result);
|
| 459 |
+
for (int i = 0; i < result->length(); i++) {
|
| 460 |
+
EXPECT_EQ(expected[i], reinterpret_cast<const T*>(p_array->values()->data())[i]);
|
| 461 |
+
}
|
| 462 |
+
}
|
| 463 |
+
|
| 464 |
+
template <typename ArrowType>
|
| 465 |
+
void ExpectArrayT(void* expected, Array* result) {
|
| 466 |
+
::arrow::PrimitiveArray* p_array = static_cast<::arrow::PrimitiveArray*>(result);
|
| 467 |
+
for (int64_t i = 0; i < result->length(); i++) {
|
| 468 |
+
EXPECT_EQ(reinterpret_cast<typename ArrowType::c_type*>(expected)[i],
|
| 469 |
+
reinterpret_cast<const typename ArrowType::c_type*>(
|
| 470 |
+
p_array->values()->data())[i]);
|
| 471 |
+
}
|
| 472 |
+
}
|
| 473 |
+
|
| 474 |
+
template <>
|
| 475 |
+
void ExpectArrayT<::arrow::BooleanType>(void* expected, Array* result) {
|
| 476 |
+
::arrow::BooleanBuilder builder;
|
| 477 |
+
ARROW_EXPECT_OK(
|
| 478 |
+
builder.AppendValues(reinterpret_cast<uint8_t*>(expected), result->length()));
|
| 479 |
+
|
| 480 |
+
std::shared_ptr<Array> expected_array;
|
| 481 |
+
ARROW_EXPECT_OK(builder.Finish(&expected_array));
|
| 482 |
+
EXPECT_TRUE(result->Equals(*expected_array));
|
| 483 |
+
}
|
| 484 |
+
|
| 485 |
+
} // namespace arrow
|
| 486 |
+
|
| 487 |
+
} // namespace parquet
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/writer.h
ADDED
|
@@ -0,0 +1,176 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <cstdint>
|
| 21 |
+
#include <memory>
|
| 22 |
+
|
| 23 |
+
#include "parquet/platform.h"
|
| 24 |
+
#include "parquet/properties.h"
|
| 25 |
+
|
| 26 |
+
namespace arrow {
|
| 27 |
+
|
| 28 |
+
class Array;
|
| 29 |
+
class ChunkedArray;
|
| 30 |
+
class RecordBatch;
|
| 31 |
+
class Schema;
|
| 32 |
+
class Table;
|
| 33 |
+
|
| 34 |
+
} // namespace arrow
|
| 35 |
+
|
| 36 |
+
namespace parquet {
|
| 37 |
+
|
| 38 |
+
class FileMetaData;
|
| 39 |
+
class ParquetFileWriter;
|
| 40 |
+
|
| 41 |
+
namespace arrow {
|
| 42 |
+
|
| 43 |
+
/// \brief Iterative FileWriter class
|
| 44 |
+
///
|
| 45 |
+
/// For basic usage, can write a Table at a time, creating one or more row
|
| 46 |
+
/// groups per write call.
|
| 47 |
+
///
|
| 48 |
+
/// For advanced usage, can write column-by-column: Start a new RowGroup or
|
| 49 |
+
/// Chunk with NewRowGroup, then write column-by-column the whole column chunk.
|
| 50 |
+
///
|
| 51 |
+
/// If PARQUET:field_id is present as a metadata key on a field, and the corresponding
|
| 52 |
+
/// value is a nonnegative integer, then it will be used as the field_id in the parquet
|
| 53 |
+
/// file.
|
| 54 |
+
class PARQUET_EXPORT FileWriter {
|
| 55 |
+
public:
|
| 56 |
+
static ::arrow::Status Make(MemoryPool* pool, std::unique_ptr<ParquetFileWriter> writer,
|
| 57 |
+
std::shared_ptr<::arrow::Schema> schema,
|
| 58 |
+
std::shared_ptr<ArrowWriterProperties> arrow_properties,
|
| 59 |
+
std::unique_ptr<FileWriter>* out);
|
| 60 |
+
|
| 61 |
+
/// \brief Try to create an Arrow to Parquet file writer.
|
| 62 |
+
///
|
| 63 |
+
/// \param schema schema of data that will be passed.
|
| 64 |
+
/// \param pool memory pool to use.
|
| 65 |
+
/// \param sink output stream to write Parquet data.
|
| 66 |
+
/// \param properties general Parquet writer properties.
|
| 67 |
+
/// \param arrow_properties Arrow-specific writer properties.
|
| 68 |
+
///
|
| 69 |
+
/// \since 11.0.0
|
| 70 |
+
static ::arrow::Result<std::unique_ptr<FileWriter>> Open(
|
| 71 |
+
const ::arrow::Schema& schema, MemoryPool* pool,
|
| 72 |
+
std::shared_ptr<::arrow::io::OutputStream> sink,
|
| 73 |
+
std::shared_ptr<WriterProperties> properties = default_writer_properties(),
|
| 74 |
+
std::shared_ptr<ArrowWriterProperties> arrow_properties =
|
| 75 |
+
default_arrow_writer_properties());
|
| 76 |
+
|
| 77 |
+
/// Return the Arrow schema to be written to.
|
| 78 |
+
virtual std::shared_ptr<::arrow::Schema> schema() const = 0;
|
| 79 |
+
|
| 80 |
+
/// \brief Write a Table to Parquet.
|
| 81 |
+
///
|
| 82 |
+
/// \param table Arrow table to write.
|
| 83 |
+
/// \param chunk_size maximum number of rows to write per row group.
|
| 84 |
+
virtual ::arrow::Status WriteTable(
|
| 85 |
+
const ::arrow::Table& table, int64_t chunk_size = DEFAULT_MAX_ROW_GROUP_LENGTH) = 0;
|
| 86 |
+
|
| 87 |
+
/// \brief Start a new row group.
|
| 88 |
+
///
|
| 89 |
+
/// Returns an error if not all columns have been written.
|
| 90 |
+
virtual ::arrow::Status NewRowGroup() = 0;
|
| 91 |
+
|
| 92 |
+
/// \brief Write ColumnChunk in row group using an array.
|
| 93 |
+
virtual ::arrow::Status WriteColumnChunk(const ::arrow::Array& data) = 0;
|
| 94 |
+
|
| 95 |
+
/// \brief Write ColumnChunk in row group using slice of a ChunkedArray
|
| 96 |
+
virtual ::arrow::Status WriteColumnChunk(
|
| 97 |
+
const std::shared_ptr<::arrow::ChunkedArray>& data, int64_t offset,
|
| 98 |
+
int64_t size) = 0;
|
| 99 |
+
|
| 100 |
+
/// \brief Write ColumnChunk in a row group using a ChunkedArray
|
| 101 |
+
virtual ::arrow::Status WriteColumnChunk(
|
| 102 |
+
const std::shared_ptr<::arrow::ChunkedArray>& data) = 0;
|
| 103 |
+
|
| 104 |
+
/// \brief Start a new buffered row group.
|
| 105 |
+
///
|
| 106 |
+
/// Returns an error if not all columns have been written.
|
| 107 |
+
virtual ::arrow::Status NewBufferedRowGroup() = 0;
|
| 108 |
+
|
| 109 |
+
/// \brief Write a RecordBatch into the buffered row group.
|
| 110 |
+
///
|
| 111 |
+
/// Multiple RecordBatches can be written into the same row group
|
| 112 |
+
/// through this method.
|
| 113 |
+
///
|
| 114 |
+
/// WriterProperties.max_row_group_length() is respected and a new
|
| 115 |
+
/// row group will be created if the current row group exceeds the
|
| 116 |
+
/// limit.
|
| 117 |
+
///
|
| 118 |
+
/// Batches get flushed to the output stream once NewBufferedRowGroup()
|
| 119 |
+
/// or Close() is called.
|
| 120 |
+
///
|
| 121 |
+
/// WARNING: If you are writing multiple files in parallel in the same
|
| 122 |
+
/// executor, deadlock may occur if ArrowWriterProperties::use_threads
|
| 123 |
+
/// is set to true to write columns in parallel. Please disable use_threads
|
| 124 |
+
/// option in this case.
|
| 125 |
+
virtual ::arrow::Status WriteRecordBatch(const ::arrow::RecordBatch& batch) = 0;
|
| 126 |
+
|
| 127 |
+
/// \brief Write the footer and close the file.
|
| 128 |
+
virtual ::arrow::Status Close() = 0;
|
| 129 |
+
virtual ~FileWriter();
|
| 130 |
+
|
| 131 |
+
virtual MemoryPool* memory_pool() const = 0;
|
| 132 |
+
/// \brief Add key-value metadata to the file.
|
| 133 |
+
/// \param[in] key_value_metadata the metadata to add.
|
| 134 |
+
/// \note This will overwrite any existing metadata with the same key.
|
| 135 |
+
/// \return Error if Close() has been called.
|
| 136 |
+
///
|
| 137 |
+
/// WARNING: If `store_schema` is enabled, `ARROW:schema` would be stored
|
| 138 |
+
/// in the key-value metadata. Overwriting this key would result in
|
| 139 |
+
/// `store_schema` being unusable during read.
|
| 140 |
+
virtual ::arrow::Status AddKeyValueMetadata(
|
| 141 |
+
const std::shared_ptr<const ::arrow::KeyValueMetadata>& key_value_metadata) = 0;
|
| 142 |
+
/// \brief Return the file metadata, only available after calling Close().
|
| 143 |
+
virtual const std::shared_ptr<FileMetaData> metadata() const = 0;
|
| 144 |
+
};
|
| 145 |
+
|
| 146 |
+
/// \brief Write Parquet file metadata only to indicated Arrow OutputStream
|
| 147 |
+
PARQUET_EXPORT
|
| 148 |
+
::arrow::Status WriteFileMetaData(const FileMetaData& file_metadata,
|
| 149 |
+
::arrow::io::OutputStream* sink);
|
| 150 |
+
|
| 151 |
+
/// \brief Write metadata-only Parquet file to indicated Arrow OutputStream
|
| 152 |
+
PARQUET_EXPORT
|
| 153 |
+
::arrow::Status WriteMetaDataFile(const FileMetaData& file_metadata,
|
| 154 |
+
::arrow::io::OutputStream* sink);
|
| 155 |
+
|
| 156 |
+
/// \brief Write a Table to Parquet.
|
| 157 |
+
///
|
| 158 |
+
/// This writes one table in a single shot. To write a Parquet file with
|
| 159 |
+
/// multiple tables iteratively, see parquet::arrow::FileWriter.
|
| 160 |
+
///
|
| 161 |
+
/// \param table Table to write.
|
| 162 |
+
/// \param pool memory pool to use.
|
| 163 |
+
/// \param sink output stream to write Parquet data.
|
| 164 |
+
/// \param chunk_size maximum number of rows to write per row group.
|
| 165 |
+
/// \param properties general Parquet writer properties.
|
| 166 |
+
/// \param arrow_properties Arrow-specific writer properties.
|
| 167 |
+
::arrow::Status PARQUET_EXPORT
|
| 168 |
+
WriteTable(const ::arrow::Table& table, MemoryPool* pool,
|
| 169 |
+
std::shared_ptr<::arrow::io::OutputStream> sink,
|
| 170 |
+
int64_t chunk_size = DEFAULT_MAX_ROW_GROUP_LENGTH,
|
| 171 |
+
std::shared_ptr<WriterProperties> properties = default_writer_properties(),
|
| 172 |
+
std::shared_ptr<ArrowWriterProperties> arrow_properties =
|
| 173 |
+
default_arrow_writer_properties());
|
| 174 |
+
|
| 175 |
+
} // namespace arrow
|
| 176 |
+
} // namespace parquet
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/benchmark_util.h
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <random>
|
| 21 |
+
#include <string>
|
| 22 |
+
#include <vector>
|
| 23 |
+
|
| 24 |
+
#include "parquet/types.h"
|
| 25 |
+
|
| 26 |
+
namespace parquet::benchmarks {
|
| 27 |
+
|
| 28 |
+
template <typename T>
|
| 29 |
+
void GenerateBenchmarkData(uint32_t size, uint32_t seed, T* data,
|
| 30 |
+
std::vector<uint8_t>* heap, uint32_t data_string_length);
|
| 31 |
+
|
| 32 |
+
#define _GENERATE_BENCHMARK_DATA_DECL(KLASS) \
|
| 33 |
+
template <> \
|
| 34 |
+
void GenerateBenchmarkData(uint32_t size, uint32_t seed, KLASS* data, \
|
| 35 |
+
std::vector<uint8_t>* heap, uint32_t data_string_length);
|
| 36 |
+
|
| 37 |
+
_GENERATE_BENCHMARK_DATA_DECL(int32_t)
|
| 38 |
+
_GENERATE_BENCHMARK_DATA_DECL(int64_t)
|
| 39 |
+
_GENERATE_BENCHMARK_DATA_DECL(float)
|
| 40 |
+
_GENERATE_BENCHMARK_DATA_DECL(double)
|
| 41 |
+
_GENERATE_BENCHMARK_DATA_DECL(ByteArray)
|
| 42 |
+
_GENERATE_BENCHMARK_DATA_DECL(FLBA)
|
| 43 |
+
_GENERATE_BENCHMARK_DATA_DECL(Int96)
|
| 44 |
+
|
| 45 |
+
#undef _GENERATE_BENCHMARK_DATA_DECL
|
| 46 |
+
|
| 47 |
+
} // namespace parquet::benchmarks
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/bloom_filter.h
ADDED
|
@@ -0,0 +1,392 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <cmath>
|
| 21 |
+
#include <cstdint>
|
| 22 |
+
#include <memory>
|
| 23 |
+
|
| 24 |
+
#include "arrow/util/bit_util.h"
|
| 25 |
+
#include "arrow/util/logging.h"
|
| 26 |
+
#include "parquet/encryption/type_fwd.h"
|
| 27 |
+
#include "parquet/hasher.h"
|
| 28 |
+
#include "parquet/platform.h"
|
| 29 |
+
#include "parquet/types.h"
|
| 30 |
+
|
| 31 |
+
namespace parquet {
|
| 32 |
+
|
| 33 |
+
// A Bloom filter is a compact structure to indicate whether an item is not in a set or
|
| 34 |
+
// probably in a set. The Bloom filter usually consists of a bit set that represents a
|
| 35 |
+
// set of elements, a hash strategy and a Bloom filter algorithm.
|
| 36 |
+
class PARQUET_EXPORT BloomFilter {
|
| 37 |
+
public:
|
| 38 |
+
// Maximum Bloom filter size, it sets to HDFS default block size 128MB
|
| 39 |
+
// This value will be reconsidered when implementing Bloom filter producer.
|
| 40 |
+
static constexpr uint32_t kMaximumBloomFilterBytes = 128 * 1024 * 1024;
|
| 41 |
+
|
| 42 |
+
/// Determine whether an element exist in set or not.
|
| 43 |
+
///
|
| 44 |
+
/// @param hash the element to contain.
|
| 45 |
+
/// @return false if value is definitely not in set, and true means PROBABLY
|
| 46 |
+
/// in set.
|
| 47 |
+
virtual bool FindHash(uint64_t hash) const = 0;
|
| 48 |
+
|
| 49 |
+
/// Insert element to set represented by Bloom filter bitset.
|
| 50 |
+
/// @param hash the hash of value to insert into Bloom filter.
|
| 51 |
+
virtual void InsertHash(uint64_t hash) = 0;
|
| 52 |
+
|
| 53 |
+
/// Insert elements to set represented by Bloom filter bitset.
|
| 54 |
+
/// @param hashes the hash values to insert into Bloom filter.
|
| 55 |
+
/// @param num_values the number of hash values to insert.
|
| 56 |
+
virtual void InsertHashes(const uint64_t* hashes, int num_values) = 0;
|
| 57 |
+
|
| 58 |
+
/// Write this Bloom filter to an output stream. A Bloom filter structure should
|
| 59 |
+
/// include bitset length, hash strategy, algorithm, and bitset.
|
| 60 |
+
///
|
| 61 |
+
/// @param sink the output stream to write
|
| 62 |
+
virtual void WriteTo(ArrowOutputStream* sink) const = 0;
|
| 63 |
+
|
| 64 |
+
/// Get the number of bytes of bitset
|
| 65 |
+
virtual uint32_t GetBitsetSize() const = 0;
|
| 66 |
+
|
| 67 |
+
/// Compute hash for 32 bits value by using its plain encoding result.
|
| 68 |
+
///
|
| 69 |
+
/// @param value the value to hash.
|
| 70 |
+
/// @return hash result.
|
| 71 |
+
virtual uint64_t Hash(int32_t value) const = 0;
|
| 72 |
+
|
| 73 |
+
/// Compute hash for 64 bits value by using its plain encoding result.
|
| 74 |
+
///
|
| 75 |
+
/// @param value the value to hash.
|
| 76 |
+
/// @return hash result.
|
| 77 |
+
virtual uint64_t Hash(int64_t value) const = 0;
|
| 78 |
+
|
| 79 |
+
/// Compute hash for float value by using its plain encoding result.
|
| 80 |
+
///
|
| 81 |
+
/// @param value the value to hash.
|
| 82 |
+
/// @return hash result.
|
| 83 |
+
virtual uint64_t Hash(float value) const = 0;
|
| 84 |
+
|
| 85 |
+
/// Compute hash for double value by using its plain encoding result.
|
| 86 |
+
///
|
| 87 |
+
/// @param value the value to hash.
|
| 88 |
+
/// @return hash result.
|
| 89 |
+
virtual uint64_t Hash(double value) const = 0;
|
| 90 |
+
|
| 91 |
+
/// Compute hash for Int96 value by using its plain encoding result.
|
| 92 |
+
///
|
| 93 |
+
/// @param value the value to hash.
|
| 94 |
+
/// @return hash result.
|
| 95 |
+
virtual uint64_t Hash(const Int96* value) const = 0;
|
| 96 |
+
uint64_t Hash(const Int96& value) const { return Hash(&value); }
|
| 97 |
+
|
| 98 |
+
/// Compute hash for ByteArray value by using its plain encoding result.
|
| 99 |
+
///
|
| 100 |
+
/// @param value the value to hash.
|
| 101 |
+
/// @return hash result.
|
| 102 |
+
virtual uint64_t Hash(const ByteArray* value) const = 0;
|
| 103 |
+
uint64_t Hash(const ByteArray& value) const { return Hash(&value); }
|
| 104 |
+
|
| 105 |
+
/// Compute hash for fixed byte array value by using its plain encoding result.
|
| 106 |
+
///
|
| 107 |
+
/// @param value the value address.
|
| 108 |
+
/// @param len the value length.
|
| 109 |
+
/// @return hash result.
|
| 110 |
+
virtual uint64_t Hash(const FLBA* value, uint32_t len) const = 0;
|
| 111 |
+
uint64_t Hash(const FLBA& value, uint32_t len) const { return Hash(&value, len); }
|
| 112 |
+
|
| 113 |
+
/// Compute hash for std::string_view value by using its plain encoding result.
|
| 114 |
+
///
|
| 115 |
+
/// @param value the value to hash.
|
| 116 |
+
/// @return hash result.
|
| 117 |
+
virtual uint64_t Hash(std::string_view value) const = 0;
|
| 118 |
+
|
| 119 |
+
/// Batch compute hashes for 32 bits values by using its plain encoding result.
|
| 120 |
+
///
|
| 121 |
+
/// @param values values a pointer to the values to hash.
|
| 122 |
+
/// @param num_values the number of values to hash.
|
| 123 |
+
/// @param hashes a pointer to the output hash values, its length should be equal to
|
| 124 |
+
/// num_values.
|
| 125 |
+
virtual void Hashes(const int32_t* values, int num_values, uint64_t* hashes) const = 0;
|
| 126 |
+
|
| 127 |
+
/// Batch compute hashes for 64 bits values by using its plain encoding result.
|
| 128 |
+
///
|
| 129 |
+
/// @param values values a pointer to the values to hash.
|
| 130 |
+
/// @param num_values the number of values to hash.
|
| 131 |
+
/// @param hashes a pointer to the output hash values, its length should be equal to
|
| 132 |
+
/// num_values.
|
| 133 |
+
virtual void Hashes(const int64_t* values, int num_values, uint64_t* hashes) const = 0;
|
| 134 |
+
|
| 135 |
+
/// Batch compute hashes for float values by using its plain encoding result.
|
| 136 |
+
///
|
| 137 |
+
/// @param values values a pointer to the values to hash.
|
| 138 |
+
/// @param num_values the number of values to hash.
|
| 139 |
+
/// @param hashes a pointer to the output hash values, its length should be equal to
|
| 140 |
+
/// num_values.
|
| 141 |
+
virtual void Hashes(const float* values, int num_values, uint64_t* hashes) const = 0;
|
| 142 |
+
|
| 143 |
+
/// Batch compute hashes for double values by using its plain encoding result.
|
| 144 |
+
///
|
| 145 |
+
/// @param values values a pointer to the values to hash.
|
| 146 |
+
/// @param num_values the number of values to hash.
|
| 147 |
+
/// @param hashes a pointer to the output hash values, its length should be equal to
|
| 148 |
+
/// num_values.
|
| 149 |
+
virtual void Hashes(const double* values, int num_values, uint64_t* hashes) const = 0;
|
| 150 |
+
|
| 151 |
+
/// Batch compute hashes for Int96 values by using its plain encoding result.
|
| 152 |
+
///
|
| 153 |
+
/// @param values values a pointer to the values to hash.
|
| 154 |
+
/// @param num_values the number of values to hash.
|
| 155 |
+
/// @param hashes a pointer to the output hash values, its length should be equal to
|
| 156 |
+
/// num_values.
|
| 157 |
+
virtual void Hashes(const Int96* values, int num_values, uint64_t* hashes) const = 0;
|
| 158 |
+
|
| 159 |
+
/// Batch compute hashes for ByteArray values by using its plain encoding result.
|
| 160 |
+
///
|
| 161 |
+
/// @param values values a pointer to the values to hash.
|
| 162 |
+
/// @param num_values the number of values to hash.
|
| 163 |
+
/// @param hashes a pointer to the output hash values, its length should be equal to
|
| 164 |
+
/// num_values.
|
| 165 |
+
virtual void Hashes(const ByteArray* values, int num_values,
|
| 166 |
+
uint64_t* hashes) const = 0;
|
| 167 |
+
|
| 168 |
+
/// Batch compute hashes for fixed byte array values by using its plain encoding result.
|
| 169 |
+
///
|
| 170 |
+
/// @param values values a pointer to the values to hash.
|
| 171 |
+
/// @param type_len the value length.
|
| 172 |
+
/// @param num_values the number of values to hash.
|
| 173 |
+
/// @param hashes a pointer to the output hash values, its length should be equal to
|
| 174 |
+
/// num_values.
|
| 175 |
+
virtual void Hashes(const FLBA* values, uint32_t type_len, int num_values,
|
| 176 |
+
uint64_t* hashes) const = 0;
|
| 177 |
+
|
| 178 |
+
virtual ~BloomFilter() = default;
|
| 179 |
+
|
| 180 |
+
protected:
|
| 181 |
+
// Hash strategy available for Bloom filter.
|
| 182 |
+
enum class HashStrategy : uint32_t { XXHASH = 0 };
|
| 183 |
+
|
| 184 |
+
// Bloom filter algorithm.
|
| 185 |
+
enum class Algorithm : uint32_t { BLOCK = 0 };
|
| 186 |
+
|
| 187 |
+
enum class CompressionStrategy : uint32_t { UNCOMPRESSED = 0 };
|
| 188 |
+
};
|
| 189 |
+
|
| 190 |
+
/// The BlockSplitBloomFilter is implemented using block-based Bloom filters from
|
| 191 |
+
/// Putze et al.'s "Cache-,Hash- and Space-Efficient Bloom filters". The basic idea is to
|
| 192 |
+
/// hash the item to a tiny Bloom filter which size fit a single cache line or smaller.
|
| 193 |
+
///
|
| 194 |
+
/// This implementation sets 8 bits in each tiny Bloom filter. Each tiny Bloom
|
| 195 |
+
/// filter is 32 bytes to take advantage of 32-byte SIMD instructions.
|
| 196 |
+
class PARQUET_EXPORT BlockSplitBloomFilter : public BloomFilter {
|
| 197 |
+
public:
|
| 198 |
+
/// The constructor of BlockSplitBloomFilter. It uses XXH64 as hash function.
|
| 199 |
+
///
|
| 200 |
+
/// \param pool memory pool to use.
|
| 201 |
+
explicit BlockSplitBloomFilter(
|
| 202 |
+
::arrow::MemoryPool* pool = ::arrow::default_memory_pool());
|
| 203 |
+
|
| 204 |
+
/// Initialize the BlockSplitBloomFilter. The range of num_bytes should be within
|
| 205 |
+
/// [kMinimumBloomFilterBytes, kMaximumBloomFilterBytes], it will be
|
| 206 |
+
/// rounded up/down to lower/upper bound if num_bytes is out of range and also
|
| 207 |
+
/// will be rounded up to a power of 2.
|
| 208 |
+
///
|
| 209 |
+
/// @param num_bytes The number of bytes to store Bloom filter bitset.
|
| 210 |
+
void Init(uint32_t num_bytes);
|
| 211 |
+
|
| 212 |
+
/// Initialize the BlockSplitBloomFilter. It copies the bitset as underlying
|
| 213 |
+
/// bitset because the given bitset may not satisfy the 32-byte alignment requirement
|
| 214 |
+
/// which may lead to segfault when performing SIMD instructions. It is the caller's
|
| 215 |
+
/// responsibility to free the bitset passed in. This is used when reconstructing
|
| 216 |
+
/// a Bloom filter from a parquet file.
|
| 217 |
+
///
|
| 218 |
+
/// @param bitset The given bitset to initialize the Bloom filter.
|
| 219 |
+
/// @param num_bytes The number of bytes of given bitset.
|
| 220 |
+
void Init(const uint8_t* bitset, uint32_t num_bytes);
|
| 221 |
+
|
| 222 |
+
/// Minimum Bloom filter size, it sets to 32 bytes to fit a tiny Bloom filter.
|
| 223 |
+
static constexpr uint32_t kMinimumBloomFilterBytes = 32;
|
| 224 |
+
|
| 225 |
+
/// Calculate optimal size according to the number of distinct values and false
|
| 226 |
+
/// positive probability.
|
| 227 |
+
///
|
| 228 |
+
/// @param ndv The number of distinct values.
|
| 229 |
+
/// @param fpp The false positive probability.
|
| 230 |
+
/// @return it always return a value between kMinimumBloomFilterBytes and
|
| 231 |
+
/// kMaximumBloomFilterBytes, and the return value is always a power of 2
|
| 232 |
+
static uint32_t OptimalNumOfBytes(uint32_t ndv, double fpp) {
|
| 233 |
+
uint32_t optimal_num_of_bits = OptimalNumOfBits(ndv, fpp);
|
| 234 |
+
ARROW_DCHECK(::arrow::bit_util::IsMultipleOf8(optimal_num_of_bits));
|
| 235 |
+
return optimal_num_of_bits >> 3;
|
| 236 |
+
}
|
| 237 |
+
|
| 238 |
+
/// Calculate optimal size according to the number of distinct values and false
|
| 239 |
+
/// positive probability.
|
| 240 |
+
///
|
| 241 |
+
/// @param ndv The number of distinct values.
|
| 242 |
+
/// @param fpp The false positive probability.
|
| 243 |
+
/// @return it always return a value between kMinimumBloomFilterBytes * 8 and
|
| 244 |
+
/// kMaximumBloomFilterBytes * 8, and the return value is always a power of 16
|
| 245 |
+
static uint32_t OptimalNumOfBits(uint32_t ndv, double fpp) {
|
| 246 |
+
ARROW_DCHECK(fpp > 0.0 && fpp < 1.0);
|
| 247 |
+
const double m = -8.0 * ndv / log(1 - pow(fpp, 1.0 / 8));
|
| 248 |
+
uint32_t num_bits;
|
| 249 |
+
|
| 250 |
+
// Handle overflow.
|
| 251 |
+
if (m < 0 || m > kMaximumBloomFilterBytes << 3) {
|
| 252 |
+
num_bits = static_cast<uint32_t>(kMaximumBloomFilterBytes << 3);
|
| 253 |
+
} else {
|
| 254 |
+
num_bits = static_cast<uint32_t>(m);
|
| 255 |
+
}
|
| 256 |
+
|
| 257 |
+
// Round up to lower bound
|
| 258 |
+
if (num_bits < kMinimumBloomFilterBytes << 3) {
|
| 259 |
+
num_bits = kMinimumBloomFilterBytes << 3;
|
| 260 |
+
}
|
| 261 |
+
|
| 262 |
+
// Get next power of 2 if bits is not power of 2.
|
| 263 |
+
if ((num_bits & (num_bits - 1)) != 0) {
|
| 264 |
+
num_bits = static_cast<uint32_t>(::arrow::bit_util::NextPower2(num_bits));
|
| 265 |
+
}
|
| 266 |
+
|
| 267 |
+
// Round down to upper bound
|
| 268 |
+
if (num_bits > kMaximumBloomFilterBytes << 3) {
|
| 269 |
+
num_bits = kMaximumBloomFilterBytes << 3;
|
| 270 |
+
}
|
| 271 |
+
|
| 272 |
+
return num_bits;
|
| 273 |
+
}
|
| 274 |
+
|
| 275 |
+
bool FindHash(uint64_t hash) const override;
|
| 276 |
+
void InsertHash(uint64_t hash) override;
|
| 277 |
+
void InsertHashes(const uint64_t* hashes, int num_values) override;
|
| 278 |
+
void WriteTo(ArrowOutputStream* sink) const override;
|
| 279 |
+
uint32_t GetBitsetSize() const override { return num_bytes_; }
|
| 280 |
+
|
| 281 |
+
uint64_t Hash(int32_t value) const override { return hasher_->Hash(value); }
|
| 282 |
+
uint64_t Hash(int64_t value) const override { return hasher_->Hash(value); }
|
| 283 |
+
uint64_t Hash(float value) const override { return hasher_->Hash(value); }
|
| 284 |
+
uint64_t Hash(double value) const override { return hasher_->Hash(value); }
|
| 285 |
+
uint64_t Hash(const Int96* value) const override { return hasher_->Hash(value); }
|
| 286 |
+
uint64_t Hash(const ByteArray* value) const override { return hasher_->Hash(value); }
|
| 287 |
+
uint64_t Hash(const FLBA* value, uint32_t len) const override {
|
| 288 |
+
return hasher_->Hash(value, len);
|
| 289 |
+
}
|
| 290 |
+
uint64_t Hash(std::string_view value) const override { return hasher_->Hash(value); }
|
| 291 |
+
|
| 292 |
+
void Hashes(const int32_t* values, int num_values, uint64_t* hashes) const override {
|
| 293 |
+
hasher_->Hashes(values, num_values, hashes);
|
| 294 |
+
}
|
| 295 |
+
void Hashes(const int64_t* values, int num_values, uint64_t* hashes) const override {
|
| 296 |
+
hasher_->Hashes(values, num_values, hashes);
|
| 297 |
+
}
|
| 298 |
+
void Hashes(const float* values, int num_values, uint64_t* hashes) const override {
|
| 299 |
+
hasher_->Hashes(values, num_values, hashes);
|
| 300 |
+
}
|
| 301 |
+
void Hashes(const double* values, int num_values, uint64_t* hashes) const override {
|
| 302 |
+
hasher_->Hashes(values, num_values, hashes);
|
| 303 |
+
}
|
| 304 |
+
void Hashes(const Int96* values, int num_values, uint64_t* hashes) const override {
|
| 305 |
+
hasher_->Hashes(values, num_values, hashes);
|
| 306 |
+
}
|
| 307 |
+
void Hashes(const ByteArray* values, int num_values, uint64_t* hashes) const override {
|
| 308 |
+
hasher_->Hashes(values, num_values, hashes);
|
| 309 |
+
}
|
| 310 |
+
void Hashes(const FLBA* values, uint32_t type_len, int num_values,
|
| 311 |
+
uint64_t* hashes) const override {
|
| 312 |
+
hasher_->Hashes(values, type_len, num_values, hashes);
|
| 313 |
+
}
|
| 314 |
+
|
| 315 |
+
uint64_t Hash(const int32_t* value) const { return hasher_->Hash(*value); }
|
| 316 |
+
uint64_t Hash(const int64_t* value) const { return hasher_->Hash(*value); }
|
| 317 |
+
uint64_t Hash(const float* value) const { return hasher_->Hash(*value); }
|
| 318 |
+
uint64_t Hash(const double* value) const { return hasher_->Hash(*value); }
|
| 319 |
+
|
| 320 |
+
/// Deserialize the Bloom filter from an input stream. It is used when reconstructing
|
| 321 |
+
/// a Bloom filter from a parquet filter.
|
| 322 |
+
///
|
| 323 |
+
/// @param properties The parquet reader properties.
|
| 324 |
+
/// @param input_stream The input stream from which to construct the bloom filter.
|
| 325 |
+
/// @param bloom_filter_length The length of the serialized bloom filter including
|
| 326 |
+
/// header.
|
| 327 |
+
/// @return The BlockSplitBloomFilter.
|
| 328 |
+
static BlockSplitBloomFilter Deserialize(
|
| 329 |
+
const ReaderProperties& properties, ArrowInputStream* input_stream,
|
| 330 |
+
std::optional<int64_t> bloom_filter_length = std::nullopt);
|
| 331 |
+
|
| 332 |
+
/// Deserialize an encrypted Bloom filter from an input stream.
|
| 333 |
+
///
|
| 334 |
+
/// The same metadata decryptor is used for both the serialized header and bitset,
|
| 335 |
+
/// while switching module AADs between the two encrypted modules.
|
| 336 |
+
///
|
| 337 |
+
/// @param properties The parquet reader properties.
|
| 338 |
+
/// @param input_stream The input stream from which to construct the bloom filter.
|
| 339 |
+
/// @param bloom_filter_length The length of the serialized bloom filter including
|
| 340 |
+
/// header.
|
| 341 |
+
/// @param decryptor Decryptor for encrypted Bloom filter modules.
|
| 342 |
+
/// @param row_group_ordinal Ordinal of the row group containing this Bloom filter.
|
| 343 |
+
/// @param column_ordinal Ordinal of the column containing this Bloom filter.
|
| 344 |
+
/// @return The BlockSplitBloomFilter.
|
| 345 |
+
static BlockSplitBloomFilter DeserializeEncrypted(
|
| 346 |
+
const ReaderProperties& properties, ArrowInputStream* input_stream,
|
| 347 |
+
std::optional<int64_t> bloom_filter_length, Decryptor* decryptor,
|
| 348 |
+
int16_t row_group_ordinal, int16_t column_ordinal);
|
| 349 |
+
|
| 350 |
+
private:
|
| 351 |
+
inline void InsertHashImpl(uint64_t hash);
|
| 352 |
+
|
| 353 |
+
// Bytes in a tiny Bloom filter block.
|
| 354 |
+
static constexpr int kBytesPerFilterBlock = 32;
|
| 355 |
+
|
| 356 |
+
// The number of bits to be set in each tiny Bloom filter
|
| 357 |
+
static constexpr int kBitsSetPerBlock = 8;
|
| 358 |
+
|
| 359 |
+
// A mask structure used to set bits in each tiny Bloom filter.
|
| 360 |
+
struct BlockMask {
|
| 361 |
+
uint32_t item[kBitsSetPerBlock];
|
| 362 |
+
};
|
| 363 |
+
|
| 364 |
+
// The block-based algorithm needs eight odd SALT values to calculate eight indexes
|
| 365 |
+
// of bit to set, one bit in each 32-bit word.
|
| 366 |
+
static constexpr uint32_t SALT[kBitsSetPerBlock] = {
|
| 367 |
+
0x47b6137bU, 0x44974d91U, 0x8824ad5bU, 0xa2b7289dU,
|
| 368 |
+
0x705495c7U, 0x2df1424bU, 0x9efc4947U, 0x5c6bfb31U};
|
| 369 |
+
|
| 370 |
+
// Memory pool to allocate aligned buffer for bitset
|
| 371 |
+
::arrow::MemoryPool* pool_;
|
| 372 |
+
|
| 373 |
+
// The underlying buffer of bitset.
|
| 374 |
+
std::shared_ptr<Buffer> data_;
|
| 375 |
+
|
| 376 |
+
// The number of bytes of Bloom filter bitset.
|
| 377 |
+
uint32_t num_bytes_;
|
| 378 |
+
|
| 379 |
+
// Hash strategy used in this Bloom filter.
|
| 380 |
+
HashStrategy hash_strategy_;
|
| 381 |
+
|
| 382 |
+
// Algorithm used in this Bloom filter.
|
| 383 |
+
Algorithm algorithm_;
|
| 384 |
+
|
| 385 |
+
// Compression used in this Bloom filter.
|
| 386 |
+
CompressionStrategy compression_strategy_;
|
| 387 |
+
|
| 388 |
+
// The hash pointer points to actual hash class used.
|
| 389 |
+
std::unique_ptr<Hasher> hasher_;
|
| 390 |
+
};
|
| 391 |
+
|
| 392 |
+
} // namespace parquet
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/bloom_filter_reader.h
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include "arrow/io/interfaces.h"
|
| 21 |
+
#include "parquet/properties.h"
|
| 22 |
+
#include "parquet/type_fwd.h"
|
| 23 |
+
|
| 24 |
+
namespace parquet {
|
| 25 |
+
|
| 26 |
+
class InternalFileDecryptor;
|
| 27 |
+
class BloomFilter;
|
| 28 |
+
|
| 29 |
+
class PARQUET_EXPORT RowGroupBloomFilterReader {
|
| 30 |
+
public:
|
| 31 |
+
virtual ~RowGroupBloomFilterReader() = default;
|
| 32 |
+
|
| 33 |
+
/// \brief Read bloom filter of a column chunk.
|
| 34 |
+
///
|
| 35 |
+
/// \param[in] i column ordinal of the column chunk.
|
| 36 |
+
/// \returns bloom filter of the column or nullptr if it does not exist.
|
| 37 |
+
/// \throws ParquetException if the index is out of bound, or read bloom
|
| 38 |
+
/// filter failed.
|
| 39 |
+
virtual std::unique_ptr<BloomFilter> GetColumnBloomFilter(int i) = 0;
|
| 40 |
+
};
|
| 41 |
+
|
| 42 |
+
/// \brief Interface for reading the bloom filter for a Parquet file.
|
| 43 |
+
class PARQUET_EXPORT BloomFilterReader {
|
| 44 |
+
public:
|
| 45 |
+
virtual ~BloomFilterReader() = default;
|
| 46 |
+
|
| 47 |
+
/// \brief Create a BloomFilterReader instance.
|
| 48 |
+
/// \returns a BloomFilterReader instance.
|
| 49 |
+
/// WARNING: The returned BloomFilterReader references to all the input parameters, so
|
| 50 |
+
/// it must not outlive all of the input parameters. Usually these input parameters
|
| 51 |
+
/// come from the same ParquetFileReader object, so it must not outlive the reader
|
| 52 |
+
/// that creates this BloomFilterReader.
|
| 53 |
+
static std::unique_ptr<BloomFilterReader> Make(
|
| 54 |
+
std::shared_ptr<::arrow::io::RandomAccessFile> input,
|
| 55 |
+
std::shared_ptr<FileMetaData> file_metadata, const ReaderProperties& properties,
|
| 56 |
+
std::shared_ptr<InternalFileDecryptor> file_decryptor = NULLPTR);
|
| 57 |
+
|
| 58 |
+
/// \brief Get the bloom filter reader of a specific row group.
|
| 59 |
+
/// \param[in] i row group ordinal to get bloom filter reader.
|
| 60 |
+
/// \returns RowGroupBloomFilterReader of the specified row group. A nullptr may or may
|
| 61 |
+
/// not be returned if the bloom filter for the row group is unavailable. It
|
| 62 |
+
/// is the caller's responsibility to check the return value of follow-up calls
|
| 63 |
+
/// to the RowGroupBloomFilterReader.
|
| 64 |
+
/// \throws ParquetException if the index is out of bound.
|
| 65 |
+
virtual std::shared_ptr<RowGroupBloomFilterReader> RowGroup(int i) = 0;
|
| 66 |
+
};
|
| 67 |
+
|
| 68 |
+
} // namespace parquet
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/bloom_filter_writer.h
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include "arrow/type_fwd.h"
|
| 21 |
+
|
| 22 |
+
#include "parquet/bloom_filter.h"
|
| 23 |
+
#include "parquet/index_location.h"
|
| 24 |
+
#include "parquet/type_fwd.h"
|
| 25 |
+
|
| 26 |
+
namespace parquet {
|
| 27 |
+
|
| 28 |
+
/// \brief Writer for updating a bloom filter with values of a specific Parquet type.
|
| 29 |
+
/// \note Boolean type is not supported.
|
| 30 |
+
template <typename ParquetType>
|
| 31 |
+
class PARQUET_EXPORT TypedBloomFilterWriter {
|
| 32 |
+
public:
|
| 33 |
+
using T = typename ParquetType::c_type;
|
| 34 |
+
|
| 35 |
+
/// \param descr The descriptor of the column to write. Must outlive this writer.
|
| 36 |
+
/// \param bloom_filter The bloom filter to update. Must outlive this writer.
|
| 37 |
+
TypedBloomFilterWriter(const ColumnDescriptor* descr, BloomFilter* bloom_filter);
|
| 38 |
+
|
| 39 |
+
/// \brief Update the bloom filter with typed values.
|
| 40 |
+
///
|
| 41 |
+
/// \param values The values to update the bloom filter with.
|
| 42 |
+
/// \param num_values The number of values to update the bloom filter with.
|
| 43 |
+
void Update(const T* values, int64_t num_values);
|
| 44 |
+
|
| 45 |
+
/// \brief Update the bloom filter with typed values that have spaces.
|
| 46 |
+
///
|
| 47 |
+
/// \param values The values to update the bloom filter with.
|
| 48 |
+
/// \param num_values The number of values to update the bloom filter with.
|
| 49 |
+
/// \param valid_bits The validity bitmap of the values.
|
| 50 |
+
/// \param valid_bits_offset The offset of the validity bitmap.
|
| 51 |
+
void UpdateSpaced(const T* values, int64_t num_values, const uint8_t* valid_bits,
|
| 52 |
+
int64_t valid_bits_offset);
|
| 53 |
+
|
| 54 |
+
/// \brief Update the bloom filter with an Arrow array.
|
| 55 |
+
///
|
| 56 |
+
/// \param values The Arrow array to update the bloom filter with.
|
| 57 |
+
void Update(const ::arrow::Array& values);
|
| 58 |
+
|
| 59 |
+
private:
|
| 60 |
+
const ColumnDescriptor* descr_;
|
| 61 |
+
BloomFilter* bloom_filter_;
|
| 62 |
+
};
|
| 63 |
+
|
| 64 |
+
/// \brief Interface for building bloom filters of a parquet file.
|
| 65 |
+
class PARQUET_EXPORT BloomFilterBuilder {
|
| 66 |
+
public:
|
| 67 |
+
virtual ~BloomFilterBuilder() = default;
|
| 68 |
+
|
| 69 |
+
/// \brief API to create a BloomFilterBuilder.
|
| 70 |
+
///
|
| 71 |
+
/// \param schema The schema of the file and it must outlive the created builder.
|
| 72 |
+
/// \param properties Properties to get bloom filter options. It must outlive the
|
| 73 |
+
/// created builder.
|
| 74 |
+
static std::unique_ptr<BloomFilterBuilder> Make(const SchemaDescriptor* schema,
|
| 75 |
+
const WriterProperties* properties);
|
| 76 |
+
|
| 77 |
+
/// \brief Start a new row group to write bloom filters, meaning that next calls
|
| 78 |
+
/// to `CreateBloomFilter` will create bloom filters for the new row group.
|
| 79 |
+
///
|
| 80 |
+
/// \throws ParquetException if `WriteTo()` has been called already.
|
| 81 |
+
virtual void AppendRowGroup() = 0;
|
| 82 |
+
|
| 83 |
+
/// \brief Create a BloomFilter of the column ordinal of the current row group.
|
| 84 |
+
///
|
| 85 |
+
/// \param column_ordinal Column ordinal for the bloom filter.
|
| 86 |
+
/// \return created BloomFilter whose ownership belongs to the builder, or nullptr if
|
| 87 |
+
/// bloom filter is not enabled for the column.
|
| 88 |
+
/// \throws ParquetException if any condition is violated:
|
| 89 |
+
/// - `AppendRowGroup()` has not been called yet
|
| 90 |
+
/// - The column ordinal is out of bound
|
| 91 |
+
/// - Bloom filter already exists for the column
|
| 92 |
+
/// - `WriteTo()` has been called
|
| 93 |
+
virtual BloomFilter* CreateBloomFilter(int32_t column_ordinal) = 0;
|
| 94 |
+
|
| 95 |
+
/// \brief Write all bloom filters to sink.
|
| 96 |
+
///
|
| 97 |
+
/// The bloom filters cannot be modified after this method is called.
|
| 98 |
+
///
|
| 99 |
+
/// \param[in] sink The output stream to write the bloom filters.
|
| 100 |
+
/// \return The location of all bloom filters.
|
| 101 |
+
/// \throws ParquetException if `WriteTo()` has been called.
|
| 102 |
+
virtual IndexLocations WriteTo(::arrow::io::OutputStream* sink) = 0;
|
| 103 |
+
};
|
| 104 |
+
|
| 105 |
+
} // namespace parquet
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_page.h
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
// This module defines an abstract interface for iterating through pages in a
|
| 19 |
+
// Parquet column chunk within a row group. It could be extended in the future
|
| 20 |
+
// to iterate through all data pages in all chunks in a file.
|
| 21 |
+
|
| 22 |
+
#pragma once
|
| 23 |
+
|
| 24 |
+
#include <cstdint>
|
| 25 |
+
#include <memory>
|
| 26 |
+
#include <optional>
|
| 27 |
+
#include <string>
|
| 28 |
+
|
| 29 |
+
#include "parquet/size_statistics.h"
|
| 30 |
+
#include "parquet/statistics.h"
|
| 31 |
+
#include "parquet/types.h"
|
| 32 |
+
|
| 33 |
+
namespace parquet {
|
| 34 |
+
|
| 35 |
+
// TODO: Parallel processing is not yet safe because of memory-ownership
|
| 36 |
+
// semantics (the PageReader may or may not own the memory referenced by a
|
| 37 |
+
// page)
|
| 38 |
+
//
|
| 39 |
+
// TODO(wesm): In the future Parquet implementations may store the crc code
|
| 40 |
+
// in format::PageHeader. parquet-mr currently does not, so we also skip it
|
| 41 |
+
// here, both on the read and write path
|
| 42 |
+
class Page {
|
| 43 |
+
public:
|
| 44 |
+
Page(const std::shared_ptr<Buffer>& buffer, PageType::type type)
|
| 45 |
+
: buffer_(buffer), type_(type) {}
|
| 46 |
+
|
| 47 |
+
virtual ~Page() = default;
|
| 48 |
+
|
| 49 |
+
PageType::type type() const { return type_; }
|
| 50 |
+
|
| 51 |
+
std::shared_ptr<Buffer> buffer() const { return buffer_; }
|
| 52 |
+
|
| 53 |
+
// @returns: a pointer to the page's data
|
| 54 |
+
const uint8_t* data() const { return buffer_->data(); }
|
| 55 |
+
|
| 56 |
+
// @returns: the total size in bytes of the page's data buffer
|
| 57 |
+
int32_t size() const { return static_cast<int32_t>(buffer_->size()); }
|
| 58 |
+
|
| 59 |
+
private:
|
| 60 |
+
std::shared_ptr<Buffer> buffer_;
|
| 61 |
+
PageType::type type_;
|
| 62 |
+
};
|
| 63 |
+
|
| 64 |
+
/// \brief Base type for DataPageV1 and DataPageV2 including common attributes
|
| 65 |
+
class DataPage : public Page {
|
| 66 |
+
public:
|
| 67 |
+
int32_t num_values() const { return num_values_; }
|
| 68 |
+
Encoding::type encoding() const { return encoding_; }
|
| 69 |
+
int64_t uncompressed_size() const { return uncompressed_size_; }
|
| 70 |
+
const EncodedStatistics& statistics() const { return statistics_; }
|
| 71 |
+
/// Return the row ordinal within the row group to the first row in the data page.
|
| 72 |
+
/// Currently it is only present from data pages created by ColumnWriter in order
|
| 73 |
+
/// to collect page index.
|
| 74 |
+
std::optional<int64_t> first_row_index() const { return first_row_index_; }
|
| 75 |
+
const SizeStatistics& size_statistics() const { return size_statistics_; }
|
| 76 |
+
|
| 77 |
+
virtual ~DataPage() = default;
|
| 78 |
+
|
| 79 |
+
protected:
|
| 80 |
+
DataPage(PageType::type type, const std::shared_ptr<Buffer>& buffer, int32_t num_values,
|
| 81 |
+
Encoding::type encoding, int64_t uncompressed_size,
|
| 82 |
+
EncodedStatistics statistics, std::optional<int64_t> first_row_index,
|
| 83 |
+
SizeStatistics size_statistics)
|
| 84 |
+
: Page(buffer, type),
|
| 85 |
+
num_values_(num_values),
|
| 86 |
+
encoding_(encoding),
|
| 87 |
+
uncompressed_size_(uncompressed_size),
|
| 88 |
+
statistics_(std::move(statistics)),
|
| 89 |
+
first_row_index_(std::move(first_row_index)),
|
| 90 |
+
size_statistics_(std::move(size_statistics)) {}
|
| 91 |
+
|
| 92 |
+
int32_t num_values_;
|
| 93 |
+
Encoding::type encoding_;
|
| 94 |
+
int64_t uncompressed_size_;
|
| 95 |
+
EncodedStatistics statistics_;
|
| 96 |
+
/// Row ordinal within the row group to the first row in the data page.
|
| 97 |
+
std::optional<int64_t> first_row_index_;
|
| 98 |
+
SizeStatistics size_statistics_;
|
| 99 |
+
};
|
| 100 |
+
|
| 101 |
+
class DataPageV1 : public DataPage {
|
| 102 |
+
public:
|
| 103 |
+
DataPageV1(const std::shared_ptr<Buffer>& buffer, int32_t num_values,
|
| 104 |
+
Encoding::type encoding, Encoding::type definition_level_encoding,
|
| 105 |
+
Encoding::type repetition_level_encoding, int64_t uncompressed_size,
|
| 106 |
+
EncodedStatistics statistics = EncodedStatistics(),
|
| 107 |
+
std::optional<int64_t> first_row_index = std::nullopt,
|
| 108 |
+
SizeStatistics size_statistics = SizeStatistics())
|
| 109 |
+
: DataPage(PageType::DATA_PAGE, buffer, num_values, encoding, uncompressed_size,
|
| 110 |
+
std::move(statistics), std::move(first_row_index),
|
| 111 |
+
std::move(size_statistics)),
|
| 112 |
+
definition_level_encoding_(definition_level_encoding),
|
| 113 |
+
repetition_level_encoding_(repetition_level_encoding) {}
|
| 114 |
+
|
| 115 |
+
Encoding::type repetition_level_encoding() const { return repetition_level_encoding_; }
|
| 116 |
+
|
| 117 |
+
Encoding::type definition_level_encoding() const { return definition_level_encoding_; }
|
| 118 |
+
|
| 119 |
+
private:
|
| 120 |
+
Encoding::type definition_level_encoding_;
|
| 121 |
+
Encoding::type repetition_level_encoding_;
|
| 122 |
+
};
|
| 123 |
+
|
| 124 |
+
class DataPageV2 : public DataPage {
|
| 125 |
+
public:
|
| 126 |
+
DataPageV2(const std::shared_ptr<Buffer>& buffer, int32_t num_values, int32_t num_nulls,
|
| 127 |
+
int32_t num_rows, Encoding::type encoding,
|
| 128 |
+
int32_t definition_levels_byte_length, int32_t repetition_levels_byte_length,
|
| 129 |
+
int64_t uncompressed_size, bool is_compressed = false,
|
| 130 |
+
EncodedStatistics statistics = EncodedStatistics(),
|
| 131 |
+
std::optional<int64_t> first_row_index = std::nullopt,
|
| 132 |
+
SizeStatistics size_statistics = SizeStatistics())
|
| 133 |
+
: DataPage(PageType::DATA_PAGE_V2, buffer, num_values, encoding, uncompressed_size,
|
| 134 |
+
std::move(statistics), std::move(first_row_index),
|
| 135 |
+
std::move(size_statistics)),
|
| 136 |
+
num_nulls_(num_nulls),
|
| 137 |
+
num_rows_(num_rows),
|
| 138 |
+
definition_levels_byte_length_(definition_levels_byte_length),
|
| 139 |
+
repetition_levels_byte_length_(repetition_levels_byte_length),
|
| 140 |
+
is_compressed_(is_compressed) {}
|
| 141 |
+
|
| 142 |
+
int32_t num_nulls() const { return num_nulls_; }
|
| 143 |
+
|
| 144 |
+
int32_t num_rows() const { return num_rows_; }
|
| 145 |
+
|
| 146 |
+
int32_t definition_levels_byte_length() const { return definition_levels_byte_length_; }
|
| 147 |
+
|
| 148 |
+
int32_t repetition_levels_byte_length() const { return repetition_levels_byte_length_; }
|
| 149 |
+
|
| 150 |
+
bool is_compressed() const { return is_compressed_; }
|
| 151 |
+
|
| 152 |
+
private:
|
| 153 |
+
int32_t num_nulls_;
|
| 154 |
+
int32_t num_rows_;
|
| 155 |
+
int32_t definition_levels_byte_length_;
|
| 156 |
+
int32_t repetition_levels_byte_length_;
|
| 157 |
+
bool is_compressed_;
|
| 158 |
+
};
|
| 159 |
+
|
| 160 |
+
class DictionaryPage : public Page {
|
| 161 |
+
public:
|
| 162 |
+
DictionaryPage(const std::shared_ptr<Buffer>& buffer, int32_t num_values,
|
| 163 |
+
Encoding::type encoding, bool is_sorted = false)
|
| 164 |
+
: Page(buffer, PageType::DICTIONARY_PAGE),
|
| 165 |
+
num_values_(num_values),
|
| 166 |
+
encoding_(encoding),
|
| 167 |
+
is_sorted_(is_sorted) {}
|
| 168 |
+
|
| 169 |
+
int32_t num_values() const { return num_values_; }
|
| 170 |
+
|
| 171 |
+
Encoding::type encoding() const { return encoding_; }
|
| 172 |
+
|
| 173 |
+
bool is_sorted() const { return is_sorted_; }
|
| 174 |
+
|
| 175 |
+
private:
|
| 176 |
+
int32_t num_values_;
|
| 177 |
+
Encoding::type encoding_;
|
| 178 |
+
bool is_sorted_;
|
| 179 |
+
};
|
| 180 |
+
|
| 181 |
+
} // namespace parquet
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_reader.h
ADDED
|
@@ -0,0 +1,458 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <cstdint>
|
| 21 |
+
#include <memory>
|
| 22 |
+
#include <utility>
|
| 23 |
+
#include <vector>
|
| 24 |
+
|
| 25 |
+
#include "arrow/type_fwd.h"
|
| 26 |
+
#include "arrow/util/macros.h"
|
| 27 |
+
#include "parquet/exception.h"
|
| 28 |
+
#include "parquet/level_conversion.h"
|
| 29 |
+
#include "parquet/metadata.h"
|
| 30 |
+
#include "parquet/platform.h"
|
| 31 |
+
#include "parquet/properties.h"
|
| 32 |
+
#include "parquet/schema.h"
|
| 33 |
+
#include "parquet/types.h"
|
| 34 |
+
|
| 35 |
+
namespace arrow {
|
| 36 |
+
|
| 37 |
+
namespace bit_util {
|
| 38 |
+
class BitReader;
|
| 39 |
+
} // namespace bit_util
|
| 40 |
+
|
| 41 |
+
namespace util {
|
| 42 |
+
template <typename T>
|
| 43 |
+
class RleBitPackedDecoder;
|
| 44 |
+
} // namespace util
|
| 45 |
+
|
| 46 |
+
} // namespace arrow
|
| 47 |
+
|
| 48 |
+
namespace parquet {
|
| 49 |
+
|
| 50 |
+
class Decryptor;
|
| 51 |
+
class Page;
|
| 52 |
+
|
| 53 |
+
// 16 MB is the default maximum page header size
|
| 54 |
+
static constexpr uint32_t kDefaultMaxPageHeaderSize = 16 * 1024 * 1024;
|
| 55 |
+
|
| 56 |
+
// 16 KB is the default expected page header size
|
| 57 |
+
static constexpr uint32_t kDefaultPageHeaderSize = 16 * 1024;
|
| 58 |
+
|
| 59 |
+
// \brief DataPageStats stores encoded statistics and number of values/rows for
|
| 60 |
+
// a page.
|
| 61 |
+
struct PARQUET_EXPORT DataPageStats {
|
| 62 |
+
DataPageStats(const EncodedStatistics* encoded_statistics, int32_t num_values,
|
| 63 |
+
std::optional<int32_t> num_rows)
|
| 64 |
+
: encoded_statistics(encoded_statistics),
|
| 65 |
+
num_values(num_values),
|
| 66 |
+
num_rows(num_rows) {}
|
| 67 |
+
|
| 68 |
+
// Encoded statistics extracted from the page header.
|
| 69 |
+
// Nullptr if there are no statistics in the page header.
|
| 70 |
+
const EncodedStatistics* encoded_statistics;
|
| 71 |
+
// Number of values stored in the page. Filled for both V1 and V2 data pages.
|
| 72 |
+
// For repeated fields, this can be greater than number of rows. For
|
| 73 |
+
// non-repeated fields, this will be the same as the number of rows.
|
| 74 |
+
int32_t num_values;
|
| 75 |
+
// Number of rows stored in the page. std::nullopt if not available.
|
| 76 |
+
std::optional<int32_t> num_rows;
|
| 77 |
+
};
|
| 78 |
+
|
| 79 |
+
class PARQUET_EXPORT LevelDecoder {
|
| 80 |
+
public:
|
| 81 |
+
LevelDecoder();
|
| 82 |
+
~LevelDecoder();
|
| 83 |
+
|
| 84 |
+
// Initialize the LevelDecoder state with new data
|
| 85 |
+
// and return the number of bytes consumed
|
| 86 |
+
int SetData(Encoding::type encoding, int16_t max_level, int num_buffered_values,
|
| 87 |
+
const uint8_t* data, int32_t data_size);
|
| 88 |
+
|
| 89 |
+
void SetDataV2(int32_t num_bytes, int16_t max_level, int num_buffered_values,
|
| 90 |
+
const uint8_t* data);
|
| 91 |
+
|
| 92 |
+
// Decodes a batch of levels into an array and returns the number of levels decoded
|
| 93 |
+
int Decode(int batch_size, int16_t* levels);
|
| 94 |
+
|
| 95 |
+
private:
|
| 96 |
+
int bit_width_;
|
| 97 |
+
int num_values_remaining_;
|
| 98 |
+
Encoding::type encoding_;
|
| 99 |
+
std::unique_ptr<::arrow::util::RleBitPackedDecoder<int16_t>> rle_decoder_;
|
| 100 |
+
std::unique_ptr<::arrow::bit_util::BitReader> bit_packed_decoder_;
|
| 101 |
+
int16_t max_level_;
|
| 102 |
+
};
|
| 103 |
+
|
| 104 |
+
struct CryptoContext {
|
| 105 |
+
bool start_decrypt_with_dictionary_page = false;
|
| 106 |
+
int16_t row_group_ordinal = -1;
|
| 107 |
+
int16_t column_ordinal = -1;
|
| 108 |
+
std::function<std::unique_ptr<Decryptor>()> meta_decryptor_factory;
|
| 109 |
+
std::function<std::unique_ptr<Decryptor>()> data_decryptor_factory;
|
| 110 |
+
};
|
| 111 |
+
|
| 112 |
+
// Abstract page iterator interface. This way, we can feed column pages to the
|
| 113 |
+
// ColumnReader through whatever mechanism we choose
|
| 114 |
+
class PARQUET_EXPORT PageReader {
|
| 115 |
+
using DataPageFilter = std::function<bool(const DataPageStats&)>;
|
| 116 |
+
|
| 117 |
+
public:
|
| 118 |
+
virtual ~PageReader() = default;
|
| 119 |
+
|
| 120 |
+
static std::unique_ptr<PageReader> Open(
|
| 121 |
+
std::shared_ptr<ArrowInputStream> stream, int64_t total_num_values,
|
| 122 |
+
Compression::type codec, bool always_compressed = false,
|
| 123 |
+
::arrow::MemoryPool* pool = ::arrow::default_memory_pool(),
|
| 124 |
+
const CryptoContext* ctx = NULLPTR);
|
| 125 |
+
static std::unique_ptr<PageReader> Open(std::shared_ptr<ArrowInputStream> stream,
|
| 126 |
+
int64_t total_num_values,
|
| 127 |
+
Compression::type codec,
|
| 128 |
+
const ReaderProperties& properties,
|
| 129 |
+
bool always_compressed = false,
|
| 130 |
+
const CryptoContext* ctx = NULLPTR);
|
| 131 |
+
|
| 132 |
+
// If data_page_filter is present (not null), NextPage() will call the
|
| 133 |
+
// callback function exactly once per page in the order the pages appear in
|
| 134 |
+
// the column. If the callback function returns true the page will be
|
| 135 |
+
// skipped. The callback will be called only if the page type is DATA_PAGE or
|
| 136 |
+
// DATA_PAGE_V2. Dictionary pages will not be skipped.
|
| 137 |
+
// Caller is responsible for checking that statistics are correct using
|
| 138 |
+
// ApplicationVersion::HasCorrectStatistics().
|
| 139 |
+
// \note API EXPERIMENTAL
|
| 140 |
+
void set_data_page_filter(DataPageFilter data_page_filter) {
|
| 141 |
+
data_page_filter_ = std::move(data_page_filter);
|
| 142 |
+
}
|
| 143 |
+
|
| 144 |
+
// @returns: shared_ptr<Page>(nullptr) on EOS, std::shared_ptr<Page>
|
| 145 |
+
// containing new Page otherwise
|
| 146 |
+
//
|
| 147 |
+
// The returned Page may contain references that aren't guaranteed to live
|
| 148 |
+
// beyond the next call to NextPage().
|
| 149 |
+
virtual std::shared_ptr<Page> NextPage() = 0;
|
| 150 |
+
|
| 151 |
+
virtual void set_max_page_header_size(uint32_t size) = 0;
|
| 152 |
+
|
| 153 |
+
protected:
|
| 154 |
+
// Callback that decides if we should skip a page or not.
|
| 155 |
+
DataPageFilter data_page_filter_;
|
| 156 |
+
};
|
| 157 |
+
|
| 158 |
+
class PARQUET_EXPORT ColumnReader {
|
| 159 |
+
public:
|
| 160 |
+
virtual ~ColumnReader() = default;
|
| 161 |
+
|
| 162 |
+
static std::shared_ptr<ColumnReader> Make(
|
| 163 |
+
const ColumnDescriptor* descr, std::unique_ptr<PageReader> pager,
|
| 164 |
+
::arrow::MemoryPool* pool = ::arrow::default_memory_pool());
|
| 165 |
+
|
| 166 |
+
// Returns true if there are still values in this column.
|
| 167 |
+
virtual bool HasNext() = 0;
|
| 168 |
+
|
| 169 |
+
virtual Type::type type() const = 0;
|
| 170 |
+
|
| 171 |
+
virtual const ColumnDescriptor* descr() const = 0;
|
| 172 |
+
|
| 173 |
+
// Get the encoding that can be exposed by this reader. If it returns
|
| 174 |
+
// dictionary encoding, then ReadBatchWithDictionary can be used to read data.
|
| 175 |
+
//
|
| 176 |
+
// \note API EXPERIMENTAL
|
| 177 |
+
virtual ExposedEncoding GetExposedEncoding() = 0;
|
| 178 |
+
|
| 179 |
+
protected:
|
| 180 |
+
friend class RowGroupReader;
|
| 181 |
+
// Set the encoding that can be exposed by this reader.
|
| 182 |
+
//
|
| 183 |
+
// \note API EXPERIMENTAL
|
| 184 |
+
virtual void SetExposedEncoding(ExposedEncoding encoding) = 0;
|
| 185 |
+
};
|
| 186 |
+
|
| 187 |
+
// API to read values from a single column. This is a main client facing API.
|
| 188 |
+
template <typename DType>
|
| 189 |
+
class TypedColumnReader : public ColumnReader {
|
| 190 |
+
public:
|
| 191 |
+
using T = typename DType::c_type;
|
| 192 |
+
|
| 193 |
+
// Read a batch of repetition levels, definition levels, and values from the
|
| 194 |
+
// column.
|
| 195 |
+
//
|
| 196 |
+
// Since null values are not stored in the values, the number of values read
|
| 197 |
+
// may be less than the number of repetition and definition levels. With
|
| 198 |
+
// nested data this is almost certainly true.
|
| 199 |
+
//
|
| 200 |
+
// Set def_levels or rep_levels to nullptr if you want to skip reading them.
|
| 201 |
+
// This is only safe if you know through some other source that there are no
|
| 202 |
+
// undefined values.
|
| 203 |
+
//
|
| 204 |
+
// To fully exhaust a row group, you must read batches until the number of
|
| 205 |
+
// values read reaches the number of stored values according to the metadata.
|
| 206 |
+
//
|
| 207 |
+
// This API is the same for both V1 and V2 of the DataPage
|
| 208 |
+
//
|
| 209 |
+
// @returns: actual number of levels read (see values_read for number of values read)
|
| 210 |
+
virtual int64_t ReadBatch(int64_t batch_size, int16_t* def_levels, int16_t* rep_levels,
|
| 211 |
+
T* values, int64_t* values_read) = 0;
|
| 212 |
+
|
| 213 |
+
// Skip reading values. This method will work for both repeated and
|
| 214 |
+
// non-repeated fields. Note that this method is skipping values and not
|
| 215 |
+
// records. This distinction is important for repeated fields, meaning that
|
| 216 |
+
// we are not skipping over the values to the next record. For example,
|
| 217 |
+
// consider the following two consecutive records containing one repeated field:
|
| 218 |
+
// {[1, 2, 3]}, {[4, 5]}. If we Skip(2), our next read value will be 3, which
|
| 219 |
+
// is inside the first record.
|
| 220 |
+
// Returns the number of values skipped.
|
| 221 |
+
virtual int64_t Skip(int64_t num_values_to_skip) = 0;
|
| 222 |
+
|
| 223 |
+
// Read a batch of repetition levels, definition levels, and indices from the
|
| 224 |
+
// column. And read the dictionary if a dictionary page is encountered during
|
| 225 |
+
// reading pages. This API is similar to ReadBatch(), with ability to read
|
| 226 |
+
// dictionary and indices. It is only valid to call this method when the reader can
|
| 227 |
+
// expose dictionary encoding. (i.e., the reader's GetExposedEncoding() returns
|
| 228 |
+
// DICTIONARY).
|
| 229 |
+
//
|
| 230 |
+
// The dictionary is read along with the data page. When there's no data page,
|
| 231 |
+
// the dictionary won't be returned.
|
| 232 |
+
//
|
| 233 |
+
// @param batch_size The batch size to read
|
| 234 |
+
// @param[out] def_levels The Parquet definition levels.
|
| 235 |
+
// @param[out] rep_levels The Parquet repetition levels.
|
| 236 |
+
// @param[out] indices The dictionary indices.
|
| 237 |
+
// @param[out] indices_read The number of indices read.
|
| 238 |
+
// @param[out] dict The pointer to dictionary values. It will return nullptr if
|
| 239 |
+
// there's no data page. Each column chunk only has one dictionary page. The dictionary
|
| 240 |
+
// is owned by the reader, so the caller is responsible for copying the dictionary
|
| 241 |
+
// values before the reader gets destroyed.
|
| 242 |
+
// @param[out] dict_len The dictionary length. It will return 0 if there's no data
|
| 243 |
+
// page.
|
| 244 |
+
// @returns: actual number of levels read (see indices_read for number of
|
| 245 |
+
// indices read
|
| 246 |
+
//
|
| 247 |
+
// \note API EXPERIMENTAL
|
| 248 |
+
virtual int64_t ReadBatchWithDictionary(int64_t batch_size, int16_t* def_levels,
|
| 249 |
+
int16_t* rep_levels, int32_t* indices,
|
| 250 |
+
int64_t* indices_read, const T** dict,
|
| 251 |
+
int32_t* dict_len) = 0;
|
| 252 |
+
};
|
| 253 |
+
|
| 254 |
+
namespace internal {
|
| 255 |
+
|
| 256 |
+
/// \brief Stateful column reader that delimits semantic records for both flat
|
| 257 |
+
/// and nested columns
|
| 258 |
+
///
|
| 259 |
+
/// \note API EXPERIMENTAL
|
| 260 |
+
/// \since 1.3.0
|
| 261 |
+
class PARQUET_EXPORT RecordReader {
|
| 262 |
+
public:
|
| 263 |
+
/// \brief Creates a record reader.
|
| 264 |
+
/// @param descr Column descriptor
|
| 265 |
+
/// @param leaf_info Level info, used to determine if a column is nullable or not
|
| 266 |
+
/// @param pool Memory pool to use for buffering values and rep/def levels
|
| 267 |
+
/// @param read_dictionary True if reading directly as Arrow dictionary-encoded
|
| 268 |
+
/// @param read_dense_for_nullable True if reading dense and not leaving space for null
|
| 269 |
+
/// values
|
| 270 |
+
/// @param arrow_type Which type to read this column as (optional). Currently
|
| 271 |
+
/// only used for byte array columns (see BinaryRecordReader::GetBuilderChunks).
|
| 272 |
+
static std::shared_ptr<RecordReader> Make(
|
| 273 |
+
const ColumnDescriptor* descr, LevelInfo leaf_info,
|
| 274 |
+
::arrow::MemoryPool* pool = ::arrow::default_memory_pool(),
|
| 275 |
+
bool read_dictionary = false, bool read_dense_for_nullable = false,
|
| 276 |
+
const std::shared_ptr<::arrow::DataType>& arrow_type = NULLPTR);
|
| 277 |
+
|
| 278 |
+
virtual ~RecordReader() = default;
|
| 279 |
+
|
| 280 |
+
/// \brief Attempt to read indicated number of records from column chunk
|
| 281 |
+
/// Note that for repeated fields, a record may have more than one value
|
| 282 |
+
/// and all of them are read. If read_dense_for_nullable() it will
|
| 283 |
+
/// not leave any space for null values. Otherwise, it will read spaced.
|
| 284 |
+
/// \return number of records read
|
| 285 |
+
virtual int64_t ReadRecords(int64_t num_records) = 0;
|
| 286 |
+
|
| 287 |
+
/// \brief Attempt to skip indicated number of records from column chunk.
|
| 288 |
+
/// Note that for repeated fields, a record may have more than one value
|
| 289 |
+
/// and all of them are skipped.
|
| 290 |
+
/// \return number of records skipped
|
| 291 |
+
virtual int64_t SkipRecords(int64_t num_records) = 0;
|
| 292 |
+
|
| 293 |
+
/// \brief Pre-allocate space for data. Results in better flat read performance
|
| 294 |
+
virtual void Reserve(int64_t num_values) = 0;
|
| 295 |
+
|
| 296 |
+
/// \brief Clear consumed values and repetition/definition levels as the
|
| 297 |
+
/// result of calling ReadRecords
|
| 298 |
+
/// For FLBA and ByteArray types, call GetBuilderChunks() to reset them.
|
| 299 |
+
virtual void Reset() = 0;
|
| 300 |
+
|
| 301 |
+
/// \brief Transfer filled values buffer to caller. A new one will be
|
| 302 |
+
/// allocated in subsequent ReadRecords calls
|
| 303 |
+
virtual std::shared_ptr<ResizableBuffer> ReleaseValues() = 0;
|
| 304 |
+
|
| 305 |
+
/// \brief Transfer filled validity bitmap buffer to caller. A new one will
|
| 306 |
+
/// be allocated in subsequent ReadRecords calls
|
| 307 |
+
virtual std::shared_ptr<ResizableBuffer> ReleaseIsValid() = 0;
|
| 308 |
+
|
| 309 |
+
/// \brief Return true if the record reader has more internal data yet to
|
| 310 |
+
/// process
|
| 311 |
+
virtual bool HasMoreData() const = 0;
|
| 312 |
+
|
| 313 |
+
/// \brief Advance record reader to the next row group. Must be set before
|
| 314 |
+
/// any records could be read/skipped.
|
| 315 |
+
/// \param[in] reader obtained from RowGroupReader::GetColumnPageReader
|
| 316 |
+
virtual void SetPageReader(std::unique_ptr<PageReader> reader) = 0;
|
| 317 |
+
|
| 318 |
+
/// \brief Returns the underlying column reader's descriptor.
|
| 319 |
+
virtual const ColumnDescriptor* descr() const = 0;
|
| 320 |
+
|
| 321 |
+
virtual void DebugPrintState() = 0;
|
| 322 |
+
|
| 323 |
+
/// \brief Returns the dictionary owned by the current decoder. Throws an
|
| 324 |
+
/// exception if the current decoder is not for dictionary encoding. The caller is
|
| 325 |
+
/// responsible for casting the returned pointer to proper type depending on the
|
| 326 |
+
/// column's physical type. An example:
|
| 327 |
+
/// const ByteArray* dict = reinterpret_cast<const ByteArray*>(ReadDictionary(&len));
|
| 328 |
+
/// or:
|
| 329 |
+
/// const float* dict = reinterpret_cast<const float*>(ReadDictionary(&len));
|
| 330 |
+
/// \param[out] dictionary_length The number of dictionary entries.
|
| 331 |
+
virtual const void* ReadDictionary(int32_t* dictionary_length) = 0;
|
| 332 |
+
|
| 333 |
+
/// \brief Decoded definition levels
|
| 334 |
+
int16_t* def_levels() const {
|
| 335 |
+
return reinterpret_cast<int16_t*>(def_levels_->mutable_data());
|
| 336 |
+
}
|
| 337 |
+
|
| 338 |
+
/// \brief Decoded repetition levels
|
| 339 |
+
int16_t* rep_levels() const {
|
| 340 |
+
return reinterpret_cast<int16_t*>(rep_levels_->mutable_data());
|
| 341 |
+
}
|
| 342 |
+
|
| 343 |
+
/// \brief Decoded values, including nulls, if any
|
| 344 |
+
/// FLBA and ByteArray types do not use this array and read into their own
|
| 345 |
+
/// builders.
|
| 346 |
+
uint8_t* values() const { return values_->mutable_data(); }
|
| 347 |
+
|
| 348 |
+
/// \brief Number of values written, including space left for nulls if any.
|
| 349 |
+
/// If this Reader was constructed with read_dense_for_nullable(), there is no space for
|
| 350 |
+
/// nulls and null_count() will be 0. There is no read-ahead/buffering for values. For
|
| 351 |
+
/// FLBA and ByteArray types this value reflects the values written with the last
|
| 352 |
+
/// ReadRecords call since those readers will reset the values after each call.
|
| 353 |
+
int64_t values_written() const { return values_written_; }
|
| 354 |
+
|
| 355 |
+
/// \brief Number of definition / repetition levels (from those that have
|
| 356 |
+
/// been decoded) that have been consumed inside the reader.
|
| 357 |
+
int64_t levels_position() const { return levels_position_; }
|
| 358 |
+
|
| 359 |
+
/// \brief Number of definition / repetition levels that have been written
|
| 360 |
+
/// internally in the reader. This may be larger than values_written() because
|
| 361 |
+
/// for repeated fields we need to look at the levels in advance to figure out
|
| 362 |
+
/// the record boundaries.
|
| 363 |
+
int64_t levels_written() const { return levels_written_; }
|
| 364 |
+
|
| 365 |
+
/// \brief Number of nulls in the leaf that we have read so far into the
|
| 366 |
+
/// values vector. This is only valid when !read_dense_for_nullable(). When
|
| 367 |
+
/// read_dense_for_nullable() it will always be 0.
|
| 368 |
+
int64_t null_count() const { return null_count_; }
|
| 369 |
+
|
| 370 |
+
/// \brief True if the leaf values are nullable
|
| 371 |
+
bool nullable_values() const { return nullable_values_; }
|
| 372 |
+
|
| 373 |
+
/// \brief True if reading directly as Arrow dictionary-encoded
|
| 374 |
+
bool read_dictionary() const { return read_dictionary_; }
|
| 375 |
+
|
| 376 |
+
/// \brief True if reading dense for nullable columns.
|
| 377 |
+
bool read_dense_for_nullable() const { return read_dense_for_nullable_; }
|
| 378 |
+
|
| 379 |
+
protected:
|
| 380 |
+
/// \brief Indicates if we can have nullable values. Note that repeated fields
|
| 381 |
+
/// may or may not be nullable.
|
| 382 |
+
bool nullable_values_;
|
| 383 |
+
|
| 384 |
+
bool at_record_start_;
|
| 385 |
+
int64_t records_read_;
|
| 386 |
+
|
| 387 |
+
/// \brief Stores values. These values are populated based on each ReadRecords
|
| 388 |
+
/// call. No extra values are buffered for the next call. SkipRecords will not
|
| 389 |
+
/// add any value to this buffer.
|
| 390 |
+
std::shared_ptr<::arrow::ResizableBuffer> values_;
|
| 391 |
+
/// \brief False for FIXED_LEN_BYTE_ARRAY and BYTE_ARRAY, in which case we
|
| 392 |
+
/// don't allocate the values buffer and we directly read into builder classes.
|
| 393 |
+
bool uses_values_;
|
| 394 |
+
|
| 395 |
+
/// \brief Values that we have read into 'values_' + 'null_count_'.
|
| 396 |
+
int64_t values_written_;
|
| 397 |
+
int64_t values_capacity_;
|
| 398 |
+
int64_t null_count_;
|
| 399 |
+
|
| 400 |
+
/// \brief Each bit corresponds to one element in 'values_' and specifies if it
|
| 401 |
+
/// is null or not null.
|
| 402 |
+
///
|
| 403 |
+
/// Not set if leaf type is not nullable or read_dense_for_nullable_ is true.
|
| 404 |
+
std::shared_ptr<::arrow::ResizableBuffer> valid_bits_;
|
| 405 |
+
|
| 406 |
+
/// \brief Buffer for definition levels. May contain more levels than
|
| 407 |
+
/// is actually read. This is because we read levels ahead to
|
| 408 |
+
/// figure out record boundaries for repeated fields.
|
| 409 |
+
/// For flat required fields, 'def_levels_' and 'rep_levels_' are not
|
| 410 |
+
/// populated. For non-repeated fields 'rep_levels_' is not populated.
|
| 411 |
+
/// 'def_levels_' and 'rep_levels_' must be of the same size if present.
|
| 412 |
+
std::shared_ptr<::arrow::ResizableBuffer> def_levels_;
|
| 413 |
+
/// \brief Buffer for repetition levels. Only populated for repeated
|
| 414 |
+
/// fields.
|
| 415 |
+
std::shared_ptr<::arrow::ResizableBuffer> rep_levels_;
|
| 416 |
+
|
| 417 |
+
/// \brief Number of definition / repetition levels that have been written
|
| 418 |
+
/// internally in the reader. This may be larger than values_written() since
|
| 419 |
+
/// for repeated fields we need to look at the levels in advance to figure out
|
| 420 |
+
/// the record boundaries.
|
| 421 |
+
int64_t levels_written_;
|
| 422 |
+
/// \brief Position of the next level that should be consumed.
|
| 423 |
+
int64_t levels_position_;
|
| 424 |
+
int64_t levels_capacity_;
|
| 425 |
+
|
| 426 |
+
bool read_dictionary_ = false;
|
| 427 |
+
// If true, we will not leave any space for the null values in the values_
|
| 428 |
+
// vector or fill nulls values in BinaryRecordReader/DictionaryRecordReader.
|
| 429 |
+
//
|
| 430 |
+
// If read_dense_for_nullable_ is true, the BinaryRecordReader/DictionaryRecordReader
|
| 431 |
+
// might still populate the validity bitmap buffer.
|
| 432 |
+
bool read_dense_for_nullable_ = false;
|
| 433 |
+
};
|
| 434 |
+
|
| 435 |
+
class BinaryRecordReader : virtual public RecordReader {
|
| 436 |
+
public:
|
| 437 |
+
virtual std::vector<std::shared_ptr<::arrow::Array>> GetBuilderChunks() = 0;
|
| 438 |
+
};
|
| 439 |
+
|
| 440 |
+
/// \brief Read records directly to dictionary-encoded Arrow form (int32
|
| 441 |
+
/// indices). Only valid for BYTE_ARRAY columns
|
| 442 |
+
class DictionaryRecordReader : virtual public RecordReader {
|
| 443 |
+
public:
|
| 444 |
+
virtual std::shared_ptr<::arrow::ChunkedArray> GetResult() = 0;
|
| 445 |
+
};
|
| 446 |
+
|
| 447 |
+
} // namespace internal
|
| 448 |
+
|
| 449 |
+
using BoolReader = TypedColumnReader<BooleanType>;
|
| 450 |
+
using Int32Reader = TypedColumnReader<Int32Type>;
|
| 451 |
+
using Int64Reader = TypedColumnReader<Int64Type>;
|
| 452 |
+
using Int96Reader = TypedColumnReader<Int96Type>;
|
| 453 |
+
using FloatReader = TypedColumnReader<FloatType>;
|
| 454 |
+
using DoubleReader = TypedColumnReader<DoubleType>;
|
| 455 |
+
using ByteArrayReader = TypedColumnReader<ByteArrayType>;
|
| 456 |
+
using FixedLenByteArrayReader = TypedColumnReader<FLBAType>;
|
| 457 |
+
|
| 458 |
+
} // namespace parquet
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_scanner.h
ADDED
|
@@ -0,0 +1,264 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <stdio.h>
|
| 21 |
+
|
| 22 |
+
#include <cstdint>
|
| 23 |
+
#include <memory>
|
| 24 |
+
#include <ostream>
|
| 25 |
+
#include <string>
|
| 26 |
+
#include <utility>
|
| 27 |
+
#include <vector>
|
| 28 |
+
|
| 29 |
+
#include "parquet/column_reader.h"
|
| 30 |
+
#include "parquet/exception.h"
|
| 31 |
+
#include "parquet/platform.h"
|
| 32 |
+
#include "parquet/schema.h"
|
| 33 |
+
#include "parquet/types.h"
|
| 34 |
+
|
| 35 |
+
namespace parquet {
|
| 36 |
+
|
| 37 |
+
static constexpr int64_t DEFAULT_SCANNER_BATCH_SIZE = 128;
|
| 38 |
+
|
| 39 |
+
class PARQUET_EXPORT Scanner {
|
| 40 |
+
public:
|
| 41 |
+
explicit Scanner(std::shared_ptr<ColumnReader> reader,
|
| 42 |
+
int64_t batch_size = DEFAULT_SCANNER_BATCH_SIZE,
|
| 43 |
+
::arrow::MemoryPool* pool = ::arrow::default_memory_pool())
|
| 44 |
+
: batch_size_(batch_size),
|
| 45 |
+
level_offset_(0),
|
| 46 |
+
levels_buffered_(0),
|
| 47 |
+
value_buffer_(AllocateBuffer(pool)),
|
| 48 |
+
value_offset_(0),
|
| 49 |
+
values_buffered_(0),
|
| 50 |
+
reader_(std::move(reader)) {
|
| 51 |
+
def_levels_.resize(
|
| 52 |
+
descr()->max_definition_level() > 0 ? static_cast<size_t>(batch_size_) : 0);
|
| 53 |
+
rep_levels_.resize(
|
| 54 |
+
descr()->max_repetition_level() > 0 ? static_cast<size_t>(batch_size_) : 0);
|
| 55 |
+
}
|
| 56 |
+
|
| 57 |
+
virtual ~Scanner() {}
|
| 58 |
+
|
| 59 |
+
static std::shared_ptr<Scanner> Make(
|
| 60 |
+
std::shared_ptr<ColumnReader> col_reader,
|
| 61 |
+
int64_t batch_size = DEFAULT_SCANNER_BATCH_SIZE,
|
| 62 |
+
::arrow::MemoryPool* pool = ::arrow::default_memory_pool());
|
| 63 |
+
|
| 64 |
+
virtual void PrintNext(std::ostream& out, int width, bool with_levels = false) = 0;
|
| 65 |
+
|
| 66 |
+
bool HasNext() { return level_offset_ < levels_buffered_ || reader_->HasNext(); }
|
| 67 |
+
|
| 68 |
+
const ColumnDescriptor* descr() const { return reader_->descr(); }
|
| 69 |
+
|
| 70 |
+
int64_t batch_size() const { return batch_size_; }
|
| 71 |
+
|
| 72 |
+
void SetBatchSize(int64_t batch_size) { batch_size_ = batch_size; }
|
| 73 |
+
|
| 74 |
+
protected:
|
| 75 |
+
int64_t batch_size_;
|
| 76 |
+
|
| 77 |
+
std::vector<int16_t> def_levels_;
|
| 78 |
+
std::vector<int16_t> rep_levels_;
|
| 79 |
+
int level_offset_;
|
| 80 |
+
int levels_buffered_;
|
| 81 |
+
|
| 82 |
+
std::shared_ptr<ResizableBuffer> value_buffer_;
|
| 83 |
+
int value_offset_;
|
| 84 |
+
int64_t values_buffered_;
|
| 85 |
+
std::shared_ptr<ColumnReader> reader_;
|
| 86 |
+
};
|
| 87 |
+
|
| 88 |
+
template <typename DType>
|
| 89 |
+
class PARQUET_TEMPLATE_CLASS_EXPORT TypedScanner : public Scanner {
|
| 90 |
+
public:
|
| 91 |
+
typedef typename DType::c_type T;
|
| 92 |
+
|
| 93 |
+
explicit TypedScanner(std::shared_ptr<ColumnReader> reader,
|
| 94 |
+
int64_t batch_size = DEFAULT_SCANNER_BATCH_SIZE,
|
| 95 |
+
::arrow::MemoryPool* pool = ::arrow::default_memory_pool())
|
| 96 |
+
: Scanner(std::move(reader), batch_size, pool) {
|
| 97 |
+
typed_reader_ = static_cast<TypedColumnReader<DType>*>(reader_.get());
|
| 98 |
+
int value_byte_size = type_traits<DType::type_num>::value_byte_size;
|
| 99 |
+
PARQUET_THROW_NOT_OK(value_buffer_->Resize(batch_size_ * value_byte_size));
|
| 100 |
+
values_ = reinterpret_cast<T*>(value_buffer_->mutable_data());
|
| 101 |
+
}
|
| 102 |
+
|
| 103 |
+
virtual ~TypedScanner() {}
|
| 104 |
+
|
| 105 |
+
bool NextLevels(int16_t* def_level, int16_t* rep_level) {
|
| 106 |
+
if (level_offset_ == levels_buffered_) {
|
| 107 |
+
levels_buffered_ = static_cast<int>(
|
| 108 |
+
typed_reader_->ReadBatch(static_cast<int>(batch_size_), def_levels_.data(),
|
| 109 |
+
rep_levels_.data(), values_, &values_buffered_));
|
| 110 |
+
|
| 111 |
+
value_offset_ = 0;
|
| 112 |
+
level_offset_ = 0;
|
| 113 |
+
if (!levels_buffered_) {
|
| 114 |
+
return false;
|
| 115 |
+
}
|
| 116 |
+
}
|
| 117 |
+
*def_level = descr()->max_definition_level() > 0 ? def_levels_[level_offset_] : 0;
|
| 118 |
+
*rep_level = descr()->max_repetition_level() > 0 ? rep_levels_[level_offset_] : 0;
|
| 119 |
+
level_offset_++;
|
| 120 |
+
return true;
|
| 121 |
+
}
|
| 122 |
+
|
| 123 |
+
bool Next(T* val, int16_t* def_level, int16_t* rep_level, bool* is_null) {
|
| 124 |
+
if (level_offset_ == levels_buffered_) {
|
| 125 |
+
if (!HasNext()) {
|
| 126 |
+
// Out of data pages
|
| 127 |
+
return false;
|
| 128 |
+
}
|
| 129 |
+
}
|
| 130 |
+
|
| 131 |
+
NextLevels(def_level, rep_level);
|
| 132 |
+
*is_null = *def_level < descr()->max_definition_level();
|
| 133 |
+
|
| 134 |
+
if (*is_null) {
|
| 135 |
+
return true;
|
| 136 |
+
}
|
| 137 |
+
|
| 138 |
+
if (value_offset_ == values_buffered_) {
|
| 139 |
+
throw ParquetException("Value was non-null, but has not been buffered");
|
| 140 |
+
}
|
| 141 |
+
*val = values_[value_offset_++];
|
| 142 |
+
return true;
|
| 143 |
+
}
|
| 144 |
+
|
| 145 |
+
// Returns true if there is a next value
|
| 146 |
+
bool NextValue(T* val, bool* is_null) {
|
| 147 |
+
if (level_offset_ == levels_buffered_) {
|
| 148 |
+
if (!HasNext()) {
|
| 149 |
+
// Out of data pages
|
| 150 |
+
return false;
|
| 151 |
+
}
|
| 152 |
+
}
|
| 153 |
+
|
| 154 |
+
// Out of values
|
| 155 |
+
int16_t def_level = -1;
|
| 156 |
+
int16_t rep_level = -1;
|
| 157 |
+
NextLevels(&def_level, &rep_level);
|
| 158 |
+
*is_null = def_level < descr()->max_definition_level();
|
| 159 |
+
|
| 160 |
+
if (*is_null) {
|
| 161 |
+
return true;
|
| 162 |
+
}
|
| 163 |
+
|
| 164 |
+
if (value_offset_ == values_buffered_) {
|
| 165 |
+
throw ParquetException("Value was non-null, but has not been buffered");
|
| 166 |
+
}
|
| 167 |
+
*val = values_[value_offset_++];
|
| 168 |
+
return true;
|
| 169 |
+
}
|
| 170 |
+
|
| 171 |
+
virtual void PrintNext(std::ostream& out, int width, bool with_levels = false) {
|
| 172 |
+
T val{};
|
| 173 |
+
int16_t def_level = -1;
|
| 174 |
+
int16_t rep_level = -1;
|
| 175 |
+
bool is_null = false;
|
| 176 |
+
char buffer[80];
|
| 177 |
+
|
| 178 |
+
if (!Next(&val, &def_level, &rep_level, &is_null)) {
|
| 179 |
+
throw ParquetException("No more values buffered");
|
| 180 |
+
}
|
| 181 |
+
|
| 182 |
+
if (with_levels) {
|
| 183 |
+
out << " D:" << def_level << " R:" << rep_level << " ";
|
| 184 |
+
if (!is_null) {
|
| 185 |
+
out << "V:";
|
| 186 |
+
}
|
| 187 |
+
}
|
| 188 |
+
|
| 189 |
+
if (is_null) {
|
| 190 |
+
std::string null_fmt = format_fwf<ByteArrayType>(width);
|
| 191 |
+
snprintf(buffer, sizeof(buffer), null_fmt.c_str(), "NULL");
|
| 192 |
+
} else {
|
| 193 |
+
FormatValue(&val, buffer, sizeof(buffer), width);
|
| 194 |
+
}
|
| 195 |
+
out << buffer;
|
| 196 |
+
}
|
| 197 |
+
|
| 198 |
+
private:
|
| 199 |
+
// The ownership of this object is expressed through the reader_ variable in the base
|
| 200 |
+
TypedColumnReader<DType>* typed_reader_;
|
| 201 |
+
|
| 202 |
+
inline void FormatValue(void* val, char* buffer, int bufsize, int width);
|
| 203 |
+
|
| 204 |
+
T* values_;
|
| 205 |
+
};
|
| 206 |
+
|
| 207 |
+
template <typename DType>
|
| 208 |
+
inline void TypedScanner<DType>::FormatValue(void* val, char* buffer, int bufsize,
|
| 209 |
+
int width) {
|
| 210 |
+
std::string fmt = format_fwf<DType>(width);
|
| 211 |
+
snprintf(buffer, bufsize, fmt.c_str(), *reinterpret_cast<T*>(val));
|
| 212 |
+
}
|
| 213 |
+
|
| 214 |
+
template <>
|
| 215 |
+
inline void TypedScanner<Int96Type>::FormatValue(void* val, char* buffer, int bufsize,
|
| 216 |
+
int width) {
|
| 217 |
+
std::string fmt = format_fwf<Int96Type>(width);
|
| 218 |
+
std::string result = Int96ToString(*reinterpret_cast<Int96*>(val));
|
| 219 |
+
snprintf(buffer, bufsize, fmt.c_str(), result.c_str());
|
| 220 |
+
}
|
| 221 |
+
|
| 222 |
+
template <>
|
| 223 |
+
inline void TypedScanner<ByteArrayType>::FormatValue(void* val, char* buffer, int bufsize,
|
| 224 |
+
int width) {
|
| 225 |
+
std::string fmt = format_fwf<ByteArrayType>(width);
|
| 226 |
+
std::string result = ByteArrayToString(*reinterpret_cast<ByteArray*>(val));
|
| 227 |
+
snprintf(buffer, bufsize, fmt.c_str(), result.c_str());
|
| 228 |
+
}
|
| 229 |
+
|
| 230 |
+
template <>
|
| 231 |
+
inline void TypedScanner<FLBAType>::FormatValue(void* val, char* buffer, int bufsize,
|
| 232 |
+
int width) {
|
| 233 |
+
std::string fmt = format_fwf<FLBAType>(width);
|
| 234 |
+
std::string result = FixedLenByteArrayToString(
|
| 235 |
+
*reinterpret_cast<FixedLenByteArray*>(val), descr()->type_length());
|
| 236 |
+
snprintf(buffer, bufsize, fmt.c_str(), result.c_str());
|
| 237 |
+
}
|
| 238 |
+
|
| 239 |
+
typedef TypedScanner<BooleanType> BoolScanner;
|
| 240 |
+
typedef TypedScanner<Int32Type> Int32Scanner;
|
| 241 |
+
typedef TypedScanner<Int64Type> Int64Scanner;
|
| 242 |
+
typedef TypedScanner<Int96Type> Int96Scanner;
|
| 243 |
+
typedef TypedScanner<FloatType> FloatScanner;
|
| 244 |
+
typedef TypedScanner<DoubleType> DoubleScanner;
|
| 245 |
+
typedef TypedScanner<ByteArrayType> ByteArrayScanner;
|
| 246 |
+
typedef TypedScanner<FLBAType> FixedLenByteArrayScanner;
|
| 247 |
+
|
| 248 |
+
template <typename RType>
|
| 249 |
+
int64_t ScanAll(int32_t batch_size, int16_t* def_levels, int16_t* rep_levels,
|
| 250 |
+
uint8_t* values, int64_t* values_buffered,
|
| 251 |
+
parquet::ColumnReader* reader) {
|
| 252 |
+
typedef typename RType::T Type;
|
| 253 |
+
auto typed_reader = static_cast<RType*>(reader);
|
| 254 |
+
auto vals = reinterpret_cast<Type*>(&values[0]);
|
| 255 |
+
return typed_reader->ReadBatch(batch_size, def_levels, rep_levels, vals,
|
| 256 |
+
values_buffered);
|
| 257 |
+
}
|
| 258 |
+
|
| 259 |
+
int64_t PARQUET_EXPORT ScanAllValues(int32_t batch_size, int16_t* def_levels,
|
| 260 |
+
int16_t* rep_levels, uint8_t* values,
|
| 261 |
+
int64_t* values_buffered,
|
| 262 |
+
parquet::ColumnReader* reader);
|
| 263 |
+
|
| 264 |
+
} // namespace parquet
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_writer.h
ADDED
|
@@ -0,0 +1,317 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <cstdint>
|
| 21 |
+
#include <cstring>
|
| 22 |
+
#include <memory>
|
| 23 |
+
|
| 24 |
+
#include "arrow/type_fwd.h"
|
| 25 |
+
#include "arrow/util/compression.h"
|
| 26 |
+
#include "parquet/exception.h"
|
| 27 |
+
#include "parquet/platform.h"
|
| 28 |
+
#include "parquet/types.h"
|
| 29 |
+
|
| 30 |
+
namespace arrow {
|
| 31 |
+
|
| 32 |
+
class Array;
|
| 33 |
+
|
| 34 |
+
namespace bit_util {
|
| 35 |
+
class BitWriter;
|
| 36 |
+
} // namespace bit_util
|
| 37 |
+
|
| 38 |
+
namespace util {
|
| 39 |
+
class RleBitPackedEncoder;
|
| 40 |
+
class CodecOptions;
|
| 41 |
+
} // namespace util
|
| 42 |
+
|
| 43 |
+
} // namespace arrow
|
| 44 |
+
|
| 45 |
+
namespace parquet {
|
| 46 |
+
|
| 47 |
+
struct ArrowWriteContext;
|
| 48 |
+
class ColumnChunkMetaDataBuilder;
|
| 49 |
+
class ColumnDescriptor;
|
| 50 |
+
class ColumnIndexBuilder;
|
| 51 |
+
class DataPage;
|
| 52 |
+
class DictionaryPage;
|
| 53 |
+
class Encryptor;
|
| 54 |
+
class OffsetIndexBuilder;
|
| 55 |
+
class WriterProperties;
|
| 56 |
+
|
| 57 |
+
class PARQUET_EXPORT LevelEncoder {
|
| 58 |
+
public:
|
| 59 |
+
LevelEncoder();
|
| 60 |
+
~LevelEncoder();
|
| 61 |
+
|
| 62 |
+
static int MaxBufferSize(Encoding::type encoding, int16_t max_level,
|
| 63 |
+
int num_buffered_values);
|
| 64 |
+
|
| 65 |
+
// Initialize the LevelEncoder.
|
| 66 |
+
void Init(Encoding::type encoding, int16_t max_level, int num_buffered_values,
|
| 67 |
+
uint8_t* data, int data_size);
|
| 68 |
+
|
| 69 |
+
// Encodes a batch of levels from an array and returns the number of levels encoded
|
| 70 |
+
int Encode(int batch_size, const int16_t* levels);
|
| 71 |
+
|
| 72 |
+
int32_t len() {
|
| 73 |
+
if (encoding_ != Encoding::RLE) {
|
| 74 |
+
throw ParquetException("Only implemented for RLE encoding");
|
| 75 |
+
}
|
| 76 |
+
return rle_length_;
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
private:
|
| 80 |
+
int bit_width_;
|
| 81 |
+
int rle_length_;
|
| 82 |
+
Encoding::type encoding_;
|
| 83 |
+
std::unique_ptr<::arrow::util::RleBitPackedEncoder> rle_encoder_;
|
| 84 |
+
std::unique_ptr<::arrow::bit_util::BitWriter> bit_packed_encoder_;
|
| 85 |
+
};
|
| 86 |
+
|
| 87 |
+
class PARQUET_EXPORT PageWriter {
|
| 88 |
+
public:
|
| 89 |
+
virtual ~PageWriter() {}
|
| 90 |
+
|
| 91 |
+
static std::unique_ptr<PageWriter> Open(
|
| 92 |
+
std::shared_ptr<ArrowOutputStream> sink, Compression::type codec,
|
| 93 |
+
ColumnChunkMetaDataBuilder* metadata, int16_t row_group_ordinal = -1,
|
| 94 |
+
int16_t column_chunk_ordinal = -1,
|
| 95 |
+
::arrow::MemoryPool* pool = ::arrow::default_memory_pool(),
|
| 96 |
+
bool buffered_row_group = false,
|
| 97 |
+
std::shared_ptr<Encryptor> header_encryptor = NULLPTR,
|
| 98 |
+
std::shared_ptr<Encryptor> data_encryptor = NULLPTR,
|
| 99 |
+
bool page_write_checksum_enabled = false,
|
| 100 |
+
// column_index_builder MUST outlive the PageWriter
|
| 101 |
+
ColumnIndexBuilder* column_index_builder = NULLPTR,
|
| 102 |
+
// offset_index_builder MUST outlive the PageWriter
|
| 103 |
+
OffsetIndexBuilder* offset_index_builder = NULLPTR,
|
| 104 |
+
const CodecOptions& codec_options = CodecOptions{});
|
| 105 |
+
|
| 106 |
+
// The Column Writer decides if dictionary encoding is used if set and
|
| 107 |
+
// if the dictionary encoding has fallen back to default encoding on reaching dictionary
|
| 108 |
+
// page limit
|
| 109 |
+
virtual void Close(bool has_dictionary, bool fallback) = 0;
|
| 110 |
+
|
| 111 |
+
// Return the number of uncompressed bytes written (including header size)
|
| 112 |
+
virtual int64_t WriteDataPage(const DataPage& page) = 0;
|
| 113 |
+
|
| 114 |
+
// Return the number of uncompressed bytes written (including header size)
|
| 115 |
+
virtual int64_t WriteDictionaryPage(const DictionaryPage& page) = 0;
|
| 116 |
+
|
| 117 |
+
/// \brief The total number of bytes written as serialized data and
|
| 118 |
+
/// dictionary pages to the sink so far.
|
| 119 |
+
virtual int64_t total_compressed_bytes_written() const = 0;
|
| 120 |
+
|
| 121 |
+
virtual bool has_compressor() = 0;
|
| 122 |
+
|
| 123 |
+
virtual void Compress(const Buffer& src_buffer, ResizableBuffer* dest_buffer) = 0;
|
| 124 |
+
};
|
| 125 |
+
|
| 126 |
+
class PARQUET_EXPORT ColumnWriter {
|
| 127 |
+
public:
|
| 128 |
+
virtual ~ColumnWriter() = default;
|
| 129 |
+
|
| 130 |
+
static std::shared_ptr<ColumnWriter> Make(ColumnChunkMetaDataBuilder*,
|
| 131 |
+
std::unique_ptr<PageWriter>,
|
| 132 |
+
const WriterProperties* properties,
|
| 133 |
+
BloomFilter* bloom_filter = NULLPTR);
|
| 134 |
+
|
| 135 |
+
/// \brief Closes the ColumnWriter, commits any buffered values to pages.
|
| 136 |
+
/// \return Total size of the column in bytes
|
| 137 |
+
virtual int64_t Close() = 0;
|
| 138 |
+
|
| 139 |
+
/// \brief The physical Parquet type of the column
|
| 140 |
+
virtual Type::type type() const = 0;
|
| 141 |
+
|
| 142 |
+
/// \brief The schema for the column
|
| 143 |
+
virtual const ColumnDescriptor* descr() const = 0;
|
| 144 |
+
|
| 145 |
+
/// \brief The number of rows written so far
|
| 146 |
+
virtual int64_t rows_written() const = 0;
|
| 147 |
+
|
| 148 |
+
/// \brief The total size of the compressed pages + page headers. Values
|
| 149 |
+
/// are still buffered and not written to a pager yet
|
| 150 |
+
///
|
| 151 |
+
/// So in un-buffered mode, it always returns 0
|
| 152 |
+
virtual int64_t total_compressed_bytes() const = 0;
|
| 153 |
+
|
| 154 |
+
/// \brief The total number of bytes written as serialized data and
|
| 155 |
+
/// dictionary pages to the ColumnChunk so far
|
| 156 |
+
/// These bytes are uncompressed bytes.
|
| 157 |
+
virtual int64_t total_bytes_written() const = 0;
|
| 158 |
+
|
| 159 |
+
/// \brief The total number of bytes written as serialized data and
|
| 160 |
+
/// dictionary pages to the ColumnChunk so far.
|
| 161 |
+
/// If the column is uncompressed, the value would be equal to
|
| 162 |
+
/// total_bytes_written().
|
| 163 |
+
virtual int64_t total_compressed_bytes_written() const = 0;
|
| 164 |
+
|
| 165 |
+
/// \brief Estimated size of the values that are not written to a page yet.
|
| 166 |
+
virtual int64_t estimated_buffered_value_bytes() const = 0;
|
| 167 |
+
|
| 168 |
+
/// \brief Estimated size of the definition levels that are not written to a page yet.
|
| 169 |
+
virtual int64_t estimated_buffered_def_level_bytes() const = 0;
|
| 170 |
+
|
| 171 |
+
/// \brief Estimated size of the repetition levels that are not written to a page yet.
|
| 172 |
+
virtual int64_t estimated_buffered_rep_level_bytes() const = 0;
|
| 173 |
+
|
| 174 |
+
/// \brief Estimated size of the dictionary that are not written to a page yet.
|
| 175 |
+
virtual int64_t estimated_buffered_dict_bytes() const = 0;
|
| 176 |
+
|
| 177 |
+
/// \brief The file-level writer properties
|
| 178 |
+
virtual const WriterProperties* properties() = 0;
|
| 179 |
+
|
| 180 |
+
/// \brief Add key-value metadata to the ColumnChunk.
|
| 181 |
+
/// \param[in] key_value_metadata the metadata to add.
|
| 182 |
+
/// \note This will overwrite any existing metadata with the same key.
|
| 183 |
+
/// \throw ParquetException if Close() has been called.
|
| 184 |
+
virtual void AddKeyValueMetadata(
|
| 185 |
+
const std::shared_ptr<const ::arrow::KeyValueMetadata>& key_value_metadata) = 0;
|
| 186 |
+
|
| 187 |
+
/// \brief Reset the ColumnChunk key-value metadata.
|
| 188 |
+
/// \throw ParquetException if Close() has been called.
|
| 189 |
+
virtual void ResetKeyValueMetadata() = 0;
|
| 190 |
+
|
| 191 |
+
/// \brief Write Apache Arrow columnar data directly to ColumnWriter. Returns
|
| 192 |
+
/// error status if the array data type is not compatible with the concrete
|
| 193 |
+
/// writer type.
|
| 194 |
+
///
|
| 195 |
+
/// leaf_array is always a primitive (possibly dictionary encoded type).
|
| 196 |
+
/// Leaf_field_nullable indicates whether the leaf array is considered nullable
|
| 197 |
+
/// according to its schema in a Table or its parent array.
|
| 198 |
+
virtual ::arrow::Status WriteArrow(const int16_t* def_levels, const int16_t* rep_levels,
|
| 199 |
+
int64_t num_levels, const ::arrow::Array& leaf_array,
|
| 200 |
+
ArrowWriteContext* ctx,
|
| 201 |
+
bool leaf_field_nullable) = 0;
|
| 202 |
+
};
|
| 203 |
+
|
| 204 |
+
// API to write values to a single column. This is the main client facing API.
|
| 205 |
+
template <typename DType>
|
| 206 |
+
class TypedColumnWriter : public ColumnWriter {
|
| 207 |
+
public:
|
| 208 |
+
using T = typename DType::c_type;
|
| 209 |
+
|
| 210 |
+
// Write a batch of repetition levels, definition levels, and values to the
|
| 211 |
+
// column.
|
| 212 |
+
// `num_values` is the number of logical leaf values.
|
| 213 |
+
// `def_levels` (resp. `rep_levels`) can be null if the column's max definition level
|
| 214 |
+
// (resp. max repetition level) is 0.
|
| 215 |
+
// If not null, each of `def_levels` and `rep_levels` must have at least
|
| 216 |
+
// `num_values`.
|
| 217 |
+
//
|
| 218 |
+
// The number of physical values written (taken from `values`) is returned.
|
| 219 |
+
// It can be smaller than `num_values` is there are some undefined values.
|
| 220 |
+
virtual int64_t WriteBatch(int64_t num_values, const int16_t* def_levels,
|
| 221 |
+
const int16_t* rep_levels, const T* values) = 0;
|
| 222 |
+
|
| 223 |
+
/// Write a batch of repetition levels, definition levels, and values to the
|
| 224 |
+
/// column.
|
| 225 |
+
///
|
| 226 |
+
/// In comparison to WriteBatch the length of repetition and definition levels
|
| 227 |
+
/// is the same as of the number of values read for max_definition_level == 1.
|
| 228 |
+
/// In the case of max_definition_level > 1, the repetition and definition
|
| 229 |
+
/// levels are larger than the values but the values include the null entries
|
| 230 |
+
/// with definition_level == (max_definition_level - 1). Thus we have to differentiate
|
| 231 |
+
/// in the parameters of this function if the input has the length of num_values or the
|
| 232 |
+
/// _number of rows in the lowest nesting level_.
|
| 233 |
+
///
|
| 234 |
+
/// In the case that the most inner node in the Parquet is required, the _number of rows
|
| 235 |
+
/// in the lowest nesting level_ is equal to the number of non-null values. If the
|
| 236 |
+
/// inner-most schema node is optional, the _number of rows in the lowest nesting level_
|
| 237 |
+
/// also includes all values with definition_level == (max_definition_level - 1).
|
| 238 |
+
///
|
| 239 |
+
/// @param num_values number of levels to write.
|
| 240 |
+
/// @param def_levels The Parquet definition levels, length is num_values
|
| 241 |
+
/// @param rep_levels The Parquet repetition levels, length is num_values
|
| 242 |
+
/// @param valid_bits Bitmap that indicates if the row is null on the lowest nesting
|
| 243 |
+
/// level. The length is number of rows in the lowest nesting level.
|
| 244 |
+
/// @param valid_bits_offset The offset in bits of the valid_bits where the
|
| 245 |
+
/// first relevant bit resides.
|
| 246 |
+
/// @param values The values in the lowest nested level including
|
| 247 |
+
/// spacing for nulls on the lowest levels; input has the length
|
| 248 |
+
/// of the number of rows on the lowest nesting level.
|
| 249 |
+
virtual void WriteBatchSpaced(int64_t num_values, const int16_t* def_levels,
|
| 250 |
+
const int16_t* rep_levels, const uint8_t* valid_bits,
|
| 251 |
+
int64_t valid_bits_offset, const T* values) = 0;
|
| 252 |
+
};
|
| 253 |
+
|
| 254 |
+
using BoolWriter = TypedColumnWriter<BooleanType>;
|
| 255 |
+
using Int32Writer = TypedColumnWriter<Int32Type>;
|
| 256 |
+
using Int64Writer = TypedColumnWriter<Int64Type>;
|
| 257 |
+
using Int96Writer = TypedColumnWriter<Int96Type>;
|
| 258 |
+
using FloatWriter = TypedColumnWriter<FloatType>;
|
| 259 |
+
using DoubleWriter = TypedColumnWriter<DoubleType>;
|
| 260 |
+
using ByteArrayWriter = TypedColumnWriter<ByteArrayType>;
|
| 261 |
+
using FixedLenByteArrayWriter = TypedColumnWriter<FLBAType>;
|
| 262 |
+
|
| 263 |
+
namespace internal {
|
| 264 |
+
|
| 265 |
+
/**
|
| 266 |
+
* Timestamp conversion constants
|
| 267 |
+
*/
|
| 268 |
+
constexpr int64_t kJulianEpochOffsetDays = INT64_C(2440588);
|
| 269 |
+
|
| 270 |
+
template <int64_t UnitPerDay, int64_t NanosecondsPerUnit>
|
| 271 |
+
inline void ArrowTimestampToImpalaTimestamp(const int64_t time, Int96* impala_timestamp) {
|
| 272 |
+
auto julian_days = static_cast<int32_t>(time / UnitPerDay + kJulianEpochOffsetDays);
|
| 273 |
+
int64_t last_day_units = time % UnitPerDay;
|
| 274 |
+
if (last_day_units < 0) {
|
| 275 |
+
--julian_days;
|
| 276 |
+
last_day_units += UnitPerDay;
|
| 277 |
+
}
|
| 278 |
+
impala_timestamp->value[2] = static_cast<uint32_t>(julian_days);
|
| 279 |
+
uint64_t last_day_nanos = static_cast<uint64_t>(last_day_units) * NanosecondsPerUnit;
|
| 280 |
+
// impala_timestamp will be unaligned every other entry so do memcpy instead
|
| 281 |
+
// of assign and reinterpret cast to avoid undefined behavior.
|
| 282 |
+
std::memcpy(impala_timestamp, &last_day_nanos, sizeof(uint64_t));
|
| 283 |
+
}
|
| 284 |
+
|
| 285 |
+
constexpr int64_t kSecondsInNanos = INT64_C(1000000000);
|
| 286 |
+
|
| 287 |
+
inline void SecondsToImpalaTimestamp(const int64_t seconds, Int96* impala_timestamp) {
|
| 288 |
+
ArrowTimestampToImpalaTimestamp<kSecondsPerDay, kSecondsInNanos>(seconds,
|
| 289 |
+
impala_timestamp);
|
| 290 |
+
}
|
| 291 |
+
|
| 292 |
+
constexpr int64_t kMillisecondsInNanos = kSecondsInNanos / INT64_C(1000);
|
| 293 |
+
|
| 294 |
+
inline void MillisecondsToImpalaTimestamp(const int64_t milliseconds,
|
| 295 |
+
Int96* impala_timestamp) {
|
| 296 |
+
ArrowTimestampToImpalaTimestamp<kMillisecondsPerDay, kMillisecondsInNanos>(
|
| 297 |
+
milliseconds, impala_timestamp);
|
| 298 |
+
}
|
| 299 |
+
|
| 300 |
+
constexpr int64_t kMicrosecondsInNanos = kMillisecondsInNanos / INT64_C(1000);
|
| 301 |
+
|
| 302 |
+
inline void MicrosecondsToImpalaTimestamp(const int64_t microseconds,
|
| 303 |
+
Int96* impala_timestamp) {
|
| 304 |
+
ArrowTimestampToImpalaTimestamp<kMicrosecondsPerDay, kMicrosecondsInNanos>(
|
| 305 |
+
microseconds, impala_timestamp);
|
| 306 |
+
}
|
| 307 |
+
|
| 308 |
+
constexpr int64_t kNanosecondsInNanos = INT64_C(1);
|
| 309 |
+
|
| 310 |
+
inline void NanosecondsToImpalaTimestamp(const int64_t nanoseconds,
|
| 311 |
+
Int96* impala_timestamp) {
|
| 312 |
+
ArrowTimestampToImpalaTimestamp<kNanosecondsPerDay, kNanosecondsInNanos>(
|
| 313 |
+
nanoseconds, impala_timestamp);
|
| 314 |
+
}
|
| 315 |
+
|
| 316 |
+
} // namespace internal
|
| 317 |
+
} // namespace parquet
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encoding.h
ADDED
|
@@ -0,0 +1,476 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <cstdint>
|
| 21 |
+
#include <cstring>
|
| 22 |
+
#include <memory>
|
| 23 |
+
#include <vector>
|
| 24 |
+
|
| 25 |
+
#include "arrow/type_fwd.h"
|
| 26 |
+
|
| 27 |
+
#include "parquet/exception.h"
|
| 28 |
+
#include "parquet/platform.h"
|
| 29 |
+
#include "parquet/types.h"
|
| 30 |
+
|
| 31 |
+
namespace arrow {
|
| 32 |
+
template <typename T>
|
| 33 |
+
class Dictionary32Builder;
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
namespace parquet {
|
| 37 |
+
|
| 38 |
+
template <typename DType>
|
| 39 |
+
class TypedEncoder;
|
| 40 |
+
|
| 41 |
+
using BooleanEncoder = TypedEncoder<BooleanType>;
|
| 42 |
+
using Int32Encoder = TypedEncoder<Int32Type>;
|
| 43 |
+
using Int64Encoder = TypedEncoder<Int64Type>;
|
| 44 |
+
using Int96Encoder = TypedEncoder<Int96Type>;
|
| 45 |
+
using FloatEncoder = TypedEncoder<FloatType>;
|
| 46 |
+
using DoubleEncoder = TypedEncoder<DoubleType>;
|
| 47 |
+
using ByteArrayEncoder = TypedEncoder<ByteArrayType>;
|
| 48 |
+
using FLBAEncoder = TypedEncoder<FLBAType>;
|
| 49 |
+
|
| 50 |
+
template <typename DType>
|
| 51 |
+
class TypedDecoder;
|
| 52 |
+
|
| 53 |
+
class BooleanDecoder;
|
| 54 |
+
using Int32Decoder = TypedDecoder<Int32Type>;
|
| 55 |
+
using Int64Decoder = TypedDecoder<Int64Type>;
|
| 56 |
+
using Int96Decoder = TypedDecoder<Int96Type>;
|
| 57 |
+
using FloatDecoder = TypedDecoder<FloatType>;
|
| 58 |
+
using DoubleDecoder = TypedDecoder<DoubleType>;
|
| 59 |
+
using ByteArrayDecoder = TypedDecoder<ByteArrayType>;
|
| 60 |
+
class FLBADecoder;
|
| 61 |
+
|
| 62 |
+
template <typename T>
|
| 63 |
+
struct EncodingTraits;
|
| 64 |
+
|
| 65 |
+
template <>
|
| 66 |
+
struct EncodingTraits<BooleanType> {
|
| 67 |
+
using Encoder = BooleanEncoder;
|
| 68 |
+
using Decoder = BooleanDecoder;
|
| 69 |
+
|
| 70 |
+
using ArrowType = ::arrow::BooleanType;
|
| 71 |
+
using Accumulator = ::arrow::BooleanBuilder;
|
| 72 |
+
struct DictAccumulator {};
|
| 73 |
+
};
|
| 74 |
+
|
| 75 |
+
template <>
|
| 76 |
+
struct EncodingTraits<Int32Type> {
|
| 77 |
+
using Encoder = Int32Encoder;
|
| 78 |
+
using Decoder = Int32Decoder;
|
| 79 |
+
|
| 80 |
+
using ArrowType = ::arrow::Int32Type;
|
| 81 |
+
using Accumulator = ::arrow::NumericBuilder<::arrow::Int32Type>;
|
| 82 |
+
using DictAccumulator = ::arrow::Dictionary32Builder<::arrow::Int32Type>;
|
| 83 |
+
};
|
| 84 |
+
|
| 85 |
+
template <>
|
| 86 |
+
struct EncodingTraits<Int64Type> {
|
| 87 |
+
using Encoder = Int64Encoder;
|
| 88 |
+
using Decoder = Int64Decoder;
|
| 89 |
+
|
| 90 |
+
using ArrowType = ::arrow::Int64Type;
|
| 91 |
+
using Accumulator = ::arrow::NumericBuilder<::arrow::Int64Type>;
|
| 92 |
+
using DictAccumulator = ::arrow::Dictionary32Builder<::arrow::Int64Type>;
|
| 93 |
+
};
|
| 94 |
+
|
| 95 |
+
template <>
|
| 96 |
+
struct EncodingTraits<Int96Type> {
|
| 97 |
+
using Encoder = Int96Encoder;
|
| 98 |
+
using Decoder = Int96Decoder;
|
| 99 |
+
|
| 100 |
+
struct Accumulator {};
|
| 101 |
+
struct DictAccumulator {};
|
| 102 |
+
};
|
| 103 |
+
|
| 104 |
+
template <>
|
| 105 |
+
struct EncodingTraits<FloatType> {
|
| 106 |
+
using Encoder = FloatEncoder;
|
| 107 |
+
using Decoder = FloatDecoder;
|
| 108 |
+
|
| 109 |
+
using ArrowType = ::arrow::FloatType;
|
| 110 |
+
using Accumulator = ::arrow::NumericBuilder<::arrow::FloatType>;
|
| 111 |
+
using DictAccumulator = ::arrow::Dictionary32Builder<::arrow::FloatType>;
|
| 112 |
+
};
|
| 113 |
+
|
| 114 |
+
template <>
|
| 115 |
+
struct EncodingTraits<DoubleType> {
|
| 116 |
+
using Encoder = DoubleEncoder;
|
| 117 |
+
using Decoder = DoubleDecoder;
|
| 118 |
+
|
| 119 |
+
using ArrowType = ::arrow::DoubleType;
|
| 120 |
+
using Accumulator = ::arrow::NumericBuilder<::arrow::DoubleType>;
|
| 121 |
+
using DictAccumulator = ::arrow::Dictionary32Builder<::arrow::DoubleType>;
|
| 122 |
+
};
|
| 123 |
+
|
| 124 |
+
template <>
|
| 125 |
+
struct EncodingTraits<ByteArrayType> {
|
| 126 |
+
using Encoder = ByteArrayEncoder;
|
| 127 |
+
using Decoder = ByteArrayDecoder;
|
| 128 |
+
|
| 129 |
+
/// \brief Internal helper class for decoding BYTE_ARRAY data
|
| 130 |
+
///
|
| 131 |
+
/// This class allows the caller to choose the concrete Arrow data type
|
| 132 |
+
/// by passing a corresponding `ArrayBuilder`.
|
| 133 |
+
/// Supported `ArrayBuilder` classes are `BinaryBuilder`, `LargeBinaryBuilder`
|
| 134 |
+
/// and `BinaryViewBuilder`.
|
| 135 |
+
/// If the builder is a `BinaryBuilder`, `chunks` can accumulate several
|
| 136 |
+
/// arrays as needed to work around the 32-bit offset limit.
|
| 137 |
+
struct Accumulator {
|
| 138 |
+
std::unique_ptr<::arrow::ArrayBuilder> builder;
|
| 139 |
+
std::vector<std::shared_ptr<::arrow::Array>> chunks;
|
| 140 |
+
};
|
| 141 |
+
using DictAccumulator = ::arrow::Dictionary32Builder<::arrow::BinaryType>;
|
| 142 |
+
};
|
| 143 |
+
|
| 144 |
+
template <>
|
| 145 |
+
struct EncodingTraits<FLBAType> {
|
| 146 |
+
using Encoder = FLBAEncoder;
|
| 147 |
+
using Decoder = FLBADecoder;
|
| 148 |
+
|
| 149 |
+
using ArrowType = ::arrow::FixedSizeBinaryType;
|
| 150 |
+
using Accumulator = ::arrow::FixedSizeBinaryBuilder;
|
| 151 |
+
using DictAccumulator = ::arrow::Dictionary32Builder<::arrow::FixedSizeBinaryType>;
|
| 152 |
+
};
|
| 153 |
+
|
| 154 |
+
class ColumnDescriptor;
|
| 155 |
+
|
| 156 |
+
// Untyped base for all encoders
|
| 157 |
+
class Encoder {
|
| 158 |
+
public:
|
| 159 |
+
virtual ~Encoder() = default;
|
| 160 |
+
|
| 161 |
+
virtual int64_t EstimatedDataEncodedSize() = 0;
|
| 162 |
+
virtual std::shared_ptr<Buffer> FlushValues() = 0;
|
| 163 |
+
virtual Encoding::type encoding() const = 0;
|
| 164 |
+
|
| 165 |
+
virtual void Put(const ::arrow::Array& values) = 0;
|
| 166 |
+
|
| 167 |
+
// Report the number of bytes written to the encoder since the last report.
|
| 168 |
+
// It only works for BYTE_ARRAY type and throw for other types.
|
| 169 |
+
// This call is not idempotent since it resets the internal counter.
|
| 170 |
+
virtual int64_t ReportUnencodedDataBytes() = 0;
|
| 171 |
+
|
| 172 |
+
virtual MemoryPool* memory_pool() const = 0;
|
| 173 |
+
};
|
| 174 |
+
|
| 175 |
+
// Base class for value encoders. Since encoders may or not have state (e.g.,
|
| 176 |
+
// dictionary encoding) we use a class instance to maintain any state.
|
| 177 |
+
//
|
| 178 |
+
// Encode interfaces are internal, subject to change without deprecation.
|
| 179 |
+
template <typename DType>
|
| 180 |
+
class TypedEncoder : virtual public Encoder {
|
| 181 |
+
public:
|
| 182 |
+
using T = typename DType::c_type;
|
| 183 |
+
|
| 184 |
+
using Encoder::Put;
|
| 185 |
+
|
| 186 |
+
virtual void Put(const T* src, int num_values) = 0;
|
| 187 |
+
|
| 188 |
+
virtual void Put(const std::vector<T>& src, int num_values = -1);
|
| 189 |
+
|
| 190 |
+
virtual void PutSpaced(const T* src, int num_values, const uint8_t* valid_bits,
|
| 191 |
+
int64_t valid_bits_offset) = 0;
|
| 192 |
+
};
|
| 193 |
+
|
| 194 |
+
template <typename DType>
|
| 195 |
+
void TypedEncoder<DType>::Put(const std::vector<T>& src, int num_values) {
|
| 196 |
+
if (num_values == -1) {
|
| 197 |
+
num_values = static_cast<int>(src.size());
|
| 198 |
+
}
|
| 199 |
+
Put(src.data(), num_values);
|
| 200 |
+
}
|
| 201 |
+
|
| 202 |
+
template <>
|
| 203 |
+
inline void TypedEncoder<BooleanType>::Put(const std::vector<bool>& src, int num_values) {
|
| 204 |
+
// NOTE(wesm): This stub is here only to satisfy the compiler; it is
|
| 205 |
+
// overridden later with the actual implementation
|
| 206 |
+
}
|
| 207 |
+
|
| 208 |
+
// Base class for dictionary encoders
|
| 209 |
+
template <typename DType>
|
| 210 |
+
class DictEncoder : virtual public TypedEncoder<DType> {
|
| 211 |
+
public:
|
| 212 |
+
/// Writes out any buffered indices to buffer preceded by the bit width of this data.
|
| 213 |
+
/// Returns the number of bytes written.
|
| 214 |
+
/// If the supplied buffer is not big enough, returns -1.
|
| 215 |
+
/// buffer must be preallocated with buffer_len bytes. Use EstimatedDataEncodedSize()
|
| 216 |
+
/// to size buffer.
|
| 217 |
+
virtual int WriteIndices(uint8_t* buffer, int buffer_len) = 0;
|
| 218 |
+
|
| 219 |
+
virtual int dict_encoded_size() const = 0;
|
| 220 |
+
|
| 221 |
+
virtual int bit_width() const = 0;
|
| 222 |
+
|
| 223 |
+
/// Writes out the encoded dictionary to buffer. buffer must be preallocated to
|
| 224 |
+
/// dict_encoded_size() bytes.
|
| 225 |
+
virtual void WriteDict(uint8_t* buffer) const = 0;
|
| 226 |
+
|
| 227 |
+
virtual int num_entries() const = 0;
|
| 228 |
+
|
| 229 |
+
/// \brief EXPERIMENTAL: Append dictionary indices into the encoder. It is
|
| 230 |
+
/// assumed (without any boundschecking) that the indices reference
|
| 231 |
+
/// preexisting dictionary values
|
| 232 |
+
/// \param[in] indices the dictionary index values. Only Int32Array currently
|
| 233 |
+
/// supported
|
| 234 |
+
virtual void PutIndices(const ::arrow::Array& indices) = 0;
|
| 235 |
+
|
| 236 |
+
/// \brief EXPERIMENTAL: Append dictionary into encoder, inserting indices
|
| 237 |
+
/// separately. Currently throws exception if the current dictionary memo is
|
| 238 |
+
/// non-empty
|
| 239 |
+
/// \param[in] values the dictionary values. Only valid for certain
|
| 240 |
+
/// Parquet/Arrow type combinations, like BYTE_ARRAY/BinaryArray
|
| 241 |
+
virtual void PutDictionary(const ::arrow::Array& values) = 0;
|
| 242 |
+
};
|
| 243 |
+
|
| 244 |
+
// ----------------------------------------------------------------------
|
| 245 |
+
// Value decoding
|
| 246 |
+
|
| 247 |
+
class Decoder {
|
| 248 |
+
public:
|
| 249 |
+
virtual ~Decoder() = default;
|
| 250 |
+
|
| 251 |
+
// Sets the data for a new page. This will be called multiple times on the same
|
| 252 |
+
// decoder and should reset all internal state.
|
| 253 |
+
//
|
| 254 |
+
// `num_values` comes from the data page header, and may be greater than the number of
|
| 255 |
+
// physical values in the data buffer if there are some omitted (null) values.
|
| 256 |
+
// `len`, on the other hand, is the size in bytes of the data buffer and
|
| 257 |
+
// directly relates to the number of physical values.
|
| 258 |
+
virtual void SetData(int num_values, const uint8_t* data, int len) = 0;
|
| 259 |
+
|
| 260 |
+
// Returns the number of values left (for the last call to SetData()). This is
|
| 261 |
+
// the number of values left in this page.
|
| 262 |
+
virtual int values_left() const = 0;
|
| 263 |
+
virtual Encoding::type encoding() const = 0;
|
| 264 |
+
};
|
| 265 |
+
|
| 266 |
+
template <typename DType>
|
| 267 |
+
class TypedDecoder : virtual public Decoder {
|
| 268 |
+
public:
|
| 269 |
+
using T = typename DType::c_type;
|
| 270 |
+
|
| 271 |
+
/// \brief Decode values into a buffer
|
| 272 |
+
///
|
| 273 |
+
/// Subclasses may override the more specialized Decode methods below.
|
| 274 |
+
///
|
| 275 |
+
/// \param[in] buffer destination for decoded values
|
| 276 |
+
/// \param[in] max_values maximum number of values to decode
|
| 277 |
+
/// \return The number of values decoded. Should be identical to max_values except
|
| 278 |
+
/// at the end of the current data page.
|
| 279 |
+
virtual int Decode(T* buffer, int max_values) = 0;
|
| 280 |
+
|
| 281 |
+
/// \brief Decode the values in this data page but leave spaces for null entries.
|
| 282 |
+
///
|
| 283 |
+
/// \param[in] buffer destination for decoded values
|
| 284 |
+
/// \param[in] num_values size of the def_levels and buffer arrays including the number
|
| 285 |
+
/// of null slots
|
| 286 |
+
/// \param[in] null_count number of null slots
|
| 287 |
+
/// \param[in] valid_bits bitmap data indicating position of valid slots
|
| 288 |
+
/// \param[in] valid_bits_offset offset into valid_bits
|
| 289 |
+
/// \return The number of values decoded, including nulls.
|
| 290 |
+
virtual int DecodeSpaced(T* buffer, int num_values, int null_count,
|
| 291 |
+
const uint8_t* valid_bits, int64_t valid_bits_offset) = 0;
|
| 292 |
+
|
| 293 |
+
/// \brief Decode into an ArrayBuilder or other accumulator
|
| 294 |
+
///
|
| 295 |
+
/// \param[in] num_values number of values to decode, including null slots
|
| 296 |
+
/// \param[in] null_count number of null slots
|
| 297 |
+
/// \param[in] valid_bits validity bitmap
|
| 298 |
+
/// \param[in] valid_bits_offset bit offset to start into the validity bitmap
|
| 299 |
+
/// \param[in] out accumulator to decode into
|
| 300 |
+
///
|
| 301 |
+
/// This function assumes the definition levels were already decoded
|
| 302 |
+
/// as a validity bitmap in the given `valid_bits`. `null_count`
|
| 303 |
+
/// is the number of 0s in `valid_bits`.
|
| 304 |
+
/// `valid_bits` must at least `valid_bits_offset + num_values` bits.
|
| 305 |
+
/// As a space optimization, it is allowed for `valid_bits` to be null
|
| 306 |
+
/// if `null_count` is zero.
|
| 307 |
+
/// This function throws a ParquetException if there are less than
|
| 308 |
+
/// `num_values - null_count` values left to decode.
|
| 309 |
+
///
|
| 310 |
+
/// \return The number of non-null values decoded
|
| 311 |
+
virtual int DecodeArrow(int num_values, int null_count, const uint8_t* valid_bits,
|
| 312 |
+
int64_t valid_bits_offset,
|
| 313 |
+
typename EncodingTraits<DType>::Accumulator* out) = 0;
|
| 314 |
+
|
| 315 |
+
/// \brief Decode into an ArrayBuilder or other accumulator ignoring nulls
|
| 316 |
+
///
|
| 317 |
+
/// \param[in] num_values number of values to decode
|
| 318 |
+
/// \param[in] out accumulator to decode into
|
| 319 |
+
///
|
| 320 |
+
/// \return The number of values decoded
|
| 321 |
+
int DecodeArrowNonNull(int num_values,
|
| 322 |
+
typename EncodingTraits<DType>::Accumulator* out) {
|
| 323 |
+
return DecodeArrow(num_values, 0, /*valid_bits=*/NULLPTR, 0, out);
|
| 324 |
+
}
|
| 325 |
+
|
| 326 |
+
/// \brief Decode into a DictionaryBuilder
|
| 327 |
+
///
|
| 328 |
+
/// This function assumes the definition levels were already decoded
|
| 329 |
+
/// as a validity bitmap in the given `valid_bits`. `null_count`
|
| 330 |
+
/// is the number of 0s in `valid_bits`.
|
| 331 |
+
/// As a space optimization, it is allowed for `valid_bits` to be null
|
| 332 |
+
/// if `null_count` is zero.
|
| 333 |
+
///
|
| 334 |
+
/// \return number of values decoded
|
| 335 |
+
virtual int DecodeArrow(int num_values, int null_count, const uint8_t* valid_bits,
|
| 336 |
+
int64_t valid_bits_offset,
|
| 337 |
+
typename EncodingTraits<DType>::DictAccumulator* builder) = 0;
|
| 338 |
+
|
| 339 |
+
/// \brief Decode into a DictionaryBuilder ignoring nulls
|
| 340 |
+
///
|
| 341 |
+
/// \return number of values decoded
|
| 342 |
+
int DecodeArrowNonNull(int num_values,
|
| 343 |
+
typename EncodingTraits<DType>::DictAccumulator* builder) {
|
| 344 |
+
return DecodeArrow(num_values, 0, /*valid_bits=*/NULLPTR, 0, builder);
|
| 345 |
+
}
|
| 346 |
+
};
|
| 347 |
+
|
| 348 |
+
template <typename DType>
|
| 349 |
+
class DictDecoder : virtual public TypedDecoder<DType> {
|
| 350 |
+
public:
|
| 351 |
+
using T = typename DType::c_type;
|
| 352 |
+
|
| 353 |
+
virtual void SetDict(TypedDecoder<DType>* dictionary) = 0;
|
| 354 |
+
|
| 355 |
+
/// \brief Insert dictionary values into the Arrow dictionary builder's memo,
|
| 356 |
+
/// but do not append any indices
|
| 357 |
+
virtual void InsertDictionary(::arrow::ArrayBuilder* builder) = 0;
|
| 358 |
+
|
| 359 |
+
/// \brief Decode only dictionary indices and append to dictionary
|
| 360 |
+
/// builder. The builder must have had the dictionary from this decoder
|
| 361 |
+
/// inserted already.
|
| 362 |
+
///
|
| 363 |
+
/// \warning Remember to reset the builder each time the dict decoder is initialized
|
| 364 |
+
/// with a new dictionary page
|
| 365 |
+
virtual int DecodeIndicesSpaced(int num_values, int null_count,
|
| 366 |
+
const uint8_t* valid_bits, int64_t valid_bits_offset,
|
| 367 |
+
::arrow::ArrayBuilder* builder) = 0;
|
| 368 |
+
|
| 369 |
+
/// \brief Decode only dictionary indices (no nulls)
|
| 370 |
+
///
|
| 371 |
+
/// \warning Remember to reset the builder each time the dict decoder is initialized
|
| 372 |
+
/// with a new dictionary page
|
| 373 |
+
virtual int DecodeIndices(int num_values, ::arrow::ArrayBuilder* builder) = 0;
|
| 374 |
+
|
| 375 |
+
/// \brief Decode only dictionary indices (no nulls). Same as above
|
| 376 |
+
/// DecodeIndices but target is an array instead of a builder.
|
| 377 |
+
///
|
| 378 |
+
/// \note API EXPERIMENTAL
|
| 379 |
+
virtual int DecodeIndices(int num_values, int32_t* indices) = 0;
|
| 380 |
+
|
| 381 |
+
/// \brief Get dictionary. The reader will call this API when it encounters a
|
| 382 |
+
/// new dictionary.
|
| 383 |
+
///
|
| 384 |
+
/// @param[out] dictionary The pointer to dictionary values. Dictionary is owned by
|
| 385 |
+
/// the decoder and is destroyed when the decoder is destroyed.
|
| 386 |
+
/// @param[out] dictionary_length The dictionary length.
|
| 387 |
+
///
|
| 388 |
+
/// \note API EXPERIMENTAL
|
| 389 |
+
virtual void GetDictionary(const T** dictionary, int32_t* dictionary_length) = 0;
|
| 390 |
+
};
|
| 391 |
+
|
| 392 |
+
// ----------------------------------------------------------------------
|
| 393 |
+
// TypedEncoder specializations, traits, and factory functions
|
| 394 |
+
|
| 395 |
+
class BooleanDecoder : virtual public TypedDecoder<BooleanType> {
|
| 396 |
+
public:
|
| 397 |
+
using TypedDecoder<BooleanType>::Decode;
|
| 398 |
+
|
| 399 |
+
/// \brief Decode and bit-pack values into a buffer
|
| 400 |
+
///
|
| 401 |
+
/// \param[in] buffer destination for decoded values
|
| 402 |
+
/// This buffer will contain bit-packed values. If
|
| 403 |
+
/// max_values is not a multiple of 8, the trailing bits
|
| 404 |
+
/// of the last byte will be undefined.
|
| 405 |
+
/// \param[in] max_values max values to decode.
|
| 406 |
+
/// \return The number of values decoded. Should be identical to max_values except
|
| 407 |
+
/// at the end of the current data page.
|
| 408 |
+
virtual int Decode(uint8_t* buffer, int max_values) = 0;
|
| 409 |
+
};
|
| 410 |
+
|
| 411 |
+
class FLBADecoder : virtual public TypedDecoder<FLBAType> {
|
| 412 |
+
public:
|
| 413 |
+
using TypedDecoder<FLBAType>::DecodeSpaced;
|
| 414 |
+
|
| 415 |
+
// TODO(wesm): As possible follow-up to PARQUET-1508, we should examine if
|
| 416 |
+
// there is value in adding specialized read methods for
|
| 417 |
+
// FIXED_LEN_BYTE_ARRAY. If only Decimal data can occur with this data type
|
| 418 |
+
// then perhaps not
|
| 419 |
+
};
|
| 420 |
+
|
| 421 |
+
PARQUET_EXPORT
|
| 422 |
+
std::unique_ptr<Encoder> MakeEncoder(
|
| 423 |
+
Type::type type_num, Encoding::type encoding, bool use_dictionary = false,
|
| 424 |
+
const ColumnDescriptor* descr = NULLPTR,
|
| 425 |
+
::arrow::MemoryPool* pool = ::arrow::default_memory_pool());
|
| 426 |
+
|
| 427 |
+
template <typename DType>
|
| 428 |
+
std::unique_ptr<typename EncodingTraits<DType>::Encoder> MakeTypedEncoder(
|
| 429 |
+
Encoding::type encoding, bool use_dictionary = false,
|
| 430 |
+
const ColumnDescriptor* descr = NULLPTR,
|
| 431 |
+
::arrow::MemoryPool* pool = ::arrow::default_memory_pool()) {
|
| 432 |
+
using OutType = typename EncodingTraits<DType>::Encoder;
|
| 433 |
+
std::unique_ptr<Encoder> base =
|
| 434 |
+
MakeEncoder(DType::type_num, encoding, use_dictionary, descr, pool);
|
| 435 |
+
return std::unique_ptr<OutType>(dynamic_cast<OutType*>(base.release()));
|
| 436 |
+
}
|
| 437 |
+
|
| 438 |
+
PARQUET_EXPORT
|
| 439 |
+
std::unique_ptr<Decoder> MakeDecoder(
|
| 440 |
+
Type::type type_num, Encoding::type encoding, const ColumnDescriptor* descr = NULLPTR,
|
| 441 |
+
::arrow::MemoryPool* pool = ::arrow::default_memory_pool());
|
| 442 |
+
|
| 443 |
+
namespace detail {
|
| 444 |
+
|
| 445 |
+
PARQUET_EXPORT
|
| 446 |
+
std::unique_ptr<Decoder> MakeDictDecoder(Type::type type_num,
|
| 447 |
+
const ColumnDescriptor* descr,
|
| 448 |
+
::arrow::MemoryPool* pool);
|
| 449 |
+
|
| 450 |
+
} // namespace detail
|
| 451 |
+
|
| 452 |
+
template <typename DType>
|
| 453 |
+
std::unique_ptr<DictDecoder<DType>> MakeDictDecoder(
|
| 454 |
+
const ColumnDescriptor* descr = NULLPTR,
|
| 455 |
+
::arrow::MemoryPool* pool = ::arrow::default_memory_pool()) {
|
| 456 |
+
using OutType = DictDecoder<DType>;
|
| 457 |
+
auto decoder = detail::MakeDictDecoder(DType::type_num, descr, pool);
|
| 458 |
+
return std::unique_ptr<OutType>(dynamic_cast<OutType*>(decoder.release()));
|
| 459 |
+
}
|
| 460 |
+
|
| 461 |
+
template <typename DType>
|
| 462 |
+
std::unique_ptr<typename EncodingTraits<DType>::Decoder> MakeTypedDecoder(
|
| 463 |
+
Encoding::type encoding, const ColumnDescriptor* descr = NULLPTR,
|
| 464 |
+
::arrow::MemoryPool* pool = ::arrow::default_memory_pool()) {
|
| 465 |
+
using OutType = typename EncodingTraits<DType>::Decoder;
|
| 466 |
+
std::unique_ptr<Decoder> base = MakeDecoder(DType::type_num, encoding, descr, pool);
|
| 467 |
+
return std::unique_ptr<OutType>(dynamic_cast<OutType*>(base.release()));
|
| 468 |
+
}
|
| 469 |
+
|
| 470 |
+
/// Return the list of supported encodings for the given physical type
|
| 471 |
+
///
|
| 472 |
+
/// Only non-dictionary encodings are returned.
|
| 473 |
+
PARQUET_EXPORT
|
| 474 |
+
std::vector<Encoding::type> SupportedEncodings(Type::type physical_type);
|
| 475 |
+
|
| 476 |
+
} // namespace parquet
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/crypto_factory.h
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <memory>
|
| 21 |
+
|
| 22 |
+
#include "parquet/encryption/encryption.h"
|
| 23 |
+
#include "parquet/encryption/file_key_wrapper.h"
|
| 24 |
+
#include "parquet/encryption/key_toolkit.h"
|
| 25 |
+
#include "parquet/encryption/kms_client_factory.h"
|
| 26 |
+
#include "parquet/platform.h"
|
| 27 |
+
|
| 28 |
+
namespace parquet::encryption {
|
| 29 |
+
|
| 30 |
+
static constexpr ParquetCipher::type kDefaultEncryptionAlgorithm =
|
| 31 |
+
ParquetCipher::AES_GCM_V1;
|
| 32 |
+
static constexpr bool kDefaultPlaintextFooter = false;
|
| 33 |
+
static constexpr bool kDefaultDoubleWrapping = true;
|
| 34 |
+
static constexpr double kDefaultCacheLifetimeSeconds = 600; // 10 minutes
|
| 35 |
+
static constexpr bool kDefaultInternalKeyMaterial = true;
|
| 36 |
+
static constexpr bool kDefaultUniformEncryption = false;
|
| 37 |
+
static constexpr int32_t kDefaultDataKeyLengthBits = 128;
|
| 38 |
+
|
| 39 |
+
struct PARQUET_EXPORT EncryptionConfiguration {
|
| 40 |
+
explicit EncryptionConfiguration(const std::string& footer_key)
|
| 41 |
+
: footer_key(footer_key) {}
|
| 42 |
+
|
| 43 |
+
/// ID of the master key for footer encryption/signing
|
| 44 |
+
std::string footer_key;
|
| 45 |
+
|
| 46 |
+
/// List of columns to encrypt, with column master key IDs (see HIVE-21848).
|
| 47 |
+
/// Format: "columnKeyID:colName,colName;columnKeyID:colName..."
|
| 48 |
+
/// Either
|
| 49 |
+
/// (1) column_keys must be set
|
| 50 |
+
/// or
|
| 51 |
+
/// (2) uniform_encryption must be set to true
|
| 52 |
+
/// If none of (1) and (2) are true, or if both are true, an exception will be
|
| 53 |
+
/// thrown.
|
| 54 |
+
std::string column_keys;
|
| 55 |
+
|
| 56 |
+
/// Encrypt footer and all columns with the same encryption key.
|
| 57 |
+
bool uniform_encryption = kDefaultUniformEncryption;
|
| 58 |
+
|
| 59 |
+
/// Parquet encryption algorithm. Can be "AES_GCM_V1" (default), or "AES_GCM_CTR_V1".
|
| 60 |
+
ParquetCipher::type encryption_algorithm = kDefaultEncryptionAlgorithm;
|
| 61 |
+
|
| 62 |
+
/// Write files with plaintext footer.
|
| 63 |
+
/// The default is false - files are written with encrypted footer.
|
| 64 |
+
bool plaintext_footer = kDefaultPlaintextFooter;
|
| 65 |
+
|
| 66 |
+
/// Use double wrapping - where data encryption keys (DEKs) are encrypted with key
|
| 67 |
+
/// encryption keys (KEKs), which in turn are encrypted with master keys.
|
| 68 |
+
/// The default is true. If set to false, use single wrapping - where DEKs are
|
| 69 |
+
/// encrypted directly with master keys.
|
| 70 |
+
bool double_wrapping = kDefaultDoubleWrapping;
|
| 71 |
+
|
| 72 |
+
/// Lifetime of cached entities (key encryption keys, local wrapping keys, KMS client
|
| 73 |
+
/// objects).
|
| 74 |
+
/// The default is 600 (10 minutes).
|
| 75 |
+
double cache_lifetime_seconds = kDefaultCacheLifetimeSeconds;
|
| 76 |
+
|
| 77 |
+
/// Store key material inside Parquet file footers; this mode doesn’t produce
|
| 78 |
+
/// additional files. By default, true. If set to false, key material is stored in
|
| 79 |
+
/// separate files in the same folder, which enables key rotation for immutable
|
| 80 |
+
/// Parquet files.
|
| 81 |
+
bool internal_key_material = kDefaultInternalKeyMaterial;
|
| 82 |
+
|
| 83 |
+
/// Length of data encryption keys (DEKs), randomly generated by parquet key
|
| 84 |
+
/// management tools. Can be 128, 192 or 256 bits.
|
| 85 |
+
/// The default is 128 bits.
|
| 86 |
+
int32_t data_key_length_bits = kDefaultDataKeyLengthBits;
|
| 87 |
+
};
|
| 88 |
+
|
| 89 |
+
struct PARQUET_EXPORT DecryptionConfiguration {
|
| 90 |
+
/// Lifetime of cached entities (key encryption keys, local wrapping keys, KMS client
|
| 91 |
+
/// objects).
|
| 92 |
+
/// The default is 600 (10 minutes).
|
| 93 |
+
double cache_lifetime_seconds = kDefaultCacheLifetimeSeconds;
|
| 94 |
+
};
|
| 95 |
+
|
| 96 |
+
/// This is a core class, that translates the parameters of high level encryption (like
|
| 97 |
+
/// the names of encrypted columns, names of master keys, etc), into parameters of low
|
| 98 |
+
/// level encryption (like the key metadata, DEK, etc). A factory that produces the low
|
| 99 |
+
/// level FileEncryptionProperties and FileDecryptionProperties objects, from the high
|
| 100 |
+
/// level parameters.
|
| 101 |
+
class PARQUET_EXPORT CryptoFactory {
|
| 102 |
+
public:
|
| 103 |
+
/// a KmsClientFactory object must be registered via this method before calling any of
|
| 104 |
+
/// GetFileEncryptionProperties()/GetFileDecryptionProperties() methods.
|
| 105 |
+
void RegisterKmsClientFactory(std::shared_ptr<KmsClientFactory> kms_client_factory);
|
| 106 |
+
|
| 107 |
+
/// Get the encryption properties for a Parquet file.
|
| 108 |
+
/// If external key material is used then a file system and path to the
|
| 109 |
+
/// parquet file must be provided.
|
| 110 |
+
std::shared_ptr<FileEncryptionProperties> GetFileEncryptionProperties(
|
| 111 |
+
const KmsConnectionConfig& kms_connection_config,
|
| 112 |
+
const EncryptionConfiguration& encryption_config, const std::string& file_path = "",
|
| 113 |
+
const std::shared_ptr<::arrow::fs::FileSystem>& file_system = NULLPTR);
|
| 114 |
+
|
| 115 |
+
/// Get decryption properties for a Parquet file.
|
| 116 |
+
/// If external key material is used then a file system and path to the
|
| 117 |
+
/// parquet file must be provided.
|
| 118 |
+
std::shared_ptr<FileDecryptionProperties> GetFileDecryptionProperties(
|
| 119 |
+
const KmsConnectionConfig& kms_connection_config,
|
| 120 |
+
const DecryptionConfiguration& decryption_config, const std::string& file_path = "",
|
| 121 |
+
const std::shared_ptr<::arrow::fs::FileSystem>& file_system = NULLPTR);
|
| 122 |
+
|
| 123 |
+
void RemoveCacheEntriesForToken(const std::string& access_token) {
|
| 124 |
+
key_toolkit_->RemoveCacheEntriesForToken(access_token);
|
| 125 |
+
}
|
| 126 |
+
|
| 127 |
+
void RemoveCacheEntriesForAllTokens() {
|
| 128 |
+
key_toolkit_->RemoveCacheEntriesForAllTokens();
|
| 129 |
+
}
|
| 130 |
+
|
| 131 |
+
/// Rotates master encryption keys for a Parquet file that uses external key material.
|
| 132 |
+
/// In single wrapping mode, data encryption keys are decrypted with the old master keys
|
| 133 |
+
/// and then re-encrypted with new master keys.
|
| 134 |
+
/// In double wrapping mode, key encryption keys are decrypted with the old master keys
|
| 135 |
+
/// and then re-encrypted with new master keys.
|
| 136 |
+
/// This relies on the KMS supporting versioning, such that the old master key is
|
| 137 |
+
/// used when unwrapping a key, and the latest version is used when wrapping a key.
|
| 138 |
+
void RotateMasterKeys(const KmsConnectionConfig& kms_connection_config,
|
| 139 |
+
const std::string& parquet_file_path,
|
| 140 |
+
const std::shared_ptr<::arrow::fs::FileSystem>& file_system,
|
| 141 |
+
bool double_wrapping = kDefaultDoubleWrapping,
|
| 142 |
+
double cache_lifetime_seconds = kDefaultCacheLifetimeSeconds);
|
| 143 |
+
|
| 144 |
+
private:
|
| 145 |
+
ColumnPathToEncryptionPropertiesMap GetColumnEncryptionProperties(
|
| 146 |
+
int dek_length, const std::string& column_keys, FileKeyWrapper* key_wrapper);
|
| 147 |
+
|
| 148 |
+
/// Key utilities object for kms client initialization and cache control
|
| 149 |
+
std::shared_ptr<KeyToolkit> key_toolkit_ = std::make_shared<KeyToolkit>();
|
| 150 |
+
};
|
| 151 |
+
|
| 152 |
+
} // namespace parquet::encryption
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/encryption.h
ADDED
|
@@ -0,0 +1,434 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <cassert>
|
| 21 |
+
#include <map>
|
| 22 |
+
#include <memory>
|
| 23 |
+
#include <span>
|
| 24 |
+
#include <string>
|
| 25 |
+
#include <utility>
|
| 26 |
+
|
| 27 |
+
#include "arrow/util/secure_string.h"
|
| 28 |
+
#include "parquet/exception.h"
|
| 29 |
+
#include "parquet/schema.h"
|
| 30 |
+
#include "parquet/types.h"
|
| 31 |
+
|
| 32 |
+
namespace parquet {
|
| 33 |
+
|
| 34 |
+
static constexpr ParquetCipher::type kDefaultEncryptionAlgorithm =
|
| 35 |
+
ParquetCipher::AES_GCM_V1;
|
| 36 |
+
static constexpr int32_t kMaximalAadMetadataLength = 256;
|
| 37 |
+
static constexpr bool kDefaultEncryptedFooter = true;
|
| 38 |
+
static constexpr bool kDefaultCheckSignature = true;
|
| 39 |
+
static constexpr bool kDefaultAllowPlaintextFiles = false;
|
| 40 |
+
static constexpr int32_t kAadFileUniqueLength = 8;
|
| 41 |
+
|
| 42 |
+
class ColumnDecryptionProperties;
|
| 43 |
+
using ColumnPathToDecryptionPropertiesMap =
|
| 44 |
+
std::map<std::string, std::shared_ptr<ColumnDecryptionProperties>>;
|
| 45 |
+
|
| 46 |
+
class ColumnEncryptionProperties;
|
| 47 |
+
using ColumnPathToEncryptionPropertiesMap =
|
| 48 |
+
std::map<std::string, std::shared_ptr<ColumnEncryptionProperties>>;
|
| 49 |
+
|
| 50 |
+
class PARQUET_EXPORT DecryptionKeyRetriever {
|
| 51 |
+
public:
|
| 52 |
+
/// \brief Retrieve a key.
|
| 53 |
+
virtual ::arrow::util::SecureString GetKey(const std::string& key_id) = 0;
|
| 54 |
+
|
| 55 |
+
virtual ~DecryptionKeyRetriever() {}
|
| 56 |
+
};
|
| 57 |
+
|
| 58 |
+
/// Simple integer key retriever
|
| 59 |
+
class PARQUET_EXPORT IntegerKeyIdRetriever : public DecryptionKeyRetriever {
|
| 60 |
+
public:
|
| 61 |
+
void PutKey(uint32_t key_id, ::arrow::util::SecureString key);
|
| 62 |
+
|
| 63 |
+
::arrow::util::SecureString GetKey(const std::string& key_id_string) override {
|
| 64 |
+
// key_id_string is string but for IntegerKeyIdRetriever it encodes
|
| 65 |
+
// a native-endian 32 bit unsigned integer key_id
|
| 66 |
+
uint32_t key_id;
|
| 67 |
+
assert(key_id_string.size() == sizeof(key_id));
|
| 68 |
+
memcpy(&key_id, key_id_string.data(), sizeof(key_id));
|
| 69 |
+
|
| 70 |
+
return key_map_.at(key_id);
|
| 71 |
+
}
|
| 72 |
+
|
| 73 |
+
private:
|
| 74 |
+
std::map<uint32_t, ::arrow::util::SecureString> key_map_;
|
| 75 |
+
};
|
| 76 |
+
|
| 77 |
+
// Simple string key retriever
|
| 78 |
+
class PARQUET_EXPORT StringKeyIdRetriever : public DecryptionKeyRetriever {
|
| 79 |
+
public:
|
| 80 |
+
void PutKey(std::string key_id, ::arrow::util::SecureString key);
|
| 81 |
+
::arrow::util::SecureString GetKey(const std::string& key_id) override;
|
| 82 |
+
|
| 83 |
+
private:
|
| 84 |
+
std::map<std::string, ::arrow::util::SecureString> key_map_;
|
| 85 |
+
};
|
| 86 |
+
|
| 87 |
+
class PARQUET_EXPORT HiddenColumnException : public ParquetException {
|
| 88 |
+
public:
|
| 89 |
+
explicit HiddenColumnException(const std::string& columnPath)
|
| 90 |
+
: ParquetException(columnPath.c_str()) {}
|
| 91 |
+
};
|
| 92 |
+
|
| 93 |
+
class PARQUET_EXPORT KeyAccessDeniedException : public ParquetException {
|
| 94 |
+
public:
|
| 95 |
+
explicit KeyAccessDeniedException(const std::string& columnPath)
|
| 96 |
+
: ParquetException(columnPath.c_str()) {}
|
| 97 |
+
};
|
| 98 |
+
|
| 99 |
+
inline std::span<const uint8_t> str2span(const std::string& str) {
|
| 100 |
+
if (str.empty()) {
|
| 101 |
+
return {};
|
| 102 |
+
}
|
| 103 |
+
|
| 104 |
+
return {reinterpret_cast<const uint8_t*>(str.data()), str.size()};
|
| 105 |
+
}
|
| 106 |
+
|
| 107 |
+
class PARQUET_EXPORT ColumnEncryptionProperties {
|
| 108 |
+
public:
|
| 109 |
+
class PARQUET_EXPORT Builder {
|
| 110 |
+
public:
|
| 111 |
+
PARQUET_DEPRECATED("name argument is ignored, use default constructor instead")
|
| 112 |
+
explicit Builder(const std::string& name) : encrypted_(true) {}
|
| 113 |
+
|
| 114 |
+
PARQUET_DEPRECATED("path argument is ignored, use default constructor instead")
|
| 115 |
+
explicit Builder(const schema::ColumnPath& path) : encrypted_(true) {}
|
| 116 |
+
|
| 117 |
+
Builder() = default;
|
| 118 |
+
|
| 119 |
+
/// Set a column-specific key.
|
| 120 |
+
/// If key is not set on an encrypted column, the column will
|
| 121 |
+
/// be encrypted with the footer key.
|
| 122 |
+
/// keyBytes Key length must be either 16, 24 or 32 bytes.
|
| 123 |
+
/// Caller is responsible for wiping out the input key array.
|
| 124 |
+
Builder* key(::arrow::util::SecureString column_key);
|
| 125 |
+
|
| 126 |
+
/// Set a key retrieval metadata.
|
| 127 |
+
/// use either key_metadata() or key_id(), not both
|
| 128 |
+
Builder* key_metadata(std::string key_metadata);
|
| 129 |
+
|
| 130 |
+
/// A convenience function to set key metadata using a string id.
|
| 131 |
+
/// Set a key retrieval metadata (converted from String).
|
| 132 |
+
/// use either key_metadata() or key_id(), not both
|
| 133 |
+
/// key_id will be converted to metadata (UTF-8 array).
|
| 134 |
+
Builder* key_id(std::string key_id);
|
| 135 |
+
|
| 136 |
+
std::shared_ptr<ColumnEncryptionProperties> build() {
|
| 137 |
+
return std::shared_ptr<ColumnEncryptionProperties>(
|
| 138 |
+
new ColumnEncryptionProperties(encrypted_, key_, key_metadata_));
|
| 139 |
+
}
|
| 140 |
+
|
| 141 |
+
private:
|
| 142 |
+
bool encrypted_ = true;
|
| 143 |
+
::arrow::util::SecureString key_;
|
| 144 |
+
std::string key_metadata_;
|
| 145 |
+
};
|
| 146 |
+
|
| 147 |
+
bool is_encrypted() const { return encrypted_; }
|
| 148 |
+
bool is_encrypted_with_footer_key() const { return encrypted_with_footer_key_; }
|
| 149 |
+
const ::arrow::util::SecureString& key() const { return key_; }
|
| 150 |
+
const std::string& key_metadata() const { return key_metadata_; }
|
| 151 |
+
|
| 152 |
+
static std::shared_ptr<ColumnEncryptionProperties> Unencrypted();
|
| 153 |
+
static std::shared_ptr<ColumnEncryptionProperties> WithFooterKey();
|
| 154 |
+
static std::shared_ptr<ColumnEncryptionProperties> WithColumnKey(
|
| 155 |
+
::arrow::util::SecureString key, std::string key_metadata = "");
|
| 156 |
+
|
| 157 |
+
private:
|
| 158 |
+
bool encrypted_;
|
| 159 |
+
bool encrypted_with_footer_key_;
|
| 160 |
+
::arrow::util::SecureString key_;
|
| 161 |
+
std::string key_metadata_;
|
| 162 |
+
explicit ColumnEncryptionProperties(bool encrypted, ::arrow::util::SecureString key,
|
| 163 |
+
std::string key_metadata);
|
| 164 |
+
};
|
| 165 |
+
|
| 166 |
+
class PARQUET_EXPORT ColumnDecryptionProperties {
|
| 167 |
+
public:
|
| 168 |
+
class PARQUET_EXPORT Builder {
|
| 169 |
+
public:
|
| 170 |
+
explicit Builder(std::string name) : column_path_(std::move(name)) {}
|
| 171 |
+
|
| 172 |
+
explicit Builder(const schema::ColumnPath& path) : Builder(path.ToDotString()) {}
|
| 173 |
+
|
| 174 |
+
/// Set an explicit column key. If applied on a file that contains
|
| 175 |
+
/// key metadata for this column the metadata will be ignored,
|
| 176 |
+
/// the column will be decrypted with this key.
|
| 177 |
+
/// key length must be either 16, 24 or 32 bytes.
|
| 178 |
+
Builder* key(::arrow::util::SecureString key);
|
| 179 |
+
|
| 180 |
+
std::shared_ptr<ColumnDecryptionProperties> build();
|
| 181 |
+
|
| 182 |
+
private:
|
| 183 |
+
std::string column_path_;
|
| 184 |
+
::arrow::util::SecureString key_;
|
| 185 |
+
};
|
| 186 |
+
|
| 187 |
+
const std::string& column_path() const { return column_path_; }
|
| 188 |
+
const ::arrow::util::SecureString& key() const { return key_; }
|
| 189 |
+
|
| 190 |
+
private:
|
| 191 |
+
std::string column_path_;
|
| 192 |
+
::arrow::util::SecureString key_;
|
| 193 |
+
|
| 194 |
+
/// This class is only required for setting explicit column decryption keys -
|
| 195 |
+
/// to override key retriever (or to provide keys when key metadata and/or
|
| 196 |
+
/// key retriever are not available)
|
| 197 |
+
explicit ColumnDecryptionProperties(std::string column_path,
|
| 198 |
+
::arrow::util::SecureString key);
|
| 199 |
+
};
|
| 200 |
+
|
| 201 |
+
class PARQUET_EXPORT AADPrefixVerifier {
|
| 202 |
+
public:
|
| 203 |
+
/// Verifies identity (AAD Prefix) of individual file,
|
| 204 |
+
/// or of file collection in a data set.
|
| 205 |
+
/// Throws exception if an AAD prefix is wrong.
|
| 206 |
+
/// In a data set, AAD Prefixes should be collected,
|
| 207 |
+
/// and then checked for missing files.
|
| 208 |
+
virtual void Verify(const std::string& aad_prefix) = 0;
|
| 209 |
+
virtual ~AADPrefixVerifier() {}
|
| 210 |
+
};
|
| 211 |
+
|
| 212 |
+
class PARQUET_EXPORT FileDecryptionProperties {
|
| 213 |
+
public:
|
| 214 |
+
class PARQUET_EXPORT Builder {
|
| 215 |
+
public:
|
| 216 |
+
Builder() {
|
| 217 |
+
check_plaintext_footer_integrity_ = kDefaultCheckSignature;
|
| 218 |
+
plaintext_files_allowed_ = kDefaultAllowPlaintextFiles;
|
| 219 |
+
}
|
| 220 |
+
|
| 221 |
+
/// Set an explicit footer key. If applied on a file that contains
|
| 222 |
+
/// footer key metadata the metadata will be ignored, the footer
|
| 223 |
+
/// will be decrypted/verified with this key.
|
| 224 |
+
/// If explicit key is not set, footer key will be fetched from
|
| 225 |
+
/// key retriever.
|
| 226 |
+
/// With explicit keys or AAD prefix, new encryption properties object must be
|
| 227 |
+
/// created for each encrypted file.
|
| 228 |
+
/// Explicit encryption keys (footer and column) are cloned.
|
| 229 |
+
/// Upon completion of file reading, the cloned encryption keys in the properties
|
| 230 |
+
/// will be wiped out (array values set to 0).
|
| 231 |
+
/// Caller is responsible for wiping out the input key array.
|
| 232 |
+
/// param footerKey Key length must be either 16, 24 or 32 bytes.
|
| 233 |
+
Builder* footer_key(::arrow::util::SecureString footer_key);
|
| 234 |
+
|
| 235 |
+
/// Set explicit column keys (decryption properties).
|
| 236 |
+
/// Its also possible to set a key retriever on this property object.
|
| 237 |
+
/// Upon file decryption, availability of explicit keys is checked before
|
| 238 |
+
/// invocation of the retriever callback.
|
| 239 |
+
/// If an explicit key is available for a footer or a column,
|
| 240 |
+
/// its key metadata will be ignored.
|
| 241 |
+
Builder* column_keys(
|
| 242 |
+
ColumnPathToDecryptionPropertiesMap column_decryption_properties);
|
| 243 |
+
|
| 244 |
+
/// Set a key retriever callback. Its also possible to
|
| 245 |
+
/// set explicit footer or column keys on this file property object.
|
| 246 |
+
/// Upon file decryption, availability of explicit keys is checked before
|
| 247 |
+
/// invocation of the retriever callback.
|
| 248 |
+
/// If an explicit key is available for a footer or a column,
|
| 249 |
+
/// its key metadata will be ignored.
|
| 250 |
+
Builder* key_retriever(std::shared_ptr<DecryptionKeyRetriever> key_retriever);
|
| 251 |
+
|
| 252 |
+
/// Skip integrity verification of plaintext footers.
|
| 253 |
+
/// If not called, integrity of plaintext footers will be checked in runtime,
|
| 254 |
+
/// and an exception will be thrown in the following situations:
|
| 255 |
+
/// - footer signing key is not available
|
| 256 |
+
/// (not passed, or not found by key retriever)
|
| 257 |
+
/// - footer content and signature don't match
|
| 258 |
+
Builder* disable_footer_signature_verification() {
|
| 259 |
+
check_plaintext_footer_integrity_ = false;
|
| 260 |
+
return this;
|
| 261 |
+
}
|
| 262 |
+
|
| 263 |
+
/// Explicitly supply the file AAD prefix.
|
| 264 |
+
/// A must when a prefix is used for file encryption, but not stored in file.
|
| 265 |
+
/// If AAD prefix is stored in file, it will be compared to the explicitly
|
| 266 |
+
/// supplied value and an exception will be thrown if they differ.
|
| 267 |
+
Builder* aad_prefix(std::string aad_prefix);
|
| 268 |
+
|
| 269 |
+
/// Set callback for verification of AAD Prefixes stored in file.
|
| 270 |
+
Builder* aad_prefix_verifier(std::shared_ptr<AADPrefixVerifier> aad_prefix_verifier);
|
| 271 |
+
|
| 272 |
+
/// By default, reading plaintext (unencrypted) files is not
|
| 273 |
+
/// allowed when using a decryptor
|
| 274 |
+
/// - in order to detect files that were not encrypted by mistake.
|
| 275 |
+
/// However, the default behavior can be overridden by calling this method.
|
| 276 |
+
/// The caller should use then a different method to ensure encryption
|
| 277 |
+
/// of files with sensitive data.
|
| 278 |
+
Builder* plaintext_files_allowed() {
|
| 279 |
+
plaintext_files_allowed_ = true;
|
| 280 |
+
return this;
|
| 281 |
+
}
|
| 282 |
+
|
| 283 |
+
std::shared_ptr<FileDecryptionProperties> build() {
|
| 284 |
+
return std::shared_ptr<FileDecryptionProperties>(new FileDecryptionProperties(
|
| 285 |
+
footer_key_, key_retriever_, check_plaintext_footer_integrity_, aad_prefix_,
|
| 286 |
+
aad_prefix_verifier_, column_decryption_properties_, plaintext_files_allowed_));
|
| 287 |
+
}
|
| 288 |
+
|
| 289 |
+
private:
|
| 290 |
+
::arrow::util::SecureString footer_key_;
|
| 291 |
+
std::string aad_prefix_;
|
| 292 |
+
std::shared_ptr<AADPrefixVerifier> aad_prefix_verifier_;
|
| 293 |
+
ColumnPathToDecryptionPropertiesMap column_decryption_properties_;
|
| 294 |
+
|
| 295 |
+
std::shared_ptr<DecryptionKeyRetriever> key_retriever_;
|
| 296 |
+
bool check_plaintext_footer_integrity_;
|
| 297 |
+
bool plaintext_files_allowed_;
|
| 298 |
+
};
|
| 299 |
+
|
| 300 |
+
const ::arrow::util::SecureString& column_key(const std::string& column_path) const;
|
| 301 |
+
|
| 302 |
+
const ::arrow::util::SecureString& footer_key() const { return footer_key_; }
|
| 303 |
+
|
| 304 |
+
const std::string& aad_prefix() const { return aad_prefix_; }
|
| 305 |
+
|
| 306 |
+
const std::shared_ptr<DecryptionKeyRetriever>& key_retriever() const {
|
| 307 |
+
return key_retriever_;
|
| 308 |
+
}
|
| 309 |
+
|
| 310 |
+
bool check_plaintext_footer_integrity() const {
|
| 311 |
+
return check_plaintext_footer_integrity_;
|
| 312 |
+
}
|
| 313 |
+
|
| 314 |
+
bool plaintext_files_allowed() const { return plaintext_files_allowed_; }
|
| 315 |
+
|
| 316 |
+
const std::shared_ptr<AADPrefixVerifier>& aad_prefix_verifier() const {
|
| 317 |
+
return aad_prefix_verifier_;
|
| 318 |
+
}
|
| 319 |
+
|
| 320 |
+
private:
|
| 321 |
+
::arrow::util::SecureString footer_key_;
|
| 322 |
+
std::string aad_prefix_;
|
| 323 |
+
std::shared_ptr<AADPrefixVerifier> aad_prefix_verifier_;
|
| 324 |
+
ColumnPathToDecryptionPropertiesMap column_decryption_properties_;
|
| 325 |
+
std::shared_ptr<DecryptionKeyRetriever> key_retriever_;
|
| 326 |
+
bool check_plaintext_footer_integrity_;
|
| 327 |
+
bool plaintext_files_allowed_;
|
| 328 |
+
|
| 329 |
+
FileDecryptionProperties(
|
| 330 |
+
::arrow::util::SecureString footer_key,
|
| 331 |
+
std::shared_ptr<DecryptionKeyRetriever> key_retriever,
|
| 332 |
+
bool check_plaintext_footer_integrity, std::string aad_prefix,
|
| 333 |
+
std::shared_ptr<AADPrefixVerifier> aad_prefix_verifier,
|
| 334 |
+
ColumnPathToDecryptionPropertiesMap column_decryption_properties,
|
| 335 |
+
bool plaintext_files_allowed);
|
| 336 |
+
};
|
| 337 |
+
|
| 338 |
+
class PARQUET_EXPORT FileEncryptionProperties {
|
| 339 |
+
public:
|
| 340 |
+
class PARQUET_EXPORT Builder {
|
| 341 |
+
public:
|
| 342 |
+
explicit Builder(::arrow::util::SecureString footer_key)
|
| 343 |
+
: parquet_cipher_(kDefaultEncryptionAlgorithm),
|
| 344 |
+
encrypted_footer_(kDefaultEncryptedFooter),
|
| 345 |
+
footer_key_(std::move(footer_key)) {
|
| 346 |
+
store_aad_prefix_in_file_ = false;
|
| 347 |
+
}
|
| 348 |
+
|
| 349 |
+
/// Create files with plaintext footer.
|
| 350 |
+
/// If not called, the files will be created with encrypted footer (default).
|
| 351 |
+
Builder* set_plaintext_footer() {
|
| 352 |
+
encrypted_footer_ = false;
|
| 353 |
+
return this;
|
| 354 |
+
}
|
| 355 |
+
|
| 356 |
+
/// Set encryption algorithm.
|
| 357 |
+
/// If not called, files will be encrypted with AES_GCM_V1 (default).
|
| 358 |
+
Builder* algorithm(ParquetCipher::type parquet_cipher) {
|
| 359 |
+
parquet_cipher_ = parquet_cipher;
|
| 360 |
+
return this;
|
| 361 |
+
}
|
| 362 |
+
|
| 363 |
+
/// Set a key retrieval metadata (converted from String).
|
| 364 |
+
/// use either footer_key_metadata or footer_key_id, not both.
|
| 365 |
+
Builder* footer_key_id(std::string key_id);
|
| 366 |
+
|
| 367 |
+
/// Set a key retrieval metadata.
|
| 368 |
+
/// use either footer_key_metadata or footer_key_id, not both.
|
| 369 |
+
Builder* footer_key_metadata(std::string footer_key_metadata);
|
| 370 |
+
|
| 371 |
+
/// Set the file AAD Prefix.
|
| 372 |
+
Builder* aad_prefix(std::string aad_prefix);
|
| 373 |
+
|
| 374 |
+
/// Skip storing AAD Prefix in file.
|
| 375 |
+
/// If not called, and if AAD Prefix is set, it will be stored.
|
| 376 |
+
Builder* disable_aad_prefix_storage();
|
| 377 |
+
|
| 378 |
+
/// Set the list of encrypted columns and their properties (keys etc).
|
| 379 |
+
/// If not called, all columns will be encrypted with the footer key.
|
| 380 |
+
/// If called, the file columns not in the list will be left unencrypted.
|
| 381 |
+
Builder* encrypted_columns(ColumnPathToEncryptionPropertiesMap encrypted_columns);
|
| 382 |
+
|
| 383 |
+
std::shared_ptr<FileEncryptionProperties> build() {
|
| 384 |
+
return std::shared_ptr<FileEncryptionProperties>(new FileEncryptionProperties(
|
| 385 |
+
parquet_cipher_, footer_key_, footer_key_metadata_, encrypted_footer_,
|
| 386 |
+
aad_prefix_, store_aad_prefix_in_file_, encrypted_columns_));
|
| 387 |
+
}
|
| 388 |
+
|
| 389 |
+
private:
|
| 390 |
+
ParquetCipher::type parquet_cipher_;
|
| 391 |
+
bool encrypted_footer_;
|
| 392 |
+
::arrow::util::SecureString footer_key_;
|
| 393 |
+
std::string footer_key_metadata_;
|
| 394 |
+
|
| 395 |
+
std::string aad_prefix_;
|
| 396 |
+
bool store_aad_prefix_in_file_;
|
| 397 |
+
ColumnPathToEncryptionPropertiesMap encrypted_columns_;
|
| 398 |
+
};
|
| 399 |
+
|
| 400 |
+
bool encrypted_footer() const { return encrypted_footer_; }
|
| 401 |
+
|
| 402 |
+
EncryptionAlgorithm algorithm() const { return algorithm_; }
|
| 403 |
+
|
| 404 |
+
const ::arrow::util::SecureString& footer_key() const { return footer_key_; }
|
| 405 |
+
|
| 406 |
+
const std::string& footer_key_metadata() const { return footer_key_metadata_; }
|
| 407 |
+
|
| 408 |
+
const std::string& file_aad() const { return file_aad_; }
|
| 409 |
+
|
| 410 |
+
std::shared_ptr<ColumnEncryptionProperties> column_encryption_properties(
|
| 411 |
+
const std::string& column_path);
|
| 412 |
+
|
| 413 |
+
const ColumnPathToEncryptionPropertiesMap& encrypted_columns() const {
|
| 414 |
+
return encrypted_columns_;
|
| 415 |
+
}
|
| 416 |
+
|
| 417 |
+
private:
|
| 418 |
+
EncryptionAlgorithm algorithm_;
|
| 419 |
+
::arrow::util::SecureString footer_key_;
|
| 420 |
+
std::string footer_key_metadata_;
|
| 421 |
+
bool encrypted_footer_;
|
| 422 |
+
std::string file_aad_;
|
| 423 |
+
std::string aad_prefix_;
|
| 424 |
+
bool store_aad_prefix_in_file_;
|
| 425 |
+
ColumnPathToEncryptionPropertiesMap encrypted_columns_;
|
| 426 |
+
|
| 427 |
+
FileEncryptionProperties(ParquetCipher::type cipher,
|
| 428 |
+
::arrow::util::SecureString footer_key,
|
| 429 |
+
std::string footer_key_metadata, bool encrypted_footer,
|
| 430 |
+
std::string aad_prefix, bool store_aad_prefix_in_file,
|
| 431 |
+
ColumnPathToEncryptionPropertiesMap encrypted_columns);
|
| 432 |
+
};
|
| 433 |
+
|
| 434 |
+
} // namespace parquet
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_key_material_store.h
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <set>
|
| 21 |
+
#include <string>
|
| 22 |
+
#include <unordered_map>
|
| 23 |
+
|
| 24 |
+
#include "arrow/filesystem/filesystem.h"
|
| 25 |
+
#include "parquet/platform.h"
|
| 26 |
+
|
| 27 |
+
namespace parquet::encryption {
|
| 28 |
+
|
| 29 |
+
/// Stores encryption key material outside the Parquet file, for example in a separate
|
| 30 |
+
/// small file in the same folder. This is important for “key rotation”, when MEKs have to
|
| 31 |
+
/// be changed (if compromised; or periodically, just in case) - without modifying the
|
| 32 |
+
/// Parquet files (often immutable).
|
| 33 |
+
class PARQUET_EXPORT FileKeyMaterialStore {
|
| 34 |
+
public:
|
| 35 |
+
/// Add key material for one encryption key.
|
| 36 |
+
virtual void AddKeyMaterial(std::string key_id_in_file, std::string key_material) = 0;
|
| 37 |
+
|
| 38 |
+
/// Get key material
|
| 39 |
+
virtual std::string GetKeyMaterial(std::string key_id_in_file) = 0;
|
| 40 |
+
|
| 41 |
+
/// After key material was added for all keys in the given Parquet file,
|
| 42 |
+
/// save material in persistent store.
|
| 43 |
+
virtual void SaveMaterial() = 0;
|
| 44 |
+
|
| 45 |
+
/// Remove key material from persistent store. Used in key rotation.
|
| 46 |
+
virtual void RemoveMaterial() = 0;
|
| 47 |
+
|
| 48 |
+
/// Move key material to another store. Used in key rotation.
|
| 49 |
+
virtual void MoveMaterialTo(std::shared_ptr<FileKeyMaterialStore> target_key_store) = 0;
|
| 50 |
+
|
| 51 |
+
/// Returns the Set of all key IDs in this store (for the given Parquet file)
|
| 52 |
+
virtual std::vector<std::string> GetKeyIDSet() = 0;
|
| 53 |
+
|
| 54 |
+
virtual ~FileKeyMaterialStore() {}
|
| 55 |
+
};
|
| 56 |
+
|
| 57 |
+
} // namespace parquet::encryption
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_key_unwrapper.h
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include "arrow/util/concurrent_map.h"
|
| 21 |
+
#include "arrow/util/secure_string.h"
|
| 22 |
+
|
| 23 |
+
#include "parquet/encryption/encryption.h"
|
| 24 |
+
#include "parquet/encryption/file_system_key_material_store.h"
|
| 25 |
+
#include "parquet/encryption/key_material.h"
|
| 26 |
+
#include "parquet/encryption/key_toolkit.h"
|
| 27 |
+
#include "parquet/encryption/key_toolkit_internal.h"
|
| 28 |
+
#include "parquet/encryption/kms_client.h"
|
| 29 |
+
#include "parquet/platform.h"
|
| 30 |
+
|
| 31 |
+
namespace parquet::encryption {
|
| 32 |
+
|
| 33 |
+
// This class will retrieve the key from "key metadata", following these steps:
|
| 34 |
+
// 1. Parse "key metadata" (see structure in KeyMetadata class).
|
| 35 |
+
// 2. Retrieve "key material" which can be stored inside or outside "key metadata".
|
| 36 |
+
// 3. Unwrap the "data encryption key" from "key material". There are 2 modes:
|
| 37 |
+
// 3.1. single wrapping: decrypt the wrapped "data encryption key" directly with "master
|
| 38 |
+
// encryption key" 3.2. double wrapping: 2 steps: 3.2.1. "key encryption key" is decrypted
|
| 39 |
+
// with "master encryption key" 3.2.2. "data encryption key" is decrypted with the above
|
| 40 |
+
// "key encryption key"
|
| 41 |
+
class PARQUET_EXPORT FileKeyUnwrapper : public DecryptionKeyRetriever {
|
| 42 |
+
public:
|
| 43 |
+
/// key_toolkit and kms_connection_config is to get KmsClient from cache or create
|
| 44 |
+
/// KmsClient if it's not in the cache yet. cache_entry_lifetime_seconds is life time of
|
| 45 |
+
/// KmsClient in the cache.
|
| 46 |
+
/// If the file uses external key material then the Parquet file path and file
|
| 47 |
+
/// system must be specified.
|
| 48 |
+
FileKeyUnwrapper(std::shared_ptr<KeyToolkit> key_toolkit,
|
| 49 |
+
const KmsConnectionConfig& kms_connection_config,
|
| 50 |
+
double cache_lifetime_seconds, const std::string& file_path = "",
|
| 51 |
+
const std::shared_ptr<::arrow::fs::FileSystem>& file_system = NULLPTR);
|
| 52 |
+
|
| 53 |
+
/// Constructor overload that takes a raw pointer to the KeyToolkit
|
| 54 |
+
FileKeyUnwrapper(KeyToolkit* key_toolkit,
|
| 55 |
+
const KmsConnectionConfig& kms_connection_config,
|
| 56 |
+
double cache_lifetime_seconds, const std::string& file_path = "",
|
| 57 |
+
const std::shared_ptr<::arrow::fs::FileSystem>& file_system = NULLPTR);
|
| 58 |
+
|
| 59 |
+
/// Constructor overload that takes a raw pointer to the KeyToolkit and
|
| 60 |
+
/// accepts an existing key_material_store rather than using
|
| 61 |
+
/// the file path and file system to create one when needed.
|
| 62 |
+
FileKeyUnwrapper(KeyToolkit* key_toolkit,
|
| 63 |
+
const KmsConnectionConfig& kms_connection_config,
|
| 64 |
+
double cache_lifetime_seconds,
|
| 65 |
+
std::shared_ptr<FileKeyMaterialStore> key_material_store);
|
| 66 |
+
|
| 67 |
+
/// Get the data key from key metadata
|
| 68 |
+
::arrow::util::SecureString GetKey(const std::string& key_metadata_bytes) override;
|
| 69 |
+
|
| 70 |
+
/// Get the data key along with the master key id from key material
|
| 71 |
+
KeyWithMasterId GetDataEncryptionKey(const KeyMaterial& key_material);
|
| 72 |
+
|
| 73 |
+
private:
|
| 74 |
+
FileKeyUnwrapper(std::shared_ptr<KeyToolkit> key_toolkit_owner, KeyToolkit* key_toolkit,
|
| 75 |
+
const KmsConnectionConfig& kms_connection_config,
|
| 76 |
+
double cache_lifetime_seconds,
|
| 77 |
+
std::shared_ptr<FileKeyMaterialStore> key_material_store,
|
| 78 |
+
const std::string& file_path,
|
| 79 |
+
const std::shared_ptr<::arrow::fs::FileSystem>& file_system);
|
| 80 |
+
|
| 81 |
+
std::shared_ptr<KmsClient> GetKmsClientFromConfigOrKeyMaterial(
|
| 82 |
+
const KeyMaterial& key_material);
|
| 83 |
+
|
| 84 |
+
/// A map of Key Encryption Key (KEK) ID -> KEK bytes, for the current token
|
| 85 |
+
std::shared_ptr<::arrow::util::ConcurrentMap<std::string, ::arrow::util::SecureString>>
|
| 86 |
+
kek_per_kek_id_;
|
| 87 |
+
std::shared_ptr<KeyToolkit> key_toolkit_owner_;
|
| 88 |
+
KeyToolkit* key_toolkit_;
|
| 89 |
+
KmsConnectionConfig kms_connection_config_;
|
| 90 |
+
const double cache_entry_lifetime_seconds_;
|
| 91 |
+
std::shared_ptr<FileKeyMaterialStore> key_material_store_;
|
| 92 |
+
const std::string file_path_;
|
| 93 |
+
std::shared_ptr<::arrow::fs::FileSystem> file_system_;
|
| 94 |
+
};
|
| 95 |
+
|
| 96 |
+
} // namespace parquet::encryption
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_key_wrapper.h
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <memory>
|
| 21 |
+
#include <string>
|
| 22 |
+
#include <unordered_map>
|
| 23 |
+
|
| 24 |
+
#include "arrow/util/concurrent_map.h"
|
| 25 |
+
|
| 26 |
+
#include "parquet/encryption/file_key_material_store.h"
|
| 27 |
+
#include "parquet/encryption/key_encryption_key.h"
|
| 28 |
+
#include "parquet/encryption/key_toolkit.h"
|
| 29 |
+
#include "parquet/encryption/kms_client.h"
|
| 30 |
+
#include "parquet/platform.h"
|
| 31 |
+
|
| 32 |
+
namespace parquet::encryption {
|
| 33 |
+
|
| 34 |
+
// This class will generate "key metadata" from "data encryption key" and "master key",
|
| 35 |
+
// following these steps:
|
| 36 |
+
// 1. Wrap "data encryption key". There are 2 modes:
|
| 37 |
+
// 1.1. single wrapping: encrypt "data encryption key" directly with "master encryption
|
| 38 |
+
// key"
|
| 39 |
+
// 1.2. double wrapping: 2 steps:
|
| 40 |
+
// 1.2.1. "key encryption key" is randomized (see KeyEncryptionKey class)
|
| 41 |
+
// 1.2.2. "data encryption key" is encrypted with the above "key encryption key"
|
| 42 |
+
// 2. Create "key material" (see structure in KeyMaterial class)
|
| 43 |
+
// 3. Create "key metadata" with "key material" inside or a reference to outside "key
|
| 44 |
+
// material" (see structure in KeyMetadata class).
|
| 45 |
+
class PARQUET_EXPORT FileKeyWrapper {
|
| 46 |
+
public:
|
| 47 |
+
static constexpr int kKeyEncryptionKeyLength = 16;
|
| 48 |
+
static constexpr int kKeyEncryptionKeyIdLength = 16;
|
| 49 |
+
|
| 50 |
+
/// key_toolkit and kms_connection_config is to get KmsClient from the cache or create
|
| 51 |
+
/// KmsClient if it's not in the cache yet. cache_entry_lifetime_seconds is life time of
|
| 52 |
+
/// KmsClient in the cache. key_material_store is to store "key material" outside
|
| 53 |
+
/// parquet file, NULL if "key material" is stored inside parquet file.
|
| 54 |
+
FileKeyWrapper(KeyToolkit* key_toolkit,
|
| 55 |
+
const KmsConnectionConfig& kms_connection_config,
|
| 56 |
+
std::shared_ptr<FileKeyMaterialStore> key_material_store,
|
| 57 |
+
double cache_entry_lifetime_seconds, bool double_wrapping);
|
| 58 |
+
|
| 59 |
+
/// Creates key_metadata field for a given data key, via wrapping the key with the
|
| 60 |
+
/// master key.
|
| 61 |
+
/// When external key material is used, an identifier is usually generated automatically
|
| 62 |
+
/// but may be specified explicitly to support key rotation,
|
| 63 |
+
/// which requires keeping the same identifiers.
|
| 64 |
+
std::string GetEncryptionKeyMetadata(const ::arrow::util::SecureString& data_key,
|
| 65 |
+
const std::string& master_key_id,
|
| 66 |
+
bool is_footer_key,
|
| 67 |
+
std::string key_id_in_file = "");
|
| 68 |
+
|
| 69 |
+
private:
|
| 70 |
+
KeyEncryptionKey CreateKeyEncryptionKey(const std::string& master_key_id);
|
| 71 |
+
|
| 72 |
+
/// A map of Master Encryption Key ID -> KeyEncryptionKey, for the current token
|
| 73 |
+
std::shared_ptr<::arrow::util::ConcurrentMap<std::string, KeyEncryptionKey>>
|
| 74 |
+
kek_per_master_key_id_;
|
| 75 |
+
|
| 76 |
+
std::shared_ptr<KmsClient> kms_client_;
|
| 77 |
+
KmsConnectionConfig kms_connection_config_;
|
| 78 |
+
std::shared_ptr<FileKeyMaterialStore> key_material_store_;
|
| 79 |
+
const double cache_entry_lifetime_seconds_;
|
| 80 |
+
const bool double_wrapping_;
|
| 81 |
+
uint16_t key_counter_;
|
| 82 |
+
};
|
| 83 |
+
|
| 84 |
+
} // namespace parquet::encryption
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_system_key_material_store.h
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <set>
|
| 21 |
+
#include <string>
|
| 22 |
+
#include <unordered_map>
|
| 23 |
+
|
| 24 |
+
#include "arrow/filesystem/filesystem.h"
|
| 25 |
+
|
| 26 |
+
#include "parquet/encryption/file_key_material_store.h"
|
| 27 |
+
#include "parquet/exception.h"
|
| 28 |
+
|
| 29 |
+
namespace parquet::encryption {
|
| 30 |
+
|
| 31 |
+
/// A FileKeyMaterialStore that stores key material in a file system file in the same
|
| 32 |
+
/// folder as the Parquet file.
|
| 33 |
+
class PARQUET_EXPORT FileSystemKeyMaterialStore : public FileKeyMaterialStore {
|
| 34 |
+
public:
|
| 35 |
+
static constexpr const char kKeyMaterialFilePrefix[] = "_KEY_MATERIAL_FOR_";
|
| 36 |
+
static constexpr const char kTempFilePrefix[] = "_TMP";
|
| 37 |
+
static constexpr const char kKeyMaterialFileSuffix[] = ".json";
|
| 38 |
+
|
| 39 |
+
FileSystemKeyMaterialStore() {}
|
| 40 |
+
FileSystemKeyMaterialStore(std::string key_material_file_path,
|
| 41 |
+
std::shared_ptr<::arrow::fs::FileSystem> file_system);
|
| 42 |
+
|
| 43 |
+
/// Creates a new file system key material store for a parquet file.
|
| 44 |
+
/// When use_tmp_prefix is true, files are saved with an extra _TMP prefix so they don't
|
| 45 |
+
/// conflict with existing external material files. This is useful during key rotation
|
| 46 |
+
/// so that temporary key material files can be created while using the existing key
|
| 47 |
+
/// material, before moving the key material to the non-temporary location.
|
| 48 |
+
static std::shared_ptr<FileSystemKeyMaterialStore> Make(
|
| 49 |
+
std::string parquet_file_path, std::shared_ptr<::arrow::fs::FileSystem> file_system,
|
| 50 |
+
bool use_tmp_prefix);
|
| 51 |
+
|
| 52 |
+
/// Add key material for one encryption key.
|
| 53 |
+
void AddKeyMaterial(std::string key_id_in_file, std::string key_material) {
|
| 54 |
+
key_material_map_.emplace(std::move(key_id_in_file), std::move(key_material));
|
| 55 |
+
}
|
| 56 |
+
|
| 57 |
+
/// Get key material
|
| 58 |
+
std::string GetKeyMaterial(std::string key_id_in_file) {
|
| 59 |
+
if (key_material_map_.empty()) {
|
| 60 |
+
LoadKeyMaterialMap();
|
| 61 |
+
}
|
| 62 |
+
auto found = key_material_map_.find(key_id_in_file);
|
| 63 |
+
if (found == key_material_map_.end()) {
|
| 64 |
+
throw ParquetException("Invalid key id");
|
| 65 |
+
}
|
| 66 |
+
return found->second;
|
| 67 |
+
}
|
| 68 |
+
|
| 69 |
+
/// After key material was added for all keys in the given Parquet file,
|
| 70 |
+
/// save material in persistent store.
|
| 71 |
+
void SaveMaterial();
|
| 72 |
+
|
| 73 |
+
/// Remove key material from persistent store. Used in key rotation.
|
| 74 |
+
void RemoveMaterial();
|
| 75 |
+
|
| 76 |
+
/// Move key material to another store. Used in key rotation.
|
| 77 |
+
void MoveMaterialTo(std::shared_ptr<FileKeyMaterialStore> target_key_store);
|
| 78 |
+
|
| 79 |
+
/// Returns the Set of all key IDs in this store (for the given Parquet file)
|
| 80 |
+
std::vector<std::string> GetKeyIDSet();
|
| 81 |
+
|
| 82 |
+
private:
|
| 83 |
+
std::string GetStorageFilePath() { return key_material_file_path_; }
|
| 84 |
+
|
| 85 |
+
std::string BuildKeyMaterialMapJson();
|
| 86 |
+
void LoadKeyMaterialMap();
|
| 87 |
+
std::string key_material_file_path_;
|
| 88 |
+
std::shared_ptr<::arrow::fs::FileSystem> file_system_;
|
| 89 |
+
/// Maps ID of a key in Parquet file and key material
|
| 90 |
+
std::unordered_map<std::string, std::string> key_material_map_;
|
| 91 |
+
};
|
| 92 |
+
|
| 93 |
+
} // namespace parquet::encryption
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_encryption_key.h
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <cstdint>
|
| 21 |
+
#include <vector>
|
| 22 |
+
|
| 23 |
+
#include "arrow/util/base64.h"
|
| 24 |
+
#include "arrow/util/secure_string.h"
|
| 25 |
+
|
| 26 |
+
namespace parquet::encryption {
|
| 27 |
+
|
| 28 |
+
// In the double wrapping mode, each "data encryption key" (DEK) is encrypted with a “key
|
| 29 |
+
// encryption key” (KEK), that in turn is encrypted with a "master encryption key" (MEK).
|
| 30 |
+
// In a writer process, a random KEK is generated for each MEK ID, and cached in a <MEK-ID
|
| 31 |
+
// : KEK> map. This allows to perform an interaction with a KMS server only once for each
|
| 32 |
+
// MEK, in order to wrap its KEK. "Data encryption key" (DEK) wrapping is performed
|
| 33 |
+
// locally, and does not involve an interaction with a KMS server.
|
| 34 |
+
class KeyEncryptionKey {
|
| 35 |
+
public:
|
| 36 |
+
KeyEncryptionKey(::arrow::util::SecureString kek_bytes, std::string kek_id,
|
| 37 |
+
std::string encoded_wrapped_kek)
|
| 38 |
+
: kek_bytes_(std::move(kek_bytes)),
|
| 39 |
+
kek_id_(std::move(kek_id)),
|
| 40 |
+
encoded_kek_id_(::arrow::util::base64_encode(kek_id_)),
|
| 41 |
+
encoded_wrapped_kek_(std::move(encoded_wrapped_kek)) {}
|
| 42 |
+
|
| 43 |
+
const ::arrow::util::SecureString& kek_bytes() const { return kek_bytes_; }
|
| 44 |
+
|
| 45 |
+
const std::string& kek_id() const { return kek_id_; }
|
| 46 |
+
|
| 47 |
+
const std::string& encoded_kek_id() const { return encoded_kek_id_; }
|
| 48 |
+
|
| 49 |
+
const std::string& encoded_wrapped_kek() const { return encoded_wrapped_kek_; }
|
| 50 |
+
|
| 51 |
+
private:
|
| 52 |
+
::arrow::util::SecureString kek_bytes_;
|
| 53 |
+
std::string kek_id_;
|
| 54 |
+
std::string encoded_kek_id_;
|
| 55 |
+
std::string encoded_wrapped_kek_;
|
| 56 |
+
};
|
| 57 |
+
|
| 58 |
+
} // namespace parquet::encryption
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_material.h
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <string>
|
| 21 |
+
|
| 22 |
+
#include "parquet/platform.h"
|
| 23 |
+
|
| 24 |
+
namespace arrow {
|
| 25 |
+
namespace json {
|
| 26 |
+
namespace internal {
|
| 27 |
+
class ObjectParser;
|
| 28 |
+
} // namespace internal
|
| 29 |
+
} // namespace json
|
| 30 |
+
} // namespace arrow
|
| 31 |
+
|
| 32 |
+
namespace parquet::encryption {
|
| 33 |
+
|
| 34 |
+
// KeyMaterial class represents the "key material", keeping the information that allows
|
| 35 |
+
// readers to recover an encryption key (see description of the KeyMetadata class). The
|
| 36 |
+
// keytools package (PARQUET-1373) implements the "envelope encryption" pattern, in a
|
| 37 |
+
// "single wrapping" or "double wrapping" mode. In the single wrapping mode, the key
|
| 38 |
+
// material is generated by encrypting the "data encryption key" (DEK) by a "master key".
|
| 39 |
+
// In the double wrapping mode, the key material is generated by encrypting the DEK by a
|
| 40 |
+
// "key encryption key" (KEK), that in turn is encrypted by a "master key".
|
| 41 |
+
//
|
| 42 |
+
// Key material is kept in a flat json object, with the following fields:
|
| 43 |
+
// 1. "keyMaterialType" - a String, with the type of key material. In the current
|
| 44 |
+
// version, only one value is allowed - "PKMT1" (stands
|
| 45 |
+
// for "parquet key management tools, version 1"). For external key material storage,
|
| 46 |
+
// this field is written in both "key metadata" and "key material" jsons. For internal
|
| 47 |
+
// key material storage, this field is written only once in the common json.
|
| 48 |
+
// 2. "isFooterKey" - a boolean. If true, means that the material belongs to a file footer
|
| 49 |
+
// key, and keeps additional information (such as
|
| 50 |
+
// KMS instance ID and URL). If false, means that the material belongs to a column
|
| 51 |
+
// key.
|
| 52 |
+
// 3. "kmsInstanceID" - a String, with the KMS Instance ID. Written only in footer key
|
| 53 |
+
// material.
|
| 54 |
+
// 4. "kmsInstanceURL" - a String, with the KMS Instance URL. Written only in footer key
|
| 55 |
+
// material.
|
| 56 |
+
// 5. "masterKeyID" - a String, with the ID of the master key used to generate the
|
| 57 |
+
// material.
|
| 58 |
+
// 6. "wrappedDEK" - a String, with the wrapped DEK (base64 encoding).
|
| 59 |
+
// 7. "doubleWrapping" - a boolean. If true, means that the material was generated in
|
| 60 |
+
// double wrapping mode.
|
| 61 |
+
// If false - in single wrapping mode.
|
| 62 |
+
// 8. "keyEncryptionKeyID" - a String, with the ID of the KEK used to generate the
|
| 63 |
+
// material. Written only in double wrapping mode.
|
| 64 |
+
// 9. "wrappedKEK" - a String, with the wrapped KEK (base64 encoding). Written only in
|
| 65 |
+
// double wrapping mode.
|
| 66 |
+
class PARQUET_EXPORT KeyMaterial {
|
| 67 |
+
public:
|
| 68 |
+
// these fields are defined in a specification and should never be changed
|
| 69 |
+
static constexpr const char kKeyMaterialTypeField[] = "keyMaterialType";
|
| 70 |
+
static constexpr const char kKeyMaterialType1[] = "PKMT1";
|
| 71 |
+
|
| 72 |
+
static constexpr const char kFooterKeyIdInFile[] = "footerKey";
|
| 73 |
+
static constexpr const char kColumnKeyIdInFilePrefix[] = "columnKey";
|
| 74 |
+
|
| 75 |
+
static constexpr const char kIsFooterKeyField[] = "isFooterKey";
|
| 76 |
+
static constexpr const char kDoubleWrappingField[] = "doubleWrapping";
|
| 77 |
+
static constexpr const char kKmsInstanceIdField[] = "kmsInstanceID";
|
| 78 |
+
static constexpr const char kKmsInstanceUrlField[] = "kmsInstanceURL";
|
| 79 |
+
static constexpr const char kMasterKeyIdField[] = "masterKeyID";
|
| 80 |
+
static constexpr const char kWrappedDataEncryptionKeyField[] = "wrappedDEK";
|
| 81 |
+
static constexpr const char kKeyEncryptionKeyIdField[] = "keyEncryptionKeyID";
|
| 82 |
+
static constexpr const char kWrappedKeyEncryptionKeyField[] = "wrappedKEK";
|
| 83 |
+
|
| 84 |
+
public:
|
| 85 |
+
KeyMaterial() = default;
|
| 86 |
+
|
| 87 |
+
static KeyMaterial Parse(const std::string& key_material_string);
|
| 88 |
+
|
| 89 |
+
static KeyMaterial Parse(
|
| 90 |
+
const ::arrow::json::internal::ObjectParser* key_material_json);
|
| 91 |
+
|
| 92 |
+
/// This method returns a json string that will be stored either inside a parquet file
|
| 93 |
+
/// or in a key material store outside the parquet file.
|
| 94 |
+
static std::string SerializeToJson(bool is_footer_key,
|
| 95 |
+
const std::string& kms_instance_id,
|
| 96 |
+
const std::string& kms_instance_url,
|
| 97 |
+
const std::string& master_key_id,
|
| 98 |
+
bool is_double_wrapped, const std::string& kek_id,
|
| 99 |
+
const std::string& encoded_wrapped_kek,
|
| 100 |
+
const std::string& encoded_wrapped_dek,
|
| 101 |
+
bool is_internal_storage);
|
| 102 |
+
|
| 103 |
+
bool is_footer_key() const { return is_footer_key_; }
|
| 104 |
+
bool is_double_wrapped() const { return is_double_wrapped_; }
|
| 105 |
+
const std::string& master_key_id() const { return master_key_id_; }
|
| 106 |
+
const std::string& wrapped_dek() const { return encoded_wrapped_dek_; }
|
| 107 |
+
const std::string& kek_id() const { return kek_id_; }
|
| 108 |
+
const std::string& wrapped_kek() const { return encoded_wrapped_kek_; }
|
| 109 |
+
const std::string& kms_instance_id() const { return kms_instance_id_; }
|
| 110 |
+
const std::string& kms_instance_url() const { return kms_instance_url_; }
|
| 111 |
+
|
| 112 |
+
private:
|
| 113 |
+
KeyMaterial(bool is_footer_key, const std::string& kms_instance_id,
|
| 114 |
+
const std::string& kms_instance_url, const std::string& master_key_id,
|
| 115 |
+
bool is_double_wrapped, const std::string& kek_id,
|
| 116 |
+
const std::string& encoded_wrapped_kek,
|
| 117 |
+
const std::string& encoded_wrapped_dek);
|
| 118 |
+
|
| 119 |
+
bool is_footer_key_;
|
| 120 |
+
std::string kms_instance_id_;
|
| 121 |
+
std::string kms_instance_url_;
|
| 122 |
+
std::string master_key_id_;
|
| 123 |
+
bool is_double_wrapped_;
|
| 124 |
+
std::string kek_id_;
|
| 125 |
+
std::string encoded_wrapped_kek_;
|
| 126 |
+
std::string encoded_wrapped_dek_;
|
| 127 |
+
};
|
| 128 |
+
|
| 129 |
+
} // namespace parquet::encryption
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_metadata.h
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <string>
|
| 21 |
+
#include <variant>
|
| 22 |
+
|
| 23 |
+
#include "parquet/encryption/key_material.h"
|
| 24 |
+
#include "parquet/exception.h"
|
| 25 |
+
#include "parquet/platform.h"
|
| 26 |
+
|
| 27 |
+
namespace parquet::encryption {
|
| 28 |
+
|
| 29 |
+
// Parquet encryption specification defines "key metadata" as an arbitrary byte array,
|
| 30 |
+
// generated by file writers for each encryption key, and passed to the low level API for
|
| 31 |
+
// storage in the file footer. The "key metadata" field is made available to file readers
|
| 32 |
+
// to enable recovery of the key. This interface can be utilized for implementation
|
| 33 |
+
// of any key management scheme.
|
| 34 |
+
//
|
| 35 |
+
// The keytools package (PARQUET-1373) implements one approach, of many possible, to key
|
| 36 |
+
// management and to generation of the "key metadata" fields. This approach, based on the
|
| 37 |
+
// "envelope encryption" pattern, allows integration with KMS servers. It keeps the actual
|
| 38 |
+
// material, required to recover a key, in a "key material" object (see the KeyMaterial
|
| 39 |
+
// class for details). This class is implemented to support version 1 of the parquet key
|
| 40 |
+
// management tools specification.
|
| 41 |
+
//
|
| 42 |
+
// KeyMetadata writes (and reads) the "key metadata" field as a flat json object,
|
| 43 |
+
// with the following fields:
|
| 44 |
+
// 1. "keyMaterialType" - a String, with the type of key material.
|
| 45 |
+
// 2. "internalStorage" - a boolean. If true, means that "key material" is kept inside the
|
| 46 |
+
// "key metadata" field. If false, "key material" is kept externally (outside Parquet
|
| 47 |
+
// files) - in this case, "key metadata" keeps a reference to the external "key material".
|
| 48 |
+
// 3. "keyReference" - a String, with the reference to the external "key material".
|
| 49 |
+
// Written only if internalStorage is false.
|
| 50 |
+
//
|
| 51 |
+
// If internalStorage is true, "key material" is a part of "key metadata", and the json
|
| 52 |
+
// keeps additional fields, described in the KeyMaterial class.
|
| 53 |
+
class PARQUET_EXPORT KeyMetadata {
|
| 54 |
+
public:
|
| 55 |
+
static constexpr const char kKeyMaterialInternalStorageField[] = "internalStorage";
|
| 56 |
+
static constexpr const char kKeyReferenceField[] = "keyReference";
|
| 57 |
+
|
| 58 |
+
/// key_metadata_bytes is the key metadata field stored in the parquet file,
|
| 59 |
+
/// in the serialized json object format.
|
| 60 |
+
static KeyMetadata Parse(const std::string& key_metadata_bytes);
|
| 61 |
+
|
| 62 |
+
static std::string CreateSerializedForExternalMaterial(
|
| 63 |
+
const std::string& key_reference);
|
| 64 |
+
|
| 65 |
+
bool key_material_stored_internally() const { return is_internal_storage_; }
|
| 66 |
+
|
| 67 |
+
const KeyMaterial& key_material() const {
|
| 68 |
+
if (!is_internal_storage_) {
|
| 69 |
+
throw ParquetException("key material is stored externally.");
|
| 70 |
+
}
|
| 71 |
+
return ::std::get<KeyMaterial>(key_material_or_reference_);
|
| 72 |
+
}
|
| 73 |
+
|
| 74 |
+
const std::string& key_reference() const {
|
| 75 |
+
if (is_internal_storage_) {
|
| 76 |
+
throw ParquetException("key material is stored internally.");
|
| 77 |
+
}
|
| 78 |
+
return ::std::get<std::string>(key_material_or_reference_);
|
| 79 |
+
}
|
| 80 |
+
|
| 81 |
+
private:
|
| 82 |
+
explicit KeyMetadata(const KeyMaterial& key_material);
|
| 83 |
+
explicit KeyMetadata(const std::string& key_reference);
|
| 84 |
+
|
| 85 |
+
bool is_internal_storage_;
|
| 86 |
+
/// If is_internal_storage_ is true, KeyMaterial is set,
|
| 87 |
+
/// else a string referencing to an outside "key material" is set.
|
| 88 |
+
::std::variant<KeyMaterial, std::string> key_material_or_reference_;
|
| 89 |
+
};
|
| 90 |
+
|
| 91 |
+
} // namespace parquet::encryption
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_toolkit.h
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <memory>
|
| 21 |
+
#include <string>
|
| 22 |
+
|
| 23 |
+
#include "parquet/encryption/key_encryption_key.h"
|
| 24 |
+
#include "parquet/encryption/kms_client.h"
|
| 25 |
+
#include "parquet/encryption/kms_client_factory.h"
|
| 26 |
+
#include "parquet/encryption/two_level_cache_with_expiration.h"
|
| 27 |
+
#include "parquet/platform.h"
|
| 28 |
+
|
| 29 |
+
namespace parquet::encryption {
|
| 30 |
+
|
| 31 |
+
static constexpr uint64_t kCacheCleanPeriodForKeyRotation = 60 * 60; // 1 hour
|
| 32 |
+
|
| 33 |
+
// KeyToolkit is a utility that keeps various tools for key management (such as key
|
| 34 |
+
// rotation, kms client instantiation, cache control, etc), plus a number of auxiliary
|
| 35 |
+
// classes for internal use.
|
| 36 |
+
class PARQUET_EXPORT KeyToolkit {
|
| 37 |
+
public:
|
| 38 |
+
KeyToolkit() { last_cache_clean_for_key_rotation_time_ = {}; }
|
| 39 |
+
|
| 40 |
+
/// KMS client two level cache: token -> KMSInstanceId -> KmsClient
|
| 41 |
+
TwoLevelCacheWithExpiration<std::shared_ptr<KmsClient>>& kms_client_cache_per_token() {
|
| 42 |
+
return kms_client_cache_;
|
| 43 |
+
}
|
| 44 |
+
/// Key encryption key two level cache for wrapping: token -> MasterEncryptionKeyId ->
|
| 45 |
+
/// KeyEncryptionKey
|
| 46 |
+
TwoLevelCacheWithExpiration<KeyEncryptionKey>& kek_write_cache_per_token() {
|
| 47 |
+
return key_encryption_key_write_cache_;
|
| 48 |
+
}
|
| 49 |
+
|
| 50 |
+
/// Key encryption key two level cache for unwrapping: token -> KeyEncryptionKeyId ->
|
| 51 |
+
/// KeyEncryptionKeyBytes
|
| 52 |
+
TwoLevelCacheWithExpiration<::arrow::util::SecureString>& kek_read_cache_per_token() {
|
| 53 |
+
return key_encryption_key_read_cache_;
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
std::shared_ptr<KmsClient> GetKmsClient(
|
| 57 |
+
const KmsConnectionConfig& kms_connection_config, double cache_entry_lifetime_ms);
|
| 58 |
+
|
| 59 |
+
/// Flush any caches that are tied to the (compromised) access_token
|
| 60 |
+
void RemoveCacheEntriesForToken(const std::string& access_token);
|
| 61 |
+
|
| 62 |
+
void RemoveCacheEntriesForAllTokens();
|
| 63 |
+
|
| 64 |
+
void RegisterKmsClientFactory(std::shared_ptr<KmsClientFactory> kms_client_factory) {
|
| 65 |
+
if (kms_client_factory_ != NULLPTR) {
|
| 66 |
+
throw ParquetException("KMS client factory has already been registered.");
|
| 67 |
+
}
|
| 68 |
+
kms_client_factory_ = std::move(kms_client_factory);
|
| 69 |
+
}
|
| 70 |
+
|
| 71 |
+
/// Key rotation. In the single wrapping mode, decrypts data keys with old master keys,
|
| 72 |
+
/// then encrypts them with new master keys. In the double wrapping mode, decrypts KEKs
|
| 73 |
+
/// (key encryption keys) with old master keys, generates new KEKs and encrypts them
|
| 74 |
+
/// with new master keys. Works only if key material is not stored internally in file
|
| 75 |
+
/// footers. Not supported in local key wrapping mode. Method can be run by multiple
|
| 76 |
+
/// threads, but each thread must work on different files.
|
| 77 |
+
void RotateMasterKeys(const KmsConnectionConfig& kms_connection_config,
|
| 78 |
+
const std::string& parquet_file_path,
|
| 79 |
+
const std::shared_ptr<::arrow::fs::FileSystem>& file_system,
|
| 80 |
+
bool double_wrapping, double cache_lifetime_seconds);
|
| 81 |
+
|
| 82 |
+
private:
|
| 83 |
+
TwoLevelCacheWithExpiration<std::shared_ptr<KmsClient>> kms_client_cache_;
|
| 84 |
+
TwoLevelCacheWithExpiration<KeyEncryptionKey> key_encryption_key_write_cache_;
|
| 85 |
+
TwoLevelCacheWithExpiration<::arrow::util::SecureString> key_encryption_key_read_cache_;
|
| 86 |
+
std::shared_ptr<KmsClientFactory> kms_client_factory_;
|
| 87 |
+
mutable ::arrow::util::Mutex last_cache_clean_for_key_rotation_time_mutex_;
|
| 88 |
+
internal::TimePoint last_cache_clean_for_key_rotation_time_;
|
| 89 |
+
};
|
| 90 |
+
|
| 91 |
+
// "data encryption key" and "master key identifier" are paired together as output when
|
| 92 |
+
// parsing from "key material"
|
| 93 |
+
class PARQUET_EXPORT KeyWithMasterId {
|
| 94 |
+
public:
|
| 95 |
+
KeyWithMasterId(::arrow::util::SecureString key_bytes, std::string master_id)
|
| 96 |
+
: key_bytes_(std::move(key_bytes)), master_id_(std::move(master_id)) {}
|
| 97 |
+
|
| 98 |
+
const ::arrow::util::SecureString& data_key() const { return key_bytes_; }
|
| 99 |
+
const std::string& master_id() const { return master_id_; }
|
| 100 |
+
|
| 101 |
+
private:
|
| 102 |
+
::arrow::util::SecureString key_bytes_;
|
| 103 |
+
std::string master_id_;
|
| 104 |
+
};
|
| 105 |
+
|
| 106 |
+
} // namespace parquet::encryption
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/kms_client.h
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <memory>
|
| 21 |
+
#include <string>
|
| 22 |
+
#include <unordered_map>
|
| 23 |
+
|
| 24 |
+
#include "arrow/util/mutex.h"
|
| 25 |
+
#include "arrow/util/secure_string.h"
|
| 26 |
+
|
| 27 |
+
#include "parquet/exception.h"
|
| 28 |
+
#include "parquet/platform.h"
|
| 29 |
+
|
| 30 |
+
namespace parquet::encryption {
|
| 31 |
+
|
| 32 |
+
/// This class wraps the key access token of a KMS server. If your token changes over
|
| 33 |
+
/// time, you should keep the reference to the KeyAccessToken object and call Refresh()
|
| 34 |
+
/// method every time you have a new token.
|
| 35 |
+
class PARQUET_EXPORT KeyAccessToken {
|
| 36 |
+
public:
|
| 37 |
+
KeyAccessToken() = default;
|
| 38 |
+
|
| 39 |
+
explicit KeyAccessToken(const std::string value) : value_(value) {}
|
| 40 |
+
|
| 41 |
+
void Refresh(const std::string& new_value) {
|
| 42 |
+
auto lock = mutex_.Lock();
|
| 43 |
+
value_ = new_value;
|
| 44 |
+
}
|
| 45 |
+
|
| 46 |
+
const std::string& value() const {
|
| 47 |
+
auto lock = mutex_.Lock();
|
| 48 |
+
return value_;
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
private:
|
| 52 |
+
std::string value_;
|
| 53 |
+
mutable ::arrow::util::Mutex mutex_;
|
| 54 |
+
};
|
| 55 |
+
|
| 56 |
+
struct PARQUET_EXPORT KmsConnectionConfig {
|
| 57 |
+
std::string kms_instance_id;
|
| 58 |
+
std::string kms_instance_url;
|
| 59 |
+
/// If the access token is changed in the future, you should keep a reference to
|
| 60 |
+
/// this object and call Refresh() on it whenever there is a new access token.
|
| 61 |
+
std::shared_ptr<KeyAccessToken> refreshable_key_access_token;
|
| 62 |
+
std::unordered_map<std::string, std::string> custom_kms_conf;
|
| 63 |
+
|
| 64 |
+
KmsConnectionConfig();
|
| 65 |
+
|
| 66 |
+
const std::string& key_access_token() const {
|
| 67 |
+
if (refreshable_key_access_token == NULLPTR ||
|
| 68 |
+
refreshable_key_access_token->value().empty()) {
|
| 69 |
+
throw ParquetException("key access token is not set!");
|
| 70 |
+
}
|
| 71 |
+
return refreshable_key_access_token->value();
|
| 72 |
+
}
|
| 73 |
+
|
| 74 |
+
void SetDefaultIfEmpty();
|
| 75 |
+
};
|
| 76 |
+
|
| 77 |
+
class PARQUET_EXPORT KmsClient {
|
| 78 |
+
public:
|
| 79 |
+
static constexpr const char kKmsInstanceIdDefault[] = "DEFAULT";
|
| 80 |
+
static constexpr const char kKmsInstanceUrlDefault[] = "DEFAULT";
|
| 81 |
+
static constexpr const char kKeyAccessTokenDefault[] = "DEFAULT";
|
| 82 |
+
|
| 83 |
+
/// \brief Wraps a key.
|
| 84 |
+
///
|
| 85 |
+
/// Encrypts it with the master key, encodes the result
|
| 86 |
+
/// and potentially adds a KMS-specific metadata.
|
| 87 |
+
virtual std::string WrapKey(const ::arrow::util::SecureString& key_bytes,
|
| 88 |
+
const std::string& master_key_identifier) = 0;
|
| 89 |
+
|
| 90 |
+
/// \brief Decrypts (unwraps) a key with the master key.
|
| 91 |
+
virtual ::arrow::util::SecureString UnwrapKey(
|
| 92 |
+
const std::string& wrapped_key, const std::string& master_key_identifier) = 0;
|
| 93 |
+
|
| 94 |
+
virtual ~KmsClient() {}
|
| 95 |
+
};
|
| 96 |
+
|
| 97 |
+
} // namespace parquet::encryption
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/kms_client_factory.h
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include "parquet/encryption/kms_client.h"
|
| 21 |
+
#include "parquet/platform.h"
|
| 22 |
+
|
| 23 |
+
namespace parquet::encryption {
|
| 24 |
+
|
| 25 |
+
class PARQUET_EXPORT KmsClientFactory {
|
| 26 |
+
public:
|
| 27 |
+
explicit KmsClientFactory(bool wrap_locally = false) : wrap_locally_(wrap_locally) {}
|
| 28 |
+
|
| 29 |
+
virtual ~KmsClientFactory() = default;
|
| 30 |
+
|
| 31 |
+
virtual std::shared_ptr<KmsClient> CreateKmsClient(
|
| 32 |
+
const KmsConnectionConfig& kms_connection_config) = 0;
|
| 33 |
+
|
| 34 |
+
protected:
|
| 35 |
+
bool wrap_locally_;
|
| 36 |
+
};
|
| 37 |
+
|
| 38 |
+
} // namespace parquet::encryption
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/local_wrap_kms_client.h
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <unordered_map>
|
| 21 |
+
#include <vector>
|
| 22 |
+
|
| 23 |
+
#include "arrow/util/concurrent_map.h"
|
| 24 |
+
|
| 25 |
+
#include "parquet/encryption/kms_client.h"
|
| 26 |
+
#include "parquet/platform.h"
|
| 27 |
+
|
| 28 |
+
namespace parquet::encryption {
|
| 29 |
+
|
| 30 |
+
/// This class supports local wrapping mode, master keys will be fetched from the KMS
|
| 31 |
+
/// server and used to encrypt other keys (data encryption keys or key encryption keys).
|
| 32 |
+
class PARQUET_EXPORT LocalWrapKmsClient : public KmsClient {
|
| 33 |
+
public:
|
| 34 |
+
static constexpr const char kLocalWrapNoKeyVersion[] = "NO_VERSION";
|
| 35 |
+
|
| 36 |
+
explicit LocalWrapKmsClient(const KmsConnectionConfig& kms_connection_config);
|
| 37 |
+
|
| 38 |
+
std::string WrapKey(const ::arrow::util::SecureString& key_bytes,
|
| 39 |
+
const std::string& master_key_identifier) override;
|
| 40 |
+
|
| 41 |
+
::arrow::util::SecureString UnwrapKey(
|
| 42 |
+
const std::string& wrapped_key, const std::string& master_key_identifier) override;
|
| 43 |
+
|
| 44 |
+
protected:
|
| 45 |
+
/// Get master key from the remote KMS server.
|
| 46 |
+
/// Note: this function might be called by multiple threads
|
| 47 |
+
virtual const ::arrow::util::SecureString& GetMasterKeyFromServer(
|
| 48 |
+
const std::string& master_key_identifier) = 0;
|
| 49 |
+
|
| 50 |
+
private:
|
| 51 |
+
/// KMS systems wrap keys by encrypting them by master keys, and attaching additional
|
| 52 |
+
/// information (such as the version number of the masker key) to the result of
|
| 53 |
+
/// encryption. The master key version is required in key rotation. Currently, the
|
| 54 |
+
/// local wrapping mode does not support key rotation (because not all KMS systems allow
|
| 55 |
+
/// to fetch a master key by its ID and version number). Still, the local wrapping mode
|
| 56 |
+
/// adds a placeholder for the master key version, that will enable support for key
|
| 57 |
+
/// rotation in this mode in the future, with appropriate KMS systems. This will also
|
| 58 |
+
/// enable backward compatibility, where future readers will be able to extract master
|
| 59 |
+
/// key version in the files written by the current code.
|
| 60 |
+
///
|
| 61 |
+
/// LocalKeyWrap class writes (and reads) the "key wrap" as a flat json with the
|
| 62 |
+
/// following fields:
|
| 63 |
+
/// 1. "masterKeyVersion" - a String, with the master key version. In the current
|
| 64 |
+
/// version, only one value is allowed - "NO_VERSION".
|
| 65 |
+
/// 2. "encryptedKey" - a String, with the key encrypted by the master key
|
| 66 |
+
/// (base64-encoded).
|
| 67 |
+
class LocalKeyWrap {
|
| 68 |
+
public:
|
| 69 |
+
static constexpr const char kLocalWrapKeyVersionField[] = "masterKeyVersion";
|
| 70 |
+
static constexpr const char kLocalWrapEncryptedKeyField[] = "encryptedKey";
|
| 71 |
+
|
| 72 |
+
LocalKeyWrap(std::string master_key_version, std::string encrypted_encoded_key);
|
| 73 |
+
|
| 74 |
+
static std::string CreateSerialized(const std::string& encrypted_encoded_key);
|
| 75 |
+
|
| 76 |
+
static LocalKeyWrap Parse(const std::string& wrapped_key);
|
| 77 |
+
|
| 78 |
+
const std::string& master_key_version() const { return master_key_version_; }
|
| 79 |
+
|
| 80 |
+
const std::string& encrypted_encoded_key() const { return encrypted_encoded_key_; }
|
| 81 |
+
|
| 82 |
+
private:
|
| 83 |
+
std::string encrypted_encoded_key_;
|
| 84 |
+
std::string master_key_version_;
|
| 85 |
+
};
|
| 86 |
+
|
| 87 |
+
const ::arrow::util::SecureString& GetKeyFromServer(const std::string& key_identifier);
|
| 88 |
+
|
| 89 |
+
protected:
|
| 90 |
+
KmsConnectionConfig kms_connection_config_;
|
| 91 |
+
::arrow::util::ConcurrentMap<std::string, ::arrow::util::SecureString>
|
| 92 |
+
master_key_cache_;
|
| 93 |
+
};
|
| 94 |
+
|
| 95 |
+
} // namespace parquet::encryption
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/test_encryption_util.h
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
// This module defines an abstract interface for iterating through pages in a
|
| 19 |
+
// Parquet column chunk within a row group. It could be extended in the future
|
| 20 |
+
// to iterate through all data pages in all chunks in a file.
|
| 21 |
+
|
| 22 |
+
#pragma once
|
| 23 |
+
|
| 24 |
+
#include <memory>
|
| 25 |
+
#include <string>
|
| 26 |
+
#include <unordered_map>
|
| 27 |
+
|
| 28 |
+
#include <gtest/gtest.h>
|
| 29 |
+
|
| 30 |
+
#include "arrow/filesystem/filesystem.h"
|
| 31 |
+
#include "arrow/filesystem/localfs.h"
|
| 32 |
+
#include "arrow/status.h"
|
| 33 |
+
#include "arrow/util/io_util.h"
|
| 34 |
+
#include "arrow/util/secure_string.h"
|
| 35 |
+
|
| 36 |
+
#include "parquet/encryption/encryption.h"
|
| 37 |
+
#include "parquet/test_util.h"
|
| 38 |
+
|
| 39 |
+
namespace parquet {
|
| 40 |
+
class ParquetFileReader;
|
| 41 |
+
namespace encryption::test {
|
| 42 |
+
|
| 43 |
+
using ::arrow::internal::TemporaryDir;
|
| 44 |
+
using ::arrow::util::SecureString;
|
| 45 |
+
|
| 46 |
+
constexpr int kFixedLength = 10;
|
| 47 |
+
|
| 48 |
+
const SecureString kFooterEncryptionKey("0123456789012345");
|
| 49 |
+
const SecureString kColumnEncryptionKey1("1234567890123450");
|
| 50 |
+
const SecureString kColumnEncryptionKey2("1234567890123451");
|
| 51 |
+
const SecureString kColumnEncryptionKey3("1234567890123452");
|
| 52 |
+
const char kFileName[] = "tester";
|
| 53 |
+
|
| 54 |
+
// Get the path of file inside parquet test data directory
|
| 55 |
+
std::string data_file(const char* file);
|
| 56 |
+
|
| 57 |
+
// A temporary directory that contains the encrypted files generated in the tests.
|
| 58 |
+
extern std::unique_ptr<TemporaryDir> temp_dir;
|
| 59 |
+
|
| 60 |
+
inline ::arrow::Result<std::unique_ptr<TemporaryDir>> temp_data_dir() {
|
| 61 |
+
return TemporaryDir::Make("parquet-encryption-test-");
|
| 62 |
+
}
|
| 63 |
+
|
| 64 |
+
const char kDoubleFieldName[] = "double_field";
|
| 65 |
+
const char kFloatFieldName[] = "float_field";
|
| 66 |
+
const char kBooleanFieldName[] = "boolean_field";
|
| 67 |
+
const char kInt32FieldName[] = "int32_field";
|
| 68 |
+
const char kInt64FieldName[] = "int64_field";
|
| 69 |
+
const char kInt96FieldName[] = "int96_field";
|
| 70 |
+
const char kByteArrayFieldName[] = "ba_field";
|
| 71 |
+
const char kFixedLenByteArrayFieldName[] = "flba_field";
|
| 72 |
+
|
| 73 |
+
const char kFooterMasterKey[] = "0123456789012345";
|
| 74 |
+
const char kFooterMasterKeyId[] = "kf";
|
| 75 |
+
const char* const kColumnMasterKeys[] = {"1234567890123450", "1234567890123451",
|
| 76 |
+
"1234567890123452", "1234567890123453",
|
| 77 |
+
"1234567890123454", "1234567890123455"};
|
| 78 |
+
const char* const kColumnMasterKeyIds[] = {"kc1", "kc2", "kc3", "kc4", "kc5", "kc6"};
|
| 79 |
+
|
| 80 |
+
// New master key values used to simulate key rotation
|
| 81 |
+
const char kNewFooterMasterKey[] = "9123456789012345";
|
| 82 |
+
const char* const kNewColumnMasterKeys[] = {"9234567890123450", "9234567890123451",
|
| 83 |
+
"9234567890123452", "9234567890123453",
|
| 84 |
+
"9234567890123454", "9234567890123455"};
|
| 85 |
+
|
| 86 |
+
// The result of this function will be used to set into TestOnlyInMemoryKmsClientFactory
|
| 87 |
+
// as the key mapping to look at.
|
| 88 |
+
std::unordered_map<std::string, SecureString> BuildKeyMap(const char* const* column_ids,
|
| 89 |
+
const char* const* column_keys,
|
| 90 |
+
const char* footer_id,
|
| 91 |
+
const char* footer_key);
|
| 92 |
+
|
| 93 |
+
// The result of this function will be used to set into EncryptionConfiguration
|
| 94 |
+
// as column keys.
|
| 95 |
+
std::string BuildColumnKeyMapping();
|
| 96 |
+
|
| 97 |
+
// FileEncryptor and FileDecryptor are helper classes to write/read an encrypted parquet
|
| 98 |
+
// file corresponding to each pair of FileEncryptionProperties/FileDecryptionProperties.
|
| 99 |
+
// FileEncryptor writes the file with fixed data values and FileDecryptor reads the file
|
| 100 |
+
// and verify the correctness of data values.
|
| 101 |
+
class FileEncryptor {
|
| 102 |
+
public:
|
| 103 |
+
FileEncryptor();
|
| 104 |
+
|
| 105 |
+
void EncryptFile(
|
| 106 |
+
std::string file,
|
| 107 |
+
std::shared_ptr<parquet::FileEncryptionProperties> encryption_configurations);
|
| 108 |
+
|
| 109 |
+
private:
|
| 110 |
+
std::shared_ptr<schema::GroupNode> SetupEncryptionSchema();
|
| 111 |
+
|
| 112 |
+
int num_rowgroups_ = 5;
|
| 113 |
+
int rows_per_rowgroup_ = 50;
|
| 114 |
+
std::shared_ptr<schema::GroupNode> schema_;
|
| 115 |
+
};
|
| 116 |
+
|
| 117 |
+
class FileDecryptor {
|
| 118 |
+
public:
|
| 119 |
+
void DecryptFile(
|
| 120 |
+
const std::string& file_name,
|
| 121 |
+
const std::shared_ptr<FileDecryptionProperties>& file_decryption_properties);
|
| 122 |
+
void DecryptPageIndex(
|
| 123 |
+
const std::string& file_name,
|
| 124 |
+
const std::shared_ptr<FileDecryptionProperties>& file_decryption_properties);
|
| 125 |
+
|
| 126 |
+
private:
|
| 127 |
+
void CheckFile(
|
| 128 |
+
parquet::ParquetFileReader* file_reader,
|
| 129 |
+
const std::shared_ptr<FileDecryptionProperties>& file_decryption_properties);
|
| 130 |
+
void CheckPageIndex(
|
| 131 |
+
parquet::ParquetFileReader* file_reader,
|
| 132 |
+
const std::shared_ptr<FileDecryptionProperties>& file_decryption_properties);
|
| 133 |
+
};
|
| 134 |
+
|
| 135 |
+
} // namespace encryption::test
|
| 136 |
+
} // namespace parquet
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/test_in_memory_kms.h
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <unordered_map>
|
| 21 |
+
|
| 22 |
+
#include "arrow/util/base64.h"
|
| 23 |
+
|
| 24 |
+
#include "parquet/encryption/kms_client_factory.h"
|
| 25 |
+
#include "parquet/encryption/local_wrap_kms_client.h"
|
| 26 |
+
#include "parquet/platform.h"
|
| 27 |
+
|
| 28 |
+
namespace parquet::encryption {
|
| 29 |
+
|
| 30 |
+
// This is a mock class, built for testing only. Don't use it as an example of
|
| 31 |
+
// LocalWrapKmsClient implementation.
|
| 32 |
+
class TestOnlyLocalWrapInMemoryKms : public LocalWrapKmsClient {
|
| 33 |
+
public:
|
| 34 |
+
explicit TestOnlyLocalWrapInMemoryKms(const KmsConnectionConfig& kms_connection_config);
|
| 35 |
+
|
| 36 |
+
static void InitializeMasterKeys(
|
| 37 |
+
const std::unordered_map<std::string, ::arrow::util::SecureString>&
|
| 38 |
+
master_keys_map);
|
| 39 |
+
|
| 40 |
+
protected:
|
| 41 |
+
const ::arrow::util::SecureString& GetMasterKeyFromServer(
|
| 42 |
+
const std::string& master_key_identifier) override;
|
| 43 |
+
|
| 44 |
+
private:
|
| 45 |
+
static std::unordered_map<std::string, ::arrow::util::SecureString> master_key_map_;
|
| 46 |
+
};
|
| 47 |
+
|
| 48 |
+
// This is a mock class, built for testing only. Don't use it as an example of KmsClient
|
| 49 |
+
// implementation.
|
| 50 |
+
class TestOnlyInServerWrapKms : public KmsClient {
|
| 51 |
+
public:
|
| 52 |
+
static void InitializeMasterKeys(
|
| 53 |
+
const std::unordered_map<std::string, ::arrow::util::SecureString>&
|
| 54 |
+
master_keys_map);
|
| 55 |
+
|
| 56 |
+
std::string WrapKey(const ::arrow::util::SecureString& key_bytes,
|
| 57 |
+
const std::string& master_key_identifier) override;
|
| 58 |
+
|
| 59 |
+
::arrow::util::SecureString UnwrapKey(
|
| 60 |
+
const std::string& wrapped_key, const std::string& master_key_identifier) override;
|
| 61 |
+
|
| 62 |
+
static void StartKeyRotation(
|
| 63 |
+
const std::unordered_map<std::string, ::arrow::util::SecureString>&
|
| 64 |
+
new_master_keys_map);
|
| 65 |
+
static void FinishKeyRotation();
|
| 66 |
+
|
| 67 |
+
private:
|
| 68 |
+
::arrow::util::SecureString GetMasterKeyFromServer(
|
| 69 |
+
const std::string& master_key_identifier);
|
| 70 |
+
|
| 71 |
+
// Different wrapping and unwrapping key maps to imitate versioning
|
| 72 |
+
// and support key rotation.
|
| 73 |
+
static std::unordered_map<std::string, ::arrow::util::SecureString>
|
| 74 |
+
unwrapping_master_key_map_;
|
| 75 |
+
static std::unordered_map<std::string, ::arrow::util::SecureString>
|
| 76 |
+
wrapping_master_key_map_;
|
| 77 |
+
};
|
| 78 |
+
|
| 79 |
+
// This is a mock class, built for testing only. Don't use it as an example of
|
| 80 |
+
// KmsClientFactory implementation.
|
| 81 |
+
class TestOnlyInMemoryKmsClientFactory : public KmsClientFactory {
|
| 82 |
+
public:
|
| 83 |
+
TestOnlyInMemoryKmsClientFactory(
|
| 84 |
+
bool wrap_locally,
|
| 85 |
+
const std::unordered_map<std::string, ::arrow::util::SecureString>& master_keys_map)
|
| 86 |
+
: KmsClientFactory(wrap_locally) {
|
| 87 |
+
TestOnlyLocalWrapInMemoryKms::InitializeMasterKeys(master_keys_map);
|
| 88 |
+
TestOnlyInServerWrapKms::InitializeMasterKeys(master_keys_map);
|
| 89 |
+
}
|
| 90 |
+
|
| 91 |
+
std::shared_ptr<KmsClient> CreateKmsClient(
|
| 92 |
+
const KmsConnectionConfig& kms_connection_config) {
|
| 93 |
+
if (wrap_locally_) {
|
| 94 |
+
return std::make_shared<TestOnlyLocalWrapInMemoryKms>(kms_connection_config);
|
| 95 |
+
} else {
|
| 96 |
+
return std::make_shared<TestOnlyInServerWrapKms>();
|
| 97 |
+
}
|
| 98 |
+
}
|
| 99 |
+
};
|
| 100 |
+
|
| 101 |
+
} // namespace parquet::encryption
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/two_level_cache_with_expiration.h
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <chrono>
|
| 21 |
+
#include <unordered_map>
|
| 22 |
+
|
| 23 |
+
#include "arrow/util/concurrent_map.h"
|
| 24 |
+
#include "arrow/util/mutex.h"
|
| 25 |
+
|
| 26 |
+
namespace parquet::encryption {
|
| 27 |
+
|
| 28 |
+
using ::arrow::util::ConcurrentMap;
|
| 29 |
+
|
| 30 |
+
namespace internal {
|
| 31 |
+
|
| 32 |
+
using TimePoint =
|
| 33 |
+
std::chrono::time_point<std::chrono::system_clock, std::chrono::duration<double>>;
|
| 34 |
+
|
| 35 |
+
inline TimePoint CurrentTimePoint() { return std::chrono::system_clock::now(); }
|
| 36 |
+
|
| 37 |
+
template <typename E>
|
| 38 |
+
class ExpiringCacheEntry {
|
| 39 |
+
public:
|
| 40 |
+
ExpiringCacheEntry() = default;
|
| 41 |
+
|
| 42 |
+
ExpiringCacheEntry(E cached_item, double expiration_interval_seconds)
|
| 43 |
+
: expiration_timestamp_(CurrentTimePoint() +
|
| 44 |
+
std::chrono::duration<double>(expiration_interval_seconds)),
|
| 45 |
+
cached_item_(std::move(cached_item)) {}
|
| 46 |
+
|
| 47 |
+
bool IsExpired() const {
|
| 48 |
+
const auto now = CurrentTimePoint();
|
| 49 |
+
return (now > expiration_timestamp_);
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
E cached_item() { return cached_item_; }
|
| 53 |
+
|
| 54 |
+
private:
|
| 55 |
+
const TimePoint expiration_timestamp_;
|
| 56 |
+
E cached_item_;
|
| 57 |
+
};
|
| 58 |
+
|
| 59 |
+
// This class is to avoid the below warning when compiling KeyToolkit class with VS2015
|
| 60 |
+
// warning C4503: decorated name length exceeded, name was truncated
|
| 61 |
+
template <typename V>
|
| 62 |
+
class ExpiringCacheMapEntry {
|
| 63 |
+
public:
|
| 64 |
+
ExpiringCacheMapEntry() = default;
|
| 65 |
+
|
| 66 |
+
explicit ExpiringCacheMapEntry(
|
| 67 |
+
std::shared_ptr<ConcurrentMap<std::string, V>> cached_item,
|
| 68 |
+
double expiration_interval_seconds)
|
| 69 |
+
: map_cache_(cached_item, expiration_interval_seconds) {}
|
| 70 |
+
|
| 71 |
+
bool IsExpired() { return map_cache_.IsExpired(); }
|
| 72 |
+
|
| 73 |
+
std::shared_ptr<ConcurrentMap<std::string, V>> cached_item() {
|
| 74 |
+
return map_cache_.cached_item();
|
| 75 |
+
}
|
| 76 |
+
|
| 77 |
+
private:
|
| 78 |
+
// ConcurrentMap object may be accessed and modified at many places at the same time,
|
| 79 |
+
// from multiple threads, or even removed from cache.
|
| 80 |
+
ExpiringCacheEntry<std::shared_ptr<ConcurrentMap<std::string, V>>> map_cache_;
|
| 81 |
+
};
|
| 82 |
+
|
| 83 |
+
} // namespace internal
|
| 84 |
+
|
| 85 |
+
// Two-level cache with expiration of internal caches according to token lifetime.
|
| 86 |
+
// External cache is per token, internal is per string key.
|
| 87 |
+
// Wrapper class around:
|
| 88 |
+
// std::unordered_map<std::string,
|
| 89 |
+
// internal::ExpiringCacheEntry<std::unordered_map<std::string, V>>>
|
| 90 |
+
// This cache is safe to be shared between threads.
|
| 91 |
+
template <typename V>
|
| 92 |
+
class TwoLevelCacheWithExpiration {
|
| 93 |
+
public:
|
| 94 |
+
TwoLevelCacheWithExpiration() {
|
| 95 |
+
last_cache_cleanup_timestamp_ = internal::CurrentTimePoint();
|
| 96 |
+
}
|
| 97 |
+
|
| 98 |
+
std::shared_ptr<ConcurrentMap<std::string, V>> GetOrCreateInternalCache(
|
| 99 |
+
const std::string& access_token, double cache_entry_lifetime_seconds) {
|
| 100 |
+
auto lock = mutex_.Lock();
|
| 101 |
+
|
| 102 |
+
auto external_cache_entry = cache_.find(access_token);
|
| 103 |
+
if (external_cache_entry == cache_.end() ||
|
| 104 |
+
external_cache_entry->second.IsExpired()) {
|
| 105 |
+
cache_.insert({access_token, internal::ExpiringCacheMapEntry<V>(
|
| 106 |
+
std::make_shared<ConcurrentMap<std::string, V>>(),
|
| 107 |
+
cache_entry_lifetime_seconds)});
|
| 108 |
+
}
|
| 109 |
+
|
| 110 |
+
return cache_[access_token].cached_item();
|
| 111 |
+
}
|
| 112 |
+
|
| 113 |
+
void CheckCacheForExpiredTokens(double cache_cleanup_period_seconds = 0.0) {
|
| 114 |
+
auto lock = mutex_.Lock();
|
| 115 |
+
|
| 116 |
+
const auto now = internal::CurrentTimePoint();
|
| 117 |
+
if (now > (last_cache_cleanup_timestamp_ +
|
| 118 |
+
std::chrono::duration<double>(cache_cleanup_period_seconds))) {
|
| 119 |
+
RemoveExpiredEntriesNoMutex();
|
| 120 |
+
last_cache_cleanup_timestamp_ = now;
|
| 121 |
+
}
|
| 122 |
+
}
|
| 123 |
+
|
| 124 |
+
void Remove(const std::string& access_token) {
|
| 125 |
+
auto lock = mutex_.Lock();
|
| 126 |
+
cache_.erase(access_token);
|
| 127 |
+
}
|
| 128 |
+
|
| 129 |
+
void Clear() {
|
| 130 |
+
auto lock = mutex_.Lock();
|
| 131 |
+
cache_.clear();
|
| 132 |
+
}
|
| 133 |
+
|
| 134 |
+
private:
|
| 135 |
+
void RemoveExpiredEntriesNoMutex() {
|
| 136 |
+
for (auto it = cache_.begin(); it != cache_.end();) {
|
| 137 |
+
if (it->second.IsExpired()) {
|
| 138 |
+
it = cache_.erase(it);
|
| 139 |
+
} else {
|
| 140 |
+
++it;
|
| 141 |
+
}
|
| 142 |
+
}
|
| 143 |
+
}
|
| 144 |
+
std::unordered_map<std::string, internal::ExpiringCacheMapEntry<V>> cache_;
|
| 145 |
+
internal::TimePoint last_cache_cleanup_timestamp_;
|
| 146 |
+
::arrow::util::Mutex mutex_;
|
| 147 |
+
};
|
| 148 |
+
|
| 149 |
+
} // namespace parquet::encryption
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/type_fwd.h
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
namespace parquet {
|
| 21 |
+
|
| 22 |
+
class Decryptor;
|
| 23 |
+
class Encryptor;
|
| 24 |
+
|
| 25 |
+
class InternalFileDecryptor;
|
| 26 |
+
class InternalFileEncryptor;
|
| 27 |
+
|
| 28 |
+
} // namespace parquet
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/exception.h
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <exception>
|
| 21 |
+
#include <sstream>
|
| 22 |
+
#include <string>
|
| 23 |
+
#include <utility>
|
| 24 |
+
|
| 25 |
+
#include "arrow/type_fwd.h"
|
| 26 |
+
#include "arrow/util/string_util.h"
|
| 27 |
+
#include "parquet/platform.h"
|
| 28 |
+
|
| 29 |
+
#ifdef _MSC_VER
|
| 30 |
+
# pragma warning(push)
|
| 31 |
+
// Disable warning for STL types usage in DLL interface
|
| 32 |
+
// https://web.archive.org/web/20130317015847/http://connect.microsoft.com/VisualStudio/feedback/details/696593/vc-10-vs-2010-basic-string-exports
|
| 33 |
+
# pragma warning(disable : 4275 4251)
|
| 34 |
+
// Disable diamond inheritance warnings
|
| 35 |
+
# pragma warning(disable : 4250)
|
| 36 |
+
// Disable macro redefinition warnings
|
| 37 |
+
# pragma warning(disable : 4005)
|
| 38 |
+
// Disable extern before exported template warnings
|
| 39 |
+
# pragma warning(disable : 4910)
|
| 40 |
+
#endif
|
| 41 |
+
|
| 42 |
+
// PARQUET-1085
|
| 43 |
+
#if !defined(ARROW_UNUSED)
|
| 44 |
+
# define ARROW_UNUSED(x) UNUSED(x)
|
| 45 |
+
#endif
|
| 46 |
+
|
| 47 |
+
// Parquet exception to Arrow Status
|
| 48 |
+
|
| 49 |
+
#define BEGIN_PARQUET_CATCH_EXCEPTIONS try {
|
| 50 |
+
#define END_PARQUET_CATCH_EXCEPTIONS \
|
| 51 |
+
} \
|
| 52 |
+
catch (const ::parquet::ParquetStatusException& e) { \
|
| 53 |
+
return e.status(); \
|
| 54 |
+
} \
|
| 55 |
+
catch (const ::parquet::ParquetException& e) { \
|
| 56 |
+
return ::arrow::Status::IOError(e.what()); \
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
// clang-format off
|
| 60 |
+
|
| 61 |
+
#define PARQUET_CATCH_NOT_OK(s) \
|
| 62 |
+
BEGIN_PARQUET_CATCH_EXCEPTIONS \
|
| 63 |
+
(s); \
|
| 64 |
+
END_PARQUET_CATCH_EXCEPTIONS
|
| 65 |
+
|
| 66 |
+
// clang-format on
|
| 67 |
+
|
| 68 |
+
#define PARQUET_CATCH_AND_RETURN(s) \
|
| 69 |
+
BEGIN_PARQUET_CATCH_EXCEPTIONS \
|
| 70 |
+
return (s); \
|
| 71 |
+
END_PARQUET_CATCH_EXCEPTIONS
|
| 72 |
+
|
| 73 |
+
// Arrow Status to Parquet exception
|
| 74 |
+
|
| 75 |
+
#define PARQUET_IGNORE_NOT_OK(s) \
|
| 76 |
+
do { \
|
| 77 |
+
::arrow::Status _s = ::arrow::ToStatus(s); \
|
| 78 |
+
ARROW_UNUSED(_s); \
|
| 79 |
+
} while (0)
|
| 80 |
+
|
| 81 |
+
#define PARQUET_THROW_NOT_OK(s) \
|
| 82 |
+
do { \
|
| 83 |
+
::arrow::Status _s = ::arrow::ToStatus(s); \
|
| 84 |
+
if (!_s.ok()) { \
|
| 85 |
+
throw ::parquet::ParquetStatusException(std::move(_s)); \
|
| 86 |
+
} \
|
| 87 |
+
} while (0)
|
| 88 |
+
|
| 89 |
+
#define PARQUET_ASSIGN_OR_THROW_IMPL(status_name, lhs, rexpr) \
|
| 90 |
+
auto status_name = (rexpr); \
|
| 91 |
+
PARQUET_THROW_NOT_OK(status_name.status()); \
|
| 92 |
+
lhs = std::move(status_name).ValueOrDie();
|
| 93 |
+
|
| 94 |
+
#define PARQUET_ASSIGN_OR_THROW(lhs, rexpr) \
|
| 95 |
+
PARQUET_ASSIGN_OR_THROW_IMPL(ARROW_ASSIGN_OR_RAISE_NAME(_error_or_value, __COUNTER__), \
|
| 96 |
+
lhs, rexpr);
|
| 97 |
+
|
| 98 |
+
namespace parquet {
|
| 99 |
+
|
| 100 |
+
class PARQUET_EXPORT ParquetException : public std::exception {
|
| 101 |
+
public:
|
| 102 |
+
PARQUET_NORETURN static void EofException(const std::string& msg = "") {
|
| 103 |
+
static std::string prefix = "Unexpected end of stream";
|
| 104 |
+
if (msg.empty()) {
|
| 105 |
+
throw ParquetException(prefix);
|
| 106 |
+
}
|
| 107 |
+
throw ParquetException(prefix, ": ", msg);
|
| 108 |
+
}
|
| 109 |
+
|
| 110 |
+
PARQUET_NORETURN static void NYI(const std::string& msg = "") {
|
| 111 |
+
throw ParquetException("Not yet implemented: ", msg, ".");
|
| 112 |
+
}
|
| 113 |
+
|
| 114 |
+
template <typename... Args>
|
| 115 |
+
explicit ParquetException(Args&&... args)
|
| 116 |
+
: msg_(::arrow::internal::JoinToString(std::forward<Args>(args)...)) {}
|
| 117 |
+
|
| 118 |
+
explicit ParquetException(std::string msg) : msg_(std::move(msg)) {}
|
| 119 |
+
|
| 120 |
+
explicit ParquetException(const char* msg, const std::exception&) : msg_(msg) {}
|
| 121 |
+
|
| 122 |
+
ParquetException(const ParquetException&) = default;
|
| 123 |
+
ParquetException& operator=(const ParquetException&) = default;
|
| 124 |
+
ParquetException(ParquetException&&) = default;
|
| 125 |
+
ParquetException& operator=(ParquetException&&) = default;
|
| 126 |
+
|
| 127 |
+
const char* what() const noexcept override { return msg_.c_str(); }
|
| 128 |
+
|
| 129 |
+
private:
|
| 130 |
+
std::string msg_;
|
| 131 |
+
};
|
| 132 |
+
|
| 133 |
+
// Support printing a ParquetException.
|
| 134 |
+
// This is needed for clang-on-MSVC as there operator<< is not defined for
|
| 135 |
+
// std::exception.
|
| 136 |
+
PARQUET_EXPORT
|
| 137 |
+
std::ostream& operator<<(std::ostream& os, const ParquetException& exception);
|
| 138 |
+
|
| 139 |
+
class ParquetStatusException : public ParquetException {
|
| 140 |
+
public:
|
| 141 |
+
explicit ParquetStatusException(::arrow::Status status)
|
| 142 |
+
: ParquetException(status.ToString()), status_(std::move(status)) {}
|
| 143 |
+
|
| 144 |
+
const ::arrow::Status& status() const { return status_; }
|
| 145 |
+
|
| 146 |
+
private:
|
| 147 |
+
::arrow::Status status_;
|
| 148 |
+
};
|
| 149 |
+
|
| 150 |
+
// This class exists for the purpose of detecting an invalid or corrupted file.
|
| 151 |
+
class ParquetInvalidOrCorruptedFileException : public ParquetStatusException {
|
| 152 |
+
public:
|
| 153 |
+
ParquetInvalidOrCorruptedFileException(const ParquetInvalidOrCorruptedFileException&) =
|
| 154 |
+
default;
|
| 155 |
+
|
| 156 |
+
template <typename Arg,
|
| 157 |
+
typename std::enable_if<
|
| 158 |
+
!std::is_base_of<ParquetInvalidOrCorruptedFileException, Arg>::value,
|
| 159 |
+
int>::type = 0,
|
| 160 |
+
typename... Args>
|
| 161 |
+
explicit ParquetInvalidOrCorruptedFileException(Arg arg, Args&&... args)
|
| 162 |
+
: ParquetStatusException(::arrow::Status::Invalid(std::forward<Arg>(arg),
|
| 163 |
+
std::forward<Args>(args)...)) {}
|
| 164 |
+
};
|
| 165 |
+
|
| 166 |
+
template <typename StatusReturnBlock>
|
| 167 |
+
void ThrowNotOk(StatusReturnBlock&& b) {
|
| 168 |
+
PARQUET_THROW_NOT_OK(b());
|
| 169 |
+
}
|
| 170 |
+
|
| 171 |
+
} // namespace parquet
|
| 172 |
+
|
| 173 |
+
#ifdef _MSC_VER
|
| 174 |
+
# pragma warning(pop)
|
| 175 |
+
#endif
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/file_reader.h
ADDED
|
@@ -0,0 +1,257 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <cstdint>
|
| 21 |
+
#include <memory>
|
| 22 |
+
#include <string>
|
| 23 |
+
#include <vector>
|
| 24 |
+
|
| 25 |
+
#include "arrow/io/caching.h"
|
| 26 |
+
#include "arrow/util/type_fwd.h"
|
| 27 |
+
#include "parquet/metadata.h" // IWYU pragma: keep
|
| 28 |
+
#include "parquet/platform.h"
|
| 29 |
+
#include "parquet/properties.h"
|
| 30 |
+
|
| 31 |
+
namespace parquet {
|
| 32 |
+
|
| 33 |
+
class ColumnReader;
|
| 34 |
+
class FileMetaData;
|
| 35 |
+
class PageIndexReader;
|
| 36 |
+
class BloomFilterReader;
|
| 37 |
+
class PageReader;
|
| 38 |
+
class RowGroupMetaData;
|
| 39 |
+
|
| 40 |
+
namespace internal {
|
| 41 |
+
class RecordReader;
|
| 42 |
+
}
|
| 43 |
+
|
| 44 |
+
class PARQUET_EXPORT RowGroupReader {
|
| 45 |
+
public:
|
| 46 |
+
// Forward declare a virtual class 'Contents' to aid dependency injection and more
|
| 47 |
+
// easily create test fixtures
|
| 48 |
+
// An implementation of the Contents class is defined in the .cc file
|
| 49 |
+
struct Contents {
|
| 50 |
+
virtual ~Contents() {}
|
| 51 |
+
virtual std::unique_ptr<PageReader> GetColumnPageReader(int i) = 0;
|
| 52 |
+
virtual const RowGroupMetaData* metadata() const = 0;
|
| 53 |
+
virtual const ReaderProperties* properties() const = 0;
|
| 54 |
+
};
|
| 55 |
+
|
| 56 |
+
explicit RowGroupReader(std::unique_ptr<Contents> contents);
|
| 57 |
+
|
| 58 |
+
// Returns the rowgroup metadata
|
| 59 |
+
const RowGroupMetaData* metadata() const;
|
| 60 |
+
|
| 61 |
+
// Construct a ColumnReader for the indicated row group-relative
|
| 62 |
+
// column. Ownership is shared with the RowGroupReader.
|
| 63 |
+
std::shared_ptr<ColumnReader> Column(int i);
|
| 64 |
+
|
| 65 |
+
// EXPERIMENTAL: Construct a RecordReader for the indicated column of the row group.
|
| 66 |
+
// Ownership is shared with the RowGroupReader.
|
| 67 |
+
std::shared_ptr<internal::RecordReader> RecordReader(int i,
|
| 68 |
+
bool read_dictionary = false);
|
| 69 |
+
|
| 70 |
+
// Construct a ColumnReader, trying to enable exposed encoding.
|
| 71 |
+
//
|
| 72 |
+
// For dictionary encoding, currently we only support column chunks that are fully
|
| 73 |
+
// dictionary encoded, i.e., all data pages in the column chunk are dictionary encoded.
|
| 74 |
+
// If a column chunk uses dictionary encoding but then falls back to plain encoding, the
|
| 75 |
+
// encoding will not be exposed.
|
| 76 |
+
//
|
| 77 |
+
// The returned column reader provides an API GetExposedEncoding() for the
|
| 78 |
+
// users to check the exposed encoding and determine how to read the batches.
|
| 79 |
+
//
|
| 80 |
+
// \note API EXPERIMENTAL
|
| 81 |
+
std::shared_ptr<ColumnReader> ColumnWithExposeEncoding(
|
| 82 |
+
int i, ExposedEncoding encoding_to_expose);
|
| 83 |
+
|
| 84 |
+
// Construct a RecordReader, trying to enable exposed encoding.
|
| 85 |
+
//
|
| 86 |
+
// For dictionary encoding, currently we only support column chunks that are
|
| 87 |
+
// fully dictionary encoded byte arrays. The caller should verify if the reader can read
|
| 88 |
+
// and expose the dictionary by checking the reader's read_dictionary(). If a column
|
| 89 |
+
// chunk uses dictionary encoding but then falls back to plain encoding, the returned
|
| 90 |
+
// reader will read decoded data without exposing the dictionary.
|
| 91 |
+
//
|
| 92 |
+
// \note API EXPERIMENTAL
|
| 93 |
+
std::shared_ptr<internal::RecordReader> RecordReaderWithExposeEncoding(
|
| 94 |
+
int i, ExposedEncoding encoding_to_expose);
|
| 95 |
+
|
| 96 |
+
std::unique_ptr<PageReader> GetColumnPageReader(int i);
|
| 97 |
+
|
| 98 |
+
private:
|
| 99 |
+
// Holds a pointer to an instance of Contents implementation
|
| 100 |
+
std::unique_ptr<Contents> contents_;
|
| 101 |
+
};
|
| 102 |
+
|
| 103 |
+
class PARQUET_EXPORT ParquetFileReader {
|
| 104 |
+
public:
|
| 105 |
+
// Declare a virtual class 'Contents' to aid dependency injection and more
|
| 106 |
+
// easily create test fixtures
|
| 107 |
+
// An implementation of the Contents class is defined in the .cc file
|
| 108 |
+
struct PARQUET_EXPORT Contents {
|
| 109 |
+
static std::unique_ptr<Contents> Open(
|
| 110 |
+
std::shared_ptr<::arrow::io::RandomAccessFile> source,
|
| 111 |
+
const ReaderProperties& props = default_reader_properties(),
|
| 112 |
+
std::shared_ptr<FileMetaData> metadata = NULLPTR);
|
| 113 |
+
|
| 114 |
+
static ::arrow::Future<std::unique_ptr<Contents>> OpenAsync(
|
| 115 |
+
std::shared_ptr<::arrow::io::RandomAccessFile> source,
|
| 116 |
+
const ReaderProperties& props = default_reader_properties(),
|
| 117 |
+
std::shared_ptr<FileMetaData> metadata = NULLPTR);
|
| 118 |
+
|
| 119 |
+
virtual ~Contents() = default;
|
| 120 |
+
// Perform any cleanup associated with the file contents
|
| 121 |
+
virtual void Close() = 0;
|
| 122 |
+
virtual std::shared_ptr<RowGroupReader> GetRowGroup(int i) = 0;
|
| 123 |
+
virtual std::shared_ptr<FileMetaData> metadata() const = 0;
|
| 124 |
+
virtual std::shared_ptr<PageIndexReader> GetPageIndexReader() = 0;
|
| 125 |
+
virtual BloomFilterReader& GetBloomFilterReader() = 0;
|
| 126 |
+
};
|
| 127 |
+
|
| 128 |
+
ParquetFileReader();
|
| 129 |
+
~ParquetFileReader();
|
| 130 |
+
|
| 131 |
+
// Create a file reader instance from an Arrow file object. Thread-safety is
|
| 132 |
+
// the responsibility of the file implementation
|
| 133 |
+
static std::unique_ptr<ParquetFileReader> Open(
|
| 134 |
+
std::shared_ptr<::arrow::io::RandomAccessFile> source,
|
| 135 |
+
const ReaderProperties& props = default_reader_properties(),
|
| 136 |
+
std::shared_ptr<FileMetaData> metadata = NULLPTR);
|
| 137 |
+
|
| 138 |
+
// API Convenience to open a serialized Parquet file on disk, using Arrow IO
|
| 139 |
+
// interfaces.
|
| 140 |
+
static std::unique_ptr<ParquetFileReader> OpenFile(
|
| 141 |
+
const std::string& path, bool memory_map = false,
|
| 142 |
+
const ReaderProperties& props = default_reader_properties(),
|
| 143 |
+
std::shared_ptr<FileMetaData> metadata = NULLPTR);
|
| 144 |
+
|
| 145 |
+
// Asynchronously open a file reader from an Arrow file object.
|
| 146 |
+
// Does not throw - all errors are reported through the Future.
|
| 147 |
+
static ::arrow::Future<std::unique_ptr<ParquetFileReader>> OpenAsync(
|
| 148 |
+
std::shared_ptr<::arrow::io::RandomAccessFile> source,
|
| 149 |
+
const ReaderProperties& props = default_reader_properties(),
|
| 150 |
+
std::shared_ptr<FileMetaData> metadata = NULLPTR);
|
| 151 |
+
|
| 152 |
+
void Open(std::unique_ptr<Contents> contents);
|
| 153 |
+
void Close();
|
| 154 |
+
|
| 155 |
+
// The RowGroupReader is owned by the FileReader
|
| 156 |
+
std::shared_ptr<RowGroupReader> RowGroup(int i);
|
| 157 |
+
|
| 158 |
+
// Returns the file metadata. Only one instance is ever created
|
| 159 |
+
std::shared_ptr<FileMetaData> metadata() const;
|
| 160 |
+
|
| 161 |
+
/// Returns the PageIndexReader. Only one instance is ever created.
|
| 162 |
+
///
|
| 163 |
+
/// If the file does not have the page index, nullptr may be returned.
|
| 164 |
+
/// Because it pays to check existence of page index in the file, it
|
| 165 |
+
/// is possible to return a non null value even if page index does
|
| 166 |
+
/// not exist. It is the caller's responsibility to check the return
|
| 167 |
+
/// value and follow-up calls to PageIndexReader.
|
| 168 |
+
///
|
| 169 |
+
/// WARNING: The returned PageIndexReader must not outlive the ParquetFileReader.
|
| 170 |
+
/// Initialize GetPageIndexReader() is not thread-safety.
|
| 171 |
+
std::shared_ptr<PageIndexReader> GetPageIndexReader();
|
| 172 |
+
|
| 173 |
+
/// Returns the BloomFilterReader. Only one instance is ever created.
|
| 174 |
+
///
|
| 175 |
+
/// WARNING: The returned BloomFilterReader must not outlive the ParquetFileReader.
|
| 176 |
+
/// Initialize GetBloomFilterReader() is not thread-safety.
|
| 177 |
+
BloomFilterReader& GetBloomFilterReader();
|
| 178 |
+
|
| 179 |
+
/// Pre-buffer the specified column indices in all row groups.
|
| 180 |
+
///
|
| 181 |
+
/// Readers can optionally call this to cache the necessary slices
|
| 182 |
+
/// of the file in-memory before deserialization. Arrow readers can
|
| 183 |
+
/// automatically do this via an option. This is intended to
|
| 184 |
+
/// increase performance when reading from high-latency filesystems
|
| 185 |
+
/// (e.g. Amazon S3).
|
| 186 |
+
///
|
| 187 |
+
/// After calling this, creating readers for row groups/column
|
| 188 |
+
/// indices that were not buffered may fail. Creating multiple
|
| 189 |
+
/// readers for the a subset of the buffered regions is
|
| 190 |
+
/// acceptable. This may be called again to buffer a different set
|
| 191 |
+
/// of row groups/columns.
|
| 192 |
+
///
|
| 193 |
+
/// If memory usage is a concern, note that data will remain
|
| 194 |
+
/// buffered in memory until either \a PreBuffer() is called again,
|
| 195 |
+
/// or the reader itself is destructed. Reading - and buffering -
|
| 196 |
+
/// only one row group at a time may be useful.
|
| 197 |
+
///
|
| 198 |
+
/// This method may throw.
|
| 199 |
+
void PreBuffer(const std::vector<int>& row_groups,
|
| 200 |
+
const std::vector<int>& column_indices,
|
| 201 |
+
const ::arrow::io::IOContext& ctx,
|
| 202 |
+
const ::arrow::io::CacheOptions& options);
|
| 203 |
+
|
| 204 |
+
/// Retrieve the list of byte ranges that would need to be read to retrieve
|
| 205 |
+
/// the data for the specified row groups and column indices.
|
| 206 |
+
///
|
| 207 |
+
/// A reader can optionally call this if they wish to handle their own
|
| 208 |
+
/// caching and management of file reads (or offload them to other readers).
|
| 209 |
+
/// Unlike PreBuffer, this method will not perform any actual caching or
|
| 210 |
+
/// reads, instead just using the file metadata to determine the byte ranges
|
| 211 |
+
/// that would need to be read if you were to consume the entirety of the column
|
| 212 |
+
/// chunks for the provided columns in the specified row groups.
|
| 213 |
+
///
|
| 214 |
+
/// If row_groups or column_indices are empty, then the result of this will be empty.
|
| 215 |
+
///
|
| 216 |
+
/// hole_size_limit represents the maximum distance, in bytes, between two
|
| 217 |
+
/// consecutive ranges; beyond this value, ranges will not be combined. The default
|
| 218 |
+
/// value is 1MB.
|
| 219 |
+
///
|
| 220 |
+
/// range_size_limit is the maximum size in bytes of a combined range; if combining
|
| 221 |
+
/// two consecutive ranges would produce a range larger than this, they are not
|
| 222 |
+
/// combined. The default values is 64MB. This *must* be larger than hole_size_limit.
|
| 223 |
+
///
|
| 224 |
+
/// This will not take into account page indexes or any other predicate push down
|
| 225 |
+
/// benefits that may be available.
|
| 226 |
+
::arrow::Result<std::vector<::arrow::io::ReadRange>> GetReadRanges(
|
| 227 |
+
const std::vector<int>& row_groups, const std::vector<int>& column_indices,
|
| 228 |
+
int64_t hole_size_limit = 1024 * 1024, int64_t range_size_limit = 64 * 1024 * 1024);
|
| 229 |
+
|
| 230 |
+
/// Wait for the specified row groups and column indices to be pre-buffered.
|
| 231 |
+
///
|
| 232 |
+
/// After the returned Future completes, reading the specified row
|
| 233 |
+
/// groups/columns will not block.
|
| 234 |
+
///
|
| 235 |
+
/// PreBuffer must be called first. This method does not throw.
|
| 236 |
+
::arrow::Future<> WhenBuffered(const std::vector<int>& row_groups,
|
| 237 |
+
const std::vector<int>& column_indices) const;
|
| 238 |
+
|
| 239 |
+
private:
|
| 240 |
+
// Holds a pointer to an instance of Contents implementation
|
| 241 |
+
std::unique_ptr<Contents> contents_;
|
| 242 |
+
};
|
| 243 |
+
|
| 244 |
+
// Read only Parquet file metadata
|
| 245 |
+
std::shared_ptr<FileMetaData> PARQUET_EXPORT
|
| 246 |
+
ReadMetaData(const std::shared_ptr<::arrow::io::RandomAccessFile>& source);
|
| 247 |
+
|
| 248 |
+
/// \brief Scan all values in file. Useful for performance testing
|
| 249 |
+
/// \param[in] columns the column numbers to scan. If empty scans all
|
| 250 |
+
/// \param[in] column_batch_size number of values to read at a time when scanning column
|
| 251 |
+
/// \param[in] reader a ParquetFileReader instance
|
| 252 |
+
/// \return number of semantic rows in file
|
| 253 |
+
PARQUET_EXPORT
|
| 254 |
+
int64_t ScanFileContents(std::vector<int> columns, const int32_t column_batch_size,
|
| 255 |
+
ParquetFileReader* reader);
|
| 256 |
+
|
| 257 |
+
} // namespace parquet
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/file_writer.h
ADDED
|
@@ -0,0 +1,260 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <cstdint>
|
| 21 |
+
#include <memory>
|
| 22 |
+
#include <utility>
|
| 23 |
+
|
| 24 |
+
#include "parquet/metadata.h"
|
| 25 |
+
#include "parquet/platform.h"
|
| 26 |
+
#include "parquet/properties.h"
|
| 27 |
+
#include "parquet/schema.h"
|
| 28 |
+
|
| 29 |
+
namespace parquet {
|
| 30 |
+
|
| 31 |
+
class ColumnWriter;
|
| 32 |
+
|
| 33 |
+
// FIXME: copied from reader-internal.cc
|
| 34 |
+
static constexpr uint8_t kParquetMagic[4] = {'P', 'A', 'R', '1'};
|
| 35 |
+
static constexpr uint8_t kParquetEMagic[4] = {'P', 'A', 'R', 'E'};
|
| 36 |
+
|
| 37 |
+
class PARQUET_EXPORT RowGroupWriter {
|
| 38 |
+
public:
|
| 39 |
+
// Estimated uncompressed byte sizes of data buffered by column writers
|
| 40 |
+
// that have not yet been serialized into pages.
|
| 41 |
+
struct BufferedStats {
|
| 42 |
+
int64_t def_level_bytes = 0;
|
| 43 |
+
int64_t rep_level_bytes = 0;
|
| 44 |
+
int64_t value_bytes = 0;
|
| 45 |
+
int64_t dict_bytes = 0;
|
| 46 |
+
};
|
| 47 |
+
|
| 48 |
+
// Forward declare a virtual class 'Contents' to aid dependency injection and more
|
| 49 |
+
// easily create test fixtures
|
| 50 |
+
// An implementation of the Contents class is defined in the .cc file
|
| 51 |
+
struct Contents {
|
| 52 |
+
virtual ~Contents() = default;
|
| 53 |
+
virtual int num_columns() const = 0;
|
| 54 |
+
virtual int64_t num_rows() const = 0;
|
| 55 |
+
|
| 56 |
+
// to be used only with ParquetFileWriter::AppendRowGroup
|
| 57 |
+
virtual ColumnWriter* NextColumn() = 0;
|
| 58 |
+
// to be used only with ParquetFileWriter::AppendBufferedRowGroup
|
| 59 |
+
virtual ColumnWriter* column(int i) = 0;
|
| 60 |
+
|
| 61 |
+
virtual int current_column() const = 0;
|
| 62 |
+
virtual void Close() = 0;
|
| 63 |
+
|
| 64 |
+
/// \brief total uncompressed bytes written by the page writer
|
| 65 |
+
virtual int64_t total_bytes_written() const = 0;
|
| 66 |
+
/// \brief total bytes still compressed but not written by the page writer
|
| 67 |
+
virtual int64_t total_compressed_bytes() const = 0;
|
| 68 |
+
/// \brief total compressed bytes written by the page writer
|
| 69 |
+
virtual int64_t total_compressed_bytes_written() const = 0;
|
| 70 |
+
/// \brief Estimated sizes of buffered data (levels, values, dict) not yet
|
| 71 |
+
/// written to pages.
|
| 72 |
+
virtual BufferedStats EstimatedBufferedStats() const = 0;
|
| 73 |
+
|
| 74 |
+
virtual bool buffered() const = 0;
|
| 75 |
+
};
|
| 76 |
+
|
| 77 |
+
explicit RowGroupWriter(std::unique_ptr<Contents> contents);
|
| 78 |
+
|
| 79 |
+
/// Construct a ColumnWriter for the indicated row group-relative column.
|
| 80 |
+
///
|
| 81 |
+
/// To be used only with ParquetFileWriter::AppendRowGroup
|
| 82 |
+
/// Ownership is solely within the RowGroupWriter. The ColumnWriter is only
|
| 83 |
+
/// valid until the next call to NextColumn or Close. As the contents are
|
| 84 |
+
/// directly written to the sink, once a new column is started, the contents
|
| 85 |
+
/// of the previous one cannot be modified anymore.
|
| 86 |
+
ColumnWriter* NextColumn();
|
| 87 |
+
/// Index of currently written column. Equal to -1 if NextColumn()
|
| 88 |
+
/// has not been called yet.
|
| 89 |
+
int current_column();
|
| 90 |
+
void Close();
|
| 91 |
+
|
| 92 |
+
int num_columns() const;
|
| 93 |
+
|
| 94 |
+
/// Construct a ColumnWriter for the indicated row group column.
|
| 95 |
+
///
|
| 96 |
+
/// To be used only with ParquetFileWriter::AppendBufferedRowGroup
|
| 97 |
+
/// Ownership is solely within the RowGroupWriter. The ColumnWriter is
|
| 98 |
+
/// valid until Close. The contents are buffered in memory and written to sink
|
| 99 |
+
/// on Close
|
| 100 |
+
ColumnWriter* column(int i);
|
| 101 |
+
|
| 102 |
+
/**
|
| 103 |
+
* Number of rows that shall be written as part of this RowGroup.
|
| 104 |
+
*/
|
| 105 |
+
int64_t num_rows() const;
|
| 106 |
+
|
| 107 |
+
/// \brief total uncompressed bytes written by the page writer
|
| 108 |
+
int64_t total_bytes_written() const;
|
| 109 |
+
/// \brief total bytes still compressed but not written by the page writer.
|
| 110 |
+
/// It will always return 0 from the SerializedPageWriter.
|
| 111 |
+
int64_t total_compressed_bytes() const;
|
| 112 |
+
/// \brief total compressed bytes written by the page writer
|
| 113 |
+
int64_t total_compressed_bytes_written() const;
|
| 114 |
+
/// \brief Estimated sizes of buffered data (levels, values, dict) not yet
|
| 115 |
+
/// written to pages.
|
| 116 |
+
BufferedStats estimated_buffered_stats() const;
|
| 117 |
+
|
| 118 |
+
/// Returns whether the current RowGroupWriter is in the buffered mode and is created
|
| 119 |
+
/// by calling ParquetFileWriter::AppendBufferedRowGroup.
|
| 120 |
+
bool buffered() const;
|
| 121 |
+
|
| 122 |
+
private:
|
| 123 |
+
// Holds a pointer to an instance of Contents implementation
|
| 124 |
+
std::unique_ptr<Contents> contents_;
|
| 125 |
+
};
|
| 126 |
+
|
| 127 |
+
PARQUET_EXPORT
|
| 128 |
+
void WriteFileMetaData(const FileMetaData& file_metadata,
|
| 129 |
+
::arrow::io::OutputStream* sink);
|
| 130 |
+
|
| 131 |
+
PARQUET_EXPORT
|
| 132 |
+
void WriteMetaDataFile(const FileMetaData& file_metadata,
|
| 133 |
+
::arrow::io::OutputStream* sink);
|
| 134 |
+
|
| 135 |
+
PARQUET_EXPORT
|
| 136 |
+
void WriteEncryptedFileMetadata(const FileMetaData& file_metadata,
|
| 137 |
+
ArrowOutputStream* sink,
|
| 138 |
+
const std::shared_ptr<Encryptor>& encryptor,
|
| 139 |
+
bool encrypt_footer);
|
| 140 |
+
|
| 141 |
+
PARQUET_EXPORT
|
| 142 |
+
void WriteEncryptedFileMetadata(const FileMetaData& file_metadata,
|
| 143 |
+
::arrow::io::OutputStream* sink,
|
| 144 |
+
const std::shared_ptr<Encryptor>& encryptor = NULLPTR,
|
| 145 |
+
bool encrypt_footer = false);
|
| 146 |
+
PARQUET_EXPORT
|
| 147 |
+
void WriteFileCryptoMetaData(const FileCryptoMetaData& crypto_metadata,
|
| 148 |
+
::arrow::io::OutputStream* sink);
|
| 149 |
+
|
| 150 |
+
class PARQUET_EXPORT ParquetFileWriter {
|
| 151 |
+
public:
|
| 152 |
+
// Forward declare a virtual class 'Contents' to aid dependency injection and more
|
| 153 |
+
// easily create test fixtures
|
| 154 |
+
// An implementation of the Contents class is defined in the .cc file
|
| 155 |
+
struct Contents {
|
| 156 |
+
Contents(std::shared_ptr<::parquet::schema::GroupNode> schema,
|
| 157 |
+
std::shared_ptr<const KeyValueMetadata> key_value_metadata)
|
| 158 |
+
: schema_(), key_value_metadata_(std::move(key_value_metadata)) {
|
| 159 |
+
schema_.Init(std::move(schema));
|
| 160 |
+
}
|
| 161 |
+
virtual ~Contents() {}
|
| 162 |
+
// Perform any cleanup associated with the file contents
|
| 163 |
+
virtual void Close() = 0;
|
| 164 |
+
|
| 165 |
+
virtual RowGroupWriter* AppendRowGroup() = 0;
|
| 166 |
+
virtual RowGroupWriter* AppendBufferedRowGroup() = 0;
|
| 167 |
+
|
| 168 |
+
virtual int64_t num_rows() const = 0;
|
| 169 |
+
virtual int num_columns() const = 0;
|
| 170 |
+
virtual int num_row_groups() const = 0;
|
| 171 |
+
|
| 172 |
+
virtual const std::shared_ptr<WriterProperties>& properties() const = 0;
|
| 173 |
+
|
| 174 |
+
const std::shared_ptr<const KeyValueMetadata>& key_value_metadata() const {
|
| 175 |
+
return key_value_metadata_;
|
| 176 |
+
}
|
| 177 |
+
|
| 178 |
+
virtual void AddKeyValueMetadata(
|
| 179 |
+
const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) = 0;
|
| 180 |
+
|
| 181 |
+
// Return const-pointer to make it clear that this object is not to be copied
|
| 182 |
+
const SchemaDescriptor* schema() const { return &schema_; }
|
| 183 |
+
|
| 184 |
+
SchemaDescriptor schema_;
|
| 185 |
+
|
| 186 |
+
/// This should be the only place this is stored. Everything else is a const reference
|
| 187 |
+
std::shared_ptr<const KeyValueMetadata> key_value_metadata_;
|
| 188 |
+
|
| 189 |
+
const std::shared_ptr<FileMetaData>& metadata() const { return file_metadata_; }
|
| 190 |
+
std::shared_ptr<FileMetaData> file_metadata_;
|
| 191 |
+
};
|
| 192 |
+
|
| 193 |
+
ParquetFileWriter();
|
| 194 |
+
~ParquetFileWriter();
|
| 195 |
+
|
| 196 |
+
static std::unique_ptr<ParquetFileWriter> Open(
|
| 197 |
+
std::shared_ptr<::arrow::io::OutputStream> sink,
|
| 198 |
+
std::shared_ptr<schema::GroupNode> schema,
|
| 199 |
+
std::shared_ptr<WriterProperties> properties = default_writer_properties(),
|
| 200 |
+
std::shared_ptr<const KeyValueMetadata> key_value_metadata = NULLPTR);
|
| 201 |
+
|
| 202 |
+
void Open(std::unique_ptr<Contents> contents);
|
| 203 |
+
void Close();
|
| 204 |
+
|
| 205 |
+
/// Construct a RowGroupWriter with an arbitrary number of rows.
|
| 206 |
+
///
|
| 207 |
+
/// Ownership is solely within the ParquetFileWriter. The RowGroupWriter is only valid
|
| 208 |
+
/// until the next call to AppendRowGroup or AppendBufferedRowGroup or Close.
|
| 209 |
+
RowGroupWriter* AppendRowGroup();
|
| 210 |
+
|
| 211 |
+
/// Construct a RowGroupWriter that buffers all the values until the RowGroup is ready.
|
| 212 |
+
/// Use this if you want to write a RowGroup based on a certain size
|
| 213 |
+
///
|
| 214 |
+
/// Ownership is solely within the ParquetFileWriter. The RowGroupWriter is only valid
|
| 215 |
+
/// until the next call to AppendRowGroup or AppendBufferedRowGroup or Close.
|
| 216 |
+
RowGroupWriter* AppendBufferedRowGroup();
|
| 217 |
+
|
| 218 |
+
/// \brief Add key-value metadata to the file.
|
| 219 |
+
/// \param[in] key_value_metadata the metadata to add.
|
| 220 |
+
/// \note This will overwrite any existing metadata with the same key(s).
|
| 221 |
+
/// \throw ParquetException if Close() has been called.
|
| 222 |
+
void AddKeyValueMetadata(
|
| 223 |
+
const std::shared_ptr<const KeyValueMetadata>& key_value_metadata);
|
| 224 |
+
|
| 225 |
+
/// Number of columns.
|
| 226 |
+
///
|
| 227 |
+
/// This number is fixed during the lifetime of the writer as it is determined via
|
| 228 |
+
/// the schema.
|
| 229 |
+
int num_columns() const;
|
| 230 |
+
|
| 231 |
+
/// Number of rows in the yet started RowGroups.
|
| 232 |
+
///
|
| 233 |
+
/// Changes on the addition of a new RowGroup.
|
| 234 |
+
int64_t num_rows() const;
|
| 235 |
+
|
| 236 |
+
/// Number of started RowGroups.
|
| 237 |
+
int num_row_groups() const;
|
| 238 |
+
|
| 239 |
+
/// Configuration passed to the writer, e.g. the used Parquet format version.
|
| 240 |
+
const std::shared_ptr<WriterProperties>& properties() const;
|
| 241 |
+
|
| 242 |
+
/// Returns the file schema descriptor
|
| 243 |
+
const SchemaDescriptor* schema() const;
|
| 244 |
+
|
| 245 |
+
/// Returns a column descriptor in schema
|
| 246 |
+
const ColumnDescriptor* descr(int i) const;
|
| 247 |
+
|
| 248 |
+
/// Returns the file custom metadata
|
| 249 |
+
const std::shared_ptr<const KeyValueMetadata>& key_value_metadata() const;
|
| 250 |
+
|
| 251 |
+
/// Returns the file metadata, only available after calling Close().
|
| 252 |
+
const std::shared_ptr<FileMetaData> metadata() const;
|
| 253 |
+
|
| 254 |
+
private:
|
| 255 |
+
// Holds a pointer to an instance of Contents implementation
|
| 256 |
+
std::unique_ptr<Contents> contents_;
|
| 257 |
+
std::shared_ptr<FileMetaData> file_metadata_;
|
| 258 |
+
};
|
| 259 |
+
|
| 260 |
+
} // namespace parquet
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/geospatial/statistics.h
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <cstdint>
|
| 21 |
+
#include <memory>
|
| 22 |
+
#include <optional>
|
| 23 |
+
|
| 24 |
+
#include "parquet/platform.h"
|
| 25 |
+
#include "parquet/types.h"
|
| 26 |
+
|
| 27 |
+
namespace parquet::geospatial {
|
| 28 |
+
|
| 29 |
+
/// \brief The maximum number of dimensions represented by a geospatial type
|
| 30 |
+
/// (i.e., X, Y, Z, and M)
|
| 31 |
+
inline constexpr int kMaxDimensions = 4;
|
| 32 |
+
|
| 33 |
+
/// \brief NaN, used to represent bounds for which predicate pushdown cannnot
|
| 34 |
+
/// be applied (e.g., because a writer did not provide bounds for a given dimension)
|
| 35 |
+
inline constexpr double kNaN = std::numeric_limits<double>::quiet_NaN();
|
| 36 |
+
|
| 37 |
+
/// \brief Structure represented encoded statistics to be written to and read from Parquet
|
| 38 |
+
/// serialized metadata.
|
| 39 |
+
///
|
| 40 |
+
/// See the Parquet Thrift definition and GeoStatistics for the specific definition
|
| 41 |
+
/// of field values.
|
| 42 |
+
struct PARQUET_EXPORT EncodedGeoStatistics {
|
| 43 |
+
bool xy_bounds_present{false};
|
| 44 |
+
double xmin{kNaN};
|
| 45 |
+
double xmax{kNaN};
|
| 46 |
+
double ymin{kNaN};
|
| 47 |
+
double ymax{kNaN};
|
| 48 |
+
|
| 49 |
+
bool z_bounds_present{false};
|
| 50 |
+
double zmin{kNaN};
|
| 51 |
+
double zmax{kNaN};
|
| 52 |
+
|
| 53 |
+
bool m_bounds_present{false};
|
| 54 |
+
double mmin{kNaN};
|
| 55 |
+
double mmax{kNaN};
|
| 56 |
+
|
| 57 |
+
bool geospatial_types_present() const { return !geospatial_types.empty(); }
|
| 58 |
+
std::vector<int32_t> geospatial_types;
|
| 59 |
+
};
|
| 60 |
+
|
| 61 |
+
class GeoStatisticsImpl;
|
| 62 |
+
|
| 63 |
+
/// \brief Base type for computing geospatial column statistics while writing a file
|
| 64 |
+
/// or representing them when reading a file
|
| 65 |
+
///
|
| 66 |
+
/// These statistics track the minimum and maximum value (omitting NaN values) of the
|
| 67 |
+
/// four possible dimensions (X, Y, Z, and M) and the distinct set of geometry
|
| 68 |
+
/// type/dimension combinations (e.g., point XY, linestring XYZM) present in the data.
|
| 69 |
+
/// Any of these individual components may be "invalid": for example, when reading a
|
| 70 |
+
/// Parquet file, information about individual components obtained from the column
|
| 71 |
+
/// chunk metadata may have been missing or deemed unusable. Orthogonally,
|
| 72 |
+
/// any of these individual components may be "empty": for example, when using
|
| 73 |
+
/// GeoStatistics to accumulate bounds whilst writing, if all geometries in a column chunk
|
| 74 |
+
/// are null, all ranges (X, Y, Z, and M) will be empty. If all geometries in a column
|
| 75 |
+
/// chunk contain only XY coordinates (the most common case), the Z and M ranges will
|
| 76 |
+
/// be empty but the X and Y ranges will contain finite bounds. Empty ranges are
|
| 77 |
+
/// considered "valid" because they are known to represent exactly zero values (in
|
| 78 |
+
/// contrast to an invalid range, whose contents is completely unknown). These concepts
|
| 79 |
+
/// are all necessary for this object to accurately represent (1) accumulated or partially
|
| 80 |
+
/// accumulated statistics during the writing process and (2) deserialized statistics read
|
| 81 |
+
/// from the column chunk metadata during the reading process.
|
| 82 |
+
///
|
| 83 |
+
/// EXPERIMENTAL
|
| 84 |
+
class PARQUET_EXPORT GeoStatistics {
|
| 85 |
+
public:
|
| 86 |
+
GeoStatistics();
|
| 87 |
+
explicit GeoStatistics(const EncodedGeoStatistics& encoded);
|
| 88 |
+
|
| 89 |
+
~GeoStatistics();
|
| 90 |
+
|
| 91 |
+
/// \brief Return true if bounds, geometry types, and validity are identical
|
| 92 |
+
bool Equals(const GeoStatistics& other) const;
|
| 93 |
+
|
| 94 |
+
/// \brief Update these statistics based on previously calculated or decoded statistics
|
| 95 |
+
///
|
| 96 |
+
/// Merging statistics with wraparound X values is not currently supported. Merging
|
| 97 |
+
/// two GeoStatistics where one or both has a wraparound X range will result in these
|
| 98 |
+
/// statistics having an X dimension marked as invalid.
|
| 99 |
+
void Merge(const GeoStatistics& other);
|
| 100 |
+
|
| 101 |
+
/// \brief Update these statistics based on values
|
| 102 |
+
void Update(const ByteArray* values, int64_t num_values);
|
| 103 |
+
|
| 104 |
+
/// \brief Update these statistics based on the non-null elements of values
|
| 105 |
+
void UpdateSpaced(const ByteArray* values, const uint8_t* valid_bits,
|
| 106 |
+
int64_t valid_bits_offset, int64_t num_spaced_values,
|
| 107 |
+
int64_t num_values);
|
| 108 |
+
|
| 109 |
+
/// \brief Update these statistics based on the non-null elements of values
|
| 110 |
+
///
|
| 111 |
+
/// Currently, BinaryArray and LargeBinaryArray input is supported.
|
| 112 |
+
void Update(const ::arrow::Array& values);
|
| 113 |
+
|
| 114 |
+
/// \brief Return these statistics to an empty state
|
| 115 |
+
void Reset();
|
| 116 |
+
|
| 117 |
+
/// \brief Encode the statistics for serializing to Thrift
|
| 118 |
+
///
|
| 119 |
+
/// If invalid WKB was encountered or if the statistics contain NaN
|
| 120 |
+
/// for any reason, Encode() will return nullopt to indicate that
|
| 121 |
+
/// statistics should not be written to thrift.
|
| 122 |
+
std::optional<EncodedGeoStatistics> Encode() const;
|
| 123 |
+
|
| 124 |
+
/// \brief Returns false if invalid WKB was encountered
|
| 125 |
+
bool is_valid() const;
|
| 126 |
+
|
| 127 |
+
/// \brief Reset existing statistics and populate them from previously-encoded ones
|
| 128 |
+
void Decode(const EncodedGeoStatistics& encoded);
|
| 129 |
+
|
| 130 |
+
/// \brief Minimum values in XYZM order
|
| 131 |
+
///
|
| 132 |
+
/// For dimensions where dimension_valid() is false, the value will be NaN. For
|
| 133 |
+
/// dimensions where dimension_empty() is true, the value will be +Inf.
|
| 134 |
+
///
|
| 135 |
+
/// For the first dimension (X) only, wraparound bounds apply where xmin > xmax. In this
|
| 136 |
+
/// case, these bounds represent the union of the intervals [xmax, Inf] and [-Inf,
|
| 137 |
+
/// xmin]. This implementation does not yet generate these types of bounds but they may
|
| 138 |
+
/// be encountered in statistics when reading a Parquet file.
|
| 139 |
+
std::array<double, kMaxDimensions> lower_bound() const;
|
| 140 |
+
|
| 141 |
+
/// \brief Maximum values in XYZM order
|
| 142 |
+
///
|
| 143 |
+
/// For dimensions where dimension_valid() is false, the value will be NaN. For
|
| 144 |
+
/// dimensions where dimension_empty() is true, the value will be -Inf.
|
| 145 |
+
///
|
| 146 |
+
/// For the first dimension (X) only, wraparound bounds apply where xmin > xmax. In this
|
| 147 |
+
/// case, these bounds represent the union of the intervals [xmax, Inf] and [-Inf,
|
| 148 |
+
/// xmin]. This implementation does not yet generate these types of bounds but they may
|
| 149 |
+
/// be encountered in statistics when reading a Parquet file.
|
| 150 |
+
std::array<double, kMaxDimensions> upper_bound() const;
|
| 151 |
+
|
| 152 |
+
/// \brief Dimension emptiness in XYZM order
|
| 153 |
+
///
|
| 154 |
+
/// True for a given dimension if and only if zero non-NaN values were encountered
|
| 155 |
+
/// in that dimension and dimension_valid() is true for that dimension.
|
| 156 |
+
///
|
| 157 |
+
/// When calculating statistics, zero or more of these values may be true because
|
| 158 |
+
/// this implementation calculates bounds for all dimensions; however, it may be
|
| 159 |
+
/// true that zero coordinates were encountered in a given dimension. For example,
|
| 160 |
+
/// dimension_empty() will return four true values if Update() was not called
|
| 161 |
+
/// or if Update() was called with only null values. If Update() was provided
|
| 162 |
+
/// one or more geometries with X and Y dimensions but not Z or M dimensions,
|
| 163 |
+
/// dimension_empty() will return true, true, false, false.
|
| 164 |
+
///
|
| 165 |
+
/// For statistics read from a Parquet file, dimension_empty() will always contain
|
| 166 |
+
/// false values because there is no mechanism to communicate an empty interval
|
| 167 |
+
/// in the Thrift metadata.
|
| 168 |
+
std::array<bool, kMaxDimensions> dimension_empty() const;
|
| 169 |
+
|
| 170 |
+
/// \brief Dimension validity (i.e. presence) in XYZM order
|
| 171 |
+
///
|
| 172 |
+
/// When calculating statistics, this will always be true because this implementation
|
| 173 |
+
/// calculates statistics for all dimensions. When reading a Parquet file, one or more
|
| 174 |
+
/// of these values may be false because the file may not have provided bounds for all
|
| 175 |
+
/// dimensions.
|
| 176 |
+
///
|
| 177 |
+
/// See documentation for dimension_empty(), lower_bound(), and/or upper_bound() for the
|
| 178 |
+
/// canonical values of those outputs for the dimensions where dimension_valid() is
|
| 179 |
+
/// false.
|
| 180 |
+
std::array<bool, kMaxDimensions> dimension_valid() const;
|
| 181 |
+
|
| 182 |
+
/// \brief Return the geometry type codes
|
| 183 |
+
///
|
| 184 |
+
/// This implementation always returns sorted output with no duplicates. When
|
| 185 |
+
/// calculating statistics, a value will always be returned (although the returned
|
| 186 |
+
/// vector may be empty if Update() was never called or was only called with null
|
| 187 |
+
/// values). When reading a Parquet file, std::nullopt may be returned because
|
| 188 |
+
/// the file may not have provided this information.
|
| 189 |
+
std::optional<std::vector<int32_t>> geometry_types() const;
|
| 190 |
+
|
| 191 |
+
/// \brief Return a string representation of these statistics
|
| 192 |
+
std::string ToString() const;
|
| 193 |
+
|
| 194 |
+
private:
|
| 195 |
+
std::unique_ptr<GeoStatisticsImpl> impl_;
|
| 196 |
+
};
|
| 197 |
+
|
| 198 |
+
} // namespace parquet::geospatial
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/hasher.h
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <cstdint>
|
| 21 |
+
#include "parquet/types.h"
|
| 22 |
+
|
| 23 |
+
namespace parquet {
|
| 24 |
+
// Abstract class for hash
|
| 25 |
+
class PARQUET_EXPORT Hasher {
|
| 26 |
+
public:
|
| 27 |
+
/// Compute hash for 32 bits value by using its plain encoding result.
|
| 28 |
+
///
|
| 29 |
+
/// @param value the value to hash.
|
| 30 |
+
/// @return hash result.
|
| 31 |
+
virtual uint64_t Hash(int32_t value) const = 0;
|
| 32 |
+
|
| 33 |
+
/// Compute hash for 64 bits value by using its plain encoding result.
|
| 34 |
+
///
|
| 35 |
+
/// @param value the value to hash.
|
| 36 |
+
/// @return hash result.
|
| 37 |
+
virtual uint64_t Hash(int64_t value) const = 0;
|
| 38 |
+
|
| 39 |
+
/// Compute hash for float value by using its plain encoding result.
|
| 40 |
+
///
|
| 41 |
+
/// @param value the value to hash.
|
| 42 |
+
/// @return hash result.
|
| 43 |
+
virtual uint64_t Hash(float value) const = 0;
|
| 44 |
+
|
| 45 |
+
/// Compute hash for double value by using its plain encoding result.
|
| 46 |
+
///
|
| 47 |
+
/// @param value the value to hash.
|
| 48 |
+
/// @return hash result.
|
| 49 |
+
virtual uint64_t Hash(double value) const = 0;
|
| 50 |
+
|
| 51 |
+
/// Compute hash for Int96 value by using its plain encoding result.
|
| 52 |
+
///
|
| 53 |
+
/// @param value the value to hash.
|
| 54 |
+
/// @return hash result.
|
| 55 |
+
virtual uint64_t Hash(const Int96* value) const = 0;
|
| 56 |
+
|
| 57 |
+
/// Compute hash for ByteArray value by using its plain encoding result.
|
| 58 |
+
///
|
| 59 |
+
/// @param value the value to hash.
|
| 60 |
+
/// @return hash result.
|
| 61 |
+
virtual uint64_t Hash(const ByteArray* value) const = 0;
|
| 62 |
+
|
| 63 |
+
/// Compute hash for fixed byte array value by using its plain encoding result.
|
| 64 |
+
///
|
| 65 |
+
/// @param value the value address.
|
| 66 |
+
/// @param len the value length.
|
| 67 |
+
virtual uint64_t Hash(const FLBA* value, uint32_t len) const = 0;
|
| 68 |
+
|
| 69 |
+
/// Compute hash for std::string_view value by using its plain encoding result.
|
| 70 |
+
///
|
| 71 |
+
/// @param value the value to hash.
|
| 72 |
+
virtual uint64_t Hash(std::string_view value) const = 0;
|
| 73 |
+
|
| 74 |
+
/// Batch compute hashes for 32 bits values by using its plain encoding result.
|
| 75 |
+
///
|
| 76 |
+
/// @param values a pointer to the values to hash.
|
| 77 |
+
/// @param num_values the number of values to hash.
|
| 78 |
+
/// @param hashes a pointer to the output hash values, its length should be equal to
|
| 79 |
+
/// num_values.
|
| 80 |
+
virtual void Hashes(const int32_t* values, int num_values, uint64_t* hashes) const = 0;
|
| 81 |
+
|
| 82 |
+
/// Batch compute hashes for 64 bits values by using its plain encoding result.
|
| 83 |
+
///
|
| 84 |
+
/// @param values a pointer to the values to hash.
|
| 85 |
+
/// @param num_values the number of values to hash.
|
| 86 |
+
/// @param hashes a pointer to the output hash values, its length should be equal to
|
| 87 |
+
/// num_values.
|
| 88 |
+
virtual void Hashes(const int64_t* values, int num_values, uint64_t* hashes) const = 0;
|
| 89 |
+
|
| 90 |
+
/// Batch compute hashes for float values by using its plain encoding result.
|
| 91 |
+
///
|
| 92 |
+
/// @param values a pointer to the values to hash.
|
| 93 |
+
/// @param num_values the number of values to hash.
|
| 94 |
+
/// @param hashes a pointer to the output hash values, its length should be equal to
|
| 95 |
+
/// num_values.
|
| 96 |
+
virtual void Hashes(const float* values, int num_values, uint64_t* hashes) const = 0;
|
| 97 |
+
|
| 98 |
+
/// Batch compute hashes for double values by using its plain encoding result.
|
| 99 |
+
///
|
| 100 |
+
/// @param values a pointer to the values to hash.
|
| 101 |
+
/// @param num_values the number of values to hash.
|
| 102 |
+
/// @param hashes a pointer to the output hash values, its length should be equal to
|
| 103 |
+
/// num_values.
|
| 104 |
+
virtual void Hashes(const double* values, int num_values, uint64_t* hashes) const = 0;
|
| 105 |
+
|
| 106 |
+
/// Batch compute hashes for Int96 values by using its plain encoding result.
|
| 107 |
+
///
|
| 108 |
+
/// @param values a pointer to the values to hash.
|
| 109 |
+
/// @param num_values the number of values to hash.
|
| 110 |
+
/// @param hashes a pointer to the output hash values, its length should be equal to
|
| 111 |
+
/// num_values.
|
| 112 |
+
virtual void Hashes(const Int96* values, int num_values, uint64_t* hashes) const = 0;
|
| 113 |
+
|
| 114 |
+
/// Batch compute hashes for ByteArray values by using its plain encoding result.
|
| 115 |
+
///
|
| 116 |
+
/// @param values a pointer to the values to hash.
|
| 117 |
+
/// @param num_values the number of values to hash.
|
| 118 |
+
/// @param hashes a pointer to the output hash values, its length should be equal to
|
| 119 |
+
/// num_values.
|
| 120 |
+
virtual void Hashes(const ByteArray* values, int num_values,
|
| 121 |
+
uint64_t* hashes) const = 0;
|
| 122 |
+
|
| 123 |
+
/// Batch compute hashes for fixed byte array values by using its plain encoding result.
|
| 124 |
+
///
|
| 125 |
+
/// @param values the value address.
|
| 126 |
+
/// @param type_len the value length.
|
| 127 |
+
/// @param num_values the number of values to hash.
|
| 128 |
+
/// @param hashes a pointer to the output hash values, its length should be equal to
|
| 129 |
+
/// num_values.
|
| 130 |
+
virtual void Hashes(const FLBA* values, uint32_t type_len, int num_values,
|
| 131 |
+
uint64_t* hashes) const = 0;
|
| 132 |
+
|
| 133 |
+
virtual ~Hasher() = default;
|
| 134 |
+
};
|
| 135 |
+
|
| 136 |
+
} // namespace parquet
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/index_location.h
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <cstdint>
|
| 21 |
+
#include <vector>
|
| 22 |
+
|
| 23 |
+
#include "parquet/platform.h"
|
| 24 |
+
|
| 25 |
+
namespace parquet {
|
| 26 |
+
|
| 27 |
+
enum class PARQUET_EXPORT IndexKind : uint8_t {
|
| 28 |
+
kColumnIndex,
|
| 29 |
+
kOffsetIndex,
|
| 30 |
+
kBloomFilter,
|
| 31 |
+
};
|
| 32 |
+
|
| 33 |
+
/// \brief Location to a Parquet index (page index or bloom filter).
|
| 34 |
+
struct PARQUET_EXPORT IndexLocation {
|
| 35 |
+
// File offset of the given index, in bytes
|
| 36 |
+
int64_t offset;
|
| 37 |
+
// Length of the given index, in bytes
|
| 38 |
+
int32_t length;
|
| 39 |
+
};
|
| 40 |
+
|
| 41 |
+
/// \brief Identifier for a column chunk.
|
| 42 |
+
struct PARQUET_EXPORT ColumnChunkId {
|
| 43 |
+
int32_t row_group_index;
|
| 44 |
+
int32_t column_index;
|
| 45 |
+
};
|
| 46 |
+
|
| 47 |
+
using IndexLocations = std::vector<std::pair<ColumnChunkId, IndexLocation>>;
|
| 48 |
+
|
| 49 |
+
} // namespace parquet
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_comparison.h
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
#pragma once
|
| 18 |
+
|
| 19 |
+
#include <algorithm>
|
| 20 |
+
#include <cstdint>
|
| 21 |
+
|
| 22 |
+
#include "parquet/platform.h"
|
| 23 |
+
|
| 24 |
+
namespace parquet::internal {
|
| 25 |
+
|
| 26 |
+
/// Builds a bitmap where each set bit indicates the corresponding level is greater
|
| 27 |
+
/// than rhs.
|
| 28 |
+
uint64_t PARQUET_EXPORT GreaterThanBitmap(const int16_t* levels, int64_t num_levels,
|
| 29 |
+
int16_t rhs);
|
| 30 |
+
|
| 31 |
+
struct MinMax {
|
| 32 |
+
int16_t min;
|
| 33 |
+
int16_t max;
|
| 34 |
+
};
|
| 35 |
+
|
| 36 |
+
MinMax FindMinMax(const int16_t* levels, int64_t num_levels);
|
| 37 |
+
|
| 38 |
+
} // namespace parquet::internal
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_comparison_inc.h
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
#pragma once
|
| 18 |
+
|
| 19 |
+
#include "arrow/util/bit_util.h"
|
| 20 |
+
#include "arrow/util/endian.h"
|
| 21 |
+
#include "parquet/level_comparison.h"
|
| 22 |
+
|
| 23 |
+
// Used to make sure ODR rule isn't violated.
|
| 24 |
+
#ifndef PARQUET_IMPL_NAMESPACE
|
| 25 |
+
# error "PARQUET_IMPL_NAMESPACE must be defined"
|
| 26 |
+
#endif
|
| 27 |
+
namespace parquet::internal::PARQUET_IMPL_NAMESPACE {
|
| 28 |
+
/// Builds a bitmap by applying predicate to the level vector provided.
|
| 29 |
+
///
|
| 30 |
+
/// \param[in] levels Rep or def level array.
|
| 31 |
+
/// \param[in] num_levels The number of levels to process (must be [0, 64])
|
| 32 |
+
/// \param[in] predicate The predicate to apply (must have the signature `bool
|
| 33 |
+
/// predicate(int16_t)`.
|
| 34 |
+
/// \returns The bitmap using least significant "bit" ordering.
|
| 35 |
+
///
|
| 36 |
+
template <typename Predicate>
|
| 37 |
+
inline uint64_t LevelsToBitmap(const int16_t* levels, int64_t num_levels,
|
| 38 |
+
Predicate predicate) {
|
| 39 |
+
// Both clang and GCC can vectorize this automatically with SSE4/AVX2.
|
| 40 |
+
uint64_t mask = 0;
|
| 41 |
+
for (int x = 0; x < num_levels; x++) {
|
| 42 |
+
mask |= static_cast<uint64_t>(predicate(levels[x]) ? 1 : 0) << x;
|
| 43 |
+
}
|
| 44 |
+
return ::arrow::bit_util::ToLittleEndian(mask);
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
inline MinMax FindMinMaxImpl(const int16_t* levels, int64_t num_levels) {
|
| 48 |
+
MinMax out{std::numeric_limits<int16_t>::max(), std::numeric_limits<int16_t>::min()};
|
| 49 |
+
for (int x = 0; x < num_levels; x++) {
|
| 50 |
+
out.min = std::min(levels[x], out.min);
|
| 51 |
+
out.max = std::max(levels[x], out.max);
|
| 52 |
+
}
|
| 53 |
+
return out;
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
inline uint64_t GreaterThanBitmapImpl(const int16_t* levels, int64_t num_levels,
|
| 57 |
+
int16_t rhs) {
|
| 58 |
+
return LevelsToBitmap(levels, num_levels, [rhs](int16_t value) { return value > rhs; });
|
| 59 |
+
}
|
| 60 |
+
|
| 61 |
+
} // namespace parquet::internal::PARQUET_IMPL_NAMESPACE
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_conversion.h
ADDED
|
@@ -0,0 +1,216 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <cstdint>
|
| 21 |
+
|
| 22 |
+
#include "arrow/util/endian.h"
|
| 23 |
+
#include "parquet/platform.h"
|
| 24 |
+
#include "parquet/schema.h"
|
| 25 |
+
|
| 26 |
+
namespace parquet::internal {
|
| 27 |
+
|
| 28 |
+
struct PARQUET_EXPORT LevelInfo {
|
| 29 |
+
LevelInfo()
|
| 30 |
+
: null_slot_usage(1), def_level(0), rep_level(0), repeated_ancestor_def_level(0) {}
|
| 31 |
+
LevelInfo(int32_t null_slots, int32_t definition_level, int32_t repetition_level,
|
| 32 |
+
int32_t repeated_ancestor_definition_level)
|
| 33 |
+
: null_slot_usage(null_slots),
|
| 34 |
+
def_level(static_cast<int16_t>(definition_level)),
|
| 35 |
+
rep_level(static_cast<int16_t>(repetition_level)),
|
| 36 |
+
repeated_ancestor_def_level(
|
| 37 |
+
static_cast<int16_t>(repeated_ancestor_definition_level)) {}
|
| 38 |
+
|
| 39 |
+
bool operator==(const LevelInfo& b) const {
|
| 40 |
+
return null_slot_usage == b.null_slot_usage && def_level == b.def_level &&
|
| 41 |
+
rep_level == b.rep_level &&
|
| 42 |
+
repeated_ancestor_def_level == b.repeated_ancestor_def_level;
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
bool HasNullableValues() const { return repeated_ancestor_def_level < def_level; }
|
| 46 |
+
|
| 47 |
+
// How many slots an undefined but present (i.e. null) element in
|
| 48 |
+
// parquet consumes when decoding to Arrow.
|
| 49 |
+
// "Slot" is used in the same context as the Arrow specification
|
| 50 |
+
// (i.e. a value holder).
|
| 51 |
+
// This is only ever >1 for descendents of FixedSizeList.
|
| 52 |
+
int32_t null_slot_usage = 1;
|
| 53 |
+
|
| 54 |
+
// The definition level at which the value for the field
|
| 55 |
+
// is considered not null (definition levels greater than
|
| 56 |
+
// or equal to this value indicate a not-null
|
| 57 |
+
// value for the field). For list fields definition levels
|
| 58 |
+
// greater than or equal to this field indicate a present,
|
| 59 |
+
// possibly null, child value.
|
| 60 |
+
int16_t def_level = 0;
|
| 61 |
+
|
| 62 |
+
// The repetition level corresponding to this element
|
| 63 |
+
// or the closest repeated ancestor. Any repetition
|
| 64 |
+
// level less than this indicates either a new list OR
|
| 65 |
+
// an empty list (which is determined in conjunction
|
| 66 |
+
// with definition levels).
|
| 67 |
+
int16_t rep_level = 0;
|
| 68 |
+
|
| 69 |
+
// The definition level indicating the level at which the closest
|
| 70 |
+
// repeated ancestor is not empty. This is used to discriminate
|
| 71 |
+
// between a value less than |def_level| being null or excluded entirely.
|
| 72 |
+
// For instance if we have an arrow schema like:
|
| 73 |
+
// list(struct(f0: int)). Then then there are the following
|
| 74 |
+
// definition levels:
|
| 75 |
+
// 0 = null list
|
| 76 |
+
// 1 = present but empty list.
|
| 77 |
+
// 2 = a null value in the list
|
| 78 |
+
// 3 = a non null struct but null integer.
|
| 79 |
+
// 4 = a present integer.
|
| 80 |
+
// When reconstructing, the struct and integer arrays'
|
| 81 |
+
// repeated_ancestor_def_level would be 2. Any
|
| 82 |
+
// def_level < 2 indicates that there isn't a corresponding
|
| 83 |
+
// child value in the list.
|
| 84 |
+
// i.e. [null, [], [null], [{f0: null}], [{f0: 1}]]
|
| 85 |
+
// has the def levels [0, 1, 2, 3, 4]. The actual
|
| 86 |
+
// struct array is only of length 3: [not-set, set, set] and
|
| 87 |
+
// the int array is also of length 3: [N/A, null, 1].
|
| 88 |
+
//
|
| 89 |
+
int16_t repeated_ancestor_def_level = 0;
|
| 90 |
+
|
| 91 |
+
/// Increments levels according to the cardinality of node.
|
| 92 |
+
void Increment(const schema::Node& node) {
|
| 93 |
+
if (node.is_repeated()) {
|
| 94 |
+
IncrementRepeated();
|
| 95 |
+
return;
|
| 96 |
+
}
|
| 97 |
+
if (node.is_optional()) {
|
| 98 |
+
IncrementOptional();
|
| 99 |
+
return;
|
| 100 |
+
}
|
| 101 |
+
}
|
| 102 |
+
|
| 103 |
+
/// Increments level for a optional node.
|
| 104 |
+
void IncrementOptional() { def_level++; }
|
| 105 |
+
|
| 106 |
+
/// Increments levels for the repeated node. Returns
|
| 107 |
+
/// the previous ancestor_list_def_level.
|
| 108 |
+
int16_t IncrementRepeated() {
|
| 109 |
+
int16_t last_repeated_ancestor = repeated_ancestor_def_level;
|
| 110 |
+
|
| 111 |
+
// Repeated fields add both a repetition and definition level. This is used
|
| 112 |
+
// to distinguish between an empty list and a list with an item in it.
|
| 113 |
+
++rep_level;
|
| 114 |
+
++def_level;
|
| 115 |
+
// For levels >= repeated_ancestor_def_level it indicates the list was
|
| 116 |
+
// non-null and had at least one element. This is important
|
| 117 |
+
// for later decoding because we need to add a slot for these
|
| 118 |
+
// values. for levels < current_def_level no slots are added
|
| 119 |
+
// to arrays.
|
| 120 |
+
repeated_ancestor_def_level = def_level;
|
| 121 |
+
return last_repeated_ancestor;
|
| 122 |
+
}
|
| 123 |
+
|
| 124 |
+
// Calculates and returns LevelInfo for a column descriptor.
|
| 125 |
+
static LevelInfo ComputeLevelInfo(const ColumnDescriptor* descr) {
|
| 126 |
+
LevelInfo level_info;
|
| 127 |
+
level_info.def_level = descr->max_definition_level();
|
| 128 |
+
level_info.rep_level = descr->max_repetition_level();
|
| 129 |
+
|
| 130 |
+
int16_t min_spaced_def_level = descr->max_definition_level();
|
| 131 |
+
const ::parquet::schema::Node* node = descr->schema_node().get();
|
| 132 |
+
while (node && !node->is_repeated()) {
|
| 133 |
+
if (node->is_optional()) {
|
| 134 |
+
min_spaced_def_level--;
|
| 135 |
+
}
|
| 136 |
+
node = node->parent();
|
| 137 |
+
}
|
| 138 |
+
level_info.repeated_ancestor_def_level = min_spaced_def_level;
|
| 139 |
+
return level_info;
|
| 140 |
+
}
|
| 141 |
+
|
| 142 |
+
friend std::ostream& operator<<(std::ostream& os, const LevelInfo& levels) {
|
| 143 |
+
// This print method is to silence valgrind issues. What's printed
|
| 144 |
+
// is not important because all asserts happen directly on
|
| 145 |
+
// members.
|
| 146 |
+
os << "{def=" << levels.def_level << ", rep=" << levels.rep_level
|
| 147 |
+
<< ", repeated_ancestor_def=" << levels.repeated_ancestor_def_level;
|
| 148 |
+
if (levels.null_slot_usage > 1) {
|
| 149 |
+
os << ", null_slot_usage=" << levels.null_slot_usage;
|
| 150 |
+
}
|
| 151 |
+
os << "}";
|
| 152 |
+
return os;
|
| 153 |
+
}
|
| 154 |
+
};
|
| 155 |
+
|
| 156 |
+
// Input/Output structure for reconstructed validity bitmaps.
|
| 157 |
+
struct PARQUET_EXPORT ValidityBitmapInputOutput {
|
| 158 |
+
// Input only.
|
| 159 |
+
// The maximum number of values_read expected (actual
|
| 160 |
+
// values read must be less than or equal to this value).
|
| 161 |
+
// If this number is exceeded methods will throw a
|
| 162 |
+
// ParquetException. Exceeding this limit indicates
|
| 163 |
+
// either a corrupt or incorrectly written file.
|
| 164 |
+
int64_t values_read_upper_bound = 0;
|
| 165 |
+
// Output only. The number of values added to the encountered
|
| 166 |
+
// (this is logically the count of the number of elements
|
| 167 |
+
// for an Arrow array).
|
| 168 |
+
int64_t values_read = 0;
|
| 169 |
+
// Input/Output. The number of nulls encountered.
|
| 170 |
+
int64_t null_count = 0;
|
| 171 |
+
// Output only. The validity bitmap to populate. Maybe be null only
|
| 172 |
+
// for DefRepLevelsToListInfo (if all that is needed is list offsets).
|
| 173 |
+
uint8_t* valid_bits = NULLPTR;
|
| 174 |
+
// Input only, offset into valid_bits to start at.
|
| 175 |
+
int64_t valid_bits_offset = 0;
|
| 176 |
+
};
|
| 177 |
+
|
| 178 |
+
// Converts def_levels to validity bitmaps for non-list arrays and structs that have
|
| 179 |
+
// at least one member that is not a list and has no list descendents.
|
| 180 |
+
// For lists use DefRepLevelsToList and structs where all descendants contain
|
| 181 |
+
// a list use DefRepLevelsToBitmap.
|
| 182 |
+
void PARQUET_EXPORT DefLevelsToBitmap(const int16_t* def_levels, int64_t num_def_levels,
|
| 183 |
+
LevelInfo level_info,
|
| 184 |
+
ValidityBitmapInputOutput* output);
|
| 185 |
+
|
| 186 |
+
// Reconstructs a validity bitmap and list offsets for a list arrays based on
|
| 187 |
+
// def/rep levels. The first element of offsets will not be modified if rep_levels
|
| 188 |
+
// starts with a new list. The first element of offsets will be used when calculating
|
| 189 |
+
// the next offset. See documentation onf DefLevelsToBitmap for when to use this
|
| 190 |
+
// method vs the other ones in this file for reconstruction.
|
| 191 |
+
//
|
| 192 |
+
// Offsets must be sized to 1 + values_read_upper_bound.
|
| 193 |
+
void PARQUET_EXPORT DefRepLevelsToList(const int16_t* def_levels,
|
| 194 |
+
const int16_t* rep_levels, int64_t num_def_levels,
|
| 195 |
+
LevelInfo level_info,
|
| 196 |
+
ValidityBitmapInputOutput* output,
|
| 197 |
+
int32_t* offsets);
|
| 198 |
+
void PARQUET_EXPORT DefRepLevelsToList(const int16_t* def_levels,
|
| 199 |
+
const int16_t* rep_levels, int64_t num_def_levels,
|
| 200 |
+
LevelInfo level_info,
|
| 201 |
+
ValidityBitmapInputOutput* output,
|
| 202 |
+
int64_t* offsets);
|
| 203 |
+
|
| 204 |
+
// Reconstructs a validity bitmap for a struct every member is a list or has
|
| 205 |
+
// a list descendant. See documentation on DefLevelsToBitmap for when more
|
| 206 |
+
// details on this method compared to the other ones defined above.
|
| 207 |
+
void PARQUET_EXPORT DefRepLevelsToBitmap(const int16_t* def_levels,
|
| 208 |
+
const int16_t* rep_levels,
|
| 209 |
+
int64_t num_def_levels, LevelInfo level_info,
|
| 210 |
+
ValidityBitmapInputOutput* output);
|
| 211 |
+
|
| 212 |
+
// This is exposed to ensure we can properly test a software simulated pext function
|
| 213 |
+
// (i.e. it isn't hidden by runtime dispatch).
|
| 214 |
+
uint64_t PARQUET_EXPORT TestOnlyExtractBitsSoftware(uint64_t bitmap, uint64_t selection);
|
| 215 |
+
|
| 216 |
+
} // namespace parquet::internal
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_conversion_inc.h
ADDED
|
@@ -0,0 +1,356 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
#pragma once
|
| 18 |
+
|
| 19 |
+
#include "parquet/level_conversion.h"
|
| 20 |
+
|
| 21 |
+
#include <algorithm>
|
| 22 |
+
#include <bit>
|
| 23 |
+
#include <cstdint>
|
| 24 |
+
#include <limits>
|
| 25 |
+
|
| 26 |
+
#include "arrow/util/bit_run_reader.h"
|
| 27 |
+
#include "arrow/util/bit_util.h"
|
| 28 |
+
#include "arrow/util/bitmap_writer.h"
|
| 29 |
+
#include "arrow/util/logging.h"
|
| 30 |
+
#include "arrow/util/simd.h"
|
| 31 |
+
#include "parquet/exception.h"
|
| 32 |
+
#include "parquet/level_comparison.h"
|
| 33 |
+
|
| 34 |
+
#ifndef PARQUET_IMPL_NAMESPACE
|
| 35 |
+
# error "PARQUET_IMPL_NAMESPACE must be defined"
|
| 36 |
+
#endif
|
| 37 |
+
|
| 38 |
+
namespace parquet::internal::PARQUET_IMPL_NAMESPACE {
|
| 39 |
+
|
| 40 |
+
// clang-format off
|
| 41 |
+
/* Python code to generate lookup table:
|
| 42 |
+
|
| 43 |
+
kLookupBits = 5
|
| 44 |
+
count = 0
|
| 45 |
+
print('constexpr int kLookupBits = {};'.format(kLookupBits))
|
| 46 |
+
print('constexpr uint8_t kPextTable[1 << kLookupBits][1 << kLookupBits] = {')
|
| 47 |
+
print(' ', end = '')
|
| 48 |
+
for mask in range(1 << kLookupBits):
|
| 49 |
+
for data in range(1 << kLookupBits):
|
| 50 |
+
bit_value = 0
|
| 51 |
+
bit_len = 0
|
| 52 |
+
for i in range(kLookupBits):
|
| 53 |
+
if mask & (1 << i):
|
| 54 |
+
bit_value |= (((data >> i) & 1) << bit_len)
|
| 55 |
+
bit_len += 1
|
| 56 |
+
out = '0x{:02X},'.format(bit_value)
|
| 57 |
+
count += 1
|
| 58 |
+
if count % (1 << kLookupBits) == 1:
|
| 59 |
+
print(' {')
|
| 60 |
+
if count % 8 == 1:
|
| 61 |
+
print(' ', end = '')
|
| 62 |
+
if count % 8 == 0:
|
| 63 |
+
print(out, end = '\n')
|
| 64 |
+
else:
|
| 65 |
+
print(out, end = ' ')
|
| 66 |
+
if count % (1 << kLookupBits) == 0:
|
| 67 |
+
print(' },', end = '')
|
| 68 |
+
print('\n};')
|
| 69 |
+
|
| 70 |
+
*/
|
| 71 |
+
// clang-format on
|
| 72 |
+
|
| 73 |
+
constexpr int kLookupBits = 5;
|
| 74 |
+
constexpr uint8_t kPextTable[1 << kLookupBits][1 << kLookupBits] = {
|
| 75 |
+
{
|
| 76 |
+
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
| 77 |
+
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
| 78 |
+
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
| 79 |
+
},
|
| 80 |
+
{
|
| 81 |
+
0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00,
|
| 82 |
+
0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01,
|
| 83 |
+
0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01,
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01,
|
| 87 |
+
0x01, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00,
|
| 88 |
+
0x01, 0x01, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01,
|
| 89 |
+
},
|
| 90 |
+
{
|
| 91 |
+
0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02,
|
| 92 |
+
0x03, 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03, 0x00, 0x01,
|
| 93 |
+
0x02, 0x03, 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03,
|
| 94 |
+
},
|
| 95 |
+
{
|
| 96 |
+
0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00,
|
| 97 |
+
0x00, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01,
|
| 98 |
+
0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01,
|
| 99 |
+
},
|
| 100 |
+
{
|
| 101 |
+
0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02, 0x03, 0x00, 0x01, 0x00,
|
| 102 |
+
0x01, 0x02, 0x03, 0x02, 0x03, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03,
|
| 103 |
+
0x02, 0x03, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02, 0x03,
|
| 104 |
+
},
|
| 105 |
+
{
|
| 106 |
+
0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x00, 0x00, 0x01,
|
| 107 |
+
0x01, 0x02, 0x02, 0x03, 0x03, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02,
|
| 108 |
+
0x03, 0x03, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03,
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x00, 0x01, 0x02,
|
| 112 |
+
0x03, 0x04, 0x05, 0x06, 0x07, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05,
|
| 113 |
+
0x06, 0x07, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01,
|
| 117 |
+
0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
| 118 |
+
0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
|
| 119 |
+
},
|
| 120 |
+
{
|
| 121 |
+
0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02,
|
| 122 |
+
0x03, 0x02, 0x03, 0x02, 0x03, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01,
|
| 123 |
+
0x00, 0x01, 0x02, 0x03, 0x02, 0x03, 0x02, 0x03, 0x02, 0x03,
|
| 124 |
+
},
|
| 125 |
+
{
|
| 126 |
+
0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03,
|
| 127 |
+
0x03, 0x02, 0x02, 0x03, 0x03, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00,
|
| 128 |
+
0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x02, 0x02, 0x03, 0x03,
|
| 129 |
+
},
|
| 130 |
+
{
|
| 131 |
+
0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
|
| 132 |
+
0x07, 0x04, 0x05, 0x06, 0x07, 0x00, 0x01, 0x02, 0x03, 0x00, 0x01,
|
| 133 |
+
0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x04, 0x05, 0x06, 0x07,
|
| 134 |
+
},
|
| 135 |
+
{
|
| 136 |
+
0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02, 0x02,
|
| 137 |
+
0x02, 0x03, 0x03, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01,
|
| 138 |
+
0x01, 0x01, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03,
|
| 139 |
+
},
|
| 140 |
+
{
|
| 141 |
+
0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02, 0x03, 0x04, 0x05, 0x04,
|
| 142 |
+
0x05, 0x06, 0x07, 0x06, 0x07, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03,
|
| 143 |
+
0x02, 0x03, 0x04, 0x05, 0x04, 0x05, 0x06, 0x07, 0x06, 0x07,
|
| 144 |
+
},
|
| 145 |
+
{
|
| 146 |
+
0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x04, 0x04, 0x05,
|
| 147 |
+
0x05, 0x06, 0x06, 0x07, 0x07, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02,
|
| 148 |
+
0x03, 0x03, 0x04, 0x04, 0x05, 0x05, 0x06, 0x06, 0x07, 0x07,
|
| 149 |
+
},
|
| 150 |
+
{
|
| 151 |
+
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A,
|
| 152 |
+
0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05,
|
| 153 |
+
0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
|
| 154 |
+
},
|
| 155 |
+
{
|
| 156 |
+
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
| 157 |
+
0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
|
| 158 |
+
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
|
| 159 |
+
},
|
| 160 |
+
{
|
| 161 |
+
0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00,
|
| 162 |
+
0x01, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02, 0x03, 0x02, 0x03,
|
| 163 |
+
0x02, 0x03, 0x02, 0x03, 0x02, 0x03, 0x02, 0x03, 0x02, 0x03,
|
| 164 |
+
},
|
| 165 |
+
{
|
| 166 |
+
0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01,
|
| 167 |
+
0x01, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x02, 0x02,
|
| 168 |
+
0x03, 0x03, 0x02, 0x02, 0x03, 0x03, 0x02, 0x02, 0x03, 0x03,
|
| 169 |
+
},
|
| 170 |
+
{
|
| 171 |
+
0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02,
|
| 172 |
+
0x03, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x04, 0x05,
|
| 173 |
+
0x06, 0x07, 0x04, 0x05, 0x06, 0x07, 0x04, 0x05, 0x06, 0x07,
|
| 174 |
+
},
|
| 175 |
+
{
|
| 176 |
+
0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00,
|
| 177 |
+
0x00, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03,
|
| 178 |
+
0x03, 0x03, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03,
|
| 179 |
+
},
|
| 180 |
+
{
|
| 181 |
+
0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02, 0x03, 0x00, 0x01, 0x00,
|
| 182 |
+
0x01, 0x02, 0x03, 0x02, 0x03, 0x04, 0x05, 0x04, 0x05, 0x06, 0x07,
|
| 183 |
+
0x06, 0x07, 0x04, 0x05, 0x04, 0x05, 0x06, 0x07, 0x06, 0x07,
|
| 184 |
+
},
|
| 185 |
+
{
|
| 186 |
+
0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x00, 0x00, 0x01,
|
| 187 |
+
0x01, 0x02, 0x02, 0x03, 0x03, 0x04, 0x04, 0x05, 0x05, 0x06, 0x06,
|
| 188 |
+
0x07, 0x07, 0x04, 0x04, 0x05, 0x05, 0x06, 0x06, 0x07, 0x07,
|
| 189 |
+
},
|
| 190 |
+
{
|
| 191 |
+
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x00, 0x01, 0x02,
|
| 192 |
+
0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D,
|
| 193 |
+
0x0E, 0x0F, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
|
| 194 |
+
},
|
| 195 |
+
{
|
| 196 |
+
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01,
|
| 197 |
+
0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,
|
| 198 |
+
0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03,
|
| 199 |
+
},
|
| 200 |
+
{
|
| 201 |
+
0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02,
|
| 202 |
+
0x03, 0x02, 0x03, 0x02, 0x03, 0x04, 0x05, 0x04, 0x05, 0x04, 0x05,
|
| 203 |
+
0x04, 0x05, 0x06, 0x07, 0x06, 0x07, 0x06, 0x07, 0x06, 0x07,
|
| 204 |
+
},
|
| 205 |
+
{
|
| 206 |
+
0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03,
|
| 207 |
+
0x03, 0x02, 0x02, 0x03, 0x03, 0x04, 0x04, 0x05, 0x05, 0x04, 0x04,
|
| 208 |
+
0x05, 0x05, 0x06, 0x06, 0x07, 0x07, 0x06, 0x06, 0x07, 0x07,
|
| 209 |
+
},
|
| 210 |
+
{
|
| 211 |
+
0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
|
| 212 |
+
0x07, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x08, 0x09,
|
| 213 |
+
0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x0C, 0x0D, 0x0E, 0x0F,
|
| 214 |
+
},
|
| 215 |
+
{
|
| 216 |
+
0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02, 0x02,
|
| 217 |
+
0x02, 0x03, 0x03, 0x03, 0x03, 0x04, 0x04, 0x04, 0x04, 0x05, 0x05,
|
| 218 |
+
0x05, 0x05, 0x06, 0x06, 0x06, 0x06, 0x07, 0x07, 0x07, 0x07,
|
| 219 |
+
},
|
| 220 |
+
{
|
| 221 |
+
0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02, 0x03, 0x04, 0x05, 0x04,
|
| 222 |
+
0x05, 0x06, 0x07, 0x06, 0x07, 0x08, 0x09, 0x08, 0x09, 0x0A, 0x0B,
|
| 223 |
+
0x0A, 0x0B, 0x0C, 0x0D, 0x0C, 0x0D, 0x0E, 0x0F, 0x0E, 0x0F,
|
| 224 |
+
},
|
| 225 |
+
{
|
| 226 |
+
0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x04, 0x04, 0x05,
|
| 227 |
+
0x05, 0x06, 0x06, 0x07, 0x07, 0x08, 0x08, 0x09, 0x09, 0x0A, 0x0A,
|
| 228 |
+
0x0B, 0x0B, 0x0C, 0x0C, 0x0D, 0x0D, 0x0E, 0x0E, 0x0F, 0x0F,
|
| 229 |
+
},
|
| 230 |
+
{
|
| 231 |
+
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A,
|
| 232 |
+
0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15,
|
| 233 |
+
0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F,
|
| 234 |
+
},
|
| 235 |
+
};
|
| 236 |
+
|
| 237 |
+
inline uint64_t ExtractBitsSoftware(uint64_t bitmap, uint64_t select_bitmap) {
|
| 238 |
+
// A software emulation of _pext_u64
|
| 239 |
+
|
| 240 |
+
// These checks should be inline and are likely to be common cases.
|
| 241 |
+
if (select_bitmap == ~uint64_t{0}) {
|
| 242 |
+
return bitmap;
|
| 243 |
+
} else if (select_bitmap == 0) {
|
| 244 |
+
return 0;
|
| 245 |
+
}
|
| 246 |
+
|
| 247 |
+
// Fallback to lookup table method
|
| 248 |
+
uint64_t bit_value = 0;
|
| 249 |
+
int bit_len = 0;
|
| 250 |
+
constexpr uint8_t kLookupMask = (1U << kLookupBits) - 1;
|
| 251 |
+
while (select_bitmap != 0) {
|
| 252 |
+
const auto mask_len = std::popcount(select_bitmap & kLookupMask);
|
| 253 |
+
const uint64_t value = kPextTable[select_bitmap & kLookupMask][bitmap & kLookupMask];
|
| 254 |
+
bit_value |= (value << bit_len);
|
| 255 |
+
bit_len += mask_len;
|
| 256 |
+
bitmap >>= kLookupBits;
|
| 257 |
+
select_bitmap >>= kLookupBits;
|
| 258 |
+
}
|
| 259 |
+
return bit_value;
|
| 260 |
+
}
|
| 261 |
+
|
| 262 |
+
#ifdef ARROW_HAVE_BMI2
|
| 263 |
+
|
| 264 |
+
// Use _pext_u64 on 64-bit builds, _pext_u32 on 32-bit builds,
|
| 265 |
+
# if UINTPTR_MAX == 0xFFFFFFFF
|
| 266 |
+
|
| 267 |
+
using extract_bitmap_t = uint32_t;
|
| 268 |
+
inline extract_bitmap_t ExtractBits(extract_bitmap_t bitmap,
|
| 269 |
+
extract_bitmap_t select_bitmap) {
|
| 270 |
+
return _pext_u32(bitmap, select_bitmap);
|
| 271 |
+
}
|
| 272 |
+
|
| 273 |
+
# else
|
| 274 |
+
|
| 275 |
+
using extract_bitmap_t = uint64_t;
|
| 276 |
+
inline extract_bitmap_t ExtractBits(extract_bitmap_t bitmap,
|
| 277 |
+
extract_bitmap_t select_bitmap) {
|
| 278 |
+
return _pext_u64(bitmap, select_bitmap);
|
| 279 |
+
}
|
| 280 |
+
|
| 281 |
+
# endif
|
| 282 |
+
|
| 283 |
+
#else // !defined(ARROW_HAVE_BMI2)
|
| 284 |
+
|
| 285 |
+
// Use 64-bit pext emulation when BMI2 isn't available.
|
| 286 |
+
using extract_bitmap_t = uint64_t;
|
| 287 |
+
inline extract_bitmap_t ExtractBits(extract_bitmap_t bitmap,
|
| 288 |
+
extract_bitmap_t select_bitmap) {
|
| 289 |
+
return ExtractBitsSoftware(bitmap, select_bitmap);
|
| 290 |
+
}
|
| 291 |
+
|
| 292 |
+
#endif
|
| 293 |
+
|
| 294 |
+
static constexpr int64_t kExtractBitsSize = 8 * sizeof(extract_bitmap_t);
|
| 295 |
+
|
| 296 |
+
template <bool has_repeated_parent>
|
| 297 |
+
int64_t DefLevelsBatchToBitmap(const int16_t* def_levels, const int64_t batch_size,
|
| 298 |
+
int64_t upper_bound_remaining, LevelInfo level_info,
|
| 299 |
+
::arrow::internal::FirstTimeBitmapWriter* writer) {
|
| 300 |
+
ARROW_DCHECK_LE(batch_size, kExtractBitsSize);
|
| 301 |
+
|
| 302 |
+
// Greater than level_info.def_level - 1 implies >= the def_level
|
| 303 |
+
auto defined_bitmap = static_cast<extract_bitmap_t>(::arrow::bit_util::FromLittleEndian(
|
| 304 |
+
internal::GreaterThanBitmap(def_levels, batch_size, level_info.def_level - 1)));
|
| 305 |
+
|
| 306 |
+
if (has_repeated_parent) {
|
| 307 |
+
// Greater than level_info.repeated_ancestor_def_level - 1 implies >= the
|
| 308 |
+
// repeated_ancestor_def_level
|
| 309 |
+
auto present_bitmap = static_cast<extract_bitmap_t>(
|
| 310 |
+
::arrow::bit_util::FromLittleEndian(internal::GreaterThanBitmap(
|
| 311 |
+
def_levels, batch_size, level_info.repeated_ancestor_def_level - 1)));
|
| 312 |
+
auto selected_bits = ExtractBits(defined_bitmap, present_bitmap);
|
| 313 |
+
int64_t selected_count = std::popcount(present_bitmap);
|
| 314 |
+
if (ARROW_PREDICT_FALSE(selected_count > upper_bound_remaining)) {
|
| 315 |
+
throw ParquetException("Values read exceeded upper bound");
|
| 316 |
+
}
|
| 317 |
+
writer->AppendWord(selected_bits, selected_count);
|
| 318 |
+
return std::popcount(selected_bits);
|
| 319 |
+
} else {
|
| 320 |
+
if (ARROW_PREDICT_FALSE(batch_size > upper_bound_remaining)) {
|
| 321 |
+
std::stringstream ss;
|
| 322 |
+
ss << "Values read exceeded upper bound";
|
| 323 |
+
throw ParquetException(ss.str());
|
| 324 |
+
}
|
| 325 |
+
|
| 326 |
+
writer->AppendWord(defined_bitmap, batch_size);
|
| 327 |
+
return std::popcount(defined_bitmap);
|
| 328 |
+
}
|
| 329 |
+
}
|
| 330 |
+
|
| 331 |
+
template <bool has_repeated_parent>
|
| 332 |
+
void DefLevelsToBitmapSimd(const int16_t* def_levels, int64_t num_def_levels,
|
| 333 |
+
LevelInfo level_info, ValidityBitmapInputOutput* output) {
|
| 334 |
+
::arrow::internal::FirstTimeBitmapWriter writer(
|
| 335 |
+
output->valid_bits,
|
| 336 |
+
/*start_offset=*/output->valid_bits_offset,
|
| 337 |
+
/*length=*/output->values_read_upper_bound);
|
| 338 |
+
int64_t set_count = 0;
|
| 339 |
+
output->values_read = 0;
|
| 340 |
+
int64_t values_read_remaining = output->values_read_upper_bound;
|
| 341 |
+
while (num_def_levels > kExtractBitsSize) {
|
| 342 |
+
set_count += DefLevelsBatchToBitmap<has_repeated_parent>(
|
| 343 |
+
def_levels, kExtractBitsSize, values_read_remaining, level_info, &writer);
|
| 344 |
+
def_levels += kExtractBitsSize;
|
| 345 |
+
num_def_levels -= kExtractBitsSize;
|
| 346 |
+
values_read_remaining = output->values_read_upper_bound - writer.position();
|
| 347 |
+
}
|
| 348 |
+
set_count += DefLevelsBatchToBitmap<has_repeated_parent>(
|
| 349 |
+
def_levels, num_def_levels, values_read_remaining, level_info, &writer);
|
| 350 |
+
|
| 351 |
+
output->values_read = writer.position();
|
| 352 |
+
output->null_count += output->values_read - set_count;
|
| 353 |
+
writer.Finish();
|
| 354 |
+
}
|
| 355 |
+
|
| 356 |
+
} // namespace parquet::internal::PARQUET_IMPL_NAMESPACE
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/metadata.h
ADDED
|
@@ -0,0 +1,536 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <cstdint>
|
| 21 |
+
#include <map>
|
| 22 |
+
#include <memory>
|
| 23 |
+
#include <optional>
|
| 24 |
+
#include <span>
|
| 25 |
+
#include <string>
|
| 26 |
+
#include <vector>
|
| 27 |
+
|
| 28 |
+
#include "parquet/encryption/type_fwd.h"
|
| 29 |
+
#include "parquet/index_location.h"
|
| 30 |
+
#include "parquet/platform.h"
|
| 31 |
+
#include "parquet/properties.h"
|
| 32 |
+
#include "parquet/type_fwd.h"
|
| 33 |
+
namespace parquet {
|
| 34 |
+
|
| 35 |
+
using KeyValueMetadata = ::arrow::KeyValueMetadata;
|
| 36 |
+
|
| 37 |
+
class PARQUET_EXPORT ApplicationVersion {
|
| 38 |
+
public:
|
| 39 |
+
// Known Versions with Issues
|
| 40 |
+
static const ApplicationVersion& PARQUET_251_FIXED_VERSION();
|
| 41 |
+
static const ApplicationVersion& PARQUET_816_FIXED_VERSION();
|
| 42 |
+
static const ApplicationVersion& PARQUET_CPP_FIXED_STATS_VERSION();
|
| 43 |
+
static const ApplicationVersion& PARQUET_MR_FIXED_STATS_VERSION();
|
| 44 |
+
static const ApplicationVersion& PARQUET_CPP_10353_FIXED_VERSION();
|
| 45 |
+
|
| 46 |
+
// Application that wrote the file. e.g. "IMPALA"
|
| 47 |
+
std::string application_;
|
| 48 |
+
// Build name
|
| 49 |
+
std::string build_;
|
| 50 |
+
|
| 51 |
+
// Version of the application that wrote the file, expressed as
|
| 52 |
+
// (<major>.<minor>.<patch>). Unmatched parts default to 0.
|
| 53 |
+
// "1.2.3" => {1, 2, 3}
|
| 54 |
+
// "1.2" => {1, 2, 0}
|
| 55 |
+
// "1.2-cdh5" => {1, 2, 0}
|
| 56 |
+
struct {
|
| 57 |
+
int major;
|
| 58 |
+
int minor;
|
| 59 |
+
int patch;
|
| 60 |
+
std::string unknown;
|
| 61 |
+
std::string pre_release;
|
| 62 |
+
std::string build_info;
|
| 63 |
+
} version;
|
| 64 |
+
|
| 65 |
+
ApplicationVersion() = default;
|
| 66 |
+
explicit ApplicationVersion(const std::string& created_by);
|
| 67 |
+
ApplicationVersion(std::string application, int major, int minor, int patch);
|
| 68 |
+
|
| 69 |
+
// Returns true if version is strictly less than other_version
|
| 70 |
+
bool VersionLt(const ApplicationVersion& other_version) const;
|
| 71 |
+
|
| 72 |
+
// Returns true if version is strictly equal with other_version
|
| 73 |
+
bool VersionEq(const ApplicationVersion& other_version) const;
|
| 74 |
+
|
| 75 |
+
// Checks if the Version has the correct statistics for a given column
|
| 76 |
+
bool HasCorrectStatistics(Type::type primitive, const EncodedStatistics& statistics,
|
| 77 |
+
SortOrder::type sort_order = SortOrder::SIGNED) const;
|
| 78 |
+
};
|
| 79 |
+
|
| 80 |
+
class PARQUET_EXPORT ColumnCryptoMetaData {
|
| 81 |
+
public:
|
| 82 |
+
static std::unique_ptr<ColumnCryptoMetaData> Make(const uint8_t* metadata);
|
| 83 |
+
~ColumnCryptoMetaData();
|
| 84 |
+
|
| 85 |
+
bool Equals(const ColumnCryptoMetaData& other) const;
|
| 86 |
+
|
| 87 |
+
std::shared_ptr<schema::ColumnPath> path_in_schema() const;
|
| 88 |
+
bool encrypted_with_footer_key() const;
|
| 89 |
+
const std::string& key_metadata() const;
|
| 90 |
+
|
| 91 |
+
private:
|
| 92 |
+
explicit ColumnCryptoMetaData(const uint8_t* metadata);
|
| 93 |
+
|
| 94 |
+
class ColumnCryptoMetaDataImpl;
|
| 95 |
+
std::unique_ptr<ColumnCryptoMetaDataImpl> impl_;
|
| 96 |
+
};
|
| 97 |
+
|
| 98 |
+
/// \brief Public struct for Thrift PageEncodingStats in ColumnChunkMetaData
|
| 99 |
+
struct PageEncodingStats {
|
| 100 |
+
PageType::type page_type;
|
| 101 |
+
Encoding::type encoding;
|
| 102 |
+
int32_t count;
|
| 103 |
+
};
|
| 104 |
+
|
| 105 |
+
/// \brief ColumnChunkMetaData is a proxy around format::ColumnChunkMetaData.
|
| 106 |
+
class PARQUET_EXPORT ColumnChunkMetaData {
|
| 107 |
+
public:
|
| 108 |
+
// API convenience to get a MetaData accessor
|
| 109 |
+
static std::unique_ptr<ColumnChunkMetaData> Make(
|
| 110 |
+
const void* metadata, const ColumnDescriptor* descr,
|
| 111 |
+
const ReaderProperties& properties = default_reader_properties(),
|
| 112 |
+
const ApplicationVersion* writer_version = NULLPTR, int16_t row_group_ordinal = -1,
|
| 113 |
+
int16_t column_ordinal = -1,
|
| 114 |
+
std::shared_ptr<InternalFileDecryptor> file_decryptor = NULLPTR);
|
| 115 |
+
|
| 116 |
+
~ColumnChunkMetaData();
|
| 117 |
+
|
| 118 |
+
bool Equals(const ColumnChunkMetaData& other) const;
|
| 119 |
+
|
| 120 |
+
// Byte offset of `ColumnMetaData` in `file_path()`.
|
| 121 |
+
//
|
| 122 |
+
// Note that the meaning of this field has been inconsistent among implementations
|
| 123 |
+
// so its use has since been deprecated in the Parquet specification. Modern
|
| 124 |
+
// implementations will set this to `0` to indicate that the `ColumnMetaData` is solely
|
| 125 |
+
// contained in the `ColumnChunk` struct.
|
| 126 |
+
int64_t file_offset() const;
|
| 127 |
+
|
| 128 |
+
// parameter is only used when a dataset is spread across multiple files
|
| 129 |
+
const std::string& file_path() const;
|
| 130 |
+
|
| 131 |
+
// column metadata
|
| 132 |
+
Type::type type() const;
|
| 133 |
+
int64_t num_values() const;
|
| 134 |
+
std::shared_ptr<schema::ColumnPath> path_in_schema() const;
|
| 135 |
+
bool is_stats_set() const;
|
| 136 |
+
bool is_geo_stats_set() const;
|
| 137 |
+
std::shared_ptr<Statistics> statistics() const;
|
| 138 |
+
std::shared_ptr<EncodedStatistics> encoded_statistics() const;
|
| 139 |
+
std::shared_ptr<SizeStatistics> size_statistics() const;
|
| 140 |
+
std::shared_ptr<geospatial::GeoStatistics> geo_statistics() const;
|
| 141 |
+
|
| 142 |
+
Compression::type compression() const;
|
| 143 |
+
// Indicate if the ColumnChunk compression is supported by the current
|
| 144 |
+
// compiled parquet library.
|
| 145 |
+
bool can_decompress() const;
|
| 146 |
+
|
| 147 |
+
const std::vector<Encoding::type>& encodings() const;
|
| 148 |
+
const std::vector<PageEncodingStats>& encoding_stats() const;
|
| 149 |
+
std::optional<int64_t> bloom_filter_offset() const;
|
| 150 |
+
std::optional<int64_t> bloom_filter_length() const;
|
| 151 |
+
bool has_dictionary_page() const;
|
| 152 |
+
int64_t dictionary_page_offset() const;
|
| 153 |
+
int64_t data_page_offset() const;
|
| 154 |
+
bool has_index_page() const;
|
| 155 |
+
int64_t index_page_offset() const;
|
| 156 |
+
int64_t total_compressed_size() const;
|
| 157 |
+
int64_t total_uncompressed_size() const;
|
| 158 |
+
std::unique_ptr<ColumnCryptoMetaData> crypto_metadata() const;
|
| 159 |
+
std::optional<IndexLocation> GetColumnIndexLocation() const;
|
| 160 |
+
std::optional<IndexLocation> GetOffsetIndexLocation() const;
|
| 161 |
+
const std::shared_ptr<const KeyValueMetadata>& key_value_metadata() const;
|
| 162 |
+
|
| 163 |
+
private:
|
| 164 |
+
explicit ColumnChunkMetaData(
|
| 165 |
+
const void* metadata, const ColumnDescriptor* descr, int16_t row_group_ordinal,
|
| 166 |
+
int16_t column_ordinal, const ReaderProperties& properties,
|
| 167 |
+
const ApplicationVersion* writer_version = NULLPTR,
|
| 168 |
+
std::shared_ptr<InternalFileDecryptor> file_decryptor = NULLPTR);
|
| 169 |
+
// PIMPL Idiom
|
| 170 |
+
class ColumnChunkMetaDataImpl;
|
| 171 |
+
std::unique_ptr<ColumnChunkMetaDataImpl> impl_;
|
| 172 |
+
};
|
| 173 |
+
|
| 174 |
+
/// \brief RowGroupMetaData is a proxy around format::RowGroupMetaData.
|
| 175 |
+
class PARQUET_EXPORT RowGroupMetaData {
|
| 176 |
+
public:
|
| 177 |
+
/// \brief Create a RowGroupMetaData from a serialized thrift message.
|
| 178 |
+
static std::unique_ptr<RowGroupMetaData> Make(
|
| 179 |
+
const void* metadata, const SchemaDescriptor* schema,
|
| 180 |
+
const ReaderProperties& properties = default_reader_properties(),
|
| 181 |
+
const ApplicationVersion* writer_version = NULLPTR,
|
| 182 |
+
std::shared_ptr<InternalFileDecryptor> file_decryptor = NULLPTR);
|
| 183 |
+
|
| 184 |
+
~RowGroupMetaData();
|
| 185 |
+
|
| 186 |
+
bool Equals(const RowGroupMetaData& other) const;
|
| 187 |
+
|
| 188 |
+
/// \brief The number of columns in this row group. The order must match the
|
| 189 |
+
/// parent's column ordering.
|
| 190 |
+
int num_columns() const;
|
| 191 |
+
|
| 192 |
+
/// \brief Return the ColumnChunkMetaData of the corresponding column ordinal.
|
| 193 |
+
///
|
| 194 |
+
/// WARNING, the returned object references memory location in it's parent
|
| 195 |
+
/// (RowGroupMetaData) object. Hence, the parent must outlive the returned
|
| 196 |
+
/// object.
|
| 197 |
+
///
|
| 198 |
+
/// \param[in] index of the ColumnChunkMetaData to retrieve.
|
| 199 |
+
///
|
| 200 |
+
/// \throws ParquetException if the index is out of bound.
|
| 201 |
+
std::unique_ptr<ColumnChunkMetaData> ColumnChunk(int index) const;
|
| 202 |
+
|
| 203 |
+
/// \brief Number of rows in this row group.
|
| 204 |
+
int64_t num_rows() const;
|
| 205 |
+
|
| 206 |
+
/// \brief Total byte size of all the uncompressed column data in this row group.
|
| 207 |
+
int64_t total_byte_size() const;
|
| 208 |
+
|
| 209 |
+
/// \brief Total byte size of all the compressed (and potentially encrypted)
|
| 210 |
+
/// column data in this row group.
|
| 211 |
+
///
|
| 212 |
+
/// This information is optional and may be 0 if omitted.
|
| 213 |
+
int64_t total_compressed_size() const;
|
| 214 |
+
|
| 215 |
+
/// \brief Byte offset from beginning of file to first page (data or
|
| 216 |
+
/// dictionary) in this row group
|
| 217 |
+
///
|
| 218 |
+
/// The file_offset field that this method exposes is optional. This method
|
| 219 |
+
/// will return 0 if that field is not set to a meaningful value.
|
| 220 |
+
int64_t file_offset() const;
|
| 221 |
+
// Return const-pointer to make it clear that this object is not to be copied
|
| 222 |
+
const SchemaDescriptor* schema() const;
|
| 223 |
+
// Indicate if all of the RowGroup's ColumnChunks can be decompressed.
|
| 224 |
+
bool can_decompress() const;
|
| 225 |
+
// Sorting columns of the row group if any.
|
| 226 |
+
std::vector<SortingColumn> sorting_columns() const;
|
| 227 |
+
|
| 228 |
+
private:
|
| 229 |
+
explicit RowGroupMetaData(
|
| 230 |
+
const void* metadata, const SchemaDescriptor* schema,
|
| 231 |
+
const ReaderProperties& properties,
|
| 232 |
+
const ApplicationVersion* writer_version = NULLPTR,
|
| 233 |
+
std::shared_ptr<InternalFileDecryptor> file_decryptor = NULLPTR);
|
| 234 |
+
// PIMPL Idiom
|
| 235 |
+
class RowGroupMetaDataImpl;
|
| 236 |
+
std::unique_ptr<RowGroupMetaDataImpl> impl_;
|
| 237 |
+
};
|
| 238 |
+
|
| 239 |
+
class FileMetaDataBuilder;
|
| 240 |
+
|
| 241 |
+
/// \brief FileMetaData is a proxy around format::FileMetaData.
|
| 242 |
+
class PARQUET_EXPORT FileMetaData {
|
| 243 |
+
public:
|
| 244 |
+
/// \brief Create a FileMetaData from a serialized thrift message.
|
| 245 |
+
static std::shared_ptr<FileMetaData> Make(
|
| 246 |
+
const void* serialized_metadata, uint32_t* inout_metadata_len,
|
| 247 |
+
const ReaderProperties& properties = default_reader_properties(),
|
| 248 |
+
std::shared_ptr<InternalFileDecryptor> file_decryptor = NULLPTR);
|
| 249 |
+
|
| 250 |
+
~FileMetaData();
|
| 251 |
+
|
| 252 |
+
bool Equals(const FileMetaData& other) const;
|
| 253 |
+
|
| 254 |
+
/// \brief The number of parquet "leaf" columns.
|
| 255 |
+
///
|
| 256 |
+
/// Parquet thrift definition requires that nested schema elements are
|
| 257 |
+
/// flattened. This method returns the number of columns in the flattened
|
| 258 |
+
/// version.
|
| 259 |
+
/// For instance, if the schema looks like this :
|
| 260 |
+
/// 0 foo.bar
|
| 261 |
+
/// foo.bar.baz 0
|
| 262 |
+
/// foo.bar.baz2 1
|
| 263 |
+
/// foo.qux 2
|
| 264 |
+
/// 1 foo2 3
|
| 265 |
+
/// 2 foo3 4
|
| 266 |
+
/// This method will return 5, because there are 5 "leaf" fields (so 5
|
| 267 |
+
/// flattened fields)
|
| 268 |
+
int num_columns() const;
|
| 269 |
+
|
| 270 |
+
/// \brief The number of flattened schema elements.
|
| 271 |
+
///
|
| 272 |
+
/// Parquet thrift definition requires that nested schema elements are
|
| 273 |
+
/// flattened. This method returns the total number of elements in the
|
| 274 |
+
/// flattened list.
|
| 275 |
+
int num_schema_elements() const;
|
| 276 |
+
|
| 277 |
+
/// \brief The total number of rows.
|
| 278 |
+
///
|
| 279 |
+
/// If the FileMetaData was obtained by calling `SubSet()`, this is the total
|
| 280 |
+
/// number of rows in the selected row groups.
|
| 281 |
+
int64_t num_rows() const;
|
| 282 |
+
|
| 283 |
+
/// \brief The number of row groups in the file.
|
| 284 |
+
///
|
| 285 |
+
/// If the FileMetaData was obtained by calling `SubSet()`, this is the number
|
| 286 |
+
/// of selected row groups.
|
| 287 |
+
int num_row_groups() const;
|
| 288 |
+
|
| 289 |
+
/// \brief Return the RowGroupMetaData of the corresponding row group ordinal.
|
| 290 |
+
///
|
| 291 |
+
/// WARNING, the returned object references memory location in it's parent
|
| 292 |
+
/// (FileMetaData) object. Hence, the parent must outlive the returned object.
|
| 293 |
+
///
|
| 294 |
+
/// \param[in] index of the RowGroup to retrieve.
|
| 295 |
+
///
|
| 296 |
+
/// \throws ParquetException if the index is out of bound.
|
| 297 |
+
std::unique_ptr<RowGroupMetaData> RowGroup(int index) const;
|
| 298 |
+
|
| 299 |
+
/// \brief Return the "version" of the file
|
| 300 |
+
///
|
| 301 |
+
/// WARNING: The value returned by this method is unreliable as 1) the Parquet
|
| 302 |
+
/// file metadata stores the version as a single integer and 2) some producers
|
| 303 |
+
/// are known to always write a hardcoded value. Therefore, you cannot use
|
| 304 |
+
/// this value to know which features are used in the file.
|
| 305 |
+
ParquetVersion::type version() const;
|
| 306 |
+
|
| 307 |
+
/// \brief Return the application's user-agent string of the writer.
|
| 308 |
+
const std::string& created_by() const;
|
| 309 |
+
|
| 310 |
+
/// \brief Return the application's version of the writer.
|
| 311 |
+
const ApplicationVersion& writer_version() const;
|
| 312 |
+
|
| 313 |
+
/// \brief Size of the original thrift encoded metadata footer.
|
| 314 |
+
uint32_t size() const;
|
| 315 |
+
|
| 316 |
+
/// \brief Indicate if all of the FileMetaData's RowGroups can be decompressed.
|
| 317 |
+
///
|
| 318 |
+
/// This will return false if any of the RowGroup's page is compressed with a
|
| 319 |
+
/// compression format which is not compiled in the current parquet library.
|
| 320 |
+
bool can_decompress() const;
|
| 321 |
+
|
| 322 |
+
bool is_encryption_algorithm_set() const;
|
| 323 |
+
EncryptionAlgorithm encryption_algorithm() const;
|
| 324 |
+
const std::string& footer_signing_key_metadata() const;
|
| 325 |
+
|
| 326 |
+
PARQUET_DEPRECATED(
|
| 327 |
+
"Deprecated in 24.0.0. If you need this functionality, please report an issue.")
|
| 328 |
+
bool VerifySignature(const void* signature);
|
| 329 |
+
|
| 330 |
+
void WriteTo(::arrow::io::OutputStream* dst,
|
| 331 |
+
const std::shared_ptr<Encryptor>& encryptor = NULLPTR) const;
|
| 332 |
+
|
| 333 |
+
/// \brief Return Thrift-serialized representation of the metadata as a
|
| 334 |
+
/// string
|
| 335 |
+
std::string SerializeToString() const;
|
| 336 |
+
|
| 337 |
+
// Return const-pointer to make it clear that this object is not to be copied
|
| 338 |
+
const SchemaDescriptor* schema() const;
|
| 339 |
+
|
| 340 |
+
const std::shared_ptr<const KeyValueMetadata>& key_value_metadata() const;
|
| 341 |
+
|
| 342 |
+
/// \brief Set a path to all ColumnChunk for all RowGroups.
|
| 343 |
+
///
|
| 344 |
+
/// Commonly used by systems (Dask, Spark) who generates an metadata-only
|
| 345 |
+
/// parquet file. The path is usually relative to said index file.
|
| 346 |
+
///
|
| 347 |
+
/// \param[in] path to set.
|
| 348 |
+
void set_file_path(const std::string& path);
|
| 349 |
+
|
| 350 |
+
/// \brief Merge row groups from another metadata file into this one.
|
| 351 |
+
///
|
| 352 |
+
/// The schema of the input FileMetaData must be equal to the
|
| 353 |
+
/// schema of this object.
|
| 354 |
+
///
|
| 355 |
+
/// This is used by systems who creates an aggregate metadata-only file by
|
| 356 |
+
/// concatenating the row groups of multiple files. This newly created
|
| 357 |
+
/// metadata file acts as an index of all available row groups.
|
| 358 |
+
///
|
| 359 |
+
/// \param[in] other FileMetaData to merge the row groups from.
|
| 360 |
+
///
|
| 361 |
+
/// \throws ParquetException if schemas are not equal.
|
| 362 |
+
void AppendRowGroups(const FileMetaData& other);
|
| 363 |
+
|
| 364 |
+
/// \brief Return a FileMetaData containing a subset of the row groups in this
|
| 365 |
+
/// FileMetaData.
|
| 366 |
+
std::shared_ptr<FileMetaData> Subset(const std::vector<int>& row_groups) const;
|
| 367 |
+
|
| 368 |
+
/// \brief Serialize metadata unencrypted as string
|
| 369 |
+
///
|
| 370 |
+
/// \param[in] scrub whether to remove sensitive information from the metadata.
|
| 371 |
+
/// \param[in] debug whether to serialize the metadata as Thrift (if false) or
|
| 372 |
+
/// debug text (if true).
|
| 373 |
+
std::string SerializeUnencrypted(bool scrub, bool debug) const;
|
| 374 |
+
|
| 375 |
+
private:
|
| 376 |
+
friend FileMetaDataBuilder;
|
| 377 |
+
friend class SerializedFile;
|
| 378 |
+
friend class SerializedRowGroup;
|
| 379 |
+
|
| 380 |
+
explicit FileMetaData(const void* serialized_metadata, uint32_t* metadata_len,
|
| 381 |
+
const ReaderProperties& properties,
|
| 382 |
+
std::shared_ptr<InternalFileDecryptor> file_decryptor = NULLPTR);
|
| 383 |
+
|
| 384 |
+
void set_file_decryptor(std::shared_ptr<InternalFileDecryptor> file_decryptor);
|
| 385 |
+
const std::shared_ptr<InternalFileDecryptor>& file_decryptor() const;
|
| 386 |
+
|
| 387 |
+
// Verify the signature of a plaintext footer.
|
| 388 |
+
static bool VerifySignature(std::span<const uint8_t> serialized_metadata,
|
| 389 |
+
std::span<const uint8_t> signature,
|
| 390 |
+
InternalFileDecryptor* file_decryptor);
|
| 391 |
+
|
| 392 |
+
// PIMPL Idiom
|
| 393 |
+
FileMetaData();
|
| 394 |
+
class FileMetaDataImpl;
|
| 395 |
+
std::unique_ptr<FileMetaDataImpl> impl_;
|
| 396 |
+
};
|
| 397 |
+
|
| 398 |
+
class PARQUET_EXPORT FileCryptoMetaData {
|
| 399 |
+
public:
|
| 400 |
+
// API convenience to get a MetaData accessor
|
| 401 |
+
static std::shared_ptr<FileCryptoMetaData> Make(
|
| 402 |
+
const uint8_t* serialized_metadata, uint32_t* metadata_len,
|
| 403 |
+
const ReaderProperties& properties = default_reader_properties());
|
| 404 |
+
~FileCryptoMetaData();
|
| 405 |
+
|
| 406 |
+
EncryptionAlgorithm encryption_algorithm() const;
|
| 407 |
+
const std::string& key_metadata() const;
|
| 408 |
+
|
| 409 |
+
void WriteTo(::arrow::io::OutputStream* dst) const;
|
| 410 |
+
|
| 411 |
+
private:
|
| 412 |
+
friend FileMetaDataBuilder;
|
| 413 |
+
FileCryptoMetaData(const uint8_t* serialized_metadata, uint32_t* metadata_len,
|
| 414 |
+
const ReaderProperties& properties);
|
| 415 |
+
|
| 416 |
+
// PIMPL Idiom
|
| 417 |
+
FileCryptoMetaData();
|
| 418 |
+
class FileCryptoMetaDataImpl;
|
| 419 |
+
std::unique_ptr<FileCryptoMetaDataImpl> impl_;
|
| 420 |
+
};
|
| 421 |
+
|
| 422 |
+
// Builder API
|
| 423 |
+
class PARQUET_EXPORT ColumnChunkMetaDataBuilder {
|
| 424 |
+
public:
|
| 425 |
+
// API convenience to get a MetaData reader
|
| 426 |
+
static std::unique_ptr<ColumnChunkMetaDataBuilder> Make(
|
| 427 |
+
std::shared_ptr<WriterProperties> props, const ColumnDescriptor* column);
|
| 428 |
+
|
| 429 |
+
static std::unique_ptr<ColumnChunkMetaDataBuilder> Make(
|
| 430 |
+
std::shared_ptr<WriterProperties> props, const ColumnDescriptor* column,
|
| 431 |
+
void* contents);
|
| 432 |
+
|
| 433 |
+
~ColumnChunkMetaDataBuilder();
|
| 434 |
+
|
| 435 |
+
// column chunk
|
| 436 |
+
// Used when a dataset is spread across multiple files
|
| 437 |
+
void set_file_path(const std::string& path);
|
| 438 |
+
|
| 439 |
+
// column metadata
|
| 440 |
+
void SetStatistics(const EncodedStatistics& stats);
|
| 441 |
+
void SetSizeStatistics(const SizeStatistics& size_stats);
|
| 442 |
+
|
| 443 |
+
// column geometry statistics
|
| 444 |
+
void SetGeoStatistics(const geospatial::EncodedGeoStatistics& geo_stats);
|
| 445 |
+
|
| 446 |
+
void SetKeyValueMetadata(std::shared_ptr<const KeyValueMetadata> key_value_metadata);
|
| 447 |
+
|
| 448 |
+
// get the column descriptor
|
| 449 |
+
const ColumnDescriptor* descr() const;
|
| 450 |
+
|
| 451 |
+
int64_t total_compressed_size() const;
|
| 452 |
+
// commit the metadata
|
| 453 |
+
|
| 454 |
+
void Finish(int64_t num_values, int64_t dictionary_page_offset,
|
| 455 |
+
int64_t index_page_offset, int64_t data_page_offset,
|
| 456 |
+
int64_t compressed_size, int64_t uncompressed_size, bool has_dictionary,
|
| 457 |
+
bool dictionary_fallback,
|
| 458 |
+
const std::map<Encoding::type, int32_t>& dict_encoding_stats_,
|
| 459 |
+
const std::map<Encoding::type, int32_t>& data_encoding_stats_,
|
| 460 |
+
const std::shared_ptr<Encryptor>& encryptor = NULLPTR);
|
| 461 |
+
|
| 462 |
+
// The metadata contents, suitable for passing to ColumnChunkMetaData::Make
|
| 463 |
+
const void* contents() const;
|
| 464 |
+
|
| 465 |
+
// For writing metadata at end of column chunk
|
| 466 |
+
void WriteTo(::arrow::io::OutputStream* sink);
|
| 467 |
+
|
| 468 |
+
private:
|
| 469 |
+
explicit ColumnChunkMetaDataBuilder(std::shared_ptr<WriterProperties> props,
|
| 470 |
+
const ColumnDescriptor* column);
|
| 471 |
+
explicit ColumnChunkMetaDataBuilder(std::shared_ptr<WriterProperties> props,
|
| 472 |
+
const ColumnDescriptor* column, void* contents);
|
| 473 |
+
// PIMPL Idiom
|
| 474 |
+
class ColumnChunkMetaDataBuilderImpl;
|
| 475 |
+
std::unique_ptr<ColumnChunkMetaDataBuilderImpl> impl_;
|
| 476 |
+
};
|
| 477 |
+
|
| 478 |
+
class PARQUET_EXPORT RowGroupMetaDataBuilder {
|
| 479 |
+
public:
|
| 480 |
+
// API convenience to get a MetaData reader
|
| 481 |
+
static std::unique_ptr<RowGroupMetaDataBuilder> Make(
|
| 482 |
+
std::shared_ptr<WriterProperties> props, const SchemaDescriptor* schema_,
|
| 483 |
+
void* contents);
|
| 484 |
+
|
| 485 |
+
~RowGroupMetaDataBuilder();
|
| 486 |
+
|
| 487 |
+
ColumnChunkMetaDataBuilder* NextColumnChunk();
|
| 488 |
+
int num_columns();
|
| 489 |
+
int64_t num_rows();
|
| 490 |
+
int current_column() const;
|
| 491 |
+
|
| 492 |
+
void set_num_rows(int64_t num_rows);
|
| 493 |
+
|
| 494 |
+
// commit the metadata
|
| 495 |
+
void Finish(int64_t total_bytes_written, int16_t row_group_ordinal = -1);
|
| 496 |
+
|
| 497 |
+
private:
|
| 498 |
+
explicit RowGroupMetaDataBuilder(std::shared_ptr<WriterProperties> props,
|
| 499 |
+
const SchemaDescriptor* schema_, void* contents);
|
| 500 |
+
// PIMPL Idiom
|
| 501 |
+
class RowGroupMetaDataBuilderImpl;
|
| 502 |
+
std::unique_ptr<RowGroupMetaDataBuilderImpl> impl_;
|
| 503 |
+
};
|
| 504 |
+
|
| 505 |
+
class PARQUET_EXPORT FileMetaDataBuilder {
|
| 506 |
+
public:
|
| 507 |
+
// API convenience to get a MetaData builder
|
| 508 |
+
static std::unique_ptr<FileMetaDataBuilder> Make(
|
| 509 |
+
const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props);
|
| 510 |
+
|
| 511 |
+
~FileMetaDataBuilder();
|
| 512 |
+
|
| 513 |
+
// The prior RowGroupMetaDataBuilder (if any) is destroyed
|
| 514 |
+
RowGroupMetaDataBuilder* AppendRowGroup();
|
| 515 |
+
|
| 516 |
+
// Set locations of all column chunks of a specific index kind.
|
| 517 |
+
void SetIndexLocations(IndexKind kind, const IndexLocations& locations);
|
| 518 |
+
|
| 519 |
+
// Complete the Thrift structure
|
| 520 |
+
std::unique_ptr<FileMetaData> Finish(
|
| 521 |
+
const std::shared_ptr<const KeyValueMetadata>& key_value_metadata = NULLPTR);
|
| 522 |
+
|
| 523 |
+
// crypto metadata
|
| 524 |
+
std::unique_ptr<FileCryptoMetaData> GetCryptoMetaData();
|
| 525 |
+
|
| 526 |
+
private:
|
| 527 |
+
explicit FileMetaDataBuilder(const SchemaDescriptor* schema,
|
| 528 |
+
std::shared_ptr<WriterProperties> props);
|
| 529 |
+
// PIMPL Idiom
|
| 530 |
+
class FileMetaDataBuilderImpl;
|
| 531 |
+
std::unique_ptr<FileMetaDataBuilderImpl> impl_;
|
| 532 |
+
};
|
| 533 |
+
|
| 534 |
+
PARQUET_EXPORT std::string ParquetVersionToString(ParquetVersion::type ver);
|
| 535 |
+
|
| 536 |
+
} // namespace parquet
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/page_index.h
ADDED
|
@@ -0,0 +1,391 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include "arrow/io/interfaces.h"
|
| 21 |
+
#include "parquet/encryption/type_fwd.h"
|
| 22 |
+
#include "parquet/index_location.h"
|
| 23 |
+
#include "parquet/type_fwd.h"
|
| 24 |
+
#include "parquet/types.h"
|
| 25 |
+
|
| 26 |
+
#include <optional>
|
| 27 |
+
#include <vector>
|
| 28 |
+
|
| 29 |
+
namespace parquet {
|
| 30 |
+
|
| 31 |
+
/// \brief ColumnIndex is a proxy around format::ColumnIndex.
|
| 32 |
+
class PARQUET_EXPORT ColumnIndex {
|
| 33 |
+
public:
|
| 34 |
+
/// \brief Create a ColumnIndex from a serialized thrift message.
|
| 35 |
+
static std::unique_ptr<ColumnIndex> Make(const ColumnDescriptor& descr,
|
| 36 |
+
const void* serialized_index,
|
| 37 |
+
uint32_t index_len,
|
| 38 |
+
const ReaderProperties& properties,
|
| 39 |
+
Decryptor* decryptor = NULLPTR);
|
| 40 |
+
|
| 41 |
+
virtual ~ColumnIndex() = default;
|
| 42 |
+
|
| 43 |
+
/// \brief A bitmap with a bit set for each data page that has only null values.
|
| 44 |
+
///
|
| 45 |
+
/// The length of this vector is equal to the number of data pages in the column.
|
| 46 |
+
virtual const std::vector<bool>& null_pages() const = 0;
|
| 47 |
+
|
| 48 |
+
/// \brief A vector of encoded lower bounds for each data page in this column.
|
| 49 |
+
///
|
| 50 |
+
/// `null_pages` should be inspected first, as only pages with non-null values
|
| 51 |
+
/// may have their lower bounds populated.
|
| 52 |
+
virtual const std::vector<std::string>& encoded_min_values() const = 0;
|
| 53 |
+
|
| 54 |
+
/// \brief A vector of encoded upper bounds for each data page in this column.
|
| 55 |
+
///
|
| 56 |
+
/// `null_pages` should be inspected first, as only pages with non-null values
|
| 57 |
+
/// may have their upper bounds populated.
|
| 58 |
+
virtual const std::vector<std::string>& encoded_max_values() const = 0;
|
| 59 |
+
|
| 60 |
+
/// \brief The ordering of lower and upper bounds.
|
| 61 |
+
///
|
| 62 |
+
/// The boundary order applies across all lower bounds, and all upper bounds,
|
| 63 |
+
/// respectively. However, the order between lower bounds and upper bounds
|
| 64 |
+
/// cannot be derived from this.
|
| 65 |
+
virtual BoundaryOrder::type boundary_order() const = 0;
|
| 66 |
+
|
| 67 |
+
/// \brief Whether per-page null count information is available.
|
| 68 |
+
virtual bool has_null_counts() const = 0;
|
| 69 |
+
|
| 70 |
+
/// \brief An optional vector with the number of null values in each data page.
|
| 71 |
+
///
|
| 72 |
+
/// `has_null_counts` should be called first to determine if this information is
|
| 73 |
+
/// available.
|
| 74 |
+
virtual const std::vector<int64_t>& null_counts() const = 0;
|
| 75 |
+
|
| 76 |
+
/// \brief A vector of page indices for non-null pages.
|
| 77 |
+
virtual const std::vector<int32_t>& non_null_page_indices() const = 0;
|
| 78 |
+
|
| 79 |
+
/// \brief Whether definition level histogram is available.
|
| 80 |
+
virtual bool has_definition_level_histograms() const = 0;
|
| 81 |
+
|
| 82 |
+
/// \brief Whether repetition level histogram is available.
|
| 83 |
+
virtual bool has_repetition_level_histograms() const = 0;
|
| 84 |
+
|
| 85 |
+
/// \brief List of definition level histograms for each page concatenated together.
|
| 86 |
+
virtual const std::vector<int64_t>& definition_level_histograms() const = 0;
|
| 87 |
+
|
| 88 |
+
/// \brief List of repetition level histograms for each page concatenated together.
|
| 89 |
+
virtual const std::vector<int64_t>& repetition_level_histograms() const = 0;
|
| 90 |
+
};
|
| 91 |
+
|
| 92 |
+
/// \brief Typed implementation of ColumnIndex.
|
| 93 |
+
template <typename DType>
|
| 94 |
+
class PARQUET_EXPORT TypedColumnIndex : public ColumnIndex {
|
| 95 |
+
public:
|
| 96 |
+
using T = typename DType::c_type;
|
| 97 |
+
|
| 98 |
+
/// \brief A vector of lower bounds for each data page in this column.
|
| 99 |
+
///
|
| 100 |
+
/// This is like `encoded_min_values`, but with the values decoded according to
|
| 101 |
+
/// the column's physical type.
|
| 102 |
+
/// `min_values` and `max_values` can be used together with `boundary_order`
|
| 103 |
+
/// in order to prune some data pages when searching for specific values.
|
| 104 |
+
virtual const std::vector<T>& min_values() const = 0;
|
| 105 |
+
|
| 106 |
+
/// \brief A vector of upper bounds for each data page in this column.
|
| 107 |
+
///
|
| 108 |
+
/// Just like `min_values`, but for upper bounds instead of lower bounds.
|
| 109 |
+
virtual const std::vector<T>& max_values() const = 0;
|
| 110 |
+
};
|
| 111 |
+
|
| 112 |
+
using BoolColumnIndex = TypedColumnIndex<BooleanType>;
|
| 113 |
+
using Int32ColumnIndex = TypedColumnIndex<Int32Type>;
|
| 114 |
+
using Int64ColumnIndex = TypedColumnIndex<Int64Type>;
|
| 115 |
+
using FloatColumnIndex = TypedColumnIndex<FloatType>;
|
| 116 |
+
using DoubleColumnIndex = TypedColumnIndex<DoubleType>;
|
| 117 |
+
using ByteArrayColumnIndex = TypedColumnIndex<ByteArrayType>;
|
| 118 |
+
using FLBAColumnIndex = TypedColumnIndex<FLBAType>;
|
| 119 |
+
|
| 120 |
+
/// \brief PageLocation is a proxy around format::PageLocation.
|
| 121 |
+
struct PARQUET_EXPORT PageLocation {
|
| 122 |
+
/// File offset of the data page.
|
| 123 |
+
int64_t offset;
|
| 124 |
+
/// Total compressed size of the data page and header.
|
| 125 |
+
int32_t compressed_page_size;
|
| 126 |
+
/// Row id of the first row in the page within the row group.
|
| 127 |
+
int64_t first_row_index;
|
| 128 |
+
};
|
| 129 |
+
|
| 130 |
+
/// \brief OffsetIndex is a proxy around format::OffsetIndex.
|
| 131 |
+
class PARQUET_EXPORT OffsetIndex {
|
| 132 |
+
public:
|
| 133 |
+
/// \brief Create a OffsetIndex from a serialized thrift message.
|
| 134 |
+
static std::unique_ptr<OffsetIndex> Make(const void* serialized_index,
|
| 135 |
+
uint32_t index_len,
|
| 136 |
+
const ReaderProperties& properties,
|
| 137 |
+
Decryptor* decryptor = NULLPTR);
|
| 138 |
+
|
| 139 |
+
virtual ~OffsetIndex() = default;
|
| 140 |
+
|
| 141 |
+
/// \brief A vector of locations for each data page in this column.
|
| 142 |
+
virtual const std::vector<PageLocation>& page_locations() const = 0;
|
| 143 |
+
|
| 144 |
+
/// \brief A vector of unencoded/uncompressed size of each page for BYTE_ARRAY types,
|
| 145 |
+
/// or empty for other types.
|
| 146 |
+
virtual const std::vector<int64_t>& unencoded_byte_array_data_bytes() const = 0;
|
| 147 |
+
};
|
| 148 |
+
|
| 149 |
+
/// \brief Interface for reading the page index for a Parquet row group.
|
| 150 |
+
class PARQUET_EXPORT RowGroupPageIndexReader {
|
| 151 |
+
public:
|
| 152 |
+
virtual ~RowGroupPageIndexReader() = default;
|
| 153 |
+
|
| 154 |
+
/// \brief Read column index of a column chunk.
|
| 155 |
+
///
|
| 156 |
+
/// \param[in] i column ordinal of the column chunk.
|
| 157 |
+
/// \returns column index of the column or nullptr if it does not exist.
|
| 158 |
+
/// \throws ParquetException if the index is out of bound.
|
| 159 |
+
virtual std::shared_ptr<ColumnIndex> GetColumnIndex(int32_t i) = 0;
|
| 160 |
+
|
| 161 |
+
/// \brief Read offset index of a column chunk.
|
| 162 |
+
///
|
| 163 |
+
/// \param[in] i column ordinal of the column chunk.
|
| 164 |
+
/// \returns offset index of the column or nullptr if it does not exist.
|
| 165 |
+
/// \throws ParquetException if the index is out of bound.
|
| 166 |
+
virtual std::shared_ptr<OffsetIndex> GetOffsetIndex(int32_t i) = 0;
|
| 167 |
+
};
|
| 168 |
+
|
| 169 |
+
struct PageIndexSelection {
|
| 170 |
+
/// Specifies whether to read the column index.
|
| 171 |
+
bool column_index = false;
|
| 172 |
+
/// Specifies whether to read the offset index.
|
| 173 |
+
bool offset_index = false;
|
| 174 |
+
};
|
| 175 |
+
|
| 176 |
+
PARQUET_EXPORT
|
| 177 |
+
std::ostream& operator<<(std::ostream& out, const PageIndexSelection& params);
|
| 178 |
+
|
| 179 |
+
struct RowGroupIndexReadRange {
|
| 180 |
+
/// Base start and total size of column index of all column chunks in a row group.
|
| 181 |
+
/// If none of the column chunks have column index, it is set to std::nullopt.
|
| 182 |
+
std::optional<::arrow::io::ReadRange> column_index = std::nullopt;
|
| 183 |
+
/// Base start and total size of offset index of all column chunks in a row group.
|
| 184 |
+
/// If none of the column chunks have offset index, it is set to std::nullopt.
|
| 185 |
+
std::optional<::arrow::io::ReadRange> offset_index = std::nullopt;
|
| 186 |
+
};
|
| 187 |
+
|
| 188 |
+
/// \brief Interface for reading the page index for a Parquet file.
|
| 189 |
+
class PARQUET_EXPORT PageIndexReader {
|
| 190 |
+
public:
|
| 191 |
+
virtual ~PageIndexReader() = default;
|
| 192 |
+
|
| 193 |
+
/// \brief Create a PageIndexReader instance.
|
| 194 |
+
/// \returns a PageIndexReader instance.
|
| 195 |
+
/// WARNING: The returned PageIndexReader references to all the input parameters, so
|
| 196 |
+
/// it must not outlive all of the input parameters. Usually these input parameters
|
| 197 |
+
/// come from the same ParquetFileReader object, so it must not outlive the reader
|
| 198 |
+
/// that creates this PageIndexReader.
|
| 199 |
+
static std::shared_ptr<PageIndexReader> Make(
|
| 200 |
+
::arrow::io::RandomAccessFile* input, std::shared_ptr<FileMetaData> file_metadata,
|
| 201 |
+
const ReaderProperties& properties,
|
| 202 |
+
InternalFileDecryptor* file_decryptor = NULLPTR);
|
| 203 |
+
|
| 204 |
+
/// \brief Get the page index reader of a specific row group.
|
| 205 |
+
/// \param[in] i row group ordinal to get page index reader.
|
| 206 |
+
/// \returns RowGroupPageIndexReader of the specified row group. A nullptr may or may
|
| 207 |
+
/// not be returned if the page index for the row group is unavailable. It is
|
| 208 |
+
/// the caller's responsibility to check the return value of follow-up calls
|
| 209 |
+
/// to the RowGroupPageIndexReader.
|
| 210 |
+
/// \throws ParquetException if the index is out of bound.
|
| 211 |
+
virtual std::shared_ptr<RowGroupPageIndexReader> RowGroup(int i) = 0;
|
| 212 |
+
|
| 213 |
+
/// \brief Advise the reader which part of page index will be read later.
|
| 214 |
+
///
|
| 215 |
+
/// The PageIndexReader can optionally prefetch and cache page index that
|
| 216 |
+
/// may be read later to get better performance.
|
| 217 |
+
///
|
| 218 |
+
/// The contract of this function is as below:
|
| 219 |
+
/// 1) If WillNeed() has not been called for a specific row group and the page index
|
| 220 |
+
/// exists, follow-up calls to get column index or offset index of all columns in
|
| 221 |
+
/// this row group SHOULD NOT FAIL, but the performance may not be optimal.
|
| 222 |
+
/// 2) If WillNeed() has been called for a specific row group, follow-up calls to get
|
| 223 |
+
/// page index are limited to columns and index type requested by WillNeed().
|
| 224 |
+
/// So it MAY FAIL if columns that are not requested by WillNeed() are requested.
|
| 225 |
+
/// 3) Later calls to WillNeed() MAY OVERRIDE previous calls of same row groups.
|
| 226 |
+
/// For example,
|
| 227 |
+
/// 1) If WillNeed() is not called for row group 0, then follow-up calls to read
|
| 228 |
+
/// column index and/or offset index of all columns of row group 0 should not
|
| 229 |
+
/// fail if its page index exists.
|
| 230 |
+
/// 2) If WillNeed() is called for columns 0 and 1 for row group 0, then follow-up
|
| 231 |
+
/// call to read page index of column 2 for row group 0 MAY FAIL even if its
|
| 232 |
+
/// page index exists.
|
| 233 |
+
/// 3) If WillNeed() is called for row group 0 with offset index only, then
|
| 234 |
+
/// follow-up call to read column index of row group 0 MAY FAIL even if
|
| 235 |
+
/// the column index of this column exists.
|
| 236 |
+
/// 4) If WillNeed() is called for columns 0 and 1 for row group 0, then later
|
| 237 |
+
/// call to WillNeed() for columns 1 and 2 for row group 0. The later one
|
| 238 |
+
/// overrides previous call and only columns 1 and 2 of row group 0 are allowed
|
| 239 |
+
/// to access.
|
| 240 |
+
///
|
| 241 |
+
/// \param[in] row_group_indices list of row group ordinal to read page index later.
|
| 242 |
+
/// \param[in] column_indices list of column ordinal to read page index later. If it is
|
| 243 |
+
/// empty, it means all columns in the row group will be read.
|
| 244 |
+
/// \param[in] selection which kind of page index is required later.
|
| 245 |
+
virtual void WillNeed(const std::vector<int32_t>& row_group_indices,
|
| 246 |
+
const std::vector<int32_t>& column_indices,
|
| 247 |
+
const PageIndexSelection& selection) = 0;
|
| 248 |
+
|
| 249 |
+
/// \brief Advise the reader page index of these row groups will not be read anymore.
|
| 250 |
+
///
|
| 251 |
+
/// The PageIndexReader implementation has the opportunity to cancel any prefetch or
|
| 252 |
+
/// release resource that are related to these row groups.
|
| 253 |
+
///
|
| 254 |
+
/// \param[in] row_group_indices list of row group ordinal that whose page index will
|
| 255 |
+
/// not be accessed anymore.
|
| 256 |
+
virtual void WillNotNeed(const std::vector<int32_t>& row_group_indices) = 0;
|
| 257 |
+
|
| 258 |
+
/// \brief Determine the column index and offset index ranges for the given row group.
|
| 259 |
+
///
|
| 260 |
+
/// \param[in] row_group_metadata row group metadata to get column chunk metadata.
|
| 261 |
+
/// \param[in] columns list of column ordinals to get page index. If the list is empty,
|
| 262 |
+
/// it means all columns in the row group.
|
| 263 |
+
/// \returns RowGroupIndexReadRange of the specified row group. Throws ParquetException
|
| 264 |
+
/// if the selected column ordinal is out of bound or metadata of page index
|
| 265 |
+
/// is corrupted.
|
| 266 |
+
static RowGroupIndexReadRange DeterminePageIndexRangesInRowGroup(
|
| 267 |
+
const RowGroupMetaData& row_group_metadata, const std::vector<int32_t>& columns);
|
| 268 |
+
};
|
| 269 |
+
|
| 270 |
+
/// \brief Interface for collecting column index of data pages in a column chunk.
|
| 271 |
+
class PARQUET_EXPORT ColumnIndexBuilder {
|
| 272 |
+
public:
|
| 273 |
+
/// \brief API convenience to create a ColumnIndexBuilder.
|
| 274 |
+
static std::unique_ptr<ColumnIndexBuilder> Make(const ColumnDescriptor* descr);
|
| 275 |
+
|
| 276 |
+
virtual ~ColumnIndexBuilder() = default;
|
| 277 |
+
|
| 278 |
+
/// \brief Add statistics of a data page.
|
| 279 |
+
///
|
| 280 |
+
/// If the ColumnIndexBuilder has seen any corrupted statistics, it will
|
| 281 |
+
/// not update statistics anymore.
|
| 282 |
+
///
|
| 283 |
+
/// \param stats Page statistics in the encoded form.
|
| 284 |
+
/// \param size_stats Size statistics of the page if available.
|
| 285 |
+
virtual void AddPage(const EncodedStatistics& stats,
|
| 286 |
+
const SizeStatistics& size_stats) = 0;
|
| 287 |
+
|
| 288 |
+
/// \brief Complete the column index.
|
| 289 |
+
///
|
| 290 |
+
/// Once called, AddPage() can no longer be called.
|
| 291 |
+
/// WriteTo() and Build() can only called after Finish() has been called.
|
| 292 |
+
virtual void Finish() = 0;
|
| 293 |
+
|
| 294 |
+
/// \brief Serialize the column index thrift message.
|
| 295 |
+
///
|
| 296 |
+
/// If the ColumnIndexBuilder has seen any corrupted statistics, it will
|
| 297 |
+
/// not write any data to the sink.
|
| 298 |
+
///
|
| 299 |
+
/// \param[out] sink output stream to write the serialized message.
|
| 300 |
+
/// \param[in] encryptor encryptor to encrypt the serialized column index.
|
| 301 |
+
virtual void WriteTo(::arrow::io::OutputStream* sink,
|
| 302 |
+
Encryptor* encryptor = NULLPTR) const = 0;
|
| 303 |
+
|
| 304 |
+
/// \brief Create a ColumnIndex directly.
|
| 305 |
+
///
|
| 306 |
+
/// \return If the ColumnIndexBuilder has seen any corrupted statistics, it simply
|
| 307 |
+
/// returns nullptr. Otherwise the column index is built and returned.
|
| 308 |
+
virtual std::unique_ptr<ColumnIndex> Build() const = 0;
|
| 309 |
+
};
|
| 310 |
+
|
| 311 |
+
/// \brief Interface for collecting offset index of data pages in a column chunk.
|
| 312 |
+
class PARQUET_EXPORT OffsetIndexBuilder {
|
| 313 |
+
public:
|
| 314 |
+
/// \brief API convenience to create a OffsetIndexBuilder.
|
| 315 |
+
static std::unique_ptr<OffsetIndexBuilder> Make();
|
| 316 |
+
|
| 317 |
+
virtual ~OffsetIndexBuilder() = default;
|
| 318 |
+
|
| 319 |
+
/// \brief Add page location and size stats of a data page.
|
| 320 |
+
virtual void AddPage(int64_t offset, int32_t compressed_page_size,
|
| 321 |
+
int64_t first_row_index,
|
| 322 |
+
std::optional<int64_t> unencoded_byte_array_length = {}) = 0;
|
| 323 |
+
|
| 324 |
+
/// \brief Add page location and size stats of a data page.
|
| 325 |
+
void AddPage(const PageLocation& page_location, const SizeStatistics& size_stats);
|
| 326 |
+
|
| 327 |
+
/// \brief Complete the offset index.
|
| 328 |
+
///
|
| 329 |
+
/// In the buffered row group mode, data pages are flushed into memory
|
| 330 |
+
/// sink and the OffsetIndexBuilder has only collected the relative offset
|
| 331 |
+
/// which requires adjustment once they are flushed to the file.
|
| 332 |
+
///
|
| 333 |
+
/// \param final_position Final stream offset to add for page offset adjustment.
|
| 334 |
+
virtual void Finish(int64_t final_position) = 0;
|
| 335 |
+
|
| 336 |
+
/// \brief Serialize the offset index thrift message.
|
| 337 |
+
///
|
| 338 |
+
/// \param[out] sink output stream to write the serialized message.
|
| 339 |
+
/// \param[in] encryptor encryptor to encrypt the serialized offset index.
|
| 340 |
+
virtual void WriteTo(::arrow::io::OutputStream* sink,
|
| 341 |
+
Encryptor* encryptor = NULLPTR) const = 0;
|
| 342 |
+
|
| 343 |
+
/// \brief Create an OffsetIndex directly.
|
| 344 |
+
virtual std::unique_ptr<OffsetIndex> Build() const = 0;
|
| 345 |
+
};
|
| 346 |
+
|
| 347 |
+
/// \brief Interface for collecting page index of a parquet file.
|
| 348 |
+
class PARQUET_EXPORT PageIndexBuilder {
|
| 349 |
+
public:
|
| 350 |
+
/// \brief API convenience to create a PageIndexBuilder.
|
| 351 |
+
static std::unique_ptr<PageIndexBuilder> Make(
|
| 352 |
+
const SchemaDescriptor* schema, InternalFileEncryptor* file_encryptor = NULLPTR);
|
| 353 |
+
|
| 354 |
+
virtual ~PageIndexBuilder() = default;
|
| 355 |
+
|
| 356 |
+
/// \brief Start a new row group.
|
| 357 |
+
virtual void AppendRowGroup() = 0;
|
| 358 |
+
|
| 359 |
+
/// \brief Get the ColumnIndexBuilder from column ordinal.
|
| 360 |
+
///
|
| 361 |
+
/// \param i Column ordinal.
|
| 362 |
+
/// \return ColumnIndexBuilder for the column and its memory ownership belongs to
|
| 363 |
+
/// the PageIndexBuilder.
|
| 364 |
+
virtual ColumnIndexBuilder* GetColumnIndexBuilder(int32_t i) = 0;
|
| 365 |
+
|
| 366 |
+
/// \brief Get the OffsetIndexBuilder from column ordinal.
|
| 367 |
+
///
|
| 368 |
+
/// \param i Column ordinal.
|
| 369 |
+
/// \return OffsetIndexBuilder for the column and its memory ownership belongs to
|
| 370 |
+
/// the PageIndexBuilder.
|
| 371 |
+
virtual OffsetIndexBuilder* GetOffsetIndexBuilder(int32_t i) = 0;
|
| 372 |
+
|
| 373 |
+
/// \brief Complete the page index builder and no more write is allowed.
|
| 374 |
+
virtual void Finish() = 0;
|
| 375 |
+
|
| 376 |
+
struct WriteResult {
|
| 377 |
+
IndexLocations column_index_locations;
|
| 378 |
+
IndexLocations offset_index_locations;
|
| 379 |
+
};
|
| 380 |
+
|
| 381 |
+
/// \brief Serialize the page index thrift message.
|
| 382 |
+
///
|
| 383 |
+
/// Only valid column indexes and offset indexes are serialized and their locations
|
| 384 |
+
/// are set.
|
| 385 |
+
///
|
| 386 |
+
/// \param[out] sink The output stream to write the page index.
|
| 387 |
+
/// \return The location of all page indexes.
|
| 388 |
+
virtual WriteResult WriteTo(::arrow::io::OutputStream* sink) const = 0;
|
| 389 |
+
};
|
| 390 |
+
|
| 391 |
+
} // namespace parquet
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/parquet_version.h
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#ifndef PARQUET_VERSION_H
|
| 19 |
+
#define PARQUET_VERSION_H
|
| 20 |
+
|
| 21 |
+
#define PARQUET_VERSION_MAJOR 24
|
| 22 |
+
#define PARQUET_VERSION_MINOR 0
|
| 23 |
+
#define PARQUET_VERSION_PATCH 0
|
| 24 |
+
|
| 25 |
+
#define PARQUET_SO_VERSION "2400"
|
| 26 |
+
#define PARQUET_FULL_SO_VERSION "2400.0.0"
|
| 27 |
+
|
| 28 |
+
// define the parquet created by version
|
| 29 |
+
#define CREATED_BY_VERSION "parquet-cpp-arrow version 24.0.0"
|
| 30 |
+
|
| 31 |
+
#endif // PARQUET_VERSION_H
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/platform.h
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <cstdint>
|
| 21 |
+
#include <memory>
|
| 22 |
+
|
| 23 |
+
#include "arrow/buffer.h" // IWYU pragma: export
|
| 24 |
+
#include "arrow/io/interfaces.h" // IWYU pragma: export
|
| 25 |
+
#include "arrow/status.h" // IWYU pragma: export
|
| 26 |
+
#include "arrow/type_fwd.h" // IWYU pragma: export
|
| 27 |
+
#include "arrow/util/macros.h" // IWYU pragma: export
|
| 28 |
+
|
| 29 |
+
#if defined(_WIN32) || defined(__CYGWIN__)
|
| 30 |
+
|
| 31 |
+
# ifdef _MSC_VER
|
| 32 |
+
# pragma warning(push)
|
| 33 |
+
// Disable warning for STL types usage in DLL interface
|
| 34 |
+
// https://web.archive.org/web/20130317015847/http://connect.microsoft.com/VisualStudio/feedback/details/696593/vc-10-vs-2010-basic-string-exports
|
| 35 |
+
# pragma warning(disable : 4275 4251)
|
| 36 |
+
// Disable diamond inheritance warnings
|
| 37 |
+
# pragma warning(disable : 4250)
|
| 38 |
+
// Disable macro redefinition warnings
|
| 39 |
+
# pragma warning(disable : 4005)
|
| 40 |
+
// Disable extern before exported template warnings
|
| 41 |
+
# pragma warning(disable : 4910)
|
| 42 |
+
# else
|
| 43 |
+
# pragma GCC diagnostic ignored "-Wattributes"
|
| 44 |
+
# endif
|
| 45 |
+
|
| 46 |
+
# ifdef PARQUET_STATIC
|
| 47 |
+
# define PARQUET_EXPORT
|
| 48 |
+
# elif defined(PARQUET_EXPORTING)
|
| 49 |
+
# define PARQUET_EXPORT __declspec(dllexport)
|
| 50 |
+
# else
|
| 51 |
+
# define PARQUET_EXPORT __declspec(dllimport)
|
| 52 |
+
# endif
|
| 53 |
+
|
| 54 |
+
# define PARQUET_NO_EXPORT
|
| 55 |
+
|
| 56 |
+
#else // Not Windows
|
| 57 |
+
# ifndef PARQUET_EXPORT
|
| 58 |
+
# define PARQUET_EXPORT __attribute__((visibility("default")))
|
| 59 |
+
# endif
|
| 60 |
+
# ifndef PARQUET_NO_EXPORT
|
| 61 |
+
# define PARQUET_NO_EXPORT __attribute__((visibility("hidden")))
|
| 62 |
+
# endif
|
| 63 |
+
#endif // Non-Windows
|
| 64 |
+
|
| 65 |
+
// This is a complicated topic, some reading on it:
|
| 66 |
+
// http://www.codesynthesis.com/~boris/blog/2010/01/18/dll-export-cxx-templates/
|
| 67 |
+
#if defined(_MSC_VER) || defined(__clang__)
|
| 68 |
+
# define PARQUET_TEMPLATE_CLASS_EXPORT
|
| 69 |
+
# define PARQUET_TEMPLATE_EXPORT PARQUET_EXPORT
|
| 70 |
+
#else
|
| 71 |
+
# define PARQUET_TEMPLATE_CLASS_EXPORT PARQUET_EXPORT
|
| 72 |
+
# define PARQUET_TEMPLATE_EXPORT
|
| 73 |
+
#endif
|
| 74 |
+
|
| 75 |
+
#define PARQUET_DISALLOW_COPY_AND_ASSIGN ARROW_DISALLOW_COPY_AND_ASSIGN
|
| 76 |
+
|
| 77 |
+
#define PARQUET_NORETURN ARROW_NORETURN
|
| 78 |
+
#define PARQUET_DEPRECATED ARROW_DEPRECATED
|
| 79 |
+
|
| 80 |
+
// If ARROW_VALGRIND set when compiling unit tests, also define
|
| 81 |
+
// PARQUET_VALGRIND
|
| 82 |
+
#ifdef ARROW_VALGRIND
|
| 83 |
+
# define PARQUET_VALGRIND
|
| 84 |
+
#endif
|
| 85 |
+
|
| 86 |
+
namespace parquet {
|
| 87 |
+
|
| 88 |
+
using Buffer = ::arrow::Buffer;
|
| 89 |
+
using Codec = ::arrow::util::Codec;
|
| 90 |
+
using CodecOptions = ::arrow::util::CodecOptions;
|
| 91 |
+
using Compression = ::arrow::Compression;
|
| 92 |
+
using MemoryPool = ::arrow::MemoryPool;
|
| 93 |
+
using MutableBuffer = ::arrow::MutableBuffer;
|
| 94 |
+
using ResizableBuffer = ::arrow::ResizableBuffer;
|
| 95 |
+
using ResizableBuffer = ::arrow::ResizableBuffer;
|
| 96 |
+
using ArrowInputFile = ::arrow::io::RandomAccessFile;
|
| 97 |
+
using ArrowInputStream = ::arrow::io::InputStream;
|
| 98 |
+
using ArrowOutputStream = ::arrow::io::OutputStream;
|
| 99 |
+
|
| 100 |
+
constexpr int64_t kDefaultOutputStreamSize = 1024;
|
| 101 |
+
|
| 102 |
+
constexpr int16_t kNonPageOrdinal = static_cast<int16_t>(-1);
|
| 103 |
+
|
| 104 |
+
PARQUET_EXPORT
|
| 105 |
+
std::shared_ptr<::arrow::io::BufferOutputStream> CreateOutputStream(
|
| 106 |
+
::arrow::MemoryPool* pool = ::arrow::default_memory_pool());
|
| 107 |
+
|
| 108 |
+
PARQUET_EXPORT
|
| 109 |
+
std::shared_ptr<ResizableBuffer> AllocateBuffer(
|
| 110 |
+
::arrow::MemoryPool* pool = ::arrow::default_memory_pool(), int64_t size = 0);
|
| 111 |
+
|
| 112 |
+
} // namespace parquet
|
| 113 |
+
|
| 114 |
+
#ifdef _MSC_VER
|
| 115 |
+
# pragma warning(pop)
|
| 116 |
+
#endif
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/printer.h
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <iosfwd>
|
| 21 |
+
#include <list>
|
| 22 |
+
|
| 23 |
+
#include "parquet/platform.h"
|
| 24 |
+
|
| 25 |
+
namespace parquet {
|
| 26 |
+
|
| 27 |
+
class ParquetFileReader;
|
| 28 |
+
|
| 29 |
+
class PARQUET_EXPORT ParquetFilePrinter {
|
| 30 |
+
private:
|
| 31 |
+
ParquetFileReader* fileReader;
|
| 32 |
+
|
| 33 |
+
public:
|
| 34 |
+
explicit ParquetFilePrinter(ParquetFileReader* reader) : fileReader(reader) {}
|
| 35 |
+
~ParquetFilePrinter() = default;
|
| 36 |
+
|
| 37 |
+
void DebugPrint(std::ostream& stream, std::list<int> selected_columns,
|
| 38 |
+
bool print_values = false, bool format_dump = false,
|
| 39 |
+
bool print_key_value_metadata = false,
|
| 40 |
+
const char* filename = "No Name");
|
| 41 |
+
|
| 42 |
+
void JSONPrint(std::ostream& stream, std::list<int> selected_columns,
|
| 43 |
+
const char* filename = "No Name");
|
| 44 |
+
};
|
| 45 |
+
|
| 46 |
+
} // namespace parquet
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/properties.h
ADDED
|
@@ -0,0 +1,1529 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <memory>
|
| 21 |
+
#include <string>
|
| 22 |
+
#include <unordered_map>
|
| 23 |
+
#include <unordered_set>
|
| 24 |
+
#include <utility>
|
| 25 |
+
|
| 26 |
+
#include "arrow/io/caching.h"
|
| 27 |
+
#include "arrow/type_fwd.h"
|
| 28 |
+
#include "arrow/util/compression.h"
|
| 29 |
+
#include "arrow/util/type_fwd.h"
|
| 30 |
+
#include "parquet/encryption/encryption.h"
|
| 31 |
+
#include "parquet/exception.h"
|
| 32 |
+
#include "parquet/parquet_version.h"
|
| 33 |
+
#include "parquet/platform.h"
|
| 34 |
+
#include "parquet/schema.h"
|
| 35 |
+
#include "parquet/type_fwd.h"
|
| 36 |
+
#include "parquet/types.h"
|
| 37 |
+
|
| 38 |
+
namespace parquet {
|
| 39 |
+
|
| 40 |
+
/// Controls serialization format of data pages. parquet-format v2.0.0
|
| 41 |
+
/// introduced a new data page metadata type DataPageV2 and serialized page
|
| 42 |
+
/// structure (for example, encoded levels are no longer compressed). Prior to
|
| 43 |
+
/// the completion of PARQUET-457 in 2020, this library did not implement
|
| 44 |
+
/// DataPageV2 correctly, so if you use the V2 data page format, you may have
|
| 45 |
+
/// forward compatibility issues (older versions of the library will be unable
|
| 46 |
+
/// to read the files). Note that some Parquet implementations do not implement
|
| 47 |
+
/// DataPageV2 at all.
|
| 48 |
+
enum class ParquetDataPageVersion { V1, V2 };
|
| 49 |
+
|
| 50 |
+
/// Controls the level of size statistics that are written to the file.
|
| 51 |
+
enum class SizeStatisticsLevel : uint8_t {
|
| 52 |
+
// No size statistics are written.
|
| 53 |
+
None = 0,
|
| 54 |
+
// Only column chunk size statistics are written.
|
| 55 |
+
ColumnChunk,
|
| 56 |
+
// Both size statistics in the column chunk and page index are written.
|
| 57 |
+
PageAndColumnChunk
|
| 58 |
+
};
|
| 59 |
+
|
| 60 |
+
/// Align the default buffer size to a small multiple of a page size.
|
| 61 |
+
constexpr int64_t kDefaultBufferSize = 4096 * 4;
|
| 62 |
+
|
| 63 |
+
constexpr int32_t kDefaultThriftStringSizeLimit = 100 * 1000 * 1000;
|
| 64 |
+
// Structs in the thrift definition are relatively large (at least 300 bytes).
|
| 65 |
+
// This limits total memory to the same order of magnitude as
|
| 66 |
+
// kDefaultStringSizeLimit.
|
| 67 |
+
constexpr int32_t kDefaultThriftContainerSizeLimit = 1000 * 1000;
|
| 68 |
+
|
| 69 |
+
// PARQUET-978: Minimize footer reads by reading 64 KB from the end of the file
|
| 70 |
+
constexpr int64_t kDefaultFooterReadSize = 64 * 1024;
|
| 71 |
+
|
| 72 |
+
class PARQUET_EXPORT ReaderProperties {
|
| 73 |
+
public:
|
| 74 |
+
explicit ReaderProperties(MemoryPool* pool = ::arrow::default_memory_pool())
|
| 75 |
+
: pool_(pool) {}
|
| 76 |
+
|
| 77 |
+
MemoryPool* memory_pool() const { return pool_; }
|
| 78 |
+
|
| 79 |
+
std::shared_ptr<ArrowInputStream> GetStream(std::shared_ptr<ArrowInputFile> source,
|
| 80 |
+
int64_t start, int64_t num_bytes);
|
| 81 |
+
|
| 82 |
+
/// Buffered stream reading allows the user to control the memory usage of
|
| 83 |
+
/// parquet readers. This ensure that all `RandomAccessFile::ReadAt` calls are
|
| 84 |
+
/// wrapped in a buffered reader that uses a fix sized buffer (of size
|
| 85 |
+
/// `buffer_size()`) instead of the full size of the ReadAt.
|
| 86 |
+
///
|
| 87 |
+
/// The primary reason for this control knobs is for resource control and not
|
| 88 |
+
/// performance.
|
| 89 |
+
bool is_buffered_stream_enabled() const { return buffered_stream_enabled_; }
|
| 90 |
+
/// Enable buffered stream reading.
|
| 91 |
+
void enable_buffered_stream() { buffered_stream_enabled_ = true; }
|
| 92 |
+
/// Disable buffered stream reading.
|
| 93 |
+
void disable_buffered_stream() { buffered_stream_enabled_ = false; }
|
| 94 |
+
|
| 95 |
+
bool read_dense_for_nullable() const { return read_dense_for_nullable_; }
|
| 96 |
+
void enable_read_dense_for_nullable() { read_dense_for_nullable_ = true; }
|
| 97 |
+
void disable_read_dense_for_nullable() { read_dense_for_nullable_ = false; }
|
| 98 |
+
|
| 99 |
+
/// Return the size of the buffered stream buffer.
|
| 100 |
+
int64_t buffer_size() const { return buffer_size_; }
|
| 101 |
+
/// Set the size of the buffered stream buffer in bytes.
|
| 102 |
+
void set_buffer_size(int64_t size) { buffer_size_ = size; }
|
| 103 |
+
|
| 104 |
+
/// \brief Return the size limit on thrift strings.
|
| 105 |
+
///
|
| 106 |
+
/// This limit helps prevent space and time bombs in files, but may need to
|
| 107 |
+
/// be increased in order to read files with especially large headers.
|
| 108 |
+
int32_t thrift_string_size_limit() const { return thrift_string_size_limit_; }
|
| 109 |
+
/// Set the size limit on thrift strings.
|
| 110 |
+
void set_thrift_string_size_limit(int32_t size) { thrift_string_size_limit_ = size; }
|
| 111 |
+
|
| 112 |
+
/// \brief Return the size limit on thrift containers.
|
| 113 |
+
///
|
| 114 |
+
/// This limit helps prevent space and time bombs in files, but may need to
|
| 115 |
+
/// be increased in order to read files with especially large headers.
|
| 116 |
+
int32_t thrift_container_size_limit() const { return thrift_container_size_limit_; }
|
| 117 |
+
/// Set the size limit on thrift containers.
|
| 118 |
+
void set_thrift_container_size_limit(int32_t size) {
|
| 119 |
+
thrift_container_size_limit_ = size;
|
| 120 |
+
}
|
| 121 |
+
|
| 122 |
+
/// Set the decryption properties.
|
| 123 |
+
void file_decryption_properties(std::shared_ptr<FileDecryptionProperties> decryption) {
|
| 124 |
+
file_decryption_properties_ = std::move(decryption);
|
| 125 |
+
}
|
| 126 |
+
/// Return the decryption properties.
|
| 127 |
+
const std::shared_ptr<FileDecryptionProperties>& file_decryption_properties() const {
|
| 128 |
+
return file_decryption_properties_;
|
| 129 |
+
}
|
| 130 |
+
|
| 131 |
+
bool page_checksum_verification() const { return page_checksum_verification_; }
|
| 132 |
+
void set_page_checksum_verification(bool check_crc) {
|
| 133 |
+
page_checksum_verification_ = check_crc;
|
| 134 |
+
}
|
| 135 |
+
|
| 136 |
+
// Set the default read size to read the footer from a file. For high latency
|
| 137 |
+
// file systems and files with large metadata (>64KB) this can increase performance
|
| 138 |
+
// by reducing the number of round-trips to retrieve the entire file metadata.
|
| 139 |
+
void set_footer_read_size(size_t size) { footer_read_size_ = size; }
|
| 140 |
+
size_t footer_read_size() const { return footer_read_size_; }
|
| 141 |
+
|
| 142 |
+
private:
|
| 143 |
+
MemoryPool* pool_;
|
| 144 |
+
int64_t buffer_size_ = kDefaultBufferSize;
|
| 145 |
+
int32_t thrift_string_size_limit_ = kDefaultThriftStringSizeLimit;
|
| 146 |
+
int32_t thrift_container_size_limit_ = kDefaultThriftContainerSizeLimit;
|
| 147 |
+
bool buffered_stream_enabled_ = false;
|
| 148 |
+
bool page_checksum_verification_ = false;
|
| 149 |
+
// Used with a RecordReader.
|
| 150 |
+
bool read_dense_for_nullable_ = false;
|
| 151 |
+
size_t footer_read_size_ = kDefaultFooterReadSize;
|
| 152 |
+
std::shared_ptr<FileDecryptionProperties> file_decryption_properties_;
|
| 153 |
+
};
|
| 154 |
+
|
| 155 |
+
ReaderProperties PARQUET_EXPORT default_reader_properties();
|
| 156 |
+
|
| 157 |
+
static constexpr int64_t kDefaultDataPageSize = 1024 * 1024;
|
| 158 |
+
static constexpr int64_t kDefaultMaxRowsPerPage = 20'000;
|
| 159 |
+
static constexpr bool DEFAULT_IS_DICTIONARY_ENABLED = true;
|
| 160 |
+
static constexpr int64_t DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT = kDefaultDataPageSize;
|
| 161 |
+
static constexpr int64_t DEFAULT_WRITE_BATCH_SIZE = 1024;
|
| 162 |
+
static constexpr int64_t DEFAULT_MAX_ROW_GROUP_LENGTH = 1024 * 1024;
|
| 163 |
+
static constexpr bool DEFAULT_ARE_STATISTICS_ENABLED = true;
|
| 164 |
+
static constexpr int64_t DEFAULT_MAX_STATISTICS_SIZE = 4096;
|
| 165 |
+
static constexpr Encoding::type DEFAULT_ENCODING = Encoding::UNKNOWN;
|
| 166 |
+
static const char DEFAULT_CREATED_BY[] = CREATED_BY_VERSION;
|
| 167 |
+
static constexpr Compression::type DEFAULT_COMPRESSION_TYPE = Compression::UNCOMPRESSED;
|
| 168 |
+
static constexpr bool DEFAULT_IS_PAGE_INDEX_ENABLED = true;
|
| 169 |
+
static constexpr SizeStatisticsLevel DEFAULT_SIZE_STATISTICS_LEVEL =
|
| 170 |
+
SizeStatisticsLevel::PageAndColumnChunk;
|
| 171 |
+
|
| 172 |
+
struct PARQUET_EXPORT BloomFilterOptions {
|
| 173 |
+
/// Expected number of distinct values (NDV) in the bloom filter.
|
| 174 |
+
///
|
| 175 |
+
/// Bloom filters are most effective for high-cardinality columns. A good default
|
| 176 |
+
/// is to set ndv equal to the number of rows. Lower values reduce disk usage but
|
| 177 |
+
/// may not be worthwhile for very small NDVs.
|
| 178 |
+
///
|
| 179 |
+
/// Increasing ndv (without increasing fpp) increases disk and memory usage.
|
| 180 |
+
int32_t ndv = 1 << 20;
|
| 181 |
+
|
| 182 |
+
/// False-positive probability (FPP) of the bloom filter.
|
| 183 |
+
///
|
| 184 |
+
/// Lower FPP values require more disk and memory space. For a fixed ndv, the
|
| 185 |
+
/// space requirement grows roughly proportional to log(1/fpp). Recommended
|
| 186 |
+
/// values are 0.1, 0.05, or 0.01. Very small values are counterproductive as
|
| 187 |
+
/// the bitset may exceed the size of the actual data. Set ndv appropriately
|
| 188 |
+
/// to minimize space usage.
|
| 189 |
+
///
|
| 190 |
+
/// Below is a table to demonstrate estimated size using common values.
|
| 191 |
+
///
|
| 192 |
+
/// | ndv | fpp | bits/key | size |
|
| 193 |
+
/// |:-----------|:------|:---------|:----------|
|
| 194 |
+
/// | 100,000 | 0.10 | 10.5 | 128 KiB |
|
| 195 |
+
/// | 100,000 | 0.05 | 10.5 | 128 KiB |
|
| 196 |
+
/// | 100,000 | 0.01 | 10.5 | 128 KiB |
|
| 197 |
+
/// | 1,000,000 | 0.10 | 8.4 | 1024 KiB |
|
| 198 |
+
/// | 1,000,000 | 0.05 | 8.4 | 1024 KiB |
|
| 199 |
+
/// | 1,000,000 | 0.01 | 16.8 | 2048 KiB |
|
| 200 |
+
/// | 10,000,000 | 0.10 | 6.7 | 8192 KiB |
|
| 201 |
+
/// | 10,000,000 | 0.05 | 13.4 | 16384 KiB |
|
| 202 |
+
/// | 10,000,000 | 0.01 | 13.4 | 16384 KiB |
|
| 203 |
+
double fpp = 0.05;
|
| 204 |
+
};
|
| 205 |
+
|
| 206 |
+
class PARQUET_EXPORT ColumnProperties {
|
| 207 |
+
public:
|
| 208 |
+
ColumnProperties(Encoding::type encoding = DEFAULT_ENCODING,
|
| 209 |
+
Compression::type codec = DEFAULT_COMPRESSION_TYPE,
|
| 210 |
+
bool dictionary_enabled = DEFAULT_IS_DICTIONARY_ENABLED,
|
| 211 |
+
bool statistics_enabled = DEFAULT_ARE_STATISTICS_ENABLED,
|
| 212 |
+
size_t max_stats_size = DEFAULT_MAX_STATISTICS_SIZE,
|
| 213 |
+
bool page_index_enabled = DEFAULT_IS_PAGE_INDEX_ENABLED)
|
| 214 |
+
: encoding_(encoding),
|
| 215 |
+
codec_(codec),
|
| 216 |
+
dictionary_enabled_(dictionary_enabled),
|
| 217 |
+
statistics_enabled_(statistics_enabled),
|
| 218 |
+
max_stats_size_(max_stats_size),
|
| 219 |
+
page_index_enabled_(page_index_enabled) {}
|
| 220 |
+
|
| 221 |
+
void set_encoding(Encoding::type encoding) { encoding_ = encoding; }
|
| 222 |
+
|
| 223 |
+
void set_compression(Compression::type codec) { codec_ = codec; }
|
| 224 |
+
|
| 225 |
+
void set_dictionary_enabled(bool dictionary_enabled) {
|
| 226 |
+
dictionary_enabled_ = dictionary_enabled;
|
| 227 |
+
}
|
| 228 |
+
|
| 229 |
+
void set_statistics_enabled(bool statistics_enabled) {
|
| 230 |
+
statistics_enabled_ = statistics_enabled;
|
| 231 |
+
}
|
| 232 |
+
|
| 233 |
+
void set_max_statistics_size(size_t max_stats_size) {
|
| 234 |
+
max_stats_size_ = max_stats_size;
|
| 235 |
+
}
|
| 236 |
+
|
| 237 |
+
void set_compression_level(int compression_level) {
|
| 238 |
+
if (!codec_options_) {
|
| 239 |
+
codec_options_ = std::make_shared<CodecOptions>();
|
| 240 |
+
}
|
| 241 |
+
codec_options_->compression_level = compression_level;
|
| 242 |
+
}
|
| 243 |
+
|
| 244 |
+
void set_codec_options(const std::shared_ptr<CodecOptions>& codec_options) {
|
| 245 |
+
codec_options_ = codec_options;
|
| 246 |
+
}
|
| 247 |
+
|
| 248 |
+
void set_page_index_enabled(bool page_index_enabled) {
|
| 249 |
+
page_index_enabled_ = page_index_enabled;
|
| 250 |
+
}
|
| 251 |
+
|
| 252 |
+
void set_bloom_filter_options(const BloomFilterOptions& bloom_filter_options) {
|
| 253 |
+
if (bloom_filter_options.fpp >= 1.0 || bloom_filter_options.fpp <= 0.0) {
|
| 254 |
+
throw ParquetException(
|
| 255 |
+
"Bloom filter false positive probability must be in (0.0, 1.0), got " +
|
| 256 |
+
std::to_string(bloom_filter_options.fpp));
|
| 257 |
+
}
|
| 258 |
+
bloom_filter_options_ = bloom_filter_options;
|
| 259 |
+
}
|
| 260 |
+
|
| 261 |
+
Encoding::type encoding() const { return encoding_; }
|
| 262 |
+
|
| 263 |
+
Compression::type compression() const { return codec_; }
|
| 264 |
+
|
| 265 |
+
bool dictionary_enabled() const { return dictionary_enabled_; }
|
| 266 |
+
|
| 267 |
+
bool statistics_enabled() const { return statistics_enabled_; }
|
| 268 |
+
|
| 269 |
+
size_t max_statistics_size() const { return max_stats_size_; }
|
| 270 |
+
|
| 271 |
+
int compression_level() const {
|
| 272 |
+
if (!codec_options_) {
|
| 273 |
+
return ::arrow::util::kUseDefaultCompressionLevel;
|
| 274 |
+
}
|
| 275 |
+
return codec_options_->compression_level;
|
| 276 |
+
}
|
| 277 |
+
|
| 278 |
+
const std::shared_ptr<CodecOptions>& codec_options() const { return codec_options_; }
|
| 279 |
+
|
| 280 |
+
bool page_index_enabled() const { return page_index_enabled_; }
|
| 281 |
+
|
| 282 |
+
std::optional<BloomFilterOptions> bloom_filter_options() const {
|
| 283 |
+
return bloom_filter_options_;
|
| 284 |
+
}
|
| 285 |
+
|
| 286 |
+
bool bloom_filter_enabled() const { return bloom_filter_options_.has_value(); }
|
| 287 |
+
|
| 288 |
+
private:
|
| 289 |
+
Encoding::type encoding_;
|
| 290 |
+
Compression::type codec_;
|
| 291 |
+
bool dictionary_enabled_;
|
| 292 |
+
bool statistics_enabled_;
|
| 293 |
+
size_t max_stats_size_;
|
| 294 |
+
std::shared_ptr<CodecOptions> codec_options_;
|
| 295 |
+
bool page_index_enabled_;
|
| 296 |
+
std::optional<BloomFilterOptions> bloom_filter_options_;
|
| 297 |
+
};
|
| 298 |
+
|
| 299 |
+
// EXPERIMENTAL: Options for content-defined chunking.
|
| 300 |
+
///
|
| 301 |
+
/// Content-defined chunking is an experimental feature that optimizes parquet
|
| 302 |
+
/// files for content addressable storage (CAS) systems by writing data pages
|
| 303 |
+
/// according to content-defined chunk boundaries. This allows for more
|
| 304 |
+
/// efficient deduplication of data across files, hence more efficient network
|
| 305 |
+
/// transfers and storage.
|
| 306 |
+
/// Each content-defined chunk is written as a separate parquet data page. The
|
| 307 |
+
/// following options control the chunks' size and the chunking process. Note
|
| 308 |
+
/// that the chunk size is calculated based on the logical value of the data,
|
| 309 |
+
/// before any encoding or compression is applied.
|
| 310 |
+
struct PARQUET_EXPORT CdcOptions {
|
| 311 |
+
/// Minimum chunk size in bytes, default is 256 KiB
|
| 312 |
+
/// The rolling hash will not be updated until this size is reached for each chunk.
|
| 313 |
+
/// Note that all data sent through the hash function is counted towards the chunk
|
| 314 |
+
/// size, including definition and repetition levels if present.
|
| 315 |
+
int64_t min_chunk_size = 256 * 1024;
|
| 316 |
+
/// Maximum chunk size in bytes, default is 1024 KiB
|
| 317 |
+
/// The chunker will create a new chunk whenever the chunk size exceeds this value.
|
| 318 |
+
/// Note that the parquet writer has a related `pagesize` property that controls
|
| 319 |
+
/// the maximum size of a parquet data page after encoding. While setting
|
| 320 |
+
/// `pagesize` to a smaller value than `max_chunk_size` doesn't affect the
|
| 321 |
+
/// chunking effectiveness, it results in more small parquet data pages.
|
| 322 |
+
int64_t max_chunk_size = 1024 * 1024;
|
| 323 |
+
/// Number of bit adjustment to the gearhash mask in order to center the chunk size
|
| 324 |
+
/// around the average size more aggressively, default is 0
|
| 325 |
+
/// Increasing the normalization level increases the probability of finding a chunk,
|
| 326 |
+
/// improving the deduplication ratio, but also increasing the number of small chunks
|
| 327 |
+
/// resulting in many small parquet data pages. The default value provides a good
|
| 328 |
+
/// balance between deduplication ratio and fragmentation.
|
| 329 |
+
/// Use norm_level=1 or norm_level=2 to reach a higher deduplication ratio at the
|
| 330 |
+
/// expense of fragmentation. Negative values can also be used to reduce the
|
| 331 |
+
/// probability of finding a chunk, resulting in larger chunks and fewer data pages.
|
| 332 |
+
/// Note that values outside [-3, 3] are not recommended, prefer using the default
|
| 333 |
+
/// value of 0 for most use cases.
|
| 334 |
+
int norm_level = 0;
|
| 335 |
+
};
|
| 336 |
+
|
| 337 |
+
class PARQUET_EXPORT WriterProperties {
|
| 338 |
+
public:
|
| 339 |
+
class PARQUET_EXPORT Builder {
|
| 340 |
+
public:
|
| 341 |
+
Builder()
|
| 342 |
+
: pool_(::arrow::default_memory_pool()),
|
| 343 |
+
dictionary_pagesize_limit_(DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT),
|
| 344 |
+
write_batch_size_(DEFAULT_WRITE_BATCH_SIZE),
|
| 345 |
+
max_row_group_length_(DEFAULT_MAX_ROW_GROUP_LENGTH),
|
| 346 |
+
pagesize_(kDefaultDataPageSize),
|
| 347 |
+
max_rows_per_page_(kDefaultMaxRowsPerPage),
|
| 348 |
+
version_(ParquetVersion::PARQUET_2_6),
|
| 349 |
+
data_page_version_(ParquetDataPageVersion::V1),
|
| 350 |
+
created_by_(DEFAULT_CREATED_BY),
|
| 351 |
+
store_decimal_as_integer_(false),
|
| 352 |
+
page_checksum_enabled_(false),
|
| 353 |
+
size_statistics_level_(DEFAULT_SIZE_STATISTICS_LEVEL),
|
| 354 |
+
content_defined_chunking_enabled_(false),
|
| 355 |
+
content_defined_chunking_options_({}) {}
|
| 356 |
+
|
| 357 |
+
explicit Builder(const WriterProperties& properties)
|
| 358 |
+
: pool_(properties.memory_pool()),
|
| 359 |
+
dictionary_pagesize_limit_(properties.dictionary_pagesize_limit()),
|
| 360 |
+
write_batch_size_(properties.write_batch_size()),
|
| 361 |
+
max_row_group_length_(properties.max_row_group_length()),
|
| 362 |
+
pagesize_(properties.data_pagesize()),
|
| 363 |
+
max_rows_per_page_(properties.max_rows_per_page()),
|
| 364 |
+
version_(properties.version()),
|
| 365 |
+
data_page_version_(properties.data_page_version()),
|
| 366 |
+
created_by_(properties.created_by()),
|
| 367 |
+
store_decimal_as_integer_(properties.store_decimal_as_integer()),
|
| 368 |
+
page_checksum_enabled_(properties.page_checksum_enabled()),
|
| 369 |
+
size_statistics_level_(properties.size_statistics_level()),
|
| 370 |
+
sorting_columns_(properties.sorting_columns()),
|
| 371 |
+
default_column_properties_(properties.default_column_properties()),
|
| 372 |
+
content_defined_chunking_enabled_(
|
| 373 |
+
properties.content_defined_chunking_enabled()),
|
| 374 |
+
content_defined_chunking_options_(
|
| 375 |
+
properties.content_defined_chunking_options()) {
|
| 376 |
+
CopyColumnSpecificProperties(properties);
|
| 377 |
+
}
|
| 378 |
+
|
| 379 |
+
/// \brief EXPERIMENTAL: Use content-defined page chunking for all columns.
|
| 380 |
+
///
|
| 381 |
+
/// Optimize parquet files for content addressable storage (CAS) systems by writing
|
| 382 |
+
/// data pages according to content-defined chunk boundaries. This allows for more
|
| 383 |
+
/// efficient deduplication of data across files, hence more efficient network
|
| 384 |
+
/// transfers and storage. The chunking is based on a rolling hash algorithm that
|
| 385 |
+
/// identifies chunk boundaries based on the actual content of the data.
|
| 386 |
+
///
|
| 387 |
+
/// Note that only the WriteArrow() interface is supported at the moment.
|
| 388 |
+
Builder* enable_content_defined_chunking() {
|
| 389 |
+
content_defined_chunking_enabled_ = true;
|
| 390 |
+
return this;
|
| 391 |
+
}
|
| 392 |
+
|
| 393 |
+
/// \brief EXPERIMENTAL: Disable content-defined page chunking for all columns.
|
| 394 |
+
Builder* disable_content_defined_chunking() {
|
| 395 |
+
content_defined_chunking_enabled_ = false;
|
| 396 |
+
return this;
|
| 397 |
+
}
|
| 398 |
+
|
| 399 |
+
/// \brief EXPERIMENTAL: Specify content-defined chunking options, see CdcOptions.
|
| 400 |
+
Builder* content_defined_chunking_options(const CdcOptions& options) {
|
| 401 |
+
content_defined_chunking_options_ = options;
|
| 402 |
+
return this;
|
| 403 |
+
}
|
| 404 |
+
|
| 405 |
+
/// Specify the memory pool for the writer. Default default_memory_pool.
|
| 406 |
+
Builder* memory_pool(MemoryPool* pool) {
|
| 407 |
+
pool_ = pool;
|
| 408 |
+
return this;
|
| 409 |
+
}
|
| 410 |
+
|
| 411 |
+
/// Enable dictionary encoding in general for all columns. Default
|
| 412 |
+
/// enabled.
|
| 413 |
+
Builder* enable_dictionary() {
|
| 414 |
+
default_column_properties_.set_dictionary_enabled(true);
|
| 415 |
+
return this;
|
| 416 |
+
}
|
| 417 |
+
|
| 418 |
+
/// Disable dictionary encoding in general for all columns. Default
|
| 419 |
+
/// enabled.
|
| 420 |
+
Builder* disable_dictionary() {
|
| 421 |
+
default_column_properties_.set_dictionary_enabled(false);
|
| 422 |
+
return this;
|
| 423 |
+
}
|
| 424 |
+
|
| 425 |
+
/// Enable dictionary encoding for column specified by `path`. Default
|
| 426 |
+
/// enabled.
|
| 427 |
+
Builder* enable_dictionary(const std::string& path) {
|
| 428 |
+
dictionary_enabled_[path] = true;
|
| 429 |
+
return this;
|
| 430 |
+
}
|
| 431 |
+
|
| 432 |
+
/// Enable dictionary encoding for column specified by `path`. Default
|
| 433 |
+
/// enabled.
|
| 434 |
+
Builder* enable_dictionary(const std::shared_ptr<schema::ColumnPath>& path) {
|
| 435 |
+
return this->enable_dictionary(path->ToDotString());
|
| 436 |
+
}
|
| 437 |
+
|
| 438 |
+
/// Disable dictionary encoding for column specified by `path`. Default
|
| 439 |
+
/// enabled.
|
| 440 |
+
Builder* disable_dictionary(const std::string& path) {
|
| 441 |
+
dictionary_enabled_[path] = false;
|
| 442 |
+
return this;
|
| 443 |
+
}
|
| 444 |
+
|
| 445 |
+
/// Disable dictionary encoding for column specified by `path`. Default
|
| 446 |
+
/// enabled.
|
| 447 |
+
Builder* disable_dictionary(const std::shared_ptr<schema::ColumnPath>& path) {
|
| 448 |
+
return this->disable_dictionary(path->ToDotString());
|
| 449 |
+
}
|
| 450 |
+
|
| 451 |
+
/// Specify the dictionary page size limit per row group. Default 1MB.
|
| 452 |
+
Builder* dictionary_pagesize_limit(int64_t dictionary_psize_limit) {
|
| 453 |
+
dictionary_pagesize_limit_ = dictionary_psize_limit;
|
| 454 |
+
return this;
|
| 455 |
+
}
|
| 456 |
+
|
| 457 |
+
/// Specify the write batch size while writing batches of Arrow values
|
| 458 |
+
/// into Parquet. Default 1024.
|
| 459 |
+
Builder* write_batch_size(int64_t write_batch_size) {
|
| 460 |
+
write_batch_size_ = write_batch_size;
|
| 461 |
+
return this;
|
| 462 |
+
}
|
| 463 |
+
|
| 464 |
+
/// Specify the max number of rows to put in a single row group.
|
| 465 |
+
/// Default 1Mi rows.
|
| 466 |
+
Builder* max_row_group_length(int64_t max_row_group_length) {
|
| 467 |
+
max_row_group_length_ = max_row_group_length;
|
| 468 |
+
return this;
|
| 469 |
+
}
|
| 470 |
+
|
| 471 |
+
/// Specify the data page size.
|
| 472 |
+
/// Default 1MB.
|
| 473 |
+
Builder* data_pagesize(int64_t pg_size) {
|
| 474 |
+
pagesize_ = pg_size;
|
| 475 |
+
return this;
|
| 476 |
+
}
|
| 477 |
+
|
| 478 |
+
/// Specify the maximum number of rows per data page.
|
| 479 |
+
/// Default 20K rows.
|
| 480 |
+
Builder* max_rows_per_page(int64_t max_rows) {
|
| 481 |
+
max_rows_per_page_ = max_rows;
|
| 482 |
+
return this;
|
| 483 |
+
}
|
| 484 |
+
|
| 485 |
+
/// Specify the data page version.
|
| 486 |
+
/// Default V1.
|
| 487 |
+
Builder* data_page_version(ParquetDataPageVersion data_page_version) {
|
| 488 |
+
data_page_version_ = data_page_version;
|
| 489 |
+
return this;
|
| 490 |
+
}
|
| 491 |
+
|
| 492 |
+
/// Specify the Parquet file version.
|
| 493 |
+
/// Default PARQUET_2_6.
|
| 494 |
+
Builder* version(ParquetVersion::type version) {
|
| 495 |
+
version_ = version;
|
| 496 |
+
return this;
|
| 497 |
+
}
|
| 498 |
+
|
| 499 |
+
Builder* created_by(const std::string& created_by) {
|
| 500 |
+
created_by_ = created_by;
|
| 501 |
+
return this;
|
| 502 |
+
}
|
| 503 |
+
|
| 504 |
+
Builder* enable_page_checksum() {
|
| 505 |
+
page_checksum_enabled_ = true;
|
| 506 |
+
return this;
|
| 507 |
+
}
|
| 508 |
+
|
| 509 |
+
Builder* disable_page_checksum() {
|
| 510 |
+
page_checksum_enabled_ = false;
|
| 511 |
+
return this;
|
| 512 |
+
}
|
| 513 |
+
|
| 514 |
+
/// \brief Define the encoding that is used when we don't utilise dictionary encoding.
|
| 515 |
+
//
|
| 516 |
+
/// This is only applied if dictionary encoding is disabled. If the dictionary grows
|
| 517 |
+
/// too large we always fall back to the PLAIN encoding.
|
| 518 |
+
Builder* encoding(Encoding::type encoding_type) {
|
| 519 |
+
if (encoding_type == Encoding::PLAIN_DICTIONARY ||
|
| 520 |
+
encoding_type == Encoding::RLE_DICTIONARY) {
|
| 521 |
+
throw ParquetException("Can't use dictionary encoding as fallback encoding");
|
| 522 |
+
}
|
| 523 |
+
|
| 524 |
+
default_column_properties_.set_encoding(encoding_type);
|
| 525 |
+
return this;
|
| 526 |
+
}
|
| 527 |
+
|
| 528 |
+
/// \brief Define the encoding that is used when we don't utilise dictionary encoding.
|
| 529 |
+
//
|
| 530 |
+
/// This is only applied if dictionary encoding is disabled. If the dictionary grows
|
| 531 |
+
/// too large we always fall back to the PLAIN encoding.
|
| 532 |
+
Builder* encoding(const std::string& path, Encoding::type encoding_type) {
|
| 533 |
+
if (encoding_type == Encoding::PLAIN_DICTIONARY ||
|
| 534 |
+
encoding_type == Encoding::RLE_DICTIONARY) {
|
| 535 |
+
throw ParquetException("Can't use dictionary encoding as fallback encoding");
|
| 536 |
+
}
|
| 537 |
+
|
| 538 |
+
encodings_[path] = encoding_type;
|
| 539 |
+
return this;
|
| 540 |
+
}
|
| 541 |
+
|
| 542 |
+
/// \brief Define the encoding that is used when we don't utilise dictionary encoding.
|
| 543 |
+
//
|
| 544 |
+
/// This is only applied if dictionary encoding is disabled. If the dictionary grows
|
| 545 |
+
/// too large we always fall back to the PLAIN encoding.
|
| 546 |
+
Builder* encoding(const std::shared_ptr<schema::ColumnPath>& path,
|
| 547 |
+
Encoding::type encoding_type) {
|
| 548 |
+
return this->encoding(path->ToDotString(), encoding_type);
|
| 549 |
+
}
|
| 550 |
+
|
| 551 |
+
/// Specify compression codec in general for all columns.
|
| 552 |
+
/// Default UNCOMPRESSED.
|
| 553 |
+
Builder* compression(Compression::type codec) {
|
| 554 |
+
default_column_properties_.set_compression(codec);
|
| 555 |
+
return this;
|
| 556 |
+
}
|
| 557 |
+
|
| 558 |
+
/// Specify max statistics size to store min max value.
|
| 559 |
+
/// Default 4KB.
|
| 560 |
+
Builder* max_statistics_size(size_t max_stats_sz) {
|
| 561 |
+
default_column_properties_.set_max_statistics_size(max_stats_sz);
|
| 562 |
+
return this;
|
| 563 |
+
}
|
| 564 |
+
|
| 565 |
+
/// Specify compression codec for the column specified by `path`.
|
| 566 |
+
/// Default UNCOMPRESSED.
|
| 567 |
+
Builder* compression(const std::string& path, Compression::type codec) {
|
| 568 |
+
codecs_[path] = codec;
|
| 569 |
+
return this;
|
| 570 |
+
}
|
| 571 |
+
|
| 572 |
+
/// Specify compression codec for the column specified by `path`.
|
| 573 |
+
/// Default UNCOMPRESSED.
|
| 574 |
+
Builder* compression(const std::shared_ptr<schema::ColumnPath>& path,
|
| 575 |
+
Compression::type codec) {
|
| 576 |
+
return this->compression(path->ToDotString(), codec);
|
| 577 |
+
}
|
| 578 |
+
|
| 579 |
+
/// \brief Specify the default compression level for the compressor in
|
| 580 |
+
/// every column. In case a column does not have an explicitly specified
|
| 581 |
+
/// compression level, the default one would be used.
|
| 582 |
+
///
|
| 583 |
+
/// The provided compression level is compressor specific. The user would
|
| 584 |
+
/// have to familiarize oneself with the available levels for the selected
|
| 585 |
+
/// compressor. If the compressor does not allow for selecting different
|
| 586 |
+
/// compression levels, calling this function would not have any effect.
|
| 587 |
+
/// Parquet and Arrow do not validate the passed compression level. If no
|
| 588 |
+
/// level is selected by the user or if the special
|
| 589 |
+
/// std::numeric_limits<int>::min() value is passed, then Arrow selects the
|
| 590 |
+
/// compression level.
|
| 591 |
+
///
|
| 592 |
+
/// If other compressor-specific options need to be set in addition to the compression
|
| 593 |
+
/// level, use the codec_options method.
|
| 594 |
+
Builder* compression_level(int compression_level) {
|
| 595 |
+
default_column_properties_.set_compression_level(compression_level);
|
| 596 |
+
return this;
|
| 597 |
+
}
|
| 598 |
+
|
| 599 |
+
/// \brief Specify a compression level for the compressor for the column
|
| 600 |
+
/// described by path.
|
| 601 |
+
///
|
| 602 |
+
/// The provided compression level is compressor specific. The user would
|
| 603 |
+
/// have to familiarize oneself with the available levels for the selected
|
| 604 |
+
/// compressor. If the compressor does not allow for selecting different
|
| 605 |
+
/// compression levels, calling this function would not have any effect.
|
| 606 |
+
/// Parquet and Arrow do not validate the passed compression level. If no
|
| 607 |
+
/// level is selected by the user or if the special
|
| 608 |
+
/// std::numeric_limits<int>::min() value is passed, then Arrow selects the
|
| 609 |
+
/// compression level.
|
| 610 |
+
Builder* compression_level(const std::string& path, int compression_level) {
|
| 611 |
+
if (!codec_options_[path]) {
|
| 612 |
+
codec_options_[path] = std::make_shared<CodecOptions>();
|
| 613 |
+
}
|
| 614 |
+
codec_options_[path]->compression_level = compression_level;
|
| 615 |
+
return this;
|
| 616 |
+
}
|
| 617 |
+
|
| 618 |
+
/// \brief Specify a compression level for the compressor for the column
|
| 619 |
+
/// described by path.
|
| 620 |
+
///
|
| 621 |
+
/// The provided compression level is compressor specific. The user would
|
| 622 |
+
/// have to familiarize oneself with the available levels for the selected
|
| 623 |
+
/// compressor. If the compressor does not allow for selecting different
|
| 624 |
+
/// compression levels, calling this function would not have any effect.
|
| 625 |
+
/// Parquet and Arrow do not validate the passed compression level. If no
|
| 626 |
+
/// level is selected by the user or if the special
|
| 627 |
+
/// std::numeric_limits<int>::min() value is passed, then Arrow selects the
|
| 628 |
+
/// compression level.
|
| 629 |
+
Builder* compression_level(const std::shared_ptr<schema::ColumnPath>& path,
|
| 630 |
+
int compression_level) {
|
| 631 |
+
return this->compression_level(path->ToDotString(), compression_level);
|
| 632 |
+
}
|
| 633 |
+
|
| 634 |
+
/// \brief Specify the default codec options for the compressor in
|
| 635 |
+
/// every column.
|
| 636 |
+
///
|
| 637 |
+
/// The codec options allow configuring the compression level as well
|
| 638 |
+
/// as other codec-specific options.
|
| 639 |
+
Builder* codec_options(
|
| 640 |
+
const std::shared_ptr<::arrow::util::CodecOptions>& codec_options) {
|
| 641 |
+
default_column_properties_.set_codec_options(codec_options);
|
| 642 |
+
return this;
|
| 643 |
+
}
|
| 644 |
+
|
| 645 |
+
/// \brief Specify the codec options for the compressor for the column
|
| 646 |
+
/// described by path.
|
| 647 |
+
Builder* codec_options(
|
| 648 |
+
const std::string& path,
|
| 649 |
+
const std::shared_ptr<::arrow::util::CodecOptions>& codec_options) {
|
| 650 |
+
codec_options_[path] = codec_options;
|
| 651 |
+
return this;
|
| 652 |
+
}
|
| 653 |
+
|
| 654 |
+
/// \brief Specify the codec options for the compressor for the column
|
| 655 |
+
/// described by path.
|
| 656 |
+
Builder* codec_options(
|
| 657 |
+
const std::shared_ptr<schema::ColumnPath>& path,
|
| 658 |
+
const std::shared_ptr<::arrow::util::CodecOptions>& codec_options) {
|
| 659 |
+
return this->codec_options(path->ToDotString(), codec_options);
|
| 660 |
+
}
|
| 661 |
+
|
| 662 |
+
/// Define the file encryption properties.
|
| 663 |
+
/// Default NULL.
|
| 664 |
+
Builder* encryption(
|
| 665 |
+
std::shared_ptr<FileEncryptionProperties> file_encryption_properties) {
|
| 666 |
+
file_encryption_properties_ = std::move(file_encryption_properties);
|
| 667 |
+
return this;
|
| 668 |
+
}
|
| 669 |
+
|
| 670 |
+
/// Enable statistics in general.
|
| 671 |
+
/// Default enabled.
|
| 672 |
+
Builder* enable_statistics() {
|
| 673 |
+
default_column_properties_.set_statistics_enabled(true);
|
| 674 |
+
return this;
|
| 675 |
+
}
|
| 676 |
+
|
| 677 |
+
/// Disable statistics in general.
|
| 678 |
+
/// Default enabled.
|
| 679 |
+
Builder* disable_statistics() {
|
| 680 |
+
default_column_properties_.set_statistics_enabled(false);
|
| 681 |
+
return this;
|
| 682 |
+
}
|
| 683 |
+
|
| 684 |
+
/// Enable statistics for the column specified by `path`.
|
| 685 |
+
/// Default enabled.
|
| 686 |
+
Builder* enable_statistics(const std::string& path) {
|
| 687 |
+
statistics_enabled_[path] = true;
|
| 688 |
+
return this;
|
| 689 |
+
}
|
| 690 |
+
|
| 691 |
+
/// Enable statistics for the column specified by `path`.
|
| 692 |
+
/// Default enabled.
|
| 693 |
+
Builder* enable_statistics(const std::shared_ptr<schema::ColumnPath>& path) {
|
| 694 |
+
return this->enable_statistics(path->ToDotString());
|
| 695 |
+
}
|
| 696 |
+
|
| 697 |
+
/// Define the sorting columns.
|
| 698 |
+
/// Default empty.
|
| 699 |
+
///
|
| 700 |
+
/// If sorting columns are set, user should ensure that records
|
| 701 |
+
/// are sorted by sorting columns. Otherwise, the storing data
|
| 702 |
+
/// will be inconsistent with sorting_columns metadata.
|
| 703 |
+
Builder* set_sorting_columns(std::vector<SortingColumn> sorting_columns) {
|
| 704 |
+
sorting_columns_ = std::move(sorting_columns);
|
| 705 |
+
return this;
|
| 706 |
+
}
|
| 707 |
+
|
| 708 |
+
/// Disable statistics for the column specified by `path`.
|
| 709 |
+
/// Default enabled.
|
| 710 |
+
Builder* disable_statistics(const std::string& path) {
|
| 711 |
+
statistics_enabled_[path] = false;
|
| 712 |
+
return this;
|
| 713 |
+
}
|
| 714 |
+
|
| 715 |
+
/// Disable statistics for the column specified by `path`.
|
| 716 |
+
/// Default enabled.
|
| 717 |
+
Builder* disable_statistics(const std::shared_ptr<schema::ColumnPath>& path) {
|
| 718 |
+
return this->disable_statistics(path->ToDotString());
|
| 719 |
+
}
|
| 720 |
+
|
| 721 |
+
/// Disable bloom filter for the column specified by `path`.
|
| 722 |
+
/// Default disabled.
|
| 723 |
+
Builder* disable_bloom_filter(const std::string& path) {
|
| 724 |
+
bloom_filter_options_.erase(path);
|
| 725 |
+
return this;
|
| 726 |
+
}
|
| 727 |
+
|
| 728 |
+
/// Disable bloom filter for the column specified by `path`.
|
| 729 |
+
/// Default disabled.
|
| 730 |
+
Builder* disable_bloom_filter(const std::shared_ptr<schema::ColumnPath>& path) {
|
| 731 |
+
return this->disable_bloom_filter(path->ToDotString());
|
| 732 |
+
}
|
| 733 |
+
|
| 734 |
+
/// Enable bloom filter for the column specified by `path`.
|
| 735 |
+
///
|
| 736 |
+
/// Default disabled.
|
| 737 |
+
///
|
| 738 |
+
/// \note Bloom filter is not supported for boolean columns. ParquetException will
|
| 739 |
+
/// be thrown during write if the column is of boolean type.
|
| 740 |
+
Builder* enable_bloom_filter(const std::string& path,
|
| 741 |
+
const BloomFilterOptions& bloom_filter_options) {
|
| 742 |
+
bloom_filter_options_[path] = bloom_filter_options;
|
| 743 |
+
return this;
|
| 744 |
+
}
|
| 745 |
+
|
| 746 |
+
/// Enable bloom filter for the column specified by `path`.
|
| 747 |
+
///
|
| 748 |
+
/// Default disabled.
|
| 749 |
+
///
|
| 750 |
+
/// \note Bloom filter is not supported for boolean columns. ParquetException will
|
| 751 |
+
/// be thrown during write if the column is of boolean type.
|
| 752 |
+
Builder* enable_bloom_filter(const std::shared_ptr<schema::ColumnPath>& path,
|
| 753 |
+
const BloomFilterOptions& bloom_filter_options) {
|
| 754 |
+
return this->enable_bloom_filter(path->ToDotString(), bloom_filter_options);
|
| 755 |
+
}
|
| 756 |
+
|
| 757 |
+
/// Allow decimals with 1 <= precision <= 18 to be stored as integers.
|
| 758 |
+
///
|
| 759 |
+
/// In Parquet, DECIMAL can be stored in any of the following physical types:
|
| 760 |
+
/// - int32: for 1 <= precision <= 9.
|
| 761 |
+
/// - int64: for 10 <= precision <= 18.
|
| 762 |
+
/// - fixed_len_byte_array: precision is limited by the array size.
|
| 763 |
+
/// Length n can store <= floor(log_10(2^(8*n - 1) - 1)) base-10 digits.
|
| 764 |
+
/// - binary: precision is unlimited. The minimum number of bytes to store
|
| 765 |
+
/// the unscaled value is used.
|
| 766 |
+
///
|
| 767 |
+
/// By default, this is DISABLED and all decimal types annotate fixed_len_byte_array.
|
| 768 |
+
///
|
| 769 |
+
/// When enabled, the C++ writer will use following physical types to store decimals:
|
| 770 |
+
/// - int32: for 1 <= precision <= 9.
|
| 771 |
+
/// - int64: for 10 <= precision <= 18.
|
| 772 |
+
/// - fixed_len_byte_array: for precision > 18.
|
| 773 |
+
///
|
| 774 |
+
/// As a consequence, decimal columns stored in integer types are more compact.
|
| 775 |
+
Builder* enable_store_decimal_as_integer() {
|
| 776 |
+
store_decimal_as_integer_ = true;
|
| 777 |
+
return this;
|
| 778 |
+
}
|
| 779 |
+
|
| 780 |
+
/// Disable decimal logical type with 1 <= precision <= 18 to be stored
|
| 781 |
+
/// as integer physical type.
|
| 782 |
+
///
|
| 783 |
+
/// Default disabled.
|
| 784 |
+
Builder* disable_store_decimal_as_integer() {
|
| 785 |
+
store_decimal_as_integer_ = false;
|
| 786 |
+
return this;
|
| 787 |
+
}
|
| 788 |
+
|
| 789 |
+
/// Enable writing page index in general for all columns. Default enabled.
|
| 790 |
+
///
|
| 791 |
+
/// Writing statistics to the page index disables the old method of writing
|
| 792 |
+
/// statistics to each data page header.
|
| 793 |
+
/// The page index makes filtering more efficient than the page header, as
|
| 794 |
+
/// it gathers all the statistics for a Parquet file in a single place,
|
| 795 |
+
/// avoiding scattered I/O.
|
| 796 |
+
///
|
| 797 |
+
/// Please check the link below for more details:
|
| 798 |
+
/// https://github.com/apache/parquet-format/blob/master/PageIndex.md
|
| 799 |
+
Builder* enable_write_page_index() {
|
| 800 |
+
default_column_properties_.set_page_index_enabled(true);
|
| 801 |
+
return this;
|
| 802 |
+
}
|
| 803 |
+
|
| 804 |
+
/// Disable writing page index in general for all columns. Default enabled.
|
| 805 |
+
Builder* disable_write_page_index() {
|
| 806 |
+
default_column_properties_.set_page_index_enabled(false);
|
| 807 |
+
return this;
|
| 808 |
+
}
|
| 809 |
+
|
| 810 |
+
/// Enable writing page index for column specified by `path`. Default enabled.
|
| 811 |
+
Builder* enable_write_page_index(const std::string& path) {
|
| 812 |
+
page_index_enabled_[path] = true;
|
| 813 |
+
return this;
|
| 814 |
+
}
|
| 815 |
+
|
| 816 |
+
/// Enable writing page index for column specified by `path`. Default enabled.
|
| 817 |
+
Builder* enable_write_page_index(const std::shared_ptr<schema::ColumnPath>& path) {
|
| 818 |
+
return this->enable_write_page_index(path->ToDotString());
|
| 819 |
+
}
|
| 820 |
+
|
| 821 |
+
/// Disable writing page index for column specified by `path`. Default enabled.
|
| 822 |
+
Builder* disable_write_page_index(const std::string& path) {
|
| 823 |
+
page_index_enabled_[path] = false;
|
| 824 |
+
return this;
|
| 825 |
+
}
|
| 826 |
+
|
| 827 |
+
/// Disable writing page index for column specified by `path`. Default enabled.
|
| 828 |
+
Builder* disable_write_page_index(const std::shared_ptr<schema::ColumnPath>& path) {
|
| 829 |
+
return this->disable_write_page_index(path->ToDotString());
|
| 830 |
+
}
|
| 831 |
+
|
| 832 |
+
/// \brief Set the level to write size statistics for all columns. Default is
|
| 833 |
+
/// PageAndColumnChunk.
|
| 834 |
+
///
|
| 835 |
+
/// \param level The level to write size statistics. Note that if page index is not
|
| 836 |
+
/// enabled, page level size statistics will not be written even if the level
|
| 837 |
+
/// is set to PageAndColumnChunk.
|
| 838 |
+
Builder* set_size_statistics_level(SizeStatisticsLevel level) {
|
| 839 |
+
size_statistics_level_ = level;
|
| 840 |
+
return this;
|
| 841 |
+
}
|
| 842 |
+
|
| 843 |
+
/// \brief Build the WriterProperties with the builder parameters.
|
| 844 |
+
/// \return The WriterProperties defined by the builder.
|
| 845 |
+
std::shared_ptr<WriterProperties> build() {
|
| 846 |
+
std::unordered_map<std::string, ColumnProperties> column_properties;
|
| 847 |
+
auto get = [&](const std::string& key) -> ColumnProperties& {
|
| 848 |
+
auto it = column_properties.find(key);
|
| 849 |
+
if (it == column_properties.end())
|
| 850 |
+
return column_properties[key] = default_column_properties_;
|
| 851 |
+
else
|
| 852 |
+
return it->second;
|
| 853 |
+
};
|
| 854 |
+
|
| 855 |
+
for (const auto& item : encodings_) get(item.first).set_encoding(item.second);
|
| 856 |
+
for (const auto& item : codecs_) get(item.first).set_compression(item.second);
|
| 857 |
+
for (const auto& item : codec_options_)
|
| 858 |
+
get(item.first).set_codec_options(item.second);
|
| 859 |
+
for (const auto& item : dictionary_enabled_)
|
| 860 |
+
get(item.first).set_dictionary_enabled(item.second);
|
| 861 |
+
for (const auto& item : statistics_enabled_)
|
| 862 |
+
get(item.first).set_statistics_enabled(item.second);
|
| 863 |
+
for (const auto& item : page_index_enabled_)
|
| 864 |
+
get(item.first).set_page_index_enabled(item.second);
|
| 865 |
+
for (const auto& item : bloom_filter_options_)
|
| 866 |
+
get(item.first).set_bloom_filter_options(item.second);
|
| 867 |
+
|
| 868 |
+
return std::shared_ptr<WriterProperties>(new WriterProperties(
|
| 869 |
+
pool_, dictionary_pagesize_limit_, write_batch_size_, max_row_group_length_,
|
| 870 |
+
pagesize_, max_rows_per_page_, version_, created_by_, page_checksum_enabled_,
|
| 871 |
+
size_statistics_level_, std::move(file_encryption_properties_),
|
| 872 |
+
default_column_properties_, column_properties, data_page_version_,
|
| 873 |
+
store_decimal_as_integer_, std::move(sorting_columns_),
|
| 874 |
+
content_defined_chunking_enabled_, content_defined_chunking_options_));
|
| 875 |
+
}
|
| 876 |
+
|
| 877 |
+
private:
|
| 878 |
+
void CopyColumnSpecificProperties(const WriterProperties& properties);
|
| 879 |
+
|
| 880 |
+
MemoryPool* pool_;
|
| 881 |
+
int64_t dictionary_pagesize_limit_;
|
| 882 |
+
int64_t write_batch_size_;
|
| 883 |
+
int64_t max_row_group_length_;
|
| 884 |
+
int64_t pagesize_;
|
| 885 |
+
int64_t max_rows_per_page_;
|
| 886 |
+
ParquetVersion::type version_;
|
| 887 |
+
ParquetDataPageVersion data_page_version_;
|
| 888 |
+
std::string created_by_;
|
| 889 |
+
bool store_decimal_as_integer_;
|
| 890 |
+
bool page_checksum_enabled_;
|
| 891 |
+
SizeStatisticsLevel size_statistics_level_;
|
| 892 |
+
|
| 893 |
+
std::shared_ptr<FileEncryptionProperties> file_encryption_properties_;
|
| 894 |
+
|
| 895 |
+
// If empty, there is no sorting columns.
|
| 896 |
+
std::vector<SortingColumn> sorting_columns_;
|
| 897 |
+
|
| 898 |
+
// Settings used for each column unless overridden in any of the maps below
|
| 899 |
+
ColumnProperties default_column_properties_;
|
| 900 |
+
std::unordered_map<std::string, Encoding::type> encodings_;
|
| 901 |
+
std::unordered_map<std::string, Compression::type> codecs_;
|
| 902 |
+
std::unordered_map<std::string, std::shared_ptr<CodecOptions>> codec_options_;
|
| 903 |
+
std::unordered_map<std::string, bool> dictionary_enabled_;
|
| 904 |
+
std::unordered_map<std::string, bool> statistics_enabled_;
|
| 905 |
+
std::unordered_map<std::string, bool> page_index_enabled_;
|
| 906 |
+
std::unordered_map<std::string, BloomFilterOptions> bloom_filter_options_;
|
| 907 |
+
|
| 908 |
+
bool content_defined_chunking_enabled_;
|
| 909 |
+
CdcOptions content_defined_chunking_options_;
|
| 910 |
+
};
|
| 911 |
+
|
| 912 |
+
inline MemoryPool* memory_pool() const { return pool_; }
|
| 913 |
+
|
| 914 |
+
inline int64_t dictionary_pagesize_limit() const { return dictionary_pagesize_limit_; }
|
| 915 |
+
|
| 916 |
+
inline int64_t write_batch_size() const { return write_batch_size_; }
|
| 917 |
+
|
| 918 |
+
inline int64_t max_row_group_length() const { return max_row_group_length_; }
|
| 919 |
+
|
| 920 |
+
inline int64_t data_pagesize() const { return pagesize_; }
|
| 921 |
+
|
| 922 |
+
inline int64_t max_rows_per_page() const { return max_rows_per_page_; }
|
| 923 |
+
|
| 924 |
+
inline ParquetDataPageVersion data_page_version() const {
|
| 925 |
+
return parquet_data_page_version_;
|
| 926 |
+
}
|
| 927 |
+
|
| 928 |
+
inline ParquetVersion::type version() const { return parquet_version_; }
|
| 929 |
+
|
| 930 |
+
inline std::string created_by() const { return parquet_created_by_; }
|
| 931 |
+
|
| 932 |
+
inline bool store_decimal_as_integer() const { return store_decimal_as_integer_; }
|
| 933 |
+
|
| 934 |
+
inline bool page_checksum_enabled() const { return page_checksum_enabled_; }
|
| 935 |
+
|
| 936 |
+
inline bool content_defined_chunking_enabled() const {
|
| 937 |
+
return content_defined_chunking_enabled_;
|
| 938 |
+
}
|
| 939 |
+
inline CdcOptions content_defined_chunking_options() const {
|
| 940 |
+
return content_defined_chunking_options_;
|
| 941 |
+
}
|
| 942 |
+
|
| 943 |
+
inline SizeStatisticsLevel size_statistics_level() const {
|
| 944 |
+
return size_statistics_level_;
|
| 945 |
+
}
|
| 946 |
+
|
| 947 |
+
inline Encoding::type dictionary_index_encoding() const {
|
| 948 |
+
if (parquet_version_ == ParquetVersion::PARQUET_1_0) {
|
| 949 |
+
return Encoding::PLAIN_DICTIONARY;
|
| 950 |
+
} else {
|
| 951 |
+
return Encoding::RLE_DICTIONARY;
|
| 952 |
+
}
|
| 953 |
+
}
|
| 954 |
+
|
| 955 |
+
inline Encoding::type dictionary_page_encoding() const {
|
| 956 |
+
if (parquet_version_ == ParquetVersion::PARQUET_1_0) {
|
| 957 |
+
return Encoding::PLAIN_DICTIONARY;
|
| 958 |
+
} else {
|
| 959 |
+
return Encoding::PLAIN;
|
| 960 |
+
}
|
| 961 |
+
}
|
| 962 |
+
|
| 963 |
+
const ColumnProperties& column_properties(
|
| 964 |
+
const std::shared_ptr<schema::ColumnPath>& path) const {
|
| 965 |
+
auto it = column_properties_.find(path->ToDotString());
|
| 966 |
+
if (it != column_properties_.end()) return it->second;
|
| 967 |
+
return default_column_properties_;
|
| 968 |
+
}
|
| 969 |
+
|
| 970 |
+
Encoding::type encoding(const std::shared_ptr<schema::ColumnPath>& path) const {
|
| 971 |
+
return column_properties(path).encoding();
|
| 972 |
+
}
|
| 973 |
+
|
| 974 |
+
Compression::type compression(const std::shared_ptr<schema::ColumnPath>& path) const {
|
| 975 |
+
return column_properties(path).compression();
|
| 976 |
+
}
|
| 977 |
+
|
| 978 |
+
int compression_level(const std::shared_ptr<schema::ColumnPath>& path) const {
|
| 979 |
+
return column_properties(path).compression_level();
|
| 980 |
+
}
|
| 981 |
+
|
| 982 |
+
const std::shared_ptr<CodecOptions> codec_options(
|
| 983 |
+
const std::shared_ptr<schema::ColumnPath>& path) const {
|
| 984 |
+
return column_properties(path).codec_options();
|
| 985 |
+
}
|
| 986 |
+
|
| 987 |
+
bool dictionary_enabled(const std::shared_ptr<schema::ColumnPath>& path) const {
|
| 988 |
+
return column_properties(path).dictionary_enabled();
|
| 989 |
+
}
|
| 990 |
+
|
| 991 |
+
const std::vector<SortingColumn>& sorting_columns() const { return sorting_columns_; }
|
| 992 |
+
|
| 993 |
+
bool statistics_enabled(const std::shared_ptr<schema::ColumnPath>& path) const {
|
| 994 |
+
return column_properties(path).statistics_enabled();
|
| 995 |
+
}
|
| 996 |
+
|
| 997 |
+
size_t max_statistics_size(const std::shared_ptr<schema::ColumnPath>& path) const {
|
| 998 |
+
return column_properties(path).max_statistics_size();
|
| 999 |
+
}
|
| 1000 |
+
|
| 1001 |
+
bool page_index_enabled(const std::shared_ptr<schema::ColumnPath>& path) const {
|
| 1002 |
+
return column_properties(path).page_index_enabled();
|
| 1003 |
+
}
|
| 1004 |
+
|
| 1005 |
+
bool page_index_enabled() const {
|
| 1006 |
+
if (default_column_properties_.page_index_enabled()) {
|
| 1007 |
+
return true;
|
| 1008 |
+
}
|
| 1009 |
+
for (const auto& item : column_properties_) {
|
| 1010 |
+
if (item.second.page_index_enabled()) {
|
| 1011 |
+
return true;
|
| 1012 |
+
}
|
| 1013 |
+
}
|
| 1014 |
+
return false;
|
| 1015 |
+
}
|
| 1016 |
+
|
| 1017 |
+
// Return whether bloom filter is enabled for any column.
|
| 1018 |
+
bool bloom_filter_enabled() const {
|
| 1019 |
+
return std::any_of(column_properties_.cbegin(), column_properties_.cend(),
|
| 1020 |
+
[](const auto& p) { return p.second.bloom_filter_enabled(); });
|
| 1021 |
+
}
|
| 1022 |
+
|
| 1023 |
+
std::optional<BloomFilterOptions> bloom_filter_options(
|
| 1024 |
+
const std::shared_ptr<schema::ColumnPath>& path) const {
|
| 1025 |
+
return column_properties(path).bloom_filter_options();
|
| 1026 |
+
}
|
| 1027 |
+
|
| 1028 |
+
inline FileEncryptionProperties* file_encryption_properties() const {
|
| 1029 |
+
return file_encryption_properties_.get();
|
| 1030 |
+
}
|
| 1031 |
+
|
| 1032 |
+
std::shared_ptr<ColumnEncryptionProperties> column_encryption_properties(
|
| 1033 |
+
const std::string& path) const {
|
| 1034 |
+
if (file_encryption_properties_) {
|
| 1035 |
+
return file_encryption_properties_->column_encryption_properties(path);
|
| 1036 |
+
} else {
|
| 1037 |
+
return NULLPTR;
|
| 1038 |
+
}
|
| 1039 |
+
}
|
| 1040 |
+
|
| 1041 |
+
// \brief Return the default column properties
|
| 1042 |
+
const ColumnProperties& default_column_properties() const {
|
| 1043 |
+
return default_column_properties_;
|
| 1044 |
+
}
|
| 1045 |
+
|
| 1046 |
+
private:
|
| 1047 |
+
explicit WriterProperties(
|
| 1048 |
+
MemoryPool* pool, int64_t dictionary_pagesize_limit, int64_t write_batch_size,
|
| 1049 |
+
int64_t max_row_group_length, int64_t pagesize, int64_t max_rows_per_page,
|
| 1050 |
+
ParquetVersion::type version, const std::string& created_by,
|
| 1051 |
+
bool page_write_checksum_enabled, SizeStatisticsLevel size_statistics_level,
|
| 1052 |
+
std::shared_ptr<FileEncryptionProperties> file_encryption_properties,
|
| 1053 |
+
const ColumnProperties& default_column_properties,
|
| 1054 |
+
const std::unordered_map<std::string, ColumnProperties>& column_properties,
|
| 1055 |
+
ParquetDataPageVersion data_page_version, bool store_short_decimal_as_integer,
|
| 1056 |
+
std::vector<SortingColumn> sorting_columns, bool content_defined_chunking_enabled,
|
| 1057 |
+
CdcOptions content_defined_chunking_options)
|
| 1058 |
+
: pool_(pool),
|
| 1059 |
+
dictionary_pagesize_limit_(dictionary_pagesize_limit),
|
| 1060 |
+
write_batch_size_(write_batch_size),
|
| 1061 |
+
max_row_group_length_(max_row_group_length),
|
| 1062 |
+
pagesize_(pagesize),
|
| 1063 |
+
max_rows_per_page_(max_rows_per_page),
|
| 1064 |
+
parquet_data_page_version_(data_page_version),
|
| 1065 |
+
parquet_version_(version),
|
| 1066 |
+
parquet_created_by_(created_by),
|
| 1067 |
+
store_decimal_as_integer_(store_short_decimal_as_integer),
|
| 1068 |
+
page_checksum_enabled_(page_write_checksum_enabled),
|
| 1069 |
+
size_statistics_level_(size_statistics_level),
|
| 1070 |
+
file_encryption_properties_(file_encryption_properties),
|
| 1071 |
+
sorting_columns_(std::move(sorting_columns)),
|
| 1072 |
+
default_column_properties_(default_column_properties),
|
| 1073 |
+
column_properties_(column_properties),
|
| 1074 |
+
content_defined_chunking_enabled_(content_defined_chunking_enabled),
|
| 1075 |
+
content_defined_chunking_options_(content_defined_chunking_options) {}
|
| 1076 |
+
|
| 1077 |
+
MemoryPool* pool_;
|
| 1078 |
+
int64_t dictionary_pagesize_limit_;
|
| 1079 |
+
int64_t write_batch_size_;
|
| 1080 |
+
int64_t max_row_group_length_;
|
| 1081 |
+
int64_t pagesize_;
|
| 1082 |
+
int64_t max_rows_per_page_;
|
| 1083 |
+
ParquetDataPageVersion parquet_data_page_version_;
|
| 1084 |
+
ParquetVersion::type parquet_version_;
|
| 1085 |
+
std::string parquet_created_by_;
|
| 1086 |
+
bool store_decimal_as_integer_;
|
| 1087 |
+
bool page_checksum_enabled_;
|
| 1088 |
+
SizeStatisticsLevel size_statistics_level_;
|
| 1089 |
+
|
| 1090 |
+
std::shared_ptr<FileEncryptionProperties> file_encryption_properties_;
|
| 1091 |
+
|
| 1092 |
+
std::vector<SortingColumn> sorting_columns_;
|
| 1093 |
+
|
| 1094 |
+
ColumnProperties default_column_properties_;
|
| 1095 |
+
std::unordered_map<std::string, ColumnProperties> column_properties_;
|
| 1096 |
+
|
| 1097 |
+
bool content_defined_chunking_enabled_;
|
| 1098 |
+
CdcOptions content_defined_chunking_options_;
|
| 1099 |
+
};
|
| 1100 |
+
|
| 1101 |
+
PARQUET_EXPORT const std::shared_ptr<WriterProperties>& default_writer_properties();
|
| 1102 |
+
|
| 1103 |
+
// ----------------------------------------------------------------------
|
| 1104 |
+
// Properties specific to Apache Arrow columnar read and write
|
| 1105 |
+
|
| 1106 |
+
static constexpr bool kArrowDefaultUseThreads = false;
|
| 1107 |
+
|
| 1108 |
+
// Default number of rows to read when using ::arrow::RecordBatchReader
|
| 1109 |
+
static constexpr int64_t kArrowDefaultBatchSize = 64 * 1024;
|
| 1110 |
+
|
| 1111 |
+
constexpr inline ::arrow::Type::type kArrowDefaultBinaryType = ::arrow::Type::BINARY;
|
| 1112 |
+
constexpr inline ::arrow::Type::type kArrowDefaultListType = ::arrow::Type::LIST;
|
| 1113 |
+
|
| 1114 |
+
/// EXPERIMENTAL: Properties for configuring FileReader behavior.
|
| 1115 |
+
class PARQUET_EXPORT ArrowReaderProperties {
|
| 1116 |
+
public:
|
| 1117 |
+
explicit ArrowReaderProperties(bool use_threads = kArrowDefaultUseThreads)
|
| 1118 |
+
: use_threads_(use_threads),
|
| 1119 |
+
read_dict_indices_(),
|
| 1120 |
+
batch_size_(kArrowDefaultBatchSize),
|
| 1121 |
+
pre_buffer_(true),
|
| 1122 |
+
cache_options_(::arrow::io::CacheOptions::LazyDefaults()),
|
| 1123 |
+
coerce_int96_timestamp_unit_(::arrow::TimeUnit::NANO),
|
| 1124 |
+
binary_type_(kArrowDefaultBinaryType),
|
| 1125 |
+
list_type_(kArrowDefaultListType),
|
| 1126 |
+
arrow_extensions_enabled_(false),
|
| 1127 |
+
should_load_statistics_(false),
|
| 1128 |
+
smallest_decimal_enabled_(false) {}
|
| 1129 |
+
|
| 1130 |
+
/// \brief Set whether to use the IO thread pool to parse columns in parallel.
|
| 1131 |
+
///
|
| 1132 |
+
/// Default is false.
|
| 1133 |
+
void set_use_threads(bool use_threads) { use_threads_ = use_threads; }
|
| 1134 |
+
/// Return whether will use multiple threads.
|
| 1135 |
+
bool use_threads() const { return use_threads_; }
|
| 1136 |
+
|
| 1137 |
+
/// \brief Set whether to read a particular column as dictionary encoded.
|
| 1138 |
+
///
|
| 1139 |
+
/// If the file metadata contains a serialized Arrow schema, then ...
|
| 1140 |
+
////
|
| 1141 |
+
/// This is only supported for columns with a Parquet physical type of
|
| 1142 |
+
/// BYTE_ARRAY, such as string or binary types.
|
| 1143 |
+
void set_read_dictionary(int column_index, bool read_dict) {
|
| 1144 |
+
if (read_dict) {
|
| 1145 |
+
read_dict_indices_.insert(column_index);
|
| 1146 |
+
} else {
|
| 1147 |
+
read_dict_indices_.erase(column_index);
|
| 1148 |
+
}
|
| 1149 |
+
}
|
| 1150 |
+
/// Return whether the column at the index will be read as dictionary.
|
| 1151 |
+
bool read_dictionary(int column_index) const {
|
| 1152 |
+
if (read_dict_indices_.find(column_index) != read_dict_indices_.end()) {
|
| 1153 |
+
return true;
|
| 1154 |
+
} else {
|
| 1155 |
+
return false;
|
| 1156 |
+
}
|
| 1157 |
+
}
|
| 1158 |
+
|
| 1159 |
+
/// \brief Set the Arrow binary type to read BYTE_ARRAY columns as.
|
| 1160 |
+
///
|
| 1161 |
+
/// Allowed values are Type::BINARY, Type::LARGE_BINARY and Type::BINARY_VIEW.
|
| 1162 |
+
/// Default is Type::BINARY.
|
| 1163 |
+
///
|
| 1164 |
+
/// If a BYTE_ARRAY column has the STRING logical type, it is read as the
|
| 1165 |
+
/// Arrow string type corresponding to the configured binary type (for example
|
| 1166 |
+
/// Type::LARGE_STRING if the configured binary type is Type::LARGE_BINARY).
|
| 1167 |
+
///
|
| 1168 |
+
/// However, if a serialized Arrow schema is found in the Parquet metadata,
|
| 1169 |
+
/// this setting is ignored and the Arrow schema takes precedence
|
| 1170 |
+
/// (see ArrowWriterProperties::store_schema).
|
| 1171 |
+
void set_binary_type(::arrow::Type::type value) { binary_type_ = value; }
|
| 1172 |
+
/// Return the Arrow binary type to read BYTE_ARRAY columns as.
|
| 1173 |
+
::arrow::Type::type binary_type() const { return binary_type_; }
|
| 1174 |
+
|
| 1175 |
+
/// \brief Set the Arrow list type to read Parquet list columns as.
|
| 1176 |
+
///
|
| 1177 |
+
/// Allowed values are Type::LIST and Type::LARGE_LIST.
|
| 1178 |
+
/// Default is Type::LIST.
|
| 1179 |
+
///
|
| 1180 |
+
/// However, if a serialized Arrow schema is found in the Parquet metadata,
|
| 1181 |
+
/// this setting is ignored and the Arrow schema takes precedence
|
| 1182 |
+
/// (see ArrowWriterProperties::store_schema).
|
| 1183 |
+
void set_list_type(::arrow::Type::type value) { list_type_ = value; }
|
| 1184 |
+
/// Return the Arrow list type to read Parquet list columns as.
|
| 1185 |
+
::arrow::Type::type list_type() const { return list_type_; }
|
| 1186 |
+
|
| 1187 |
+
/// \brief Set the maximum number of rows to read into a record batch.
|
| 1188 |
+
///
|
| 1189 |
+
/// Will only be fewer rows when there are no more rows in the file.
|
| 1190 |
+
/// Note that some APIs such as ReadTable may ignore this setting.
|
| 1191 |
+
void set_batch_size(int64_t batch_size) { batch_size_ = batch_size; }
|
| 1192 |
+
/// Return the batch size in rows.
|
| 1193 |
+
///
|
| 1194 |
+
/// Note that some APIs such as ReadTable may ignore this setting.
|
| 1195 |
+
int64_t batch_size() const { return batch_size_; }
|
| 1196 |
+
|
| 1197 |
+
/// Enable read coalescing (default false).
|
| 1198 |
+
///
|
| 1199 |
+
/// When enabled, the Arrow reader will pre-buffer necessary regions
|
| 1200 |
+
/// of the file in-memory. This is intended to improve performance on
|
| 1201 |
+
/// high-latency filesystems (e.g. Amazon S3).
|
| 1202 |
+
void set_pre_buffer(bool pre_buffer) { pre_buffer_ = pre_buffer; }
|
| 1203 |
+
/// Return whether read coalescing is enabled.
|
| 1204 |
+
bool pre_buffer() const { return pre_buffer_; }
|
| 1205 |
+
|
| 1206 |
+
/// Set options for read coalescing. This can be used to tune the
|
| 1207 |
+
/// implementation for characteristics of different filesystems.
|
| 1208 |
+
void set_cache_options(::arrow::io::CacheOptions options) { cache_options_ = options; }
|
| 1209 |
+
/// Return the options for read coalescing.
|
| 1210 |
+
const ::arrow::io::CacheOptions& cache_options() const { return cache_options_; }
|
| 1211 |
+
|
| 1212 |
+
/// Set execution context for read coalescing.
|
| 1213 |
+
void set_io_context(const ::arrow::io::IOContext& ctx) { io_context_ = ctx; }
|
| 1214 |
+
/// Return the execution context used for read coalescing.
|
| 1215 |
+
const ::arrow::io::IOContext& io_context() const { return io_context_; }
|
| 1216 |
+
|
| 1217 |
+
/// Set timestamp unit to use for deprecated INT96-encoded timestamps
|
| 1218 |
+
/// (default is NANO).
|
| 1219 |
+
void set_coerce_int96_timestamp_unit(::arrow::TimeUnit::type unit) {
|
| 1220 |
+
coerce_int96_timestamp_unit_ = unit;
|
| 1221 |
+
}
|
| 1222 |
+
|
| 1223 |
+
::arrow::TimeUnit::type coerce_int96_timestamp_unit() const {
|
| 1224 |
+
return coerce_int96_timestamp_unit_;
|
| 1225 |
+
}
|
| 1226 |
+
|
| 1227 |
+
/// Enable Parquet-supported Arrow extension types.
|
| 1228 |
+
///
|
| 1229 |
+
/// When enabled, Parquet logical types will be mapped to their corresponding Arrow
|
| 1230 |
+
/// extension types at read time, if such exist. Currently only arrow::extension::json()
|
| 1231 |
+
/// extension type is supported. Columns whose LogicalType is JSON will be interpreted
|
| 1232 |
+
/// as arrow::extension::json(), with storage type inferred from the serialized Arrow
|
| 1233 |
+
/// schema if present, or `utf8` by default.
|
| 1234 |
+
void set_arrow_extensions_enabled(bool extensions_enabled) {
|
| 1235 |
+
arrow_extensions_enabled_ = extensions_enabled;
|
| 1236 |
+
}
|
| 1237 |
+
bool get_arrow_extensions_enabled() const { return arrow_extensions_enabled_; }
|
| 1238 |
+
|
| 1239 |
+
/// \brief Set whether to load statistics as much as possible.
|
| 1240 |
+
///
|
| 1241 |
+
/// Default is false.
|
| 1242 |
+
void set_should_load_statistics(bool should_load_statistics) {
|
| 1243 |
+
should_load_statistics_ = should_load_statistics;
|
| 1244 |
+
}
|
| 1245 |
+
/// Return whether loading statistics as much as possible.
|
| 1246 |
+
bool should_load_statistics() const { return should_load_statistics_; }
|
| 1247 |
+
|
| 1248 |
+
/// \brief Set whether to infer Decimal32/64 from Parquet decimal logical types.
|
| 1249 |
+
///
|
| 1250 |
+
/// Default is false for compatibility, meaning that only Decimal128 and Decimal256
|
| 1251 |
+
/// can be inferred.
|
| 1252 |
+
void set_smallest_decimal_enabled(bool smallest_decimal_enable) {
|
| 1253 |
+
smallest_decimal_enabled_ = smallest_decimal_enable;
|
| 1254 |
+
}
|
| 1255 |
+
/// \brief Whether to infer Decimal32/64 from Parquet decimal logical types.
|
| 1256 |
+
///
|
| 1257 |
+
/// When enabled, Parquet decimal columns will be inferred as the smallest possible
|
| 1258 |
+
/// Arrow Decimal type.
|
| 1259 |
+
/// When disabled, Parquet decimal columns will be inferred as either Decimal128 or
|
| 1260 |
+
/// Decimal256, but not Decimal32/64.
|
| 1261 |
+
///
|
| 1262 |
+
/// Note: if an Arrow schema is found in the Parquet metadata, it will take priority and
|
| 1263 |
+
/// this setting will be ignored.
|
| 1264 |
+
bool smallest_decimal_enabled() const { return smallest_decimal_enabled_; }
|
| 1265 |
+
|
| 1266 |
+
private:
|
| 1267 |
+
bool use_threads_;
|
| 1268 |
+
std::unordered_set<int> read_dict_indices_;
|
| 1269 |
+
int64_t batch_size_;
|
| 1270 |
+
bool pre_buffer_;
|
| 1271 |
+
::arrow::io::IOContext io_context_;
|
| 1272 |
+
::arrow::io::CacheOptions cache_options_;
|
| 1273 |
+
::arrow::TimeUnit::type coerce_int96_timestamp_unit_;
|
| 1274 |
+
::arrow::Type::type binary_type_;
|
| 1275 |
+
::arrow::Type::type list_type_;
|
| 1276 |
+
bool arrow_extensions_enabled_;
|
| 1277 |
+
bool should_load_statistics_;
|
| 1278 |
+
bool smallest_decimal_enabled_;
|
| 1279 |
+
};
|
| 1280 |
+
|
| 1281 |
+
/// EXPERIMENTAL: Constructs the default ArrowReaderProperties
|
| 1282 |
+
PARQUET_EXPORT
|
| 1283 |
+
ArrowReaderProperties default_arrow_reader_properties();
|
| 1284 |
+
|
| 1285 |
+
class PARQUET_EXPORT ArrowWriterProperties {
|
| 1286 |
+
public:
|
| 1287 |
+
enum EngineVersion {
|
| 1288 |
+
V1, // Supports only nested lists.
|
| 1289 |
+
V2 // Full support for all nesting combinations
|
| 1290 |
+
};
|
| 1291 |
+
class Builder {
|
| 1292 |
+
public:
|
| 1293 |
+
Builder()
|
| 1294 |
+
: write_timestamps_as_int96_(false),
|
| 1295 |
+
coerce_timestamps_enabled_(false),
|
| 1296 |
+
coerce_timestamps_unit_(::arrow::TimeUnit::SECOND),
|
| 1297 |
+
truncated_timestamps_allowed_(false),
|
| 1298 |
+
store_schema_(false),
|
| 1299 |
+
compliant_nested_types_(true),
|
| 1300 |
+
engine_version_(V2),
|
| 1301 |
+
use_threads_(kArrowDefaultUseThreads),
|
| 1302 |
+
executor_(NULLPTR),
|
| 1303 |
+
write_time_adjusted_to_utc_(false) {}
|
| 1304 |
+
|
| 1305 |
+
/// \brief Disable writing legacy int96 timestamps (default disabled).
|
| 1306 |
+
Builder* disable_deprecated_int96_timestamps() {
|
| 1307 |
+
write_timestamps_as_int96_ = false;
|
| 1308 |
+
return this;
|
| 1309 |
+
}
|
| 1310 |
+
|
| 1311 |
+
/// \brief Enable writing legacy int96 timestamps (default disabled).
|
| 1312 |
+
///
|
| 1313 |
+
/// May be turned on to write timestamps compatible with older Parquet writers.
|
| 1314 |
+
/// This takes precedent over coerce_timestamps.
|
| 1315 |
+
Builder* enable_deprecated_int96_timestamps() {
|
| 1316 |
+
write_timestamps_as_int96_ = true;
|
| 1317 |
+
return this;
|
| 1318 |
+
}
|
| 1319 |
+
|
| 1320 |
+
/// \brief Coerce all timestamps to the specified time unit.
|
| 1321 |
+
/// \param unit time unit to truncate to.
|
| 1322 |
+
/// For Parquet versions 1.0 and 2.4, nanoseconds are casted to microseconds.
|
| 1323 |
+
Builder* coerce_timestamps(::arrow::TimeUnit::type unit) {
|
| 1324 |
+
coerce_timestamps_enabled_ = true;
|
| 1325 |
+
coerce_timestamps_unit_ = unit;
|
| 1326 |
+
return this;
|
| 1327 |
+
}
|
| 1328 |
+
|
| 1329 |
+
/// \brief Allow loss of data when truncating timestamps.
|
| 1330 |
+
///
|
| 1331 |
+
/// This is disallowed by default and an error will be returned.
|
| 1332 |
+
Builder* allow_truncated_timestamps() {
|
| 1333 |
+
truncated_timestamps_allowed_ = true;
|
| 1334 |
+
return this;
|
| 1335 |
+
}
|
| 1336 |
+
|
| 1337 |
+
/// \brief Disallow loss of data when truncating timestamps (default).
|
| 1338 |
+
Builder* disallow_truncated_timestamps() {
|
| 1339 |
+
truncated_timestamps_allowed_ = false;
|
| 1340 |
+
return this;
|
| 1341 |
+
}
|
| 1342 |
+
|
| 1343 |
+
/// \brief EXPERIMENTAL: Write binary serialized Arrow schema to the file,
|
| 1344 |
+
/// to enable certain read options (like "read_dictionary") to be set
|
| 1345 |
+
/// automatically
|
| 1346 |
+
Builder* store_schema() {
|
| 1347 |
+
store_schema_ = true;
|
| 1348 |
+
return this;
|
| 1349 |
+
}
|
| 1350 |
+
|
| 1351 |
+
/// \brief When enabled, will not preserve Arrow field names for list types.
|
| 1352 |
+
///
|
| 1353 |
+
/// Instead of using the field names Arrow uses for the values array of
|
| 1354 |
+
/// list types (default "item"), will use "element", as is specified in
|
| 1355 |
+
/// the Parquet spec.
|
| 1356 |
+
///
|
| 1357 |
+
/// This is enabled by default.
|
| 1358 |
+
Builder* enable_compliant_nested_types() {
|
| 1359 |
+
compliant_nested_types_ = true;
|
| 1360 |
+
return this;
|
| 1361 |
+
}
|
| 1362 |
+
|
| 1363 |
+
/// Preserve Arrow list field name.
|
| 1364 |
+
Builder* disable_compliant_nested_types() {
|
| 1365 |
+
compliant_nested_types_ = false;
|
| 1366 |
+
return this;
|
| 1367 |
+
}
|
| 1368 |
+
|
| 1369 |
+
/// Set the version of the Parquet writer engine.
|
| 1370 |
+
Builder* set_engine_version(EngineVersion version) {
|
| 1371 |
+
engine_version_ = version;
|
| 1372 |
+
return this;
|
| 1373 |
+
}
|
| 1374 |
+
|
| 1375 |
+
/// \brief Set whether to use multiple threads to write columns
|
| 1376 |
+
/// in parallel in the buffered row group mode.
|
| 1377 |
+
///
|
| 1378 |
+
/// WARNING: If writing multiple files in parallel in the same
|
| 1379 |
+
/// executor, deadlock may occur if use_threads is true. Please
|
| 1380 |
+
/// disable it in this case.
|
| 1381 |
+
///
|
| 1382 |
+
/// Default is false.
|
| 1383 |
+
Builder* set_use_threads(bool use_threads) {
|
| 1384 |
+
use_threads_ = use_threads;
|
| 1385 |
+
return this;
|
| 1386 |
+
}
|
| 1387 |
+
|
| 1388 |
+
/// \brief Set the executor to write columns in parallel in the
|
| 1389 |
+
/// buffered row group mode.
|
| 1390 |
+
///
|
| 1391 |
+
/// Default is nullptr and the default cpu executor will be used.
|
| 1392 |
+
Builder* set_executor(::arrow::internal::Executor* executor) {
|
| 1393 |
+
executor_ = executor;
|
| 1394 |
+
return this;
|
| 1395 |
+
}
|
| 1396 |
+
|
| 1397 |
+
/// \brief Set the value of isAdjustedTOUTC when writing a TIME column
|
| 1398 |
+
///
|
| 1399 |
+
/// Default is false because Arrow TIME data is expressed in an unspecified timezone.
|
| 1400 |
+
/// Note this setting doesn't affect TIMESTAMP data.
|
| 1401 |
+
Builder* set_time_adjusted_to_utc(bool adjusted) {
|
| 1402 |
+
write_time_adjusted_to_utc_ = adjusted;
|
| 1403 |
+
return this;
|
| 1404 |
+
}
|
| 1405 |
+
|
| 1406 |
+
/// Create the final properties.
|
| 1407 |
+
std::shared_ptr<ArrowWriterProperties> build() {
|
| 1408 |
+
return std::shared_ptr<ArrowWriterProperties>(new ArrowWriterProperties(
|
| 1409 |
+
write_timestamps_as_int96_, coerce_timestamps_enabled_, coerce_timestamps_unit_,
|
| 1410 |
+
truncated_timestamps_allowed_, store_schema_, compliant_nested_types_,
|
| 1411 |
+
engine_version_, use_threads_, executor_, write_time_adjusted_to_utc_));
|
| 1412 |
+
}
|
| 1413 |
+
|
| 1414 |
+
private:
|
| 1415 |
+
bool write_timestamps_as_int96_;
|
| 1416 |
+
|
| 1417 |
+
bool coerce_timestamps_enabled_;
|
| 1418 |
+
::arrow::TimeUnit::type coerce_timestamps_unit_;
|
| 1419 |
+
bool truncated_timestamps_allowed_;
|
| 1420 |
+
|
| 1421 |
+
bool store_schema_;
|
| 1422 |
+
bool compliant_nested_types_;
|
| 1423 |
+
EngineVersion engine_version_;
|
| 1424 |
+
|
| 1425 |
+
bool use_threads_;
|
| 1426 |
+
::arrow::internal::Executor* executor_;
|
| 1427 |
+
|
| 1428 |
+
bool write_time_adjusted_to_utc_;
|
| 1429 |
+
};
|
| 1430 |
+
|
| 1431 |
+
bool support_deprecated_int96_timestamps() const { return write_timestamps_as_int96_; }
|
| 1432 |
+
|
| 1433 |
+
bool coerce_timestamps_enabled() const { return coerce_timestamps_enabled_; }
|
| 1434 |
+
::arrow::TimeUnit::type coerce_timestamps_unit() const {
|
| 1435 |
+
return coerce_timestamps_unit_;
|
| 1436 |
+
}
|
| 1437 |
+
|
| 1438 |
+
bool truncated_timestamps_allowed() const { return truncated_timestamps_allowed_; }
|
| 1439 |
+
|
| 1440 |
+
bool store_schema() const { return store_schema_; }
|
| 1441 |
+
|
| 1442 |
+
/// \brief Enable nested type naming according to the parquet specification.
|
| 1443 |
+
///
|
| 1444 |
+
/// Older versions of arrow wrote out field names for nested lists based on the name
|
| 1445 |
+
/// of the field. According to the parquet specification they should always be
|
| 1446 |
+
/// "element".
|
| 1447 |
+
bool compliant_nested_types() const { return compliant_nested_types_; }
|
| 1448 |
+
|
| 1449 |
+
/// \brief The underlying engine version to use when writing Arrow data.
|
| 1450 |
+
///
|
| 1451 |
+
/// V2 is currently the latest V1 is considered deprecated but left in
|
| 1452 |
+
/// place in case there are bugs detected in V2.
|
| 1453 |
+
EngineVersion engine_version() const { return engine_version_; }
|
| 1454 |
+
|
| 1455 |
+
/// \brief Returns whether the writer will use multiple threads
|
| 1456 |
+
/// to write columns in parallel in the buffered row group mode.
|
| 1457 |
+
bool use_threads() const { return use_threads_; }
|
| 1458 |
+
|
| 1459 |
+
/// \brief Returns the executor used to write columns in parallel.
|
| 1460 |
+
::arrow::internal::Executor* executor() const;
|
| 1461 |
+
|
| 1462 |
+
/// \brief The value of isAdjustedTOUTC when writing a TIME column
|
| 1463 |
+
///
|
| 1464 |
+
/// Note this setting doesn't affect TIMESTAMP data.
|
| 1465 |
+
bool write_time_adjusted_to_utc() const { return write_time_adjusted_to_utc_; }
|
| 1466 |
+
|
| 1467 |
+
private:
|
| 1468 |
+
explicit ArrowWriterProperties(bool write_nanos_as_int96,
|
| 1469 |
+
bool coerce_timestamps_enabled,
|
| 1470 |
+
::arrow::TimeUnit::type coerce_timestamps_unit,
|
| 1471 |
+
bool truncated_timestamps_allowed, bool store_schema,
|
| 1472 |
+
bool compliant_nested_types,
|
| 1473 |
+
EngineVersion engine_version, bool use_threads,
|
| 1474 |
+
::arrow::internal::Executor* executor,
|
| 1475 |
+
bool write_time_adjusted_to_utc)
|
| 1476 |
+
: write_timestamps_as_int96_(write_nanos_as_int96),
|
| 1477 |
+
coerce_timestamps_enabled_(coerce_timestamps_enabled),
|
| 1478 |
+
coerce_timestamps_unit_(coerce_timestamps_unit),
|
| 1479 |
+
truncated_timestamps_allowed_(truncated_timestamps_allowed),
|
| 1480 |
+
store_schema_(store_schema),
|
| 1481 |
+
compliant_nested_types_(compliant_nested_types),
|
| 1482 |
+
engine_version_(engine_version),
|
| 1483 |
+
use_threads_(use_threads),
|
| 1484 |
+
executor_(executor),
|
| 1485 |
+
write_time_adjusted_to_utc_(write_time_adjusted_to_utc) {}
|
| 1486 |
+
|
| 1487 |
+
const bool write_timestamps_as_int96_;
|
| 1488 |
+
const bool coerce_timestamps_enabled_;
|
| 1489 |
+
const ::arrow::TimeUnit::type coerce_timestamps_unit_;
|
| 1490 |
+
const bool truncated_timestamps_allowed_;
|
| 1491 |
+
const bool store_schema_;
|
| 1492 |
+
const bool compliant_nested_types_;
|
| 1493 |
+
const EngineVersion engine_version_;
|
| 1494 |
+
const bool use_threads_;
|
| 1495 |
+
::arrow::internal::Executor* executor_;
|
| 1496 |
+
const bool write_time_adjusted_to_utc_;
|
| 1497 |
+
};
|
| 1498 |
+
|
| 1499 |
+
/// \brief State object used for writing Arrow data directly to a Parquet
|
| 1500 |
+
/// column chunk. API possibly not stable
|
| 1501 |
+
struct ArrowWriteContext {
|
| 1502 |
+
ArrowWriteContext(MemoryPool* memory_pool, ArrowWriterProperties* properties)
|
| 1503 |
+
: memory_pool(memory_pool),
|
| 1504 |
+
properties(properties),
|
| 1505 |
+
data_buffer(AllocateBuffer(memory_pool)),
|
| 1506 |
+
def_levels_buffer(AllocateBuffer(memory_pool)) {}
|
| 1507 |
+
|
| 1508 |
+
template <typename T>
|
| 1509 |
+
::arrow::Status GetScratchData(const int64_t num_values, T** out) {
|
| 1510 |
+
ARROW_RETURN_NOT_OK(this->data_buffer->Resize(num_values * sizeof(T), false));
|
| 1511 |
+
*out = reinterpret_cast<T*>(this->data_buffer->mutable_data());
|
| 1512 |
+
return ::arrow::Status::OK();
|
| 1513 |
+
}
|
| 1514 |
+
|
| 1515 |
+
MemoryPool* memory_pool;
|
| 1516 |
+
const ArrowWriterProperties* properties;
|
| 1517 |
+
|
| 1518 |
+
// Buffer used for storing the data of an array converted to the physical type
|
| 1519 |
+
// as expected by parquet-cpp.
|
| 1520 |
+
std::shared_ptr<ResizableBuffer> data_buffer;
|
| 1521 |
+
|
| 1522 |
+
// We use the shared ownership of this buffer
|
| 1523 |
+
std::shared_ptr<ResizableBuffer> def_levels_buffer;
|
| 1524 |
+
};
|
| 1525 |
+
|
| 1526 |
+
PARQUET_EXPORT
|
| 1527 |
+
std::shared_ptr<ArrowWriterProperties> default_arrow_writer_properties();
|
| 1528 |
+
|
| 1529 |
+
} // namespace parquet
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/schema.h
ADDED
|
@@ -0,0 +1,494 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
// This module contains the logical parquet-cpp types (independent of Thrift
|
| 19 |
+
// structures), schema nodes, and related type tools
|
| 20 |
+
|
| 21 |
+
#pragma once
|
| 22 |
+
|
| 23 |
+
#include <cstdint>
|
| 24 |
+
#include <memory>
|
| 25 |
+
#include <ostream>
|
| 26 |
+
#include <string>
|
| 27 |
+
#include <unordered_map>
|
| 28 |
+
#include <utility>
|
| 29 |
+
#include <vector>
|
| 30 |
+
|
| 31 |
+
#include "parquet/platform.h"
|
| 32 |
+
#include "parquet/types.h"
|
| 33 |
+
#include "parquet/windows_fixup.h" // for OPTIONAL
|
| 34 |
+
|
| 35 |
+
namespace parquet {
|
| 36 |
+
|
| 37 |
+
class SchemaDescriptor;
|
| 38 |
+
|
| 39 |
+
namespace schema {
|
| 40 |
+
|
| 41 |
+
class Node;
|
| 42 |
+
|
| 43 |
+
// List encodings: using the terminology from Impala to define different styles
|
| 44 |
+
// of representing logical lists (a.k.a. ARRAY types) in Parquet schemas. Since
|
| 45 |
+
// the converted type named in the Parquet metadata is ConvertedType::LIST we
|
| 46 |
+
// use that terminology here. It also helps distinguish from the *_ARRAY
|
| 47 |
+
// primitive types.
|
| 48 |
+
//
|
| 49 |
+
// One-level encoding: Only allows required lists with required cells
|
| 50 |
+
// repeated value_type name
|
| 51 |
+
//
|
| 52 |
+
// Two-level encoding: Enables optional lists with only required cells
|
| 53 |
+
// <required/optional> group list
|
| 54 |
+
// repeated value_type item
|
| 55 |
+
//
|
| 56 |
+
// Three-level encoding: Enables optional lists with optional cells
|
| 57 |
+
// <required/optional> group bag
|
| 58 |
+
// repeated group list
|
| 59 |
+
// <required/optional> value_type item
|
| 60 |
+
//
|
| 61 |
+
// 2- and 1-level encoding are respectively equivalent to 3-level encoding with
|
| 62 |
+
// the non-repeated nodes set to required.
|
| 63 |
+
//
|
| 64 |
+
// The "official" encoding recommended in the Parquet spec is the 3-level, and
|
| 65 |
+
// we use that as the default when creating list types. For semantic completeness
|
| 66 |
+
// we allow the other two. Since all types of encodings will occur "in the
|
| 67 |
+
// wild" we need to be able to interpret the associated definition levels in
|
| 68 |
+
// the context of the actual encoding used in the file.
|
| 69 |
+
//
|
| 70 |
+
// NB: Some Parquet writers may not set ConvertedType::LIST on the repeated
|
| 71 |
+
// SchemaElement, which could make things challenging if we are trying to infer
|
| 72 |
+
// that a sequence of nodes semantically represents an array according to one
|
| 73 |
+
// of these encodings (versus a struct containing an array). We should refuse
|
| 74 |
+
// the temptation to guess, as they say.
|
| 75 |
+
struct ListEncoding {
|
| 76 |
+
enum type { ONE_LEVEL, TWO_LEVEL, THREE_LEVEL };
|
| 77 |
+
};
|
| 78 |
+
|
| 79 |
+
class PARQUET_EXPORT ColumnPath {
|
| 80 |
+
public:
|
| 81 |
+
ColumnPath() : path_() {}
|
| 82 |
+
explicit ColumnPath(const std::vector<std::string>& path) : path_(path) {}
|
| 83 |
+
explicit ColumnPath(std::vector<std::string>&& path) : path_(std::move(path)) {}
|
| 84 |
+
|
| 85 |
+
static std::shared_ptr<ColumnPath> FromDotString(const std::string& dotstring);
|
| 86 |
+
static std::shared_ptr<ColumnPath> FromNode(const Node& node);
|
| 87 |
+
|
| 88 |
+
std::shared_ptr<ColumnPath> extend(const std::string& node_name) const;
|
| 89 |
+
std::string ToDotString() const;
|
| 90 |
+
const std::vector<std::string>& ToDotVector() const;
|
| 91 |
+
|
| 92 |
+
protected:
|
| 93 |
+
std::vector<std::string> path_;
|
| 94 |
+
};
|
| 95 |
+
|
| 96 |
+
// Base class for logical schema types. A type has a name, repetition level,
|
| 97 |
+
// and optionally a logical type (ConvertedType in Parquet metadata parlance)
|
| 98 |
+
class PARQUET_EXPORT Node {
|
| 99 |
+
public:
|
| 100 |
+
enum type { PRIMITIVE, GROUP };
|
| 101 |
+
|
| 102 |
+
virtual ~Node() {}
|
| 103 |
+
|
| 104 |
+
bool is_primitive() const { return type_ == Node::PRIMITIVE; }
|
| 105 |
+
|
| 106 |
+
bool is_group() const { return type_ == Node::GROUP; }
|
| 107 |
+
|
| 108 |
+
bool is_optional() const { return repetition_ == Repetition::OPTIONAL; }
|
| 109 |
+
|
| 110 |
+
bool is_repeated() const { return repetition_ == Repetition::REPEATED; }
|
| 111 |
+
|
| 112 |
+
bool is_required() const { return repetition_ == Repetition::REQUIRED; }
|
| 113 |
+
|
| 114 |
+
virtual bool Equals(const Node* other) const = 0;
|
| 115 |
+
|
| 116 |
+
const std::string& name() const { return name_; }
|
| 117 |
+
|
| 118 |
+
Node::type node_type() const { return type_; }
|
| 119 |
+
|
| 120 |
+
Repetition::type repetition() const { return repetition_; }
|
| 121 |
+
|
| 122 |
+
ConvertedType::type converted_type() const { return converted_type_; }
|
| 123 |
+
|
| 124 |
+
const std::shared_ptr<const LogicalType>& logical_type() const { return logical_type_; }
|
| 125 |
+
|
| 126 |
+
/// \brief The field_id value for the serialized SchemaElement. If the
|
| 127 |
+
/// field_id is less than 0 (e.g. -1), it will not be set when serialized to
|
| 128 |
+
/// Thrift.
|
| 129 |
+
int field_id() const { return field_id_; }
|
| 130 |
+
|
| 131 |
+
const Node* parent() const { return parent_; }
|
| 132 |
+
|
| 133 |
+
const std::shared_ptr<ColumnPath> path() const;
|
| 134 |
+
|
| 135 |
+
virtual void ToParquet(void* element) const = 0;
|
| 136 |
+
|
| 137 |
+
// Node::Visitor abstract class for walking schemas with the visitor pattern
|
| 138 |
+
class Visitor {
|
| 139 |
+
public:
|
| 140 |
+
virtual ~Visitor() {}
|
| 141 |
+
|
| 142 |
+
virtual void Visit(Node* node) = 0;
|
| 143 |
+
};
|
| 144 |
+
class ConstVisitor {
|
| 145 |
+
public:
|
| 146 |
+
virtual ~ConstVisitor() {}
|
| 147 |
+
|
| 148 |
+
virtual void Visit(const Node* node) = 0;
|
| 149 |
+
};
|
| 150 |
+
|
| 151 |
+
virtual void Visit(Visitor* visitor) = 0;
|
| 152 |
+
virtual void VisitConst(ConstVisitor* visitor) const = 0;
|
| 153 |
+
|
| 154 |
+
protected:
|
| 155 |
+
friend class GroupNode;
|
| 156 |
+
|
| 157 |
+
Node(Node::type type, const std::string& name, Repetition::type repetition,
|
| 158 |
+
ConvertedType::type converted_type = ConvertedType::NONE, int field_id = -1)
|
| 159 |
+
: type_(type),
|
| 160 |
+
name_(name),
|
| 161 |
+
repetition_(repetition),
|
| 162 |
+
converted_type_(converted_type),
|
| 163 |
+
field_id_(field_id),
|
| 164 |
+
parent_(NULLPTR) {}
|
| 165 |
+
|
| 166 |
+
Node(Node::type type, const std::string& name, Repetition::type repetition,
|
| 167 |
+
std::shared_ptr<const LogicalType> logical_type, int field_id = -1)
|
| 168 |
+
: type_(type),
|
| 169 |
+
name_(name),
|
| 170 |
+
repetition_(repetition),
|
| 171 |
+
logical_type_(std::move(logical_type)),
|
| 172 |
+
field_id_(field_id),
|
| 173 |
+
parent_(NULLPTR) {}
|
| 174 |
+
|
| 175 |
+
Node::type type_;
|
| 176 |
+
std::string name_;
|
| 177 |
+
Repetition::type repetition_;
|
| 178 |
+
ConvertedType::type converted_type_{ConvertedType::NONE};
|
| 179 |
+
std::shared_ptr<const LogicalType> logical_type_;
|
| 180 |
+
int field_id_;
|
| 181 |
+
// Nodes should not be shared, they have a single parent.
|
| 182 |
+
const Node* parent_;
|
| 183 |
+
|
| 184 |
+
bool EqualsInternal(const Node* other) const;
|
| 185 |
+
void SetParent(const Node* p_parent);
|
| 186 |
+
|
| 187 |
+
private:
|
| 188 |
+
PARQUET_DISALLOW_COPY_AND_ASSIGN(Node);
|
| 189 |
+
};
|
| 190 |
+
|
| 191 |
+
// Save our breath all over the place with these typedefs
|
| 192 |
+
using NodePtr = std::shared_ptr<Node>;
|
| 193 |
+
using NodeVector = std::vector<NodePtr>;
|
| 194 |
+
|
| 195 |
+
// A type that is one of the primitive Parquet storage types. In addition to
|
| 196 |
+
// the other type metadata (name, repetition level, logical type), also has the
|
| 197 |
+
// physical storage type and their type-specific metadata (byte width, decimal
|
| 198 |
+
// parameters)
|
| 199 |
+
class PARQUET_EXPORT PrimitiveNode : public Node {
|
| 200 |
+
public:
|
| 201 |
+
static std::unique_ptr<Node> FromParquet(const void* opaque_element);
|
| 202 |
+
|
| 203 |
+
// A field_id -1 (or any negative value) will be serialized as null in Thrift
|
| 204 |
+
static inline NodePtr Make(const std::string& name, Repetition::type repetition,
|
| 205 |
+
Type::type type,
|
| 206 |
+
ConvertedType::type converted_type = ConvertedType::NONE,
|
| 207 |
+
int length = -1, int precision = -1, int scale = -1,
|
| 208 |
+
int field_id = -1) {
|
| 209 |
+
return NodePtr(new PrimitiveNode(name, repetition, type, converted_type, length,
|
| 210 |
+
precision, scale, field_id));
|
| 211 |
+
}
|
| 212 |
+
|
| 213 |
+
// If no logical type, pass LogicalType::None() or nullptr
|
| 214 |
+
// A field_id -1 (or any negative value) will be serialized as null in Thrift
|
| 215 |
+
static inline NodePtr Make(const std::string& name, Repetition::type repetition,
|
| 216 |
+
std::shared_ptr<const LogicalType> logical_type,
|
| 217 |
+
Type::type primitive_type, int primitive_length = -1,
|
| 218 |
+
int field_id = -1) {
|
| 219 |
+
return NodePtr(new PrimitiveNode(name, repetition, std::move(logical_type),
|
| 220 |
+
primitive_type, primitive_length, field_id));
|
| 221 |
+
}
|
| 222 |
+
|
| 223 |
+
bool Equals(const Node* other) const override;
|
| 224 |
+
|
| 225 |
+
Type::type physical_type() const { return physical_type_; }
|
| 226 |
+
|
| 227 |
+
ColumnOrder column_order() const { return column_order_; }
|
| 228 |
+
|
| 229 |
+
void SetColumnOrder(ColumnOrder column_order) { column_order_ = column_order; }
|
| 230 |
+
|
| 231 |
+
int32_t type_length() const { return type_length_; }
|
| 232 |
+
|
| 233 |
+
const DecimalMetadata& decimal_metadata() const { return decimal_metadata_; }
|
| 234 |
+
|
| 235 |
+
void ToParquet(void* element) const override;
|
| 236 |
+
void Visit(Visitor* visitor) override;
|
| 237 |
+
void VisitConst(ConstVisitor* visitor) const override;
|
| 238 |
+
|
| 239 |
+
private:
|
| 240 |
+
PrimitiveNode(const std::string& name, Repetition::type repetition, Type::type type,
|
| 241 |
+
ConvertedType::type converted_type = ConvertedType::NONE, int length = -1,
|
| 242 |
+
int precision = -1, int scale = -1, int field_id = -1);
|
| 243 |
+
|
| 244 |
+
PrimitiveNode(const std::string& name, Repetition::type repetition,
|
| 245 |
+
std::shared_ptr<const LogicalType> logical_type,
|
| 246 |
+
Type::type primitive_type, int primitive_length = -1, int field_id = -1);
|
| 247 |
+
|
| 248 |
+
Type::type physical_type_;
|
| 249 |
+
int32_t type_length_;
|
| 250 |
+
DecimalMetadata decimal_metadata_;
|
| 251 |
+
ColumnOrder column_order_;
|
| 252 |
+
|
| 253 |
+
// For FIXED_LEN_BYTE_ARRAY
|
| 254 |
+
void SetTypeLength(int32_t length) { type_length_ = length; }
|
| 255 |
+
|
| 256 |
+
bool EqualsInternal(const PrimitiveNode* other) const;
|
| 257 |
+
|
| 258 |
+
FRIEND_TEST(TestPrimitiveNode, Attrs);
|
| 259 |
+
FRIEND_TEST(TestPrimitiveNode, Equals);
|
| 260 |
+
FRIEND_TEST(TestPrimitiveNode, PhysicalLogicalMapping);
|
| 261 |
+
FRIEND_TEST(TestPrimitiveNode, FromParquet);
|
| 262 |
+
};
|
| 263 |
+
|
| 264 |
+
class PARQUET_EXPORT GroupNode : public Node {
|
| 265 |
+
public:
|
| 266 |
+
static std::unique_ptr<Node> FromParquet(const void* opaque_element,
|
| 267 |
+
NodeVector fields = {});
|
| 268 |
+
|
| 269 |
+
// A field_id -1 (or any negative value) will be serialized as null in Thrift
|
| 270 |
+
static inline NodePtr Make(const std::string& name, Repetition::type repetition,
|
| 271 |
+
const NodeVector& fields,
|
| 272 |
+
ConvertedType::type converted_type = ConvertedType::NONE,
|
| 273 |
+
int field_id = -1) {
|
| 274 |
+
return NodePtr(new GroupNode(name, repetition, fields, converted_type, field_id));
|
| 275 |
+
}
|
| 276 |
+
|
| 277 |
+
// If no logical type, pass nullptr
|
| 278 |
+
// A field_id -1 (or any negative value) will be serialized as null in Thrift
|
| 279 |
+
static inline NodePtr Make(const std::string& name, Repetition::type repetition,
|
| 280 |
+
const NodeVector& fields,
|
| 281 |
+
std::shared_ptr<const LogicalType> logical_type,
|
| 282 |
+
int field_id = -1) {
|
| 283 |
+
return NodePtr(
|
| 284 |
+
new GroupNode(name, repetition, fields, std::move(logical_type), field_id));
|
| 285 |
+
}
|
| 286 |
+
|
| 287 |
+
bool Equals(const Node* other) const override;
|
| 288 |
+
|
| 289 |
+
const NodePtr& field(int i) const { return fields_[i]; }
|
| 290 |
+
// Get the index of a field by its name, or negative value if not found.
|
| 291 |
+
// If several fields share the same name, it is unspecified which one
|
| 292 |
+
// is returned.
|
| 293 |
+
int FieldIndex(const std::string& name) const;
|
| 294 |
+
// Get the index of a field by its node, or negative value if not found.
|
| 295 |
+
int FieldIndex(const Node& node) const;
|
| 296 |
+
|
| 297 |
+
int field_count() const { return static_cast<int>(fields_.size()); }
|
| 298 |
+
|
| 299 |
+
void ToParquet(void* element) const override;
|
| 300 |
+
void Visit(Visitor* visitor) override;
|
| 301 |
+
void VisitConst(ConstVisitor* visitor) const override;
|
| 302 |
+
|
| 303 |
+
/// \brief Return true if this node or any child node has REPEATED repetition
|
| 304 |
+
/// type
|
| 305 |
+
bool HasRepeatedFields() const;
|
| 306 |
+
|
| 307 |
+
private:
|
| 308 |
+
GroupNode(const std::string& name, Repetition::type repetition,
|
| 309 |
+
const NodeVector& fields,
|
| 310 |
+
ConvertedType::type converted_type = ConvertedType::NONE, int field_id = -1);
|
| 311 |
+
|
| 312 |
+
GroupNode(const std::string& name, Repetition::type repetition,
|
| 313 |
+
const NodeVector& fields, std::shared_ptr<const LogicalType> logical_type,
|
| 314 |
+
int field_id = -1);
|
| 315 |
+
|
| 316 |
+
NodeVector fields_;
|
| 317 |
+
bool EqualsInternal(const GroupNode* other) const;
|
| 318 |
+
|
| 319 |
+
// Mapping between field name to the field index
|
| 320 |
+
std::unordered_multimap<std::string, int> field_name_to_idx_;
|
| 321 |
+
|
| 322 |
+
FRIEND_TEST(TestGroupNode, Attrs);
|
| 323 |
+
FRIEND_TEST(TestGroupNode, Equals);
|
| 324 |
+
FRIEND_TEST(TestGroupNode, FieldIndex);
|
| 325 |
+
FRIEND_TEST(TestGroupNode, FieldIndexDuplicateName);
|
| 326 |
+
};
|
| 327 |
+
|
| 328 |
+
// ----------------------------------------------------------------------
|
| 329 |
+
// Convenience primitive type factory functions
|
| 330 |
+
|
| 331 |
+
#define PRIMITIVE_FACTORY(FuncName, TYPE) \
|
| 332 |
+
static inline NodePtr FuncName(const std::string& name, \
|
| 333 |
+
Repetition::type repetition = Repetition::OPTIONAL, \
|
| 334 |
+
int field_id = -1) { \
|
| 335 |
+
return PrimitiveNode::Make(name, repetition, Type::TYPE, ConvertedType::NONE, \
|
| 336 |
+
/*length=*/-1, /*precision=*/-1, /*scale=*/-1, field_id); \
|
| 337 |
+
}
|
| 338 |
+
|
| 339 |
+
PRIMITIVE_FACTORY(Boolean, BOOLEAN)
|
| 340 |
+
PRIMITIVE_FACTORY(Int32, INT32)
|
| 341 |
+
PRIMITIVE_FACTORY(Int64, INT64)
|
| 342 |
+
PRIMITIVE_FACTORY(Int96, INT96)
|
| 343 |
+
PRIMITIVE_FACTORY(Float, FLOAT)
|
| 344 |
+
PRIMITIVE_FACTORY(Double, DOUBLE)
|
| 345 |
+
PRIMITIVE_FACTORY(ByteArray, BYTE_ARRAY)
|
| 346 |
+
|
| 347 |
+
void PARQUET_EXPORT PrintSchema(const schema::Node* schema, std::ostream& stream,
|
| 348 |
+
int indent_width = 2);
|
| 349 |
+
|
| 350 |
+
} // namespace schema
|
| 351 |
+
|
| 352 |
+
// The ColumnDescriptor encapsulates information necessary to interpret
|
| 353 |
+
// primitive column data in the context of a particular schema. We have to
|
| 354 |
+
// examine the node structure of a column's path to the root in the schema tree
|
| 355 |
+
// to be able to reassemble the nested structure from the repetition and
|
| 356 |
+
// definition levels.
|
| 357 |
+
class PARQUET_EXPORT ColumnDescriptor {
|
| 358 |
+
public:
|
| 359 |
+
ColumnDescriptor(schema::NodePtr node, int16_t max_definition_level,
|
| 360 |
+
int16_t max_repetition_level,
|
| 361 |
+
const SchemaDescriptor* schema_descr = NULLPTR);
|
| 362 |
+
|
| 363 |
+
bool Equals(const ColumnDescriptor& other) const;
|
| 364 |
+
|
| 365 |
+
int16_t max_definition_level() const { return max_definition_level_; }
|
| 366 |
+
|
| 367 |
+
int16_t max_repetition_level() const { return max_repetition_level_; }
|
| 368 |
+
|
| 369 |
+
Type::type physical_type() const { return primitive_node_->physical_type(); }
|
| 370 |
+
|
| 371 |
+
ConvertedType::type converted_type() const { return primitive_node_->converted_type(); }
|
| 372 |
+
|
| 373 |
+
const std::shared_ptr<const LogicalType>& logical_type() const {
|
| 374 |
+
return primitive_node_->logical_type();
|
| 375 |
+
}
|
| 376 |
+
|
| 377 |
+
ColumnOrder column_order() const { return primitive_node_->column_order(); }
|
| 378 |
+
|
| 379 |
+
SortOrder::type sort_order() const {
|
| 380 |
+
const auto& la = logical_type();
|
| 381 |
+
auto pt = physical_type();
|
| 382 |
+
return la ? GetSortOrder(la, pt) : GetSortOrder(converted_type(), pt);
|
| 383 |
+
}
|
| 384 |
+
|
| 385 |
+
const std::string& name() const { return primitive_node_->name(); }
|
| 386 |
+
|
| 387 |
+
const std::shared_ptr<schema::ColumnPath> path() const;
|
| 388 |
+
|
| 389 |
+
const schema::NodePtr& schema_node() const { return node_; }
|
| 390 |
+
|
| 391 |
+
std::string ToString() const;
|
| 392 |
+
|
| 393 |
+
int type_length() const;
|
| 394 |
+
|
| 395 |
+
int type_precision() const;
|
| 396 |
+
|
| 397 |
+
int type_scale() const;
|
| 398 |
+
|
| 399 |
+
private:
|
| 400 |
+
schema::NodePtr node_;
|
| 401 |
+
const schema::PrimitiveNode* primitive_node_;
|
| 402 |
+
|
| 403 |
+
int16_t max_definition_level_;
|
| 404 |
+
int16_t max_repetition_level_;
|
| 405 |
+
};
|
| 406 |
+
|
| 407 |
+
// Container for the converted Parquet schema with a computed information from
|
| 408 |
+
// the schema analysis needed for file reading
|
| 409 |
+
//
|
| 410 |
+
// * Column index to Node
|
| 411 |
+
// * Max repetition / definition levels for each primitive node
|
| 412 |
+
//
|
| 413 |
+
// The ColumnDescriptor objects produced by this class can be used to assist in
|
| 414 |
+
// the reconstruction of fully materialized data structures from the
|
| 415 |
+
// repetition-definition level encoding of nested data
|
| 416 |
+
//
|
| 417 |
+
// TODO(wesm): this object can be recomputed from a Schema
|
| 418 |
+
class PARQUET_EXPORT SchemaDescriptor {
|
| 419 |
+
public:
|
| 420 |
+
SchemaDescriptor() = default;
|
| 421 |
+
~SchemaDescriptor() = default;
|
| 422 |
+
|
| 423 |
+
// Analyze the schema
|
| 424 |
+
void Init(std::unique_ptr<schema::Node> schema);
|
| 425 |
+
void Init(schema::NodePtr schema);
|
| 426 |
+
|
| 427 |
+
const ColumnDescriptor* Column(int i) const;
|
| 428 |
+
|
| 429 |
+
// Get the index of a column by its dotstring path, or negative value if not found.
|
| 430 |
+
// If several columns share the same dotstring path, it is unspecified which one
|
| 431 |
+
// is returned.
|
| 432 |
+
int ColumnIndex(const std::string& node_path) const;
|
| 433 |
+
// Get the index of a column by its node, or negative value if not found.
|
| 434 |
+
int ColumnIndex(const schema::Node& node) const;
|
| 435 |
+
|
| 436 |
+
bool Equals(const SchemaDescriptor& other, std::ostream* diff_output = NULLPTR) const;
|
| 437 |
+
|
| 438 |
+
// The number of physical columns appearing in the file
|
| 439 |
+
int num_columns() const { return static_cast<int>(leaves_.size()); }
|
| 440 |
+
|
| 441 |
+
const schema::NodePtr& schema_root() const { return schema_; }
|
| 442 |
+
|
| 443 |
+
const schema::GroupNode* group_node() const { return group_node_; }
|
| 444 |
+
|
| 445 |
+
// Returns the root (child of the schema root) node of the leaf(column) node
|
| 446 |
+
const schema::Node* GetColumnRoot(int i) const;
|
| 447 |
+
|
| 448 |
+
const std::string& name() const { return group_node_->name(); }
|
| 449 |
+
|
| 450 |
+
std::string ToString() const;
|
| 451 |
+
|
| 452 |
+
void updateColumnOrders(const std::vector<ColumnOrder>& column_orders);
|
| 453 |
+
|
| 454 |
+
/// \brief Return column index corresponding to a particular
|
| 455 |
+
/// PrimitiveNode. Returns -1 if not found
|
| 456 |
+
int GetColumnIndex(const schema::PrimitiveNode& node) const;
|
| 457 |
+
|
| 458 |
+
/// \brief Return true if any field or their children have REPEATED repetition
|
| 459 |
+
/// type
|
| 460 |
+
bool HasRepeatedFields() const;
|
| 461 |
+
|
| 462 |
+
private:
|
| 463 |
+
friend class ColumnDescriptor;
|
| 464 |
+
|
| 465 |
+
// Root Node
|
| 466 |
+
schema::NodePtr schema_;
|
| 467 |
+
// Root Node
|
| 468 |
+
// Would never be NULLPTR.
|
| 469 |
+
const schema::GroupNode* group_node_;
|
| 470 |
+
|
| 471 |
+
void BuildTree(const schema::NodePtr& node, int16_t max_def_level,
|
| 472 |
+
int16_t max_rep_level, const schema::NodePtr& base);
|
| 473 |
+
|
| 474 |
+
// Result of leaf node / tree analysis
|
| 475 |
+
std::vector<ColumnDescriptor> leaves_;
|
| 476 |
+
|
| 477 |
+
std::unordered_map<const schema::PrimitiveNode*, int> node_to_leaf_index_;
|
| 478 |
+
|
| 479 |
+
// Mapping between leaf nodes and root group of leaf (first node
|
| 480 |
+
// below the schema's root group)
|
| 481 |
+
//
|
| 482 |
+
// For example, the leaf `a.b.c.d` would have a link back to `a`
|
| 483 |
+
//
|
| 484 |
+
// -- a <------
|
| 485 |
+
// -- -- b |
|
| 486 |
+
// -- -- -- c |
|
| 487 |
+
// -- -- -- -- d
|
| 488 |
+
std::unordered_map<int, schema::NodePtr> leaf_to_base_;
|
| 489 |
+
|
| 490 |
+
// Mapping between ColumnPath DotString to the leaf index
|
| 491 |
+
std::unordered_multimap<std::string, int> leaf_to_idx_;
|
| 492 |
+
};
|
| 493 |
+
|
| 494 |
+
} // namespace parquet
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/size_statistics.h
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <cstdint>
|
| 21 |
+
#include <iosfwd>
|
| 22 |
+
#include <optional>
|
| 23 |
+
#include <span>
|
| 24 |
+
#include <vector>
|
| 25 |
+
|
| 26 |
+
#include "parquet/platform.h"
|
| 27 |
+
#include "parquet/type_fwd.h"
|
| 28 |
+
|
| 29 |
+
namespace parquet {
|
| 30 |
+
|
| 31 |
+
/// A structure for capturing metadata for estimating the unencoded,
|
| 32 |
+
/// uncompressed size of data written. This is useful for readers to estimate
|
| 33 |
+
/// how much memory is needed to reconstruct data in their memory model and for
|
| 34 |
+
/// fine-grained filter push down on nested structures (the histograms contained
|
| 35 |
+
/// in this structure can help determine the number of nulls at a particular
|
| 36 |
+
/// nesting level and maximum length of lists).
|
| 37 |
+
struct PARQUET_EXPORT SizeStatistics {
|
| 38 |
+
/// When present, there is expected to be one element corresponding to each
|
| 39 |
+
/// definition (i.e. size=max definition+1) where each element
|
| 40 |
+
/// represents the number of times the definition level was observed in the
|
| 41 |
+
/// data.
|
| 42 |
+
///
|
| 43 |
+
/// This field may be omitted (a.k.a. zero-length vector) if max_definition_level
|
| 44 |
+
/// is 0 without loss of information.
|
| 45 |
+
std::vector<int64_t> definition_level_histogram;
|
| 46 |
+
|
| 47 |
+
/// Same as definition_level_histogram except for repetition levels.
|
| 48 |
+
///
|
| 49 |
+
/// This field may be omitted (a.k.a. zero-length vector) if max_repetition_level
|
| 50 |
+
/// is 0 without loss of information.
|
| 51 |
+
std::vector<int64_t> repetition_level_histogram;
|
| 52 |
+
|
| 53 |
+
/// The number of physical bytes stored for BYTE_ARRAY data values assuming
|
| 54 |
+
/// no encoding. This is exclusive of the bytes needed to store the length of
|
| 55 |
+
/// each byte array. In other words, this field is equivalent to the `(size
|
| 56 |
+
/// of PLAIN-ENCODING the byte array values) - (4 bytes * number of values
|
| 57 |
+
/// written)`. To determine unencoded sizes of other types readers can use
|
| 58 |
+
/// schema information multiplied by the number of non-null and null values.
|
| 59 |
+
/// The number of null/non-null values can be inferred from the histograms
|
| 60 |
+
/// below.
|
| 61 |
+
///
|
| 62 |
+
/// For example, if a column chunk is dictionary-encoded with dictionary
|
| 63 |
+
/// ["a", "bc", "cde"], and a data page contains the indices [0, 0, 1, 2],
|
| 64 |
+
/// then this value for that data page should be 7 (1 + 1 + 2 + 3).
|
| 65 |
+
///
|
| 66 |
+
/// This field should only be set for types that use BYTE_ARRAY as their
|
| 67 |
+
/// physical type.
|
| 68 |
+
std::optional<int64_t> unencoded_byte_array_data_bytes;
|
| 69 |
+
|
| 70 |
+
/// \brief Check if the SizeStatistics is set.
|
| 71 |
+
bool is_set() const {
|
| 72 |
+
return !repetition_level_histogram.empty() || !definition_level_histogram.empty() ||
|
| 73 |
+
unencoded_byte_array_data_bytes.has_value();
|
| 74 |
+
}
|
| 75 |
+
|
| 76 |
+
/// \brief Increment the unencoded byte array data bytes.
|
| 77 |
+
void IncrementUnencodedByteArrayDataBytes(int64_t value);
|
| 78 |
+
|
| 79 |
+
/// \brief Merge two SizeStatistics.
|
| 80 |
+
/// \throws ParquetException if SizeStatistics to merge is not compatible.
|
| 81 |
+
void Merge(const SizeStatistics& other);
|
| 82 |
+
|
| 83 |
+
/// \brief Validate the SizeStatistics
|
| 84 |
+
/// \throws ParquetException if the histograms don't have the right length,
|
| 85 |
+
/// or if unencoded_byte_array_data_bytes is present for a non-BYTE_ARRAY column.
|
| 86 |
+
void Validate(const ColumnDescriptor* descr) const;
|
| 87 |
+
|
| 88 |
+
/// \brief Reset the SizeStatistics to be empty.
|
| 89 |
+
void Reset();
|
| 90 |
+
|
| 91 |
+
/// \brief Make an empty SizeStatistics object for specific type.
|
| 92 |
+
static std::unique_ptr<SizeStatistics> Make(const ColumnDescriptor* descr);
|
| 93 |
+
};
|
| 94 |
+
|
| 95 |
+
PARQUET_EXPORT
|
| 96 |
+
std::ostream& operator<<(std::ostream&, const SizeStatistics&);
|
| 97 |
+
|
| 98 |
+
PARQUET_EXPORT
|
| 99 |
+
void UpdateLevelHistogram(std::span<const int16_t> levels, std::span<int64_t> histogram);
|
| 100 |
+
|
| 101 |
+
} // namespace parquet
|