techprotrade commited on
Commit
41323cd
·
verified ·
1 Parent(s): f102409

Add loan-advisor-agent (part 3)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +3 -0
  2. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/api/writer.h +25 -0
  3. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/reader.h +424 -0
  4. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/schema.h +184 -0
  5. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/test_util.h +487 -0
  6. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/writer.h +176 -0
  7. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/benchmark_util.h +47 -0
  8. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/bloom_filter.h +392 -0
  9. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/bloom_filter_reader.h +68 -0
  10. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/bloom_filter_writer.h +105 -0
  11. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_page.h +181 -0
  12. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_reader.h +458 -0
  13. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_scanner.h +264 -0
  14. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_writer.h +317 -0
  15. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encoding.h +476 -0
  16. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/crypto_factory.h +152 -0
  17. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/encryption.h +434 -0
  18. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_key_material_store.h +57 -0
  19. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_key_unwrapper.h +96 -0
  20. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_key_wrapper.h +84 -0
  21. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_system_key_material_store.h +93 -0
  22. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_encryption_key.h +58 -0
  23. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_material.h +129 -0
  24. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_metadata.h +91 -0
  25. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_toolkit.h +106 -0
  26. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/kms_client.h +97 -0
  27. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/kms_client_factory.h +38 -0
  28. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/local_wrap_kms_client.h +95 -0
  29. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/test_encryption_util.h +136 -0
  30. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/test_in_memory_kms.h +101 -0
  31. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/two_level_cache_with_expiration.h +149 -0
  32. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/type_fwd.h +28 -0
  33. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/exception.h +175 -0
  34. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/file_reader.h +257 -0
  35. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/file_writer.h +260 -0
  36. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/geospatial/statistics.h +198 -0
  37. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/hasher.h +136 -0
  38. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/index_location.h +49 -0
  39. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_comparison.h +38 -0
  40. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_comparison_inc.h +61 -0
  41. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_conversion.h +216 -0
  42. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_conversion_inc.h +356 -0
  43. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/metadata.h +536 -0
  44. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/page_index.h +391 -0
  45. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/parquet_version.h +31 -0
  46. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/platform.h +116 -0
  47. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/printer.h +46 -0
  48. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/properties.h +1529 -0
  49. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/schema.h +494 -0
  50. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/size_statistics.h +101 -0
.gitattributes CHANGED
@@ -537,3 +537,6 @@ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_python.dll filter=lfs d
537
  loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_python.lib filter=lfs diff=lfs merge=lfs -text
538
  loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_substrait.dll filter=lfs diff=lfs merge=lfs -text
539
  loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_substrait.lib filter=lfs diff=lfs merge=lfs -text
 
 
 
 
537
  loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_python.lib filter=lfs diff=lfs merge=lfs -text
538
  loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_substrait.dll filter=lfs diff=lfs merge=lfs -text
539
  loan-advisor-agent/.venv/Lib/site-packages/pyarrow/arrow_substrait.lib filter=lfs diff=lfs merge=lfs -text
540
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/lib.cp312-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
541
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/parquet.dll filter=lfs diff=lfs merge=lfs -text
542
+ loan-advisor-agent/.venv/Lib/site-packages/pyarrow/parquet.lib filter=lfs diff=lfs merge=lfs -text
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/api/writer.h ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include "parquet/api/io.h"
21
+ #include "parquet/api/schema.h"
22
+ #include "parquet/column_writer.h"
23
+ #include "parquet/exception.h"
24
+ #include "parquet/file_writer.h"
25
+ #include "parquet/statistics.h"
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/reader.h ADDED
@@ -0,0 +1,424 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <cstdint>
21
+ // N.B. we don't include async_generator.h as it's relatively heavy
22
+ #include <functional>
23
+ #include <memory>
24
+ #include <vector>
25
+
26
+ #include "parquet/file_reader.h"
27
+ #include "parquet/platform.h"
28
+ #include "parquet/properties.h"
29
+
30
+ namespace arrow {
31
+
32
+ class ChunkedArray;
33
+ class KeyValueMetadata;
34
+ class RecordBatchReader;
35
+ struct Scalar;
36
+ class Schema;
37
+ class Table;
38
+ class RecordBatch;
39
+
40
+ } // namespace arrow
41
+
42
+ namespace parquet {
43
+
44
+ class FileMetaData;
45
+ class SchemaDescriptor;
46
+
47
+ namespace arrow {
48
+
49
+ class ColumnChunkReader;
50
+ class ColumnReader;
51
+ struct SchemaManifest;
52
+ class RowGroupReader;
53
+
54
+ /// \brief Arrow read adapter class for deserializing Parquet files as Arrow row batches.
55
+ ///
56
+ /// This interfaces caters for different use cases and thus provides different
57
+ /// interfaces. In its most simplistic form, we cater for a user that wants to
58
+ /// read the whole Parquet at once with the `FileReader::ReadTable` method.
59
+ ///
60
+ /// More advanced users that also want to implement parallelism on top of each
61
+ /// single Parquet files should do this on the RowGroup level. For this, they can
62
+ /// call `FileReader::RowGroup(i)->ReadTable` to receive only the specified
63
+ /// RowGroup as a table.
64
+ ///
65
+ /// In the most advanced situation, where a consumer wants to independently read
66
+ /// RowGroups in parallel and consume each column individually, they can call
67
+ /// `FileReader::RowGroup(i)->Column(j)->Read` and receive an `arrow::Column`
68
+ /// instance.
69
+ ///
70
+ /// Finally, one can also get a stream of record batches using
71
+ /// `FileReader::GetRecordBatchReader()`. This can internally decode columns
72
+ /// in parallel if use_threads was enabled in the ArrowReaderProperties.
73
+ ///
74
+ /// The parquet format supports an optional integer field_id which can be assigned
75
+ /// to a field. Arrow will convert these field IDs to a metadata key named
76
+ /// PARQUET:field_id on the appropriate field.
77
+ // TODO(wesm): nested data does not always make sense with this user
78
+ // interface unless you are only reading a single leaf node from a branch of
79
+ // a table. For example:
80
+ //
81
+ // repeated group data {
82
+ // optional group record {
83
+ // optional int32 val1;
84
+ // optional byte_array val2;
85
+ // optional bool val3;
86
+ // }
87
+ // optional int32 val4;
88
+ // }
89
+ //
90
+ // In the Parquet file, there are 4 leaf nodes:
91
+ //
92
+ // * data.record.val1
93
+ // * data.record.val2
94
+ // * data.record.val3
95
+ // * data.val4
96
+ //
97
+ // When materializing this data in an Arrow array, we would have:
98
+ //
99
+ // data: list<struct<
100
+ // record: struct<
101
+ // val1: int32,
102
+ // val2: string (= list<uint8>),
103
+ // val3: bool,
104
+ // >,
105
+ // val4: int32
106
+ // >>
107
+ //
108
+ // However, in the Parquet format, each leaf node has its own repetition and
109
+ // definition levels describing the structure of the intermediate nodes in
110
+ // this array structure. Thus, we will need to scan the leaf data for a group
111
+ // of leaf nodes part of the same type tree to create a single result Arrow
112
+ // nested array structure.
113
+ //
114
+ // This is additionally complicated "chunky" repeated fields or very large byte
115
+ // arrays
116
+ class PARQUET_EXPORT FileReader {
117
+ public:
118
+ /// Factory function to create a FileReader from a ParquetFileReader and properties
119
+ /// \deprecated Deprecated in 23.0.0. Use arrow::Result version instead.
120
+ ARROW_DEPRECATED("Deprecated in 23.0.0. Use arrow::Result version instead.")
121
+ static ::arrow::Status Make(::arrow::MemoryPool* pool,
122
+ std::unique_ptr<ParquetFileReader> reader,
123
+ const ArrowReaderProperties& properties,
124
+ std::unique_ptr<FileReader>* out);
125
+
126
+ /// Factory function to create a FileReader from a ParquetFileReader
127
+ /// \deprecated Deprecated in 23.0.0. Use arrow::Result version instead.
128
+ ARROW_DEPRECATED("Deprecated in 23.0.0. Use arrow::Result version instead.")
129
+ static ::arrow::Status Make(::arrow::MemoryPool* pool,
130
+ std::unique_ptr<ParquetFileReader> reader,
131
+ std::unique_ptr<FileReader>* out);
132
+
133
+ /// Factory function to create a FileReader from a ParquetFileReader and properties
134
+ static ::arrow::Result<std::unique_ptr<FileReader>> Make(
135
+ ::arrow::MemoryPool* pool, std::unique_ptr<ParquetFileReader> reader,
136
+ const ArrowReaderProperties& properties);
137
+
138
+ /// Factory function to create a FileReader from a ParquetFileReader
139
+ static ::arrow::Result<std::unique_ptr<FileReader>> Make(
140
+ ::arrow::MemoryPool* pool, std::unique_ptr<ParquetFileReader> reader);
141
+
142
+ // Since the distribution of columns amongst a Parquet file's row groups may
143
+ // be uneven (the number of values in each column chunk can be different), we
144
+ // provide a column-oriented read interface. The ColumnReader hides the
145
+ // details of paging through the file's row groups and yielding
146
+ // fully-materialized arrow::Array instances
147
+ //
148
+ // Returns error status if the column of interest is not flat.
149
+ // The indicated column index is relative to the schema
150
+ virtual ::arrow::Status GetColumn(int i, std::unique_ptr<ColumnReader>* out) = 0;
151
+
152
+ /// \brief Return arrow schema for all the columns.
153
+ virtual ::arrow::Status GetSchema(std::shared_ptr<::arrow::Schema>* out) = 0;
154
+
155
+ /// \brief Read column as a whole into a chunked array.
156
+ ///
157
+ /// The index i refers the index of the top level schema field, which may
158
+ /// be nested or flat - e.g.
159
+ ///
160
+ /// 0 foo.bar
161
+ /// foo.bar.baz
162
+ /// foo.qux
163
+ /// 1 foo2
164
+ /// 2 foo3
165
+ ///
166
+ /// i=0 will read the entire foo struct, i=1 the foo2 primitive column etc
167
+ virtual ::arrow::Status ReadColumn(int i,
168
+ std::shared_ptr<::arrow::ChunkedArray>* out) = 0;
169
+
170
+ /// \brief Return a RecordBatchReader of all row groups and columns.
171
+ virtual ::arrow::Result<std::unique_ptr<::arrow::RecordBatchReader>>
172
+ GetRecordBatchReader() = 0;
173
+
174
+ /// \brief Return a RecordBatchReader of row groups selected from row_group_indices.
175
+ ///
176
+ /// Note that the ordering in row_group_indices matters. FileReaders must outlive
177
+ /// their RecordBatchReaders.
178
+ ///
179
+ /// \returns error Result if row_group_indices contains an invalid index
180
+ virtual ::arrow::Result<std::unique_ptr<::arrow::RecordBatchReader>>
181
+ GetRecordBatchReader(const std::vector<int>& row_group_indices) = 0;
182
+
183
+ /// \brief Return a RecordBatchReader of row groups selected from
184
+ /// row_group_indices, whose columns are selected by column_indices.
185
+ ///
186
+ /// Note that the ordering in row_group_indices and column_indices
187
+ /// matter. FileReaders must outlive their RecordBatchReaders.
188
+ ///
189
+ /// \returns error Result if either row_group_indices or column_indices
190
+ /// contains an invalid index
191
+ virtual ::arrow::Result<std::unique_ptr<::arrow::RecordBatchReader>>
192
+ GetRecordBatchReader(const std::vector<int>& row_group_indices,
193
+ const std::vector<int>& column_indices) = 0;
194
+
195
+ /// \brief Return a RecordBatchReader of row groups selected from
196
+ /// row_group_indices, whose columns are selected by column_indices.
197
+ ///
198
+ /// Note that the ordering in row_group_indices and column_indices
199
+ /// matter. FileReaders must outlive their RecordBatchReaders.
200
+ ///
201
+ /// \param row_group_indices which row groups to read (order determines read order).
202
+ /// \param column_indices which columns to read (order determines output schema).
203
+ /// \param[out] out record batch stream from parquet data.
204
+ ///
205
+ /// \returns error Status if either row_group_indices or column_indices
206
+ /// contains an invalid index
207
+ /// \deprecated Deprecated in 21.0.0. Use arrow::Result version instead.
208
+ ARROW_DEPRECATED("Deprecated in 21.0.0. Use arrow::Result version instead.")
209
+ ::arrow::Status GetRecordBatchReader(const std::vector<int>& row_group_indices,
210
+ const std::vector<int>& column_indices,
211
+ std::shared_ptr<::arrow::RecordBatchReader>* out);
212
+
213
+ /// \deprecated Deprecated in 21.0.0. Use arrow::Result version instead.
214
+ ARROW_DEPRECATED("Deprecated in 21.0.0. Use arrow::Result version instead.")
215
+ ::arrow::Status GetRecordBatchReader(const std::vector<int>& row_group_indices,
216
+ std::shared_ptr<::arrow::RecordBatchReader>* out);
217
+
218
+ /// \deprecated Deprecated in 21.0.0. Use arrow::Result version instead.
219
+ ARROW_DEPRECATED("Deprecated in 21.0.0. Use arrow::Result version instead.")
220
+ ::arrow::Status GetRecordBatchReader(std::shared_ptr<::arrow::RecordBatchReader>* out);
221
+
222
+ /// \brief Return a generator of record batches.
223
+ ///
224
+ /// The FileReader must outlive the generator, so this requires that you pass in a
225
+ /// shared_ptr.
226
+ ///
227
+ /// \returns error Result if either row_group_indices or column_indices contains an
228
+ /// invalid index
229
+ virtual ::arrow::Result<
230
+ std::function<::arrow::Future<std::shared_ptr<::arrow::RecordBatch>>()>>
231
+ GetRecordBatchGenerator(std::shared_ptr<FileReader> reader,
232
+ const std::vector<int> row_group_indices,
233
+ const std::vector<int> column_indices,
234
+ ::arrow::internal::Executor* cpu_executor = NULLPTR,
235
+ int64_t rows_to_readahead = 0) = 0;
236
+
237
+ /// Read all columns into a Table
238
+ virtual ::arrow::Result<std::shared_ptr<::arrow::Table>> ReadTable() = 0;
239
+
240
+ /// \deprecated Deprecated in 24.0.0. Use arrow::Result version instead.
241
+ ARROW_DEPRECATED("Deprecated in 24.0.0. Use arrow::Result version instead.")
242
+ ::arrow::Status ReadTable(std::shared_ptr<::arrow::Table>* out);
243
+
244
+ /// \brief Read the given columns into a Table
245
+ ///
246
+ /// The indicated column indices are relative to the internal representation
247
+ /// of the parquet table. For instance :
248
+ /// 0 foo.bar
249
+ /// foo.bar.baz 0
250
+ /// foo.bar.baz2 1
251
+ /// foo.qux 2
252
+ /// 1 foo2 3
253
+ /// 2 foo3 4
254
+ ///
255
+ /// i=0 will read foo.bar.baz, i=1 will read only foo.bar.baz2 and so on.
256
+ /// Only leaf fields have indices; foo itself doesn't have an index.
257
+ /// To get the index for a particular leaf field, one can use
258
+ /// manifest().schema_fields to get the top level fields, and then walk the
259
+ /// tree to identify the relevant leaf fields and access its column_index.
260
+ /// To get the total number of leaf fields, use FileMetadata.num_columns().
261
+ virtual ::arrow::Result<std::shared_ptr<::arrow::Table>> ReadTable(
262
+ const std::vector<int>& column_indices) = 0;
263
+
264
+ /// \deprecated Deprecated in 24.0.0. Use arrow::Result version instead.
265
+ ARROW_DEPRECATED("Deprecated in 24.0.0. Use arrow::Result version instead.")
266
+ ::arrow::Status ReadTable(const std::vector<int>& column_indices,
267
+ std::shared_ptr<::arrow::Table>* out);
268
+
269
+ /// \brief Read the given row group columns into a Table
270
+ virtual ::arrow::Result<std::shared_ptr<::arrow::Table>> ReadRowGroup(
271
+ int i, const std::vector<int>& column_indices) = 0;
272
+
273
+ /// \brief Read the given row group into a Table
274
+ virtual ::arrow::Result<std::shared_ptr<::arrow::Table>> ReadRowGroup(int i) = 0;
275
+
276
+ /// \brief Read the given row groups columns into a Table
277
+ virtual ::arrow::Result<std::shared_ptr<::arrow::Table>> ReadRowGroups(
278
+ const std::vector<int>& row_groups, const std::vector<int>& column_indices) = 0;
279
+
280
+ /// \brief Read the given row groups into a Table
281
+ virtual ::arrow::Result<std::shared_ptr<::arrow::Table>> ReadRowGroups(
282
+ const std::vector<int>& row_groups) = 0;
283
+
284
+ /// \deprecated Deprecated in 24.0.0. Use arrow::Result version instead.
285
+ ARROW_DEPRECATED("Deprecated in 24.0.0. Use arrow::Result version instead.")
286
+ ::arrow::Status ReadRowGroup(int i, const std::vector<int>& column_indices,
287
+ std::shared_ptr<::arrow::Table>* out);
288
+
289
+ /// \deprecated Deprecated in 24.0.0. Use arrow::Result version instead.
290
+ ARROW_DEPRECATED("Deprecated in 24.0.0. Use arrow::Result version instead.")
291
+ ::arrow::Status ReadRowGroup(int i, std::shared_ptr<::arrow::Table>* out);
292
+
293
+ /// \deprecated Deprecated in 24.0.0. Use arrow::Result version instead.
294
+ ARROW_DEPRECATED("Deprecated in 24.0.0. Use arrow::Result version instead.")
295
+ ::arrow::Status ReadRowGroups(const std::vector<int>& row_groups,
296
+ const std::vector<int>& column_indices,
297
+ std::shared_ptr<::arrow::Table>* out);
298
+
299
+ /// \deprecated Deprecated in 24.0.0. Use arrow::Result version instead.
300
+ ARROW_DEPRECATED("Deprecated in 24.0.0. Use arrow::Result version instead.")
301
+ ::arrow::Status ReadRowGroups(const std::vector<int>& row_groups,
302
+ std::shared_ptr<::arrow::Table>* out);
303
+
304
+ /// \brief Scan file contents with one thread, return number of rows
305
+ virtual ::arrow::Status ScanContents(std::vector<int> columns,
306
+ const int32_t column_batch_size,
307
+ int64_t* num_rows) = 0;
308
+
309
+ /// \brief Return a reader for the RowGroup, this object must not outlive the
310
+ /// FileReader.
311
+ virtual std::shared_ptr<RowGroupReader> RowGroup(int row_group_index) = 0;
312
+
313
+ /// \brief The number of row groups in the file
314
+ virtual int num_row_groups() const = 0;
315
+
316
+ virtual ParquetFileReader* parquet_reader() const = 0;
317
+
318
+ /// Set whether to use multiple threads during reads of multiple columns.
319
+ /// By default only one thread is used.
320
+ virtual void set_use_threads(bool use_threads) = 0;
321
+
322
+ /// Set number of records to read per batch for the RecordBatchReader.
323
+ virtual void set_batch_size(int64_t batch_size) = 0;
324
+
325
+ virtual const ArrowReaderProperties& properties() const = 0;
326
+
327
+ virtual const SchemaManifest& manifest() const = 0;
328
+
329
+ virtual ~FileReader() = default;
330
+ };
331
+
332
+ class RowGroupReader {
333
+ public:
334
+ virtual ~RowGroupReader() = default;
335
+ virtual std::shared_ptr<ColumnChunkReader> Column(int column_index) = 0;
336
+ virtual ::arrow::Status ReadTable(const std::vector<int>& column_indices,
337
+ std::shared_ptr<::arrow::Table>* out) = 0;
338
+ virtual ::arrow::Status ReadTable(std::shared_ptr<::arrow::Table>* out) = 0;
339
+
340
+ private:
341
+ struct Iterator;
342
+ };
343
+
344
+ class ColumnChunkReader {
345
+ public:
346
+ virtual ~ColumnChunkReader() = default;
347
+ virtual ::arrow::Status Read(std::shared_ptr<::arrow::ChunkedArray>* out) = 0;
348
+ };
349
+
350
+ // At this point, the column reader is a stream iterator. It only knows how to
351
+ // read the next batch of values for a particular column from the file until it
352
+ // runs out.
353
+ //
354
+ // We also do not expose any internal Parquet details, such as row groups. This
355
+ // might change in the future.
356
+ class PARQUET_EXPORT ColumnReader {
357
+ public:
358
+ virtual ~ColumnReader() = default;
359
+
360
+ // Scan the next array of the indicated size. The actual size of the
361
+ // returned array may be less than the passed size depending how much data is
362
+ // available in the file.
363
+ //
364
+ // When all the data in the file has been exhausted, the result is set to
365
+ // nullptr.
366
+ //
367
+ // Returns Status::OK on a successful read, including if you have exhausted
368
+ // the data available in the file.
369
+ virtual ::arrow::Status NextBatch(int64_t batch_size,
370
+ std::shared_ptr<::arrow::ChunkedArray>* out) = 0;
371
+ };
372
+
373
+ /// \brief Experimental helper class for bindings (like Python) that struggle
374
+ /// either with std::move or C++ exceptions
375
+ class PARQUET_EXPORT FileReaderBuilder {
376
+ public:
377
+ FileReaderBuilder();
378
+
379
+ /// Create FileReaderBuilder from Arrow file and optional properties / metadata
380
+ ::arrow::Status Open(std::shared_ptr<::arrow::io::RandomAccessFile> file,
381
+ const ReaderProperties& properties = default_reader_properties(),
382
+ std::shared_ptr<FileMetaData> metadata = NULLPTR);
383
+
384
+ /// Create FileReaderBuilder from file path and optional properties / metadata
385
+ ::arrow::Status OpenFile(const std::string& path, bool memory_map = false,
386
+ const ReaderProperties& props = default_reader_properties(),
387
+ std::shared_ptr<FileMetaData> metadata = NULLPTR);
388
+
389
+ ParquetFileReader* raw_reader() { return raw_reader_.get(); }
390
+
391
+ /// Set Arrow MemoryPool for memory allocation
392
+ FileReaderBuilder* memory_pool(::arrow::MemoryPool* pool);
393
+ /// Set Arrow reader properties
394
+ FileReaderBuilder* properties(const ArrowReaderProperties& arg_properties);
395
+ /// Build FileReader instance
396
+ ::arrow::Status Build(std::unique_ptr<FileReader>* out);
397
+ ::arrow::Result<std::unique_ptr<FileReader>> Build();
398
+
399
+ private:
400
+ ::arrow::MemoryPool* pool_;
401
+ ArrowReaderProperties properties_;
402
+ std::unique_ptr<ParquetFileReader> raw_reader_;
403
+ };
404
+
405
+ /// \defgroup parquet-arrow-reader-factories Factory functions for Parquet Arrow readers
406
+ ///
407
+ /// @{
408
+
409
+ /// \brief Build FileReader from Arrow file and MemoryPool
410
+ ///
411
+ /// Advanced settings are supported through the FileReaderBuilder class.
412
+ PARQUET_EXPORT
413
+ ::arrow::Result<std::unique_ptr<FileReader>> OpenFile(
414
+ std::shared_ptr<::arrow::io::RandomAccessFile>, ::arrow::MemoryPool* allocator);
415
+
416
+ /// @}
417
+
418
+ PARQUET_EXPORT
419
+ ::arrow::Status StatisticsAsScalars(const Statistics& Statistics,
420
+ std::shared_ptr<::arrow::Scalar>* min,
421
+ std::shared_ptr<::arrow::Scalar>* max);
422
+
423
+ } // namespace arrow
424
+ } // namespace parquet
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/schema.h ADDED
@@ -0,0 +1,184 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <cassert>
21
+ #include <memory>
22
+ #include <unordered_map>
23
+ #include <unordered_set>
24
+ #include <vector>
25
+
26
+ #include "arrow/result.h"
27
+ #include "arrow/status.h"
28
+ #include "arrow/type.h"
29
+ #include "arrow/type_fwd.h"
30
+
31
+ #include "parquet/level_conversion.h"
32
+ #include "parquet/platform.h"
33
+ #include "parquet/schema.h"
34
+
35
+ namespace parquet {
36
+
37
+ class ArrowReaderProperties;
38
+ class ArrowWriterProperties;
39
+ class WriterProperties;
40
+
41
+ namespace arrow {
42
+
43
+ /// \defgroup arrow-to-parquet-schema-conversion Functions to convert an Arrow
44
+ /// schema into a Parquet schema.
45
+ ///
46
+ /// @{
47
+
48
+ PARQUET_EXPORT
49
+ ::arrow::Status FieldToNode(const std::shared_ptr<::arrow::Field>& field,
50
+ const WriterProperties& properties,
51
+ const ArrowWriterProperties& arrow_properties,
52
+ schema::NodePtr* out);
53
+
54
+ PARQUET_EXPORT
55
+ ::arrow::Status ToParquetSchema(const ::arrow::Schema* arrow_schema,
56
+ const WriterProperties& properties,
57
+ const ArrowWriterProperties& arrow_properties,
58
+ std::shared_ptr<SchemaDescriptor>* out);
59
+
60
+ PARQUET_EXPORT
61
+ ::arrow::Status ToParquetSchema(const ::arrow::Schema* arrow_schema,
62
+ const WriterProperties& properties,
63
+ std::shared_ptr<SchemaDescriptor>* out);
64
+
65
+ /// @}
66
+
67
+ /// \defgroup parquet-to-arrow-schema-conversion Functions to convert a Parquet
68
+ /// schema into an Arrow schema.
69
+ ///
70
+ /// @{
71
+
72
+ PARQUET_EXPORT
73
+ ::arrow::Status FromParquetSchema(
74
+ const SchemaDescriptor* parquet_schema, const ArrowReaderProperties& properties,
75
+ const std::shared_ptr<const ::arrow::KeyValueMetadata>& key_value_metadata,
76
+ std::shared_ptr<::arrow::Schema>* out);
77
+
78
+ PARQUET_EXPORT
79
+ ::arrow::Status FromParquetSchema(const SchemaDescriptor* parquet_schema,
80
+ const ArrowReaderProperties& properties,
81
+ std::shared_ptr<::arrow::Schema>* out);
82
+
83
+ PARQUET_EXPORT
84
+ ::arrow::Status FromParquetSchema(const SchemaDescriptor* parquet_schema,
85
+ std::shared_ptr<::arrow::Schema>* out);
86
+
87
+ /// @}
88
+
89
+ /// \brief Bridge between an arrow::Field and parquet column indices.
90
+ struct PARQUET_EXPORT SchemaField {
91
+ std::shared_ptr<::arrow::Field> field;
92
+ std::vector<SchemaField> children;
93
+
94
+ // Only set for leaf nodes
95
+ int column_index = -1;
96
+
97
+ parquet::internal::LevelInfo level_info;
98
+
99
+ bool is_leaf() const { return column_index != -1; }
100
+ };
101
+
102
+ /// \brief Bridge between a parquet Schema and an arrow Schema.
103
+ ///
104
+ /// Expose parquet columns as a tree structure. Useful traverse and link
105
+ /// between arrow's Schema and parquet's Schema.
106
+ struct PARQUET_EXPORT SchemaManifest {
107
+ static ::arrow::Status Make(
108
+ const SchemaDescriptor* schema,
109
+ const std::shared_ptr<const ::arrow::KeyValueMetadata>& metadata,
110
+ const ArrowReaderProperties& properties, SchemaManifest* manifest);
111
+
112
+ const SchemaDescriptor* descr;
113
+ std::shared_ptr<::arrow::Schema> origin_schema;
114
+ std::shared_ptr<const ::arrow::KeyValueMetadata> schema_metadata;
115
+ std::vector<SchemaField> schema_fields;
116
+
117
+ std::unordered_map<int, const SchemaField*> column_index_to_field;
118
+ std::unordered_map<const SchemaField*, const SchemaField*> child_to_parent;
119
+
120
+ ::arrow::Status GetColumnField(int column_index, const SchemaField** out) const {
121
+ auto it = column_index_to_field.find(column_index);
122
+ if (it == column_index_to_field.end()) {
123
+ return ::arrow::Status::KeyError("Column index ", column_index,
124
+ " not found in schema manifest, may be malformed");
125
+ }
126
+ *out = it->second;
127
+ return ::arrow::Status::OK();
128
+ }
129
+
130
+ const SchemaField* GetParent(const SchemaField* field) const {
131
+ // Returns nullptr also if not found
132
+ auto it = child_to_parent.find(field);
133
+ if (it == child_to_parent.end()) {
134
+ return NULLPTR;
135
+ }
136
+ return it->second;
137
+ }
138
+
139
+ /// Coalesce a list of field indices (relative to the equivalent arrow::Schema) which
140
+ /// correspond to the column root (first node below the parquet schema's root group) of
141
+ /// each leaf referenced in column_indices.
142
+ ///
143
+ /// For example, for leaves `a.b.c`, `a.b.d.e`, and `i.j.k` (column_indices=[0,1,3])
144
+ /// the roots are `a` and `i` (return=[0,2]).
145
+ ///
146
+ /// root
147
+ /// -- a <------
148
+ /// -- -- b | |
149
+ /// -- -- -- c |
150
+ /// -- -- -- d |
151
+ /// -- -- -- -- e
152
+ /// -- f
153
+ /// -- -- g
154
+ /// -- -- -- h
155
+ /// -- i <---
156
+ /// -- -- j |
157
+ /// -- -- -- k
158
+ ::arrow::Result<std::vector<int>> GetFieldIndices(
159
+ const std::vector<int>& column_indices) const {
160
+ const schema::GroupNode* group = descr->group_node();
161
+ std::unordered_set<int> already_added;
162
+
163
+ std::vector<int> out;
164
+ for (int column_idx : column_indices) {
165
+ if (column_idx < 0 || column_idx >= descr->num_columns()) {
166
+ return ::arrow::Status::IndexError("Column index ", column_idx, " is not valid");
167
+ }
168
+
169
+ auto field_node = descr->GetColumnRoot(column_idx);
170
+ auto field_idx = group->FieldIndex(*field_node);
171
+ if (field_idx == -1) {
172
+ return ::arrow::Status::IndexError("Column index ", column_idx, " is not valid");
173
+ }
174
+
175
+ if (already_added.insert(field_idx).second) {
176
+ out.push_back(field_idx);
177
+ }
178
+ }
179
+ return out;
180
+ }
181
+ };
182
+
183
+ } // namespace arrow
184
+ } // namespace parquet
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/test_util.h ADDED
@@ -0,0 +1,487 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <limits>
21
+ #include <memory>
22
+ #include <random>
23
+ #include <string>
24
+ #include <utility>
25
+ #include <vector>
26
+
27
+ #include "arrow/array.h"
28
+ #include "arrow/array/builder_binary.h"
29
+ #include "arrow/array/builder_decimal.h"
30
+ #include "arrow/array/builder_primitive.h"
31
+ #include "arrow/testing/gtest_util.h"
32
+ #include "arrow/testing/random.h"
33
+ #include "arrow/type_fwd.h"
34
+ #include "arrow/type_traits.h"
35
+ #include "arrow/util/decimal.h"
36
+ #include "arrow/util/float16.h"
37
+ #include "parquet/column_reader.h"
38
+ #include "parquet/test_util.h"
39
+
40
+ namespace parquet {
41
+
42
+ using internal::RecordReader;
43
+
44
+ namespace arrow {
45
+
46
+ using ::arrow::Array;
47
+ using ::arrow::ChunkedArray;
48
+ using ::arrow::Status;
49
+
50
+ template <typename T, int32_t PRECISION, typename = ::arrow::enable_if_decimal<T>>
51
+ struct DecimalWithPrecisionAndScale {
52
+ using type = T;
53
+ static_assert(PRECISION >= T::kMinPrecision && PRECISION <= T::kMaxPrecision,
54
+ "Invalid precision value");
55
+ static constexpr ::arrow::Type::type type_id = T::type_id;
56
+ static constexpr int32_t precision = PRECISION;
57
+ static constexpr int32_t scale = PRECISION - 1;
58
+ };
59
+ template <int32_t PRECISION>
60
+ using Decimal32WithPrecisionAndScale =
61
+ DecimalWithPrecisionAndScale<::arrow::Decimal32Type, PRECISION>;
62
+ template <int32_t PRECISION>
63
+ using Decimal64WithPrecisionAndScale =
64
+ DecimalWithPrecisionAndScale<::arrow::Decimal64Type, PRECISION>;
65
+ template <int32_t PRECISION>
66
+ using Decimal128WithPrecisionAndScale =
67
+ DecimalWithPrecisionAndScale<::arrow::Decimal128Type, PRECISION>;
68
+ template <int32_t PRECISION>
69
+ using Decimal256WithPrecisionAndScale =
70
+ DecimalWithPrecisionAndScale<::arrow::Decimal256Type, PRECISION>;
71
+
72
+ template <class ArrowType>
73
+ ::arrow::enable_if_floating_point<ArrowType, Status> NonNullArray(
74
+ size_t size, std::shared_ptr<Array>* out) {
75
+ using c_type = typename ArrowType::c_type;
76
+ std::vector<c_type> values;
77
+ if constexpr (::arrow::is_half_float_type<ArrowType>::value) {
78
+ values.resize(size);
79
+ test::random_float16_numbers(static_cast<int>(size), 0, ::arrow::util::Float16(0.0f),
80
+ ::arrow::util::Float16(1.0f), values.data());
81
+ } else {
82
+ ::arrow::random_real(size, 0, static_cast<c_type>(0), static_cast<c_type>(1),
83
+ &values);
84
+ }
85
+ ::arrow::NumericBuilder<ArrowType> builder;
86
+ RETURN_NOT_OK(builder.AppendValues(values.data(), values.size()));
87
+ return builder.Finish(out);
88
+ }
89
+
90
+ template <class ArrowType>
91
+ ::arrow::enable_if_integer<ArrowType, Status> NonNullArray(size_t size,
92
+ std::shared_ptr<Array>* out) {
93
+ std::vector<typename ArrowType::c_type> values;
94
+ ::arrow::randint(size, 0, 64, &values);
95
+
96
+ // Passing data type so this will work with TimestampType too
97
+ ::arrow::NumericBuilder<ArrowType> builder(std::make_shared<ArrowType>(),
98
+ ::arrow::default_memory_pool());
99
+ RETURN_NOT_OK(builder.AppendValues(values.data(), values.size()));
100
+ return builder.Finish(out);
101
+ }
102
+
103
+ template <class ArrowType>
104
+ ::arrow::enable_if_date<ArrowType, Status> NonNullArray(size_t size,
105
+ std::shared_ptr<Array>* out) {
106
+ std::vector<typename ArrowType::c_type> values;
107
+ ::arrow::randint(size, 0, 24, &values);
108
+ for (size_t i = 0; i < size; i++) {
109
+ values[i] *= 86400000;
110
+ }
111
+
112
+ // Passing data type so this will work with TimestampType too
113
+ ::arrow::NumericBuilder<ArrowType> builder(std::make_shared<ArrowType>(),
114
+ ::arrow::default_memory_pool());
115
+ RETURN_NOT_OK(builder.AppendValues(values.data(), values.size()));
116
+ return builder.Finish(out);
117
+ }
118
+
119
+ template <class ArrowType>
120
+ ::arrow::enable_if_base_binary<ArrowType, Status> NonNullArray(
121
+ size_t size, std::shared_ptr<Array>* out) {
122
+ using BuilderType = typename ::arrow::TypeTraits<ArrowType>::BuilderType;
123
+ BuilderType builder;
124
+ for (size_t i = 0; i < size; i++) {
125
+ RETURN_NOT_OK(builder.Append("test-string"));
126
+ }
127
+ return builder.Finish(out);
128
+ }
129
+
130
+ template <typename ArrowType>
131
+ ::arrow::enable_if_fixed_size_binary<ArrowType, Status> NonNullArray(
132
+ size_t size, std::shared_ptr<Array>* out) {
133
+ using BuilderType = typename ::arrow::TypeTraits<ArrowType>::BuilderType;
134
+ // set byte_width to the length of "fixed": 5
135
+ // todo: find a way to generate test data with more diversity.
136
+ BuilderType builder(::arrow::fixed_size_binary(5));
137
+ for (size_t i = 0; i < size; i++) {
138
+ RETURN_NOT_OK(builder.Append("fixed"));
139
+ }
140
+ return builder.Finish(out);
141
+ }
142
+
143
+ template <int32_t byte_width>
144
+ static void random_decimals(int64_t n, uint32_t seed, int32_t precision, uint8_t* out) {
145
+ auto gen = ::arrow::random::RandomArrayGenerator(seed);
146
+ std::shared_ptr<Array> decimals;
147
+ if constexpr (byte_width == 4) {
148
+ decimals = gen.Decimal32(::arrow::decimal32(precision, 0), n);
149
+ } else if constexpr (byte_width == 8) {
150
+ decimals = gen.Decimal64(::arrow::decimal64(precision, 0), n);
151
+ } else if constexpr (byte_width == 16) {
152
+ decimals = gen.Decimal128(::arrow::decimal128(precision, 0), n);
153
+ } else {
154
+ decimals = gen.Decimal256(::arrow::decimal256(precision, 0), n);
155
+ }
156
+ std::memcpy(out, decimals->data()->GetValues<uint8_t>(1, 0), byte_width * n);
157
+ }
158
+
159
+ template <typename ArrowType, int32_t precision = ArrowType::precision>
160
+ ::arrow::enable_if_t<std::is_same_v<ArrowType, DecimalWithPrecisionAndScale<
161
+ typename ArrowType::type, precision>>,
162
+ Status>
163
+ NonNullArray(size_t size, std::shared_ptr<Array>* out) {
164
+ constexpr int32_t kDecimalPrecision = precision;
165
+ constexpr int32_t kDecimalScale = ArrowType::scale;
166
+
167
+ const auto type =
168
+ std::make_shared<typename ArrowType::type>(kDecimalPrecision, kDecimalScale);
169
+ const int32_t byte_width = type->byte_width();
170
+
171
+ constexpr int32_t seed = 0;
172
+
173
+ ARROW_ASSIGN_OR_RAISE(auto out_buf, ::arrow::AllocateBuffer(size * byte_width));
174
+ random_decimals<ArrowType::type::kByteWidth>(size, seed, kDecimalPrecision,
175
+ out_buf->mutable_data());
176
+
177
+ using Builder = typename ::arrow::TypeTraits<typename ArrowType::type>::BuilderType;
178
+ Builder builder(type);
179
+ RETURN_NOT_OK(builder.AppendValues(out_buf->data(), size));
180
+ return builder.Finish(out);
181
+ }
182
+
183
+ template <class ArrowType>
184
+ ::arrow::enable_if_boolean<ArrowType, Status> NonNullArray(size_t size,
185
+ std::shared_ptr<Array>* out) {
186
+ std::vector<uint8_t> values;
187
+ ::arrow::randint(size, 0, 1, &values);
188
+ ::arrow::BooleanBuilder builder;
189
+ RETURN_NOT_OK(builder.AppendValues(values.data(), values.size()));
190
+ return builder.Finish(out);
191
+ }
192
+
193
+ // This helper function only supports (size/2) nulls.
194
+ template <typename ArrowType>
195
+ ::arrow::enable_if_floating_point<ArrowType, Status> NullableArray(
196
+ size_t size, size_t num_nulls, uint32_t seed, std::shared_ptr<Array>* out) {
197
+ using c_type = typename ArrowType::c_type;
198
+ std::vector<c_type> values;
199
+ if constexpr (::arrow::is_half_float_type<ArrowType>::value) {
200
+ values.resize(size);
201
+ test::random_float16_numbers(static_cast<int>(size), 0, ::arrow::util::Float16(-1e4f),
202
+ ::arrow::util::Float16(1e4f), values.data());
203
+ } else {
204
+ ::arrow::random_real(size, seed, static_cast<c_type>(-1e10),
205
+ static_cast<c_type>(1e10), &values);
206
+ }
207
+ std::vector<uint8_t> valid_bytes(size, 1);
208
+
209
+ for (size_t i = 0; i < num_nulls; i++) {
210
+ valid_bytes[i * 2] = 0;
211
+ }
212
+
213
+ ::arrow::NumericBuilder<ArrowType> builder;
214
+ if (values.size() > 0) {
215
+ RETURN_NOT_OK(builder.AppendValues(values.data(), values.size(), valid_bytes.data()));
216
+ }
217
+ return builder.Finish(out);
218
+ }
219
+
220
+ // This helper function only supports (size/2) nulls.
221
+ template <typename ArrowType>
222
+ ::arrow::enable_if_integer<ArrowType, Status> NullableArray(size_t size, size_t num_nulls,
223
+ uint32_t seed,
224
+ std::shared_ptr<Array>* out) {
225
+ std::vector<typename ArrowType::c_type> values;
226
+
227
+ // Seed is random in Arrow right now
228
+ (void)seed;
229
+ ::arrow::randint(size, 0, 64, &values);
230
+ std::vector<uint8_t> valid_bytes(size, 1);
231
+
232
+ for (size_t i = 0; i < num_nulls; i++) {
233
+ valid_bytes[i * 2] = 0;
234
+ }
235
+
236
+ // Passing data type so this will work with TimestampType too
237
+ ::arrow::NumericBuilder<ArrowType> builder(std::make_shared<ArrowType>(),
238
+ ::arrow::default_memory_pool());
239
+ RETURN_NOT_OK(builder.AppendValues(values.data(), values.size(), valid_bytes.data()));
240
+ return builder.Finish(out);
241
+ }
242
+
243
+ template <typename ArrowType>
244
+ ::arrow::enable_if_date<ArrowType, Status> NullableArray(size_t size, size_t num_nulls,
245
+ uint32_t seed,
246
+ std::shared_ptr<Array>* out) {
247
+ std::vector<typename ArrowType::c_type> values;
248
+
249
+ // Seed is random in Arrow right now
250
+ (void)seed;
251
+ ::arrow::randint(size, 0, 24, &values);
252
+ for (size_t i = 0; i < size; i++) {
253
+ values[i] *= 86400000;
254
+ }
255
+ std::vector<uint8_t> valid_bytes(size, 1);
256
+
257
+ for (size_t i = 0; i < num_nulls; i++) {
258
+ valid_bytes[i * 2] = 0;
259
+ }
260
+
261
+ // Passing data type so this will work with TimestampType too
262
+ ::arrow::NumericBuilder<ArrowType> builder(std::make_shared<ArrowType>(),
263
+ ::arrow::default_memory_pool());
264
+ RETURN_NOT_OK(builder.AppendValues(values.data(), values.size(), valid_bytes.data()));
265
+ return builder.Finish(out);
266
+ }
267
+
268
+ // This helper function only supports (size/2) nulls yet.
269
+ template <typename ArrowType>
270
+ ::arrow::enable_if_base_binary<ArrowType, Status> NullableArray(
271
+ size_t size, size_t num_nulls, uint32_t seed, std::shared_ptr<::arrow::Array>* out) {
272
+ std::vector<uint8_t> valid_bytes(size, 1);
273
+
274
+ for (size_t i = 0; i < num_nulls; i++) {
275
+ valid_bytes[i * 2] = 0;
276
+ }
277
+
278
+ using BuilderType = typename ::arrow::TypeTraits<ArrowType>::BuilderType;
279
+ BuilderType builder;
280
+
281
+ const int kBufferSize = 10;
282
+ uint8_t buffer[kBufferSize];
283
+ for (size_t i = 0; i < size; i++) {
284
+ if (!valid_bytes[i]) {
285
+ RETURN_NOT_OK(builder.AppendNull());
286
+ } else {
287
+ ::arrow::random_bytes(kBufferSize, seed + static_cast<uint32_t>(i), buffer);
288
+ if (ArrowType::is_utf8) {
289
+ // Trivially force data to be valid UTF8 by making it all ASCII
290
+ for (auto& byte : buffer) {
291
+ byte &= 0x7f;
292
+ }
293
+ }
294
+ RETURN_NOT_OK(builder.Append(buffer, kBufferSize));
295
+ }
296
+ }
297
+ return builder.Finish(out);
298
+ }
299
+
300
+ // This helper function only supports (size/2) nulls yet,
301
+ // same as NullableArray<String|Binary>(..)
302
+ template <typename ArrowType>
303
+ ::arrow::enable_if_fixed_size_binary<ArrowType, Status> NullableArray(
304
+ size_t size, size_t num_nulls, uint32_t seed, std::shared_ptr<::arrow::Array>* out) {
305
+ std::vector<uint8_t> valid_bytes(size, 1);
306
+
307
+ for (size_t i = 0; i < num_nulls; i++) {
308
+ valid_bytes[i * 2] = 0;
309
+ }
310
+
311
+ using BuilderType = typename ::arrow::TypeTraits<ArrowType>::BuilderType;
312
+ const int byte_width = 10;
313
+ BuilderType builder(::arrow::fixed_size_binary(byte_width));
314
+
315
+ const int kBufferSize = byte_width;
316
+ uint8_t buffer[kBufferSize];
317
+ for (size_t i = 0; i < size; i++) {
318
+ if (!valid_bytes[i]) {
319
+ RETURN_NOT_OK(builder.AppendNull());
320
+ } else {
321
+ ::arrow::random_bytes(kBufferSize, seed + static_cast<uint32_t>(i), buffer);
322
+ RETURN_NOT_OK(builder.Append(buffer));
323
+ }
324
+ }
325
+ return builder.Finish(out);
326
+ }
327
+
328
+ template <typename ArrowType, int32_t precision = ArrowType::precision>
329
+ ::arrow::enable_if_t<std::is_same_v<ArrowType, DecimalWithPrecisionAndScale<
330
+ typename ArrowType::type, precision>>,
331
+ Status>
332
+ NullableArray(size_t size, size_t num_nulls, uint32_t seed,
333
+ std::shared_ptr<::arrow::Array>* out) {
334
+ std::vector<uint8_t> valid_bytes(size, '\1');
335
+
336
+ for (size_t i = 0; i < num_nulls; ++i) {
337
+ valid_bytes[i * 2] = '\0';
338
+ }
339
+
340
+ constexpr int32_t kDecimalPrecision = precision;
341
+ constexpr int32_t kDecimalScale = ArrowType::scale;
342
+
343
+ const auto type =
344
+ std::make_shared<typename ArrowType::type>(kDecimalPrecision, kDecimalScale);
345
+ const int32_t byte_width = type->byte_width();
346
+
347
+ ARROW_ASSIGN_OR_RAISE(auto out_buf, ::arrow::AllocateBuffer(size * byte_width));
348
+ random_decimals<ArrowType::type::kByteWidth>(size, seed, precision,
349
+ out_buf->mutable_data());
350
+
351
+ using Builder = typename ::arrow::TypeTraits<typename ArrowType::type>::BuilderType;
352
+ Builder builder(type);
353
+ RETURN_NOT_OK(builder.AppendValues(out_buf->data(), size, valid_bytes.data()));
354
+ return builder.Finish(out);
355
+ }
356
+
357
+ // This helper function only supports (size/2) nulls yet.
358
+ template <class ArrowType>
359
+ ::arrow::enable_if_boolean<ArrowType, Status> NullableArray(size_t size, size_t num_nulls,
360
+ uint32_t seed,
361
+ std::shared_ptr<Array>* out) {
362
+ std::vector<uint8_t> values;
363
+
364
+ // Seed is random in Arrow right now
365
+ (void)seed;
366
+
367
+ ::arrow::randint(size, 0, 1, &values);
368
+ std::vector<uint8_t> valid_bytes(size, 1);
369
+
370
+ for (size_t i = 0; i < num_nulls; i++) {
371
+ valid_bytes[i * 2] = 0;
372
+ }
373
+
374
+ ::arrow::BooleanBuilder builder;
375
+ RETURN_NOT_OK(builder.AppendValues(values.data(), values.size(), valid_bytes.data()));
376
+ return builder.Finish(out);
377
+ }
378
+
379
+ /// Wrap an Array into a ListArray by splitting it up into size lists.
380
+ ///
381
+ /// This helper function only supports (size/2) nulls.
382
+ Status MakeListArray(const std::shared_ptr<Array>& values, int64_t size,
383
+ int64_t null_count, const std::string& item_name,
384
+ bool nullable_values, std::shared_ptr<::arrow::ListArray>* out) {
385
+ // We always include an empty list
386
+ int64_t non_null_entries = size - null_count - 1;
387
+ int64_t length_per_entry = values->length() / non_null_entries;
388
+
389
+ auto offsets = AllocateBuffer();
390
+ RETURN_NOT_OK(offsets->Resize((size + 1) * sizeof(int32_t)));
391
+ int32_t* offsets_ptr = reinterpret_cast<int32_t*>(offsets->mutable_data());
392
+
393
+ auto null_bitmap = AllocateBuffer();
394
+ int64_t bitmap_size = ::arrow::bit_util::BytesForBits(size);
395
+ RETURN_NOT_OK(null_bitmap->Resize(bitmap_size));
396
+ uint8_t* null_bitmap_ptr = null_bitmap->mutable_data();
397
+ memset(null_bitmap_ptr, 0, bitmap_size);
398
+
399
+ int32_t current_offset = 0;
400
+ for (int64_t i = 0; i < size; i++) {
401
+ offsets_ptr[i] = current_offset;
402
+ if (!(((i % 2) == 0) && ((i / 2) < null_count))) {
403
+ // Non-null list (list with index 1 is always empty).
404
+ ::arrow::bit_util::SetBit(null_bitmap_ptr, i);
405
+ if (i != 1) {
406
+ current_offset += static_cast<int32_t>(length_per_entry);
407
+ }
408
+ }
409
+ }
410
+ offsets_ptr[size] = static_cast<int32_t>(values->length());
411
+
412
+ auto value_field = ::arrow::field(item_name, values->type(), nullable_values);
413
+ *out = std::make_shared<::arrow::ListArray>(::arrow::list(value_field), size, offsets,
414
+ values, null_bitmap, null_count);
415
+
416
+ return Status::OK();
417
+ }
418
+
419
+ // Make an array containing only empty lists, with a null values array
420
+ Status MakeEmptyListsArray(int64_t size, std::shared_ptr<Array>* out_array) {
421
+ // Allocate an offsets buffer containing only zeroes
422
+ const int64_t offsets_nbytes = (size + 1) * sizeof(int32_t);
423
+ ARROW_ASSIGN_OR_RAISE(auto offsets_buffer, ::arrow::AllocateBuffer(offsets_nbytes));
424
+ memset(offsets_buffer->mutable_data(), 0, offsets_nbytes);
425
+
426
+ auto value_field =
427
+ ::arrow::field("item", ::arrow::float64(), false /* nullable_values */);
428
+ auto list_type = ::arrow::list(value_field);
429
+
430
+ std::vector<std::shared_ptr<Buffer>> child_buffers = {nullptr /* null bitmap */,
431
+ nullptr /* values */};
432
+ auto child_data =
433
+ ::arrow::ArrayData::Make(value_field->type(), 0, std::move(child_buffers));
434
+
435
+ std::vector<std::shared_ptr<Buffer>> buffers = {nullptr /* bitmap */,
436
+ std::move(offsets_buffer)};
437
+ auto array_data = ::arrow::ArrayData::Make(list_type, size, std::move(buffers));
438
+ array_data->child_data.push_back(child_data);
439
+
440
+ *out_array = ::arrow::MakeArray(array_data);
441
+ return Status::OK();
442
+ }
443
+
444
+ std::shared_ptr<::arrow::Table> MakeSimpleTable(
445
+ const std::shared_ptr<ChunkedArray>& values, bool nullable) {
446
+ auto schema = ::arrow::schema({::arrow::field("col", values->type(), nullable)});
447
+ return ::arrow::Table::Make(schema, {values});
448
+ }
449
+
450
+ std::shared_ptr<::arrow::Table> MakeSimpleTable(const std::shared_ptr<Array>& values,
451
+ bool nullable) {
452
+ auto carr = std::make_shared<::arrow::ChunkedArray>(values);
453
+ return MakeSimpleTable(carr, nullable);
454
+ }
455
+
456
+ template <typename T>
457
+ void ExpectArray(T* expected, Array* result) {
458
+ auto p_array = static_cast<::arrow::PrimitiveArray*>(result);
459
+ for (int i = 0; i < result->length(); i++) {
460
+ EXPECT_EQ(expected[i], reinterpret_cast<const T*>(p_array->values()->data())[i]);
461
+ }
462
+ }
463
+
464
+ template <typename ArrowType>
465
+ void ExpectArrayT(void* expected, Array* result) {
466
+ ::arrow::PrimitiveArray* p_array = static_cast<::arrow::PrimitiveArray*>(result);
467
+ for (int64_t i = 0; i < result->length(); i++) {
468
+ EXPECT_EQ(reinterpret_cast<typename ArrowType::c_type*>(expected)[i],
469
+ reinterpret_cast<const typename ArrowType::c_type*>(
470
+ p_array->values()->data())[i]);
471
+ }
472
+ }
473
+
474
+ template <>
475
+ void ExpectArrayT<::arrow::BooleanType>(void* expected, Array* result) {
476
+ ::arrow::BooleanBuilder builder;
477
+ ARROW_EXPECT_OK(
478
+ builder.AppendValues(reinterpret_cast<uint8_t*>(expected), result->length()));
479
+
480
+ std::shared_ptr<Array> expected_array;
481
+ ARROW_EXPECT_OK(builder.Finish(&expected_array));
482
+ EXPECT_TRUE(result->Equals(*expected_array));
483
+ }
484
+
485
+ } // namespace arrow
486
+
487
+ } // namespace parquet
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/arrow/writer.h ADDED
@@ -0,0 +1,176 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <cstdint>
21
+ #include <memory>
22
+
23
+ #include "parquet/platform.h"
24
+ #include "parquet/properties.h"
25
+
26
+ namespace arrow {
27
+
28
+ class Array;
29
+ class ChunkedArray;
30
+ class RecordBatch;
31
+ class Schema;
32
+ class Table;
33
+
34
+ } // namespace arrow
35
+
36
+ namespace parquet {
37
+
38
+ class FileMetaData;
39
+ class ParquetFileWriter;
40
+
41
+ namespace arrow {
42
+
43
+ /// \brief Iterative FileWriter class
44
+ ///
45
+ /// For basic usage, can write a Table at a time, creating one or more row
46
+ /// groups per write call.
47
+ ///
48
+ /// For advanced usage, can write column-by-column: Start a new RowGroup or
49
+ /// Chunk with NewRowGroup, then write column-by-column the whole column chunk.
50
+ ///
51
+ /// If PARQUET:field_id is present as a metadata key on a field, and the corresponding
52
+ /// value is a nonnegative integer, then it will be used as the field_id in the parquet
53
+ /// file.
54
+ class PARQUET_EXPORT FileWriter {
55
+ public:
56
+ static ::arrow::Status Make(MemoryPool* pool, std::unique_ptr<ParquetFileWriter> writer,
57
+ std::shared_ptr<::arrow::Schema> schema,
58
+ std::shared_ptr<ArrowWriterProperties> arrow_properties,
59
+ std::unique_ptr<FileWriter>* out);
60
+
61
+ /// \brief Try to create an Arrow to Parquet file writer.
62
+ ///
63
+ /// \param schema schema of data that will be passed.
64
+ /// \param pool memory pool to use.
65
+ /// \param sink output stream to write Parquet data.
66
+ /// \param properties general Parquet writer properties.
67
+ /// \param arrow_properties Arrow-specific writer properties.
68
+ ///
69
+ /// \since 11.0.0
70
+ static ::arrow::Result<std::unique_ptr<FileWriter>> Open(
71
+ const ::arrow::Schema& schema, MemoryPool* pool,
72
+ std::shared_ptr<::arrow::io::OutputStream> sink,
73
+ std::shared_ptr<WriterProperties> properties = default_writer_properties(),
74
+ std::shared_ptr<ArrowWriterProperties> arrow_properties =
75
+ default_arrow_writer_properties());
76
+
77
+ /// Return the Arrow schema to be written to.
78
+ virtual std::shared_ptr<::arrow::Schema> schema() const = 0;
79
+
80
+ /// \brief Write a Table to Parquet.
81
+ ///
82
+ /// \param table Arrow table to write.
83
+ /// \param chunk_size maximum number of rows to write per row group.
84
+ virtual ::arrow::Status WriteTable(
85
+ const ::arrow::Table& table, int64_t chunk_size = DEFAULT_MAX_ROW_GROUP_LENGTH) = 0;
86
+
87
+ /// \brief Start a new row group.
88
+ ///
89
+ /// Returns an error if not all columns have been written.
90
+ virtual ::arrow::Status NewRowGroup() = 0;
91
+
92
+ /// \brief Write ColumnChunk in row group using an array.
93
+ virtual ::arrow::Status WriteColumnChunk(const ::arrow::Array& data) = 0;
94
+
95
+ /// \brief Write ColumnChunk in row group using slice of a ChunkedArray
96
+ virtual ::arrow::Status WriteColumnChunk(
97
+ const std::shared_ptr<::arrow::ChunkedArray>& data, int64_t offset,
98
+ int64_t size) = 0;
99
+
100
+ /// \brief Write ColumnChunk in a row group using a ChunkedArray
101
+ virtual ::arrow::Status WriteColumnChunk(
102
+ const std::shared_ptr<::arrow::ChunkedArray>& data) = 0;
103
+
104
+ /// \brief Start a new buffered row group.
105
+ ///
106
+ /// Returns an error if not all columns have been written.
107
+ virtual ::arrow::Status NewBufferedRowGroup() = 0;
108
+
109
+ /// \brief Write a RecordBatch into the buffered row group.
110
+ ///
111
+ /// Multiple RecordBatches can be written into the same row group
112
+ /// through this method.
113
+ ///
114
+ /// WriterProperties.max_row_group_length() is respected and a new
115
+ /// row group will be created if the current row group exceeds the
116
+ /// limit.
117
+ ///
118
+ /// Batches get flushed to the output stream once NewBufferedRowGroup()
119
+ /// or Close() is called.
120
+ ///
121
+ /// WARNING: If you are writing multiple files in parallel in the same
122
+ /// executor, deadlock may occur if ArrowWriterProperties::use_threads
123
+ /// is set to true to write columns in parallel. Please disable use_threads
124
+ /// option in this case.
125
+ virtual ::arrow::Status WriteRecordBatch(const ::arrow::RecordBatch& batch) = 0;
126
+
127
+ /// \brief Write the footer and close the file.
128
+ virtual ::arrow::Status Close() = 0;
129
+ virtual ~FileWriter();
130
+
131
+ virtual MemoryPool* memory_pool() const = 0;
132
+ /// \brief Add key-value metadata to the file.
133
+ /// \param[in] key_value_metadata the metadata to add.
134
+ /// \note This will overwrite any existing metadata with the same key.
135
+ /// \return Error if Close() has been called.
136
+ ///
137
+ /// WARNING: If `store_schema` is enabled, `ARROW:schema` would be stored
138
+ /// in the key-value metadata. Overwriting this key would result in
139
+ /// `store_schema` being unusable during read.
140
+ virtual ::arrow::Status AddKeyValueMetadata(
141
+ const std::shared_ptr<const ::arrow::KeyValueMetadata>& key_value_metadata) = 0;
142
+ /// \brief Return the file metadata, only available after calling Close().
143
+ virtual const std::shared_ptr<FileMetaData> metadata() const = 0;
144
+ };
145
+
146
+ /// \brief Write Parquet file metadata only to indicated Arrow OutputStream
147
+ PARQUET_EXPORT
148
+ ::arrow::Status WriteFileMetaData(const FileMetaData& file_metadata,
149
+ ::arrow::io::OutputStream* sink);
150
+
151
+ /// \brief Write metadata-only Parquet file to indicated Arrow OutputStream
152
+ PARQUET_EXPORT
153
+ ::arrow::Status WriteMetaDataFile(const FileMetaData& file_metadata,
154
+ ::arrow::io::OutputStream* sink);
155
+
156
+ /// \brief Write a Table to Parquet.
157
+ ///
158
+ /// This writes one table in a single shot. To write a Parquet file with
159
+ /// multiple tables iteratively, see parquet::arrow::FileWriter.
160
+ ///
161
+ /// \param table Table to write.
162
+ /// \param pool memory pool to use.
163
+ /// \param sink output stream to write Parquet data.
164
+ /// \param chunk_size maximum number of rows to write per row group.
165
+ /// \param properties general Parquet writer properties.
166
+ /// \param arrow_properties Arrow-specific writer properties.
167
+ ::arrow::Status PARQUET_EXPORT
168
+ WriteTable(const ::arrow::Table& table, MemoryPool* pool,
169
+ std::shared_ptr<::arrow::io::OutputStream> sink,
170
+ int64_t chunk_size = DEFAULT_MAX_ROW_GROUP_LENGTH,
171
+ std::shared_ptr<WriterProperties> properties = default_writer_properties(),
172
+ std::shared_ptr<ArrowWriterProperties> arrow_properties =
173
+ default_arrow_writer_properties());
174
+
175
+ } // namespace arrow
176
+ } // namespace parquet
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/benchmark_util.h ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <random>
21
+ #include <string>
22
+ #include <vector>
23
+
24
+ #include "parquet/types.h"
25
+
26
+ namespace parquet::benchmarks {
27
+
28
+ template <typename T>
29
+ void GenerateBenchmarkData(uint32_t size, uint32_t seed, T* data,
30
+ std::vector<uint8_t>* heap, uint32_t data_string_length);
31
+
32
+ #define _GENERATE_BENCHMARK_DATA_DECL(KLASS) \
33
+ template <> \
34
+ void GenerateBenchmarkData(uint32_t size, uint32_t seed, KLASS* data, \
35
+ std::vector<uint8_t>* heap, uint32_t data_string_length);
36
+
37
+ _GENERATE_BENCHMARK_DATA_DECL(int32_t)
38
+ _GENERATE_BENCHMARK_DATA_DECL(int64_t)
39
+ _GENERATE_BENCHMARK_DATA_DECL(float)
40
+ _GENERATE_BENCHMARK_DATA_DECL(double)
41
+ _GENERATE_BENCHMARK_DATA_DECL(ByteArray)
42
+ _GENERATE_BENCHMARK_DATA_DECL(FLBA)
43
+ _GENERATE_BENCHMARK_DATA_DECL(Int96)
44
+
45
+ #undef _GENERATE_BENCHMARK_DATA_DECL
46
+
47
+ } // namespace parquet::benchmarks
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/bloom_filter.h ADDED
@@ -0,0 +1,392 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <cmath>
21
+ #include <cstdint>
22
+ #include <memory>
23
+
24
+ #include "arrow/util/bit_util.h"
25
+ #include "arrow/util/logging.h"
26
+ #include "parquet/encryption/type_fwd.h"
27
+ #include "parquet/hasher.h"
28
+ #include "parquet/platform.h"
29
+ #include "parquet/types.h"
30
+
31
+ namespace parquet {
32
+
33
+ // A Bloom filter is a compact structure to indicate whether an item is not in a set or
34
+ // probably in a set. The Bloom filter usually consists of a bit set that represents a
35
+ // set of elements, a hash strategy and a Bloom filter algorithm.
36
+ class PARQUET_EXPORT BloomFilter {
37
+ public:
38
+ // Maximum Bloom filter size, it sets to HDFS default block size 128MB
39
+ // This value will be reconsidered when implementing Bloom filter producer.
40
+ static constexpr uint32_t kMaximumBloomFilterBytes = 128 * 1024 * 1024;
41
+
42
+ /// Determine whether an element exist in set or not.
43
+ ///
44
+ /// @param hash the element to contain.
45
+ /// @return false if value is definitely not in set, and true means PROBABLY
46
+ /// in set.
47
+ virtual bool FindHash(uint64_t hash) const = 0;
48
+
49
+ /// Insert element to set represented by Bloom filter bitset.
50
+ /// @param hash the hash of value to insert into Bloom filter.
51
+ virtual void InsertHash(uint64_t hash) = 0;
52
+
53
+ /// Insert elements to set represented by Bloom filter bitset.
54
+ /// @param hashes the hash values to insert into Bloom filter.
55
+ /// @param num_values the number of hash values to insert.
56
+ virtual void InsertHashes(const uint64_t* hashes, int num_values) = 0;
57
+
58
+ /// Write this Bloom filter to an output stream. A Bloom filter structure should
59
+ /// include bitset length, hash strategy, algorithm, and bitset.
60
+ ///
61
+ /// @param sink the output stream to write
62
+ virtual void WriteTo(ArrowOutputStream* sink) const = 0;
63
+
64
+ /// Get the number of bytes of bitset
65
+ virtual uint32_t GetBitsetSize() const = 0;
66
+
67
+ /// Compute hash for 32 bits value by using its plain encoding result.
68
+ ///
69
+ /// @param value the value to hash.
70
+ /// @return hash result.
71
+ virtual uint64_t Hash(int32_t value) const = 0;
72
+
73
+ /// Compute hash for 64 bits value by using its plain encoding result.
74
+ ///
75
+ /// @param value the value to hash.
76
+ /// @return hash result.
77
+ virtual uint64_t Hash(int64_t value) const = 0;
78
+
79
+ /// Compute hash for float value by using its plain encoding result.
80
+ ///
81
+ /// @param value the value to hash.
82
+ /// @return hash result.
83
+ virtual uint64_t Hash(float value) const = 0;
84
+
85
+ /// Compute hash for double value by using its plain encoding result.
86
+ ///
87
+ /// @param value the value to hash.
88
+ /// @return hash result.
89
+ virtual uint64_t Hash(double value) const = 0;
90
+
91
+ /// Compute hash for Int96 value by using its plain encoding result.
92
+ ///
93
+ /// @param value the value to hash.
94
+ /// @return hash result.
95
+ virtual uint64_t Hash(const Int96* value) const = 0;
96
+ uint64_t Hash(const Int96& value) const { return Hash(&value); }
97
+
98
+ /// Compute hash for ByteArray value by using its plain encoding result.
99
+ ///
100
+ /// @param value the value to hash.
101
+ /// @return hash result.
102
+ virtual uint64_t Hash(const ByteArray* value) const = 0;
103
+ uint64_t Hash(const ByteArray& value) const { return Hash(&value); }
104
+
105
+ /// Compute hash for fixed byte array value by using its plain encoding result.
106
+ ///
107
+ /// @param value the value address.
108
+ /// @param len the value length.
109
+ /// @return hash result.
110
+ virtual uint64_t Hash(const FLBA* value, uint32_t len) const = 0;
111
+ uint64_t Hash(const FLBA& value, uint32_t len) const { return Hash(&value, len); }
112
+
113
+ /// Compute hash for std::string_view value by using its plain encoding result.
114
+ ///
115
+ /// @param value the value to hash.
116
+ /// @return hash result.
117
+ virtual uint64_t Hash(std::string_view value) const = 0;
118
+
119
+ /// Batch compute hashes for 32 bits values by using its plain encoding result.
120
+ ///
121
+ /// @param values values a pointer to the values to hash.
122
+ /// @param num_values the number of values to hash.
123
+ /// @param hashes a pointer to the output hash values, its length should be equal to
124
+ /// num_values.
125
+ virtual void Hashes(const int32_t* values, int num_values, uint64_t* hashes) const = 0;
126
+
127
+ /// Batch compute hashes for 64 bits values by using its plain encoding result.
128
+ ///
129
+ /// @param values values a pointer to the values to hash.
130
+ /// @param num_values the number of values to hash.
131
+ /// @param hashes a pointer to the output hash values, its length should be equal to
132
+ /// num_values.
133
+ virtual void Hashes(const int64_t* values, int num_values, uint64_t* hashes) const = 0;
134
+
135
+ /// Batch compute hashes for float values by using its plain encoding result.
136
+ ///
137
+ /// @param values values a pointer to the values to hash.
138
+ /// @param num_values the number of values to hash.
139
+ /// @param hashes a pointer to the output hash values, its length should be equal to
140
+ /// num_values.
141
+ virtual void Hashes(const float* values, int num_values, uint64_t* hashes) const = 0;
142
+
143
+ /// Batch compute hashes for double values by using its plain encoding result.
144
+ ///
145
+ /// @param values values a pointer to the values to hash.
146
+ /// @param num_values the number of values to hash.
147
+ /// @param hashes a pointer to the output hash values, its length should be equal to
148
+ /// num_values.
149
+ virtual void Hashes(const double* values, int num_values, uint64_t* hashes) const = 0;
150
+
151
+ /// Batch compute hashes for Int96 values by using its plain encoding result.
152
+ ///
153
+ /// @param values values a pointer to the values to hash.
154
+ /// @param num_values the number of values to hash.
155
+ /// @param hashes a pointer to the output hash values, its length should be equal to
156
+ /// num_values.
157
+ virtual void Hashes(const Int96* values, int num_values, uint64_t* hashes) const = 0;
158
+
159
+ /// Batch compute hashes for ByteArray values by using its plain encoding result.
160
+ ///
161
+ /// @param values values a pointer to the values to hash.
162
+ /// @param num_values the number of values to hash.
163
+ /// @param hashes a pointer to the output hash values, its length should be equal to
164
+ /// num_values.
165
+ virtual void Hashes(const ByteArray* values, int num_values,
166
+ uint64_t* hashes) const = 0;
167
+
168
+ /// Batch compute hashes for fixed byte array values by using its plain encoding result.
169
+ ///
170
+ /// @param values values a pointer to the values to hash.
171
+ /// @param type_len the value length.
172
+ /// @param num_values the number of values to hash.
173
+ /// @param hashes a pointer to the output hash values, its length should be equal to
174
+ /// num_values.
175
+ virtual void Hashes(const FLBA* values, uint32_t type_len, int num_values,
176
+ uint64_t* hashes) const = 0;
177
+
178
+ virtual ~BloomFilter() = default;
179
+
180
+ protected:
181
+ // Hash strategy available for Bloom filter.
182
+ enum class HashStrategy : uint32_t { XXHASH = 0 };
183
+
184
+ // Bloom filter algorithm.
185
+ enum class Algorithm : uint32_t { BLOCK = 0 };
186
+
187
+ enum class CompressionStrategy : uint32_t { UNCOMPRESSED = 0 };
188
+ };
189
+
190
+ /// The BlockSplitBloomFilter is implemented using block-based Bloom filters from
191
+ /// Putze et al.'s "Cache-,Hash- and Space-Efficient Bloom filters". The basic idea is to
192
+ /// hash the item to a tiny Bloom filter which size fit a single cache line or smaller.
193
+ ///
194
+ /// This implementation sets 8 bits in each tiny Bloom filter. Each tiny Bloom
195
+ /// filter is 32 bytes to take advantage of 32-byte SIMD instructions.
196
+ class PARQUET_EXPORT BlockSplitBloomFilter : public BloomFilter {
197
+ public:
198
+ /// The constructor of BlockSplitBloomFilter. It uses XXH64 as hash function.
199
+ ///
200
+ /// \param pool memory pool to use.
201
+ explicit BlockSplitBloomFilter(
202
+ ::arrow::MemoryPool* pool = ::arrow::default_memory_pool());
203
+
204
+ /// Initialize the BlockSplitBloomFilter. The range of num_bytes should be within
205
+ /// [kMinimumBloomFilterBytes, kMaximumBloomFilterBytes], it will be
206
+ /// rounded up/down to lower/upper bound if num_bytes is out of range and also
207
+ /// will be rounded up to a power of 2.
208
+ ///
209
+ /// @param num_bytes The number of bytes to store Bloom filter bitset.
210
+ void Init(uint32_t num_bytes);
211
+
212
+ /// Initialize the BlockSplitBloomFilter. It copies the bitset as underlying
213
+ /// bitset because the given bitset may not satisfy the 32-byte alignment requirement
214
+ /// which may lead to segfault when performing SIMD instructions. It is the caller's
215
+ /// responsibility to free the bitset passed in. This is used when reconstructing
216
+ /// a Bloom filter from a parquet file.
217
+ ///
218
+ /// @param bitset The given bitset to initialize the Bloom filter.
219
+ /// @param num_bytes The number of bytes of given bitset.
220
+ void Init(const uint8_t* bitset, uint32_t num_bytes);
221
+
222
+ /// Minimum Bloom filter size, it sets to 32 bytes to fit a tiny Bloom filter.
223
+ static constexpr uint32_t kMinimumBloomFilterBytes = 32;
224
+
225
+ /// Calculate optimal size according to the number of distinct values and false
226
+ /// positive probability.
227
+ ///
228
+ /// @param ndv The number of distinct values.
229
+ /// @param fpp The false positive probability.
230
+ /// @return it always return a value between kMinimumBloomFilterBytes and
231
+ /// kMaximumBloomFilterBytes, and the return value is always a power of 2
232
+ static uint32_t OptimalNumOfBytes(uint32_t ndv, double fpp) {
233
+ uint32_t optimal_num_of_bits = OptimalNumOfBits(ndv, fpp);
234
+ ARROW_DCHECK(::arrow::bit_util::IsMultipleOf8(optimal_num_of_bits));
235
+ return optimal_num_of_bits >> 3;
236
+ }
237
+
238
+ /// Calculate optimal size according to the number of distinct values and false
239
+ /// positive probability.
240
+ ///
241
+ /// @param ndv The number of distinct values.
242
+ /// @param fpp The false positive probability.
243
+ /// @return it always return a value between kMinimumBloomFilterBytes * 8 and
244
+ /// kMaximumBloomFilterBytes * 8, and the return value is always a power of 16
245
+ static uint32_t OptimalNumOfBits(uint32_t ndv, double fpp) {
246
+ ARROW_DCHECK(fpp > 0.0 && fpp < 1.0);
247
+ const double m = -8.0 * ndv / log(1 - pow(fpp, 1.0 / 8));
248
+ uint32_t num_bits;
249
+
250
+ // Handle overflow.
251
+ if (m < 0 || m > kMaximumBloomFilterBytes << 3) {
252
+ num_bits = static_cast<uint32_t>(kMaximumBloomFilterBytes << 3);
253
+ } else {
254
+ num_bits = static_cast<uint32_t>(m);
255
+ }
256
+
257
+ // Round up to lower bound
258
+ if (num_bits < kMinimumBloomFilterBytes << 3) {
259
+ num_bits = kMinimumBloomFilterBytes << 3;
260
+ }
261
+
262
+ // Get next power of 2 if bits is not power of 2.
263
+ if ((num_bits & (num_bits - 1)) != 0) {
264
+ num_bits = static_cast<uint32_t>(::arrow::bit_util::NextPower2(num_bits));
265
+ }
266
+
267
+ // Round down to upper bound
268
+ if (num_bits > kMaximumBloomFilterBytes << 3) {
269
+ num_bits = kMaximumBloomFilterBytes << 3;
270
+ }
271
+
272
+ return num_bits;
273
+ }
274
+
275
+ bool FindHash(uint64_t hash) const override;
276
+ void InsertHash(uint64_t hash) override;
277
+ void InsertHashes(const uint64_t* hashes, int num_values) override;
278
+ void WriteTo(ArrowOutputStream* sink) const override;
279
+ uint32_t GetBitsetSize() const override { return num_bytes_; }
280
+
281
+ uint64_t Hash(int32_t value) const override { return hasher_->Hash(value); }
282
+ uint64_t Hash(int64_t value) const override { return hasher_->Hash(value); }
283
+ uint64_t Hash(float value) const override { return hasher_->Hash(value); }
284
+ uint64_t Hash(double value) const override { return hasher_->Hash(value); }
285
+ uint64_t Hash(const Int96* value) const override { return hasher_->Hash(value); }
286
+ uint64_t Hash(const ByteArray* value) const override { return hasher_->Hash(value); }
287
+ uint64_t Hash(const FLBA* value, uint32_t len) const override {
288
+ return hasher_->Hash(value, len);
289
+ }
290
+ uint64_t Hash(std::string_view value) const override { return hasher_->Hash(value); }
291
+
292
+ void Hashes(const int32_t* values, int num_values, uint64_t* hashes) const override {
293
+ hasher_->Hashes(values, num_values, hashes);
294
+ }
295
+ void Hashes(const int64_t* values, int num_values, uint64_t* hashes) const override {
296
+ hasher_->Hashes(values, num_values, hashes);
297
+ }
298
+ void Hashes(const float* values, int num_values, uint64_t* hashes) const override {
299
+ hasher_->Hashes(values, num_values, hashes);
300
+ }
301
+ void Hashes(const double* values, int num_values, uint64_t* hashes) const override {
302
+ hasher_->Hashes(values, num_values, hashes);
303
+ }
304
+ void Hashes(const Int96* values, int num_values, uint64_t* hashes) const override {
305
+ hasher_->Hashes(values, num_values, hashes);
306
+ }
307
+ void Hashes(const ByteArray* values, int num_values, uint64_t* hashes) const override {
308
+ hasher_->Hashes(values, num_values, hashes);
309
+ }
310
+ void Hashes(const FLBA* values, uint32_t type_len, int num_values,
311
+ uint64_t* hashes) const override {
312
+ hasher_->Hashes(values, type_len, num_values, hashes);
313
+ }
314
+
315
+ uint64_t Hash(const int32_t* value) const { return hasher_->Hash(*value); }
316
+ uint64_t Hash(const int64_t* value) const { return hasher_->Hash(*value); }
317
+ uint64_t Hash(const float* value) const { return hasher_->Hash(*value); }
318
+ uint64_t Hash(const double* value) const { return hasher_->Hash(*value); }
319
+
320
+ /// Deserialize the Bloom filter from an input stream. It is used when reconstructing
321
+ /// a Bloom filter from a parquet filter.
322
+ ///
323
+ /// @param properties The parquet reader properties.
324
+ /// @param input_stream The input stream from which to construct the bloom filter.
325
+ /// @param bloom_filter_length The length of the serialized bloom filter including
326
+ /// header.
327
+ /// @return The BlockSplitBloomFilter.
328
+ static BlockSplitBloomFilter Deserialize(
329
+ const ReaderProperties& properties, ArrowInputStream* input_stream,
330
+ std::optional<int64_t> bloom_filter_length = std::nullopt);
331
+
332
+ /// Deserialize an encrypted Bloom filter from an input stream.
333
+ ///
334
+ /// The same metadata decryptor is used for both the serialized header and bitset,
335
+ /// while switching module AADs between the two encrypted modules.
336
+ ///
337
+ /// @param properties The parquet reader properties.
338
+ /// @param input_stream The input stream from which to construct the bloom filter.
339
+ /// @param bloom_filter_length The length of the serialized bloom filter including
340
+ /// header.
341
+ /// @param decryptor Decryptor for encrypted Bloom filter modules.
342
+ /// @param row_group_ordinal Ordinal of the row group containing this Bloom filter.
343
+ /// @param column_ordinal Ordinal of the column containing this Bloom filter.
344
+ /// @return The BlockSplitBloomFilter.
345
+ static BlockSplitBloomFilter DeserializeEncrypted(
346
+ const ReaderProperties& properties, ArrowInputStream* input_stream,
347
+ std::optional<int64_t> bloom_filter_length, Decryptor* decryptor,
348
+ int16_t row_group_ordinal, int16_t column_ordinal);
349
+
350
+ private:
351
+ inline void InsertHashImpl(uint64_t hash);
352
+
353
+ // Bytes in a tiny Bloom filter block.
354
+ static constexpr int kBytesPerFilterBlock = 32;
355
+
356
+ // The number of bits to be set in each tiny Bloom filter
357
+ static constexpr int kBitsSetPerBlock = 8;
358
+
359
+ // A mask structure used to set bits in each tiny Bloom filter.
360
+ struct BlockMask {
361
+ uint32_t item[kBitsSetPerBlock];
362
+ };
363
+
364
+ // The block-based algorithm needs eight odd SALT values to calculate eight indexes
365
+ // of bit to set, one bit in each 32-bit word.
366
+ static constexpr uint32_t SALT[kBitsSetPerBlock] = {
367
+ 0x47b6137bU, 0x44974d91U, 0x8824ad5bU, 0xa2b7289dU,
368
+ 0x705495c7U, 0x2df1424bU, 0x9efc4947U, 0x5c6bfb31U};
369
+
370
+ // Memory pool to allocate aligned buffer for bitset
371
+ ::arrow::MemoryPool* pool_;
372
+
373
+ // The underlying buffer of bitset.
374
+ std::shared_ptr<Buffer> data_;
375
+
376
+ // The number of bytes of Bloom filter bitset.
377
+ uint32_t num_bytes_;
378
+
379
+ // Hash strategy used in this Bloom filter.
380
+ HashStrategy hash_strategy_;
381
+
382
+ // Algorithm used in this Bloom filter.
383
+ Algorithm algorithm_;
384
+
385
+ // Compression used in this Bloom filter.
386
+ CompressionStrategy compression_strategy_;
387
+
388
+ // The hash pointer points to actual hash class used.
389
+ std::unique_ptr<Hasher> hasher_;
390
+ };
391
+
392
+ } // namespace parquet
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/bloom_filter_reader.h ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include "arrow/io/interfaces.h"
21
+ #include "parquet/properties.h"
22
+ #include "parquet/type_fwd.h"
23
+
24
+ namespace parquet {
25
+
26
+ class InternalFileDecryptor;
27
+ class BloomFilter;
28
+
29
+ class PARQUET_EXPORT RowGroupBloomFilterReader {
30
+ public:
31
+ virtual ~RowGroupBloomFilterReader() = default;
32
+
33
+ /// \brief Read bloom filter of a column chunk.
34
+ ///
35
+ /// \param[in] i column ordinal of the column chunk.
36
+ /// \returns bloom filter of the column or nullptr if it does not exist.
37
+ /// \throws ParquetException if the index is out of bound, or read bloom
38
+ /// filter failed.
39
+ virtual std::unique_ptr<BloomFilter> GetColumnBloomFilter(int i) = 0;
40
+ };
41
+
42
+ /// \brief Interface for reading the bloom filter for a Parquet file.
43
+ class PARQUET_EXPORT BloomFilterReader {
44
+ public:
45
+ virtual ~BloomFilterReader() = default;
46
+
47
+ /// \brief Create a BloomFilterReader instance.
48
+ /// \returns a BloomFilterReader instance.
49
+ /// WARNING: The returned BloomFilterReader references to all the input parameters, so
50
+ /// it must not outlive all of the input parameters. Usually these input parameters
51
+ /// come from the same ParquetFileReader object, so it must not outlive the reader
52
+ /// that creates this BloomFilterReader.
53
+ static std::unique_ptr<BloomFilterReader> Make(
54
+ std::shared_ptr<::arrow::io::RandomAccessFile> input,
55
+ std::shared_ptr<FileMetaData> file_metadata, const ReaderProperties& properties,
56
+ std::shared_ptr<InternalFileDecryptor> file_decryptor = NULLPTR);
57
+
58
+ /// \brief Get the bloom filter reader of a specific row group.
59
+ /// \param[in] i row group ordinal to get bloom filter reader.
60
+ /// \returns RowGroupBloomFilterReader of the specified row group. A nullptr may or may
61
+ /// not be returned if the bloom filter for the row group is unavailable. It
62
+ /// is the caller's responsibility to check the return value of follow-up calls
63
+ /// to the RowGroupBloomFilterReader.
64
+ /// \throws ParquetException if the index is out of bound.
65
+ virtual std::shared_ptr<RowGroupBloomFilterReader> RowGroup(int i) = 0;
66
+ };
67
+
68
+ } // namespace parquet
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/bloom_filter_writer.h ADDED
@@ -0,0 +1,105 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include "arrow/type_fwd.h"
21
+
22
+ #include "parquet/bloom_filter.h"
23
+ #include "parquet/index_location.h"
24
+ #include "parquet/type_fwd.h"
25
+
26
+ namespace parquet {
27
+
28
+ /// \brief Writer for updating a bloom filter with values of a specific Parquet type.
29
+ /// \note Boolean type is not supported.
30
+ template <typename ParquetType>
31
+ class PARQUET_EXPORT TypedBloomFilterWriter {
32
+ public:
33
+ using T = typename ParquetType::c_type;
34
+
35
+ /// \param descr The descriptor of the column to write. Must outlive this writer.
36
+ /// \param bloom_filter The bloom filter to update. Must outlive this writer.
37
+ TypedBloomFilterWriter(const ColumnDescriptor* descr, BloomFilter* bloom_filter);
38
+
39
+ /// \brief Update the bloom filter with typed values.
40
+ ///
41
+ /// \param values The values to update the bloom filter with.
42
+ /// \param num_values The number of values to update the bloom filter with.
43
+ void Update(const T* values, int64_t num_values);
44
+
45
+ /// \brief Update the bloom filter with typed values that have spaces.
46
+ ///
47
+ /// \param values The values to update the bloom filter with.
48
+ /// \param num_values The number of values to update the bloom filter with.
49
+ /// \param valid_bits The validity bitmap of the values.
50
+ /// \param valid_bits_offset The offset of the validity bitmap.
51
+ void UpdateSpaced(const T* values, int64_t num_values, const uint8_t* valid_bits,
52
+ int64_t valid_bits_offset);
53
+
54
+ /// \brief Update the bloom filter with an Arrow array.
55
+ ///
56
+ /// \param values The Arrow array to update the bloom filter with.
57
+ void Update(const ::arrow::Array& values);
58
+
59
+ private:
60
+ const ColumnDescriptor* descr_;
61
+ BloomFilter* bloom_filter_;
62
+ };
63
+
64
+ /// \brief Interface for building bloom filters of a parquet file.
65
+ class PARQUET_EXPORT BloomFilterBuilder {
66
+ public:
67
+ virtual ~BloomFilterBuilder() = default;
68
+
69
+ /// \brief API to create a BloomFilterBuilder.
70
+ ///
71
+ /// \param schema The schema of the file and it must outlive the created builder.
72
+ /// \param properties Properties to get bloom filter options. It must outlive the
73
+ /// created builder.
74
+ static std::unique_ptr<BloomFilterBuilder> Make(const SchemaDescriptor* schema,
75
+ const WriterProperties* properties);
76
+
77
+ /// \brief Start a new row group to write bloom filters, meaning that next calls
78
+ /// to `CreateBloomFilter` will create bloom filters for the new row group.
79
+ ///
80
+ /// \throws ParquetException if `WriteTo()` has been called already.
81
+ virtual void AppendRowGroup() = 0;
82
+
83
+ /// \brief Create a BloomFilter of the column ordinal of the current row group.
84
+ ///
85
+ /// \param column_ordinal Column ordinal for the bloom filter.
86
+ /// \return created BloomFilter whose ownership belongs to the builder, or nullptr if
87
+ /// bloom filter is not enabled for the column.
88
+ /// \throws ParquetException if any condition is violated:
89
+ /// - `AppendRowGroup()` has not been called yet
90
+ /// - The column ordinal is out of bound
91
+ /// - Bloom filter already exists for the column
92
+ /// - `WriteTo()` has been called
93
+ virtual BloomFilter* CreateBloomFilter(int32_t column_ordinal) = 0;
94
+
95
+ /// \brief Write all bloom filters to sink.
96
+ ///
97
+ /// The bloom filters cannot be modified after this method is called.
98
+ ///
99
+ /// \param[in] sink The output stream to write the bloom filters.
100
+ /// \return The location of all bloom filters.
101
+ /// \throws ParquetException if `WriteTo()` has been called.
102
+ virtual IndexLocations WriteTo(::arrow::io::OutputStream* sink) = 0;
103
+ };
104
+
105
+ } // namespace parquet
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_page.h ADDED
@@ -0,0 +1,181 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ // This module defines an abstract interface for iterating through pages in a
19
+ // Parquet column chunk within a row group. It could be extended in the future
20
+ // to iterate through all data pages in all chunks in a file.
21
+
22
+ #pragma once
23
+
24
+ #include <cstdint>
25
+ #include <memory>
26
+ #include <optional>
27
+ #include <string>
28
+
29
+ #include "parquet/size_statistics.h"
30
+ #include "parquet/statistics.h"
31
+ #include "parquet/types.h"
32
+
33
+ namespace parquet {
34
+
35
+ // TODO: Parallel processing is not yet safe because of memory-ownership
36
+ // semantics (the PageReader may or may not own the memory referenced by a
37
+ // page)
38
+ //
39
+ // TODO(wesm): In the future Parquet implementations may store the crc code
40
+ // in format::PageHeader. parquet-mr currently does not, so we also skip it
41
+ // here, both on the read and write path
42
+ class Page {
43
+ public:
44
+ Page(const std::shared_ptr<Buffer>& buffer, PageType::type type)
45
+ : buffer_(buffer), type_(type) {}
46
+
47
+ virtual ~Page() = default;
48
+
49
+ PageType::type type() const { return type_; }
50
+
51
+ std::shared_ptr<Buffer> buffer() const { return buffer_; }
52
+
53
+ // @returns: a pointer to the page's data
54
+ const uint8_t* data() const { return buffer_->data(); }
55
+
56
+ // @returns: the total size in bytes of the page's data buffer
57
+ int32_t size() const { return static_cast<int32_t>(buffer_->size()); }
58
+
59
+ private:
60
+ std::shared_ptr<Buffer> buffer_;
61
+ PageType::type type_;
62
+ };
63
+
64
+ /// \brief Base type for DataPageV1 and DataPageV2 including common attributes
65
+ class DataPage : public Page {
66
+ public:
67
+ int32_t num_values() const { return num_values_; }
68
+ Encoding::type encoding() const { return encoding_; }
69
+ int64_t uncompressed_size() const { return uncompressed_size_; }
70
+ const EncodedStatistics& statistics() const { return statistics_; }
71
+ /// Return the row ordinal within the row group to the first row in the data page.
72
+ /// Currently it is only present from data pages created by ColumnWriter in order
73
+ /// to collect page index.
74
+ std::optional<int64_t> first_row_index() const { return first_row_index_; }
75
+ const SizeStatistics& size_statistics() const { return size_statistics_; }
76
+
77
+ virtual ~DataPage() = default;
78
+
79
+ protected:
80
+ DataPage(PageType::type type, const std::shared_ptr<Buffer>& buffer, int32_t num_values,
81
+ Encoding::type encoding, int64_t uncompressed_size,
82
+ EncodedStatistics statistics, std::optional<int64_t> first_row_index,
83
+ SizeStatistics size_statistics)
84
+ : Page(buffer, type),
85
+ num_values_(num_values),
86
+ encoding_(encoding),
87
+ uncompressed_size_(uncompressed_size),
88
+ statistics_(std::move(statistics)),
89
+ first_row_index_(std::move(first_row_index)),
90
+ size_statistics_(std::move(size_statistics)) {}
91
+
92
+ int32_t num_values_;
93
+ Encoding::type encoding_;
94
+ int64_t uncompressed_size_;
95
+ EncodedStatistics statistics_;
96
+ /// Row ordinal within the row group to the first row in the data page.
97
+ std::optional<int64_t> first_row_index_;
98
+ SizeStatistics size_statistics_;
99
+ };
100
+
101
+ class DataPageV1 : public DataPage {
102
+ public:
103
+ DataPageV1(const std::shared_ptr<Buffer>& buffer, int32_t num_values,
104
+ Encoding::type encoding, Encoding::type definition_level_encoding,
105
+ Encoding::type repetition_level_encoding, int64_t uncompressed_size,
106
+ EncodedStatistics statistics = EncodedStatistics(),
107
+ std::optional<int64_t> first_row_index = std::nullopt,
108
+ SizeStatistics size_statistics = SizeStatistics())
109
+ : DataPage(PageType::DATA_PAGE, buffer, num_values, encoding, uncompressed_size,
110
+ std::move(statistics), std::move(first_row_index),
111
+ std::move(size_statistics)),
112
+ definition_level_encoding_(definition_level_encoding),
113
+ repetition_level_encoding_(repetition_level_encoding) {}
114
+
115
+ Encoding::type repetition_level_encoding() const { return repetition_level_encoding_; }
116
+
117
+ Encoding::type definition_level_encoding() const { return definition_level_encoding_; }
118
+
119
+ private:
120
+ Encoding::type definition_level_encoding_;
121
+ Encoding::type repetition_level_encoding_;
122
+ };
123
+
124
+ class DataPageV2 : public DataPage {
125
+ public:
126
+ DataPageV2(const std::shared_ptr<Buffer>& buffer, int32_t num_values, int32_t num_nulls,
127
+ int32_t num_rows, Encoding::type encoding,
128
+ int32_t definition_levels_byte_length, int32_t repetition_levels_byte_length,
129
+ int64_t uncompressed_size, bool is_compressed = false,
130
+ EncodedStatistics statistics = EncodedStatistics(),
131
+ std::optional<int64_t> first_row_index = std::nullopt,
132
+ SizeStatistics size_statistics = SizeStatistics())
133
+ : DataPage(PageType::DATA_PAGE_V2, buffer, num_values, encoding, uncompressed_size,
134
+ std::move(statistics), std::move(first_row_index),
135
+ std::move(size_statistics)),
136
+ num_nulls_(num_nulls),
137
+ num_rows_(num_rows),
138
+ definition_levels_byte_length_(definition_levels_byte_length),
139
+ repetition_levels_byte_length_(repetition_levels_byte_length),
140
+ is_compressed_(is_compressed) {}
141
+
142
+ int32_t num_nulls() const { return num_nulls_; }
143
+
144
+ int32_t num_rows() const { return num_rows_; }
145
+
146
+ int32_t definition_levels_byte_length() const { return definition_levels_byte_length_; }
147
+
148
+ int32_t repetition_levels_byte_length() const { return repetition_levels_byte_length_; }
149
+
150
+ bool is_compressed() const { return is_compressed_; }
151
+
152
+ private:
153
+ int32_t num_nulls_;
154
+ int32_t num_rows_;
155
+ int32_t definition_levels_byte_length_;
156
+ int32_t repetition_levels_byte_length_;
157
+ bool is_compressed_;
158
+ };
159
+
160
+ class DictionaryPage : public Page {
161
+ public:
162
+ DictionaryPage(const std::shared_ptr<Buffer>& buffer, int32_t num_values,
163
+ Encoding::type encoding, bool is_sorted = false)
164
+ : Page(buffer, PageType::DICTIONARY_PAGE),
165
+ num_values_(num_values),
166
+ encoding_(encoding),
167
+ is_sorted_(is_sorted) {}
168
+
169
+ int32_t num_values() const { return num_values_; }
170
+
171
+ Encoding::type encoding() const { return encoding_; }
172
+
173
+ bool is_sorted() const { return is_sorted_; }
174
+
175
+ private:
176
+ int32_t num_values_;
177
+ Encoding::type encoding_;
178
+ bool is_sorted_;
179
+ };
180
+
181
+ } // namespace parquet
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_reader.h ADDED
@@ -0,0 +1,458 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <cstdint>
21
+ #include <memory>
22
+ #include <utility>
23
+ #include <vector>
24
+
25
+ #include "arrow/type_fwd.h"
26
+ #include "arrow/util/macros.h"
27
+ #include "parquet/exception.h"
28
+ #include "parquet/level_conversion.h"
29
+ #include "parquet/metadata.h"
30
+ #include "parquet/platform.h"
31
+ #include "parquet/properties.h"
32
+ #include "parquet/schema.h"
33
+ #include "parquet/types.h"
34
+
35
+ namespace arrow {
36
+
37
+ namespace bit_util {
38
+ class BitReader;
39
+ } // namespace bit_util
40
+
41
+ namespace util {
42
+ template <typename T>
43
+ class RleBitPackedDecoder;
44
+ } // namespace util
45
+
46
+ } // namespace arrow
47
+
48
+ namespace parquet {
49
+
50
+ class Decryptor;
51
+ class Page;
52
+
53
+ // 16 MB is the default maximum page header size
54
+ static constexpr uint32_t kDefaultMaxPageHeaderSize = 16 * 1024 * 1024;
55
+
56
+ // 16 KB is the default expected page header size
57
+ static constexpr uint32_t kDefaultPageHeaderSize = 16 * 1024;
58
+
59
+ // \brief DataPageStats stores encoded statistics and number of values/rows for
60
+ // a page.
61
+ struct PARQUET_EXPORT DataPageStats {
62
+ DataPageStats(const EncodedStatistics* encoded_statistics, int32_t num_values,
63
+ std::optional<int32_t> num_rows)
64
+ : encoded_statistics(encoded_statistics),
65
+ num_values(num_values),
66
+ num_rows(num_rows) {}
67
+
68
+ // Encoded statistics extracted from the page header.
69
+ // Nullptr if there are no statistics in the page header.
70
+ const EncodedStatistics* encoded_statistics;
71
+ // Number of values stored in the page. Filled for both V1 and V2 data pages.
72
+ // For repeated fields, this can be greater than number of rows. For
73
+ // non-repeated fields, this will be the same as the number of rows.
74
+ int32_t num_values;
75
+ // Number of rows stored in the page. std::nullopt if not available.
76
+ std::optional<int32_t> num_rows;
77
+ };
78
+
79
+ class PARQUET_EXPORT LevelDecoder {
80
+ public:
81
+ LevelDecoder();
82
+ ~LevelDecoder();
83
+
84
+ // Initialize the LevelDecoder state with new data
85
+ // and return the number of bytes consumed
86
+ int SetData(Encoding::type encoding, int16_t max_level, int num_buffered_values,
87
+ const uint8_t* data, int32_t data_size);
88
+
89
+ void SetDataV2(int32_t num_bytes, int16_t max_level, int num_buffered_values,
90
+ const uint8_t* data);
91
+
92
+ // Decodes a batch of levels into an array and returns the number of levels decoded
93
+ int Decode(int batch_size, int16_t* levels);
94
+
95
+ private:
96
+ int bit_width_;
97
+ int num_values_remaining_;
98
+ Encoding::type encoding_;
99
+ std::unique_ptr<::arrow::util::RleBitPackedDecoder<int16_t>> rle_decoder_;
100
+ std::unique_ptr<::arrow::bit_util::BitReader> bit_packed_decoder_;
101
+ int16_t max_level_;
102
+ };
103
+
104
+ struct CryptoContext {
105
+ bool start_decrypt_with_dictionary_page = false;
106
+ int16_t row_group_ordinal = -1;
107
+ int16_t column_ordinal = -1;
108
+ std::function<std::unique_ptr<Decryptor>()> meta_decryptor_factory;
109
+ std::function<std::unique_ptr<Decryptor>()> data_decryptor_factory;
110
+ };
111
+
112
+ // Abstract page iterator interface. This way, we can feed column pages to the
113
+ // ColumnReader through whatever mechanism we choose
114
+ class PARQUET_EXPORT PageReader {
115
+ using DataPageFilter = std::function<bool(const DataPageStats&)>;
116
+
117
+ public:
118
+ virtual ~PageReader() = default;
119
+
120
+ static std::unique_ptr<PageReader> Open(
121
+ std::shared_ptr<ArrowInputStream> stream, int64_t total_num_values,
122
+ Compression::type codec, bool always_compressed = false,
123
+ ::arrow::MemoryPool* pool = ::arrow::default_memory_pool(),
124
+ const CryptoContext* ctx = NULLPTR);
125
+ static std::unique_ptr<PageReader> Open(std::shared_ptr<ArrowInputStream> stream,
126
+ int64_t total_num_values,
127
+ Compression::type codec,
128
+ const ReaderProperties& properties,
129
+ bool always_compressed = false,
130
+ const CryptoContext* ctx = NULLPTR);
131
+
132
+ // If data_page_filter is present (not null), NextPage() will call the
133
+ // callback function exactly once per page in the order the pages appear in
134
+ // the column. If the callback function returns true the page will be
135
+ // skipped. The callback will be called only if the page type is DATA_PAGE or
136
+ // DATA_PAGE_V2. Dictionary pages will not be skipped.
137
+ // Caller is responsible for checking that statistics are correct using
138
+ // ApplicationVersion::HasCorrectStatistics().
139
+ // \note API EXPERIMENTAL
140
+ void set_data_page_filter(DataPageFilter data_page_filter) {
141
+ data_page_filter_ = std::move(data_page_filter);
142
+ }
143
+
144
+ // @returns: shared_ptr<Page>(nullptr) on EOS, std::shared_ptr<Page>
145
+ // containing new Page otherwise
146
+ //
147
+ // The returned Page may contain references that aren't guaranteed to live
148
+ // beyond the next call to NextPage().
149
+ virtual std::shared_ptr<Page> NextPage() = 0;
150
+
151
+ virtual void set_max_page_header_size(uint32_t size) = 0;
152
+
153
+ protected:
154
+ // Callback that decides if we should skip a page or not.
155
+ DataPageFilter data_page_filter_;
156
+ };
157
+
158
+ class PARQUET_EXPORT ColumnReader {
159
+ public:
160
+ virtual ~ColumnReader() = default;
161
+
162
+ static std::shared_ptr<ColumnReader> Make(
163
+ const ColumnDescriptor* descr, std::unique_ptr<PageReader> pager,
164
+ ::arrow::MemoryPool* pool = ::arrow::default_memory_pool());
165
+
166
+ // Returns true if there are still values in this column.
167
+ virtual bool HasNext() = 0;
168
+
169
+ virtual Type::type type() const = 0;
170
+
171
+ virtual const ColumnDescriptor* descr() const = 0;
172
+
173
+ // Get the encoding that can be exposed by this reader. If it returns
174
+ // dictionary encoding, then ReadBatchWithDictionary can be used to read data.
175
+ //
176
+ // \note API EXPERIMENTAL
177
+ virtual ExposedEncoding GetExposedEncoding() = 0;
178
+
179
+ protected:
180
+ friend class RowGroupReader;
181
+ // Set the encoding that can be exposed by this reader.
182
+ //
183
+ // \note API EXPERIMENTAL
184
+ virtual void SetExposedEncoding(ExposedEncoding encoding) = 0;
185
+ };
186
+
187
+ // API to read values from a single column. This is a main client facing API.
188
+ template <typename DType>
189
+ class TypedColumnReader : public ColumnReader {
190
+ public:
191
+ using T = typename DType::c_type;
192
+
193
+ // Read a batch of repetition levels, definition levels, and values from the
194
+ // column.
195
+ //
196
+ // Since null values are not stored in the values, the number of values read
197
+ // may be less than the number of repetition and definition levels. With
198
+ // nested data this is almost certainly true.
199
+ //
200
+ // Set def_levels or rep_levels to nullptr if you want to skip reading them.
201
+ // This is only safe if you know through some other source that there are no
202
+ // undefined values.
203
+ //
204
+ // To fully exhaust a row group, you must read batches until the number of
205
+ // values read reaches the number of stored values according to the metadata.
206
+ //
207
+ // This API is the same for both V1 and V2 of the DataPage
208
+ //
209
+ // @returns: actual number of levels read (see values_read for number of values read)
210
+ virtual int64_t ReadBatch(int64_t batch_size, int16_t* def_levels, int16_t* rep_levels,
211
+ T* values, int64_t* values_read) = 0;
212
+
213
+ // Skip reading values. This method will work for both repeated and
214
+ // non-repeated fields. Note that this method is skipping values and not
215
+ // records. This distinction is important for repeated fields, meaning that
216
+ // we are not skipping over the values to the next record. For example,
217
+ // consider the following two consecutive records containing one repeated field:
218
+ // {[1, 2, 3]}, {[4, 5]}. If we Skip(2), our next read value will be 3, which
219
+ // is inside the first record.
220
+ // Returns the number of values skipped.
221
+ virtual int64_t Skip(int64_t num_values_to_skip) = 0;
222
+
223
+ // Read a batch of repetition levels, definition levels, and indices from the
224
+ // column. And read the dictionary if a dictionary page is encountered during
225
+ // reading pages. This API is similar to ReadBatch(), with ability to read
226
+ // dictionary and indices. It is only valid to call this method when the reader can
227
+ // expose dictionary encoding. (i.e., the reader's GetExposedEncoding() returns
228
+ // DICTIONARY).
229
+ //
230
+ // The dictionary is read along with the data page. When there's no data page,
231
+ // the dictionary won't be returned.
232
+ //
233
+ // @param batch_size The batch size to read
234
+ // @param[out] def_levels The Parquet definition levels.
235
+ // @param[out] rep_levels The Parquet repetition levels.
236
+ // @param[out] indices The dictionary indices.
237
+ // @param[out] indices_read The number of indices read.
238
+ // @param[out] dict The pointer to dictionary values. It will return nullptr if
239
+ // there's no data page. Each column chunk only has one dictionary page. The dictionary
240
+ // is owned by the reader, so the caller is responsible for copying the dictionary
241
+ // values before the reader gets destroyed.
242
+ // @param[out] dict_len The dictionary length. It will return 0 if there's no data
243
+ // page.
244
+ // @returns: actual number of levels read (see indices_read for number of
245
+ // indices read
246
+ //
247
+ // \note API EXPERIMENTAL
248
+ virtual int64_t ReadBatchWithDictionary(int64_t batch_size, int16_t* def_levels,
249
+ int16_t* rep_levels, int32_t* indices,
250
+ int64_t* indices_read, const T** dict,
251
+ int32_t* dict_len) = 0;
252
+ };
253
+
254
+ namespace internal {
255
+
256
+ /// \brief Stateful column reader that delimits semantic records for both flat
257
+ /// and nested columns
258
+ ///
259
+ /// \note API EXPERIMENTAL
260
+ /// \since 1.3.0
261
+ class PARQUET_EXPORT RecordReader {
262
+ public:
263
+ /// \brief Creates a record reader.
264
+ /// @param descr Column descriptor
265
+ /// @param leaf_info Level info, used to determine if a column is nullable or not
266
+ /// @param pool Memory pool to use for buffering values and rep/def levels
267
+ /// @param read_dictionary True if reading directly as Arrow dictionary-encoded
268
+ /// @param read_dense_for_nullable True if reading dense and not leaving space for null
269
+ /// values
270
+ /// @param arrow_type Which type to read this column as (optional). Currently
271
+ /// only used for byte array columns (see BinaryRecordReader::GetBuilderChunks).
272
+ static std::shared_ptr<RecordReader> Make(
273
+ const ColumnDescriptor* descr, LevelInfo leaf_info,
274
+ ::arrow::MemoryPool* pool = ::arrow::default_memory_pool(),
275
+ bool read_dictionary = false, bool read_dense_for_nullable = false,
276
+ const std::shared_ptr<::arrow::DataType>& arrow_type = NULLPTR);
277
+
278
+ virtual ~RecordReader() = default;
279
+
280
+ /// \brief Attempt to read indicated number of records from column chunk
281
+ /// Note that for repeated fields, a record may have more than one value
282
+ /// and all of them are read. If read_dense_for_nullable() it will
283
+ /// not leave any space for null values. Otherwise, it will read spaced.
284
+ /// \return number of records read
285
+ virtual int64_t ReadRecords(int64_t num_records) = 0;
286
+
287
+ /// \brief Attempt to skip indicated number of records from column chunk.
288
+ /// Note that for repeated fields, a record may have more than one value
289
+ /// and all of them are skipped.
290
+ /// \return number of records skipped
291
+ virtual int64_t SkipRecords(int64_t num_records) = 0;
292
+
293
+ /// \brief Pre-allocate space for data. Results in better flat read performance
294
+ virtual void Reserve(int64_t num_values) = 0;
295
+
296
+ /// \brief Clear consumed values and repetition/definition levels as the
297
+ /// result of calling ReadRecords
298
+ /// For FLBA and ByteArray types, call GetBuilderChunks() to reset them.
299
+ virtual void Reset() = 0;
300
+
301
+ /// \brief Transfer filled values buffer to caller. A new one will be
302
+ /// allocated in subsequent ReadRecords calls
303
+ virtual std::shared_ptr<ResizableBuffer> ReleaseValues() = 0;
304
+
305
+ /// \brief Transfer filled validity bitmap buffer to caller. A new one will
306
+ /// be allocated in subsequent ReadRecords calls
307
+ virtual std::shared_ptr<ResizableBuffer> ReleaseIsValid() = 0;
308
+
309
+ /// \brief Return true if the record reader has more internal data yet to
310
+ /// process
311
+ virtual bool HasMoreData() const = 0;
312
+
313
+ /// \brief Advance record reader to the next row group. Must be set before
314
+ /// any records could be read/skipped.
315
+ /// \param[in] reader obtained from RowGroupReader::GetColumnPageReader
316
+ virtual void SetPageReader(std::unique_ptr<PageReader> reader) = 0;
317
+
318
+ /// \brief Returns the underlying column reader's descriptor.
319
+ virtual const ColumnDescriptor* descr() const = 0;
320
+
321
+ virtual void DebugPrintState() = 0;
322
+
323
+ /// \brief Returns the dictionary owned by the current decoder. Throws an
324
+ /// exception if the current decoder is not for dictionary encoding. The caller is
325
+ /// responsible for casting the returned pointer to proper type depending on the
326
+ /// column's physical type. An example:
327
+ /// const ByteArray* dict = reinterpret_cast<const ByteArray*>(ReadDictionary(&len));
328
+ /// or:
329
+ /// const float* dict = reinterpret_cast<const float*>(ReadDictionary(&len));
330
+ /// \param[out] dictionary_length The number of dictionary entries.
331
+ virtual const void* ReadDictionary(int32_t* dictionary_length) = 0;
332
+
333
+ /// \brief Decoded definition levels
334
+ int16_t* def_levels() const {
335
+ return reinterpret_cast<int16_t*>(def_levels_->mutable_data());
336
+ }
337
+
338
+ /// \brief Decoded repetition levels
339
+ int16_t* rep_levels() const {
340
+ return reinterpret_cast<int16_t*>(rep_levels_->mutable_data());
341
+ }
342
+
343
+ /// \brief Decoded values, including nulls, if any
344
+ /// FLBA and ByteArray types do not use this array and read into their own
345
+ /// builders.
346
+ uint8_t* values() const { return values_->mutable_data(); }
347
+
348
+ /// \brief Number of values written, including space left for nulls if any.
349
+ /// If this Reader was constructed with read_dense_for_nullable(), there is no space for
350
+ /// nulls and null_count() will be 0. There is no read-ahead/buffering for values. For
351
+ /// FLBA and ByteArray types this value reflects the values written with the last
352
+ /// ReadRecords call since those readers will reset the values after each call.
353
+ int64_t values_written() const { return values_written_; }
354
+
355
+ /// \brief Number of definition / repetition levels (from those that have
356
+ /// been decoded) that have been consumed inside the reader.
357
+ int64_t levels_position() const { return levels_position_; }
358
+
359
+ /// \brief Number of definition / repetition levels that have been written
360
+ /// internally in the reader. This may be larger than values_written() because
361
+ /// for repeated fields we need to look at the levels in advance to figure out
362
+ /// the record boundaries.
363
+ int64_t levels_written() const { return levels_written_; }
364
+
365
+ /// \brief Number of nulls in the leaf that we have read so far into the
366
+ /// values vector. This is only valid when !read_dense_for_nullable(). When
367
+ /// read_dense_for_nullable() it will always be 0.
368
+ int64_t null_count() const { return null_count_; }
369
+
370
+ /// \brief True if the leaf values are nullable
371
+ bool nullable_values() const { return nullable_values_; }
372
+
373
+ /// \brief True if reading directly as Arrow dictionary-encoded
374
+ bool read_dictionary() const { return read_dictionary_; }
375
+
376
+ /// \brief True if reading dense for nullable columns.
377
+ bool read_dense_for_nullable() const { return read_dense_for_nullable_; }
378
+
379
+ protected:
380
+ /// \brief Indicates if we can have nullable values. Note that repeated fields
381
+ /// may or may not be nullable.
382
+ bool nullable_values_;
383
+
384
+ bool at_record_start_;
385
+ int64_t records_read_;
386
+
387
+ /// \brief Stores values. These values are populated based on each ReadRecords
388
+ /// call. No extra values are buffered for the next call. SkipRecords will not
389
+ /// add any value to this buffer.
390
+ std::shared_ptr<::arrow::ResizableBuffer> values_;
391
+ /// \brief False for FIXED_LEN_BYTE_ARRAY and BYTE_ARRAY, in which case we
392
+ /// don't allocate the values buffer and we directly read into builder classes.
393
+ bool uses_values_;
394
+
395
+ /// \brief Values that we have read into 'values_' + 'null_count_'.
396
+ int64_t values_written_;
397
+ int64_t values_capacity_;
398
+ int64_t null_count_;
399
+
400
+ /// \brief Each bit corresponds to one element in 'values_' and specifies if it
401
+ /// is null or not null.
402
+ ///
403
+ /// Not set if leaf type is not nullable or read_dense_for_nullable_ is true.
404
+ std::shared_ptr<::arrow::ResizableBuffer> valid_bits_;
405
+
406
+ /// \brief Buffer for definition levels. May contain more levels than
407
+ /// is actually read. This is because we read levels ahead to
408
+ /// figure out record boundaries for repeated fields.
409
+ /// For flat required fields, 'def_levels_' and 'rep_levels_' are not
410
+ /// populated. For non-repeated fields 'rep_levels_' is not populated.
411
+ /// 'def_levels_' and 'rep_levels_' must be of the same size if present.
412
+ std::shared_ptr<::arrow::ResizableBuffer> def_levels_;
413
+ /// \brief Buffer for repetition levels. Only populated for repeated
414
+ /// fields.
415
+ std::shared_ptr<::arrow::ResizableBuffer> rep_levels_;
416
+
417
+ /// \brief Number of definition / repetition levels that have been written
418
+ /// internally in the reader. This may be larger than values_written() since
419
+ /// for repeated fields we need to look at the levels in advance to figure out
420
+ /// the record boundaries.
421
+ int64_t levels_written_;
422
+ /// \brief Position of the next level that should be consumed.
423
+ int64_t levels_position_;
424
+ int64_t levels_capacity_;
425
+
426
+ bool read_dictionary_ = false;
427
+ // If true, we will not leave any space for the null values in the values_
428
+ // vector or fill nulls values in BinaryRecordReader/DictionaryRecordReader.
429
+ //
430
+ // If read_dense_for_nullable_ is true, the BinaryRecordReader/DictionaryRecordReader
431
+ // might still populate the validity bitmap buffer.
432
+ bool read_dense_for_nullable_ = false;
433
+ };
434
+
435
+ class BinaryRecordReader : virtual public RecordReader {
436
+ public:
437
+ virtual std::vector<std::shared_ptr<::arrow::Array>> GetBuilderChunks() = 0;
438
+ };
439
+
440
+ /// \brief Read records directly to dictionary-encoded Arrow form (int32
441
+ /// indices). Only valid for BYTE_ARRAY columns
442
+ class DictionaryRecordReader : virtual public RecordReader {
443
+ public:
444
+ virtual std::shared_ptr<::arrow::ChunkedArray> GetResult() = 0;
445
+ };
446
+
447
+ } // namespace internal
448
+
449
+ using BoolReader = TypedColumnReader<BooleanType>;
450
+ using Int32Reader = TypedColumnReader<Int32Type>;
451
+ using Int64Reader = TypedColumnReader<Int64Type>;
452
+ using Int96Reader = TypedColumnReader<Int96Type>;
453
+ using FloatReader = TypedColumnReader<FloatType>;
454
+ using DoubleReader = TypedColumnReader<DoubleType>;
455
+ using ByteArrayReader = TypedColumnReader<ByteArrayType>;
456
+ using FixedLenByteArrayReader = TypedColumnReader<FLBAType>;
457
+
458
+ } // namespace parquet
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_scanner.h ADDED
@@ -0,0 +1,264 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <stdio.h>
21
+
22
+ #include <cstdint>
23
+ #include <memory>
24
+ #include <ostream>
25
+ #include <string>
26
+ #include <utility>
27
+ #include <vector>
28
+
29
+ #include "parquet/column_reader.h"
30
+ #include "parquet/exception.h"
31
+ #include "parquet/platform.h"
32
+ #include "parquet/schema.h"
33
+ #include "parquet/types.h"
34
+
35
+ namespace parquet {
36
+
37
+ static constexpr int64_t DEFAULT_SCANNER_BATCH_SIZE = 128;
38
+
39
+ class PARQUET_EXPORT Scanner {
40
+ public:
41
+ explicit Scanner(std::shared_ptr<ColumnReader> reader,
42
+ int64_t batch_size = DEFAULT_SCANNER_BATCH_SIZE,
43
+ ::arrow::MemoryPool* pool = ::arrow::default_memory_pool())
44
+ : batch_size_(batch_size),
45
+ level_offset_(0),
46
+ levels_buffered_(0),
47
+ value_buffer_(AllocateBuffer(pool)),
48
+ value_offset_(0),
49
+ values_buffered_(0),
50
+ reader_(std::move(reader)) {
51
+ def_levels_.resize(
52
+ descr()->max_definition_level() > 0 ? static_cast<size_t>(batch_size_) : 0);
53
+ rep_levels_.resize(
54
+ descr()->max_repetition_level() > 0 ? static_cast<size_t>(batch_size_) : 0);
55
+ }
56
+
57
+ virtual ~Scanner() {}
58
+
59
+ static std::shared_ptr<Scanner> Make(
60
+ std::shared_ptr<ColumnReader> col_reader,
61
+ int64_t batch_size = DEFAULT_SCANNER_BATCH_SIZE,
62
+ ::arrow::MemoryPool* pool = ::arrow::default_memory_pool());
63
+
64
+ virtual void PrintNext(std::ostream& out, int width, bool with_levels = false) = 0;
65
+
66
+ bool HasNext() { return level_offset_ < levels_buffered_ || reader_->HasNext(); }
67
+
68
+ const ColumnDescriptor* descr() const { return reader_->descr(); }
69
+
70
+ int64_t batch_size() const { return batch_size_; }
71
+
72
+ void SetBatchSize(int64_t batch_size) { batch_size_ = batch_size; }
73
+
74
+ protected:
75
+ int64_t batch_size_;
76
+
77
+ std::vector<int16_t> def_levels_;
78
+ std::vector<int16_t> rep_levels_;
79
+ int level_offset_;
80
+ int levels_buffered_;
81
+
82
+ std::shared_ptr<ResizableBuffer> value_buffer_;
83
+ int value_offset_;
84
+ int64_t values_buffered_;
85
+ std::shared_ptr<ColumnReader> reader_;
86
+ };
87
+
88
+ template <typename DType>
89
+ class PARQUET_TEMPLATE_CLASS_EXPORT TypedScanner : public Scanner {
90
+ public:
91
+ typedef typename DType::c_type T;
92
+
93
+ explicit TypedScanner(std::shared_ptr<ColumnReader> reader,
94
+ int64_t batch_size = DEFAULT_SCANNER_BATCH_SIZE,
95
+ ::arrow::MemoryPool* pool = ::arrow::default_memory_pool())
96
+ : Scanner(std::move(reader), batch_size, pool) {
97
+ typed_reader_ = static_cast<TypedColumnReader<DType>*>(reader_.get());
98
+ int value_byte_size = type_traits<DType::type_num>::value_byte_size;
99
+ PARQUET_THROW_NOT_OK(value_buffer_->Resize(batch_size_ * value_byte_size));
100
+ values_ = reinterpret_cast<T*>(value_buffer_->mutable_data());
101
+ }
102
+
103
+ virtual ~TypedScanner() {}
104
+
105
+ bool NextLevels(int16_t* def_level, int16_t* rep_level) {
106
+ if (level_offset_ == levels_buffered_) {
107
+ levels_buffered_ = static_cast<int>(
108
+ typed_reader_->ReadBatch(static_cast<int>(batch_size_), def_levels_.data(),
109
+ rep_levels_.data(), values_, &values_buffered_));
110
+
111
+ value_offset_ = 0;
112
+ level_offset_ = 0;
113
+ if (!levels_buffered_) {
114
+ return false;
115
+ }
116
+ }
117
+ *def_level = descr()->max_definition_level() > 0 ? def_levels_[level_offset_] : 0;
118
+ *rep_level = descr()->max_repetition_level() > 0 ? rep_levels_[level_offset_] : 0;
119
+ level_offset_++;
120
+ return true;
121
+ }
122
+
123
+ bool Next(T* val, int16_t* def_level, int16_t* rep_level, bool* is_null) {
124
+ if (level_offset_ == levels_buffered_) {
125
+ if (!HasNext()) {
126
+ // Out of data pages
127
+ return false;
128
+ }
129
+ }
130
+
131
+ NextLevels(def_level, rep_level);
132
+ *is_null = *def_level < descr()->max_definition_level();
133
+
134
+ if (*is_null) {
135
+ return true;
136
+ }
137
+
138
+ if (value_offset_ == values_buffered_) {
139
+ throw ParquetException("Value was non-null, but has not been buffered");
140
+ }
141
+ *val = values_[value_offset_++];
142
+ return true;
143
+ }
144
+
145
+ // Returns true if there is a next value
146
+ bool NextValue(T* val, bool* is_null) {
147
+ if (level_offset_ == levels_buffered_) {
148
+ if (!HasNext()) {
149
+ // Out of data pages
150
+ return false;
151
+ }
152
+ }
153
+
154
+ // Out of values
155
+ int16_t def_level = -1;
156
+ int16_t rep_level = -1;
157
+ NextLevels(&def_level, &rep_level);
158
+ *is_null = def_level < descr()->max_definition_level();
159
+
160
+ if (*is_null) {
161
+ return true;
162
+ }
163
+
164
+ if (value_offset_ == values_buffered_) {
165
+ throw ParquetException("Value was non-null, but has not been buffered");
166
+ }
167
+ *val = values_[value_offset_++];
168
+ return true;
169
+ }
170
+
171
+ virtual void PrintNext(std::ostream& out, int width, bool with_levels = false) {
172
+ T val{};
173
+ int16_t def_level = -1;
174
+ int16_t rep_level = -1;
175
+ bool is_null = false;
176
+ char buffer[80];
177
+
178
+ if (!Next(&val, &def_level, &rep_level, &is_null)) {
179
+ throw ParquetException("No more values buffered");
180
+ }
181
+
182
+ if (with_levels) {
183
+ out << " D:" << def_level << " R:" << rep_level << " ";
184
+ if (!is_null) {
185
+ out << "V:";
186
+ }
187
+ }
188
+
189
+ if (is_null) {
190
+ std::string null_fmt = format_fwf<ByteArrayType>(width);
191
+ snprintf(buffer, sizeof(buffer), null_fmt.c_str(), "NULL");
192
+ } else {
193
+ FormatValue(&val, buffer, sizeof(buffer), width);
194
+ }
195
+ out << buffer;
196
+ }
197
+
198
+ private:
199
+ // The ownership of this object is expressed through the reader_ variable in the base
200
+ TypedColumnReader<DType>* typed_reader_;
201
+
202
+ inline void FormatValue(void* val, char* buffer, int bufsize, int width);
203
+
204
+ T* values_;
205
+ };
206
+
207
+ template <typename DType>
208
+ inline void TypedScanner<DType>::FormatValue(void* val, char* buffer, int bufsize,
209
+ int width) {
210
+ std::string fmt = format_fwf<DType>(width);
211
+ snprintf(buffer, bufsize, fmt.c_str(), *reinterpret_cast<T*>(val));
212
+ }
213
+
214
+ template <>
215
+ inline void TypedScanner<Int96Type>::FormatValue(void* val, char* buffer, int bufsize,
216
+ int width) {
217
+ std::string fmt = format_fwf<Int96Type>(width);
218
+ std::string result = Int96ToString(*reinterpret_cast<Int96*>(val));
219
+ snprintf(buffer, bufsize, fmt.c_str(), result.c_str());
220
+ }
221
+
222
+ template <>
223
+ inline void TypedScanner<ByteArrayType>::FormatValue(void* val, char* buffer, int bufsize,
224
+ int width) {
225
+ std::string fmt = format_fwf<ByteArrayType>(width);
226
+ std::string result = ByteArrayToString(*reinterpret_cast<ByteArray*>(val));
227
+ snprintf(buffer, bufsize, fmt.c_str(), result.c_str());
228
+ }
229
+
230
+ template <>
231
+ inline void TypedScanner<FLBAType>::FormatValue(void* val, char* buffer, int bufsize,
232
+ int width) {
233
+ std::string fmt = format_fwf<FLBAType>(width);
234
+ std::string result = FixedLenByteArrayToString(
235
+ *reinterpret_cast<FixedLenByteArray*>(val), descr()->type_length());
236
+ snprintf(buffer, bufsize, fmt.c_str(), result.c_str());
237
+ }
238
+
239
+ typedef TypedScanner<BooleanType> BoolScanner;
240
+ typedef TypedScanner<Int32Type> Int32Scanner;
241
+ typedef TypedScanner<Int64Type> Int64Scanner;
242
+ typedef TypedScanner<Int96Type> Int96Scanner;
243
+ typedef TypedScanner<FloatType> FloatScanner;
244
+ typedef TypedScanner<DoubleType> DoubleScanner;
245
+ typedef TypedScanner<ByteArrayType> ByteArrayScanner;
246
+ typedef TypedScanner<FLBAType> FixedLenByteArrayScanner;
247
+
248
+ template <typename RType>
249
+ int64_t ScanAll(int32_t batch_size, int16_t* def_levels, int16_t* rep_levels,
250
+ uint8_t* values, int64_t* values_buffered,
251
+ parquet::ColumnReader* reader) {
252
+ typedef typename RType::T Type;
253
+ auto typed_reader = static_cast<RType*>(reader);
254
+ auto vals = reinterpret_cast<Type*>(&values[0]);
255
+ return typed_reader->ReadBatch(batch_size, def_levels, rep_levels, vals,
256
+ values_buffered);
257
+ }
258
+
259
+ int64_t PARQUET_EXPORT ScanAllValues(int32_t batch_size, int16_t* def_levels,
260
+ int16_t* rep_levels, uint8_t* values,
261
+ int64_t* values_buffered,
262
+ parquet::ColumnReader* reader);
263
+
264
+ } // namespace parquet
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/column_writer.h ADDED
@@ -0,0 +1,317 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <cstdint>
21
+ #include <cstring>
22
+ #include <memory>
23
+
24
+ #include "arrow/type_fwd.h"
25
+ #include "arrow/util/compression.h"
26
+ #include "parquet/exception.h"
27
+ #include "parquet/platform.h"
28
+ #include "parquet/types.h"
29
+
30
+ namespace arrow {
31
+
32
+ class Array;
33
+
34
+ namespace bit_util {
35
+ class BitWriter;
36
+ } // namespace bit_util
37
+
38
+ namespace util {
39
+ class RleBitPackedEncoder;
40
+ class CodecOptions;
41
+ } // namespace util
42
+
43
+ } // namespace arrow
44
+
45
+ namespace parquet {
46
+
47
+ struct ArrowWriteContext;
48
+ class ColumnChunkMetaDataBuilder;
49
+ class ColumnDescriptor;
50
+ class ColumnIndexBuilder;
51
+ class DataPage;
52
+ class DictionaryPage;
53
+ class Encryptor;
54
+ class OffsetIndexBuilder;
55
+ class WriterProperties;
56
+
57
+ class PARQUET_EXPORT LevelEncoder {
58
+ public:
59
+ LevelEncoder();
60
+ ~LevelEncoder();
61
+
62
+ static int MaxBufferSize(Encoding::type encoding, int16_t max_level,
63
+ int num_buffered_values);
64
+
65
+ // Initialize the LevelEncoder.
66
+ void Init(Encoding::type encoding, int16_t max_level, int num_buffered_values,
67
+ uint8_t* data, int data_size);
68
+
69
+ // Encodes a batch of levels from an array and returns the number of levels encoded
70
+ int Encode(int batch_size, const int16_t* levels);
71
+
72
+ int32_t len() {
73
+ if (encoding_ != Encoding::RLE) {
74
+ throw ParquetException("Only implemented for RLE encoding");
75
+ }
76
+ return rle_length_;
77
+ }
78
+
79
+ private:
80
+ int bit_width_;
81
+ int rle_length_;
82
+ Encoding::type encoding_;
83
+ std::unique_ptr<::arrow::util::RleBitPackedEncoder> rle_encoder_;
84
+ std::unique_ptr<::arrow::bit_util::BitWriter> bit_packed_encoder_;
85
+ };
86
+
87
+ class PARQUET_EXPORT PageWriter {
88
+ public:
89
+ virtual ~PageWriter() {}
90
+
91
+ static std::unique_ptr<PageWriter> Open(
92
+ std::shared_ptr<ArrowOutputStream> sink, Compression::type codec,
93
+ ColumnChunkMetaDataBuilder* metadata, int16_t row_group_ordinal = -1,
94
+ int16_t column_chunk_ordinal = -1,
95
+ ::arrow::MemoryPool* pool = ::arrow::default_memory_pool(),
96
+ bool buffered_row_group = false,
97
+ std::shared_ptr<Encryptor> header_encryptor = NULLPTR,
98
+ std::shared_ptr<Encryptor> data_encryptor = NULLPTR,
99
+ bool page_write_checksum_enabled = false,
100
+ // column_index_builder MUST outlive the PageWriter
101
+ ColumnIndexBuilder* column_index_builder = NULLPTR,
102
+ // offset_index_builder MUST outlive the PageWriter
103
+ OffsetIndexBuilder* offset_index_builder = NULLPTR,
104
+ const CodecOptions& codec_options = CodecOptions{});
105
+
106
+ // The Column Writer decides if dictionary encoding is used if set and
107
+ // if the dictionary encoding has fallen back to default encoding on reaching dictionary
108
+ // page limit
109
+ virtual void Close(bool has_dictionary, bool fallback) = 0;
110
+
111
+ // Return the number of uncompressed bytes written (including header size)
112
+ virtual int64_t WriteDataPage(const DataPage& page) = 0;
113
+
114
+ // Return the number of uncompressed bytes written (including header size)
115
+ virtual int64_t WriteDictionaryPage(const DictionaryPage& page) = 0;
116
+
117
+ /// \brief The total number of bytes written as serialized data and
118
+ /// dictionary pages to the sink so far.
119
+ virtual int64_t total_compressed_bytes_written() const = 0;
120
+
121
+ virtual bool has_compressor() = 0;
122
+
123
+ virtual void Compress(const Buffer& src_buffer, ResizableBuffer* dest_buffer) = 0;
124
+ };
125
+
126
+ class PARQUET_EXPORT ColumnWriter {
127
+ public:
128
+ virtual ~ColumnWriter() = default;
129
+
130
+ static std::shared_ptr<ColumnWriter> Make(ColumnChunkMetaDataBuilder*,
131
+ std::unique_ptr<PageWriter>,
132
+ const WriterProperties* properties,
133
+ BloomFilter* bloom_filter = NULLPTR);
134
+
135
+ /// \brief Closes the ColumnWriter, commits any buffered values to pages.
136
+ /// \return Total size of the column in bytes
137
+ virtual int64_t Close() = 0;
138
+
139
+ /// \brief The physical Parquet type of the column
140
+ virtual Type::type type() const = 0;
141
+
142
+ /// \brief The schema for the column
143
+ virtual const ColumnDescriptor* descr() const = 0;
144
+
145
+ /// \brief The number of rows written so far
146
+ virtual int64_t rows_written() const = 0;
147
+
148
+ /// \brief The total size of the compressed pages + page headers. Values
149
+ /// are still buffered and not written to a pager yet
150
+ ///
151
+ /// So in un-buffered mode, it always returns 0
152
+ virtual int64_t total_compressed_bytes() const = 0;
153
+
154
+ /// \brief The total number of bytes written as serialized data and
155
+ /// dictionary pages to the ColumnChunk so far
156
+ /// These bytes are uncompressed bytes.
157
+ virtual int64_t total_bytes_written() const = 0;
158
+
159
+ /// \brief The total number of bytes written as serialized data and
160
+ /// dictionary pages to the ColumnChunk so far.
161
+ /// If the column is uncompressed, the value would be equal to
162
+ /// total_bytes_written().
163
+ virtual int64_t total_compressed_bytes_written() const = 0;
164
+
165
+ /// \brief Estimated size of the values that are not written to a page yet.
166
+ virtual int64_t estimated_buffered_value_bytes() const = 0;
167
+
168
+ /// \brief Estimated size of the definition levels that are not written to a page yet.
169
+ virtual int64_t estimated_buffered_def_level_bytes() const = 0;
170
+
171
+ /// \brief Estimated size of the repetition levels that are not written to a page yet.
172
+ virtual int64_t estimated_buffered_rep_level_bytes() const = 0;
173
+
174
+ /// \brief Estimated size of the dictionary that are not written to a page yet.
175
+ virtual int64_t estimated_buffered_dict_bytes() const = 0;
176
+
177
+ /// \brief The file-level writer properties
178
+ virtual const WriterProperties* properties() = 0;
179
+
180
+ /// \brief Add key-value metadata to the ColumnChunk.
181
+ /// \param[in] key_value_metadata the metadata to add.
182
+ /// \note This will overwrite any existing metadata with the same key.
183
+ /// \throw ParquetException if Close() has been called.
184
+ virtual void AddKeyValueMetadata(
185
+ const std::shared_ptr<const ::arrow::KeyValueMetadata>& key_value_metadata) = 0;
186
+
187
+ /// \brief Reset the ColumnChunk key-value metadata.
188
+ /// \throw ParquetException if Close() has been called.
189
+ virtual void ResetKeyValueMetadata() = 0;
190
+
191
+ /// \brief Write Apache Arrow columnar data directly to ColumnWriter. Returns
192
+ /// error status if the array data type is not compatible with the concrete
193
+ /// writer type.
194
+ ///
195
+ /// leaf_array is always a primitive (possibly dictionary encoded type).
196
+ /// Leaf_field_nullable indicates whether the leaf array is considered nullable
197
+ /// according to its schema in a Table or its parent array.
198
+ virtual ::arrow::Status WriteArrow(const int16_t* def_levels, const int16_t* rep_levels,
199
+ int64_t num_levels, const ::arrow::Array& leaf_array,
200
+ ArrowWriteContext* ctx,
201
+ bool leaf_field_nullable) = 0;
202
+ };
203
+
204
+ // API to write values to a single column. This is the main client facing API.
205
+ template <typename DType>
206
+ class TypedColumnWriter : public ColumnWriter {
207
+ public:
208
+ using T = typename DType::c_type;
209
+
210
+ // Write a batch of repetition levels, definition levels, and values to the
211
+ // column.
212
+ // `num_values` is the number of logical leaf values.
213
+ // `def_levels` (resp. `rep_levels`) can be null if the column's max definition level
214
+ // (resp. max repetition level) is 0.
215
+ // If not null, each of `def_levels` and `rep_levels` must have at least
216
+ // `num_values`.
217
+ //
218
+ // The number of physical values written (taken from `values`) is returned.
219
+ // It can be smaller than `num_values` is there are some undefined values.
220
+ virtual int64_t WriteBatch(int64_t num_values, const int16_t* def_levels,
221
+ const int16_t* rep_levels, const T* values) = 0;
222
+
223
+ /// Write a batch of repetition levels, definition levels, and values to the
224
+ /// column.
225
+ ///
226
+ /// In comparison to WriteBatch the length of repetition and definition levels
227
+ /// is the same as of the number of values read for max_definition_level == 1.
228
+ /// In the case of max_definition_level > 1, the repetition and definition
229
+ /// levels are larger than the values but the values include the null entries
230
+ /// with definition_level == (max_definition_level - 1). Thus we have to differentiate
231
+ /// in the parameters of this function if the input has the length of num_values or the
232
+ /// _number of rows in the lowest nesting level_.
233
+ ///
234
+ /// In the case that the most inner node in the Parquet is required, the _number of rows
235
+ /// in the lowest nesting level_ is equal to the number of non-null values. If the
236
+ /// inner-most schema node is optional, the _number of rows in the lowest nesting level_
237
+ /// also includes all values with definition_level == (max_definition_level - 1).
238
+ ///
239
+ /// @param num_values number of levels to write.
240
+ /// @param def_levels The Parquet definition levels, length is num_values
241
+ /// @param rep_levels The Parquet repetition levels, length is num_values
242
+ /// @param valid_bits Bitmap that indicates if the row is null on the lowest nesting
243
+ /// level. The length is number of rows in the lowest nesting level.
244
+ /// @param valid_bits_offset The offset in bits of the valid_bits where the
245
+ /// first relevant bit resides.
246
+ /// @param values The values in the lowest nested level including
247
+ /// spacing for nulls on the lowest levels; input has the length
248
+ /// of the number of rows on the lowest nesting level.
249
+ virtual void WriteBatchSpaced(int64_t num_values, const int16_t* def_levels,
250
+ const int16_t* rep_levels, const uint8_t* valid_bits,
251
+ int64_t valid_bits_offset, const T* values) = 0;
252
+ };
253
+
254
+ using BoolWriter = TypedColumnWriter<BooleanType>;
255
+ using Int32Writer = TypedColumnWriter<Int32Type>;
256
+ using Int64Writer = TypedColumnWriter<Int64Type>;
257
+ using Int96Writer = TypedColumnWriter<Int96Type>;
258
+ using FloatWriter = TypedColumnWriter<FloatType>;
259
+ using DoubleWriter = TypedColumnWriter<DoubleType>;
260
+ using ByteArrayWriter = TypedColumnWriter<ByteArrayType>;
261
+ using FixedLenByteArrayWriter = TypedColumnWriter<FLBAType>;
262
+
263
+ namespace internal {
264
+
265
+ /**
266
+ * Timestamp conversion constants
267
+ */
268
+ constexpr int64_t kJulianEpochOffsetDays = INT64_C(2440588);
269
+
270
+ template <int64_t UnitPerDay, int64_t NanosecondsPerUnit>
271
+ inline void ArrowTimestampToImpalaTimestamp(const int64_t time, Int96* impala_timestamp) {
272
+ auto julian_days = static_cast<int32_t>(time / UnitPerDay + kJulianEpochOffsetDays);
273
+ int64_t last_day_units = time % UnitPerDay;
274
+ if (last_day_units < 0) {
275
+ --julian_days;
276
+ last_day_units += UnitPerDay;
277
+ }
278
+ impala_timestamp->value[2] = static_cast<uint32_t>(julian_days);
279
+ uint64_t last_day_nanos = static_cast<uint64_t>(last_day_units) * NanosecondsPerUnit;
280
+ // impala_timestamp will be unaligned every other entry so do memcpy instead
281
+ // of assign and reinterpret cast to avoid undefined behavior.
282
+ std::memcpy(impala_timestamp, &last_day_nanos, sizeof(uint64_t));
283
+ }
284
+
285
+ constexpr int64_t kSecondsInNanos = INT64_C(1000000000);
286
+
287
+ inline void SecondsToImpalaTimestamp(const int64_t seconds, Int96* impala_timestamp) {
288
+ ArrowTimestampToImpalaTimestamp<kSecondsPerDay, kSecondsInNanos>(seconds,
289
+ impala_timestamp);
290
+ }
291
+
292
+ constexpr int64_t kMillisecondsInNanos = kSecondsInNanos / INT64_C(1000);
293
+
294
+ inline void MillisecondsToImpalaTimestamp(const int64_t milliseconds,
295
+ Int96* impala_timestamp) {
296
+ ArrowTimestampToImpalaTimestamp<kMillisecondsPerDay, kMillisecondsInNanos>(
297
+ milliseconds, impala_timestamp);
298
+ }
299
+
300
+ constexpr int64_t kMicrosecondsInNanos = kMillisecondsInNanos / INT64_C(1000);
301
+
302
+ inline void MicrosecondsToImpalaTimestamp(const int64_t microseconds,
303
+ Int96* impala_timestamp) {
304
+ ArrowTimestampToImpalaTimestamp<kMicrosecondsPerDay, kMicrosecondsInNanos>(
305
+ microseconds, impala_timestamp);
306
+ }
307
+
308
+ constexpr int64_t kNanosecondsInNanos = INT64_C(1);
309
+
310
+ inline void NanosecondsToImpalaTimestamp(const int64_t nanoseconds,
311
+ Int96* impala_timestamp) {
312
+ ArrowTimestampToImpalaTimestamp<kNanosecondsPerDay, kNanosecondsInNanos>(
313
+ nanoseconds, impala_timestamp);
314
+ }
315
+
316
+ } // namespace internal
317
+ } // namespace parquet
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encoding.h ADDED
@@ -0,0 +1,476 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <cstdint>
21
+ #include <cstring>
22
+ #include <memory>
23
+ #include <vector>
24
+
25
+ #include "arrow/type_fwd.h"
26
+
27
+ #include "parquet/exception.h"
28
+ #include "parquet/platform.h"
29
+ #include "parquet/types.h"
30
+
31
+ namespace arrow {
32
+ template <typename T>
33
+ class Dictionary32Builder;
34
+ }
35
+
36
+ namespace parquet {
37
+
38
+ template <typename DType>
39
+ class TypedEncoder;
40
+
41
+ using BooleanEncoder = TypedEncoder<BooleanType>;
42
+ using Int32Encoder = TypedEncoder<Int32Type>;
43
+ using Int64Encoder = TypedEncoder<Int64Type>;
44
+ using Int96Encoder = TypedEncoder<Int96Type>;
45
+ using FloatEncoder = TypedEncoder<FloatType>;
46
+ using DoubleEncoder = TypedEncoder<DoubleType>;
47
+ using ByteArrayEncoder = TypedEncoder<ByteArrayType>;
48
+ using FLBAEncoder = TypedEncoder<FLBAType>;
49
+
50
+ template <typename DType>
51
+ class TypedDecoder;
52
+
53
+ class BooleanDecoder;
54
+ using Int32Decoder = TypedDecoder<Int32Type>;
55
+ using Int64Decoder = TypedDecoder<Int64Type>;
56
+ using Int96Decoder = TypedDecoder<Int96Type>;
57
+ using FloatDecoder = TypedDecoder<FloatType>;
58
+ using DoubleDecoder = TypedDecoder<DoubleType>;
59
+ using ByteArrayDecoder = TypedDecoder<ByteArrayType>;
60
+ class FLBADecoder;
61
+
62
+ template <typename T>
63
+ struct EncodingTraits;
64
+
65
+ template <>
66
+ struct EncodingTraits<BooleanType> {
67
+ using Encoder = BooleanEncoder;
68
+ using Decoder = BooleanDecoder;
69
+
70
+ using ArrowType = ::arrow::BooleanType;
71
+ using Accumulator = ::arrow::BooleanBuilder;
72
+ struct DictAccumulator {};
73
+ };
74
+
75
+ template <>
76
+ struct EncodingTraits<Int32Type> {
77
+ using Encoder = Int32Encoder;
78
+ using Decoder = Int32Decoder;
79
+
80
+ using ArrowType = ::arrow::Int32Type;
81
+ using Accumulator = ::arrow::NumericBuilder<::arrow::Int32Type>;
82
+ using DictAccumulator = ::arrow::Dictionary32Builder<::arrow::Int32Type>;
83
+ };
84
+
85
+ template <>
86
+ struct EncodingTraits<Int64Type> {
87
+ using Encoder = Int64Encoder;
88
+ using Decoder = Int64Decoder;
89
+
90
+ using ArrowType = ::arrow::Int64Type;
91
+ using Accumulator = ::arrow::NumericBuilder<::arrow::Int64Type>;
92
+ using DictAccumulator = ::arrow::Dictionary32Builder<::arrow::Int64Type>;
93
+ };
94
+
95
+ template <>
96
+ struct EncodingTraits<Int96Type> {
97
+ using Encoder = Int96Encoder;
98
+ using Decoder = Int96Decoder;
99
+
100
+ struct Accumulator {};
101
+ struct DictAccumulator {};
102
+ };
103
+
104
+ template <>
105
+ struct EncodingTraits<FloatType> {
106
+ using Encoder = FloatEncoder;
107
+ using Decoder = FloatDecoder;
108
+
109
+ using ArrowType = ::arrow::FloatType;
110
+ using Accumulator = ::arrow::NumericBuilder<::arrow::FloatType>;
111
+ using DictAccumulator = ::arrow::Dictionary32Builder<::arrow::FloatType>;
112
+ };
113
+
114
+ template <>
115
+ struct EncodingTraits<DoubleType> {
116
+ using Encoder = DoubleEncoder;
117
+ using Decoder = DoubleDecoder;
118
+
119
+ using ArrowType = ::arrow::DoubleType;
120
+ using Accumulator = ::arrow::NumericBuilder<::arrow::DoubleType>;
121
+ using DictAccumulator = ::arrow::Dictionary32Builder<::arrow::DoubleType>;
122
+ };
123
+
124
+ template <>
125
+ struct EncodingTraits<ByteArrayType> {
126
+ using Encoder = ByteArrayEncoder;
127
+ using Decoder = ByteArrayDecoder;
128
+
129
+ /// \brief Internal helper class for decoding BYTE_ARRAY data
130
+ ///
131
+ /// This class allows the caller to choose the concrete Arrow data type
132
+ /// by passing a corresponding `ArrayBuilder`.
133
+ /// Supported `ArrayBuilder` classes are `BinaryBuilder`, `LargeBinaryBuilder`
134
+ /// and `BinaryViewBuilder`.
135
+ /// If the builder is a `BinaryBuilder`, `chunks` can accumulate several
136
+ /// arrays as needed to work around the 32-bit offset limit.
137
+ struct Accumulator {
138
+ std::unique_ptr<::arrow::ArrayBuilder> builder;
139
+ std::vector<std::shared_ptr<::arrow::Array>> chunks;
140
+ };
141
+ using DictAccumulator = ::arrow::Dictionary32Builder<::arrow::BinaryType>;
142
+ };
143
+
144
+ template <>
145
+ struct EncodingTraits<FLBAType> {
146
+ using Encoder = FLBAEncoder;
147
+ using Decoder = FLBADecoder;
148
+
149
+ using ArrowType = ::arrow::FixedSizeBinaryType;
150
+ using Accumulator = ::arrow::FixedSizeBinaryBuilder;
151
+ using DictAccumulator = ::arrow::Dictionary32Builder<::arrow::FixedSizeBinaryType>;
152
+ };
153
+
154
+ class ColumnDescriptor;
155
+
156
+ // Untyped base for all encoders
157
+ class Encoder {
158
+ public:
159
+ virtual ~Encoder() = default;
160
+
161
+ virtual int64_t EstimatedDataEncodedSize() = 0;
162
+ virtual std::shared_ptr<Buffer> FlushValues() = 0;
163
+ virtual Encoding::type encoding() const = 0;
164
+
165
+ virtual void Put(const ::arrow::Array& values) = 0;
166
+
167
+ // Report the number of bytes written to the encoder since the last report.
168
+ // It only works for BYTE_ARRAY type and throw for other types.
169
+ // This call is not idempotent since it resets the internal counter.
170
+ virtual int64_t ReportUnencodedDataBytes() = 0;
171
+
172
+ virtual MemoryPool* memory_pool() const = 0;
173
+ };
174
+
175
+ // Base class for value encoders. Since encoders may or not have state (e.g.,
176
+ // dictionary encoding) we use a class instance to maintain any state.
177
+ //
178
+ // Encode interfaces are internal, subject to change without deprecation.
179
+ template <typename DType>
180
+ class TypedEncoder : virtual public Encoder {
181
+ public:
182
+ using T = typename DType::c_type;
183
+
184
+ using Encoder::Put;
185
+
186
+ virtual void Put(const T* src, int num_values) = 0;
187
+
188
+ virtual void Put(const std::vector<T>& src, int num_values = -1);
189
+
190
+ virtual void PutSpaced(const T* src, int num_values, const uint8_t* valid_bits,
191
+ int64_t valid_bits_offset) = 0;
192
+ };
193
+
194
+ template <typename DType>
195
+ void TypedEncoder<DType>::Put(const std::vector<T>& src, int num_values) {
196
+ if (num_values == -1) {
197
+ num_values = static_cast<int>(src.size());
198
+ }
199
+ Put(src.data(), num_values);
200
+ }
201
+
202
+ template <>
203
+ inline void TypedEncoder<BooleanType>::Put(const std::vector<bool>& src, int num_values) {
204
+ // NOTE(wesm): This stub is here only to satisfy the compiler; it is
205
+ // overridden later with the actual implementation
206
+ }
207
+
208
+ // Base class for dictionary encoders
209
+ template <typename DType>
210
+ class DictEncoder : virtual public TypedEncoder<DType> {
211
+ public:
212
+ /// Writes out any buffered indices to buffer preceded by the bit width of this data.
213
+ /// Returns the number of bytes written.
214
+ /// If the supplied buffer is not big enough, returns -1.
215
+ /// buffer must be preallocated with buffer_len bytes. Use EstimatedDataEncodedSize()
216
+ /// to size buffer.
217
+ virtual int WriteIndices(uint8_t* buffer, int buffer_len) = 0;
218
+
219
+ virtual int dict_encoded_size() const = 0;
220
+
221
+ virtual int bit_width() const = 0;
222
+
223
+ /// Writes out the encoded dictionary to buffer. buffer must be preallocated to
224
+ /// dict_encoded_size() bytes.
225
+ virtual void WriteDict(uint8_t* buffer) const = 0;
226
+
227
+ virtual int num_entries() const = 0;
228
+
229
+ /// \brief EXPERIMENTAL: Append dictionary indices into the encoder. It is
230
+ /// assumed (without any boundschecking) that the indices reference
231
+ /// preexisting dictionary values
232
+ /// \param[in] indices the dictionary index values. Only Int32Array currently
233
+ /// supported
234
+ virtual void PutIndices(const ::arrow::Array& indices) = 0;
235
+
236
+ /// \brief EXPERIMENTAL: Append dictionary into encoder, inserting indices
237
+ /// separately. Currently throws exception if the current dictionary memo is
238
+ /// non-empty
239
+ /// \param[in] values the dictionary values. Only valid for certain
240
+ /// Parquet/Arrow type combinations, like BYTE_ARRAY/BinaryArray
241
+ virtual void PutDictionary(const ::arrow::Array& values) = 0;
242
+ };
243
+
244
+ // ----------------------------------------------------------------------
245
+ // Value decoding
246
+
247
+ class Decoder {
248
+ public:
249
+ virtual ~Decoder() = default;
250
+
251
+ // Sets the data for a new page. This will be called multiple times on the same
252
+ // decoder and should reset all internal state.
253
+ //
254
+ // `num_values` comes from the data page header, and may be greater than the number of
255
+ // physical values in the data buffer if there are some omitted (null) values.
256
+ // `len`, on the other hand, is the size in bytes of the data buffer and
257
+ // directly relates to the number of physical values.
258
+ virtual void SetData(int num_values, const uint8_t* data, int len) = 0;
259
+
260
+ // Returns the number of values left (for the last call to SetData()). This is
261
+ // the number of values left in this page.
262
+ virtual int values_left() const = 0;
263
+ virtual Encoding::type encoding() const = 0;
264
+ };
265
+
266
+ template <typename DType>
267
+ class TypedDecoder : virtual public Decoder {
268
+ public:
269
+ using T = typename DType::c_type;
270
+
271
+ /// \brief Decode values into a buffer
272
+ ///
273
+ /// Subclasses may override the more specialized Decode methods below.
274
+ ///
275
+ /// \param[in] buffer destination for decoded values
276
+ /// \param[in] max_values maximum number of values to decode
277
+ /// \return The number of values decoded. Should be identical to max_values except
278
+ /// at the end of the current data page.
279
+ virtual int Decode(T* buffer, int max_values) = 0;
280
+
281
+ /// \brief Decode the values in this data page but leave spaces for null entries.
282
+ ///
283
+ /// \param[in] buffer destination for decoded values
284
+ /// \param[in] num_values size of the def_levels and buffer arrays including the number
285
+ /// of null slots
286
+ /// \param[in] null_count number of null slots
287
+ /// \param[in] valid_bits bitmap data indicating position of valid slots
288
+ /// \param[in] valid_bits_offset offset into valid_bits
289
+ /// \return The number of values decoded, including nulls.
290
+ virtual int DecodeSpaced(T* buffer, int num_values, int null_count,
291
+ const uint8_t* valid_bits, int64_t valid_bits_offset) = 0;
292
+
293
+ /// \brief Decode into an ArrayBuilder or other accumulator
294
+ ///
295
+ /// \param[in] num_values number of values to decode, including null slots
296
+ /// \param[in] null_count number of null slots
297
+ /// \param[in] valid_bits validity bitmap
298
+ /// \param[in] valid_bits_offset bit offset to start into the validity bitmap
299
+ /// \param[in] out accumulator to decode into
300
+ ///
301
+ /// This function assumes the definition levels were already decoded
302
+ /// as a validity bitmap in the given `valid_bits`. `null_count`
303
+ /// is the number of 0s in `valid_bits`.
304
+ /// `valid_bits` must at least `valid_bits_offset + num_values` bits.
305
+ /// As a space optimization, it is allowed for `valid_bits` to be null
306
+ /// if `null_count` is zero.
307
+ /// This function throws a ParquetException if there are less than
308
+ /// `num_values - null_count` values left to decode.
309
+ ///
310
+ /// \return The number of non-null values decoded
311
+ virtual int DecodeArrow(int num_values, int null_count, const uint8_t* valid_bits,
312
+ int64_t valid_bits_offset,
313
+ typename EncodingTraits<DType>::Accumulator* out) = 0;
314
+
315
+ /// \brief Decode into an ArrayBuilder or other accumulator ignoring nulls
316
+ ///
317
+ /// \param[in] num_values number of values to decode
318
+ /// \param[in] out accumulator to decode into
319
+ ///
320
+ /// \return The number of values decoded
321
+ int DecodeArrowNonNull(int num_values,
322
+ typename EncodingTraits<DType>::Accumulator* out) {
323
+ return DecodeArrow(num_values, 0, /*valid_bits=*/NULLPTR, 0, out);
324
+ }
325
+
326
+ /// \brief Decode into a DictionaryBuilder
327
+ ///
328
+ /// This function assumes the definition levels were already decoded
329
+ /// as a validity bitmap in the given `valid_bits`. `null_count`
330
+ /// is the number of 0s in `valid_bits`.
331
+ /// As a space optimization, it is allowed for `valid_bits` to be null
332
+ /// if `null_count` is zero.
333
+ ///
334
+ /// \return number of values decoded
335
+ virtual int DecodeArrow(int num_values, int null_count, const uint8_t* valid_bits,
336
+ int64_t valid_bits_offset,
337
+ typename EncodingTraits<DType>::DictAccumulator* builder) = 0;
338
+
339
+ /// \brief Decode into a DictionaryBuilder ignoring nulls
340
+ ///
341
+ /// \return number of values decoded
342
+ int DecodeArrowNonNull(int num_values,
343
+ typename EncodingTraits<DType>::DictAccumulator* builder) {
344
+ return DecodeArrow(num_values, 0, /*valid_bits=*/NULLPTR, 0, builder);
345
+ }
346
+ };
347
+
348
+ template <typename DType>
349
+ class DictDecoder : virtual public TypedDecoder<DType> {
350
+ public:
351
+ using T = typename DType::c_type;
352
+
353
+ virtual void SetDict(TypedDecoder<DType>* dictionary) = 0;
354
+
355
+ /// \brief Insert dictionary values into the Arrow dictionary builder's memo,
356
+ /// but do not append any indices
357
+ virtual void InsertDictionary(::arrow::ArrayBuilder* builder) = 0;
358
+
359
+ /// \brief Decode only dictionary indices and append to dictionary
360
+ /// builder. The builder must have had the dictionary from this decoder
361
+ /// inserted already.
362
+ ///
363
+ /// \warning Remember to reset the builder each time the dict decoder is initialized
364
+ /// with a new dictionary page
365
+ virtual int DecodeIndicesSpaced(int num_values, int null_count,
366
+ const uint8_t* valid_bits, int64_t valid_bits_offset,
367
+ ::arrow::ArrayBuilder* builder) = 0;
368
+
369
+ /// \brief Decode only dictionary indices (no nulls)
370
+ ///
371
+ /// \warning Remember to reset the builder each time the dict decoder is initialized
372
+ /// with a new dictionary page
373
+ virtual int DecodeIndices(int num_values, ::arrow::ArrayBuilder* builder) = 0;
374
+
375
+ /// \brief Decode only dictionary indices (no nulls). Same as above
376
+ /// DecodeIndices but target is an array instead of a builder.
377
+ ///
378
+ /// \note API EXPERIMENTAL
379
+ virtual int DecodeIndices(int num_values, int32_t* indices) = 0;
380
+
381
+ /// \brief Get dictionary. The reader will call this API when it encounters a
382
+ /// new dictionary.
383
+ ///
384
+ /// @param[out] dictionary The pointer to dictionary values. Dictionary is owned by
385
+ /// the decoder and is destroyed when the decoder is destroyed.
386
+ /// @param[out] dictionary_length The dictionary length.
387
+ ///
388
+ /// \note API EXPERIMENTAL
389
+ virtual void GetDictionary(const T** dictionary, int32_t* dictionary_length) = 0;
390
+ };
391
+
392
+ // ----------------------------------------------------------------------
393
+ // TypedEncoder specializations, traits, and factory functions
394
+
395
+ class BooleanDecoder : virtual public TypedDecoder<BooleanType> {
396
+ public:
397
+ using TypedDecoder<BooleanType>::Decode;
398
+
399
+ /// \brief Decode and bit-pack values into a buffer
400
+ ///
401
+ /// \param[in] buffer destination for decoded values
402
+ /// This buffer will contain bit-packed values. If
403
+ /// max_values is not a multiple of 8, the trailing bits
404
+ /// of the last byte will be undefined.
405
+ /// \param[in] max_values max values to decode.
406
+ /// \return The number of values decoded. Should be identical to max_values except
407
+ /// at the end of the current data page.
408
+ virtual int Decode(uint8_t* buffer, int max_values) = 0;
409
+ };
410
+
411
+ class FLBADecoder : virtual public TypedDecoder<FLBAType> {
412
+ public:
413
+ using TypedDecoder<FLBAType>::DecodeSpaced;
414
+
415
+ // TODO(wesm): As possible follow-up to PARQUET-1508, we should examine if
416
+ // there is value in adding specialized read methods for
417
+ // FIXED_LEN_BYTE_ARRAY. If only Decimal data can occur with this data type
418
+ // then perhaps not
419
+ };
420
+
421
+ PARQUET_EXPORT
422
+ std::unique_ptr<Encoder> MakeEncoder(
423
+ Type::type type_num, Encoding::type encoding, bool use_dictionary = false,
424
+ const ColumnDescriptor* descr = NULLPTR,
425
+ ::arrow::MemoryPool* pool = ::arrow::default_memory_pool());
426
+
427
+ template <typename DType>
428
+ std::unique_ptr<typename EncodingTraits<DType>::Encoder> MakeTypedEncoder(
429
+ Encoding::type encoding, bool use_dictionary = false,
430
+ const ColumnDescriptor* descr = NULLPTR,
431
+ ::arrow::MemoryPool* pool = ::arrow::default_memory_pool()) {
432
+ using OutType = typename EncodingTraits<DType>::Encoder;
433
+ std::unique_ptr<Encoder> base =
434
+ MakeEncoder(DType::type_num, encoding, use_dictionary, descr, pool);
435
+ return std::unique_ptr<OutType>(dynamic_cast<OutType*>(base.release()));
436
+ }
437
+
438
+ PARQUET_EXPORT
439
+ std::unique_ptr<Decoder> MakeDecoder(
440
+ Type::type type_num, Encoding::type encoding, const ColumnDescriptor* descr = NULLPTR,
441
+ ::arrow::MemoryPool* pool = ::arrow::default_memory_pool());
442
+
443
+ namespace detail {
444
+
445
+ PARQUET_EXPORT
446
+ std::unique_ptr<Decoder> MakeDictDecoder(Type::type type_num,
447
+ const ColumnDescriptor* descr,
448
+ ::arrow::MemoryPool* pool);
449
+
450
+ } // namespace detail
451
+
452
+ template <typename DType>
453
+ std::unique_ptr<DictDecoder<DType>> MakeDictDecoder(
454
+ const ColumnDescriptor* descr = NULLPTR,
455
+ ::arrow::MemoryPool* pool = ::arrow::default_memory_pool()) {
456
+ using OutType = DictDecoder<DType>;
457
+ auto decoder = detail::MakeDictDecoder(DType::type_num, descr, pool);
458
+ return std::unique_ptr<OutType>(dynamic_cast<OutType*>(decoder.release()));
459
+ }
460
+
461
+ template <typename DType>
462
+ std::unique_ptr<typename EncodingTraits<DType>::Decoder> MakeTypedDecoder(
463
+ Encoding::type encoding, const ColumnDescriptor* descr = NULLPTR,
464
+ ::arrow::MemoryPool* pool = ::arrow::default_memory_pool()) {
465
+ using OutType = typename EncodingTraits<DType>::Decoder;
466
+ std::unique_ptr<Decoder> base = MakeDecoder(DType::type_num, encoding, descr, pool);
467
+ return std::unique_ptr<OutType>(dynamic_cast<OutType*>(base.release()));
468
+ }
469
+
470
+ /// Return the list of supported encodings for the given physical type
471
+ ///
472
+ /// Only non-dictionary encodings are returned.
473
+ PARQUET_EXPORT
474
+ std::vector<Encoding::type> SupportedEncodings(Type::type physical_type);
475
+
476
+ } // namespace parquet
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/crypto_factory.h ADDED
@@ -0,0 +1,152 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <memory>
21
+
22
+ #include "parquet/encryption/encryption.h"
23
+ #include "parquet/encryption/file_key_wrapper.h"
24
+ #include "parquet/encryption/key_toolkit.h"
25
+ #include "parquet/encryption/kms_client_factory.h"
26
+ #include "parquet/platform.h"
27
+
28
+ namespace parquet::encryption {
29
+
30
+ static constexpr ParquetCipher::type kDefaultEncryptionAlgorithm =
31
+ ParquetCipher::AES_GCM_V1;
32
+ static constexpr bool kDefaultPlaintextFooter = false;
33
+ static constexpr bool kDefaultDoubleWrapping = true;
34
+ static constexpr double kDefaultCacheLifetimeSeconds = 600; // 10 minutes
35
+ static constexpr bool kDefaultInternalKeyMaterial = true;
36
+ static constexpr bool kDefaultUniformEncryption = false;
37
+ static constexpr int32_t kDefaultDataKeyLengthBits = 128;
38
+
39
+ struct PARQUET_EXPORT EncryptionConfiguration {
40
+ explicit EncryptionConfiguration(const std::string& footer_key)
41
+ : footer_key(footer_key) {}
42
+
43
+ /// ID of the master key for footer encryption/signing
44
+ std::string footer_key;
45
+
46
+ /// List of columns to encrypt, with column master key IDs (see HIVE-21848).
47
+ /// Format: "columnKeyID:colName,colName;columnKeyID:colName..."
48
+ /// Either
49
+ /// (1) column_keys must be set
50
+ /// or
51
+ /// (2) uniform_encryption must be set to true
52
+ /// If none of (1) and (2) are true, or if both are true, an exception will be
53
+ /// thrown.
54
+ std::string column_keys;
55
+
56
+ /// Encrypt footer and all columns with the same encryption key.
57
+ bool uniform_encryption = kDefaultUniformEncryption;
58
+
59
+ /// Parquet encryption algorithm. Can be "AES_GCM_V1" (default), or "AES_GCM_CTR_V1".
60
+ ParquetCipher::type encryption_algorithm = kDefaultEncryptionAlgorithm;
61
+
62
+ /// Write files with plaintext footer.
63
+ /// The default is false - files are written with encrypted footer.
64
+ bool plaintext_footer = kDefaultPlaintextFooter;
65
+
66
+ /// Use double wrapping - where data encryption keys (DEKs) are encrypted with key
67
+ /// encryption keys (KEKs), which in turn are encrypted with master keys.
68
+ /// The default is true. If set to false, use single wrapping - where DEKs are
69
+ /// encrypted directly with master keys.
70
+ bool double_wrapping = kDefaultDoubleWrapping;
71
+
72
+ /// Lifetime of cached entities (key encryption keys, local wrapping keys, KMS client
73
+ /// objects).
74
+ /// The default is 600 (10 minutes).
75
+ double cache_lifetime_seconds = kDefaultCacheLifetimeSeconds;
76
+
77
+ /// Store key material inside Parquet file footers; this mode doesn’t produce
78
+ /// additional files. By default, true. If set to false, key material is stored in
79
+ /// separate files in the same folder, which enables key rotation for immutable
80
+ /// Parquet files.
81
+ bool internal_key_material = kDefaultInternalKeyMaterial;
82
+
83
+ /// Length of data encryption keys (DEKs), randomly generated by parquet key
84
+ /// management tools. Can be 128, 192 or 256 bits.
85
+ /// The default is 128 bits.
86
+ int32_t data_key_length_bits = kDefaultDataKeyLengthBits;
87
+ };
88
+
89
+ struct PARQUET_EXPORT DecryptionConfiguration {
90
+ /// Lifetime of cached entities (key encryption keys, local wrapping keys, KMS client
91
+ /// objects).
92
+ /// The default is 600 (10 minutes).
93
+ double cache_lifetime_seconds = kDefaultCacheLifetimeSeconds;
94
+ };
95
+
96
+ /// This is a core class, that translates the parameters of high level encryption (like
97
+ /// the names of encrypted columns, names of master keys, etc), into parameters of low
98
+ /// level encryption (like the key metadata, DEK, etc). A factory that produces the low
99
+ /// level FileEncryptionProperties and FileDecryptionProperties objects, from the high
100
+ /// level parameters.
101
+ class PARQUET_EXPORT CryptoFactory {
102
+ public:
103
+ /// a KmsClientFactory object must be registered via this method before calling any of
104
+ /// GetFileEncryptionProperties()/GetFileDecryptionProperties() methods.
105
+ void RegisterKmsClientFactory(std::shared_ptr<KmsClientFactory> kms_client_factory);
106
+
107
+ /// Get the encryption properties for a Parquet file.
108
+ /// If external key material is used then a file system and path to the
109
+ /// parquet file must be provided.
110
+ std::shared_ptr<FileEncryptionProperties> GetFileEncryptionProperties(
111
+ const KmsConnectionConfig& kms_connection_config,
112
+ const EncryptionConfiguration& encryption_config, const std::string& file_path = "",
113
+ const std::shared_ptr<::arrow::fs::FileSystem>& file_system = NULLPTR);
114
+
115
+ /// Get decryption properties for a Parquet file.
116
+ /// If external key material is used then a file system and path to the
117
+ /// parquet file must be provided.
118
+ std::shared_ptr<FileDecryptionProperties> GetFileDecryptionProperties(
119
+ const KmsConnectionConfig& kms_connection_config,
120
+ const DecryptionConfiguration& decryption_config, const std::string& file_path = "",
121
+ const std::shared_ptr<::arrow::fs::FileSystem>& file_system = NULLPTR);
122
+
123
+ void RemoveCacheEntriesForToken(const std::string& access_token) {
124
+ key_toolkit_->RemoveCacheEntriesForToken(access_token);
125
+ }
126
+
127
+ void RemoveCacheEntriesForAllTokens() {
128
+ key_toolkit_->RemoveCacheEntriesForAllTokens();
129
+ }
130
+
131
+ /// Rotates master encryption keys for a Parquet file that uses external key material.
132
+ /// In single wrapping mode, data encryption keys are decrypted with the old master keys
133
+ /// and then re-encrypted with new master keys.
134
+ /// In double wrapping mode, key encryption keys are decrypted with the old master keys
135
+ /// and then re-encrypted with new master keys.
136
+ /// This relies on the KMS supporting versioning, such that the old master key is
137
+ /// used when unwrapping a key, and the latest version is used when wrapping a key.
138
+ void RotateMasterKeys(const KmsConnectionConfig& kms_connection_config,
139
+ const std::string& parquet_file_path,
140
+ const std::shared_ptr<::arrow::fs::FileSystem>& file_system,
141
+ bool double_wrapping = kDefaultDoubleWrapping,
142
+ double cache_lifetime_seconds = kDefaultCacheLifetimeSeconds);
143
+
144
+ private:
145
+ ColumnPathToEncryptionPropertiesMap GetColumnEncryptionProperties(
146
+ int dek_length, const std::string& column_keys, FileKeyWrapper* key_wrapper);
147
+
148
+ /// Key utilities object for kms client initialization and cache control
149
+ std::shared_ptr<KeyToolkit> key_toolkit_ = std::make_shared<KeyToolkit>();
150
+ };
151
+
152
+ } // namespace parquet::encryption
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/encryption.h ADDED
@@ -0,0 +1,434 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <cassert>
21
+ #include <map>
22
+ #include <memory>
23
+ #include <span>
24
+ #include <string>
25
+ #include <utility>
26
+
27
+ #include "arrow/util/secure_string.h"
28
+ #include "parquet/exception.h"
29
+ #include "parquet/schema.h"
30
+ #include "parquet/types.h"
31
+
32
+ namespace parquet {
33
+
34
+ static constexpr ParquetCipher::type kDefaultEncryptionAlgorithm =
35
+ ParquetCipher::AES_GCM_V1;
36
+ static constexpr int32_t kMaximalAadMetadataLength = 256;
37
+ static constexpr bool kDefaultEncryptedFooter = true;
38
+ static constexpr bool kDefaultCheckSignature = true;
39
+ static constexpr bool kDefaultAllowPlaintextFiles = false;
40
+ static constexpr int32_t kAadFileUniqueLength = 8;
41
+
42
+ class ColumnDecryptionProperties;
43
+ using ColumnPathToDecryptionPropertiesMap =
44
+ std::map<std::string, std::shared_ptr<ColumnDecryptionProperties>>;
45
+
46
+ class ColumnEncryptionProperties;
47
+ using ColumnPathToEncryptionPropertiesMap =
48
+ std::map<std::string, std::shared_ptr<ColumnEncryptionProperties>>;
49
+
50
+ class PARQUET_EXPORT DecryptionKeyRetriever {
51
+ public:
52
+ /// \brief Retrieve a key.
53
+ virtual ::arrow::util::SecureString GetKey(const std::string& key_id) = 0;
54
+
55
+ virtual ~DecryptionKeyRetriever() {}
56
+ };
57
+
58
+ /// Simple integer key retriever
59
+ class PARQUET_EXPORT IntegerKeyIdRetriever : public DecryptionKeyRetriever {
60
+ public:
61
+ void PutKey(uint32_t key_id, ::arrow::util::SecureString key);
62
+
63
+ ::arrow::util::SecureString GetKey(const std::string& key_id_string) override {
64
+ // key_id_string is string but for IntegerKeyIdRetriever it encodes
65
+ // a native-endian 32 bit unsigned integer key_id
66
+ uint32_t key_id;
67
+ assert(key_id_string.size() == sizeof(key_id));
68
+ memcpy(&key_id, key_id_string.data(), sizeof(key_id));
69
+
70
+ return key_map_.at(key_id);
71
+ }
72
+
73
+ private:
74
+ std::map<uint32_t, ::arrow::util::SecureString> key_map_;
75
+ };
76
+
77
+ // Simple string key retriever
78
+ class PARQUET_EXPORT StringKeyIdRetriever : public DecryptionKeyRetriever {
79
+ public:
80
+ void PutKey(std::string key_id, ::arrow::util::SecureString key);
81
+ ::arrow::util::SecureString GetKey(const std::string& key_id) override;
82
+
83
+ private:
84
+ std::map<std::string, ::arrow::util::SecureString> key_map_;
85
+ };
86
+
87
+ class PARQUET_EXPORT HiddenColumnException : public ParquetException {
88
+ public:
89
+ explicit HiddenColumnException(const std::string& columnPath)
90
+ : ParquetException(columnPath.c_str()) {}
91
+ };
92
+
93
+ class PARQUET_EXPORT KeyAccessDeniedException : public ParquetException {
94
+ public:
95
+ explicit KeyAccessDeniedException(const std::string& columnPath)
96
+ : ParquetException(columnPath.c_str()) {}
97
+ };
98
+
99
+ inline std::span<const uint8_t> str2span(const std::string& str) {
100
+ if (str.empty()) {
101
+ return {};
102
+ }
103
+
104
+ return {reinterpret_cast<const uint8_t*>(str.data()), str.size()};
105
+ }
106
+
107
+ class PARQUET_EXPORT ColumnEncryptionProperties {
108
+ public:
109
+ class PARQUET_EXPORT Builder {
110
+ public:
111
+ PARQUET_DEPRECATED("name argument is ignored, use default constructor instead")
112
+ explicit Builder(const std::string& name) : encrypted_(true) {}
113
+
114
+ PARQUET_DEPRECATED("path argument is ignored, use default constructor instead")
115
+ explicit Builder(const schema::ColumnPath& path) : encrypted_(true) {}
116
+
117
+ Builder() = default;
118
+
119
+ /// Set a column-specific key.
120
+ /// If key is not set on an encrypted column, the column will
121
+ /// be encrypted with the footer key.
122
+ /// keyBytes Key length must be either 16, 24 or 32 bytes.
123
+ /// Caller is responsible for wiping out the input key array.
124
+ Builder* key(::arrow::util::SecureString column_key);
125
+
126
+ /// Set a key retrieval metadata.
127
+ /// use either key_metadata() or key_id(), not both
128
+ Builder* key_metadata(std::string key_metadata);
129
+
130
+ /// A convenience function to set key metadata using a string id.
131
+ /// Set a key retrieval metadata (converted from String).
132
+ /// use either key_metadata() or key_id(), not both
133
+ /// key_id will be converted to metadata (UTF-8 array).
134
+ Builder* key_id(std::string key_id);
135
+
136
+ std::shared_ptr<ColumnEncryptionProperties> build() {
137
+ return std::shared_ptr<ColumnEncryptionProperties>(
138
+ new ColumnEncryptionProperties(encrypted_, key_, key_metadata_));
139
+ }
140
+
141
+ private:
142
+ bool encrypted_ = true;
143
+ ::arrow::util::SecureString key_;
144
+ std::string key_metadata_;
145
+ };
146
+
147
+ bool is_encrypted() const { return encrypted_; }
148
+ bool is_encrypted_with_footer_key() const { return encrypted_with_footer_key_; }
149
+ const ::arrow::util::SecureString& key() const { return key_; }
150
+ const std::string& key_metadata() const { return key_metadata_; }
151
+
152
+ static std::shared_ptr<ColumnEncryptionProperties> Unencrypted();
153
+ static std::shared_ptr<ColumnEncryptionProperties> WithFooterKey();
154
+ static std::shared_ptr<ColumnEncryptionProperties> WithColumnKey(
155
+ ::arrow::util::SecureString key, std::string key_metadata = "");
156
+
157
+ private:
158
+ bool encrypted_;
159
+ bool encrypted_with_footer_key_;
160
+ ::arrow::util::SecureString key_;
161
+ std::string key_metadata_;
162
+ explicit ColumnEncryptionProperties(bool encrypted, ::arrow::util::SecureString key,
163
+ std::string key_metadata);
164
+ };
165
+
166
+ class PARQUET_EXPORT ColumnDecryptionProperties {
167
+ public:
168
+ class PARQUET_EXPORT Builder {
169
+ public:
170
+ explicit Builder(std::string name) : column_path_(std::move(name)) {}
171
+
172
+ explicit Builder(const schema::ColumnPath& path) : Builder(path.ToDotString()) {}
173
+
174
+ /// Set an explicit column key. If applied on a file that contains
175
+ /// key metadata for this column the metadata will be ignored,
176
+ /// the column will be decrypted with this key.
177
+ /// key length must be either 16, 24 or 32 bytes.
178
+ Builder* key(::arrow::util::SecureString key);
179
+
180
+ std::shared_ptr<ColumnDecryptionProperties> build();
181
+
182
+ private:
183
+ std::string column_path_;
184
+ ::arrow::util::SecureString key_;
185
+ };
186
+
187
+ const std::string& column_path() const { return column_path_; }
188
+ const ::arrow::util::SecureString& key() const { return key_; }
189
+
190
+ private:
191
+ std::string column_path_;
192
+ ::arrow::util::SecureString key_;
193
+
194
+ /// This class is only required for setting explicit column decryption keys -
195
+ /// to override key retriever (or to provide keys when key metadata and/or
196
+ /// key retriever are not available)
197
+ explicit ColumnDecryptionProperties(std::string column_path,
198
+ ::arrow::util::SecureString key);
199
+ };
200
+
201
+ class PARQUET_EXPORT AADPrefixVerifier {
202
+ public:
203
+ /// Verifies identity (AAD Prefix) of individual file,
204
+ /// or of file collection in a data set.
205
+ /// Throws exception if an AAD prefix is wrong.
206
+ /// In a data set, AAD Prefixes should be collected,
207
+ /// and then checked for missing files.
208
+ virtual void Verify(const std::string& aad_prefix) = 0;
209
+ virtual ~AADPrefixVerifier() {}
210
+ };
211
+
212
+ class PARQUET_EXPORT FileDecryptionProperties {
213
+ public:
214
+ class PARQUET_EXPORT Builder {
215
+ public:
216
+ Builder() {
217
+ check_plaintext_footer_integrity_ = kDefaultCheckSignature;
218
+ plaintext_files_allowed_ = kDefaultAllowPlaintextFiles;
219
+ }
220
+
221
+ /// Set an explicit footer key. If applied on a file that contains
222
+ /// footer key metadata the metadata will be ignored, the footer
223
+ /// will be decrypted/verified with this key.
224
+ /// If explicit key is not set, footer key will be fetched from
225
+ /// key retriever.
226
+ /// With explicit keys or AAD prefix, new encryption properties object must be
227
+ /// created for each encrypted file.
228
+ /// Explicit encryption keys (footer and column) are cloned.
229
+ /// Upon completion of file reading, the cloned encryption keys in the properties
230
+ /// will be wiped out (array values set to 0).
231
+ /// Caller is responsible for wiping out the input key array.
232
+ /// param footerKey Key length must be either 16, 24 or 32 bytes.
233
+ Builder* footer_key(::arrow::util::SecureString footer_key);
234
+
235
+ /// Set explicit column keys (decryption properties).
236
+ /// Its also possible to set a key retriever on this property object.
237
+ /// Upon file decryption, availability of explicit keys is checked before
238
+ /// invocation of the retriever callback.
239
+ /// If an explicit key is available for a footer or a column,
240
+ /// its key metadata will be ignored.
241
+ Builder* column_keys(
242
+ ColumnPathToDecryptionPropertiesMap column_decryption_properties);
243
+
244
+ /// Set a key retriever callback. Its also possible to
245
+ /// set explicit footer or column keys on this file property object.
246
+ /// Upon file decryption, availability of explicit keys is checked before
247
+ /// invocation of the retriever callback.
248
+ /// If an explicit key is available for a footer or a column,
249
+ /// its key metadata will be ignored.
250
+ Builder* key_retriever(std::shared_ptr<DecryptionKeyRetriever> key_retriever);
251
+
252
+ /// Skip integrity verification of plaintext footers.
253
+ /// If not called, integrity of plaintext footers will be checked in runtime,
254
+ /// and an exception will be thrown in the following situations:
255
+ /// - footer signing key is not available
256
+ /// (not passed, or not found by key retriever)
257
+ /// - footer content and signature don't match
258
+ Builder* disable_footer_signature_verification() {
259
+ check_plaintext_footer_integrity_ = false;
260
+ return this;
261
+ }
262
+
263
+ /// Explicitly supply the file AAD prefix.
264
+ /// A must when a prefix is used for file encryption, but not stored in file.
265
+ /// If AAD prefix is stored in file, it will be compared to the explicitly
266
+ /// supplied value and an exception will be thrown if they differ.
267
+ Builder* aad_prefix(std::string aad_prefix);
268
+
269
+ /// Set callback for verification of AAD Prefixes stored in file.
270
+ Builder* aad_prefix_verifier(std::shared_ptr<AADPrefixVerifier> aad_prefix_verifier);
271
+
272
+ /// By default, reading plaintext (unencrypted) files is not
273
+ /// allowed when using a decryptor
274
+ /// - in order to detect files that were not encrypted by mistake.
275
+ /// However, the default behavior can be overridden by calling this method.
276
+ /// The caller should use then a different method to ensure encryption
277
+ /// of files with sensitive data.
278
+ Builder* plaintext_files_allowed() {
279
+ plaintext_files_allowed_ = true;
280
+ return this;
281
+ }
282
+
283
+ std::shared_ptr<FileDecryptionProperties> build() {
284
+ return std::shared_ptr<FileDecryptionProperties>(new FileDecryptionProperties(
285
+ footer_key_, key_retriever_, check_plaintext_footer_integrity_, aad_prefix_,
286
+ aad_prefix_verifier_, column_decryption_properties_, plaintext_files_allowed_));
287
+ }
288
+
289
+ private:
290
+ ::arrow::util::SecureString footer_key_;
291
+ std::string aad_prefix_;
292
+ std::shared_ptr<AADPrefixVerifier> aad_prefix_verifier_;
293
+ ColumnPathToDecryptionPropertiesMap column_decryption_properties_;
294
+
295
+ std::shared_ptr<DecryptionKeyRetriever> key_retriever_;
296
+ bool check_plaintext_footer_integrity_;
297
+ bool plaintext_files_allowed_;
298
+ };
299
+
300
+ const ::arrow::util::SecureString& column_key(const std::string& column_path) const;
301
+
302
+ const ::arrow::util::SecureString& footer_key() const { return footer_key_; }
303
+
304
+ const std::string& aad_prefix() const { return aad_prefix_; }
305
+
306
+ const std::shared_ptr<DecryptionKeyRetriever>& key_retriever() const {
307
+ return key_retriever_;
308
+ }
309
+
310
+ bool check_plaintext_footer_integrity() const {
311
+ return check_plaintext_footer_integrity_;
312
+ }
313
+
314
+ bool plaintext_files_allowed() const { return plaintext_files_allowed_; }
315
+
316
+ const std::shared_ptr<AADPrefixVerifier>& aad_prefix_verifier() const {
317
+ return aad_prefix_verifier_;
318
+ }
319
+
320
+ private:
321
+ ::arrow::util::SecureString footer_key_;
322
+ std::string aad_prefix_;
323
+ std::shared_ptr<AADPrefixVerifier> aad_prefix_verifier_;
324
+ ColumnPathToDecryptionPropertiesMap column_decryption_properties_;
325
+ std::shared_ptr<DecryptionKeyRetriever> key_retriever_;
326
+ bool check_plaintext_footer_integrity_;
327
+ bool plaintext_files_allowed_;
328
+
329
+ FileDecryptionProperties(
330
+ ::arrow::util::SecureString footer_key,
331
+ std::shared_ptr<DecryptionKeyRetriever> key_retriever,
332
+ bool check_plaintext_footer_integrity, std::string aad_prefix,
333
+ std::shared_ptr<AADPrefixVerifier> aad_prefix_verifier,
334
+ ColumnPathToDecryptionPropertiesMap column_decryption_properties,
335
+ bool plaintext_files_allowed);
336
+ };
337
+
338
+ class PARQUET_EXPORT FileEncryptionProperties {
339
+ public:
340
+ class PARQUET_EXPORT Builder {
341
+ public:
342
+ explicit Builder(::arrow::util::SecureString footer_key)
343
+ : parquet_cipher_(kDefaultEncryptionAlgorithm),
344
+ encrypted_footer_(kDefaultEncryptedFooter),
345
+ footer_key_(std::move(footer_key)) {
346
+ store_aad_prefix_in_file_ = false;
347
+ }
348
+
349
+ /// Create files with plaintext footer.
350
+ /// If not called, the files will be created with encrypted footer (default).
351
+ Builder* set_plaintext_footer() {
352
+ encrypted_footer_ = false;
353
+ return this;
354
+ }
355
+
356
+ /// Set encryption algorithm.
357
+ /// If not called, files will be encrypted with AES_GCM_V1 (default).
358
+ Builder* algorithm(ParquetCipher::type parquet_cipher) {
359
+ parquet_cipher_ = parquet_cipher;
360
+ return this;
361
+ }
362
+
363
+ /// Set a key retrieval metadata (converted from String).
364
+ /// use either footer_key_metadata or footer_key_id, not both.
365
+ Builder* footer_key_id(std::string key_id);
366
+
367
+ /// Set a key retrieval metadata.
368
+ /// use either footer_key_metadata or footer_key_id, not both.
369
+ Builder* footer_key_metadata(std::string footer_key_metadata);
370
+
371
+ /// Set the file AAD Prefix.
372
+ Builder* aad_prefix(std::string aad_prefix);
373
+
374
+ /// Skip storing AAD Prefix in file.
375
+ /// If not called, and if AAD Prefix is set, it will be stored.
376
+ Builder* disable_aad_prefix_storage();
377
+
378
+ /// Set the list of encrypted columns and their properties (keys etc).
379
+ /// If not called, all columns will be encrypted with the footer key.
380
+ /// If called, the file columns not in the list will be left unencrypted.
381
+ Builder* encrypted_columns(ColumnPathToEncryptionPropertiesMap encrypted_columns);
382
+
383
+ std::shared_ptr<FileEncryptionProperties> build() {
384
+ return std::shared_ptr<FileEncryptionProperties>(new FileEncryptionProperties(
385
+ parquet_cipher_, footer_key_, footer_key_metadata_, encrypted_footer_,
386
+ aad_prefix_, store_aad_prefix_in_file_, encrypted_columns_));
387
+ }
388
+
389
+ private:
390
+ ParquetCipher::type parquet_cipher_;
391
+ bool encrypted_footer_;
392
+ ::arrow::util::SecureString footer_key_;
393
+ std::string footer_key_metadata_;
394
+
395
+ std::string aad_prefix_;
396
+ bool store_aad_prefix_in_file_;
397
+ ColumnPathToEncryptionPropertiesMap encrypted_columns_;
398
+ };
399
+
400
+ bool encrypted_footer() const { return encrypted_footer_; }
401
+
402
+ EncryptionAlgorithm algorithm() const { return algorithm_; }
403
+
404
+ const ::arrow::util::SecureString& footer_key() const { return footer_key_; }
405
+
406
+ const std::string& footer_key_metadata() const { return footer_key_metadata_; }
407
+
408
+ const std::string& file_aad() const { return file_aad_; }
409
+
410
+ std::shared_ptr<ColumnEncryptionProperties> column_encryption_properties(
411
+ const std::string& column_path);
412
+
413
+ const ColumnPathToEncryptionPropertiesMap& encrypted_columns() const {
414
+ return encrypted_columns_;
415
+ }
416
+
417
+ private:
418
+ EncryptionAlgorithm algorithm_;
419
+ ::arrow::util::SecureString footer_key_;
420
+ std::string footer_key_metadata_;
421
+ bool encrypted_footer_;
422
+ std::string file_aad_;
423
+ std::string aad_prefix_;
424
+ bool store_aad_prefix_in_file_;
425
+ ColumnPathToEncryptionPropertiesMap encrypted_columns_;
426
+
427
+ FileEncryptionProperties(ParquetCipher::type cipher,
428
+ ::arrow::util::SecureString footer_key,
429
+ std::string footer_key_metadata, bool encrypted_footer,
430
+ std::string aad_prefix, bool store_aad_prefix_in_file,
431
+ ColumnPathToEncryptionPropertiesMap encrypted_columns);
432
+ };
433
+
434
+ } // namespace parquet
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_key_material_store.h ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <set>
21
+ #include <string>
22
+ #include <unordered_map>
23
+
24
+ #include "arrow/filesystem/filesystem.h"
25
+ #include "parquet/platform.h"
26
+
27
+ namespace parquet::encryption {
28
+
29
+ /// Stores encryption key material outside the Parquet file, for example in a separate
30
+ /// small file in the same folder. This is important for “key rotation”, when MEKs have to
31
+ /// be changed (if compromised; or periodically, just in case) - without modifying the
32
+ /// Parquet files (often immutable).
33
+ class PARQUET_EXPORT FileKeyMaterialStore {
34
+ public:
35
+ /// Add key material for one encryption key.
36
+ virtual void AddKeyMaterial(std::string key_id_in_file, std::string key_material) = 0;
37
+
38
+ /// Get key material
39
+ virtual std::string GetKeyMaterial(std::string key_id_in_file) = 0;
40
+
41
+ /// After key material was added for all keys in the given Parquet file,
42
+ /// save material in persistent store.
43
+ virtual void SaveMaterial() = 0;
44
+
45
+ /// Remove key material from persistent store. Used in key rotation.
46
+ virtual void RemoveMaterial() = 0;
47
+
48
+ /// Move key material to another store. Used in key rotation.
49
+ virtual void MoveMaterialTo(std::shared_ptr<FileKeyMaterialStore> target_key_store) = 0;
50
+
51
+ /// Returns the Set of all key IDs in this store (for the given Parquet file)
52
+ virtual std::vector<std::string> GetKeyIDSet() = 0;
53
+
54
+ virtual ~FileKeyMaterialStore() {}
55
+ };
56
+
57
+ } // namespace parquet::encryption
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_key_unwrapper.h ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include "arrow/util/concurrent_map.h"
21
+ #include "arrow/util/secure_string.h"
22
+
23
+ #include "parquet/encryption/encryption.h"
24
+ #include "parquet/encryption/file_system_key_material_store.h"
25
+ #include "parquet/encryption/key_material.h"
26
+ #include "parquet/encryption/key_toolkit.h"
27
+ #include "parquet/encryption/key_toolkit_internal.h"
28
+ #include "parquet/encryption/kms_client.h"
29
+ #include "parquet/platform.h"
30
+
31
+ namespace parquet::encryption {
32
+
33
+ // This class will retrieve the key from "key metadata", following these steps:
34
+ // 1. Parse "key metadata" (see structure in KeyMetadata class).
35
+ // 2. Retrieve "key material" which can be stored inside or outside "key metadata".
36
+ // 3. Unwrap the "data encryption key" from "key material". There are 2 modes:
37
+ // 3.1. single wrapping: decrypt the wrapped "data encryption key" directly with "master
38
+ // encryption key" 3.2. double wrapping: 2 steps: 3.2.1. "key encryption key" is decrypted
39
+ // with "master encryption key" 3.2.2. "data encryption key" is decrypted with the above
40
+ // "key encryption key"
41
+ class PARQUET_EXPORT FileKeyUnwrapper : public DecryptionKeyRetriever {
42
+ public:
43
+ /// key_toolkit and kms_connection_config is to get KmsClient from cache or create
44
+ /// KmsClient if it's not in the cache yet. cache_entry_lifetime_seconds is life time of
45
+ /// KmsClient in the cache.
46
+ /// If the file uses external key material then the Parquet file path and file
47
+ /// system must be specified.
48
+ FileKeyUnwrapper(std::shared_ptr<KeyToolkit> key_toolkit,
49
+ const KmsConnectionConfig& kms_connection_config,
50
+ double cache_lifetime_seconds, const std::string& file_path = "",
51
+ const std::shared_ptr<::arrow::fs::FileSystem>& file_system = NULLPTR);
52
+
53
+ /// Constructor overload that takes a raw pointer to the KeyToolkit
54
+ FileKeyUnwrapper(KeyToolkit* key_toolkit,
55
+ const KmsConnectionConfig& kms_connection_config,
56
+ double cache_lifetime_seconds, const std::string& file_path = "",
57
+ const std::shared_ptr<::arrow::fs::FileSystem>& file_system = NULLPTR);
58
+
59
+ /// Constructor overload that takes a raw pointer to the KeyToolkit and
60
+ /// accepts an existing key_material_store rather than using
61
+ /// the file path and file system to create one when needed.
62
+ FileKeyUnwrapper(KeyToolkit* key_toolkit,
63
+ const KmsConnectionConfig& kms_connection_config,
64
+ double cache_lifetime_seconds,
65
+ std::shared_ptr<FileKeyMaterialStore> key_material_store);
66
+
67
+ /// Get the data key from key metadata
68
+ ::arrow::util::SecureString GetKey(const std::string& key_metadata_bytes) override;
69
+
70
+ /// Get the data key along with the master key id from key material
71
+ KeyWithMasterId GetDataEncryptionKey(const KeyMaterial& key_material);
72
+
73
+ private:
74
+ FileKeyUnwrapper(std::shared_ptr<KeyToolkit> key_toolkit_owner, KeyToolkit* key_toolkit,
75
+ const KmsConnectionConfig& kms_connection_config,
76
+ double cache_lifetime_seconds,
77
+ std::shared_ptr<FileKeyMaterialStore> key_material_store,
78
+ const std::string& file_path,
79
+ const std::shared_ptr<::arrow::fs::FileSystem>& file_system);
80
+
81
+ std::shared_ptr<KmsClient> GetKmsClientFromConfigOrKeyMaterial(
82
+ const KeyMaterial& key_material);
83
+
84
+ /// A map of Key Encryption Key (KEK) ID -> KEK bytes, for the current token
85
+ std::shared_ptr<::arrow::util::ConcurrentMap<std::string, ::arrow::util::SecureString>>
86
+ kek_per_kek_id_;
87
+ std::shared_ptr<KeyToolkit> key_toolkit_owner_;
88
+ KeyToolkit* key_toolkit_;
89
+ KmsConnectionConfig kms_connection_config_;
90
+ const double cache_entry_lifetime_seconds_;
91
+ std::shared_ptr<FileKeyMaterialStore> key_material_store_;
92
+ const std::string file_path_;
93
+ std::shared_ptr<::arrow::fs::FileSystem> file_system_;
94
+ };
95
+
96
+ } // namespace parquet::encryption
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_key_wrapper.h ADDED
@@ -0,0 +1,84 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <memory>
21
+ #include <string>
22
+ #include <unordered_map>
23
+
24
+ #include "arrow/util/concurrent_map.h"
25
+
26
+ #include "parquet/encryption/file_key_material_store.h"
27
+ #include "parquet/encryption/key_encryption_key.h"
28
+ #include "parquet/encryption/key_toolkit.h"
29
+ #include "parquet/encryption/kms_client.h"
30
+ #include "parquet/platform.h"
31
+
32
+ namespace parquet::encryption {
33
+
34
+ // This class will generate "key metadata" from "data encryption key" and "master key",
35
+ // following these steps:
36
+ // 1. Wrap "data encryption key". There are 2 modes:
37
+ // 1.1. single wrapping: encrypt "data encryption key" directly with "master encryption
38
+ // key"
39
+ // 1.2. double wrapping: 2 steps:
40
+ // 1.2.1. "key encryption key" is randomized (see KeyEncryptionKey class)
41
+ // 1.2.2. "data encryption key" is encrypted with the above "key encryption key"
42
+ // 2. Create "key material" (see structure in KeyMaterial class)
43
+ // 3. Create "key metadata" with "key material" inside or a reference to outside "key
44
+ // material" (see structure in KeyMetadata class).
45
+ class PARQUET_EXPORT FileKeyWrapper {
46
+ public:
47
+ static constexpr int kKeyEncryptionKeyLength = 16;
48
+ static constexpr int kKeyEncryptionKeyIdLength = 16;
49
+
50
+ /// key_toolkit and kms_connection_config is to get KmsClient from the cache or create
51
+ /// KmsClient if it's not in the cache yet. cache_entry_lifetime_seconds is life time of
52
+ /// KmsClient in the cache. key_material_store is to store "key material" outside
53
+ /// parquet file, NULL if "key material" is stored inside parquet file.
54
+ FileKeyWrapper(KeyToolkit* key_toolkit,
55
+ const KmsConnectionConfig& kms_connection_config,
56
+ std::shared_ptr<FileKeyMaterialStore> key_material_store,
57
+ double cache_entry_lifetime_seconds, bool double_wrapping);
58
+
59
+ /// Creates key_metadata field for a given data key, via wrapping the key with the
60
+ /// master key.
61
+ /// When external key material is used, an identifier is usually generated automatically
62
+ /// but may be specified explicitly to support key rotation,
63
+ /// which requires keeping the same identifiers.
64
+ std::string GetEncryptionKeyMetadata(const ::arrow::util::SecureString& data_key,
65
+ const std::string& master_key_id,
66
+ bool is_footer_key,
67
+ std::string key_id_in_file = "");
68
+
69
+ private:
70
+ KeyEncryptionKey CreateKeyEncryptionKey(const std::string& master_key_id);
71
+
72
+ /// A map of Master Encryption Key ID -> KeyEncryptionKey, for the current token
73
+ std::shared_ptr<::arrow::util::ConcurrentMap<std::string, KeyEncryptionKey>>
74
+ kek_per_master_key_id_;
75
+
76
+ std::shared_ptr<KmsClient> kms_client_;
77
+ KmsConnectionConfig kms_connection_config_;
78
+ std::shared_ptr<FileKeyMaterialStore> key_material_store_;
79
+ const double cache_entry_lifetime_seconds_;
80
+ const bool double_wrapping_;
81
+ uint16_t key_counter_;
82
+ };
83
+
84
+ } // namespace parquet::encryption
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/file_system_key_material_store.h ADDED
@@ -0,0 +1,93 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <set>
21
+ #include <string>
22
+ #include <unordered_map>
23
+
24
+ #include "arrow/filesystem/filesystem.h"
25
+
26
+ #include "parquet/encryption/file_key_material_store.h"
27
+ #include "parquet/exception.h"
28
+
29
+ namespace parquet::encryption {
30
+
31
+ /// A FileKeyMaterialStore that stores key material in a file system file in the same
32
+ /// folder as the Parquet file.
33
+ class PARQUET_EXPORT FileSystemKeyMaterialStore : public FileKeyMaterialStore {
34
+ public:
35
+ static constexpr const char kKeyMaterialFilePrefix[] = "_KEY_MATERIAL_FOR_";
36
+ static constexpr const char kTempFilePrefix[] = "_TMP";
37
+ static constexpr const char kKeyMaterialFileSuffix[] = ".json";
38
+
39
+ FileSystemKeyMaterialStore() {}
40
+ FileSystemKeyMaterialStore(std::string key_material_file_path,
41
+ std::shared_ptr<::arrow::fs::FileSystem> file_system);
42
+
43
+ /// Creates a new file system key material store for a parquet file.
44
+ /// When use_tmp_prefix is true, files are saved with an extra _TMP prefix so they don't
45
+ /// conflict with existing external material files. This is useful during key rotation
46
+ /// so that temporary key material files can be created while using the existing key
47
+ /// material, before moving the key material to the non-temporary location.
48
+ static std::shared_ptr<FileSystemKeyMaterialStore> Make(
49
+ std::string parquet_file_path, std::shared_ptr<::arrow::fs::FileSystem> file_system,
50
+ bool use_tmp_prefix);
51
+
52
+ /// Add key material for one encryption key.
53
+ void AddKeyMaterial(std::string key_id_in_file, std::string key_material) {
54
+ key_material_map_.emplace(std::move(key_id_in_file), std::move(key_material));
55
+ }
56
+
57
+ /// Get key material
58
+ std::string GetKeyMaterial(std::string key_id_in_file) {
59
+ if (key_material_map_.empty()) {
60
+ LoadKeyMaterialMap();
61
+ }
62
+ auto found = key_material_map_.find(key_id_in_file);
63
+ if (found == key_material_map_.end()) {
64
+ throw ParquetException("Invalid key id");
65
+ }
66
+ return found->second;
67
+ }
68
+
69
+ /// After key material was added for all keys in the given Parquet file,
70
+ /// save material in persistent store.
71
+ void SaveMaterial();
72
+
73
+ /// Remove key material from persistent store. Used in key rotation.
74
+ void RemoveMaterial();
75
+
76
+ /// Move key material to another store. Used in key rotation.
77
+ void MoveMaterialTo(std::shared_ptr<FileKeyMaterialStore> target_key_store);
78
+
79
+ /// Returns the Set of all key IDs in this store (for the given Parquet file)
80
+ std::vector<std::string> GetKeyIDSet();
81
+
82
+ private:
83
+ std::string GetStorageFilePath() { return key_material_file_path_; }
84
+
85
+ std::string BuildKeyMaterialMapJson();
86
+ void LoadKeyMaterialMap();
87
+ std::string key_material_file_path_;
88
+ std::shared_ptr<::arrow::fs::FileSystem> file_system_;
89
+ /// Maps ID of a key in Parquet file and key material
90
+ std::unordered_map<std::string, std::string> key_material_map_;
91
+ };
92
+
93
+ } // namespace parquet::encryption
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_encryption_key.h ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <cstdint>
21
+ #include <vector>
22
+
23
+ #include "arrow/util/base64.h"
24
+ #include "arrow/util/secure_string.h"
25
+
26
+ namespace parquet::encryption {
27
+
28
+ // In the double wrapping mode, each "data encryption key" (DEK) is encrypted with a “key
29
+ // encryption key” (KEK), that in turn is encrypted with a "master encryption key" (MEK).
30
+ // In a writer process, a random KEK is generated for each MEK ID, and cached in a <MEK-ID
31
+ // : KEK> map. This allows to perform an interaction with a KMS server only once for each
32
+ // MEK, in order to wrap its KEK. "Data encryption key" (DEK) wrapping is performed
33
+ // locally, and does not involve an interaction with a KMS server.
34
+ class KeyEncryptionKey {
35
+ public:
36
+ KeyEncryptionKey(::arrow::util::SecureString kek_bytes, std::string kek_id,
37
+ std::string encoded_wrapped_kek)
38
+ : kek_bytes_(std::move(kek_bytes)),
39
+ kek_id_(std::move(kek_id)),
40
+ encoded_kek_id_(::arrow::util::base64_encode(kek_id_)),
41
+ encoded_wrapped_kek_(std::move(encoded_wrapped_kek)) {}
42
+
43
+ const ::arrow::util::SecureString& kek_bytes() const { return kek_bytes_; }
44
+
45
+ const std::string& kek_id() const { return kek_id_; }
46
+
47
+ const std::string& encoded_kek_id() const { return encoded_kek_id_; }
48
+
49
+ const std::string& encoded_wrapped_kek() const { return encoded_wrapped_kek_; }
50
+
51
+ private:
52
+ ::arrow::util::SecureString kek_bytes_;
53
+ std::string kek_id_;
54
+ std::string encoded_kek_id_;
55
+ std::string encoded_wrapped_kek_;
56
+ };
57
+
58
+ } // namespace parquet::encryption
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_material.h ADDED
@@ -0,0 +1,129 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <string>
21
+
22
+ #include "parquet/platform.h"
23
+
24
+ namespace arrow {
25
+ namespace json {
26
+ namespace internal {
27
+ class ObjectParser;
28
+ } // namespace internal
29
+ } // namespace json
30
+ } // namespace arrow
31
+
32
+ namespace parquet::encryption {
33
+
34
+ // KeyMaterial class represents the "key material", keeping the information that allows
35
+ // readers to recover an encryption key (see description of the KeyMetadata class). The
36
+ // keytools package (PARQUET-1373) implements the "envelope encryption" pattern, in a
37
+ // "single wrapping" or "double wrapping" mode. In the single wrapping mode, the key
38
+ // material is generated by encrypting the "data encryption key" (DEK) by a "master key".
39
+ // In the double wrapping mode, the key material is generated by encrypting the DEK by a
40
+ // "key encryption key" (KEK), that in turn is encrypted by a "master key".
41
+ //
42
+ // Key material is kept in a flat json object, with the following fields:
43
+ // 1. "keyMaterialType" - a String, with the type of key material. In the current
44
+ // version, only one value is allowed - "PKMT1" (stands
45
+ // for "parquet key management tools, version 1"). For external key material storage,
46
+ // this field is written in both "key metadata" and "key material" jsons. For internal
47
+ // key material storage, this field is written only once in the common json.
48
+ // 2. "isFooterKey" - a boolean. If true, means that the material belongs to a file footer
49
+ // key, and keeps additional information (such as
50
+ // KMS instance ID and URL). If false, means that the material belongs to a column
51
+ // key.
52
+ // 3. "kmsInstanceID" - a String, with the KMS Instance ID. Written only in footer key
53
+ // material.
54
+ // 4. "kmsInstanceURL" - a String, with the KMS Instance URL. Written only in footer key
55
+ // material.
56
+ // 5. "masterKeyID" - a String, with the ID of the master key used to generate the
57
+ // material.
58
+ // 6. "wrappedDEK" - a String, with the wrapped DEK (base64 encoding).
59
+ // 7. "doubleWrapping" - a boolean. If true, means that the material was generated in
60
+ // double wrapping mode.
61
+ // If false - in single wrapping mode.
62
+ // 8. "keyEncryptionKeyID" - a String, with the ID of the KEK used to generate the
63
+ // material. Written only in double wrapping mode.
64
+ // 9. "wrappedKEK" - a String, with the wrapped KEK (base64 encoding). Written only in
65
+ // double wrapping mode.
66
+ class PARQUET_EXPORT KeyMaterial {
67
+ public:
68
+ // these fields are defined in a specification and should never be changed
69
+ static constexpr const char kKeyMaterialTypeField[] = "keyMaterialType";
70
+ static constexpr const char kKeyMaterialType1[] = "PKMT1";
71
+
72
+ static constexpr const char kFooterKeyIdInFile[] = "footerKey";
73
+ static constexpr const char kColumnKeyIdInFilePrefix[] = "columnKey";
74
+
75
+ static constexpr const char kIsFooterKeyField[] = "isFooterKey";
76
+ static constexpr const char kDoubleWrappingField[] = "doubleWrapping";
77
+ static constexpr const char kKmsInstanceIdField[] = "kmsInstanceID";
78
+ static constexpr const char kKmsInstanceUrlField[] = "kmsInstanceURL";
79
+ static constexpr const char kMasterKeyIdField[] = "masterKeyID";
80
+ static constexpr const char kWrappedDataEncryptionKeyField[] = "wrappedDEK";
81
+ static constexpr const char kKeyEncryptionKeyIdField[] = "keyEncryptionKeyID";
82
+ static constexpr const char kWrappedKeyEncryptionKeyField[] = "wrappedKEK";
83
+
84
+ public:
85
+ KeyMaterial() = default;
86
+
87
+ static KeyMaterial Parse(const std::string& key_material_string);
88
+
89
+ static KeyMaterial Parse(
90
+ const ::arrow::json::internal::ObjectParser* key_material_json);
91
+
92
+ /// This method returns a json string that will be stored either inside a parquet file
93
+ /// or in a key material store outside the parquet file.
94
+ static std::string SerializeToJson(bool is_footer_key,
95
+ const std::string& kms_instance_id,
96
+ const std::string& kms_instance_url,
97
+ const std::string& master_key_id,
98
+ bool is_double_wrapped, const std::string& kek_id,
99
+ const std::string& encoded_wrapped_kek,
100
+ const std::string& encoded_wrapped_dek,
101
+ bool is_internal_storage);
102
+
103
+ bool is_footer_key() const { return is_footer_key_; }
104
+ bool is_double_wrapped() const { return is_double_wrapped_; }
105
+ const std::string& master_key_id() const { return master_key_id_; }
106
+ const std::string& wrapped_dek() const { return encoded_wrapped_dek_; }
107
+ const std::string& kek_id() const { return kek_id_; }
108
+ const std::string& wrapped_kek() const { return encoded_wrapped_kek_; }
109
+ const std::string& kms_instance_id() const { return kms_instance_id_; }
110
+ const std::string& kms_instance_url() const { return kms_instance_url_; }
111
+
112
+ private:
113
+ KeyMaterial(bool is_footer_key, const std::string& kms_instance_id,
114
+ const std::string& kms_instance_url, const std::string& master_key_id,
115
+ bool is_double_wrapped, const std::string& kek_id,
116
+ const std::string& encoded_wrapped_kek,
117
+ const std::string& encoded_wrapped_dek);
118
+
119
+ bool is_footer_key_;
120
+ std::string kms_instance_id_;
121
+ std::string kms_instance_url_;
122
+ std::string master_key_id_;
123
+ bool is_double_wrapped_;
124
+ std::string kek_id_;
125
+ std::string encoded_wrapped_kek_;
126
+ std::string encoded_wrapped_dek_;
127
+ };
128
+
129
+ } // namespace parquet::encryption
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_metadata.h ADDED
@@ -0,0 +1,91 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <string>
21
+ #include <variant>
22
+
23
+ #include "parquet/encryption/key_material.h"
24
+ #include "parquet/exception.h"
25
+ #include "parquet/platform.h"
26
+
27
+ namespace parquet::encryption {
28
+
29
+ // Parquet encryption specification defines "key metadata" as an arbitrary byte array,
30
+ // generated by file writers for each encryption key, and passed to the low level API for
31
+ // storage in the file footer. The "key metadata" field is made available to file readers
32
+ // to enable recovery of the key. This interface can be utilized for implementation
33
+ // of any key management scheme.
34
+ //
35
+ // The keytools package (PARQUET-1373) implements one approach, of many possible, to key
36
+ // management and to generation of the "key metadata" fields. This approach, based on the
37
+ // "envelope encryption" pattern, allows integration with KMS servers. It keeps the actual
38
+ // material, required to recover a key, in a "key material" object (see the KeyMaterial
39
+ // class for details). This class is implemented to support version 1 of the parquet key
40
+ // management tools specification.
41
+ //
42
+ // KeyMetadata writes (and reads) the "key metadata" field as a flat json object,
43
+ // with the following fields:
44
+ // 1. "keyMaterialType" - a String, with the type of key material.
45
+ // 2. "internalStorage" - a boolean. If true, means that "key material" is kept inside the
46
+ // "key metadata" field. If false, "key material" is kept externally (outside Parquet
47
+ // files) - in this case, "key metadata" keeps a reference to the external "key material".
48
+ // 3. "keyReference" - a String, with the reference to the external "key material".
49
+ // Written only if internalStorage is false.
50
+ //
51
+ // If internalStorage is true, "key material" is a part of "key metadata", and the json
52
+ // keeps additional fields, described in the KeyMaterial class.
53
+ class PARQUET_EXPORT KeyMetadata {
54
+ public:
55
+ static constexpr const char kKeyMaterialInternalStorageField[] = "internalStorage";
56
+ static constexpr const char kKeyReferenceField[] = "keyReference";
57
+
58
+ /// key_metadata_bytes is the key metadata field stored in the parquet file,
59
+ /// in the serialized json object format.
60
+ static KeyMetadata Parse(const std::string& key_metadata_bytes);
61
+
62
+ static std::string CreateSerializedForExternalMaterial(
63
+ const std::string& key_reference);
64
+
65
+ bool key_material_stored_internally() const { return is_internal_storage_; }
66
+
67
+ const KeyMaterial& key_material() const {
68
+ if (!is_internal_storage_) {
69
+ throw ParquetException("key material is stored externally.");
70
+ }
71
+ return ::std::get<KeyMaterial>(key_material_or_reference_);
72
+ }
73
+
74
+ const std::string& key_reference() const {
75
+ if (is_internal_storage_) {
76
+ throw ParquetException("key material is stored internally.");
77
+ }
78
+ return ::std::get<std::string>(key_material_or_reference_);
79
+ }
80
+
81
+ private:
82
+ explicit KeyMetadata(const KeyMaterial& key_material);
83
+ explicit KeyMetadata(const std::string& key_reference);
84
+
85
+ bool is_internal_storage_;
86
+ /// If is_internal_storage_ is true, KeyMaterial is set,
87
+ /// else a string referencing to an outside "key material" is set.
88
+ ::std::variant<KeyMaterial, std::string> key_material_or_reference_;
89
+ };
90
+
91
+ } // namespace parquet::encryption
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/key_toolkit.h ADDED
@@ -0,0 +1,106 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <memory>
21
+ #include <string>
22
+
23
+ #include "parquet/encryption/key_encryption_key.h"
24
+ #include "parquet/encryption/kms_client.h"
25
+ #include "parquet/encryption/kms_client_factory.h"
26
+ #include "parquet/encryption/two_level_cache_with_expiration.h"
27
+ #include "parquet/platform.h"
28
+
29
+ namespace parquet::encryption {
30
+
31
+ static constexpr uint64_t kCacheCleanPeriodForKeyRotation = 60 * 60; // 1 hour
32
+
33
+ // KeyToolkit is a utility that keeps various tools for key management (such as key
34
+ // rotation, kms client instantiation, cache control, etc), plus a number of auxiliary
35
+ // classes for internal use.
36
+ class PARQUET_EXPORT KeyToolkit {
37
+ public:
38
+ KeyToolkit() { last_cache_clean_for_key_rotation_time_ = {}; }
39
+
40
+ /// KMS client two level cache: token -> KMSInstanceId -> KmsClient
41
+ TwoLevelCacheWithExpiration<std::shared_ptr<KmsClient>>& kms_client_cache_per_token() {
42
+ return kms_client_cache_;
43
+ }
44
+ /// Key encryption key two level cache for wrapping: token -> MasterEncryptionKeyId ->
45
+ /// KeyEncryptionKey
46
+ TwoLevelCacheWithExpiration<KeyEncryptionKey>& kek_write_cache_per_token() {
47
+ return key_encryption_key_write_cache_;
48
+ }
49
+
50
+ /// Key encryption key two level cache for unwrapping: token -> KeyEncryptionKeyId ->
51
+ /// KeyEncryptionKeyBytes
52
+ TwoLevelCacheWithExpiration<::arrow::util::SecureString>& kek_read_cache_per_token() {
53
+ return key_encryption_key_read_cache_;
54
+ }
55
+
56
+ std::shared_ptr<KmsClient> GetKmsClient(
57
+ const KmsConnectionConfig& kms_connection_config, double cache_entry_lifetime_ms);
58
+
59
+ /// Flush any caches that are tied to the (compromised) access_token
60
+ void RemoveCacheEntriesForToken(const std::string& access_token);
61
+
62
+ void RemoveCacheEntriesForAllTokens();
63
+
64
+ void RegisterKmsClientFactory(std::shared_ptr<KmsClientFactory> kms_client_factory) {
65
+ if (kms_client_factory_ != NULLPTR) {
66
+ throw ParquetException("KMS client factory has already been registered.");
67
+ }
68
+ kms_client_factory_ = std::move(kms_client_factory);
69
+ }
70
+
71
+ /// Key rotation. In the single wrapping mode, decrypts data keys with old master keys,
72
+ /// then encrypts them with new master keys. In the double wrapping mode, decrypts KEKs
73
+ /// (key encryption keys) with old master keys, generates new KEKs and encrypts them
74
+ /// with new master keys. Works only if key material is not stored internally in file
75
+ /// footers. Not supported in local key wrapping mode. Method can be run by multiple
76
+ /// threads, but each thread must work on different files.
77
+ void RotateMasterKeys(const KmsConnectionConfig& kms_connection_config,
78
+ const std::string& parquet_file_path,
79
+ const std::shared_ptr<::arrow::fs::FileSystem>& file_system,
80
+ bool double_wrapping, double cache_lifetime_seconds);
81
+
82
+ private:
83
+ TwoLevelCacheWithExpiration<std::shared_ptr<KmsClient>> kms_client_cache_;
84
+ TwoLevelCacheWithExpiration<KeyEncryptionKey> key_encryption_key_write_cache_;
85
+ TwoLevelCacheWithExpiration<::arrow::util::SecureString> key_encryption_key_read_cache_;
86
+ std::shared_ptr<KmsClientFactory> kms_client_factory_;
87
+ mutable ::arrow::util::Mutex last_cache_clean_for_key_rotation_time_mutex_;
88
+ internal::TimePoint last_cache_clean_for_key_rotation_time_;
89
+ };
90
+
91
+ // "data encryption key" and "master key identifier" are paired together as output when
92
+ // parsing from "key material"
93
+ class PARQUET_EXPORT KeyWithMasterId {
94
+ public:
95
+ KeyWithMasterId(::arrow::util::SecureString key_bytes, std::string master_id)
96
+ : key_bytes_(std::move(key_bytes)), master_id_(std::move(master_id)) {}
97
+
98
+ const ::arrow::util::SecureString& data_key() const { return key_bytes_; }
99
+ const std::string& master_id() const { return master_id_; }
100
+
101
+ private:
102
+ ::arrow::util::SecureString key_bytes_;
103
+ std::string master_id_;
104
+ };
105
+
106
+ } // namespace parquet::encryption
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/kms_client.h ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <memory>
21
+ #include <string>
22
+ #include <unordered_map>
23
+
24
+ #include "arrow/util/mutex.h"
25
+ #include "arrow/util/secure_string.h"
26
+
27
+ #include "parquet/exception.h"
28
+ #include "parquet/platform.h"
29
+
30
+ namespace parquet::encryption {
31
+
32
+ /// This class wraps the key access token of a KMS server. If your token changes over
33
+ /// time, you should keep the reference to the KeyAccessToken object and call Refresh()
34
+ /// method every time you have a new token.
35
+ class PARQUET_EXPORT KeyAccessToken {
36
+ public:
37
+ KeyAccessToken() = default;
38
+
39
+ explicit KeyAccessToken(const std::string value) : value_(value) {}
40
+
41
+ void Refresh(const std::string& new_value) {
42
+ auto lock = mutex_.Lock();
43
+ value_ = new_value;
44
+ }
45
+
46
+ const std::string& value() const {
47
+ auto lock = mutex_.Lock();
48
+ return value_;
49
+ }
50
+
51
+ private:
52
+ std::string value_;
53
+ mutable ::arrow::util::Mutex mutex_;
54
+ };
55
+
56
+ struct PARQUET_EXPORT KmsConnectionConfig {
57
+ std::string kms_instance_id;
58
+ std::string kms_instance_url;
59
+ /// If the access token is changed in the future, you should keep a reference to
60
+ /// this object and call Refresh() on it whenever there is a new access token.
61
+ std::shared_ptr<KeyAccessToken> refreshable_key_access_token;
62
+ std::unordered_map<std::string, std::string> custom_kms_conf;
63
+
64
+ KmsConnectionConfig();
65
+
66
+ const std::string& key_access_token() const {
67
+ if (refreshable_key_access_token == NULLPTR ||
68
+ refreshable_key_access_token->value().empty()) {
69
+ throw ParquetException("key access token is not set!");
70
+ }
71
+ return refreshable_key_access_token->value();
72
+ }
73
+
74
+ void SetDefaultIfEmpty();
75
+ };
76
+
77
+ class PARQUET_EXPORT KmsClient {
78
+ public:
79
+ static constexpr const char kKmsInstanceIdDefault[] = "DEFAULT";
80
+ static constexpr const char kKmsInstanceUrlDefault[] = "DEFAULT";
81
+ static constexpr const char kKeyAccessTokenDefault[] = "DEFAULT";
82
+
83
+ /// \brief Wraps a key.
84
+ ///
85
+ /// Encrypts it with the master key, encodes the result
86
+ /// and potentially adds a KMS-specific metadata.
87
+ virtual std::string WrapKey(const ::arrow::util::SecureString& key_bytes,
88
+ const std::string& master_key_identifier) = 0;
89
+
90
+ /// \brief Decrypts (unwraps) a key with the master key.
91
+ virtual ::arrow::util::SecureString UnwrapKey(
92
+ const std::string& wrapped_key, const std::string& master_key_identifier) = 0;
93
+
94
+ virtual ~KmsClient() {}
95
+ };
96
+
97
+ } // namespace parquet::encryption
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/kms_client_factory.h ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include "parquet/encryption/kms_client.h"
21
+ #include "parquet/platform.h"
22
+
23
+ namespace parquet::encryption {
24
+
25
+ class PARQUET_EXPORT KmsClientFactory {
26
+ public:
27
+ explicit KmsClientFactory(bool wrap_locally = false) : wrap_locally_(wrap_locally) {}
28
+
29
+ virtual ~KmsClientFactory() = default;
30
+
31
+ virtual std::shared_ptr<KmsClient> CreateKmsClient(
32
+ const KmsConnectionConfig& kms_connection_config) = 0;
33
+
34
+ protected:
35
+ bool wrap_locally_;
36
+ };
37
+
38
+ } // namespace parquet::encryption
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/local_wrap_kms_client.h ADDED
@@ -0,0 +1,95 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <unordered_map>
21
+ #include <vector>
22
+
23
+ #include "arrow/util/concurrent_map.h"
24
+
25
+ #include "parquet/encryption/kms_client.h"
26
+ #include "parquet/platform.h"
27
+
28
+ namespace parquet::encryption {
29
+
30
+ /// This class supports local wrapping mode, master keys will be fetched from the KMS
31
+ /// server and used to encrypt other keys (data encryption keys or key encryption keys).
32
+ class PARQUET_EXPORT LocalWrapKmsClient : public KmsClient {
33
+ public:
34
+ static constexpr const char kLocalWrapNoKeyVersion[] = "NO_VERSION";
35
+
36
+ explicit LocalWrapKmsClient(const KmsConnectionConfig& kms_connection_config);
37
+
38
+ std::string WrapKey(const ::arrow::util::SecureString& key_bytes,
39
+ const std::string& master_key_identifier) override;
40
+
41
+ ::arrow::util::SecureString UnwrapKey(
42
+ const std::string& wrapped_key, const std::string& master_key_identifier) override;
43
+
44
+ protected:
45
+ /// Get master key from the remote KMS server.
46
+ /// Note: this function might be called by multiple threads
47
+ virtual const ::arrow::util::SecureString& GetMasterKeyFromServer(
48
+ const std::string& master_key_identifier) = 0;
49
+
50
+ private:
51
+ /// KMS systems wrap keys by encrypting them by master keys, and attaching additional
52
+ /// information (such as the version number of the masker key) to the result of
53
+ /// encryption. The master key version is required in key rotation. Currently, the
54
+ /// local wrapping mode does not support key rotation (because not all KMS systems allow
55
+ /// to fetch a master key by its ID and version number). Still, the local wrapping mode
56
+ /// adds a placeholder for the master key version, that will enable support for key
57
+ /// rotation in this mode in the future, with appropriate KMS systems. This will also
58
+ /// enable backward compatibility, where future readers will be able to extract master
59
+ /// key version in the files written by the current code.
60
+ ///
61
+ /// LocalKeyWrap class writes (and reads) the "key wrap" as a flat json with the
62
+ /// following fields:
63
+ /// 1. "masterKeyVersion" - a String, with the master key version. In the current
64
+ /// version, only one value is allowed - "NO_VERSION".
65
+ /// 2. "encryptedKey" - a String, with the key encrypted by the master key
66
+ /// (base64-encoded).
67
+ class LocalKeyWrap {
68
+ public:
69
+ static constexpr const char kLocalWrapKeyVersionField[] = "masterKeyVersion";
70
+ static constexpr const char kLocalWrapEncryptedKeyField[] = "encryptedKey";
71
+
72
+ LocalKeyWrap(std::string master_key_version, std::string encrypted_encoded_key);
73
+
74
+ static std::string CreateSerialized(const std::string& encrypted_encoded_key);
75
+
76
+ static LocalKeyWrap Parse(const std::string& wrapped_key);
77
+
78
+ const std::string& master_key_version() const { return master_key_version_; }
79
+
80
+ const std::string& encrypted_encoded_key() const { return encrypted_encoded_key_; }
81
+
82
+ private:
83
+ std::string encrypted_encoded_key_;
84
+ std::string master_key_version_;
85
+ };
86
+
87
+ const ::arrow::util::SecureString& GetKeyFromServer(const std::string& key_identifier);
88
+
89
+ protected:
90
+ KmsConnectionConfig kms_connection_config_;
91
+ ::arrow::util::ConcurrentMap<std::string, ::arrow::util::SecureString>
92
+ master_key_cache_;
93
+ };
94
+
95
+ } // namespace parquet::encryption
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/test_encryption_util.h ADDED
@@ -0,0 +1,136 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ // This module defines an abstract interface for iterating through pages in a
19
+ // Parquet column chunk within a row group. It could be extended in the future
20
+ // to iterate through all data pages in all chunks in a file.
21
+
22
+ #pragma once
23
+
24
+ #include <memory>
25
+ #include <string>
26
+ #include <unordered_map>
27
+
28
+ #include <gtest/gtest.h>
29
+
30
+ #include "arrow/filesystem/filesystem.h"
31
+ #include "arrow/filesystem/localfs.h"
32
+ #include "arrow/status.h"
33
+ #include "arrow/util/io_util.h"
34
+ #include "arrow/util/secure_string.h"
35
+
36
+ #include "parquet/encryption/encryption.h"
37
+ #include "parquet/test_util.h"
38
+
39
+ namespace parquet {
40
+ class ParquetFileReader;
41
+ namespace encryption::test {
42
+
43
+ using ::arrow::internal::TemporaryDir;
44
+ using ::arrow::util::SecureString;
45
+
46
+ constexpr int kFixedLength = 10;
47
+
48
+ const SecureString kFooterEncryptionKey("0123456789012345");
49
+ const SecureString kColumnEncryptionKey1("1234567890123450");
50
+ const SecureString kColumnEncryptionKey2("1234567890123451");
51
+ const SecureString kColumnEncryptionKey3("1234567890123452");
52
+ const char kFileName[] = "tester";
53
+
54
+ // Get the path of file inside parquet test data directory
55
+ std::string data_file(const char* file);
56
+
57
+ // A temporary directory that contains the encrypted files generated in the tests.
58
+ extern std::unique_ptr<TemporaryDir> temp_dir;
59
+
60
+ inline ::arrow::Result<std::unique_ptr<TemporaryDir>> temp_data_dir() {
61
+ return TemporaryDir::Make("parquet-encryption-test-");
62
+ }
63
+
64
+ const char kDoubleFieldName[] = "double_field";
65
+ const char kFloatFieldName[] = "float_field";
66
+ const char kBooleanFieldName[] = "boolean_field";
67
+ const char kInt32FieldName[] = "int32_field";
68
+ const char kInt64FieldName[] = "int64_field";
69
+ const char kInt96FieldName[] = "int96_field";
70
+ const char kByteArrayFieldName[] = "ba_field";
71
+ const char kFixedLenByteArrayFieldName[] = "flba_field";
72
+
73
+ const char kFooterMasterKey[] = "0123456789012345";
74
+ const char kFooterMasterKeyId[] = "kf";
75
+ const char* const kColumnMasterKeys[] = {"1234567890123450", "1234567890123451",
76
+ "1234567890123452", "1234567890123453",
77
+ "1234567890123454", "1234567890123455"};
78
+ const char* const kColumnMasterKeyIds[] = {"kc1", "kc2", "kc3", "kc4", "kc5", "kc6"};
79
+
80
+ // New master key values used to simulate key rotation
81
+ const char kNewFooterMasterKey[] = "9123456789012345";
82
+ const char* const kNewColumnMasterKeys[] = {"9234567890123450", "9234567890123451",
83
+ "9234567890123452", "9234567890123453",
84
+ "9234567890123454", "9234567890123455"};
85
+
86
+ // The result of this function will be used to set into TestOnlyInMemoryKmsClientFactory
87
+ // as the key mapping to look at.
88
+ std::unordered_map<std::string, SecureString> BuildKeyMap(const char* const* column_ids,
89
+ const char* const* column_keys,
90
+ const char* footer_id,
91
+ const char* footer_key);
92
+
93
+ // The result of this function will be used to set into EncryptionConfiguration
94
+ // as column keys.
95
+ std::string BuildColumnKeyMapping();
96
+
97
+ // FileEncryptor and FileDecryptor are helper classes to write/read an encrypted parquet
98
+ // file corresponding to each pair of FileEncryptionProperties/FileDecryptionProperties.
99
+ // FileEncryptor writes the file with fixed data values and FileDecryptor reads the file
100
+ // and verify the correctness of data values.
101
+ class FileEncryptor {
102
+ public:
103
+ FileEncryptor();
104
+
105
+ void EncryptFile(
106
+ std::string file,
107
+ std::shared_ptr<parquet::FileEncryptionProperties> encryption_configurations);
108
+
109
+ private:
110
+ std::shared_ptr<schema::GroupNode> SetupEncryptionSchema();
111
+
112
+ int num_rowgroups_ = 5;
113
+ int rows_per_rowgroup_ = 50;
114
+ std::shared_ptr<schema::GroupNode> schema_;
115
+ };
116
+
117
+ class FileDecryptor {
118
+ public:
119
+ void DecryptFile(
120
+ const std::string& file_name,
121
+ const std::shared_ptr<FileDecryptionProperties>& file_decryption_properties);
122
+ void DecryptPageIndex(
123
+ const std::string& file_name,
124
+ const std::shared_ptr<FileDecryptionProperties>& file_decryption_properties);
125
+
126
+ private:
127
+ void CheckFile(
128
+ parquet::ParquetFileReader* file_reader,
129
+ const std::shared_ptr<FileDecryptionProperties>& file_decryption_properties);
130
+ void CheckPageIndex(
131
+ parquet::ParquetFileReader* file_reader,
132
+ const std::shared_ptr<FileDecryptionProperties>& file_decryption_properties);
133
+ };
134
+
135
+ } // namespace encryption::test
136
+ } // namespace parquet
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/test_in_memory_kms.h ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <unordered_map>
21
+
22
+ #include "arrow/util/base64.h"
23
+
24
+ #include "parquet/encryption/kms_client_factory.h"
25
+ #include "parquet/encryption/local_wrap_kms_client.h"
26
+ #include "parquet/platform.h"
27
+
28
+ namespace parquet::encryption {
29
+
30
+ // This is a mock class, built for testing only. Don't use it as an example of
31
+ // LocalWrapKmsClient implementation.
32
+ class TestOnlyLocalWrapInMemoryKms : public LocalWrapKmsClient {
33
+ public:
34
+ explicit TestOnlyLocalWrapInMemoryKms(const KmsConnectionConfig& kms_connection_config);
35
+
36
+ static void InitializeMasterKeys(
37
+ const std::unordered_map<std::string, ::arrow::util::SecureString>&
38
+ master_keys_map);
39
+
40
+ protected:
41
+ const ::arrow::util::SecureString& GetMasterKeyFromServer(
42
+ const std::string& master_key_identifier) override;
43
+
44
+ private:
45
+ static std::unordered_map<std::string, ::arrow::util::SecureString> master_key_map_;
46
+ };
47
+
48
+ // This is a mock class, built for testing only. Don't use it as an example of KmsClient
49
+ // implementation.
50
+ class TestOnlyInServerWrapKms : public KmsClient {
51
+ public:
52
+ static void InitializeMasterKeys(
53
+ const std::unordered_map<std::string, ::arrow::util::SecureString>&
54
+ master_keys_map);
55
+
56
+ std::string WrapKey(const ::arrow::util::SecureString& key_bytes,
57
+ const std::string& master_key_identifier) override;
58
+
59
+ ::arrow::util::SecureString UnwrapKey(
60
+ const std::string& wrapped_key, const std::string& master_key_identifier) override;
61
+
62
+ static void StartKeyRotation(
63
+ const std::unordered_map<std::string, ::arrow::util::SecureString>&
64
+ new_master_keys_map);
65
+ static void FinishKeyRotation();
66
+
67
+ private:
68
+ ::arrow::util::SecureString GetMasterKeyFromServer(
69
+ const std::string& master_key_identifier);
70
+
71
+ // Different wrapping and unwrapping key maps to imitate versioning
72
+ // and support key rotation.
73
+ static std::unordered_map<std::string, ::arrow::util::SecureString>
74
+ unwrapping_master_key_map_;
75
+ static std::unordered_map<std::string, ::arrow::util::SecureString>
76
+ wrapping_master_key_map_;
77
+ };
78
+
79
+ // This is a mock class, built for testing only. Don't use it as an example of
80
+ // KmsClientFactory implementation.
81
+ class TestOnlyInMemoryKmsClientFactory : public KmsClientFactory {
82
+ public:
83
+ TestOnlyInMemoryKmsClientFactory(
84
+ bool wrap_locally,
85
+ const std::unordered_map<std::string, ::arrow::util::SecureString>& master_keys_map)
86
+ : KmsClientFactory(wrap_locally) {
87
+ TestOnlyLocalWrapInMemoryKms::InitializeMasterKeys(master_keys_map);
88
+ TestOnlyInServerWrapKms::InitializeMasterKeys(master_keys_map);
89
+ }
90
+
91
+ std::shared_ptr<KmsClient> CreateKmsClient(
92
+ const KmsConnectionConfig& kms_connection_config) {
93
+ if (wrap_locally_) {
94
+ return std::make_shared<TestOnlyLocalWrapInMemoryKms>(kms_connection_config);
95
+ } else {
96
+ return std::make_shared<TestOnlyInServerWrapKms>();
97
+ }
98
+ }
99
+ };
100
+
101
+ } // namespace parquet::encryption
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/two_level_cache_with_expiration.h ADDED
@@ -0,0 +1,149 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <chrono>
21
+ #include <unordered_map>
22
+
23
+ #include "arrow/util/concurrent_map.h"
24
+ #include "arrow/util/mutex.h"
25
+
26
+ namespace parquet::encryption {
27
+
28
+ using ::arrow::util::ConcurrentMap;
29
+
30
+ namespace internal {
31
+
32
+ using TimePoint =
33
+ std::chrono::time_point<std::chrono::system_clock, std::chrono::duration<double>>;
34
+
35
+ inline TimePoint CurrentTimePoint() { return std::chrono::system_clock::now(); }
36
+
37
+ template <typename E>
38
+ class ExpiringCacheEntry {
39
+ public:
40
+ ExpiringCacheEntry() = default;
41
+
42
+ ExpiringCacheEntry(E cached_item, double expiration_interval_seconds)
43
+ : expiration_timestamp_(CurrentTimePoint() +
44
+ std::chrono::duration<double>(expiration_interval_seconds)),
45
+ cached_item_(std::move(cached_item)) {}
46
+
47
+ bool IsExpired() const {
48
+ const auto now = CurrentTimePoint();
49
+ return (now > expiration_timestamp_);
50
+ }
51
+
52
+ E cached_item() { return cached_item_; }
53
+
54
+ private:
55
+ const TimePoint expiration_timestamp_;
56
+ E cached_item_;
57
+ };
58
+
59
+ // This class is to avoid the below warning when compiling KeyToolkit class with VS2015
60
+ // warning C4503: decorated name length exceeded, name was truncated
61
+ template <typename V>
62
+ class ExpiringCacheMapEntry {
63
+ public:
64
+ ExpiringCacheMapEntry() = default;
65
+
66
+ explicit ExpiringCacheMapEntry(
67
+ std::shared_ptr<ConcurrentMap<std::string, V>> cached_item,
68
+ double expiration_interval_seconds)
69
+ : map_cache_(cached_item, expiration_interval_seconds) {}
70
+
71
+ bool IsExpired() { return map_cache_.IsExpired(); }
72
+
73
+ std::shared_ptr<ConcurrentMap<std::string, V>> cached_item() {
74
+ return map_cache_.cached_item();
75
+ }
76
+
77
+ private:
78
+ // ConcurrentMap object may be accessed and modified at many places at the same time,
79
+ // from multiple threads, or even removed from cache.
80
+ ExpiringCacheEntry<std::shared_ptr<ConcurrentMap<std::string, V>>> map_cache_;
81
+ };
82
+
83
+ } // namespace internal
84
+
85
+ // Two-level cache with expiration of internal caches according to token lifetime.
86
+ // External cache is per token, internal is per string key.
87
+ // Wrapper class around:
88
+ // std::unordered_map<std::string,
89
+ // internal::ExpiringCacheEntry<std::unordered_map<std::string, V>>>
90
+ // This cache is safe to be shared between threads.
91
+ template <typename V>
92
+ class TwoLevelCacheWithExpiration {
93
+ public:
94
+ TwoLevelCacheWithExpiration() {
95
+ last_cache_cleanup_timestamp_ = internal::CurrentTimePoint();
96
+ }
97
+
98
+ std::shared_ptr<ConcurrentMap<std::string, V>> GetOrCreateInternalCache(
99
+ const std::string& access_token, double cache_entry_lifetime_seconds) {
100
+ auto lock = mutex_.Lock();
101
+
102
+ auto external_cache_entry = cache_.find(access_token);
103
+ if (external_cache_entry == cache_.end() ||
104
+ external_cache_entry->second.IsExpired()) {
105
+ cache_.insert({access_token, internal::ExpiringCacheMapEntry<V>(
106
+ std::make_shared<ConcurrentMap<std::string, V>>(),
107
+ cache_entry_lifetime_seconds)});
108
+ }
109
+
110
+ return cache_[access_token].cached_item();
111
+ }
112
+
113
+ void CheckCacheForExpiredTokens(double cache_cleanup_period_seconds = 0.0) {
114
+ auto lock = mutex_.Lock();
115
+
116
+ const auto now = internal::CurrentTimePoint();
117
+ if (now > (last_cache_cleanup_timestamp_ +
118
+ std::chrono::duration<double>(cache_cleanup_period_seconds))) {
119
+ RemoveExpiredEntriesNoMutex();
120
+ last_cache_cleanup_timestamp_ = now;
121
+ }
122
+ }
123
+
124
+ void Remove(const std::string& access_token) {
125
+ auto lock = mutex_.Lock();
126
+ cache_.erase(access_token);
127
+ }
128
+
129
+ void Clear() {
130
+ auto lock = mutex_.Lock();
131
+ cache_.clear();
132
+ }
133
+
134
+ private:
135
+ void RemoveExpiredEntriesNoMutex() {
136
+ for (auto it = cache_.begin(); it != cache_.end();) {
137
+ if (it->second.IsExpired()) {
138
+ it = cache_.erase(it);
139
+ } else {
140
+ ++it;
141
+ }
142
+ }
143
+ }
144
+ std::unordered_map<std::string, internal::ExpiringCacheMapEntry<V>> cache_;
145
+ internal::TimePoint last_cache_cleanup_timestamp_;
146
+ ::arrow::util::Mutex mutex_;
147
+ };
148
+
149
+ } // namespace parquet::encryption
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/encryption/type_fwd.h ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ namespace parquet {
21
+
22
+ class Decryptor;
23
+ class Encryptor;
24
+
25
+ class InternalFileDecryptor;
26
+ class InternalFileEncryptor;
27
+
28
+ } // namespace parquet
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/exception.h ADDED
@@ -0,0 +1,175 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <exception>
21
+ #include <sstream>
22
+ #include <string>
23
+ #include <utility>
24
+
25
+ #include "arrow/type_fwd.h"
26
+ #include "arrow/util/string_util.h"
27
+ #include "parquet/platform.h"
28
+
29
+ #ifdef _MSC_VER
30
+ # pragma warning(push)
31
+ // Disable warning for STL types usage in DLL interface
32
+ // https://web.archive.org/web/20130317015847/http://connect.microsoft.com/VisualStudio/feedback/details/696593/vc-10-vs-2010-basic-string-exports
33
+ # pragma warning(disable : 4275 4251)
34
+ // Disable diamond inheritance warnings
35
+ # pragma warning(disable : 4250)
36
+ // Disable macro redefinition warnings
37
+ # pragma warning(disable : 4005)
38
+ // Disable extern before exported template warnings
39
+ # pragma warning(disable : 4910)
40
+ #endif
41
+
42
+ // PARQUET-1085
43
+ #if !defined(ARROW_UNUSED)
44
+ # define ARROW_UNUSED(x) UNUSED(x)
45
+ #endif
46
+
47
+ // Parquet exception to Arrow Status
48
+
49
+ #define BEGIN_PARQUET_CATCH_EXCEPTIONS try {
50
+ #define END_PARQUET_CATCH_EXCEPTIONS \
51
+ } \
52
+ catch (const ::parquet::ParquetStatusException& e) { \
53
+ return e.status(); \
54
+ } \
55
+ catch (const ::parquet::ParquetException& e) { \
56
+ return ::arrow::Status::IOError(e.what()); \
57
+ }
58
+
59
+ // clang-format off
60
+
61
+ #define PARQUET_CATCH_NOT_OK(s) \
62
+ BEGIN_PARQUET_CATCH_EXCEPTIONS \
63
+ (s); \
64
+ END_PARQUET_CATCH_EXCEPTIONS
65
+
66
+ // clang-format on
67
+
68
+ #define PARQUET_CATCH_AND_RETURN(s) \
69
+ BEGIN_PARQUET_CATCH_EXCEPTIONS \
70
+ return (s); \
71
+ END_PARQUET_CATCH_EXCEPTIONS
72
+
73
+ // Arrow Status to Parquet exception
74
+
75
+ #define PARQUET_IGNORE_NOT_OK(s) \
76
+ do { \
77
+ ::arrow::Status _s = ::arrow::ToStatus(s); \
78
+ ARROW_UNUSED(_s); \
79
+ } while (0)
80
+
81
+ #define PARQUET_THROW_NOT_OK(s) \
82
+ do { \
83
+ ::arrow::Status _s = ::arrow::ToStatus(s); \
84
+ if (!_s.ok()) { \
85
+ throw ::parquet::ParquetStatusException(std::move(_s)); \
86
+ } \
87
+ } while (0)
88
+
89
+ #define PARQUET_ASSIGN_OR_THROW_IMPL(status_name, lhs, rexpr) \
90
+ auto status_name = (rexpr); \
91
+ PARQUET_THROW_NOT_OK(status_name.status()); \
92
+ lhs = std::move(status_name).ValueOrDie();
93
+
94
+ #define PARQUET_ASSIGN_OR_THROW(lhs, rexpr) \
95
+ PARQUET_ASSIGN_OR_THROW_IMPL(ARROW_ASSIGN_OR_RAISE_NAME(_error_or_value, __COUNTER__), \
96
+ lhs, rexpr);
97
+
98
+ namespace parquet {
99
+
100
+ class PARQUET_EXPORT ParquetException : public std::exception {
101
+ public:
102
+ PARQUET_NORETURN static void EofException(const std::string& msg = "") {
103
+ static std::string prefix = "Unexpected end of stream";
104
+ if (msg.empty()) {
105
+ throw ParquetException(prefix);
106
+ }
107
+ throw ParquetException(prefix, ": ", msg);
108
+ }
109
+
110
+ PARQUET_NORETURN static void NYI(const std::string& msg = "") {
111
+ throw ParquetException("Not yet implemented: ", msg, ".");
112
+ }
113
+
114
+ template <typename... Args>
115
+ explicit ParquetException(Args&&... args)
116
+ : msg_(::arrow::internal::JoinToString(std::forward<Args>(args)...)) {}
117
+
118
+ explicit ParquetException(std::string msg) : msg_(std::move(msg)) {}
119
+
120
+ explicit ParquetException(const char* msg, const std::exception&) : msg_(msg) {}
121
+
122
+ ParquetException(const ParquetException&) = default;
123
+ ParquetException& operator=(const ParquetException&) = default;
124
+ ParquetException(ParquetException&&) = default;
125
+ ParquetException& operator=(ParquetException&&) = default;
126
+
127
+ const char* what() const noexcept override { return msg_.c_str(); }
128
+
129
+ private:
130
+ std::string msg_;
131
+ };
132
+
133
+ // Support printing a ParquetException.
134
+ // This is needed for clang-on-MSVC as there operator<< is not defined for
135
+ // std::exception.
136
+ PARQUET_EXPORT
137
+ std::ostream& operator<<(std::ostream& os, const ParquetException& exception);
138
+
139
+ class ParquetStatusException : public ParquetException {
140
+ public:
141
+ explicit ParquetStatusException(::arrow::Status status)
142
+ : ParquetException(status.ToString()), status_(std::move(status)) {}
143
+
144
+ const ::arrow::Status& status() const { return status_; }
145
+
146
+ private:
147
+ ::arrow::Status status_;
148
+ };
149
+
150
+ // This class exists for the purpose of detecting an invalid or corrupted file.
151
+ class ParquetInvalidOrCorruptedFileException : public ParquetStatusException {
152
+ public:
153
+ ParquetInvalidOrCorruptedFileException(const ParquetInvalidOrCorruptedFileException&) =
154
+ default;
155
+
156
+ template <typename Arg,
157
+ typename std::enable_if<
158
+ !std::is_base_of<ParquetInvalidOrCorruptedFileException, Arg>::value,
159
+ int>::type = 0,
160
+ typename... Args>
161
+ explicit ParquetInvalidOrCorruptedFileException(Arg arg, Args&&... args)
162
+ : ParquetStatusException(::arrow::Status::Invalid(std::forward<Arg>(arg),
163
+ std::forward<Args>(args)...)) {}
164
+ };
165
+
166
+ template <typename StatusReturnBlock>
167
+ void ThrowNotOk(StatusReturnBlock&& b) {
168
+ PARQUET_THROW_NOT_OK(b());
169
+ }
170
+
171
+ } // namespace parquet
172
+
173
+ #ifdef _MSC_VER
174
+ # pragma warning(pop)
175
+ #endif
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/file_reader.h ADDED
@@ -0,0 +1,257 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <cstdint>
21
+ #include <memory>
22
+ #include <string>
23
+ #include <vector>
24
+
25
+ #include "arrow/io/caching.h"
26
+ #include "arrow/util/type_fwd.h"
27
+ #include "parquet/metadata.h" // IWYU pragma: keep
28
+ #include "parquet/platform.h"
29
+ #include "parquet/properties.h"
30
+
31
+ namespace parquet {
32
+
33
+ class ColumnReader;
34
+ class FileMetaData;
35
+ class PageIndexReader;
36
+ class BloomFilterReader;
37
+ class PageReader;
38
+ class RowGroupMetaData;
39
+
40
+ namespace internal {
41
+ class RecordReader;
42
+ }
43
+
44
+ class PARQUET_EXPORT RowGroupReader {
45
+ public:
46
+ // Forward declare a virtual class 'Contents' to aid dependency injection and more
47
+ // easily create test fixtures
48
+ // An implementation of the Contents class is defined in the .cc file
49
+ struct Contents {
50
+ virtual ~Contents() {}
51
+ virtual std::unique_ptr<PageReader> GetColumnPageReader(int i) = 0;
52
+ virtual const RowGroupMetaData* metadata() const = 0;
53
+ virtual const ReaderProperties* properties() const = 0;
54
+ };
55
+
56
+ explicit RowGroupReader(std::unique_ptr<Contents> contents);
57
+
58
+ // Returns the rowgroup metadata
59
+ const RowGroupMetaData* metadata() const;
60
+
61
+ // Construct a ColumnReader for the indicated row group-relative
62
+ // column. Ownership is shared with the RowGroupReader.
63
+ std::shared_ptr<ColumnReader> Column(int i);
64
+
65
+ // EXPERIMENTAL: Construct a RecordReader for the indicated column of the row group.
66
+ // Ownership is shared with the RowGroupReader.
67
+ std::shared_ptr<internal::RecordReader> RecordReader(int i,
68
+ bool read_dictionary = false);
69
+
70
+ // Construct a ColumnReader, trying to enable exposed encoding.
71
+ //
72
+ // For dictionary encoding, currently we only support column chunks that are fully
73
+ // dictionary encoded, i.e., all data pages in the column chunk are dictionary encoded.
74
+ // If a column chunk uses dictionary encoding but then falls back to plain encoding, the
75
+ // encoding will not be exposed.
76
+ //
77
+ // The returned column reader provides an API GetExposedEncoding() for the
78
+ // users to check the exposed encoding and determine how to read the batches.
79
+ //
80
+ // \note API EXPERIMENTAL
81
+ std::shared_ptr<ColumnReader> ColumnWithExposeEncoding(
82
+ int i, ExposedEncoding encoding_to_expose);
83
+
84
+ // Construct a RecordReader, trying to enable exposed encoding.
85
+ //
86
+ // For dictionary encoding, currently we only support column chunks that are
87
+ // fully dictionary encoded byte arrays. The caller should verify if the reader can read
88
+ // and expose the dictionary by checking the reader's read_dictionary(). If a column
89
+ // chunk uses dictionary encoding but then falls back to plain encoding, the returned
90
+ // reader will read decoded data without exposing the dictionary.
91
+ //
92
+ // \note API EXPERIMENTAL
93
+ std::shared_ptr<internal::RecordReader> RecordReaderWithExposeEncoding(
94
+ int i, ExposedEncoding encoding_to_expose);
95
+
96
+ std::unique_ptr<PageReader> GetColumnPageReader(int i);
97
+
98
+ private:
99
+ // Holds a pointer to an instance of Contents implementation
100
+ std::unique_ptr<Contents> contents_;
101
+ };
102
+
103
+ class PARQUET_EXPORT ParquetFileReader {
104
+ public:
105
+ // Declare a virtual class 'Contents' to aid dependency injection and more
106
+ // easily create test fixtures
107
+ // An implementation of the Contents class is defined in the .cc file
108
+ struct PARQUET_EXPORT Contents {
109
+ static std::unique_ptr<Contents> Open(
110
+ std::shared_ptr<::arrow::io::RandomAccessFile> source,
111
+ const ReaderProperties& props = default_reader_properties(),
112
+ std::shared_ptr<FileMetaData> metadata = NULLPTR);
113
+
114
+ static ::arrow::Future<std::unique_ptr<Contents>> OpenAsync(
115
+ std::shared_ptr<::arrow::io::RandomAccessFile> source,
116
+ const ReaderProperties& props = default_reader_properties(),
117
+ std::shared_ptr<FileMetaData> metadata = NULLPTR);
118
+
119
+ virtual ~Contents() = default;
120
+ // Perform any cleanup associated with the file contents
121
+ virtual void Close() = 0;
122
+ virtual std::shared_ptr<RowGroupReader> GetRowGroup(int i) = 0;
123
+ virtual std::shared_ptr<FileMetaData> metadata() const = 0;
124
+ virtual std::shared_ptr<PageIndexReader> GetPageIndexReader() = 0;
125
+ virtual BloomFilterReader& GetBloomFilterReader() = 0;
126
+ };
127
+
128
+ ParquetFileReader();
129
+ ~ParquetFileReader();
130
+
131
+ // Create a file reader instance from an Arrow file object. Thread-safety is
132
+ // the responsibility of the file implementation
133
+ static std::unique_ptr<ParquetFileReader> Open(
134
+ std::shared_ptr<::arrow::io::RandomAccessFile> source,
135
+ const ReaderProperties& props = default_reader_properties(),
136
+ std::shared_ptr<FileMetaData> metadata = NULLPTR);
137
+
138
+ // API Convenience to open a serialized Parquet file on disk, using Arrow IO
139
+ // interfaces.
140
+ static std::unique_ptr<ParquetFileReader> OpenFile(
141
+ const std::string& path, bool memory_map = false,
142
+ const ReaderProperties& props = default_reader_properties(),
143
+ std::shared_ptr<FileMetaData> metadata = NULLPTR);
144
+
145
+ // Asynchronously open a file reader from an Arrow file object.
146
+ // Does not throw - all errors are reported through the Future.
147
+ static ::arrow::Future<std::unique_ptr<ParquetFileReader>> OpenAsync(
148
+ std::shared_ptr<::arrow::io::RandomAccessFile> source,
149
+ const ReaderProperties& props = default_reader_properties(),
150
+ std::shared_ptr<FileMetaData> metadata = NULLPTR);
151
+
152
+ void Open(std::unique_ptr<Contents> contents);
153
+ void Close();
154
+
155
+ // The RowGroupReader is owned by the FileReader
156
+ std::shared_ptr<RowGroupReader> RowGroup(int i);
157
+
158
+ // Returns the file metadata. Only one instance is ever created
159
+ std::shared_ptr<FileMetaData> metadata() const;
160
+
161
+ /// Returns the PageIndexReader. Only one instance is ever created.
162
+ ///
163
+ /// If the file does not have the page index, nullptr may be returned.
164
+ /// Because it pays to check existence of page index in the file, it
165
+ /// is possible to return a non null value even if page index does
166
+ /// not exist. It is the caller's responsibility to check the return
167
+ /// value and follow-up calls to PageIndexReader.
168
+ ///
169
+ /// WARNING: The returned PageIndexReader must not outlive the ParquetFileReader.
170
+ /// Initialize GetPageIndexReader() is not thread-safety.
171
+ std::shared_ptr<PageIndexReader> GetPageIndexReader();
172
+
173
+ /// Returns the BloomFilterReader. Only one instance is ever created.
174
+ ///
175
+ /// WARNING: The returned BloomFilterReader must not outlive the ParquetFileReader.
176
+ /// Initialize GetBloomFilterReader() is not thread-safety.
177
+ BloomFilterReader& GetBloomFilterReader();
178
+
179
+ /// Pre-buffer the specified column indices in all row groups.
180
+ ///
181
+ /// Readers can optionally call this to cache the necessary slices
182
+ /// of the file in-memory before deserialization. Arrow readers can
183
+ /// automatically do this via an option. This is intended to
184
+ /// increase performance when reading from high-latency filesystems
185
+ /// (e.g. Amazon S3).
186
+ ///
187
+ /// After calling this, creating readers for row groups/column
188
+ /// indices that were not buffered may fail. Creating multiple
189
+ /// readers for the a subset of the buffered regions is
190
+ /// acceptable. This may be called again to buffer a different set
191
+ /// of row groups/columns.
192
+ ///
193
+ /// If memory usage is a concern, note that data will remain
194
+ /// buffered in memory until either \a PreBuffer() is called again,
195
+ /// or the reader itself is destructed. Reading - and buffering -
196
+ /// only one row group at a time may be useful.
197
+ ///
198
+ /// This method may throw.
199
+ void PreBuffer(const std::vector<int>& row_groups,
200
+ const std::vector<int>& column_indices,
201
+ const ::arrow::io::IOContext& ctx,
202
+ const ::arrow::io::CacheOptions& options);
203
+
204
+ /// Retrieve the list of byte ranges that would need to be read to retrieve
205
+ /// the data for the specified row groups and column indices.
206
+ ///
207
+ /// A reader can optionally call this if they wish to handle their own
208
+ /// caching and management of file reads (or offload them to other readers).
209
+ /// Unlike PreBuffer, this method will not perform any actual caching or
210
+ /// reads, instead just using the file metadata to determine the byte ranges
211
+ /// that would need to be read if you were to consume the entirety of the column
212
+ /// chunks for the provided columns in the specified row groups.
213
+ ///
214
+ /// If row_groups or column_indices are empty, then the result of this will be empty.
215
+ ///
216
+ /// hole_size_limit represents the maximum distance, in bytes, between two
217
+ /// consecutive ranges; beyond this value, ranges will not be combined. The default
218
+ /// value is 1MB.
219
+ ///
220
+ /// range_size_limit is the maximum size in bytes of a combined range; if combining
221
+ /// two consecutive ranges would produce a range larger than this, they are not
222
+ /// combined. The default values is 64MB. This *must* be larger than hole_size_limit.
223
+ ///
224
+ /// This will not take into account page indexes or any other predicate push down
225
+ /// benefits that may be available.
226
+ ::arrow::Result<std::vector<::arrow::io::ReadRange>> GetReadRanges(
227
+ const std::vector<int>& row_groups, const std::vector<int>& column_indices,
228
+ int64_t hole_size_limit = 1024 * 1024, int64_t range_size_limit = 64 * 1024 * 1024);
229
+
230
+ /// Wait for the specified row groups and column indices to be pre-buffered.
231
+ ///
232
+ /// After the returned Future completes, reading the specified row
233
+ /// groups/columns will not block.
234
+ ///
235
+ /// PreBuffer must be called first. This method does not throw.
236
+ ::arrow::Future<> WhenBuffered(const std::vector<int>& row_groups,
237
+ const std::vector<int>& column_indices) const;
238
+
239
+ private:
240
+ // Holds a pointer to an instance of Contents implementation
241
+ std::unique_ptr<Contents> contents_;
242
+ };
243
+
244
+ // Read only Parquet file metadata
245
+ std::shared_ptr<FileMetaData> PARQUET_EXPORT
246
+ ReadMetaData(const std::shared_ptr<::arrow::io::RandomAccessFile>& source);
247
+
248
+ /// \brief Scan all values in file. Useful for performance testing
249
+ /// \param[in] columns the column numbers to scan. If empty scans all
250
+ /// \param[in] column_batch_size number of values to read at a time when scanning column
251
+ /// \param[in] reader a ParquetFileReader instance
252
+ /// \return number of semantic rows in file
253
+ PARQUET_EXPORT
254
+ int64_t ScanFileContents(std::vector<int> columns, const int32_t column_batch_size,
255
+ ParquetFileReader* reader);
256
+
257
+ } // namespace parquet
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/file_writer.h ADDED
@@ -0,0 +1,260 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <cstdint>
21
+ #include <memory>
22
+ #include <utility>
23
+
24
+ #include "parquet/metadata.h"
25
+ #include "parquet/platform.h"
26
+ #include "parquet/properties.h"
27
+ #include "parquet/schema.h"
28
+
29
+ namespace parquet {
30
+
31
+ class ColumnWriter;
32
+
33
+ // FIXME: copied from reader-internal.cc
34
+ static constexpr uint8_t kParquetMagic[4] = {'P', 'A', 'R', '1'};
35
+ static constexpr uint8_t kParquetEMagic[4] = {'P', 'A', 'R', 'E'};
36
+
37
+ class PARQUET_EXPORT RowGroupWriter {
38
+ public:
39
+ // Estimated uncompressed byte sizes of data buffered by column writers
40
+ // that have not yet been serialized into pages.
41
+ struct BufferedStats {
42
+ int64_t def_level_bytes = 0;
43
+ int64_t rep_level_bytes = 0;
44
+ int64_t value_bytes = 0;
45
+ int64_t dict_bytes = 0;
46
+ };
47
+
48
+ // Forward declare a virtual class 'Contents' to aid dependency injection and more
49
+ // easily create test fixtures
50
+ // An implementation of the Contents class is defined in the .cc file
51
+ struct Contents {
52
+ virtual ~Contents() = default;
53
+ virtual int num_columns() const = 0;
54
+ virtual int64_t num_rows() const = 0;
55
+
56
+ // to be used only with ParquetFileWriter::AppendRowGroup
57
+ virtual ColumnWriter* NextColumn() = 0;
58
+ // to be used only with ParquetFileWriter::AppendBufferedRowGroup
59
+ virtual ColumnWriter* column(int i) = 0;
60
+
61
+ virtual int current_column() const = 0;
62
+ virtual void Close() = 0;
63
+
64
+ /// \brief total uncompressed bytes written by the page writer
65
+ virtual int64_t total_bytes_written() const = 0;
66
+ /// \brief total bytes still compressed but not written by the page writer
67
+ virtual int64_t total_compressed_bytes() const = 0;
68
+ /// \brief total compressed bytes written by the page writer
69
+ virtual int64_t total_compressed_bytes_written() const = 0;
70
+ /// \brief Estimated sizes of buffered data (levels, values, dict) not yet
71
+ /// written to pages.
72
+ virtual BufferedStats EstimatedBufferedStats() const = 0;
73
+
74
+ virtual bool buffered() const = 0;
75
+ };
76
+
77
+ explicit RowGroupWriter(std::unique_ptr<Contents> contents);
78
+
79
+ /// Construct a ColumnWriter for the indicated row group-relative column.
80
+ ///
81
+ /// To be used only with ParquetFileWriter::AppendRowGroup
82
+ /// Ownership is solely within the RowGroupWriter. The ColumnWriter is only
83
+ /// valid until the next call to NextColumn or Close. As the contents are
84
+ /// directly written to the sink, once a new column is started, the contents
85
+ /// of the previous one cannot be modified anymore.
86
+ ColumnWriter* NextColumn();
87
+ /// Index of currently written column. Equal to -1 if NextColumn()
88
+ /// has not been called yet.
89
+ int current_column();
90
+ void Close();
91
+
92
+ int num_columns() const;
93
+
94
+ /// Construct a ColumnWriter for the indicated row group column.
95
+ ///
96
+ /// To be used only with ParquetFileWriter::AppendBufferedRowGroup
97
+ /// Ownership is solely within the RowGroupWriter. The ColumnWriter is
98
+ /// valid until Close. The contents are buffered in memory and written to sink
99
+ /// on Close
100
+ ColumnWriter* column(int i);
101
+
102
+ /**
103
+ * Number of rows that shall be written as part of this RowGroup.
104
+ */
105
+ int64_t num_rows() const;
106
+
107
+ /// \brief total uncompressed bytes written by the page writer
108
+ int64_t total_bytes_written() const;
109
+ /// \brief total bytes still compressed but not written by the page writer.
110
+ /// It will always return 0 from the SerializedPageWriter.
111
+ int64_t total_compressed_bytes() const;
112
+ /// \brief total compressed bytes written by the page writer
113
+ int64_t total_compressed_bytes_written() const;
114
+ /// \brief Estimated sizes of buffered data (levels, values, dict) not yet
115
+ /// written to pages.
116
+ BufferedStats estimated_buffered_stats() const;
117
+
118
+ /// Returns whether the current RowGroupWriter is in the buffered mode and is created
119
+ /// by calling ParquetFileWriter::AppendBufferedRowGroup.
120
+ bool buffered() const;
121
+
122
+ private:
123
+ // Holds a pointer to an instance of Contents implementation
124
+ std::unique_ptr<Contents> contents_;
125
+ };
126
+
127
+ PARQUET_EXPORT
128
+ void WriteFileMetaData(const FileMetaData& file_metadata,
129
+ ::arrow::io::OutputStream* sink);
130
+
131
+ PARQUET_EXPORT
132
+ void WriteMetaDataFile(const FileMetaData& file_metadata,
133
+ ::arrow::io::OutputStream* sink);
134
+
135
+ PARQUET_EXPORT
136
+ void WriteEncryptedFileMetadata(const FileMetaData& file_metadata,
137
+ ArrowOutputStream* sink,
138
+ const std::shared_ptr<Encryptor>& encryptor,
139
+ bool encrypt_footer);
140
+
141
+ PARQUET_EXPORT
142
+ void WriteEncryptedFileMetadata(const FileMetaData& file_metadata,
143
+ ::arrow::io::OutputStream* sink,
144
+ const std::shared_ptr<Encryptor>& encryptor = NULLPTR,
145
+ bool encrypt_footer = false);
146
+ PARQUET_EXPORT
147
+ void WriteFileCryptoMetaData(const FileCryptoMetaData& crypto_metadata,
148
+ ::arrow::io::OutputStream* sink);
149
+
150
+ class PARQUET_EXPORT ParquetFileWriter {
151
+ public:
152
+ // Forward declare a virtual class 'Contents' to aid dependency injection and more
153
+ // easily create test fixtures
154
+ // An implementation of the Contents class is defined in the .cc file
155
+ struct Contents {
156
+ Contents(std::shared_ptr<::parquet::schema::GroupNode> schema,
157
+ std::shared_ptr<const KeyValueMetadata> key_value_metadata)
158
+ : schema_(), key_value_metadata_(std::move(key_value_metadata)) {
159
+ schema_.Init(std::move(schema));
160
+ }
161
+ virtual ~Contents() {}
162
+ // Perform any cleanup associated with the file contents
163
+ virtual void Close() = 0;
164
+
165
+ virtual RowGroupWriter* AppendRowGroup() = 0;
166
+ virtual RowGroupWriter* AppendBufferedRowGroup() = 0;
167
+
168
+ virtual int64_t num_rows() const = 0;
169
+ virtual int num_columns() const = 0;
170
+ virtual int num_row_groups() const = 0;
171
+
172
+ virtual const std::shared_ptr<WriterProperties>& properties() const = 0;
173
+
174
+ const std::shared_ptr<const KeyValueMetadata>& key_value_metadata() const {
175
+ return key_value_metadata_;
176
+ }
177
+
178
+ virtual void AddKeyValueMetadata(
179
+ const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) = 0;
180
+
181
+ // Return const-pointer to make it clear that this object is not to be copied
182
+ const SchemaDescriptor* schema() const { return &schema_; }
183
+
184
+ SchemaDescriptor schema_;
185
+
186
+ /// This should be the only place this is stored. Everything else is a const reference
187
+ std::shared_ptr<const KeyValueMetadata> key_value_metadata_;
188
+
189
+ const std::shared_ptr<FileMetaData>& metadata() const { return file_metadata_; }
190
+ std::shared_ptr<FileMetaData> file_metadata_;
191
+ };
192
+
193
+ ParquetFileWriter();
194
+ ~ParquetFileWriter();
195
+
196
+ static std::unique_ptr<ParquetFileWriter> Open(
197
+ std::shared_ptr<::arrow::io::OutputStream> sink,
198
+ std::shared_ptr<schema::GroupNode> schema,
199
+ std::shared_ptr<WriterProperties> properties = default_writer_properties(),
200
+ std::shared_ptr<const KeyValueMetadata> key_value_metadata = NULLPTR);
201
+
202
+ void Open(std::unique_ptr<Contents> contents);
203
+ void Close();
204
+
205
+ /// Construct a RowGroupWriter with an arbitrary number of rows.
206
+ ///
207
+ /// Ownership is solely within the ParquetFileWriter. The RowGroupWriter is only valid
208
+ /// until the next call to AppendRowGroup or AppendBufferedRowGroup or Close.
209
+ RowGroupWriter* AppendRowGroup();
210
+
211
+ /// Construct a RowGroupWriter that buffers all the values until the RowGroup is ready.
212
+ /// Use this if you want to write a RowGroup based on a certain size
213
+ ///
214
+ /// Ownership is solely within the ParquetFileWriter. The RowGroupWriter is only valid
215
+ /// until the next call to AppendRowGroup or AppendBufferedRowGroup or Close.
216
+ RowGroupWriter* AppendBufferedRowGroup();
217
+
218
+ /// \brief Add key-value metadata to the file.
219
+ /// \param[in] key_value_metadata the metadata to add.
220
+ /// \note This will overwrite any existing metadata with the same key(s).
221
+ /// \throw ParquetException if Close() has been called.
222
+ void AddKeyValueMetadata(
223
+ const std::shared_ptr<const KeyValueMetadata>& key_value_metadata);
224
+
225
+ /// Number of columns.
226
+ ///
227
+ /// This number is fixed during the lifetime of the writer as it is determined via
228
+ /// the schema.
229
+ int num_columns() const;
230
+
231
+ /// Number of rows in the yet started RowGroups.
232
+ ///
233
+ /// Changes on the addition of a new RowGroup.
234
+ int64_t num_rows() const;
235
+
236
+ /// Number of started RowGroups.
237
+ int num_row_groups() const;
238
+
239
+ /// Configuration passed to the writer, e.g. the used Parquet format version.
240
+ const std::shared_ptr<WriterProperties>& properties() const;
241
+
242
+ /// Returns the file schema descriptor
243
+ const SchemaDescriptor* schema() const;
244
+
245
+ /// Returns a column descriptor in schema
246
+ const ColumnDescriptor* descr(int i) const;
247
+
248
+ /// Returns the file custom metadata
249
+ const std::shared_ptr<const KeyValueMetadata>& key_value_metadata() const;
250
+
251
+ /// Returns the file metadata, only available after calling Close().
252
+ const std::shared_ptr<FileMetaData> metadata() const;
253
+
254
+ private:
255
+ // Holds a pointer to an instance of Contents implementation
256
+ std::unique_ptr<Contents> contents_;
257
+ std::shared_ptr<FileMetaData> file_metadata_;
258
+ };
259
+
260
+ } // namespace parquet
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/geospatial/statistics.h ADDED
@@ -0,0 +1,198 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <cstdint>
21
+ #include <memory>
22
+ #include <optional>
23
+
24
+ #include "parquet/platform.h"
25
+ #include "parquet/types.h"
26
+
27
+ namespace parquet::geospatial {
28
+
29
+ /// \brief The maximum number of dimensions represented by a geospatial type
30
+ /// (i.e., X, Y, Z, and M)
31
+ inline constexpr int kMaxDimensions = 4;
32
+
33
+ /// \brief NaN, used to represent bounds for which predicate pushdown cannnot
34
+ /// be applied (e.g., because a writer did not provide bounds for a given dimension)
35
+ inline constexpr double kNaN = std::numeric_limits<double>::quiet_NaN();
36
+
37
+ /// \brief Structure represented encoded statistics to be written to and read from Parquet
38
+ /// serialized metadata.
39
+ ///
40
+ /// See the Parquet Thrift definition and GeoStatistics for the specific definition
41
+ /// of field values.
42
+ struct PARQUET_EXPORT EncodedGeoStatistics {
43
+ bool xy_bounds_present{false};
44
+ double xmin{kNaN};
45
+ double xmax{kNaN};
46
+ double ymin{kNaN};
47
+ double ymax{kNaN};
48
+
49
+ bool z_bounds_present{false};
50
+ double zmin{kNaN};
51
+ double zmax{kNaN};
52
+
53
+ bool m_bounds_present{false};
54
+ double mmin{kNaN};
55
+ double mmax{kNaN};
56
+
57
+ bool geospatial_types_present() const { return !geospatial_types.empty(); }
58
+ std::vector<int32_t> geospatial_types;
59
+ };
60
+
61
+ class GeoStatisticsImpl;
62
+
63
+ /// \brief Base type for computing geospatial column statistics while writing a file
64
+ /// or representing them when reading a file
65
+ ///
66
+ /// These statistics track the minimum and maximum value (omitting NaN values) of the
67
+ /// four possible dimensions (X, Y, Z, and M) and the distinct set of geometry
68
+ /// type/dimension combinations (e.g., point XY, linestring XYZM) present in the data.
69
+ /// Any of these individual components may be "invalid": for example, when reading a
70
+ /// Parquet file, information about individual components obtained from the column
71
+ /// chunk metadata may have been missing or deemed unusable. Orthogonally,
72
+ /// any of these individual components may be "empty": for example, when using
73
+ /// GeoStatistics to accumulate bounds whilst writing, if all geometries in a column chunk
74
+ /// are null, all ranges (X, Y, Z, and M) will be empty. If all geometries in a column
75
+ /// chunk contain only XY coordinates (the most common case), the Z and M ranges will
76
+ /// be empty but the X and Y ranges will contain finite bounds. Empty ranges are
77
+ /// considered "valid" because they are known to represent exactly zero values (in
78
+ /// contrast to an invalid range, whose contents is completely unknown). These concepts
79
+ /// are all necessary for this object to accurately represent (1) accumulated or partially
80
+ /// accumulated statistics during the writing process and (2) deserialized statistics read
81
+ /// from the column chunk metadata during the reading process.
82
+ ///
83
+ /// EXPERIMENTAL
84
+ class PARQUET_EXPORT GeoStatistics {
85
+ public:
86
+ GeoStatistics();
87
+ explicit GeoStatistics(const EncodedGeoStatistics& encoded);
88
+
89
+ ~GeoStatistics();
90
+
91
+ /// \brief Return true if bounds, geometry types, and validity are identical
92
+ bool Equals(const GeoStatistics& other) const;
93
+
94
+ /// \brief Update these statistics based on previously calculated or decoded statistics
95
+ ///
96
+ /// Merging statistics with wraparound X values is not currently supported. Merging
97
+ /// two GeoStatistics where one or both has a wraparound X range will result in these
98
+ /// statistics having an X dimension marked as invalid.
99
+ void Merge(const GeoStatistics& other);
100
+
101
+ /// \brief Update these statistics based on values
102
+ void Update(const ByteArray* values, int64_t num_values);
103
+
104
+ /// \brief Update these statistics based on the non-null elements of values
105
+ void UpdateSpaced(const ByteArray* values, const uint8_t* valid_bits,
106
+ int64_t valid_bits_offset, int64_t num_spaced_values,
107
+ int64_t num_values);
108
+
109
+ /// \brief Update these statistics based on the non-null elements of values
110
+ ///
111
+ /// Currently, BinaryArray and LargeBinaryArray input is supported.
112
+ void Update(const ::arrow::Array& values);
113
+
114
+ /// \brief Return these statistics to an empty state
115
+ void Reset();
116
+
117
+ /// \brief Encode the statistics for serializing to Thrift
118
+ ///
119
+ /// If invalid WKB was encountered or if the statistics contain NaN
120
+ /// for any reason, Encode() will return nullopt to indicate that
121
+ /// statistics should not be written to thrift.
122
+ std::optional<EncodedGeoStatistics> Encode() const;
123
+
124
+ /// \brief Returns false if invalid WKB was encountered
125
+ bool is_valid() const;
126
+
127
+ /// \brief Reset existing statistics and populate them from previously-encoded ones
128
+ void Decode(const EncodedGeoStatistics& encoded);
129
+
130
+ /// \brief Minimum values in XYZM order
131
+ ///
132
+ /// For dimensions where dimension_valid() is false, the value will be NaN. For
133
+ /// dimensions where dimension_empty() is true, the value will be +Inf.
134
+ ///
135
+ /// For the first dimension (X) only, wraparound bounds apply where xmin > xmax. In this
136
+ /// case, these bounds represent the union of the intervals [xmax, Inf] and [-Inf,
137
+ /// xmin]. This implementation does not yet generate these types of bounds but they may
138
+ /// be encountered in statistics when reading a Parquet file.
139
+ std::array<double, kMaxDimensions> lower_bound() const;
140
+
141
+ /// \brief Maximum values in XYZM order
142
+ ///
143
+ /// For dimensions where dimension_valid() is false, the value will be NaN. For
144
+ /// dimensions where dimension_empty() is true, the value will be -Inf.
145
+ ///
146
+ /// For the first dimension (X) only, wraparound bounds apply where xmin > xmax. In this
147
+ /// case, these bounds represent the union of the intervals [xmax, Inf] and [-Inf,
148
+ /// xmin]. This implementation does not yet generate these types of bounds but they may
149
+ /// be encountered in statistics when reading a Parquet file.
150
+ std::array<double, kMaxDimensions> upper_bound() const;
151
+
152
+ /// \brief Dimension emptiness in XYZM order
153
+ ///
154
+ /// True for a given dimension if and only if zero non-NaN values were encountered
155
+ /// in that dimension and dimension_valid() is true for that dimension.
156
+ ///
157
+ /// When calculating statistics, zero or more of these values may be true because
158
+ /// this implementation calculates bounds for all dimensions; however, it may be
159
+ /// true that zero coordinates were encountered in a given dimension. For example,
160
+ /// dimension_empty() will return four true values if Update() was not called
161
+ /// or if Update() was called with only null values. If Update() was provided
162
+ /// one or more geometries with X and Y dimensions but not Z or M dimensions,
163
+ /// dimension_empty() will return true, true, false, false.
164
+ ///
165
+ /// For statistics read from a Parquet file, dimension_empty() will always contain
166
+ /// false values because there is no mechanism to communicate an empty interval
167
+ /// in the Thrift metadata.
168
+ std::array<bool, kMaxDimensions> dimension_empty() const;
169
+
170
+ /// \brief Dimension validity (i.e. presence) in XYZM order
171
+ ///
172
+ /// When calculating statistics, this will always be true because this implementation
173
+ /// calculates statistics for all dimensions. When reading a Parquet file, one or more
174
+ /// of these values may be false because the file may not have provided bounds for all
175
+ /// dimensions.
176
+ ///
177
+ /// See documentation for dimension_empty(), lower_bound(), and/or upper_bound() for the
178
+ /// canonical values of those outputs for the dimensions where dimension_valid() is
179
+ /// false.
180
+ std::array<bool, kMaxDimensions> dimension_valid() const;
181
+
182
+ /// \brief Return the geometry type codes
183
+ ///
184
+ /// This implementation always returns sorted output with no duplicates. When
185
+ /// calculating statistics, a value will always be returned (although the returned
186
+ /// vector may be empty if Update() was never called or was only called with null
187
+ /// values). When reading a Parquet file, std::nullopt may be returned because
188
+ /// the file may not have provided this information.
189
+ std::optional<std::vector<int32_t>> geometry_types() const;
190
+
191
+ /// \brief Return a string representation of these statistics
192
+ std::string ToString() const;
193
+
194
+ private:
195
+ std::unique_ptr<GeoStatisticsImpl> impl_;
196
+ };
197
+
198
+ } // namespace parquet::geospatial
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/hasher.h ADDED
@@ -0,0 +1,136 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <cstdint>
21
+ #include "parquet/types.h"
22
+
23
+ namespace parquet {
24
+ // Abstract class for hash
25
+ class PARQUET_EXPORT Hasher {
26
+ public:
27
+ /// Compute hash for 32 bits value by using its plain encoding result.
28
+ ///
29
+ /// @param value the value to hash.
30
+ /// @return hash result.
31
+ virtual uint64_t Hash(int32_t value) const = 0;
32
+
33
+ /// Compute hash for 64 bits value by using its plain encoding result.
34
+ ///
35
+ /// @param value the value to hash.
36
+ /// @return hash result.
37
+ virtual uint64_t Hash(int64_t value) const = 0;
38
+
39
+ /// Compute hash for float value by using its plain encoding result.
40
+ ///
41
+ /// @param value the value to hash.
42
+ /// @return hash result.
43
+ virtual uint64_t Hash(float value) const = 0;
44
+
45
+ /// Compute hash for double value by using its plain encoding result.
46
+ ///
47
+ /// @param value the value to hash.
48
+ /// @return hash result.
49
+ virtual uint64_t Hash(double value) const = 0;
50
+
51
+ /// Compute hash for Int96 value by using its plain encoding result.
52
+ ///
53
+ /// @param value the value to hash.
54
+ /// @return hash result.
55
+ virtual uint64_t Hash(const Int96* value) const = 0;
56
+
57
+ /// Compute hash for ByteArray value by using its plain encoding result.
58
+ ///
59
+ /// @param value the value to hash.
60
+ /// @return hash result.
61
+ virtual uint64_t Hash(const ByteArray* value) const = 0;
62
+
63
+ /// Compute hash for fixed byte array value by using its plain encoding result.
64
+ ///
65
+ /// @param value the value address.
66
+ /// @param len the value length.
67
+ virtual uint64_t Hash(const FLBA* value, uint32_t len) const = 0;
68
+
69
+ /// Compute hash for std::string_view value by using its plain encoding result.
70
+ ///
71
+ /// @param value the value to hash.
72
+ virtual uint64_t Hash(std::string_view value) const = 0;
73
+
74
+ /// Batch compute hashes for 32 bits values by using its plain encoding result.
75
+ ///
76
+ /// @param values a pointer to the values to hash.
77
+ /// @param num_values the number of values to hash.
78
+ /// @param hashes a pointer to the output hash values, its length should be equal to
79
+ /// num_values.
80
+ virtual void Hashes(const int32_t* values, int num_values, uint64_t* hashes) const = 0;
81
+
82
+ /// Batch compute hashes for 64 bits values by using its plain encoding result.
83
+ ///
84
+ /// @param values a pointer to the values to hash.
85
+ /// @param num_values the number of values to hash.
86
+ /// @param hashes a pointer to the output hash values, its length should be equal to
87
+ /// num_values.
88
+ virtual void Hashes(const int64_t* values, int num_values, uint64_t* hashes) const = 0;
89
+
90
+ /// Batch compute hashes for float values by using its plain encoding result.
91
+ ///
92
+ /// @param values a pointer to the values to hash.
93
+ /// @param num_values the number of values to hash.
94
+ /// @param hashes a pointer to the output hash values, its length should be equal to
95
+ /// num_values.
96
+ virtual void Hashes(const float* values, int num_values, uint64_t* hashes) const = 0;
97
+
98
+ /// Batch compute hashes for double values by using its plain encoding result.
99
+ ///
100
+ /// @param values a pointer to the values to hash.
101
+ /// @param num_values the number of values to hash.
102
+ /// @param hashes a pointer to the output hash values, its length should be equal to
103
+ /// num_values.
104
+ virtual void Hashes(const double* values, int num_values, uint64_t* hashes) const = 0;
105
+
106
+ /// Batch compute hashes for Int96 values by using its plain encoding result.
107
+ ///
108
+ /// @param values a pointer to the values to hash.
109
+ /// @param num_values the number of values to hash.
110
+ /// @param hashes a pointer to the output hash values, its length should be equal to
111
+ /// num_values.
112
+ virtual void Hashes(const Int96* values, int num_values, uint64_t* hashes) const = 0;
113
+
114
+ /// Batch compute hashes for ByteArray values by using its plain encoding result.
115
+ ///
116
+ /// @param values a pointer to the values to hash.
117
+ /// @param num_values the number of values to hash.
118
+ /// @param hashes a pointer to the output hash values, its length should be equal to
119
+ /// num_values.
120
+ virtual void Hashes(const ByteArray* values, int num_values,
121
+ uint64_t* hashes) const = 0;
122
+
123
+ /// Batch compute hashes for fixed byte array values by using its plain encoding result.
124
+ ///
125
+ /// @param values the value address.
126
+ /// @param type_len the value length.
127
+ /// @param num_values the number of values to hash.
128
+ /// @param hashes a pointer to the output hash values, its length should be equal to
129
+ /// num_values.
130
+ virtual void Hashes(const FLBA* values, uint32_t type_len, int num_values,
131
+ uint64_t* hashes) const = 0;
132
+
133
+ virtual ~Hasher() = default;
134
+ };
135
+
136
+ } // namespace parquet
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/index_location.h ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <cstdint>
21
+ #include <vector>
22
+
23
+ #include "parquet/platform.h"
24
+
25
+ namespace parquet {
26
+
27
+ enum class PARQUET_EXPORT IndexKind : uint8_t {
28
+ kColumnIndex,
29
+ kOffsetIndex,
30
+ kBloomFilter,
31
+ };
32
+
33
+ /// \brief Location to a Parquet index (page index or bloom filter).
34
+ struct PARQUET_EXPORT IndexLocation {
35
+ // File offset of the given index, in bytes
36
+ int64_t offset;
37
+ // Length of the given index, in bytes
38
+ int32_t length;
39
+ };
40
+
41
+ /// \brief Identifier for a column chunk.
42
+ struct PARQUET_EXPORT ColumnChunkId {
43
+ int32_t row_group_index;
44
+ int32_t column_index;
45
+ };
46
+
47
+ using IndexLocations = std::vector<std::pair<ColumnChunkId, IndexLocation>>;
48
+
49
+ } // namespace parquet
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_comparison.h ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+ #pragma once
18
+
19
+ #include <algorithm>
20
+ #include <cstdint>
21
+
22
+ #include "parquet/platform.h"
23
+
24
+ namespace parquet::internal {
25
+
26
+ /// Builds a bitmap where each set bit indicates the corresponding level is greater
27
+ /// than rhs.
28
+ uint64_t PARQUET_EXPORT GreaterThanBitmap(const int16_t* levels, int64_t num_levels,
29
+ int16_t rhs);
30
+
31
+ struct MinMax {
32
+ int16_t min;
33
+ int16_t max;
34
+ };
35
+
36
+ MinMax FindMinMax(const int16_t* levels, int64_t num_levels);
37
+
38
+ } // namespace parquet::internal
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_comparison_inc.h ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+ #pragma once
18
+
19
+ #include "arrow/util/bit_util.h"
20
+ #include "arrow/util/endian.h"
21
+ #include "parquet/level_comparison.h"
22
+
23
+ // Used to make sure ODR rule isn't violated.
24
+ #ifndef PARQUET_IMPL_NAMESPACE
25
+ # error "PARQUET_IMPL_NAMESPACE must be defined"
26
+ #endif
27
+ namespace parquet::internal::PARQUET_IMPL_NAMESPACE {
28
+ /// Builds a bitmap by applying predicate to the level vector provided.
29
+ ///
30
+ /// \param[in] levels Rep or def level array.
31
+ /// \param[in] num_levels The number of levels to process (must be [0, 64])
32
+ /// \param[in] predicate The predicate to apply (must have the signature `bool
33
+ /// predicate(int16_t)`.
34
+ /// \returns The bitmap using least significant "bit" ordering.
35
+ ///
36
+ template <typename Predicate>
37
+ inline uint64_t LevelsToBitmap(const int16_t* levels, int64_t num_levels,
38
+ Predicate predicate) {
39
+ // Both clang and GCC can vectorize this automatically with SSE4/AVX2.
40
+ uint64_t mask = 0;
41
+ for (int x = 0; x < num_levels; x++) {
42
+ mask |= static_cast<uint64_t>(predicate(levels[x]) ? 1 : 0) << x;
43
+ }
44
+ return ::arrow::bit_util::ToLittleEndian(mask);
45
+ }
46
+
47
+ inline MinMax FindMinMaxImpl(const int16_t* levels, int64_t num_levels) {
48
+ MinMax out{std::numeric_limits<int16_t>::max(), std::numeric_limits<int16_t>::min()};
49
+ for (int x = 0; x < num_levels; x++) {
50
+ out.min = std::min(levels[x], out.min);
51
+ out.max = std::max(levels[x], out.max);
52
+ }
53
+ return out;
54
+ }
55
+
56
+ inline uint64_t GreaterThanBitmapImpl(const int16_t* levels, int64_t num_levels,
57
+ int16_t rhs) {
58
+ return LevelsToBitmap(levels, num_levels, [rhs](int16_t value) { return value > rhs; });
59
+ }
60
+
61
+ } // namespace parquet::internal::PARQUET_IMPL_NAMESPACE
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_conversion.h ADDED
@@ -0,0 +1,216 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <cstdint>
21
+
22
+ #include "arrow/util/endian.h"
23
+ #include "parquet/platform.h"
24
+ #include "parquet/schema.h"
25
+
26
+ namespace parquet::internal {
27
+
28
+ struct PARQUET_EXPORT LevelInfo {
29
+ LevelInfo()
30
+ : null_slot_usage(1), def_level(0), rep_level(0), repeated_ancestor_def_level(0) {}
31
+ LevelInfo(int32_t null_slots, int32_t definition_level, int32_t repetition_level,
32
+ int32_t repeated_ancestor_definition_level)
33
+ : null_slot_usage(null_slots),
34
+ def_level(static_cast<int16_t>(definition_level)),
35
+ rep_level(static_cast<int16_t>(repetition_level)),
36
+ repeated_ancestor_def_level(
37
+ static_cast<int16_t>(repeated_ancestor_definition_level)) {}
38
+
39
+ bool operator==(const LevelInfo& b) const {
40
+ return null_slot_usage == b.null_slot_usage && def_level == b.def_level &&
41
+ rep_level == b.rep_level &&
42
+ repeated_ancestor_def_level == b.repeated_ancestor_def_level;
43
+ }
44
+
45
+ bool HasNullableValues() const { return repeated_ancestor_def_level < def_level; }
46
+
47
+ // How many slots an undefined but present (i.e. null) element in
48
+ // parquet consumes when decoding to Arrow.
49
+ // "Slot" is used in the same context as the Arrow specification
50
+ // (i.e. a value holder).
51
+ // This is only ever >1 for descendents of FixedSizeList.
52
+ int32_t null_slot_usage = 1;
53
+
54
+ // The definition level at which the value for the field
55
+ // is considered not null (definition levels greater than
56
+ // or equal to this value indicate a not-null
57
+ // value for the field). For list fields definition levels
58
+ // greater than or equal to this field indicate a present,
59
+ // possibly null, child value.
60
+ int16_t def_level = 0;
61
+
62
+ // The repetition level corresponding to this element
63
+ // or the closest repeated ancestor. Any repetition
64
+ // level less than this indicates either a new list OR
65
+ // an empty list (which is determined in conjunction
66
+ // with definition levels).
67
+ int16_t rep_level = 0;
68
+
69
+ // The definition level indicating the level at which the closest
70
+ // repeated ancestor is not empty. This is used to discriminate
71
+ // between a value less than |def_level| being null or excluded entirely.
72
+ // For instance if we have an arrow schema like:
73
+ // list(struct(f0: int)). Then then there are the following
74
+ // definition levels:
75
+ // 0 = null list
76
+ // 1 = present but empty list.
77
+ // 2 = a null value in the list
78
+ // 3 = a non null struct but null integer.
79
+ // 4 = a present integer.
80
+ // When reconstructing, the struct and integer arrays'
81
+ // repeated_ancestor_def_level would be 2. Any
82
+ // def_level < 2 indicates that there isn't a corresponding
83
+ // child value in the list.
84
+ // i.e. [null, [], [null], [{f0: null}], [{f0: 1}]]
85
+ // has the def levels [0, 1, 2, 3, 4]. The actual
86
+ // struct array is only of length 3: [not-set, set, set] and
87
+ // the int array is also of length 3: [N/A, null, 1].
88
+ //
89
+ int16_t repeated_ancestor_def_level = 0;
90
+
91
+ /// Increments levels according to the cardinality of node.
92
+ void Increment(const schema::Node& node) {
93
+ if (node.is_repeated()) {
94
+ IncrementRepeated();
95
+ return;
96
+ }
97
+ if (node.is_optional()) {
98
+ IncrementOptional();
99
+ return;
100
+ }
101
+ }
102
+
103
+ /// Increments level for a optional node.
104
+ void IncrementOptional() { def_level++; }
105
+
106
+ /// Increments levels for the repeated node. Returns
107
+ /// the previous ancestor_list_def_level.
108
+ int16_t IncrementRepeated() {
109
+ int16_t last_repeated_ancestor = repeated_ancestor_def_level;
110
+
111
+ // Repeated fields add both a repetition and definition level. This is used
112
+ // to distinguish between an empty list and a list with an item in it.
113
+ ++rep_level;
114
+ ++def_level;
115
+ // For levels >= repeated_ancestor_def_level it indicates the list was
116
+ // non-null and had at least one element. This is important
117
+ // for later decoding because we need to add a slot for these
118
+ // values. for levels < current_def_level no slots are added
119
+ // to arrays.
120
+ repeated_ancestor_def_level = def_level;
121
+ return last_repeated_ancestor;
122
+ }
123
+
124
+ // Calculates and returns LevelInfo for a column descriptor.
125
+ static LevelInfo ComputeLevelInfo(const ColumnDescriptor* descr) {
126
+ LevelInfo level_info;
127
+ level_info.def_level = descr->max_definition_level();
128
+ level_info.rep_level = descr->max_repetition_level();
129
+
130
+ int16_t min_spaced_def_level = descr->max_definition_level();
131
+ const ::parquet::schema::Node* node = descr->schema_node().get();
132
+ while (node && !node->is_repeated()) {
133
+ if (node->is_optional()) {
134
+ min_spaced_def_level--;
135
+ }
136
+ node = node->parent();
137
+ }
138
+ level_info.repeated_ancestor_def_level = min_spaced_def_level;
139
+ return level_info;
140
+ }
141
+
142
+ friend std::ostream& operator<<(std::ostream& os, const LevelInfo& levels) {
143
+ // This print method is to silence valgrind issues. What's printed
144
+ // is not important because all asserts happen directly on
145
+ // members.
146
+ os << "{def=" << levels.def_level << ", rep=" << levels.rep_level
147
+ << ", repeated_ancestor_def=" << levels.repeated_ancestor_def_level;
148
+ if (levels.null_slot_usage > 1) {
149
+ os << ", null_slot_usage=" << levels.null_slot_usage;
150
+ }
151
+ os << "}";
152
+ return os;
153
+ }
154
+ };
155
+
156
+ // Input/Output structure for reconstructed validity bitmaps.
157
+ struct PARQUET_EXPORT ValidityBitmapInputOutput {
158
+ // Input only.
159
+ // The maximum number of values_read expected (actual
160
+ // values read must be less than or equal to this value).
161
+ // If this number is exceeded methods will throw a
162
+ // ParquetException. Exceeding this limit indicates
163
+ // either a corrupt or incorrectly written file.
164
+ int64_t values_read_upper_bound = 0;
165
+ // Output only. The number of values added to the encountered
166
+ // (this is logically the count of the number of elements
167
+ // for an Arrow array).
168
+ int64_t values_read = 0;
169
+ // Input/Output. The number of nulls encountered.
170
+ int64_t null_count = 0;
171
+ // Output only. The validity bitmap to populate. Maybe be null only
172
+ // for DefRepLevelsToListInfo (if all that is needed is list offsets).
173
+ uint8_t* valid_bits = NULLPTR;
174
+ // Input only, offset into valid_bits to start at.
175
+ int64_t valid_bits_offset = 0;
176
+ };
177
+
178
+ // Converts def_levels to validity bitmaps for non-list arrays and structs that have
179
+ // at least one member that is not a list and has no list descendents.
180
+ // For lists use DefRepLevelsToList and structs where all descendants contain
181
+ // a list use DefRepLevelsToBitmap.
182
+ void PARQUET_EXPORT DefLevelsToBitmap(const int16_t* def_levels, int64_t num_def_levels,
183
+ LevelInfo level_info,
184
+ ValidityBitmapInputOutput* output);
185
+
186
+ // Reconstructs a validity bitmap and list offsets for a list arrays based on
187
+ // def/rep levels. The first element of offsets will not be modified if rep_levels
188
+ // starts with a new list. The first element of offsets will be used when calculating
189
+ // the next offset. See documentation onf DefLevelsToBitmap for when to use this
190
+ // method vs the other ones in this file for reconstruction.
191
+ //
192
+ // Offsets must be sized to 1 + values_read_upper_bound.
193
+ void PARQUET_EXPORT DefRepLevelsToList(const int16_t* def_levels,
194
+ const int16_t* rep_levels, int64_t num_def_levels,
195
+ LevelInfo level_info,
196
+ ValidityBitmapInputOutput* output,
197
+ int32_t* offsets);
198
+ void PARQUET_EXPORT DefRepLevelsToList(const int16_t* def_levels,
199
+ const int16_t* rep_levels, int64_t num_def_levels,
200
+ LevelInfo level_info,
201
+ ValidityBitmapInputOutput* output,
202
+ int64_t* offsets);
203
+
204
+ // Reconstructs a validity bitmap for a struct every member is a list or has
205
+ // a list descendant. See documentation on DefLevelsToBitmap for when more
206
+ // details on this method compared to the other ones defined above.
207
+ void PARQUET_EXPORT DefRepLevelsToBitmap(const int16_t* def_levels,
208
+ const int16_t* rep_levels,
209
+ int64_t num_def_levels, LevelInfo level_info,
210
+ ValidityBitmapInputOutput* output);
211
+
212
+ // This is exposed to ensure we can properly test a software simulated pext function
213
+ // (i.e. it isn't hidden by runtime dispatch).
214
+ uint64_t PARQUET_EXPORT TestOnlyExtractBitsSoftware(uint64_t bitmap, uint64_t selection);
215
+
216
+ } // namespace parquet::internal
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/level_conversion_inc.h ADDED
@@ -0,0 +1,356 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+ #pragma once
18
+
19
+ #include "parquet/level_conversion.h"
20
+
21
+ #include <algorithm>
22
+ #include <bit>
23
+ #include <cstdint>
24
+ #include <limits>
25
+
26
+ #include "arrow/util/bit_run_reader.h"
27
+ #include "arrow/util/bit_util.h"
28
+ #include "arrow/util/bitmap_writer.h"
29
+ #include "arrow/util/logging.h"
30
+ #include "arrow/util/simd.h"
31
+ #include "parquet/exception.h"
32
+ #include "parquet/level_comparison.h"
33
+
34
+ #ifndef PARQUET_IMPL_NAMESPACE
35
+ # error "PARQUET_IMPL_NAMESPACE must be defined"
36
+ #endif
37
+
38
+ namespace parquet::internal::PARQUET_IMPL_NAMESPACE {
39
+
40
+ // clang-format off
41
+ /* Python code to generate lookup table:
42
+
43
+ kLookupBits = 5
44
+ count = 0
45
+ print('constexpr int kLookupBits = {};'.format(kLookupBits))
46
+ print('constexpr uint8_t kPextTable[1 << kLookupBits][1 << kLookupBits] = {')
47
+ print(' ', end = '')
48
+ for mask in range(1 << kLookupBits):
49
+ for data in range(1 << kLookupBits):
50
+ bit_value = 0
51
+ bit_len = 0
52
+ for i in range(kLookupBits):
53
+ if mask & (1 << i):
54
+ bit_value |= (((data >> i) & 1) << bit_len)
55
+ bit_len += 1
56
+ out = '0x{:02X},'.format(bit_value)
57
+ count += 1
58
+ if count % (1 << kLookupBits) == 1:
59
+ print(' {')
60
+ if count % 8 == 1:
61
+ print(' ', end = '')
62
+ if count % 8 == 0:
63
+ print(out, end = '\n')
64
+ else:
65
+ print(out, end = ' ')
66
+ if count % (1 << kLookupBits) == 0:
67
+ print(' },', end = '')
68
+ print('\n};')
69
+
70
+ */
71
+ // clang-format on
72
+
73
+ constexpr int kLookupBits = 5;
74
+ constexpr uint8_t kPextTable[1 << kLookupBits][1 << kLookupBits] = {
75
+ {
76
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
77
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
78
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
79
+ },
80
+ {
81
+ 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00,
82
+ 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01,
83
+ 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01,
84
+ },
85
+ {
86
+ 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01,
87
+ 0x01, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00,
88
+ 0x01, 0x01, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01,
89
+ },
90
+ {
91
+ 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02,
92
+ 0x03, 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03, 0x00, 0x01,
93
+ 0x02, 0x03, 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03,
94
+ },
95
+ {
96
+ 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00,
97
+ 0x00, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01,
98
+ 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01,
99
+ },
100
+ {
101
+ 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02, 0x03, 0x00, 0x01, 0x00,
102
+ 0x01, 0x02, 0x03, 0x02, 0x03, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03,
103
+ 0x02, 0x03, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02, 0x03,
104
+ },
105
+ {
106
+ 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x00, 0x00, 0x01,
107
+ 0x01, 0x02, 0x02, 0x03, 0x03, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02,
108
+ 0x03, 0x03, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03,
109
+ },
110
+ {
111
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x00, 0x01, 0x02,
112
+ 0x03, 0x04, 0x05, 0x06, 0x07, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05,
113
+ 0x06, 0x07, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
114
+ },
115
+ {
116
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01,
117
+ 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
118
+ 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
119
+ },
120
+ {
121
+ 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02,
122
+ 0x03, 0x02, 0x03, 0x02, 0x03, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01,
123
+ 0x00, 0x01, 0x02, 0x03, 0x02, 0x03, 0x02, 0x03, 0x02, 0x03,
124
+ },
125
+ {
126
+ 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03,
127
+ 0x03, 0x02, 0x02, 0x03, 0x03, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00,
128
+ 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x02, 0x02, 0x03, 0x03,
129
+ },
130
+ {
131
+ 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
132
+ 0x07, 0x04, 0x05, 0x06, 0x07, 0x00, 0x01, 0x02, 0x03, 0x00, 0x01,
133
+ 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x04, 0x05, 0x06, 0x07,
134
+ },
135
+ {
136
+ 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02, 0x02,
137
+ 0x02, 0x03, 0x03, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01,
138
+ 0x01, 0x01, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03,
139
+ },
140
+ {
141
+ 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02, 0x03, 0x04, 0x05, 0x04,
142
+ 0x05, 0x06, 0x07, 0x06, 0x07, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03,
143
+ 0x02, 0x03, 0x04, 0x05, 0x04, 0x05, 0x06, 0x07, 0x06, 0x07,
144
+ },
145
+ {
146
+ 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x04, 0x04, 0x05,
147
+ 0x05, 0x06, 0x06, 0x07, 0x07, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02,
148
+ 0x03, 0x03, 0x04, 0x04, 0x05, 0x05, 0x06, 0x06, 0x07, 0x07,
149
+ },
150
+ {
151
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A,
152
+ 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05,
153
+ 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
154
+ },
155
+ {
156
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
157
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
158
+ 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
159
+ },
160
+ {
161
+ 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00,
162
+ 0x01, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02, 0x03, 0x02, 0x03,
163
+ 0x02, 0x03, 0x02, 0x03, 0x02, 0x03, 0x02, 0x03, 0x02, 0x03,
164
+ },
165
+ {
166
+ 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01,
167
+ 0x01, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x02, 0x02,
168
+ 0x03, 0x03, 0x02, 0x02, 0x03, 0x03, 0x02, 0x02, 0x03, 0x03,
169
+ },
170
+ {
171
+ 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02,
172
+ 0x03, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x04, 0x05,
173
+ 0x06, 0x07, 0x04, 0x05, 0x06, 0x07, 0x04, 0x05, 0x06, 0x07,
174
+ },
175
+ {
176
+ 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00,
177
+ 0x00, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03,
178
+ 0x03, 0x03, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03,
179
+ },
180
+ {
181
+ 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02, 0x03, 0x00, 0x01, 0x00,
182
+ 0x01, 0x02, 0x03, 0x02, 0x03, 0x04, 0x05, 0x04, 0x05, 0x06, 0x07,
183
+ 0x06, 0x07, 0x04, 0x05, 0x04, 0x05, 0x06, 0x07, 0x06, 0x07,
184
+ },
185
+ {
186
+ 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x00, 0x00, 0x01,
187
+ 0x01, 0x02, 0x02, 0x03, 0x03, 0x04, 0x04, 0x05, 0x05, 0x06, 0x06,
188
+ 0x07, 0x07, 0x04, 0x04, 0x05, 0x05, 0x06, 0x06, 0x07, 0x07,
189
+ },
190
+ {
191
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x00, 0x01, 0x02,
192
+ 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D,
193
+ 0x0E, 0x0F, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
194
+ },
195
+ {
196
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01,
197
+ 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,
198
+ 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03,
199
+ },
200
+ {
201
+ 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02,
202
+ 0x03, 0x02, 0x03, 0x02, 0x03, 0x04, 0x05, 0x04, 0x05, 0x04, 0x05,
203
+ 0x04, 0x05, 0x06, 0x07, 0x06, 0x07, 0x06, 0x07, 0x06, 0x07,
204
+ },
205
+ {
206
+ 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03,
207
+ 0x03, 0x02, 0x02, 0x03, 0x03, 0x04, 0x04, 0x05, 0x05, 0x04, 0x04,
208
+ 0x05, 0x05, 0x06, 0x06, 0x07, 0x07, 0x06, 0x06, 0x07, 0x07,
209
+ },
210
+ {
211
+ 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
212
+ 0x07, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x08, 0x09,
213
+ 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x0C, 0x0D, 0x0E, 0x0F,
214
+ },
215
+ {
216
+ 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02, 0x02,
217
+ 0x02, 0x03, 0x03, 0x03, 0x03, 0x04, 0x04, 0x04, 0x04, 0x05, 0x05,
218
+ 0x05, 0x05, 0x06, 0x06, 0x06, 0x06, 0x07, 0x07, 0x07, 0x07,
219
+ },
220
+ {
221
+ 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x02, 0x03, 0x04, 0x05, 0x04,
222
+ 0x05, 0x06, 0x07, 0x06, 0x07, 0x08, 0x09, 0x08, 0x09, 0x0A, 0x0B,
223
+ 0x0A, 0x0B, 0x0C, 0x0D, 0x0C, 0x0D, 0x0E, 0x0F, 0x0E, 0x0F,
224
+ },
225
+ {
226
+ 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x04, 0x04, 0x05,
227
+ 0x05, 0x06, 0x06, 0x07, 0x07, 0x08, 0x08, 0x09, 0x09, 0x0A, 0x0A,
228
+ 0x0B, 0x0B, 0x0C, 0x0C, 0x0D, 0x0D, 0x0E, 0x0E, 0x0F, 0x0F,
229
+ },
230
+ {
231
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A,
232
+ 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15,
233
+ 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F,
234
+ },
235
+ };
236
+
237
+ inline uint64_t ExtractBitsSoftware(uint64_t bitmap, uint64_t select_bitmap) {
238
+ // A software emulation of _pext_u64
239
+
240
+ // These checks should be inline and are likely to be common cases.
241
+ if (select_bitmap == ~uint64_t{0}) {
242
+ return bitmap;
243
+ } else if (select_bitmap == 0) {
244
+ return 0;
245
+ }
246
+
247
+ // Fallback to lookup table method
248
+ uint64_t bit_value = 0;
249
+ int bit_len = 0;
250
+ constexpr uint8_t kLookupMask = (1U << kLookupBits) - 1;
251
+ while (select_bitmap != 0) {
252
+ const auto mask_len = std::popcount(select_bitmap & kLookupMask);
253
+ const uint64_t value = kPextTable[select_bitmap & kLookupMask][bitmap & kLookupMask];
254
+ bit_value |= (value << bit_len);
255
+ bit_len += mask_len;
256
+ bitmap >>= kLookupBits;
257
+ select_bitmap >>= kLookupBits;
258
+ }
259
+ return bit_value;
260
+ }
261
+
262
+ #ifdef ARROW_HAVE_BMI2
263
+
264
+ // Use _pext_u64 on 64-bit builds, _pext_u32 on 32-bit builds,
265
+ # if UINTPTR_MAX == 0xFFFFFFFF
266
+
267
+ using extract_bitmap_t = uint32_t;
268
+ inline extract_bitmap_t ExtractBits(extract_bitmap_t bitmap,
269
+ extract_bitmap_t select_bitmap) {
270
+ return _pext_u32(bitmap, select_bitmap);
271
+ }
272
+
273
+ # else
274
+
275
+ using extract_bitmap_t = uint64_t;
276
+ inline extract_bitmap_t ExtractBits(extract_bitmap_t bitmap,
277
+ extract_bitmap_t select_bitmap) {
278
+ return _pext_u64(bitmap, select_bitmap);
279
+ }
280
+
281
+ # endif
282
+
283
+ #else // !defined(ARROW_HAVE_BMI2)
284
+
285
+ // Use 64-bit pext emulation when BMI2 isn't available.
286
+ using extract_bitmap_t = uint64_t;
287
+ inline extract_bitmap_t ExtractBits(extract_bitmap_t bitmap,
288
+ extract_bitmap_t select_bitmap) {
289
+ return ExtractBitsSoftware(bitmap, select_bitmap);
290
+ }
291
+
292
+ #endif
293
+
294
+ static constexpr int64_t kExtractBitsSize = 8 * sizeof(extract_bitmap_t);
295
+
296
+ template <bool has_repeated_parent>
297
+ int64_t DefLevelsBatchToBitmap(const int16_t* def_levels, const int64_t batch_size,
298
+ int64_t upper_bound_remaining, LevelInfo level_info,
299
+ ::arrow::internal::FirstTimeBitmapWriter* writer) {
300
+ ARROW_DCHECK_LE(batch_size, kExtractBitsSize);
301
+
302
+ // Greater than level_info.def_level - 1 implies >= the def_level
303
+ auto defined_bitmap = static_cast<extract_bitmap_t>(::arrow::bit_util::FromLittleEndian(
304
+ internal::GreaterThanBitmap(def_levels, batch_size, level_info.def_level - 1)));
305
+
306
+ if (has_repeated_parent) {
307
+ // Greater than level_info.repeated_ancestor_def_level - 1 implies >= the
308
+ // repeated_ancestor_def_level
309
+ auto present_bitmap = static_cast<extract_bitmap_t>(
310
+ ::arrow::bit_util::FromLittleEndian(internal::GreaterThanBitmap(
311
+ def_levels, batch_size, level_info.repeated_ancestor_def_level - 1)));
312
+ auto selected_bits = ExtractBits(defined_bitmap, present_bitmap);
313
+ int64_t selected_count = std::popcount(present_bitmap);
314
+ if (ARROW_PREDICT_FALSE(selected_count > upper_bound_remaining)) {
315
+ throw ParquetException("Values read exceeded upper bound");
316
+ }
317
+ writer->AppendWord(selected_bits, selected_count);
318
+ return std::popcount(selected_bits);
319
+ } else {
320
+ if (ARROW_PREDICT_FALSE(batch_size > upper_bound_remaining)) {
321
+ std::stringstream ss;
322
+ ss << "Values read exceeded upper bound";
323
+ throw ParquetException(ss.str());
324
+ }
325
+
326
+ writer->AppendWord(defined_bitmap, batch_size);
327
+ return std::popcount(defined_bitmap);
328
+ }
329
+ }
330
+
331
+ template <bool has_repeated_parent>
332
+ void DefLevelsToBitmapSimd(const int16_t* def_levels, int64_t num_def_levels,
333
+ LevelInfo level_info, ValidityBitmapInputOutput* output) {
334
+ ::arrow::internal::FirstTimeBitmapWriter writer(
335
+ output->valid_bits,
336
+ /*start_offset=*/output->valid_bits_offset,
337
+ /*length=*/output->values_read_upper_bound);
338
+ int64_t set_count = 0;
339
+ output->values_read = 0;
340
+ int64_t values_read_remaining = output->values_read_upper_bound;
341
+ while (num_def_levels > kExtractBitsSize) {
342
+ set_count += DefLevelsBatchToBitmap<has_repeated_parent>(
343
+ def_levels, kExtractBitsSize, values_read_remaining, level_info, &writer);
344
+ def_levels += kExtractBitsSize;
345
+ num_def_levels -= kExtractBitsSize;
346
+ values_read_remaining = output->values_read_upper_bound - writer.position();
347
+ }
348
+ set_count += DefLevelsBatchToBitmap<has_repeated_parent>(
349
+ def_levels, num_def_levels, values_read_remaining, level_info, &writer);
350
+
351
+ output->values_read = writer.position();
352
+ output->null_count += output->values_read - set_count;
353
+ writer.Finish();
354
+ }
355
+
356
+ } // namespace parquet::internal::PARQUET_IMPL_NAMESPACE
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/metadata.h ADDED
@@ -0,0 +1,536 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <cstdint>
21
+ #include <map>
22
+ #include <memory>
23
+ #include <optional>
24
+ #include <span>
25
+ #include <string>
26
+ #include <vector>
27
+
28
+ #include "parquet/encryption/type_fwd.h"
29
+ #include "parquet/index_location.h"
30
+ #include "parquet/platform.h"
31
+ #include "parquet/properties.h"
32
+ #include "parquet/type_fwd.h"
33
+ namespace parquet {
34
+
35
+ using KeyValueMetadata = ::arrow::KeyValueMetadata;
36
+
37
+ class PARQUET_EXPORT ApplicationVersion {
38
+ public:
39
+ // Known Versions with Issues
40
+ static const ApplicationVersion& PARQUET_251_FIXED_VERSION();
41
+ static const ApplicationVersion& PARQUET_816_FIXED_VERSION();
42
+ static const ApplicationVersion& PARQUET_CPP_FIXED_STATS_VERSION();
43
+ static const ApplicationVersion& PARQUET_MR_FIXED_STATS_VERSION();
44
+ static const ApplicationVersion& PARQUET_CPP_10353_FIXED_VERSION();
45
+
46
+ // Application that wrote the file. e.g. "IMPALA"
47
+ std::string application_;
48
+ // Build name
49
+ std::string build_;
50
+
51
+ // Version of the application that wrote the file, expressed as
52
+ // (<major>.<minor>.<patch>). Unmatched parts default to 0.
53
+ // "1.2.3" => {1, 2, 3}
54
+ // "1.2" => {1, 2, 0}
55
+ // "1.2-cdh5" => {1, 2, 0}
56
+ struct {
57
+ int major;
58
+ int minor;
59
+ int patch;
60
+ std::string unknown;
61
+ std::string pre_release;
62
+ std::string build_info;
63
+ } version;
64
+
65
+ ApplicationVersion() = default;
66
+ explicit ApplicationVersion(const std::string& created_by);
67
+ ApplicationVersion(std::string application, int major, int minor, int patch);
68
+
69
+ // Returns true if version is strictly less than other_version
70
+ bool VersionLt(const ApplicationVersion& other_version) const;
71
+
72
+ // Returns true if version is strictly equal with other_version
73
+ bool VersionEq(const ApplicationVersion& other_version) const;
74
+
75
+ // Checks if the Version has the correct statistics for a given column
76
+ bool HasCorrectStatistics(Type::type primitive, const EncodedStatistics& statistics,
77
+ SortOrder::type sort_order = SortOrder::SIGNED) const;
78
+ };
79
+
80
+ class PARQUET_EXPORT ColumnCryptoMetaData {
81
+ public:
82
+ static std::unique_ptr<ColumnCryptoMetaData> Make(const uint8_t* metadata);
83
+ ~ColumnCryptoMetaData();
84
+
85
+ bool Equals(const ColumnCryptoMetaData& other) const;
86
+
87
+ std::shared_ptr<schema::ColumnPath> path_in_schema() const;
88
+ bool encrypted_with_footer_key() const;
89
+ const std::string& key_metadata() const;
90
+
91
+ private:
92
+ explicit ColumnCryptoMetaData(const uint8_t* metadata);
93
+
94
+ class ColumnCryptoMetaDataImpl;
95
+ std::unique_ptr<ColumnCryptoMetaDataImpl> impl_;
96
+ };
97
+
98
+ /// \brief Public struct for Thrift PageEncodingStats in ColumnChunkMetaData
99
+ struct PageEncodingStats {
100
+ PageType::type page_type;
101
+ Encoding::type encoding;
102
+ int32_t count;
103
+ };
104
+
105
+ /// \brief ColumnChunkMetaData is a proxy around format::ColumnChunkMetaData.
106
+ class PARQUET_EXPORT ColumnChunkMetaData {
107
+ public:
108
+ // API convenience to get a MetaData accessor
109
+ static std::unique_ptr<ColumnChunkMetaData> Make(
110
+ const void* metadata, const ColumnDescriptor* descr,
111
+ const ReaderProperties& properties = default_reader_properties(),
112
+ const ApplicationVersion* writer_version = NULLPTR, int16_t row_group_ordinal = -1,
113
+ int16_t column_ordinal = -1,
114
+ std::shared_ptr<InternalFileDecryptor> file_decryptor = NULLPTR);
115
+
116
+ ~ColumnChunkMetaData();
117
+
118
+ bool Equals(const ColumnChunkMetaData& other) const;
119
+
120
+ // Byte offset of `ColumnMetaData` in `file_path()`.
121
+ //
122
+ // Note that the meaning of this field has been inconsistent among implementations
123
+ // so its use has since been deprecated in the Parquet specification. Modern
124
+ // implementations will set this to `0` to indicate that the `ColumnMetaData` is solely
125
+ // contained in the `ColumnChunk` struct.
126
+ int64_t file_offset() const;
127
+
128
+ // parameter is only used when a dataset is spread across multiple files
129
+ const std::string& file_path() const;
130
+
131
+ // column metadata
132
+ Type::type type() const;
133
+ int64_t num_values() const;
134
+ std::shared_ptr<schema::ColumnPath> path_in_schema() const;
135
+ bool is_stats_set() const;
136
+ bool is_geo_stats_set() const;
137
+ std::shared_ptr<Statistics> statistics() const;
138
+ std::shared_ptr<EncodedStatistics> encoded_statistics() const;
139
+ std::shared_ptr<SizeStatistics> size_statistics() const;
140
+ std::shared_ptr<geospatial::GeoStatistics> geo_statistics() const;
141
+
142
+ Compression::type compression() const;
143
+ // Indicate if the ColumnChunk compression is supported by the current
144
+ // compiled parquet library.
145
+ bool can_decompress() const;
146
+
147
+ const std::vector<Encoding::type>& encodings() const;
148
+ const std::vector<PageEncodingStats>& encoding_stats() const;
149
+ std::optional<int64_t> bloom_filter_offset() const;
150
+ std::optional<int64_t> bloom_filter_length() const;
151
+ bool has_dictionary_page() const;
152
+ int64_t dictionary_page_offset() const;
153
+ int64_t data_page_offset() const;
154
+ bool has_index_page() const;
155
+ int64_t index_page_offset() const;
156
+ int64_t total_compressed_size() const;
157
+ int64_t total_uncompressed_size() const;
158
+ std::unique_ptr<ColumnCryptoMetaData> crypto_metadata() const;
159
+ std::optional<IndexLocation> GetColumnIndexLocation() const;
160
+ std::optional<IndexLocation> GetOffsetIndexLocation() const;
161
+ const std::shared_ptr<const KeyValueMetadata>& key_value_metadata() const;
162
+
163
+ private:
164
+ explicit ColumnChunkMetaData(
165
+ const void* metadata, const ColumnDescriptor* descr, int16_t row_group_ordinal,
166
+ int16_t column_ordinal, const ReaderProperties& properties,
167
+ const ApplicationVersion* writer_version = NULLPTR,
168
+ std::shared_ptr<InternalFileDecryptor> file_decryptor = NULLPTR);
169
+ // PIMPL Idiom
170
+ class ColumnChunkMetaDataImpl;
171
+ std::unique_ptr<ColumnChunkMetaDataImpl> impl_;
172
+ };
173
+
174
+ /// \brief RowGroupMetaData is a proxy around format::RowGroupMetaData.
175
+ class PARQUET_EXPORT RowGroupMetaData {
176
+ public:
177
+ /// \brief Create a RowGroupMetaData from a serialized thrift message.
178
+ static std::unique_ptr<RowGroupMetaData> Make(
179
+ const void* metadata, const SchemaDescriptor* schema,
180
+ const ReaderProperties& properties = default_reader_properties(),
181
+ const ApplicationVersion* writer_version = NULLPTR,
182
+ std::shared_ptr<InternalFileDecryptor> file_decryptor = NULLPTR);
183
+
184
+ ~RowGroupMetaData();
185
+
186
+ bool Equals(const RowGroupMetaData& other) const;
187
+
188
+ /// \brief The number of columns in this row group. The order must match the
189
+ /// parent's column ordering.
190
+ int num_columns() const;
191
+
192
+ /// \brief Return the ColumnChunkMetaData of the corresponding column ordinal.
193
+ ///
194
+ /// WARNING, the returned object references memory location in it's parent
195
+ /// (RowGroupMetaData) object. Hence, the parent must outlive the returned
196
+ /// object.
197
+ ///
198
+ /// \param[in] index of the ColumnChunkMetaData to retrieve.
199
+ ///
200
+ /// \throws ParquetException if the index is out of bound.
201
+ std::unique_ptr<ColumnChunkMetaData> ColumnChunk(int index) const;
202
+
203
+ /// \brief Number of rows in this row group.
204
+ int64_t num_rows() const;
205
+
206
+ /// \brief Total byte size of all the uncompressed column data in this row group.
207
+ int64_t total_byte_size() const;
208
+
209
+ /// \brief Total byte size of all the compressed (and potentially encrypted)
210
+ /// column data in this row group.
211
+ ///
212
+ /// This information is optional and may be 0 if omitted.
213
+ int64_t total_compressed_size() const;
214
+
215
+ /// \brief Byte offset from beginning of file to first page (data or
216
+ /// dictionary) in this row group
217
+ ///
218
+ /// The file_offset field that this method exposes is optional. This method
219
+ /// will return 0 if that field is not set to a meaningful value.
220
+ int64_t file_offset() const;
221
+ // Return const-pointer to make it clear that this object is not to be copied
222
+ const SchemaDescriptor* schema() const;
223
+ // Indicate if all of the RowGroup's ColumnChunks can be decompressed.
224
+ bool can_decompress() const;
225
+ // Sorting columns of the row group if any.
226
+ std::vector<SortingColumn> sorting_columns() const;
227
+
228
+ private:
229
+ explicit RowGroupMetaData(
230
+ const void* metadata, const SchemaDescriptor* schema,
231
+ const ReaderProperties& properties,
232
+ const ApplicationVersion* writer_version = NULLPTR,
233
+ std::shared_ptr<InternalFileDecryptor> file_decryptor = NULLPTR);
234
+ // PIMPL Idiom
235
+ class RowGroupMetaDataImpl;
236
+ std::unique_ptr<RowGroupMetaDataImpl> impl_;
237
+ };
238
+
239
+ class FileMetaDataBuilder;
240
+
241
+ /// \brief FileMetaData is a proxy around format::FileMetaData.
242
+ class PARQUET_EXPORT FileMetaData {
243
+ public:
244
+ /// \brief Create a FileMetaData from a serialized thrift message.
245
+ static std::shared_ptr<FileMetaData> Make(
246
+ const void* serialized_metadata, uint32_t* inout_metadata_len,
247
+ const ReaderProperties& properties = default_reader_properties(),
248
+ std::shared_ptr<InternalFileDecryptor> file_decryptor = NULLPTR);
249
+
250
+ ~FileMetaData();
251
+
252
+ bool Equals(const FileMetaData& other) const;
253
+
254
+ /// \brief The number of parquet "leaf" columns.
255
+ ///
256
+ /// Parquet thrift definition requires that nested schema elements are
257
+ /// flattened. This method returns the number of columns in the flattened
258
+ /// version.
259
+ /// For instance, if the schema looks like this :
260
+ /// 0 foo.bar
261
+ /// foo.bar.baz 0
262
+ /// foo.bar.baz2 1
263
+ /// foo.qux 2
264
+ /// 1 foo2 3
265
+ /// 2 foo3 4
266
+ /// This method will return 5, because there are 5 "leaf" fields (so 5
267
+ /// flattened fields)
268
+ int num_columns() const;
269
+
270
+ /// \brief The number of flattened schema elements.
271
+ ///
272
+ /// Parquet thrift definition requires that nested schema elements are
273
+ /// flattened. This method returns the total number of elements in the
274
+ /// flattened list.
275
+ int num_schema_elements() const;
276
+
277
+ /// \brief The total number of rows.
278
+ ///
279
+ /// If the FileMetaData was obtained by calling `SubSet()`, this is the total
280
+ /// number of rows in the selected row groups.
281
+ int64_t num_rows() const;
282
+
283
+ /// \brief The number of row groups in the file.
284
+ ///
285
+ /// If the FileMetaData was obtained by calling `SubSet()`, this is the number
286
+ /// of selected row groups.
287
+ int num_row_groups() const;
288
+
289
+ /// \brief Return the RowGroupMetaData of the corresponding row group ordinal.
290
+ ///
291
+ /// WARNING, the returned object references memory location in it's parent
292
+ /// (FileMetaData) object. Hence, the parent must outlive the returned object.
293
+ ///
294
+ /// \param[in] index of the RowGroup to retrieve.
295
+ ///
296
+ /// \throws ParquetException if the index is out of bound.
297
+ std::unique_ptr<RowGroupMetaData> RowGroup(int index) const;
298
+
299
+ /// \brief Return the "version" of the file
300
+ ///
301
+ /// WARNING: The value returned by this method is unreliable as 1) the Parquet
302
+ /// file metadata stores the version as a single integer and 2) some producers
303
+ /// are known to always write a hardcoded value. Therefore, you cannot use
304
+ /// this value to know which features are used in the file.
305
+ ParquetVersion::type version() const;
306
+
307
+ /// \brief Return the application's user-agent string of the writer.
308
+ const std::string& created_by() const;
309
+
310
+ /// \brief Return the application's version of the writer.
311
+ const ApplicationVersion& writer_version() const;
312
+
313
+ /// \brief Size of the original thrift encoded metadata footer.
314
+ uint32_t size() const;
315
+
316
+ /// \brief Indicate if all of the FileMetaData's RowGroups can be decompressed.
317
+ ///
318
+ /// This will return false if any of the RowGroup's page is compressed with a
319
+ /// compression format which is not compiled in the current parquet library.
320
+ bool can_decompress() const;
321
+
322
+ bool is_encryption_algorithm_set() const;
323
+ EncryptionAlgorithm encryption_algorithm() const;
324
+ const std::string& footer_signing_key_metadata() const;
325
+
326
+ PARQUET_DEPRECATED(
327
+ "Deprecated in 24.0.0. If you need this functionality, please report an issue.")
328
+ bool VerifySignature(const void* signature);
329
+
330
+ void WriteTo(::arrow::io::OutputStream* dst,
331
+ const std::shared_ptr<Encryptor>& encryptor = NULLPTR) const;
332
+
333
+ /// \brief Return Thrift-serialized representation of the metadata as a
334
+ /// string
335
+ std::string SerializeToString() const;
336
+
337
+ // Return const-pointer to make it clear that this object is not to be copied
338
+ const SchemaDescriptor* schema() const;
339
+
340
+ const std::shared_ptr<const KeyValueMetadata>& key_value_metadata() const;
341
+
342
+ /// \brief Set a path to all ColumnChunk for all RowGroups.
343
+ ///
344
+ /// Commonly used by systems (Dask, Spark) who generates an metadata-only
345
+ /// parquet file. The path is usually relative to said index file.
346
+ ///
347
+ /// \param[in] path to set.
348
+ void set_file_path(const std::string& path);
349
+
350
+ /// \brief Merge row groups from another metadata file into this one.
351
+ ///
352
+ /// The schema of the input FileMetaData must be equal to the
353
+ /// schema of this object.
354
+ ///
355
+ /// This is used by systems who creates an aggregate metadata-only file by
356
+ /// concatenating the row groups of multiple files. This newly created
357
+ /// metadata file acts as an index of all available row groups.
358
+ ///
359
+ /// \param[in] other FileMetaData to merge the row groups from.
360
+ ///
361
+ /// \throws ParquetException if schemas are not equal.
362
+ void AppendRowGroups(const FileMetaData& other);
363
+
364
+ /// \brief Return a FileMetaData containing a subset of the row groups in this
365
+ /// FileMetaData.
366
+ std::shared_ptr<FileMetaData> Subset(const std::vector<int>& row_groups) const;
367
+
368
+ /// \brief Serialize metadata unencrypted as string
369
+ ///
370
+ /// \param[in] scrub whether to remove sensitive information from the metadata.
371
+ /// \param[in] debug whether to serialize the metadata as Thrift (if false) or
372
+ /// debug text (if true).
373
+ std::string SerializeUnencrypted(bool scrub, bool debug) const;
374
+
375
+ private:
376
+ friend FileMetaDataBuilder;
377
+ friend class SerializedFile;
378
+ friend class SerializedRowGroup;
379
+
380
+ explicit FileMetaData(const void* serialized_metadata, uint32_t* metadata_len,
381
+ const ReaderProperties& properties,
382
+ std::shared_ptr<InternalFileDecryptor> file_decryptor = NULLPTR);
383
+
384
+ void set_file_decryptor(std::shared_ptr<InternalFileDecryptor> file_decryptor);
385
+ const std::shared_ptr<InternalFileDecryptor>& file_decryptor() const;
386
+
387
+ // Verify the signature of a plaintext footer.
388
+ static bool VerifySignature(std::span<const uint8_t> serialized_metadata,
389
+ std::span<const uint8_t> signature,
390
+ InternalFileDecryptor* file_decryptor);
391
+
392
+ // PIMPL Idiom
393
+ FileMetaData();
394
+ class FileMetaDataImpl;
395
+ std::unique_ptr<FileMetaDataImpl> impl_;
396
+ };
397
+
398
+ class PARQUET_EXPORT FileCryptoMetaData {
399
+ public:
400
+ // API convenience to get a MetaData accessor
401
+ static std::shared_ptr<FileCryptoMetaData> Make(
402
+ const uint8_t* serialized_metadata, uint32_t* metadata_len,
403
+ const ReaderProperties& properties = default_reader_properties());
404
+ ~FileCryptoMetaData();
405
+
406
+ EncryptionAlgorithm encryption_algorithm() const;
407
+ const std::string& key_metadata() const;
408
+
409
+ void WriteTo(::arrow::io::OutputStream* dst) const;
410
+
411
+ private:
412
+ friend FileMetaDataBuilder;
413
+ FileCryptoMetaData(const uint8_t* serialized_metadata, uint32_t* metadata_len,
414
+ const ReaderProperties& properties);
415
+
416
+ // PIMPL Idiom
417
+ FileCryptoMetaData();
418
+ class FileCryptoMetaDataImpl;
419
+ std::unique_ptr<FileCryptoMetaDataImpl> impl_;
420
+ };
421
+
422
+ // Builder API
423
+ class PARQUET_EXPORT ColumnChunkMetaDataBuilder {
424
+ public:
425
+ // API convenience to get a MetaData reader
426
+ static std::unique_ptr<ColumnChunkMetaDataBuilder> Make(
427
+ std::shared_ptr<WriterProperties> props, const ColumnDescriptor* column);
428
+
429
+ static std::unique_ptr<ColumnChunkMetaDataBuilder> Make(
430
+ std::shared_ptr<WriterProperties> props, const ColumnDescriptor* column,
431
+ void* contents);
432
+
433
+ ~ColumnChunkMetaDataBuilder();
434
+
435
+ // column chunk
436
+ // Used when a dataset is spread across multiple files
437
+ void set_file_path(const std::string& path);
438
+
439
+ // column metadata
440
+ void SetStatistics(const EncodedStatistics& stats);
441
+ void SetSizeStatistics(const SizeStatistics& size_stats);
442
+
443
+ // column geometry statistics
444
+ void SetGeoStatistics(const geospatial::EncodedGeoStatistics& geo_stats);
445
+
446
+ void SetKeyValueMetadata(std::shared_ptr<const KeyValueMetadata> key_value_metadata);
447
+
448
+ // get the column descriptor
449
+ const ColumnDescriptor* descr() const;
450
+
451
+ int64_t total_compressed_size() const;
452
+ // commit the metadata
453
+
454
+ void Finish(int64_t num_values, int64_t dictionary_page_offset,
455
+ int64_t index_page_offset, int64_t data_page_offset,
456
+ int64_t compressed_size, int64_t uncompressed_size, bool has_dictionary,
457
+ bool dictionary_fallback,
458
+ const std::map<Encoding::type, int32_t>& dict_encoding_stats_,
459
+ const std::map<Encoding::type, int32_t>& data_encoding_stats_,
460
+ const std::shared_ptr<Encryptor>& encryptor = NULLPTR);
461
+
462
+ // The metadata contents, suitable for passing to ColumnChunkMetaData::Make
463
+ const void* contents() const;
464
+
465
+ // For writing metadata at end of column chunk
466
+ void WriteTo(::arrow::io::OutputStream* sink);
467
+
468
+ private:
469
+ explicit ColumnChunkMetaDataBuilder(std::shared_ptr<WriterProperties> props,
470
+ const ColumnDescriptor* column);
471
+ explicit ColumnChunkMetaDataBuilder(std::shared_ptr<WriterProperties> props,
472
+ const ColumnDescriptor* column, void* contents);
473
+ // PIMPL Idiom
474
+ class ColumnChunkMetaDataBuilderImpl;
475
+ std::unique_ptr<ColumnChunkMetaDataBuilderImpl> impl_;
476
+ };
477
+
478
+ class PARQUET_EXPORT RowGroupMetaDataBuilder {
479
+ public:
480
+ // API convenience to get a MetaData reader
481
+ static std::unique_ptr<RowGroupMetaDataBuilder> Make(
482
+ std::shared_ptr<WriterProperties> props, const SchemaDescriptor* schema_,
483
+ void* contents);
484
+
485
+ ~RowGroupMetaDataBuilder();
486
+
487
+ ColumnChunkMetaDataBuilder* NextColumnChunk();
488
+ int num_columns();
489
+ int64_t num_rows();
490
+ int current_column() const;
491
+
492
+ void set_num_rows(int64_t num_rows);
493
+
494
+ // commit the metadata
495
+ void Finish(int64_t total_bytes_written, int16_t row_group_ordinal = -1);
496
+
497
+ private:
498
+ explicit RowGroupMetaDataBuilder(std::shared_ptr<WriterProperties> props,
499
+ const SchemaDescriptor* schema_, void* contents);
500
+ // PIMPL Idiom
501
+ class RowGroupMetaDataBuilderImpl;
502
+ std::unique_ptr<RowGroupMetaDataBuilderImpl> impl_;
503
+ };
504
+
505
+ class PARQUET_EXPORT FileMetaDataBuilder {
506
+ public:
507
+ // API convenience to get a MetaData builder
508
+ static std::unique_ptr<FileMetaDataBuilder> Make(
509
+ const SchemaDescriptor* schema, std::shared_ptr<WriterProperties> props);
510
+
511
+ ~FileMetaDataBuilder();
512
+
513
+ // The prior RowGroupMetaDataBuilder (if any) is destroyed
514
+ RowGroupMetaDataBuilder* AppendRowGroup();
515
+
516
+ // Set locations of all column chunks of a specific index kind.
517
+ void SetIndexLocations(IndexKind kind, const IndexLocations& locations);
518
+
519
+ // Complete the Thrift structure
520
+ std::unique_ptr<FileMetaData> Finish(
521
+ const std::shared_ptr<const KeyValueMetadata>& key_value_metadata = NULLPTR);
522
+
523
+ // crypto metadata
524
+ std::unique_ptr<FileCryptoMetaData> GetCryptoMetaData();
525
+
526
+ private:
527
+ explicit FileMetaDataBuilder(const SchemaDescriptor* schema,
528
+ std::shared_ptr<WriterProperties> props);
529
+ // PIMPL Idiom
530
+ class FileMetaDataBuilderImpl;
531
+ std::unique_ptr<FileMetaDataBuilderImpl> impl_;
532
+ };
533
+
534
+ PARQUET_EXPORT std::string ParquetVersionToString(ParquetVersion::type ver);
535
+
536
+ } // namespace parquet
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/page_index.h ADDED
@@ -0,0 +1,391 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include "arrow/io/interfaces.h"
21
+ #include "parquet/encryption/type_fwd.h"
22
+ #include "parquet/index_location.h"
23
+ #include "parquet/type_fwd.h"
24
+ #include "parquet/types.h"
25
+
26
+ #include <optional>
27
+ #include <vector>
28
+
29
+ namespace parquet {
30
+
31
+ /// \brief ColumnIndex is a proxy around format::ColumnIndex.
32
+ class PARQUET_EXPORT ColumnIndex {
33
+ public:
34
+ /// \brief Create a ColumnIndex from a serialized thrift message.
35
+ static std::unique_ptr<ColumnIndex> Make(const ColumnDescriptor& descr,
36
+ const void* serialized_index,
37
+ uint32_t index_len,
38
+ const ReaderProperties& properties,
39
+ Decryptor* decryptor = NULLPTR);
40
+
41
+ virtual ~ColumnIndex() = default;
42
+
43
+ /// \brief A bitmap with a bit set for each data page that has only null values.
44
+ ///
45
+ /// The length of this vector is equal to the number of data pages in the column.
46
+ virtual const std::vector<bool>& null_pages() const = 0;
47
+
48
+ /// \brief A vector of encoded lower bounds for each data page in this column.
49
+ ///
50
+ /// `null_pages` should be inspected first, as only pages with non-null values
51
+ /// may have their lower bounds populated.
52
+ virtual const std::vector<std::string>& encoded_min_values() const = 0;
53
+
54
+ /// \brief A vector of encoded upper bounds for each data page in this column.
55
+ ///
56
+ /// `null_pages` should be inspected first, as only pages with non-null values
57
+ /// may have their upper bounds populated.
58
+ virtual const std::vector<std::string>& encoded_max_values() const = 0;
59
+
60
+ /// \brief The ordering of lower and upper bounds.
61
+ ///
62
+ /// The boundary order applies across all lower bounds, and all upper bounds,
63
+ /// respectively. However, the order between lower bounds and upper bounds
64
+ /// cannot be derived from this.
65
+ virtual BoundaryOrder::type boundary_order() const = 0;
66
+
67
+ /// \brief Whether per-page null count information is available.
68
+ virtual bool has_null_counts() const = 0;
69
+
70
+ /// \brief An optional vector with the number of null values in each data page.
71
+ ///
72
+ /// `has_null_counts` should be called first to determine if this information is
73
+ /// available.
74
+ virtual const std::vector<int64_t>& null_counts() const = 0;
75
+
76
+ /// \brief A vector of page indices for non-null pages.
77
+ virtual const std::vector<int32_t>& non_null_page_indices() const = 0;
78
+
79
+ /// \brief Whether definition level histogram is available.
80
+ virtual bool has_definition_level_histograms() const = 0;
81
+
82
+ /// \brief Whether repetition level histogram is available.
83
+ virtual bool has_repetition_level_histograms() const = 0;
84
+
85
+ /// \brief List of definition level histograms for each page concatenated together.
86
+ virtual const std::vector<int64_t>& definition_level_histograms() const = 0;
87
+
88
+ /// \brief List of repetition level histograms for each page concatenated together.
89
+ virtual const std::vector<int64_t>& repetition_level_histograms() const = 0;
90
+ };
91
+
92
+ /// \brief Typed implementation of ColumnIndex.
93
+ template <typename DType>
94
+ class PARQUET_EXPORT TypedColumnIndex : public ColumnIndex {
95
+ public:
96
+ using T = typename DType::c_type;
97
+
98
+ /// \brief A vector of lower bounds for each data page in this column.
99
+ ///
100
+ /// This is like `encoded_min_values`, but with the values decoded according to
101
+ /// the column's physical type.
102
+ /// `min_values` and `max_values` can be used together with `boundary_order`
103
+ /// in order to prune some data pages when searching for specific values.
104
+ virtual const std::vector<T>& min_values() const = 0;
105
+
106
+ /// \brief A vector of upper bounds for each data page in this column.
107
+ ///
108
+ /// Just like `min_values`, but for upper bounds instead of lower bounds.
109
+ virtual const std::vector<T>& max_values() const = 0;
110
+ };
111
+
112
+ using BoolColumnIndex = TypedColumnIndex<BooleanType>;
113
+ using Int32ColumnIndex = TypedColumnIndex<Int32Type>;
114
+ using Int64ColumnIndex = TypedColumnIndex<Int64Type>;
115
+ using FloatColumnIndex = TypedColumnIndex<FloatType>;
116
+ using DoubleColumnIndex = TypedColumnIndex<DoubleType>;
117
+ using ByteArrayColumnIndex = TypedColumnIndex<ByteArrayType>;
118
+ using FLBAColumnIndex = TypedColumnIndex<FLBAType>;
119
+
120
+ /// \brief PageLocation is a proxy around format::PageLocation.
121
+ struct PARQUET_EXPORT PageLocation {
122
+ /// File offset of the data page.
123
+ int64_t offset;
124
+ /// Total compressed size of the data page and header.
125
+ int32_t compressed_page_size;
126
+ /// Row id of the first row in the page within the row group.
127
+ int64_t first_row_index;
128
+ };
129
+
130
+ /// \brief OffsetIndex is a proxy around format::OffsetIndex.
131
+ class PARQUET_EXPORT OffsetIndex {
132
+ public:
133
+ /// \brief Create a OffsetIndex from a serialized thrift message.
134
+ static std::unique_ptr<OffsetIndex> Make(const void* serialized_index,
135
+ uint32_t index_len,
136
+ const ReaderProperties& properties,
137
+ Decryptor* decryptor = NULLPTR);
138
+
139
+ virtual ~OffsetIndex() = default;
140
+
141
+ /// \brief A vector of locations for each data page in this column.
142
+ virtual const std::vector<PageLocation>& page_locations() const = 0;
143
+
144
+ /// \brief A vector of unencoded/uncompressed size of each page for BYTE_ARRAY types,
145
+ /// or empty for other types.
146
+ virtual const std::vector<int64_t>& unencoded_byte_array_data_bytes() const = 0;
147
+ };
148
+
149
+ /// \brief Interface for reading the page index for a Parquet row group.
150
+ class PARQUET_EXPORT RowGroupPageIndexReader {
151
+ public:
152
+ virtual ~RowGroupPageIndexReader() = default;
153
+
154
+ /// \brief Read column index of a column chunk.
155
+ ///
156
+ /// \param[in] i column ordinal of the column chunk.
157
+ /// \returns column index of the column or nullptr if it does not exist.
158
+ /// \throws ParquetException if the index is out of bound.
159
+ virtual std::shared_ptr<ColumnIndex> GetColumnIndex(int32_t i) = 0;
160
+
161
+ /// \brief Read offset index of a column chunk.
162
+ ///
163
+ /// \param[in] i column ordinal of the column chunk.
164
+ /// \returns offset index of the column or nullptr if it does not exist.
165
+ /// \throws ParquetException if the index is out of bound.
166
+ virtual std::shared_ptr<OffsetIndex> GetOffsetIndex(int32_t i) = 0;
167
+ };
168
+
169
+ struct PageIndexSelection {
170
+ /// Specifies whether to read the column index.
171
+ bool column_index = false;
172
+ /// Specifies whether to read the offset index.
173
+ bool offset_index = false;
174
+ };
175
+
176
+ PARQUET_EXPORT
177
+ std::ostream& operator<<(std::ostream& out, const PageIndexSelection& params);
178
+
179
+ struct RowGroupIndexReadRange {
180
+ /// Base start and total size of column index of all column chunks in a row group.
181
+ /// If none of the column chunks have column index, it is set to std::nullopt.
182
+ std::optional<::arrow::io::ReadRange> column_index = std::nullopt;
183
+ /// Base start and total size of offset index of all column chunks in a row group.
184
+ /// If none of the column chunks have offset index, it is set to std::nullopt.
185
+ std::optional<::arrow::io::ReadRange> offset_index = std::nullopt;
186
+ };
187
+
188
+ /// \brief Interface for reading the page index for a Parquet file.
189
+ class PARQUET_EXPORT PageIndexReader {
190
+ public:
191
+ virtual ~PageIndexReader() = default;
192
+
193
+ /// \brief Create a PageIndexReader instance.
194
+ /// \returns a PageIndexReader instance.
195
+ /// WARNING: The returned PageIndexReader references to all the input parameters, so
196
+ /// it must not outlive all of the input parameters. Usually these input parameters
197
+ /// come from the same ParquetFileReader object, so it must not outlive the reader
198
+ /// that creates this PageIndexReader.
199
+ static std::shared_ptr<PageIndexReader> Make(
200
+ ::arrow::io::RandomAccessFile* input, std::shared_ptr<FileMetaData> file_metadata,
201
+ const ReaderProperties& properties,
202
+ InternalFileDecryptor* file_decryptor = NULLPTR);
203
+
204
+ /// \brief Get the page index reader of a specific row group.
205
+ /// \param[in] i row group ordinal to get page index reader.
206
+ /// \returns RowGroupPageIndexReader of the specified row group. A nullptr may or may
207
+ /// not be returned if the page index for the row group is unavailable. It is
208
+ /// the caller's responsibility to check the return value of follow-up calls
209
+ /// to the RowGroupPageIndexReader.
210
+ /// \throws ParquetException if the index is out of bound.
211
+ virtual std::shared_ptr<RowGroupPageIndexReader> RowGroup(int i) = 0;
212
+
213
+ /// \brief Advise the reader which part of page index will be read later.
214
+ ///
215
+ /// The PageIndexReader can optionally prefetch and cache page index that
216
+ /// may be read later to get better performance.
217
+ ///
218
+ /// The contract of this function is as below:
219
+ /// 1) If WillNeed() has not been called for a specific row group and the page index
220
+ /// exists, follow-up calls to get column index or offset index of all columns in
221
+ /// this row group SHOULD NOT FAIL, but the performance may not be optimal.
222
+ /// 2) If WillNeed() has been called for a specific row group, follow-up calls to get
223
+ /// page index are limited to columns and index type requested by WillNeed().
224
+ /// So it MAY FAIL if columns that are not requested by WillNeed() are requested.
225
+ /// 3) Later calls to WillNeed() MAY OVERRIDE previous calls of same row groups.
226
+ /// For example,
227
+ /// 1) If WillNeed() is not called for row group 0, then follow-up calls to read
228
+ /// column index and/or offset index of all columns of row group 0 should not
229
+ /// fail if its page index exists.
230
+ /// 2) If WillNeed() is called for columns 0 and 1 for row group 0, then follow-up
231
+ /// call to read page index of column 2 for row group 0 MAY FAIL even if its
232
+ /// page index exists.
233
+ /// 3) If WillNeed() is called for row group 0 with offset index only, then
234
+ /// follow-up call to read column index of row group 0 MAY FAIL even if
235
+ /// the column index of this column exists.
236
+ /// 4) If WillNeed() is called for columns 0 and 1 for row group 0, then later
237
+ /// call to WillNeed() for columns 1 and 2 for row group 0. The later one
238
+ /// overrides previous call and only columns 1 and 2 of row group 0 are allowed
239
+ /// to access.
240
+ ///
241
+ /// \param[in] row_group_indices list of row group ordinal to read page index later.
242
+ /// \param[in] column_indices list of column ordinal to read page index later. If it is
243
+ /// empty, it means all columns in the row group will be read.
244
+ /// \param[in] selection which kind of page index is required later.
245
+ virtual void WillNeed(const std::vector<int32_t>& row_group_indices,
246
+ const std::vector<int32_t>& column_indices,
247
+ const PageIndexSelection& selection) = 0;
248
+
249
+ /// \brief Advise the reader page index of these row groups will not be read anymore.
250
+ ///
251
+ /// The PageIndexReader implementation has the opportunity to cancel any prefetch or
252
+ /// release resource that are related to these row groups.
253
+ ///
254
+ /// \param[in] row_group_indices list of row group ordinal that whose page index will
255
+ /// not be accessed anymore.
256
+ virtual void WillNotNeed(const std::vector<int32_t>& row_group_indices) = 0;
257
+
258
+ /// \brief Determine the column index and offset index ranges for the given row group.
259
+ ///
260
+ /// \param[in] row_group_metadata row group metadata to get column chunk metadata.
261
+ /// \param[in] columns list of column ordinals to get page index. If the list is empty,
262
+ /// it means all columns in the row group.
263
+ /// \returns RowGroupIndexReadRange of the specified row group. Throws ParquetException
264
+ /// if the selected column ordinal is out of bound or metadata of page index
265
+ /// is corrupted.
266
+ static RowGroupIndexReadRange DeterminePageIndexRangesInRowGroup(
267
+ const RowGroupMetaData& row_group_metadata, const std::vector<int32_t>& columns);
268
+ };
269
+
270
+ /// \brief Interface for collecting column index of data pages in a column chunk.
271
+ class PARQUET_EXPORT ColumnIndexBuilder {
272
+ public:
273
+ /// \brief API convenience to create a ColumnIndexBuilder.
274
+ static std::unique_ptr<ColumnIndexBuilder> Make(const ColumnDescriptor* descr);
275
+
276
+ virtual ~ColumnIndexBuilder() = default;
277
+
278
+ /// \brief Add statistics of a data page.
279
+ ///
280
+ /// If the ColumnIndexBuilder has seen any corrupted statistics, it will
281
+ /// not update statistics anymore.
282
+ ///
283
+ /// \param stats Page statistics in the encoded form.
284
+ /// \param size_stats Size statistics of the page if available.
285
+ virtual void AddPage(const EncodedStatistics& stats,
286
+ const SizeStatistics& size_stats) = 0;
287
+
288
+ /// \brief Complete the column index.
289
+ ///
290
+ /// Once called, AddPage() can no longer be called.
291
+ /// WriteTo() and Build() can only called after Finish() has been called.
292
+ virtual void Finish() = 0;
293
+
294
+ /// \brief Serialize the column index thrift message.
295
+ ///
296
+ /// If the ColumnIndexBuilder has seen any corrupted statistics, it will
297
+ /// not write any data to the sink.
298
+ ///
299
+ /// \param[out] sink output stream to write the serialized message.
300
+ /// \param[in] encryptor encryptor to encrypt the serialized column index.
301
+ virtual void WriteTo(::arrow::io::OutputStream* sink,
302
+ Encryptor* encryptor = NULLPTR) const = 0;
303
+
304
+ /// \brief Create a ColumnIndex directly.
305
+ ///
306
+ /// \return If the ColumnIndexBuilder has seen any corrupted statistics, it simply
307
+ /// returns nullptr. Otherwise the column index is built and returned.
308
+ virtual std::unique_ptr<ColumnIndex> Build() const = 0;
309
+ };
310
+
311
+ /// \brief Interface for collecting offset index of data pages in a column chunk.
312
+ class PARQUET_EXPORT OffsetIndexBuilder {
313
+ public:
314
+ /// \brief API convenience to create a OffsetIndexBuilder.
315
+ static std::unique_ptr<OffsetIndexBuilder> Make();
316
+
317
+ virtual ~OffsetIndexBuilder() = default;
318
+
319
+ /// \brief Add page location and size stats of a data page.
320
+ virtual void AddPage(int64_t offset, int32_t compressed_page_size,
321
+ int64_t first_row_index,
322
+ std::optional<int64_t> unencoded_byte_array_length = {}) = 0;
323
+
324
+ /// \brief Add page location and size stats of a data page.
325
+ void AddPage(const PageLocation& page_location, const SizeStatistics& size_stats);
326
+
327
+ /// \brief Complete the offset index.
328
+ ///
329
+ /// In the buffered row group mode, data pages are flushed into memory
330
+ /// sink and the OffsetIndexBuilder has only collected the relative offset
331
+ /// which requires adjustment once they are flushed to the file.
332
+ ///
333
+ /// \param final_position Final stream offset to add for page offset adjustment.
334
+ virtual void Finish(int64_t final_position) = 0;
335
+
336
+ /// \brief Serialize the offset index thrift message.
337
+ ///
338
+ /// \param[out] sink output stream to write the serialized message.
339
+ /// \param[in] encryptor encryptor to encrypt the serialized offset index.
340
+ virtual void WriteTo(::arrow::io::OutputStream* sink,
341
+ Encryptor* encryptor = NULLPTR) const = 0;
342
+
343
+ /// \brief Create an OffsetIndex directly.
344
+ virtual std::unique_ptr<OffsetIndex> Build() const = 0;
345
+ };
346
+
347
+ /// \brief Interface for collecting page index of a parquet file.
348
+ class PARQUET_EXPORT PageIndexBuilder {
349
+ public:
350
+ /// \brief API convenience to create a PageIndexBuilder.
351
+ static std::unique_ptr<PageIndexBuilder> Make(
352
+ const SchemaDescriptor* schema, InternalFileEncryptor* file_encryptor = NULLPTR);
353
+
354
+ virtual ~PageIndexBuilder() = default;
355
+
356
+ /// \brief Start a new row group.
357
+ virtual void AppendRowGroup() = 0;
358
+
359
+ /// \brief Get the ColumnIndexBuilder from column ordinal.
360
+ ///
361
+ /// \param i Column ordinal.
362
+ /// \return ColumnIndexBuilder for the column and its memory ownership belongs to
363
+ /// the PageIndexBuilder.
364
+ virtual ColumnIndexBuilder* GetColumnIndexBuilder(int32_t i) = 0;
365
+
366
+ /// \brief Get the OffsetIndexBuilder from column ordinal.
367
+ ///
368
+ /// \param i Column ordinal.
369
+ /// \return OffsetIndexBuilder for the column and its memory ownership belongs to
370
+ /// the PageIndexBuilder.
371
+ virtual OffsetIndexBuilder* GetOffsetIndexBuilder(int32_t i) = 0;
372
+
373
+ /// \brief Complete the page index builder and no more write is allowed.
374
+ virtual void Finish() = 0;
375
+
376
+ struct WriteResult {
377
+ IndexLocations column_index_locations;
378
+ IndexLocations offset_index_locations;
379
+ };
380
+
381
+ /// \brief Serialize the page index thrift message.
382
+ ///
383
+ /// Only valid column indexes and offset indexes are serialized and their locations
384
+ /// are set.
385
+ ///
386
+ /// \param[out] sink The output stream to write the page index.
387
+ /// \return The location of all page indexes.
388
+ virtual WriteResult WriteTo(::arrow::io::OutputStream* sink) const = 0;
389
+ };
390
+
391
+ } // namespace parquet
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/parquet_version.h ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #ifndef PARQUET_VERSION_H
19
+ #define PARQUET_VERSION_H
20
+
21
+ #define PARQUET_VERSION_MAJOR 24
22
+ #define PARQUET_VERSION_MINOR 0
23
+ #define PARQUET_VERSION_PATCH 0
24
+
25
+ #define PARQUET_SO_VERSION "2400"
26
+ #define PARQUET_FULL_SO_VERSION "2400.0.0"
27
+
28
+ // define the parquet created by version
29
+ #define CREATED_BY_VERSION "parquet-cpp-arrow version 24.0.0"
30
+
31
+ #endif // PARQUET_VERSION_H
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/platform.h ADDED
@@ -0,0 +1,116 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <cstdint>
21
+ #include <memory>
22
+
23
+ #include "arrow/buffer.h" // IWYU pragma: export
24
+ #include "arrow/io/interfaces.h" // IWYU pragma: export
25
+ #include "arrow/status.h" // IWYU pragma: export
26
+ #include "arrow/type_fwd.h" // IWYU pragma: export
27
+ #include "arrow/util/macros.h" // IWYU pragma: export
28
+
29
+ #if defined(_WIN32) || defined(__CYGWIN__)
30
+
31
+ # ifdef _MSC_VER
32
+ # pragma warning(push)
33
+ // Disable warning for STL types usage in DLL interface
34
+ // https://web.archive.org/web/20130317015847/http://connect.microsoft.com/VisualStudio/feedback/details/696593/vc-10-vs-2010-basic-string-exports
35
+ # pragma warning(disable : 4275 4251)
36
+ // Disable diamond inheritance warnings
37
+ # pragma warning(disable : 4250)
38
+ // Disable macro redefinition warnings
39
+ # pragma warning(disable : 4005)
40
+ // Disable extern before exported template warnings
41
+ # pragma warning(disable : 4910)
42
+ # else
43
+ # pragma GCC diagnostic ignored "-Wattributes"
44
+ # endif
45
+
46
+ # ifdef PARQUET_STATIC
47
+ # define PARQUET_EXPORT
48
+ # elif defined(PARQUET_EXPORTING)
49
+ # define PARQUET_EXPORT __declspec(dllexport)
50
+ # else
51
+ # define PARQUET_EXPORT __declspec(dllimport)
52
+ # endif
53
+
54
+ # define PARQUET_NO_EXPORT
55
+
56
+ #else // Not Windows
57
+ # ifndef PARQUET_EXPORT
58
+ # define PARQUET_EXPORT __attribute__((visibility("default")))
59
+ # endif
60
+ # ifndef PARQUET_NO_EXPORT
61
+ # define PARQUET_NO_EXPORT __attribute__((visibility("hidden")))
62
+ # endif
63
+ #endif // Non-Windows
64
+
65
+ // This is a complicated topic, some reading on it:
66
+ // http://www.codesynthesis.com/~boris/blog/2010/01/18/dll-export-cxx-templates/
67
+ #if defined(_MSC_VER) || defined(__clang__)
68
+ # define PARQUET_TEMPLATE_CLASS_EXPORT
69
+ # define PARQUET_TEMPLATE_EXPORT PARQUET_EXPORT
70
+ #else
71
+ # define PARQUET_TEMPLATE_CLASS_EXPORT PARQUET_EXPORT
72
+ # define PARQUET_TEMPLATE_EXPORT
73
+ #endif
74
+
75
+ #define PARQUET_DISALLOW_COPY_AND_ASSIGN ARROW_DISALLOW_COPY_AND_ASSIGN
76
+
77
+ #define PARQUET_NORETURN ARROW_NORETURN
78
+ #define PARQUET_DEPRECATED ARROW_DEPRECATED
79
+
80
+ // If ARROW_VALGRIND set when compiling unit tests, also define
81
+ // PARQUET_VALGRIND
82
+ #ifdef ARROW_VALGRIND
83
+ # define PARQUET_VALGRIND
84
+ #endif
85
+
86
+ namespace parquet {
87
+
88
+ using Buffer = ::arrow::Buffer;
89
+ using Codec = ::arrow::util::Codec;
90
+ using CodecOptions = ::arrow::util::CodecOptions;
91
+ using Compression = ::arrow::Compression;
92
+ using MemoryPool = ::arrow::MemoryPool;
93
+ using MutableBuffer = ::arrow::MutableBuffer;
94
+ using ResizableBuffer = ::arrow::ResizableBuffer;
95
+ using ResizableBuffer = ::arrow::ResizableBuffer;
96
+ using ArrowInputFile = ::arrow::io::RandomAccessFile;
97
+ using ArrowInputStream = ::arrow::io::InputStream;
98
+ using ArrowOutputStream = ::arrow::io::OutputStream;
99
+
100
+ constexpr int64_t kDefaultOutputStreamSize = 1024;
101
+
102
+ constexpr int16_t kNonPageOrdinal = static_cast<int16_t>(-1);
103
+
104
+ PARQUET_EXPORT
105
+ std::shared_ptr<::arrow::io::BufferOutputStream> CreateOutputStream(
106
+ ::arrow::MemoryPool* pool = ::arrow::default_memory_pool());
107
+
108
+ PARQUET_EXPORT
109
+ std::shared_ptr<ResizableBuffer> AllocateBuffer(
110
+ ::arrow::MemoryPool* pool = ::arrow::default_memory_pool(), int64_t size = 0);
111
+
112
+ } // namespace parquet
113
+
114
+ #ifdef _MSC_VER
115
+ # pragma warning(pop)
116
+ #endif
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/printer.h ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <iosfwd>
21
+ #include <list>
22
+
23
+ #include "parquet/platform.h"
24
+
25
+ namespace parquet {
26
+
27
+ class ParquetFileReader;
28
+
29
+ class PARQUET_EXPORT ParquetFilePrinter {
30
+ private:
31
+ ParquetFileReader* fileReader;
32
+
33
+ public:
34
+ explicit ParquetFilePrinter(ParquetFileReader* reader) : fileReader(reader) {}
35
+ ~ParquetFilePrinter() = default;
36
+
37
+ void DebugPrint(std::ostream& stream, std::list<int> selected_columns,
38
+ bool print_values = false, bool format_dump = false,
39
+ bool print_key_value_metadata = false,
40
+ const char* filename = "No Name");
41
+
42
+ void JSONPrint(std::ostream& stream, std::list<int> selected_columns,
43
+ const char* filename = "No Name");
44
+ };
45
+
46
+ } // namespace parquet
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/properties.h ADDED
@@ -0,0 +1,1529 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <memory>
21
+ #include <string>
22
+ #include <unordered_map>
23
+ #include <unordered_set>
24
+ #include <utility>
25
+
26
+ #include "arrow/io/caching.h"
27
+ #include "arrow/type_fwd.h"
28
+ #include "arrow/util/compression.h"
29
+ #include "arrow/util/type_fwd.h"
30
+ #include "parquet/encryption/encryption.h"
31
+ #include "parquet/exception.h"
32
+ #include "parquet/parquet_version.h"
33
+ #include "parquet/platform.h"
34
+ #include "parquet/schema.h"
35
+ #include "parquet/type_fwd.h"
36
+ #include "parquet/types.h"
37
+
38
+ namespace parquet {
39
+
40
+ /// Controls serialization format of data pages. parquet-format v2.0.0
41
+ /// introduced a new data page metadata type DataPageV2 and serialized page
42
+ /// structure (for example, encoded levels are no longer compressed). Prior to
43
+ /// the completion of PARQUET-457 in 2020, this library did not implement
44
+ /// DataPageV2 correctly, so if you use the V2 data page format, you may have
45
+ /// forward compatibility issues (older versions of the library will be unable
46
+ /// to read the files). Note that some Parquet implementations do not implement
47
+ /// DataPageV2 at all.
48
+ enum class ParquetDataPageVersion { V1, V2 };
49
+
50
+ /// Controls the level of size statistics that are written to the file.
51
+ enum class SizeStatisticsLevel : uint8_t {
52
+ // No size statistics are written.
53
+ None = 0,
54
+ // Only column chunk size statistics are written.
55
+ ColumnChunk,
56
+ // Both size statistics in the column chunk and page index are written.
57
+ PageAndColumnChunk
58
+ };
59
+
60
+ /// Align the default buffer size to a small multiple of a page size.
61
+ constexpr int64_t kDefaultBufferSize = 4096 * 4;
62
+
63
+ constexpr int32_t kDefaultThriftStringSizeLimit = 100 * 1000 * 1000;
64
+ // Structs in the thrift definition are relatively large (at least 300 bytes).
65
+ // This limits total memory to the same order of magnitude as
66
+ // kDefaultStringSizeLimit.
67
+ constexpr int32_t kDefaultThriftContainerSizeLimit = 1000 * 1000;
68
+
69
+ // PARQUET-978: Minimize footer reads by reading 64 KB from the end of the file
70
+ constexpr int64_t kDefaultFooterReadSize = 64 * 1024;
71
+
72
+ class PARQUET_EXPORT ReaderProperties {
73
+ public:
74
+ explicit ReaderProperties(MemoryPool* pool = ::arrow::default_memory_pool())
75
+ : pool_(pool) {}
76
+
77
+ MemoryPool* memory_pool() const { return pool_; }
78
+
79
+ std::shared_ptr<ArrowInputStream> GetStream(std::shared_ptr<ArrowInputFile> source,
80
+ int64_t start, int64_t num_bytes);
81
+
82
+ /// Buffered stream reading allows the user to control the memory usage of
83
+ /// parquet readers. This ensure that all `RandomAccessFile::ReadAt` calls are
84
+ /// wrapped in a buffered reader that uses a fix sized buffer (of size
85
+ /// `buffer_size()`) instead of the full size of the ReadAt.
86
+ ///
87
+ /// The primary reason for this control knobs is for resource control and not
88
+ /// performance.
89
+ bool is_buffered_stream_enabled() const { return buffered_stream_enabled_; }
90
+ /// Enable buffered stream reading.
91
+ void enable_buffered_stream() { buffered_stream_enabled_ = true; }
92
+ /// Disable buffered stream reading.
93
+ void disable_buffered_stream() { buffered_stream_enabled_ = false; }
94
+
95
+ bool read_dense_for_nullable() const { return read_dense_for_nullable_; }
96
+ void enable_read_dense_for_nullable() { read_dense_for_nullable_ = true; }
97
+ void disable_read_dense_for_nullable() { read_dense_for_nullable_ = false; }
98
+
99
+ /// Return the size of the buffered stream buffer.
100
+ int64_t buffer_size() const { return buffer_size_; }
101
+ /// Set the size of the buffered stream buffer in bytes.
102
+ void set_buffer_size(int64_t size) { buffer_size_ = size; }
103
+
104
+ /// \brief Return the size limit on thrift strings.
105
+ ///
106
+ /// This limit helps prevent space and time bombs in files, but may need to
107
+ /// be increased in order to read files with especially large headers.
108
+ int32_t thrift_string_size_limit() const { return thrift_string_size_limit_; }
109
+ /// Set the size limit on thrift strings.
110
+ void set_thrift_string_size_limit(int32_t size) { thrift_string_size_limit_ = size; }
111
+
112
+ /// \brief Return the size limit on thrift containers.
113
+ ///
114
+ /// This limit helps prevent space and time bombs in files, but may need to
115
+ /// be increased in order to read files with especially large headers.
116
+ int32_t thrift_container_size_limit() const { return thrift_container_size_limit_; }
117
+ /// Set the size limit on thrift containers.
118
+ void set_thrift_container_size_limit(int32_t size) {
119
+ thrift_container_size_limit_ = size;
120
+ }
121
+
122
+ /// Set the decryption properties.
123
+ void file_decryption_properties(std::shared_ptr<FileDecryptionProperties> decryption) {
124
+ file_decryption_properties_ = std::move(decryption);
125
+ }
126
+ /// Return the decryption properties.
127
+ const std::shared_ptr<FileDecryptionProperties>& file_decryption_properties() const {
128
+ return file_decryption_properties_;
129
+ }
130
+
131
+ bool page_checksum_verification() const { return page_checksum_verification_; }
132
+ void set_page_checksum_verification(bool check_crc) {
133
+ page_checksum_verification_ = check_crc;
134
+ }
135
+
136
+ // Set the default read size to read the footer from a file. For high latency
137
+ // file systems and files with large metadata (>64KB) this can increase performance
138
+ // by reducing the number of round-trips to retrieve the entire file metadata.
139
+ void set_footer_read_size(size_t size) { footer_read_size_ = size; }
140
+ size_t footer_read_size() const { return footer_read_size_; }
141
+
142
+ private:
143
+ MemoryPool* pool_;
144
+ int64_t buffer_size_ = kDefaultBufferSize;
145
+ int32_t thrift_string_size_limit_ = kDefaultThriftStringSizeLimit;
146
+ int32_t thrift_container_size_limit_ = kDefaultThriftContainerSizeLimit;
147
+ bool buffered_stream_enabled_ = false;
148
+ bool page_checksum_verification_ = false;
149
+ // Used with a RecordReader.
150
+ bool read_dense_for_nullable_ = false;
151
+ size_t footer_read_size_ = kDefaultFooterReadSize;
152
+ std::shared_ptr<FileDecryptionProperties> file_decryption_properties_;
153
+ };
154
+
155
+ ReaderProperties PARQUET_EXPORT default_reader_properties();
156
+
157
+ static constexpr int64_t kDefaultDataPageSize = 1024 * 1024;
158
+ static constexpr int64_t kDefaultMaxRowsPerPage = 20'000;
159
+ static constexpr bool DEFAULT_IS_DICTIONARY_ENABLED = true;
160
+ static constexpr int64_t DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT = kDefaultDataPageSize;
161
+ static constexpr int64_t DEFAULT_WRITE_BATCH_SIZE = 1024;
162
+ static constexpr int64_t DEFAULT_MAX_ROW_GROUP_LENGTH = 1024 * 1024;
163
+ static constexpr bool DEFAULT_ARE_STATISTICS_ENABLED = true;
164
+ static constexpr int64_t DEFAULT_MAX_STATISTICS_SIZE = 4096;
165
+ static constexpr Encoding::type DEFAULT_ENCODING = Encoding::UNKNOWN;
166
+ static const char DEFAULT_CREATED_BY[] = CREATED_BY_VERSION;
167
+ static constexpr Compression::type DEFAULT_COMPRESSION_TYPE = Compression::UNCOMPRESSED;
168
+ static constexpr bool DEFAULT_IS_PAGE_INDEX_ENABLED = true;
169
+ static constexpr SizeStatisticsLevel DEFAULT_SIZE_STATISTICS_LEVEL =
170
+ SizeStatisticsLevel::PageAndColumnChunk;
171
+
172
+ struct PARQUET_EXPORT BloomFilterOptions {
173
+ /// Expected number of distinct values (NDV) in the bloom filter.
174
+ ///
175
+ /// Bloom filters are most effective for high-cardinality columns. A good default
176
+ /// is to set ndv equal to the number of rows. Lower values reduce disk usage but
177
+ /// may not be worthwhile for very small NDVs.
178
+ ///
179
+ /// Increasing ndv (without increasing fpp) increases disk and memory usage.
180
+ int32_t ndv = 1 << 20;
181
+
182
+ /// False-positive probability (FPP) of the bloom filter.
183
+ ///
184
+ /// Lower FPP values require more disk and memory space. For a fixed ndv, the
185
+ /// space requirement grows roughly proportional to log(1/fpp). Recommended
186
+ /// values are 0.1, 0.05, or 0.01. Very small values are counterproductive as
187
+ /// the bitset may exceed the size of the actual data. Set ndv appropriately
188
+ /// to minimize space usage.
189
+ ///
190
+ /// Below is a table to demonstrate estimated size using common values.
191
+ ///
192
+ /// | ndv | fpp | bits/key | size |
193
+ /// |:-----------|:------|:---------|:----------|
194
+ /// | 100,000 | 0.10 | 10.5 | 128 KiB |
195
+ /// | 100,000 | 0.05 | 10.5 | 128 KiB |
196
+ /// | 100,000 | 0.01 | 10.5 | 128 KiB |
197
+ /// | 1,000,000 | 0.10 | 8.4 | 1024 KiB |
198
+ /// | 1,000,000 | 0.05 | 8.4 | 1024 KiB |
199
+ /// | 1,000,000 | 0.01 | 16.8 | 2048 KiB |
200
+ /// | 10,000,000 | 0.10 | 6.7 | 8192 KiB |
201
+ /// | 10,000,000 | 0.05 | 13.4 | 16384 KiB |
202
+ /// | 10,000,000 | 0.01 | 13.4 | 16384 KiB |
203
+ double fpp = 0.05;
204
+ };
205
+
206
+ class PARQUET_EXPORT ColumnProperties {
207
+ public:
208
+ ColumnProperties(Encoding::type encoding = DEFAULT_ENCODING,
209
+ Compression::type codec = DEFAULT_COMPRESSION_TYPE,
210
+ bool dictionary_enabled = DEFAULT_IS_DICTIONARY_ENABLED,
211
+ bool statistics_enabled = DEFAULT_ARE_STATISTICS_ENABLED,
212
+ size_t max_stats_size = DEFAULT_MAX_STATISTICS_SIZE,
213
+ bool page_index_enabled = DEFAULT_IS_PAGE_INDEX_ENABLED)
214
+ : encoding_(encoding),
215
+ codec_(codec),
216
+ dictionary_enabled_(dictionary_enabled),
217
+ statistics_enabled_(statistics_enabled),
218
+ max_stats_size_(max_stats_size),
219
+ page_index_enabled_(page_index_enabled) {}
220
+
221
+ void set_encoding(Encoding::type encoding) { encoding_ = encoding; }
222
+
223
+ void set_compression(Compression::type codec) { codec_ = codec; }
224
+
225
+ void set_dictionary_enabled(bool dictionary_enabled) {
226
+ dictionary_enabled_ = dictionary_enabled;
227
+ }
228
+
229
+ void set_statistics_enabled(bool statistics_enabled) {
230
+ statistics_enabled_ = statistics_enabled;
231
+ }
232
+
233
+ void set_max_statistics_size(size_t max_stats_size) {
234
+ max_stats_size_ = max_stats_size;
235
+ }
236
+
237
+ void set_compression_level(int compression_level) {
238
+ if (!codec_options_) {
239
+ codec_options_ = std::make_shared<CodecOptions>();
240
+ }
241
+ codec_options_->compression_level = compression_level;
242
+ }
243
+
244
+ void set_codec_options(const std::shared_ptr<CodecOptions>& codec_options) {
245
+ codec_options_ = codec_options;
246
+ }
247
+
248
+ void set_page_index_enabled(bool page_index_enabled) {
249
+ page_index_enabled_ = page_index_enabled;
250
+ }
251
+
252
+ void set_bloom_filter_options(const BloomFilterOptions& bloom_filter_options) {
253
+ if (bloom_filter_options.fpp >= 1.0 || bloom_filter_options.fpp <= 0.0) {
254
+ throw ParquetException(
255
+ "Bloom filter false positive probability must be in (0.0, 1.0), got " +
256
+ std::to_string(bloom_filter_options.fpp));
257
+ }
258
+ bloom_filter_options_ = bloom_filter_options;
259
+ }
260
+
261
+ Encoding::type encoding() const { return encoding_; }
262
+
263
+ Compression::type compression() const { return codec_; }
264
+
265
+ bool dictionary_enabled() const { return dictionary_enabled_; }
266
+
267
+ bool statistics_enabled() const { return statistics_enabled_; }
268
+
269
+ size_t max_statistics_size() const { return max_stats_size_; }
270
+
271
+ int compression_level() const {
272
+ if (!codec_options_) {
273
+ return ::arrow::util::kUseDefaultCompressionLevel;
274
+ }
275
+ return codec_options_->compression_level;
276
+ }
277
+
278
+ const std::shared_ptr<CodecOptions>& codec_options() const { return codec_options_; }
279
+
280
+ bool page_index_enabled() const { return page_index_enabled_; }
281
+
282
+ std::optional<BloomFilterOptions> bloom_filter_options() const {
283
+ return bloom_filter_options_;
284
+ }
285
+
286
+ bool bloom_filter_enabled() const { return bloom_filter_options_.has_value(); }
287
+
288
+ private:
289
+ Encoding::type encoding_;
290
+ Compression::type codec_;
291
+ bool dictionary_enabled_;
292
+ bool statistics_enabled_;
293
+ size_t max_stats_size_;
294
+ std::shared_ptr<CodecOptions> codec_options_;
295
+ bool page_index_enabled_;
296
+ std::optional<BloomFilterOptions> bloom_filter_options_;
297
+ };
298
+
299
+ // EXPERIMENTAL: Options for content-defined chunking.
300
+ ///
301
+ /// Content-defined chunking is an experimental feature that optimizes parquet
302
+ /// files for content addressable storage (CAS) systems by writing data pages
303
+ /// according to content-defined chunk boundaries. This allows for more
304
+ /// efficient deduplication of data across files, hence more efficient network
305
+ /// transfers and storage.
306
+ /// Each content-defined chunk is written as a separate parquet data page. The
307
+ /// following options control the chunks' size and the chunking process. Note
308
+ /// that the chunk size is calculated based on the logical value of the data,
309
+ /// before any encoding or compression is applied.
310
+ struct PARQUET_EXPORT CdcOptions {
311
+ /// Minimum chunk size in bytes, default is 256 KiB
312
+ /// The rolling hash will not be updated until this size is reached for each chunk.
313
+ /// Note that all data sent through the hash function is counted towards the chunk
314
+ /// size, including definition and repetition levels if present.
315
+ int64_t min_chunk_size = 256 * 1024;
316
+ /// Maximum chunk size in bytes, default is 1024 KiB
317
+ /// The chunker will create a new chunk whenever the chunk size exceeds this value.
318
+ /// Note that the parquet writer has a related `pagesize` property that controls
319
+ /// the maximum size of a parquet data page after encoding. While setting
320
+ /// `pagesize` to a smaller value than `max_chunk_size` doesn't affect the
321
+ /// chunking effectiveness, it results in more small parquet data pages.
322
+ int64_t max_chunk_size = 1024 * 1024;
323
+ /// Number of bit adjustment to the gearhash mask in order to center the chunk size
324
+ /// around the average size more aggressively, default is 0
325
+ /// Increasing the normalization level increases the probability of finding a chunk,
326
+ /// improving the deduplication ratio, but also increasing the number of small chunks
327
+ /// resulting in many small parquet data pages. The default value provides a good
328
+ /// balance between deduplication ratio and fragmentation.
329
+ /// Use norm_level=1 or norm_level=2 to reach a higher deduplication ratio at the
330
+ /// expense of fragmentation. Negative values can also be used to reduce the
331
+ /// probability of finding a chunk, resulting in larger chunks and fewer data pages.
332
+ /// Note that values outside [-3, 3] are not recommended, prefer using the default
333
+ /// value of 0 for most use cases.
334
+ int norm_level = 0;
335
+ };
336
+
337
+ class PARQUET_EXPORT WriterProperties {
338
+ public:
339
+ class PARQUET_EXPORT Builder {
340
+ public:
341
+ Builder()
342
+ : pool_(::arrow::default_memory_pool()),
343
+ dictionary_pagesize_limit_(DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT),
344
+ write_batch_size_(DEFAULT_WRITE_BATCH_SIZE),
345
+ max_row_group_length_(DEFAULT_MAX_ROW_GROUP_LENGTH),
346
+ pagesize_(kDefaultDataPageSize),
347
+ max_rows_per_page_(kDefaultMaxRowsPerPage),
348
+ version_(ParquetVersion::PARQUET_2_6),
349
+ data_page_version_(ParquetDataPageVersion::V1),
350
+ created_by_(DEFAULT_CREATED_BY),
351
+ store_decimal_as_integer_(false),
352
+ page_checksum_enabled_(false),
353
+ size_statistics_level_(DEFAULT_SIZE_STATISTICS_LEVEL),
354
+ content_defined_chunking_enabled_(false),
355
+ content_defined_chunking_options_({}) {}
356
+
357
+ explicit Builder(const WriterProperties& properties)
358
+ : pool_(properties.memory_pool()),
359
+ dictionary_pagesize_limit_(properties.dictionary_pagesize_limit()),
360
+ write_batch_size_(properties.write_batch_size()),
361
+ max_row_group_length_(properties.max_row_group_length()),
362
+ pagesize_(properties.data_pagesize()),
363
+ max_rows_per_page_(properties.max_rows_per_page()),
364
+ version_(properties.version()),
365
+ data_page_version_(properties.data_page_version()),
366
+ created_by_(properties.created_by()),
367
+ store_decimal_as_integer_(properties.store_decimal_as_integer()),
368
+ page_checksum_enabled_(properties.page_checksum_enabled()),
369
+ size_statistics_level_(properties.size_statistics_level()),
370
+ sorting_columns_(properties.sorting_columns()),
371
+ default_column_properties_(properties.default_column_properties()),
372
+ content_defined_chunking_enabled_(
373
+ properties.content_defined_chunking_enabled()),
374
+ content_defined_chunking_options_(
375
+ properties.content_defined_chunking_options()) {
376
+ CopyColumnSpecificProperties(properties);
377
+ }
378
+
379
+ /// \brief EXPERIMENTAL: Use content-defined page chunking for all columns.
380
+ ///
381
+ /// Optimize parquet files for content addressable storage (CAS) systems by writing
382
+ /// data pages according to content-defined chunk boundaries. This allows for more
383
+ /// efficient deduplication of data across files, hence more efficient network
384
+ /// transfers and storage. The chunking is based on a rolling hash algorithm that
385
+ /// identifies chunk boundaries based on the actual content of the data.
386
+ ///
387
+ /// Note that only the WriteArrow() interface is supported at the moment.
388
+ Builder* enable_content_defined_chunking() {
389
+ content_defined_chunking_enabled_ = true;
390
+ return this;
391
+ }
392
+
393
+ /// \brief EXPERIMENTAL: Disable content-defined page chunking for all columns.
394
+ Builder* disable_content_defined_chunking() {
395
+ content_defined_chunking_enabled_ = false;
396
+ return this;
397
+ }
398
+
399
+ /// \brief EXPERIMENTAL: Specify content-defined chunking options, see CdcOptions.
400
+ Builder* content_defined_chunking_options(const CdcOptions& options) {
401
+ content_defined_chunking_options_ = options;
402
+ return this;
403
+ }
404
+
405
+ /// Specify the memory pool for the writer. Default default_memory_pool.
406
+ Builder* memory_pool(MemoryPool* pool) {
407
+ pool_ = pool;
408
+ return this;
409
+ }
410
+
411
+ /// Enable dictionary encoding in general for all columns. Default
412
+ /// enabled.
413
+ Builder* enable_dictionary() {
414
+ default_column_properties_.set_dictionary_enabled(true);
415
+ return this;
416
+ }
417
+
418
+ /// Disable dictionary encoding in general for all columns. Default
419
+ /// enabled.
420
+ Builder* disable_dictionary() {
421
+ default_column_properties_.set_dictionary_enabled(false);
422
+ return this;
423
+ }
424
+
425
+ /// Enable dictionary encoding for column specified by `path`. Default
426
+ /// enabled.
427
+ Builder* enable_dictionary(const std::string& path) {
428
+ dictionary_enabled_[path] = true;
429
+ return this;
430
+ }
431
+
432
+ /// Enable dictionary encoding for column specified by `path`. Default
433
+ /// enabled.
434
+ Builder* enable_dictionary(const std::shared_ptr<schema::ColumnPath>& path) {
435
+ return this->enable_dictionary(path->ToDotString());
436
+ }
437
+
438
+ /// Disable dictionary encoding for column specified by `path`. Default
439
+ /// enabled.
440
+ Builder* disable_dictionary(const std::string& path) {
441
+ dictionary_enabled_[path] = false;
442
+ return this;
443
+ }
444
+
445
+ /// Disable dictionary encoding for column specified by `path`. Default
446
+ /// enabled.
447
+ Builder* disable_dictionary(const std::shared_ptr<schema::ColumnPath>& path) {
448
+ return this->disable_dictionary(path->ToDotString());
449
+ }
450
+
451
+ /// Specify the dictionary page size limit per row group. Default 1MB.
452
+ Builder* dictionary_pagesize_limit(int64_t dictionary_psize_limit) {
453
+ dictionary_pagesize_limit_ = dictionary_psize_limit;
454
+ return this;
455
+ }
456
+
457
+ /// Specify the write batch size while writing batches of Arrow values
458
+ /// into Parquet. Default 1024.
459
+ Builder* write_batch_size(int64_t write_batch_size) {
460
+ write_batch_size_ = write_batch_size;
461
+ return this;
462
+ }
463
+
464
+ /// Specify the max number of rows to put in a single row group.
465
+ /// Default 1Mi rows.
466
+ Builder* max_row_group_length(int64_t max_row_group_length) {
467
+ max_row_group_length_ = max_row_group_length;
468
+ return this;
469
+ }
470
+
471
+ /// Specify the data page size.
472
+ /// Default 1MB.
473
+ Builder* data_pagesize(int64_t pg_size) {
474
+ pagesize_ = pg_size;
475
+ return this;
476
+ }
477
+
478
+ /// Specify the maximum number of rows per data page.
479
+ /// Default 20K rows.
480
+ Builder* max_rows_per_page(int64_t max_rows) {
481
+ max_rows_per_page_ = max_rows;
482
+ return this;
483
+ }
484
+
485
+ /// Specify the data page version.
486
+ /// Default V1.
487
+ Builder* data_page_version(ParquetDataPageVersion data_page_version) {
488
+ data_page_version_ = data_page_version;
489
+ return this;
490
+ }
491
+
492
+ /// Specify the Parquet file version.
493
+ /// Default PARQUET_2_6.
494
+ Builder* version(ParquetVersion::type version) {
495
+ version_ = version;
496
+ return this;
497
+ }
498
+
499
+ Builder* created_by(const std::string& created_by) {
500
+ created_by_ = created_by;
501
+ return this;
502
+ }
503
+
504
+ Builder* enable_page_checksum() {
505
+ page_checksum_enabled_ = true;
506
+ return this;
507
+ }
508
+
509
+ Builder* disable_page_checksum() {
510
+ page_checksum_enabled_ = false;
511
+ return this;
512
+ }
513
+
514
+ /// \brief Define the encoding that is used when we don't utilise dictionary encoding.
515
+ //
516
+ /// This is only applied if dictionary encoding is disabled. If the dictionary grows
517
+ /// too large we always fall back to the PLAIN encoding.
518
+ Builder* encoding(Encoding::type encoding_type) {
519
+ if (encoding_type == Encoding::PLAIN_DICTIONARY ||
520
+ encoding_type == Encoding::RLE_DICTIONARY) {
521
+ throw ParquetException("Can't use dictionary encoding as fallback encoding");
522
+ }
523
+
524
+ default_column_properties_.set_encoding(encoding_type);
525
+ return this;
526
+ }
527
+
528
+ /// \brief Define the encoding that is used when we don't utilise dictionary encoding.
529
+ //
530
+ /// This is only applied if dictionary encoding is disabled. If the dictionary grows
531
+ /// too large we always fall back to the PLAIN encoding.
532
+ Builder* encoding(const std::string& path, Encoding::type encoding_type) {
533
+ if (encoding_type == Encoding::PLAIN_DICTIONARY ||
534
+ encoding_type == Encoding::RLE_DICTIONARY) {
535
+ throw ParquetException("Can't use dictionary encoding as fallback encoding");
536
+ }
537
+
538
+ encodings_[path] = encoding_type;
539
+ return this;
540
+ }
541
+
542
+ /// \brief Define the encoding that is used when we don't utilise dictionary encoding.
543
+ //
544
+ /// This is only applied if dictionary encoding is disabled. If the dictionary grows
545
+ /// too large we always fall back to the PLAIN encoding.
546
+ Builder* encoding(const std::shared_ptr<schema::ColumnPath>& path,
547
+ Encoding::type encoding_type) {
548
+ return this->encoding(path->ToDotString(), encoding_type);
549
+ }
550
+
551
+ /// Specify compression codec in general for all columns.
552
+ /// Default UNCOMPRESSED.
553
+ Builder* compression(Compression::type codec) {
554
+ default_column_properties_.set_compression(codec);
555
+ return this;
556
+ }
557
+
558
+ /// Specify max statistics size to store min max value.
559
+ /// Default 4KB.
560
+ Builder* max_statistics_size(size_t max_stats_sz) {
561
+ default_column_properties_.set_max_statistics_size(max_stats_sz);
562
+ return this;
563
+ }
564
+
565
+ /// Specify compression codec for the column specified by `path`.
566
+ /// Default UNCOMPRESSED.
567
+ Builder* compression(const std::string& path, Compression::type codec) {
568
+ codecs_[path] = codec;
569
+ return this;
570
+ }
571
+
572
+ /// Specify compression codec for the column specified by `path`.
573
+ /// Default UNCOMPRESSED.
574
+ Builder* compression(const std::shared_ptr<schema::ColumnPath>& path,
575
+ Compression::type codec) {
576
+ return this->compression(path->ToDotString(), codec);
577
+ }
578
+
579
+ /// \brief Specify the default compression level for the compressor in
580
+ /// every column. In case a column does not have an explicitly specified
581
+ /// compression level, the default one would be used.
582
+ ///
583
+ /// The provided compression level is compressor specific. The user would
584
+ /// have to familiarize oneself with the available levels for the selected
585
+ /// compressor. If the compressor does not allow for selecting different
586
+ /// compression levels, calling this function would not have any effect.
587
+ /// Parquet and Arrow do not validate the passed compression level. If no
588
+ /// level is selected by the user or if the special
589
+ /// std::numeric_limits<int>::min() value is passed, then Arrow selects the
590
+ /// compression level.
591
+ ///
592
+ /// If other compressor-specific options need to be set in addition to the compression
593
+ /// level, use the codec_options method.
594
+ Builder* compression_level(int compression_level) {
595
+ default_column_properties_.set_compression_level(compression_level);
596
+ return this;
597
+ }
598
+
599
+ /// \brief Specify a compression level for the compressor for the column
600
+ /// described by path.
601
+ ///
602
+ /// The provided compression level is compressor specific. The user would
603
+ /// have to familiarize oneself with the available levels for the selected
604
+ /// compressor. If the compressor does not allow for selecting different
605
+ /// compression levels, calling this function would not have any effect.
606
+ /// Parquet and Arrow do not validate the passed compression level. If no
607
+ /// level is selected by the user or if the special
608
+ /// std::numeric_limits<int>::min() value is passed, then Arrow selects the
609
+ /// compression level.
610
+ Builder* compression_level(const std::string& path, int compression_level) {
611
+ if (!codec_options_[path]) {
612
+ codec_options_[path] = std::make_shared<CodecOptions>();
613
+ }
614
+ codec_options_[path]->compression_level = compression_level;
615
+ return this;
616
+ }
617
+
618
+ /// \brief Specify a compression level for the compressor for the column
619
+ /// described by path.
620
+ ///
621
+ /// The provided compression level is compressor specific. The user would
622
+ /// have to familiarize oneself with the available levels for the selected
623
+ /// compressor. If the compressor does not allow for selecting different
624
+ /// compression levels, calling this function would not have any effect.
625
+ /// Parquet and Arrow do not validate the passed compression level. If no
626
+ /// level is selected by the user or if the special
627
+ /// std::numeric_limits<int>::min() value is passed, then Arrow selects the
628
+ /// compression level.
629
+ Builder* compression_level(const std::shared_ptr<schema::ColumnPath>& path,
630
+ int compression_level) {
631
+ return this->compression_level(path->ToDotString(), compression_level);
632
+ }
633
+
634
+ /// \brief Specify the default codec options for the compressor in
635
+ /// every column.
636
+ ///
637
+ /// The codec options allow configuring the compression level as well
638
+ /// as other codec-specific options.
639
+ Builder* codec_options(
640
+ const std::shared_ptr<::arrow::util::CodecOptions>& codec_options) {
641
+ default_column_properties_.set_codec_options(codec_options);
642
+ return this;
643
+ }
644
+
645
+ /// \brief Specify the codec options for the compressor for the column
646
+ /// described by path.
647
+ Builder* codec_options(
648
+ const std::string& path,
649
+ const std::shared_ptr<::arrow::util::CodecOptions>& codec_options) {
650
+ codec_options_[path] = codec_options;
651
+ return this;
652
+ }
653
+
654
+ /// \brief Specify the codec options for the compressor for the column
655
+ /// described by path.
656
+ Builder* codec_options(
657
+ const std::shared_ptr<schema::ColumnPath>& path,
658
+ const std::shared_ptr<::arrow::util::CodecOptions>& codec_options) {
659
+ return this->codec_options(path->ToDotString(), codec_options);
660
+ }
661
+
662
+ /// Define the file encryption properties.
663
+ /// Default NULL.
664
+ Builder* encryption(
665
+ std::shared_ptr<FileEncryptionProperties> file_encryption_properties) {
666
+ file_encryption_properties_ = std::move(file_encryption_properties);
667
+ return this;
668
+ }
669
+
670
+ /// Enable statistics in general.
671
+ /// Default enabled.
672
+ Builder* enable_statistics() {
673
+ default_column_properties_.set_statistics_enabled(true);
674
+ return this;
675
+ }
676
+
677
+ /// Disable statistics in general.
678
+ /// Default enabled.
679
+ Builder* disable_statistics() {
680
+ default_column_properties_.set_statistics_enabled(false);
681
+ return this;
682
+ }
683
+
684
+ /// Enable statistics for the column specified by `path`.
685
+ /// Default enabled.
686
+ Builder* enable_statistics(const std::string& path) {
687
+ statistics_enabled_[path] = true;
688
+ return this;
689
+ }
690
+
691
+ /// Enable statistics for the column specified by `path`.
692
+ /// Default enabled.
693
+ Builder* enable_statistics(const std::shared_ptr<schema::ColumnPath>& path) {
694
+ return this->enable_statistics(path->ToDotString());
695
+ }
696
+
697
+ /// Define the sorting columns.
698
+ /// Default empty.
699
+ ///
700
+ /// If sorting columns are set, user should ensure that records
701
+ /// are sorted by sorting columns. Otherwise, the storing data
702
+ /// will be inconsistent with sorting_columns metadata.
703
+ Builder* set_sorting_columns(std::vector<SortingColumn> sorting_columns) {
704
+ sorting_columns_ = std::move(sorting_columns);
705
+ return this;
706
+ }
707
+
708
+ /// Disable statistics for the column specified by `path`.
709
+ /// Default enabled.
710
+ Builder* disable_statistics(const std::string& path) {
711
+ statistics_enabled_[path] = false;
712
+ return this;
713
+ }
714
+
715
+ /// Disable statistics for the column specified by `path`.
716
+ /// Default enabled.
717
+ Builder* disable_statistics(const std::shared_ptr<schema::ColumnPath>& path) {
718
+ return this->disable_statistics(path->ToDotString());
719
+ }
720
+
721
+ /// Disable bloom filter for the column specified by `path`.
722
+ /// Default disabled.
723
+ Builder* disable_bloom_filter(const std::string& path) {
724
+ bloom_filter_options_.erase(path);
725
+ return this;
726
+ }
727
+
728
+ /// Disable bloom filter for the column specified by `path`.
729
+ /// Default disabled.
730
+ Builder* disable_bloom_filter(const std::shared_ptr<schema::ColumnPath>& path) {
731
+ return this->disable_bloom_filter(path->ToDotString());
732
+ }
733
+
734
+ /// Enable bloom filter for the column specified by `path`.
735
+ ///
736
+ /// Default disabled.
737
+ ///
738
+ /// \note Bloom filter is not supported for boolean columns. ParquetException will
739
+ /// be thrown during write if the column is of boolean type.
740
+ Builder* enable_bloom_filter(const std::string& path,
741
+ const BloomFilterOptions& bloom_filter_options) {
742
+ bloom_filter_options_[path] = bloom_filter_options;
743
+ return this;
744
+ }
745
+
746
+ /// Enable bloom filter for the column specified by `path`.
747
+ ///
748
+ /// Default disabled.
749
+ ///
750
+ /// \note Bloom filter is not supported for boolean columns. ParquetException will
751
+ /// be thrown during write if the column is of boolean type.
752
+ Builder* enable_bloom_filter(const std::shared_ptr<schema::ColumnPath>& path,
753
+ const BloomFilterOptions& bloom_filter_options) {
754
+ return this->enable_bloom_filter(path->ToDotString(), bloom_filter_options);
755
+ }
756
+
757
+ /// Allow decimals with 1 <= precision <= 18 to be stored as integers.
758
+ ///
759
+ /// In Parquet, DECIMAL can be stored in any of the following physical types:
760
+ /// - int32: for 1 <= precision <= 9.
761
+ /// - int64: for 10 <= precision <= 18.
762
+ /// - fixed_len_byte_array: precision is limited by the array size.
763
+ /// Length n can store <= floor(log_10(2^(8*n - 1) - 1)) base-10 digits.
764
+ /// - binary: precision is unlimited. The minimum number of bytes to store
765
+ /// the unscaled value is used.
766
+ ///
767
+ /// By default, this is DISABLED and all decimal types annotate fixed_len_byte_array.
768
+ ///
769
+ /// When enabled, the C++ writer will use following physical types to store decimals:
770
+ /// - int32: for 1 <= precision <= 9.
771
+ /// - int64: for 10 <= precision <= 18.
772
+ /// - fixed_len_byte_array: for precision > 18.
773
+ ///
774
+ /// As a consequence, decimal columns stored in integer types are more compact.
775
+ Builder* enable_store_decimal_as_integer() {
776
+ store_decimal_as_integer_ = true;
777
+ return this;
778
+ }
779
+
780
+ /// Disable decimal logical type with 1 <= precision <= 18 to be stored
781
+ /// as integer physical type.
782
+ ///
783
+ /// Default disabled.
784
+ Builder* disable_store_decimal_as_integer() {
785
+ store_decimal_as_integer_ = false;
786
+ return this;
787
+ }
788
+
789
+ /// Enable writing page index in general for all columns. Default enabled.
790
+ ///
791
+ /// Writing statistics to the page index disables the old method of writing
792
+ /// statistics to each data page header.
793
+ /// The page index makes filtering more efficient than the page header, as
794
+ /// it gathers all the statistics for a Parquet file in a single place,
795
+ /// avoiding scattered I/O.
796
+ ///
797
+ /// Please check the link below for more details:
798
+ /// https://github.com/apache/parquet-format/blob/master/PageIndex.md
799
+ Builder* enable_write_page_index() {
800
+ default_column_properties_.set_page_index_enabled(true);
801
+ return this;
802
+ }
803
+
804
+ /// Disable writing page index in general for all columns. Default enabled.
805
+ Builder* disable_write_page_index() {
806
+ default_column_properties_.set_page_index_enabled(false);
807
+ return this;
808
+ }
809
+
810
+ /// Enable writing page index for column specified by `path`. Default enabled.
811
+ Builder* enable_write_page_index(const std::string& path) {
812
+ page_index_enabled_[path] = true;
813
+ return this;
814
+ }
815
+
816
+ /// Enable writing page index for column specified by `path`. Default enabled.
817
+ Builder* enable_write_page_index(const std::shared_ptr<schema::ColumnPath>& path) {
818
+ return this->enable_write_page_index(path->ToDotString());
819
+ }
820
+
821
+ /// Disable writing page index for column specified by `path`. Default enabled.
822
+ Builder* disable_write_page_index(const std::string& path) {
823
+ page_index_enabled_[path] = false;
824
+ return this;
825
+ }
826
+
827
+ /// Disable writing page index for column specified by `path`. Default enabled.
828
+ Builder* disable_write_page_index(const std::shared_ptr<schema::ColumnPath>& path) {
829
+ return this->disable_write_page_index(path->ToDotString());
830
+ }
831
+
832
+ /// \brief Set the level to write size statistics for all columns. Default is
833
+ /// PageAndColumnChunk.
834
+ ///
835
+ /// \param level The level to write size statistics. Note that if page index is not
836
+ /// enabled, page level size statistics will not be written even if the level
837
+ /// is set to PageAndColumnChunk.
838
+ Builder* set_size_statistics_level(SizeStatisticsLevel level) {
839
+ size_statistics_level_ = level;
840
+ return this;
841
+ }
842
+
843
+ /// \brief Build the WriterProperties with the builder parameters.
844
+ /// \return The WriterProperties defined by the builder.
845
+ std::shared_ptr<WriterProperties> build() {
846
+ std::unordered_map<std::string, ColumnProperties> column_properties;
847
+ auto get = [&](const std::string& key) -> ColumnProperties& {
848
+ auto it = column_properties.find(key);
849
+ if (it == column_properties.end())
850
+ return column_properties[key] = default_column_properties_;
851
+ else
852
+ return it->second;
853
+ };
854
+
855
+ for (const auto& item : encodings_) get(item.first).set_encoding(item.second);
856
+ for (const auto& item : codecs_) get(item.first).set_compression(item.second);
857
+ for (const auto& item : codec_options_)
858
+ get(item.first).set_codec_options(item.second);
859
+ for (const auto& item : dictionary_enabled_)
860
+ get(item.first).set_dictionary_enabled(item.second);
861
+ for (const auto& item : statistics_enabled_)
862
+ get(item.first).set_statistics_enabled(item.second);
863
+ for (const auto& item : page_index_enabled_)
864
+ get(item.first).set_page_index_enabled(item.second);
865
+ for (const auto& item : bloom_filter_options_)
866
+ get(item.first).set_bloom_filter_options(item.second);
867
+
868
+ return std::shared_ptr<WriterProperties>(new WriterProperties(
869
+ pool_, dictionary_pagesize_limit_, write_batch_size_, max_row_group_length_,
870
+ pagesize_, max_rows_per_page_, version_, created_by_, page_checksum_enabled_,
871
+ size_statistics_level_, std::move(file_encryption_properties_),
872
+ default_column_properties_, column_properties, data_page_version_,
873
+ store_decimal_as_integer_, std::move(sorting_columns_),
874
+ content_defined_chunking_enabled_, content_defined_chunking_options_));
875
+ }
876
+
877
+ private:
878
+ void CopyColumnSpecificProperties(const WriterProperties& properties);
879
+
880
+ MemoryPool* pool_;
881
+ int64_t dictionary_pagesize_limit_;
882
+ int64_t write_batch_size_;
883
+ int64_t max_row_group_length_;
884
+ int64_t pagesize_;
885
+ int64_t max_rows_per_page_;
886
+ ParquetVersion::type version_;
887
+ ParquetDataPageVersion data_page_version_;
888
+ std::string created_by_;
889
+ bool store_decimal_as_integer_;
890
+ bool page_checksum_enabled_;
891
+ SizeStatisticsLevel size_statistics_level_;
892
+
893
+ std::shared_ptr<FileEncryptionProperties> file_encryption_properties_;
894
+
895
+ // If empty, there is no sorting columns.
896
+ std::vector<SortingColumn> sorting_columns_;
897
+
898
+ // Settings used for each column unless overridden in any of the maps below
899
+ ColumnProperties default_column_properties_;
900
+ std::unordered_map<std::string, Encoding::type> encodings_;
901
+ std::unordered_map<std::string, Compression::type> codecs_;
902
+ std::unordered_map<std::string, std::shared_ptr<CodecOptions>> codec_options_;
903
+ std::unordered_map<std::string, bool> dictionary_enabled_;
904
+ std::unordered_map<std::string, bool> statistics_enabled_;
905
+ std::unordered_map<std::string, bool> page_index_enabled_;
906
+ std::unordered_map<std::string, BloomFilterOptions> bloom_filter_options_;
907
+
908
+ bool content_defined_chunking_enabled_;
909
+ CdcOptions content_defined_chunking_options_;
910
+ };
911
+
912
+ inline MemoryPool* memory_pool() const { return pool_; }
913
+
914
+ inline int64_t dictionary_pagesize_limit() const { return dictionary_pagesize_limit_; }
915
+
916
+ inline int64_t write_batch_size() const { return write_batch_size_; }
917
+
918
+ inline int64_t max_row_group_length() const { return max_row_group_length_; }
919
+
920
+ inline int64_t data_pagesize() const { return pagesize_; }
921
+
922
+ inline int64_t max_rows_per_page() const { return max_rows_per_page_; }
923
+
924
+ inline ParquetDataPageVersion data_page_version() const {
925
+ return parquet_data_page_version_;
926
+ }
927
+
928
+ inline ParquetVersion::type version() const { return parquet_version_; }
929
+
930
+ inline std::string created_by() const { return parquet_created_by_; }
931
+
932
+ inline bool store_decimal_as_integer() const { return store_decimal_as_integer_; }
933
+
934
+ inline bool page_checksum_enabled() const { return page_checksum_enabled_; }
935
+
936
+ inline bool content_defined_chunking_enabled() const {
937
+ return content_defined_chunking_enabled_;
938
+ }
939
+ inline CdcOptions content_defined_chunking_options() const {
940
+ return content_defined_chunking_options_;
941
+ }
942
+
943
+ inline SizeStatisticsLevel size_statistics_level() const {
944
+ return size_statistics_level_;
945
+ }
946
+
947
+ inline Encoding::type dictionary_index_encoding() const {
948
+ if (parquet_version_ == ParquetVersion::PARQUET_1_0) {
949
+ return Encoding::PLAIN_DICTIONARY;
950
+ } else {
951
+ return Encoding::RLE_DICTIONARY;
952
+ }
953
+ }
954
+
955
+ inline Encoding::type dictionary_page_encoding() const {
956
+ if (parquet_version_ == ParquetVersion::PARQUET_1_0) {
957
+ return Encoding::PLAIN_DICTIONARY;
958
+ } else {
959
+ return Encoding::PLAIN;
960
+ }
961
+ }
962
+
963
+ const ColumnProperties& column_properties(
964
+ const std::shared_ptr<schema::ColumnPath>& path) const {
965
+ auto it = column_properties_.find(path->ToDotString());
966
+ if (it != column_properties_.end()) return it->second;
967
+ return default_column_properties_;
968
+ }
969
+
970
+ Encoding::type encoding(const std::shared_ptr<schema::ColumnPath>& path) const {
971
+ return column_properties(path).encoding();
972
+ }
973
+
974
+ Compression::type compression(const std::shared_ptr<schema::ColumnPath>& path) const {
975
+ return column_properties(path).compression();
976
+ }
977
+
978
+ int compression_level(const std::shared_ptr<schema::ColumnPath>& path) const {
979
+ return column_properties(path).compression_level();
980
+ }
981
+
982
+ const std::shared_ptr<CodecOptions> codec_options(
983
+ const std::shared_ptr<schema::ColumnPath>& path) const {
984
+ return column_properties(path).codec_options();
985
+ }
986
+
987
+ bool dictionary_enabled(const std::shared_ptr<schema::ColumnPath>& path) const {
988
+ return column_properties(path).dictionary_enabled();
989
+ }
990
+
991
+ const std::vector<SortingColumn>& sorting_columns() const { return sorting_columns_; }
992
+
993
+ bool statistics_enabled(const std::shared_ptr<schema::ColumnPath>& path) const {
994
+ return column_properties(path).statistics_enabled();
995
+ }
996
+
997
+ size_t max_statistics_size(const std::shared_ptr<schema::ColumnPath>& path) const {
998
+ return column_properties(path).max_statistics_size();
999
+ }
1000
+
1001
+ bool page_index_enabled(const std::shared_ptr<schema::ColumnPath>& path) const {
1002
+ return column_properties(path).page_index_enabled();
1003
+ }
1004
+
1005
+ bool page_index_enabled() const {
1006
+ if (default_column_properties_.page_index_enabled()) {
1007
+ return true;
1008
+ }
1009
+ for (const auto& item : column_properties_) {
1010
+ if (item.second.page_index_enabled()) {
1011
+ return true;
1012
+ }
1013
+ }
1014
+ return false;
1015
+ }
1016
+
1017
+ // Return whether bloom filter is enabled for any column.
1018
+ bool bloom_filter_enabled() const {
1019
+ return std::any_of(column_properties_.cbegin(), column_properties_.cend(),
1020
+ [](const auto& p) { return p.second.bloom_filter_enabled(); });
1021
+ }
1022
+
1023
+ std::optional<BloomFilterOptions> bloom_filter_options(
1024
+ const std::shared_ptr<schema::ColumnPath>& path) const {
1025
+ return column_properties(path).bloom_filter_options();
1026
+ }
1027
+
1028
+ inline FileEncryptionProperties* file_encryption_properties() const {
1029
+ return file_encryption_properties_.get();
1030
+ }
1031
+
1032
+ std::shared_ptr<ColumnEncryptionProperties> column_encryption_properties(
1033
+ const std::string& path) const {
1034
+ if (file_encryption_properties_) {
1035
+ return file_encryption_properties_->column_encryption_properties(path);
1036
+ } else {
1037
+ return NULLPTR;
1038
+ }
1039
+ }
1040
+
1041
+ // \brief Return the default column properties
1042
+ const ColumnProperties& default_column_properties() const {
1043
+ return default_column_properties_;
1044
+ }
1045
+
1046
+ private:
1047
+ explicit WriterProperties(
1048
+ MemoryPool* pool, int64_t dictionary_pagesize_limit, int64_t write_batch_size,
1049
+ int64_t max_row_group_length, int64_t pagesize, int64_t max_rows_per_page,
1050
+ ParquetVersion::type version, const std::string& created_by,
1051
+ bool page_write_checksum_enabled, SizeStatisticsLevel size_statistics_level,
1052
+ std::shared_ptr<FileEncryptionProperties> file_encryption_properties,
1053
+ const ColumnProperties& default_column_properties,
1054
+ const std::unordered_map<std::string, ColumnProperties>& column_properties,
1055
+ ParquetDataPageVersion data_page_version, bool store_short_decimal_as_integer,
1056
+ std::vector<SortingColumn> sorting_columns, bool content_defined_chunking_enabled,
1057
+ CdcOptions content_defined_chunking_options)
1058
+ : pool_(pool),
1059
+ dictionary_pagesize_limit_(dictionary_pagesize_limit),
1060
+ write_batch_size_(write_batch_size),
1061
+ max_row_group_length_(max_row_group_length),
1062
+ pagesize_(pagesize),
1063
+ max_rows_per_page_(max_rows_per_page),
1064
+ parquet_data_page_version_(data_page_version),
1065
+ parquet_version_(version),
1066
+ parquet_created_by_(created_by),
1067
+ store_decimal_as_integer_(store_short_decimal_as_integer),
1068
+ page_checksum_enabled_(page_write_checksum_enabled),
1069
+ size_statistics_level_(size_statistics_level),
1070
+ file_encryption_properties_(file_encryption_properties),
1071
+ sorting_columns_(std::move(sorting_columns)),
1072
+ default_column_properties_(default_column_properties),
1073
+ column_properties_(column_properties),
1074
+ content_defined_chunking_enabled_(content_defined_chunking_enabled),
1075
+ content_defined_chunking_options_(content_defined_chunking_options) {}
1076
+
1077
+ MemoryPool* pool_;
1078
+ int64_t dictionary_pagesize_limit_;
1079
+ int64_t write_batch_size_;
1080
+ int64_t max_row_group_length_;
1081
+ int64_t pagesize_;
1082
+ int64_t max_rows_per_page_;
1083
+ ParquetDataPageVersion parquet_data_page_version_;
1084
+ ParquetVersion::type parquet_version_;
1085
+ std::string parquet_created_by_;
1086
+ bool store_decimal_as_integer_;
1087
+ bool page_checksum_enabled_;
1088
+ SizeStatisticsLevel size_statistics_level_;
1089
+
1090
+ std::shared_ptr<FileEncryptionProperties> file_encryption_properties_;
1091
+
1092
+ std::vector<SortingColumn> sorting_columns_;
1093
+
1094
+ ColumnProperties default_column_properties_;
1095
+ std::unordered_map<std::string, ColumnProperties> column_properties_;
1096
+
1097
+ bool content_defined_chunking_enabled_;
1098
+ CdcOptions content_defined_chunking_options_;
1099
+ };
1100
+
1101
+ PARQUET_EXPORT const std::shared_ptr<WriterProperties>& default_writer_properties();
1102
+
1103
+ // ----------------------------------------------------------------------
1104
+ // Properties specific to Apache Arrow columnar read and write
1105
+
1106
+ static constexpr bool kArrowDefaultUseThreads = false;
1107
+
1108
+ // Default number of rows to read when using ::arrow::RecordBatchReader
1109
+ static constexpr int64_t kArrowDefaultBatchSize = 64 * 1024;
1110
+
1111
+ constexpr inline ::arrow::Type::type kArrowDefaultBinaryType = ::arrow::Type::BINARY;
1112
+ constexpr inline ::arrow::Type::type kArrowDefaultListType = ::arrow::Type::LIST;
1113
+
1114
+ /// EXPERIMENTAL: Properties for configuring FileReader behavior.
1115
+ class PARQUET_EXPORT ArrowReaderProperties {
1116
+ public:
1117
+ explicit ArrowReaderProperties(bool use_threads = kArrowDefaultUseThreads)
1118
+ : use_threads_(use_threads),
1119
+ read_dict_indices_(),
1120
+ batch_size_(kArrowDefaultBatchSize),
1121
+ pre_buffer_(true),
1122
+ cache_options_(::arrow::io::CacheOptions::LazyDefaults()),
1123
+ coerce_int96_timestamp_unit_(::arrow::TimeUnit::NANO),
1124
+ binary_type_(kArrowDefaultBinaryType),
1125
+ list_type_(kArrowDefaultListType),
1126
+ arrow_extensions_enabled_(false),
1127
+ should_load_statistics_(false),
1128
+ smallest_decimal_enabled_(false) {}
1129
+
1130
+ /// \brief Set whether to use the IO thread pool to parse columns in parallel.
1131
+ ///
1132
+ /// Default is false.
1133
+ void set_use_threads(bool use_threads) { use_threads_ = use_threads; }
1134
+ /// Return whether will use multiple threads.
1135
+ bool use_threads() const { return use_threads_; }
1136
+
1137
+ /// \brief Set whether to read a particular column as dictionary encoded.
1138
+ ///
1139
+ /// If the file metadata contains a serialized Arrow schema, then ...
1140
+ ////
1141
+ /// This is only supported for columns with a Parquet physical type of
1142
+ /// BYTE_ARRAY, such as string or binary types.
1143
+ void set_read_dictionary(int column_index, bool read_dict) {
1144
+ if (read_dict) {
1145
+ read_dict_indices_.insert(column_index);
1146
+ } else {
1147
+ read_dict_indices_.erase(column_index);
1148
+ }
1149
+ }
1150
+ /// Return whether the column at the index will be read as dictionary.
1151
+ bool read_dictionary(int column_index) const {
1152
+ if (read_dict_indices_.find(column_index) != read_dict_indices_.end()) {
1153
+ return true;
1154
+ } else {
1155
+ return false;
1156
+ }
1157
+ }
1158
+
1159
+ /// \brief Set the Arrow binary type to read BYTE_ARRAY columns as.
1160
+ ///
1161
+ /// Allowed values are Type::BINARY, Type::LARGE_BINARY and Type::BINARY_VIEW.
1162
+ /// Default is Type::BINARY.
1163
+ ///
1164
+ /// If a BYTE_ARRAY column has the STRING logical type, it is read as the
1165
+ /// Arrow string type corresponding to the configured binary type (for example
1166
+ /// Type::LARGE_STRING if the configured binary type is Type::LARGE_BINARY).
1167
+ ///
1168
+ /// However, if a serialized Arrow schema is found in the Parquet metadata,
1169
+ /// this setting is ignored and the Arrow schema takes precedence
1170
+ /// (see ArrowWriterProperties::store_schema).
1171
+ void set_binary_type(::arrow::Type::type value) { binary_type_ = value; }
1172
+ /// Return the Arrow binary type to read BYTE_ARRAY columns as.
1173
+ ::arrow::Type::type binary_type() const { return binary_type_; }
1174
+
1175
+ /// \brief Set the Arrow list type to read Parquet list columns as.
1176
+ ///
1177
+ /// Allowed values are Type::LIST and Type::LARGE_LIST.
1178
+ /// Default is Type::LIST.
1179
+ ///
1180
+ /// However, if a serialized Arrow schema is found in the Parquet metadata,
1181
+ /// this setting is ignored and the Arrow schema takes precedence
1182
+ /// (see ArrowWriterProperties::store_schema).
1183
+ void set_list_type(::arrow::Type::type value) { list_type_ = value; }
1184
+ /// Return the Arrow list type to read Parquet list columns as.
1185
+ ::arrow::Type::type list_type() const { return list_type_; }
1186
+
1187
+ /// \brief Set the maximum number of rows to read into a record batch.
1188
+ ///
1189
+ /// Will only be fewer rows when there are no more rows in the file.
1190
+ /// Note that some APIs such as ReadTable may ignore this setting.
1191
+ void set_batch_size(int64_t batch_size) { batch_size_ = batch_size; }
1192
+ /// Return the batch size in rows.
1193
+ ///
1194
+ /// Note that some APIs such as ReadTable may ignore this setting.
1195
+ int64_t batch_size() const { return batch_size_; }
1196
+
1197
+ /// Enable read coalescing (default false).
1198
+ ///
1199
+ /// When enabled, the Arrow reader will pre-buffer necessary regions
1200
+ /// of the file in-memory. This is intended to improve performance on
1201
+ /// high-latency filesystems (e.g. Amazon S3).
1202
+ void set_pre_buffer(bool pre_buffer) { pre_buffer_ = pre_buffer; }
1203
+ /// Return whether read coalescing is enabled.
1204
+ bool pre_buffer() const { return pre_buffer_; }
1205
+
1206
+ /// Set options for read coalescing. This can be used to tune the
1207
+ /// implementation for characteristics of different filesystems.
1208
+ void set_cache_options(::arrow::io::CacheOptions options) { cache_options_ = options; }
1209
+ /// Return the options for read coalescing.
1210
+ const ::arrow::io::CacheOptions& cache_options() const { return cache_options_; }
1211
+
1212
+ /// Set execution context for read coalescing.
1213
+ void set_io_context(const ::arrow::io::IOContext& ctx) { io_context_ = ctx; }
1214
+ /// Return the execution context used for read coalescing.
1215
+ const ::arrow::io::IOContext& io_context() const { return io_context_; }
1216
+
1217
+ /// Set timestamp unit to use for deprecated INT96-encoded timestamps
1218
+ /// (default is NANO).
1219
+ void set_coerce_int96_timestamp_unit(::arrow::TimeUnit::type unit) {
1220
+ coerce_int96_timestamp_unit_ = unit;
1221
+ }
1222
+
1223
+ ::arrow::TimeUnit::type coerce_int96_timestamp_unit() const {
1224
+ return coerce_int96_timestamp_unit_;
1225
+ }
1226
+
1227
+ /// Enable Parquet-supported Arrow extension types.
1228
+ ///
1229
+ /// When enabled, Parquet logical types will be mapped to their corresponding Arrow
1230
+ /// extension types at read time, if such exist. Currently only arrow::extension::json()
1231
+ /// extension type is supported. Columns whose LogicalType is JSON will be interpreted
1232
+ /// as arrow::extension::json(), with storage type inferred from the serialized Arrow
1233
+ /// schema if present, or `utf8` by default.
1234
+ void set_arrow_extensions_enabled(bool extensions_enabled) {
1235
+ arrow_extensions_enabled_ = extensions_enabled;
1236
+ }
1237
+ bool get_arrow_extensions_enabled() const { return arrow_extensions_enabled_; }
1238
+
1239
+ /// \brief Set whether to load statistics as much as possible.
1240
+ ///
1241
+ /// Default is false.
1242
+ void set_should_load_statistics(bool should_load_statistics) {
1243
+ should_load_statistics_ = should_load_statistics;
1244
+ }
1245
+ /// Return whether loading statistics as much as possible.
1246
+ bool should_load_statistics() const { return should_load_statistics_; }
1247
+
1248
+ /// \brief Set whether to infer Decimal32/64 from Parquet decimal logical types.
1249
+ ///
1250
+ /// Default is false for compatibility, meaning that only Decimal128 and Decimal256
1251
+ /// can be inferred.
1252
+ void set_smallest_decimal_enabled(bool smallest_decimal_enable) {
1253
+ smallest_decimal_enabled_ = smallest_decimal_enable;
1254
+ }
1255
+ /// \brief Whether to infer Decimal32/64 from Parquet decimal logical types.
1256
+ ///
1257
+ /// When enabled, Parquet decimal columns will be inferred as the smallest possible
1258
+ /// Arrow Decimal type.
1259
+ /// When disabled, Parquet decimal columns will be inferred as either Decimal128 or
1260
+ /// Decimal256, but not Decimal32/64.
1261
+ ///
1262
+ /// Note: if an Arrow schema is found in the Parquet metadata, it will take priority and
1263
+ /// this setting will be ignored.
1264
+ bool smallest_decimal_enabled() const { return smallest_decimal_enabled_; }
1265
+
1266
+ private:
1267
+ bool use_threads_;
1268
+ std::unordered_set<int> read_dict_indices_;
1269
+ int64_t batch_size_;
1270
+ bool pre_buffer_;
1271
+ ::arrow::io::IOContext io_context_;
1272
+ ::arrow::io::CacheOptions cache_options_;
1273
+ ::arrow::TimeUnit::type coerce_int96_timestamp_unit_;
1274
+ ::arrow::Type::type binary_type_;
1275
+ ::arrow::Type::type list_type_;
1276
+ bool arrow_extensions_enabled_;
1277
+ bool should_load_statistics_;
1278
+ bool smallest_decimal_enabled_;
1279
+ };
1280
+
1281
+ /// EXPERIMENTAL: Constructs the default ArrowReaderProperties
1282
+ PARQUET_EXPORT
1283
+ ArrowReaderProperties default_arrow_reader_properties();
1284
+
1285
+ class PARQUET_EXPORT ArrowWriterProperties {
1286
+ public:
1287
+ enum EngineVersion {
1288
+ V1, // Supports only nested lists.
1289
+ V2 // Full support for all nesting combinations
1290
+ };
1291
+ class Builder {
1292
+ public:
1293
+ Builder()
1294
+ : write_timestamps_as_int96_(false),
1295
+ coerce_timestamps_enabled_(false),
1296
+ coerce_timestamps_unit_(::arrow::TimeUnit::SECOND),
1297
+ truncated_timestamps_allowed_(false),
1298
+ store_schema_(false),
1299
+ compliant_nested_types_(true),
1300
+ engine_version_(V2),
1301
+ use_threads_(kArrowDefaultUseThreads),
1302
+ executor_(NULLPTR),
1303
+ write_time_adjusted_to_utc_(false) {}
1304
+
1305
+ /// \brief Disable writing legacy int96 timestamps (default disabled).
1306
+ Builder* disable_deprecated_int96_timestamps() {
1307
+ write_timestamps_as_int96_ = false;
1308
+ return this;
1309
+ }
1310
+
1311
+ /// \brief Enable writing legacy int96 timestamps (default disabled).
1312
+ ///
1313
+ /// May be turned on to write timestamps compatible with older Parquet writers.
1314
+ /// This takes precedent over coerce_timestamps.
1315
+ Builder* enable_deprecated_int96_timestamps() {
1316
+ write_timestamps_as_int96_ = true;
1317
+ return this;
1318
+ }
1319
+
1320
+ /// \brief Coerce all timestamps to the specified time unit.
1321
+ /// \param unit time unit to truncate to.
1322
+ /// For Parquet versions 1.0 and 2.4, nanoseconds are casted to microseconds.
1323
+ Builder* coerce_timestamps(::arrow::TimeUnit::type unit) {
1324
+ coerce_timestamps_enabled_ = true;
1325
+ coerce_timestamps_unit_ = unit;
1326
+ return this;
1327
+ }
1328
+
1329
+ /// \brief Allow loss of data when truncating timestamps.
1330
+ ///
1331
+ /// This is disallowed by default and an error will be returned.
1332
+ Builder* allow_truncated_timestamps() {
1333
+ truncated_timestamps_allowed_ = true;
1334
+ return this;
1335
+ }
1336
+
1337
+ /// \brief Disallow loss of data when truncating timestamps (default).
1338
+ Builder* disallow_truncated_timestamps() {
1339
+ truncated_timestamps_allowed_ = false;
1340
+ return this;
1341
+ }
1342
+
1343
+ /// \brief EXPERIMENTAL: Write binary serialized Arrow schema to the file,
1344
+ /// to enable certain read options (like "read_dictionary") to be set
1345
+ /// automatically
1346
+ Builder* store_schema() {
1347
+ store_schema_ = true;
1348
+ return this;
1349
+ }
1350
+
1351
+ /// \brief When enabled, will not preserve Arrow field names for list types.
1352
+ ///
1353
+ /// Instead of using the field names Arrow uses for the values array of
1354
+ /// list types (default "item"), will use "element", as is specified in
1355
+ /// the Parquet spec.
1356
+ ///
1357
+ /// This is enabled by default.
1358
+ Builder* enable_compliant_nested_types() {
1359
+ compliant_nested_types_ = true;
1360
+ return this;
1361
+ }
1362
+
1363
+ /// Preserve Arrow list field name.
1364
+ Builder* disable_compliant_nested_types() {
1365
+ compliant_nested_types_ = false;
1366
+ return this;
1367
+ }
1368
+
1369
+ /// Set the version of the Parquet writer engine.
1370
+ Builder* set_engine_version(EngineVersion version) {
1371
+ engine_version_ = version;
1372
+ return this;
1373
+ }
1374
+
1375
+ /// \brief Set whether to use multiple threads to write columns
1376
+ /// in parallel in the buffered row group mode.
1377
+ ///
1378
+ /// WARNING: If writing multiple files in parallel in the same
1379
+ /// executor, deadlock may occur if use_threads is true. Please
1380
+ /// disable it in this case.
1381
+ ///
1382
+ /// Default is false.
1383
+ Builder* set_use_threads(bool use_threads) {
1384
+ use_threads_ = use_threads;
1385
+ return this;
1386
+ }
1387
+
1388
+ /// \brief Set the executor to write columns in parallel in the
1389
+ /// buffered row group mode.
1390
+ ///
1391
+ /// Default is nullptr and the default cpu executor will be used.
1392
+ Builder* set_executor(::arrow::internal::Executor* executor) {
1393
+ executor_ = executor;
1394
+ return this;
1395
+ }
1396
+
1397
+ /// \brief Set the value of isAdjustedTOUTC when writing a TIME column
1398
+ ///
1399
+ /// Default is false because Arrow TIME data is expressed in an unspecified timezone.
1400
+ /// Note this setting doesn't affect TIMESTAMP data.
1401
+ Builder* set_time_adjusted_to_utc(bool adjusted) {
1402
+ write_time_adjusted_to_utc_ = adjusted;
1403
+ return this;
1404
+ }
1405
+
1406
+ /// Create the final properties.
1407
+ std::shared_ptr<ArrowWriterProperties> build() {
1408
+ return std::shared_ptr<ArrowWriterProperties>(new ArrowWriterProperties(
1409
+ write_timestamps_as_int96_, coerce_timestamps_enabled_, coerce_timestamps_unit_,
1410
+ truncated_timestamps_allowed_, store_schema_, compliant_nested_types_,
1411
+ engine_version_, use_threads_, executor_, write_time_adjusted_to_utc_));
1412
+ }
1413
+
1414
+ private:
1415
+ bool write_timestamps_as_int96_;
1416
+
1417
+ bool coerce_timestamps_enabled_;
1418
+ ::arrow::TimeUnit::type coerce_timestamps_unit_;
1419
+ bool truncated_timestamps_allowed_;
1420
+
1421
+ bool store_schema_;
1422
+ bool compliant_nested_types_;
1423
+ EngineVersion engine_version_;
1424
+
1425
+ bool use_threads_;
1426
+ ::arrow::internal::Executor* executor_;
1427
+
1428
+ bool write_time_adjusted_to_utc_;
1429
+ };
1430
+
1431
+ bool support_deprecated_int96_timestamps() const { return write_timestamps_as_int96_; }
1432
+
1433
+ bool coerce_timestamps_enabled() const { return coerce_timestamps_enabled_; }
1434
+ ::arrow::TimeUnit::type coerce_timestamps_unit() const {
1435
+ return coerce_timestamps_unit_;
1436
+ }
1437
+
1438
+ bool truncated_timestamps_allowed() const { return truncated_timestamps_allowed_; }
1439
+
1440
+ bool store_schema() const { return store_schema_; }
1441
+
1442
+ /// \brief Enable nested type naming according to the parquet specification.
1443
+ ///
1444
+ /// Older versions of arrow wrote out field names for nested lists based on the name
1445
+ /// of the field. According to the parquet specification they should always be
1446
+ /// "element".
1447
+ bool compliant_nested_types() const { return compliant_nested_types_; }
1448
+
1449
+ /// \brief The underlying engine version to use when writing Arrow data.
1450
+ ///
1451
+ /// V2 is currently the latest V1 is considered deprecated but left in
1452
+ /// place in case there are bugs detected in V2.
1453
+ EngineVersion engine_version() const { return engine_version_; }
1454
+
1455
+ /// \brief Returns whether the writer will use multiple threads
1456
+ /// to write columns in parallel in the buffered row group mode.
1457
+ bool use_threads() const { return use_threads_; }
1458
+
1459
+ /// \brief Returns the executor used to write columns in parallel.
1460
+ ::arrow::internal::Executor* executor() const;
1461
+
1462
+ /// \brief The value of isAdjustedTOUTC when writing a TIME column
1463
+ ///
1464
+ /// Note this setting doesn't affect TIMESTAMP data.
1465
+ bool write_time_adjusted_to_utc() const { return write_time_adjusted_to_utc_; }
1466
+
1467
+ private:
1468
+ explicit ArrowWriterProperties(bool write_nanos_as_int96,
1469
+ bool coerce_timestamps_enabled,
1470
+ ::arrow::TimeUnit::type coerce_timestamps_unit,
1471
+ bool truncated_timestamps_allowed, bool store_schema,
1472
+ bool compliant_nested_types,
1473
+ EngineVersion engine_version, bool use_threads,
1474
+ ::arrow::internal::Executor* executor,
1475
+ bool write_time_adjusted_to_utc)
1476
+ : write_timestamps_as_int96_(write_nanos_as_int96),
1477
+ coerce_timestamps_enabled_(coerce_timestamps_enabled),
1478
+ coerce_timestamps_unit_(coerce_timestamps_unit),
1479
+ truncated_timestamps_allowed_(truncated_timestamps_allowed),
1480
+ store_schema_(store_schema),
1481
+ compliant_nested_types_(compliant_nested_types),
1482
+ engine_version_(engine_version),
1483
+ use_threads_(use_threads),
1484
+ executor_(executor),
1485
+ write_time_adjusted_to_utc_(write_time_adjusted_to_utc) {}
1486
+
1487
+ const bool write_timestamps_as_int96_;
1488
+ const bool coerce_timestamps_enabled_;
1489
+ const ::arrow::TimeUnit::type coerce_timestamps_unit_;
1490
+ const bool truncated_timestamps_allowed_;
1491
+ const bool store_schema_;
1492
+ const bool compliant_nested_types_;
1493
+ const EngineVersion engine_version_;
1494
+ const bool use_threads_;
1495
+ ::arrow::internal::Executor* executor_;
1496
+ const bool write_time_adjusted_to_utc_;
1497
+ };
1498
+
1499
+ /// \brief State object used for writing Arrow data directly to a Parquet
1500
+ /// column chunk. API possibly not stable
1501
+ struct ArrowWriteContext {
1502
+ ArrowWriteContext(MemoryPool* memory_pool, ArrowWriterProperties* properties)
1503
+ : memory_pool(memory_pool),
1504
+ properties(properties),
1505
+ data_buffer(AllocateBuffer(memory_pool)),
1506
+ def_levels_buffer(AllocateBuffer(memory_pool)) {}
1507
+
1508
+ template <typename T>
1509
+ ::arrow::Status GetScratchData(const int64_t num_values, T** out) {
1510
+ ARROW_RETURN_NOT_OK(this->data_buffer->Resize(num_values * sizeof(T), false));
1511
+ *out = reinterpret_cast<T*>(this->data_buffer->mutable_data());
1512
+ return ::arrow::Status::OK();
1513
+ }
1514
+
1515
+ MemoryPool* memory_pool;
1516
+ const ArrowWriterProperties* properties;
1517
+
1518
+ // Buffer used for storing the data of an array converted to the physical type
1519
+ // as expected by parquet-cpp.
1520
+ std::shared_ptr<ResizableBuffer> data_buffer;
1521
+
1522
+ // We use the shared ownership of this buffer
1523
+ std::shared_ptr<ResizableBuffer> def_levels_buffer;
1524
+ };
1525
+
1526
+ PARQUET_EXPORT
1527
+ std::shared_ptr<ArrowWriterProperties> default_arrow_writer_properties();
1528
+
1529
+ } // namespace parquet
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/schema.h ADDED
@@ -0,0 +1,494 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ // This module contains the logical parquet-cpp types (independent of Thrift
19
+ // structures), schema nodes, and related type tools
20
+
21
+ #pragma once
22
+
23
+ #include <cstdint>
24
+ #include <memory>
25
+ #include <ostream>
26
+ #include <string>
27
+ #include <unordered_map>
28
+ #include <utility>
29
+ #include <vector>
30
+
31
+ #include "parquet/platform.h"
32
+ #include "parquet/types.h"
33
+ #include "parquet/windows_fixup.h" // for OPTIONAL
34
+
35
+ namespace parquet {
36
+
37
+ class SchemaDescriptor;
38
+
39
+ namespace schema {
40
+
41
+ class Node;
42
+
43
+ // List encodings: using the terminology from Impala to define different styles
44
+ // of representing logical lists (a.k.a. ARRAY types) in Parquet schemas. Since
45
+ // the converted type named in the Parquet metadata is ConvertedType::LIST we
46
+ // use that terminology here. It also helps distinguish from the *_ARRAY
47
+ // primitive types.
48
+ //
49
+ // One-level encoding: Only allows required lists with required cells
50
+ // repeated value_type name
51
+ //
52
+ // Two-level encoding: Enables optional lists with only required cells
53
+ // <required/optional> group list
54
+ // repeated value_type item
55
+ //
56
+ // Three-level encoding: Enables optional lists with optional cells
57
+ // <required/optional> group bag
58
+ // repeated group list
59
+ // <required/optional> value_type item
60
+ //
61
+ // 2- and 1-level encoding are respectively equivalent to 3-level encoding with
62
+ // the non-repeated nodes set to required.
63
+ //
64
+ // The "official" encoding recommended in the Parquet spec is the 3-level, and
65
+ // we use that as the default when creating list types. For semantic completeness
66
+ // we allow the other two. Since all types of encodings will occur "in the
67
+ // wild" we need to be able to interpret the associated definition levels in
68
+ // the context of the actual encoding used in the file.
69
+ //
70
+ // NB: Some Parquet writers may not set ConvertedType::LIST on the repeated
71
+ // SchemaElement, which could make things challenging if we are trying to infer
72
+ // that a sequence of nodes semantically represents an array according to one
73
+ // of these encodings (versus a struct containing an array). We should refuse
74
+ // the temptation to guess, as they say.
75
+ struct ListEncoding {
76
+ enum type { ONE_LEVEL, TWO_LEVEL, THREE_LEVEL };
77
+ };
78
+
79
+ class PARQUET_EXPORT ColumnPath {
80
+ public:
81
+ ColumnPath() : path_() {}
82
+ explicit ColumnPath(const std::vector<std::string>& path) : path_(path) {}
83
+ explicit ColumnPath(std::vector<std::string>&& path) : path_(std::move(path)) {}
84
+
85
+ static std::shared_ptr<ColumnPath> FromDotString(const std::string& dotstring);
86
+ static std::shared_ptr<ColumnPath> FromNode(const Node& node);
87
+
88
+ std::shared_ptr<ColumnPath> extend(const std::string& node_name) const;
89
+ std::string ToDotString() const;
90
+ const std::vector<std::string>& ToDotVector() const;
91
+
92
+ protected:
93
+ std::vector<std::string> path_;
94
+ };
95
+
96
+ // Base class for logical schema types. A type has a name, repetition level,
97
+ // and optionally a logical type (ConvertedType in Parquet metadata parlance)
98
+ class PARQUET_EXPORT Node {
99
+ public:
100
+ enum type { PRIMITIVE, GROUP };
101
+
102
+ virtual ~Node() {}
103
+
104
+ bool is_primitive() const { return type_ == Node::PRIMITIVE; }
105
+
106
+ bool is_group() const { return type_ == Node::GROUP; }
107
+
108
+ bool is_optional() const { return repetition_ == Repetition::OPTIONAL; }
109
+
110
+ bool is_repeated() const { return repetition_ == Repetition::REPEATED; }
111
+
112
+ bool is_required() const { return repetition_ == Repetition::REQUIRED; }
113
+
114
+ virtual bool Equals(const Node* other) const = 0;
115
+
116
+ const std::string& name() const { return name_; }
117
+
118
+ Node::type node_type() const { return type_; }
119
+
120
+ Repetition::type repetition() const { return repetition_; }
121
+
122
+ ConvertedType::type converted_type() const { return converted_type_; }
123
+
124
+ const std::shared_ptr<const LogicalType>& logical_type() const { return logical_type_; }
125
+
126
+ /// \brief The field_id value for the serialized SchemaElement. If the
127
+ /// field_id is less than 0 (e.g. -1), it will not be set when serialized to
128
+ /// Thrift.
129
+ int field_id() const { return field_id_; }
130
+
131
+ const Node* parent() const { return parent_; }
132
+
133
+ const std::shared_ptr<ColumnPath> path() const;
134
+
135
+ virtual void ToParquet(void* element) const = 0;
136
+
137
+ // Node::Visitor abstract class for walking schemas with the visitor pattern
138
+ class Visitor {
139
+ public:
140
+ virtual ~Visitor() {}
141
+
142
+ virtual void Visit(Node* node) = 0;
143
+ };
144
+ class ConstVisitor {
145
+ public:
146
+ virtual ~ConstVisitor() {}
147
+
148
+ virtual void Visit(const Node* node) = 0;
149
+ };
150
+
151
+ virtual void Visit(Visitor* visitor) = 0;
152
+ virtual void VisitConst(ConstVisitor* visitor) const = 0;
153
+
154
+ protected:
155
+ friend class GroupNode;
156
+
157
+ Node(Node::type type, const std::string& name, Repetition::type repetition,
158
+ ConvertedType::type converted_type = ConvertedType::NONE, int field_id = -1)
159
+ : type_(type),
160
+ name_(name),
161
+ repetition_(repetition),
162
+ converted_type_(converted_type),
163
+ field_id_(field_id),
164
+ parent_(NULLPTR) {}
165
+
166
+ Node(Node::type type, const std::string& name, Repetition::type repetition,
167
+ std::shared_ptr<const LogicalType> logical_type, int field_id = -1)
168
+ : type_(type),
169
+ name_(name),
170
+ repetition_(repetition),
171
+ logical_type_(std::move(logical_type)),
172
+ field_id_(field_id),
173
+ parent_(NULLPTR) {}
174
+
175
+ Node::type type_;
176
+ std::string name_;
177
+ Repetition::type repetition_;
178
+ ConvertedType::type converted_type_{ConvertedType::NONE};
179
+ std::shared_ptr<const LogicalType> logical_type_;
180
+ int field_id_;
181
+ // Nodes should not be shared, they have a single parent.
182
+ const Node* parent_;
183
+
184
+ bool EqualsInternal(const Node* other) const;
185
+ void SetParent(const Node* p_parent);
186
+
187
+ private:
188
+ PARQUET_DISALLOW_COPY_AND_ASSIGN(Node);
189
+ };
190
+
191
+ // Save our breath all over the place with these typedefs
192
+ using NodePtr = std::shared_ptr<Node>;
193
+ using NodeVector = std::vector<NodePtr>;
194
+
195
+ // A type that is one of the primitive Parquet storage types. In addition to
196
+ // the other type metadata (name, repetition level, logical type), also has the
197
+ // physical storage type and their type-specific metadata (byte width, decimal
198
+ // parameters)
199
+ class PARQUET_EXPORT PrimitiveNode : public Node {
200
+ public:
201
+ static std::unique_ptr<Node> FromParquet(const void* opaque_element);
202
+
203
+ // A field_id -1 (or any negative value) will be serialized as null in Thrift
204
+ static inline NodePtr Make(const std::string& name, Repetition::type repetition,
205
+ Type::type type,
206
+ ConvertedType::type converted_type = ConvertedType::NONE,
207
+ int length = -1, int precision = -1, int scale = -1,
208
+ int field_id = -1) {
209
+ return NodePtr(new PrimitiveNode(name, repetition, type, converted_type, length,
210
+ precision, scale, field_id));
211
+ }
212
+
213
+ // If no logical type, pass LogicalType::None() or nullptr
214
+ // A field_id -1 (or any negative value) will be serialized as null in Thrift
215
+ static inline NodePtr Make(const std::string& name, Repetition::type repetition,
216
+ std::shared_ptr<const LogicalType> logical_type,
217
+ Type::type primitive_type, int primitive_length = -1,
218
+ int field_id = -1) {
219
+ return NodePtr(new PrimitiveNode(name, repetition, std::move(logical_type),
220
+ primitive_type, primitive_length, field_id));
221
+ }
222
+
223
+ bool Equals(const Node* other) const override;
224
+
225
+ Type::type physical_type() const { return physical_type_; }
226
+
227
+ ColumnOrder column_order() const { return column_order_; }
228
+
229
+ void SetColumnOrder(ColumnOrder column_order) { column_order_ = column_order; }
230
+
231
+ int32_t type_length() const { return type_length_; }
232
+
233
+ const DecimalMetadata& decimal_metadata() const { return decimal_metadata_; }
234
+
235
+ void ToParquet(void* element) const override;
236
+ void Visit(Visitor* visitor) override;
237
+ void VisitConst(ConstVisitor* visitor) const override;
238
+
239
+ private:
240
+ PrimitiveNode(const std::string& name, Repetition::type repetition, Type::type type,
241
+ ConvertedType::type converted_type = ConvertedType::NONE, int length = -1,
242
+ int precision = -1, int scale = -1, int field_id = -1);
243
+
244
+ PrimitiveNode(const std::string& name, Repetition::type repetition,
245
+ std::shared_ptr<const LogicalType> logical_type,
246
+ Type::type primitive_type, int primitive_length = -1, int field_id = -1);
247
+
248
+ Type::type physical_type_;
249
+ int32_t type_length_;
250
+ DecimalMetadata decimal_metadata_;
251
+ ColumnOrder column_order_;
252
+
253
+ // For FIXED_LEN_BYTE_ARRAY
254
+ void SetTypeLength(int32_t length) { type_length_ = length; }
255
+
256
+ bool EqualsInternal(const PrimitiveNode* other) const;
257
+
258
+ FRIEND_TEST(TestPrimitiveNode, Attrs);
259
+ FRIEND_TEST(TestPrimitiveNode, Equals);
260
+ FRIEND_TEST(TestPrimitiveNode, PhysicalLogicalMapping);
261
+ FRIEND_TEST(TestPrimitiveNode, FromParquet);
262
+ };
263
+
264
+ class PARQUET_EXPORT GroupNode : public Node {
265
+ public:
266
+ static std::unique_ptr<Node> FromParquet(const void* opaque_element,
267
+ NodeVector fields = {});
268
+
269
+ // A field_id -1 (or any negative value) will be serialized as null in Thrift
270
+ static inline NodePtr Make(const std::string& name, Repetition::type repetition,
271
+ const NodeVector& fields,
272
+ ConvertedType::type converted_type = ConvertedType::NONE,
273
+ int field_id = -1) {
274
+ return NodePtr(new GroupNode(name, repetition, fields, converted_type, field_id));
275
+ }
276
+
277
+ // If no logical type, pass nullptr
278
+ // A field_id -1 (or any negative value) will be serialized as null in Thrift
279
+ static inline NodePtr Make(const std::string& name, Repetition::type repetition,
280
+ const NodeVector& fields,
281
+ std::shared_ptr<const LogicalType> logical_type,
282
+ int field_id = -1) {
283
+ return NodePtr(
284
+ new GroupNode(name, repetition, fields, std::move(logical_type), field_id));
285
+ }
286
+
287
+ bool Equals(const Node* other) const override;
288
+
289
+ const NodePtr& field(int i) const { return fields_[i]; }
290
+ // Get the index of a field by its name, or negative value if not found.
291
+ // If several fields share the same name, it is unspecified which one
292
+ // is returned.
293
+ int FieldIndex(const std::string& name) const;
294
+ // Get the index of a field by its node, or negative value if not found.
295
+ int FieldIndex(const Node& node) const;
296
+
297
+ int field_count() const { return static_cast<int>(fields_.size()); }
298
+
299
+ void ToParquet(void* element) const override;
300
+ void Visit(Visitor* visitor) override;
301
+ void VisitConst(ConstVisitor* visitor) const override;
302
+
303
+ /// \brief Return true if this node or any child node has REPEATED repetition
304
+ /// type
305
+ bool HasRepeatedFields() const;
306
+
307
+ private:
308
+ GroupNode(const std::string& name, Repetition::type repetition,
309
+ const NodeVector& fields,
310
+ ConvertedType::type converted_type = ConvertedType::NONE, int field_id = -1);
311
+
312
+ GroupNode(const std::string& name, Repetition::type repetition,
313
+ const NodeVector& fields, std::shared_ptr<const LogicalType> logical_type,
314
+ int field_id = -1);
315
+
316
+ NodeVector fields_;
317
+ bool EqualsInternal(const GroupNode* other) const;
318
+
319
+ // Mapping between field name to the field index
320
+ std::unordered_multimap<std::string, int> field_name_to_idx_;
321
+
322
+ FRIEND_TEST(TestGroupNode, Attrs);
323
+ FRIEND_TEST(TestGroupNode, Equals);
324
+ FRIEND_TEST(TestGroupNode, FieldIndex);
325
+ FRIEND_TEST(TestGroupNode, FieldIndexDuplicateName);
326
+ };
327
+
328
+ // ----------------------------------------------------------------------
329
+ // Convenience primitive type factory functions
330
+
331
+ #define PRIMITIVE_FACTORY(FuncName, TYPE) \
332
+ static inline NodePtr FuncName(const std::string& name, \
333
+ Repetition::type repetition = Repetition::OPTIONAL, \
334
+ int field_id = -1) { \
335
+ return PrimitiveNode::Make(name, repetition, Type::TYPE, ConvertedType::NONE, \
336
+ /*length=*/-1, /*precision=*/-1, /*scale=*/-1, field_id); \
337
+ }
338
+
339
+ PRIMITIVE_FACTORY(Boolean, BOOLEAN)
340
+ PRIMITIVE_FACTORY(Int32, INT32)
341
+ PRIMITIVE_FACTORY(Int64, INT64)
342
+ PRIMITIVE_FACTORY(Int96, INT96)
343
+ PRIMITIVE_FACTORY(Float, FLOAT)
344
+ PRIMITIVE_FACTORY(Double, DOUBLE)
345
+ PRIMITIVE_FACTORY(ByteArray, BYTE_ARRAY)
346
+
347
+ void PARQUET_EXPORT PrintSchema(const schema::Node* schema, std::ostream& stream,
348
+ int indent_width = 2);
349
+
350
+ } // namespace schema
351
+
352
+ // The ColumnDescriptor encapsulates information necessary to interpret
353
+ // primitive column data in the context of a particular schema. We have to
354
+ // examine the node structure of a column's path to the root in the schema tree
355
+ // to be able to reassemble the nested structure from the repetition and
356
+ // definition levels.
357
+ class PARQUET_EXPORT ColumnDescriptor {
358
+ public:
359
+ ColumnDescriptor(schema::NodePtr node, int16_t max_definition_level,
360
+ int16_t max_repetition_level,
361
+ const SchemaDescriptor* schema_descr = NULLPTR);
362
+
363
+ bool Equals(const ColumnDescriptor& other) const;
364
+
365
+ int16_t max_definition_level() const { return max_definition_level_; }
366
+
367
+ int16_t max_repetition_level() const { return max_repetition_level_; }
368
+
369
+ Type::type physical_type() const { return primitive_node_->physical_type(); }
370
+
371
+ ConvertedType::type converted_type() const { return primitive_node_->converted_type(); }
372
+
373
+ const std::shared_ptr<const LogicalType>& logical_type() const {
374
+ return primitive_node_->logical_type();
375
+ }
376
+
377
+ ColumnOrder column_order() const { return primitive_node_->column_order(); }
378
+
379
+ SortOrder::type sort_order() const {
380
+ const auto& la = logical_type();
381
+ auto pt = physical_type();
382
+ return la ? GetSortOrder(la, pt) : GetSortOrder(converted_type(), pt);
383
+ }
384
+
385
+ const std::string& name() const { return primitive_node_->name(); }
386
+
387
+ const std::shared_ptr<schema::ColumnPath> path() const;
388
+
389
+ const schema::NodePtr& schema_node() const { return node_; }
390
+
391
+ std::string ToString() const;
392
+
393
+ int type_length() const;
394
+
395
+ int type_precision() const;
396
+
397
+ int type_scale() const;
398
+
399
+ private:
400
+ schema::NodePtr node_;
401
+ const schema::PrimitiveNode* primitive_node_;
402
+
403
+ int16_t max_definition_level_;
404
+ int16_t max_repetition_level_;
405
+ };
406
+
407
+ // Container for the converted Parquet schema with a computed information from
408
+ // the schema analysis needed for file reading
409
+ //
410
+ // * Column index to Node
411
+ // * Max repetition / definition levels for each primitive node
412
+ //
413
+ // The ColumnDescriptor objects produced by this class can be used to assist in
414
+ // the reconstruction of fully materialized data structures from the
415
+ // repetition-definition level encoding of nested data
416
+ //
417
+ // TODO(wesm): this object can be recomputed from a Schema
418
+ class PARQUET_EXPORT SchemaDescriptor {
419
+ public:
420
+ SchemaDescriptor() = default;
421
+ ~SchemaDescriptor() = default;
422
+
423
+ // Analyze the schema
424
+ void Init(std::unique_ptr<schema::Node> schema);
425
+ void Init(schema::NodePtr schema);
426
+
427
+ const ColumnDescriptor* Column(int i) const;
428
+
429
+ // Get the index of a column by its dotstring path, or negative value if not found.
430
+ // If several columns share the same dotstring path, it is unspecified which one
431
+ // is returned.
432
+ int ColumnIndex(const std::string& node_path) const;
433
+ // Get the index of a column by its node, or negative value if not found.
434
+ int ColumnIndex(const schema::Node& node) const;
435
+
436
+ bool Equals(const SchemaDescriptor& other, std::ostream* diff_output = NULLPTR) const;
437
+
438
+ // The number of physical columns appearing in the file
439
+ int num_columns() const { return static_cast<int>(leaves_.size()); }
440
+
441
+ const schema::NodePtr& schema_root() const { return schema_; }
442
+
443
+ const schema::GroupNode* group_node() const { return group_node_; }
444
+
445
+ // Returns the root (child of the schema root) node of the leaf(column) node
446
+ const schema::Node* GetColumnRoot(int i) const;
447
+
448
+ const std::string& name() const { return group_node_->name(); }
449
+
450
+ std::string ToString() const;
451
+
452
+ void updateColumnOrders(const std::vector<ColumnOrder>& column_orders);
453
+
454
+ /// \brief Return column index corresponding to a particular
455
+ /// PrimitiveNode. Returns -1 if not found
456
+ int GetColumnIndex(const schema::PrimitiveNode& node) const;
457
+
458
+ /// \brief Return true if any field or their children have REPEATED repetition
459
+ /// type
460
+ bool HasRepeatedFields() const;
461
+
462
+ private:
463
+ friend class ColumnDescriptor;
464
+
465
+ // Root Node
466
+ schema::NodePtr schema_;
467
+ // Root Node
468
+ // Would never be NULLPTR.
469
+ const schema::GroupNode* group_node_;
470
+
471
+ void BuildTree(const schema::NodePtr& node, int16_t max_def_level,
472
+ int16_t max_rep_level, const schema::NodePtr& base);
473
+
474
+ // Result of leaf node / tree analysis
475
+ std::vector<ColumnDescriptor> leaves_;
476
+
477
+ std::unordered_map<const schema::PrimitiveNode*, int> node_to_leaf_index_;
478
+
479
+ // Mapping between leaf nodes and root group of leaf (first node
480
+ // below the schema's root group)
481
+ //
482
+ // For example, the leaf `a.b.c.d` would have a link back to `a`
483
+ //
484
+ // -- a <------
485
+ // -- -- b |
486
+ // -- -- -- c |
487
+ // -- -- -- -- d
488
+ std::unordered_map<int, schema::NodePtr> leaf_to_base_;
489
+
490
+ // Mapping between ColumnPath DotString to the leaf index
491
+ std::unordered_multimap<std::string, int> leaf_to_idx_;
492
+ };
493
+
494
+ } // namespace parquet
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/parquet/size_statistics.h ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <cstdint>
21
+ #include <iosfwd>
22
+ #include <optional>
23
+ #include <span>
24
+ #include <vector>
25
+
26
+ #include "parquet/platform.h"
27
+ #include "parquet/type_fwd.h"
28
+
29
+ namespace parquet {
30
+
31
+ /// A structure for capturing metadata for estimating the unencoded,
32
+ /// uncompressed size of data written. This is useful for readers to estimate
33
+ /// how much memory is needed to reconstruct data in their memory model and for
34
+ /// fine-grained filter push down on nested structures (the histograms contained
35
+ /// in this structure can help determine the number of nulls at a particular
36
+ /// nesting level and maximum length of lists).
37
+ struct PARQUET_EXPORT SizeStatistics {
38
+ /// When present, there is expected to be one element corresponding to each
39
+ /// definition (i.e. size=max definition+1) where each element
40
+ /// represents the number of times the definition level was observed in the
41
+ /// data.
42
+ ///
43
+ /// This field may be omitted (a.k.a. zero-length vector) if max_definition_level
44
+ /// is 0 without loss of information.
45
+ std::vector<int64_t> definition_level_histogram;
46
+
47
+ /// Same as definition_level_histogram except for repetition levels.
48
+ ///
49
+ /// This field may be omitted (a.k.a. zero-length vector) if max_repetition_level
50
+ /// is 0 without loss of information.
51
+ std::vector<int64_t> repetition_level_histogram;
52
+
53
+ /// The number of physical bytes stored for BYTE_ARRAY data values assuming
54
+ /// no encoding. This is exclusive of the bytes needed to store the length of
55
+ /// each byte array. In other words, this field is equivalent to the `(size
56
+ /// of PLAIN-ENCODING the byte array values) - (4 bytes * number of values
57
+ /// written)`. To determine unencoded sizes of other types readers can use
58
+ /// schema information multiplied by the number of non-null and null values.
59
+ /// The number of null/non-null values can be inferred from the histograms
60
+ /// below.
61
+ ///
62
+ /// For example, if a column chunk is dictionary-encoded with dictionary
63
+ /// ["a", "bc", "cde"], and a data page contains the indices [0, 0, 1, 2],
64
+ /// then this value for that data page should be 7 (1 + 1 + 2 + 3).
65
+ ///
66
+ /// This field should only be set for types that use BYTE_ARRAY as their
67
+ /// physical type.
68
+ std::optional<int64_t> unencoded_byte_array_data_bytes;
69
+
70
+ /// \brief Check if the SizeStatistics is set.
71
+ bool is_set() const {
72
+ return !repetition_level_histogram.empty() || !definition_level_histogram.empty() ||
73
+ unencoded_byte_array_data_bytes.has_value();
74
+ }
75
+
76
+ /// \brief Increment the unencoded byte array data bytes.
77
+ void IncrementUnencodedByteArrayDataBytes(int64_t value);
78
+
79
+ /// \brief Merge two SizeStatistics.
80
+ /// \throws ParquetException if SizeStatistics to merge is not compatible.
81
+ void Merge(const SizeStatistics& other);
82
+
83
+ /// \brief Validate the SizeStatistics
84
+ /// \throws ParquetException if the histograms don't have the right length,
85
+ /// or if unencoded_byte_array_data_bytes is present for a non-BYTE_ARRAY column.
86
+ void Validate(const ColumnDescriptor* descr) const;
87
+
88
+ /// \brief Reset the SizeStatistics to be empty.
89
+ void Reset();
90
+
91
+ /// \brief Make an empty SizeStatistics object for specific type.
92
+ static std::unique_ptr<SizeStatistics> Make(const ColumnDescriptor* descr);
93
+ };
94
+
95
+ PARQUET_EXPORT
96
+ std::ostream& operator<<(std::ostream&, const SizeStatistics&);
97
+
98
+ PARQUET_EXPORT
99
+ void UpdateLevelHistogram(std::span<const int16_t> levels, std::span<int64_t> histogram);
100
+
101
+ } // namespace parquet