techprotrade commited on
Commit
f102409
·
verified ·
1 Parent(s): 2b47bd3

Add loan-advisor-agent (part 2)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client.h +434 -0
  2. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_auth.h +62 -0
  3. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_cookie_middleware.h +33 -0
  4. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_middleware.h +78 -0
  5. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_tracing_middleware.h +34 -0
  6. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/middleware.h +75 -0
  7. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/otel_logging.h +33 -0
  8. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/platform.h +31 -0
  9. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server.h +327 -0
  10. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server_auth.h +107 -0
  11. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server_middleware.h +82 -0
  12. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server_tracing_middleware.h +68 -0
  13. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_auth_handlers.h +89 -0
  14. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_definitions.h +318 -0
  15. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_flight_server.h +92 -0
  16. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_util.h +188 -0
  17. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/transport.h +298 -0
  18. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/transport_server.h +133 -0
  19. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/type_fwd.h +65 -0
  20. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/types.h +1314 -0
  21. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/types_async.h +76 -0
  22. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/visibility.h +48 -0
  23. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/api.h +25 -0
  24. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/buffered.h +168 -0
  25. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/caching.h +157 -0
  26. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/compressed.h +124 -0
  27. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/concurrency.h +263 -0
  28. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/file.h +221 -0
  29. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/hdfs.h +284 -0
  30. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/interfaces.h +362 -0
  31. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/memory.h +197 -0
  32. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/mman.h +169 -0
  33. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/slow.h +118 -0
  34. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/stdio.h +82 -0
  35. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/test_common.h +69 -0
  36. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/transform.h +60 -0
  37. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/type_fwd.h +77 -0
  38. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/api.h +24 -0
  39. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/dictionary.h +177 -0
  40. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/feather.h +150 -0
  41. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/message.h +585 -0
  42. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/options.h +201 -0
  43. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/reader.h +638 -0
  44. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/test_common.h +198 -0
  45. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/type_fwd.h +68 -0
  46. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/util.h +41 -0
  47. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/writer.h +475 -0
  48. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/api.h +21 -0
  49. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/chunked_builder.h +68 -0
  50. loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/chunker.h +35 -0
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client.h ADDED
@@ -0,0 +1,434 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ /// \brief Implementation of Flight RPC client.
19
+
20
+ #pragma once
21
+
22
+ #include <chrono>
23
+ #include <memory>
24
+ #include <string>
25
+ #include <utility>
26
+ #include <variant>
27
+ #include <vector>
28
+
29
+ #include "arrow/ipc/options.h"
30
+ #include "arrow/ipc/reader.h"
31
+ #include "arrow/ipc/writer.h"
32
+ #include "arrow/result.h"
33
+ #include "arrow/status.h"
34
+ #include "arrow/util/cancel.h"
35
+
36
+ #include "arrow/flight/type_fwd.h"
37
+ #include "arrow/flight/types.h" // IWYU pragma: keep
38
+ #include "arrow/flight/visibility.h"
39
+
40
+ namespace arrow {
41
+
42
+ class RecordBatch;
43
+ class Schema;
44
+
45
+ namespace flight {
46
+
47
+ /// \brief A duration type for Flight call timeouts.
48
+ typedef std::chrono::duration<double, std::chrono::seconds::period> TimeoutDuration;
49
+
50
+ /// \brief Hints to the underlying RPC layer for Arrow Flight calls.
51
+ class ARROW_FLIGHT_EXPORT FlightCallOptions {
52
+ public:
53
+ /// Create a default set of call options.
54
+ FlightCallOptions();
55
+
56
+ /// \brief An optional timeout for this call. Negative durations
57
+ /// mean an implementation-defined default behavior will be used
58
+ /// instead. This is the default value.
59
+ TimeoutDuration timeout;
60
+
61
+ /// \brief IPC reader options, if applicable for the call.
62
+ ipc::IpcReadOptions read_options;
63
+
64
+ /// \brief IPC writer options, if applicable for the call.
65
+ ipc::IpcWriteOptions write_options;
66
+
67
+ /// \brief Headers for client to add to context.
68
+ std::vector<std::pair<std::string, std::string>> headers;
69
+
70
+ /// \brief A token to enable interactive user cancellation of long-running requests.
71
+ StopToken stop_token;
72
+
73
+ /// \brief An optional memory manager to control where to allocate incoming data.
74
+ std::shared_ptr<MemoryManager> memory_manager;
75
+ };
76
+
77
+ /// \brief Indicate that the client attempted to write a message
78
+ /// larger than the soft limit set via write_size_limit_bytes.
79
+ class ARROW_FLIGHT_EXPORT FlightWriteSizeStatusDetail : public arrow::StatusDetail {
80
+ public:
81
+ explicit FlightWriteSizeStatusDetail(int64_t limit, int64_t actual)
82
+ : limit_(limit), actual_(actual) {}
83
+ const char* type_id() const override;
84
+ std::string ToString() const override;
85
+ int64_t limit() const { return limit_; }
86
+ int64_t actual() const { return actual_; }
87
+
88
+ /// \brief Extract this status detail from a status, or return
89
+ /// nullptr if the status doesn't contain this status detail.
90
+ static std::shared_ptr<FlightWriteSizeStatusDetail> UnwrapStatus(
91
+ const arrow::Status& status);
92
+
93
+ private:
94
+ int64_t limit_;
95
+ int64_t actual_;
96
+ };
97
+
98
+ struct ARROW_FLIGHT_EXPORT FlightClientOptions {
99
+ /// \brief Root certificates to use for validating server
100
+ /// certificates.
101
+ std::string tls_root_certs;
102
+ /// \brief Override the hostname checked by TLS. Use with caution.
103
+ std::string override_hostname;
104
+ /// \brief The client certificate to use if using Mutual TLS
105
+ std::string cert_chain;
106
+ /// \brief The private key associated with the client certificate for Mutual TLS
107
+ std::string private_key;
108
+ /// \brief A list of client middleware to apply.
109
+ std::vector<std::shared_ptr<ClientMiddlewareFactory>> middleware;
110
+ /// \brief A soft limit on the number of bytes to write in a single
111
+ /// batch when sending Arrow data to a server.
112
+ ///
113
+ /// Used to help limit server memory consumption. Only enabled if
114
+ /// positive. When enabled, FlightStreamWriter.Write* may yield a
115
+ /// IOError with error detail FlightWriteSizeStatusDetail.
116
+ int64_t write_size_limit_bytes = 0;
117
+
118
+ /// \brief Generic connection options, passed to the underlying
119
+ /// transport; interpretation is implementation-dependent.
120
+ std::vector<std::pair<std::string, std::variant<int, std::string>>> generic_options;
121
+
122
+ /// \brief Use TLS without validating the server certificate. Use with caution.
123
+ bool disable_server_verification = false;
124
+
125
+ /// \brief Get default options.
126
+ static FlightClientOptions Defaults();
127
+ };
128
+
129
+ /// \brief A RecordBatchReader exposing Flight metadata and cancel
130
+ /// operations.
131
+ class ARROW_FLIGHT_EXPORT FlightStreamReader : public MetadataRecordBatchReader {
132
+ public:
133
+ /// \brief Try to cancel the call.
134
+ virtual void Cancel() = 0;
135
+
136
+ using MetadataRecordBatchReader::ToRecordBatches;
137
+ /// \brief Consume entire stream as a vector of record batches
138
+ virtual arrow::Result<std::vector<std::shared_ptr<RecordBatch>>> ToRecordBatches(
139
+ const StopToken& stop_token) = 0;
140
+
141
+ using MetadataRecordBatchReader::ToTable;
142
+ /// \brief Consume entire stream as a Table
143
+ arrow::Result<std::shared_ptr<Table>> ToTable(const StopToken& stop_token);
144
+
145
+ using MetadataRecordBatchReader::stats;
146
+ /// \brief Return current read statistics
147
+ virtual arrow::ipc::ReadStats stats() const = 0;
148
+ };
149
+
150
+ // Silence warning
151
+ // "non dll-interface class RecordBatchReader used as base for dll-interface class"
152
+ #ifdef _MSC_VER
153
+ # pragma warning(push)
154
+ # pragma warning(disable : 4275)
155
+ #endif
156
+
157
+ /// \brief A RecordBatchWriter that also allows sending
158
+ /// application-defined metadata via the Flight protocol.
159
+ class ARROW_FLIGHT_EXPORT FlightStreamWriter : public MetadataRecordBatchWriter {
160
+ public:
161
+ /// \brief Indicate that the application is done writing to this stream.
162
+ ///
163
+ /// The application may not write to this stream after calling
164
+ /// this. This differs from closing the stream because this writer
165
+ /// may represent only one half of a readable and writable stream.
166
+ virtual Status DoneWriting() = 0;
167
+ };
168
+
169
+ #ifdef _MSC_VER
170
+ # pragma warning(pop)
171
+ #endif
172
+
173
+ /// \brief A reader for application-specific metadata sent back to the
174
+ /// client during an upload.
175
+ class ARROW_FLIGHT_EXPORT FlightMetadataReader {
176
+ public:
177
+ virtual ~FlightMetadataReader();
178
+ /// \brief Read a message from the server.
179
+ virtual Status ReadMetadata(std::shared_ptr<Buffer>* out) = 0;
180
+ };
181
+
182
+ /// \brief Client class for Arrow Flight RPC services.
183
+ class ARROW_FLIGHT_EXPORT FlightClient {
184
+ public:
185
+ ~FlightClient();
186
+
187
+ /// \brief Connect to an unauthenticated flight service
188
+ /// \param[in] location the URI
189
+ /// \return Arrow result with the created FlightClient, OK status may not indicate that
190
+ /// the connection was successful
191
+ static arrow::Result<std::unique_ptr<FlightClient>> Connect(const Location& location);
192
+
193
+ /// \brief Connect to an unauthenticated flight service
194
+ /// \param[in] location the URI
195
+ /// \param[in] options Other options for setting up the client
196
+ /// \return Arrow result with the created FlightClient, OK status may not indicate that
197
+ /// the connection was successful
198
+ static arrow::Result<std::unique_ptr<FlightClient>> Connect(
199
+ const Location& location, const FlightClientOptions& options);
200
+
201
+ /// \brief Authenticate to the server using the given handler.
202
+ /// \param[in] options Per-RPC options
203
+ /// \param[in] auth_handler The authentication mechanism to use
204
+ /// \return Status OK if the client authenticated successfully
205
+ Status Authenticate(const FlightCallOptions& options,
206
+ std::unique_ptr<ClientAuthHandler> auth_handler);
207
+
208
+ /// \brief Authenticate to the server using basic HTTP style authentication.
209
+ /// \param[in] options Per-RPC options
210
+ /// \param[in] username Username to use
211
+ /// \param[in] password Password to use
212
+ /// \return Arrow result with bearer token and status OK if client authenticated
213
+ /// successfully
214
+ arrow::Result<std::pair<std::string, std::string>> AuthenticateBasicToken(
215
+ const FlightCallOptions& options, const std::string& username,
216
+ const std::string& password);
217
+
218
+ /// \brief Perform the indicated action, returning an iterator to the stream
219
+ /// of results, if any
220
+ /// \param[in] options Per-RPC options
221
+ /// \param[in] action the action to be performed
222
+ /// \return Arrow result with an iterator object for reading the returned results
223
+ arrow::Result<std::unique_ptr<ResultStream>> DoAction(const FlightCallOptions& options,
224
+ const Action& action);
225
+ arrow::Result<std::unique_ptr<ResultStream>> DoAction(const Action& action) {
226
+ return DoAction({}, action);
227
+ }
228
+
229
+ /// \brief Perform the CancelFlightInfo action, returning a
230
+ /// CancelFlightInfoResult
231
+ ///
232
+ /// \param[in] options Per-RPC options
233
+ /// \param[in] request The CancelFlightInfoRequest
234
+ /// \return Arrow result with a CancelFlightInfoResult
235
+ arrow::Result<CancelFlightInfoResult> CancelFlightInfo(
236
+ const FlightCallOptions& options, const CancelFlightInfoRequest& request);
237
+ arrow::Result<CancelFlightInfoResult> CancelFlightInfo(
238
+ const CancelFlightInfoRequest& request) {
239
+ return CancelFlightInfo({}, request);
240
+ }
241
+
242
+ /// \brief Perform the RenewFlightEndpoint action, returning a renewed
243
+ /// FlightEndpoint
244
+ ///
245
+ /// \param[in] options Per-RPC options
246
+ /// \param[in] request The RenewFlightEndpointRequest
247
+ /// \return Arrow result with a renewed FlightEndpoint
248
+ arrow::Result<FlightEndpoint> RenewFlightEndpoint(
249
+ const FlightCallOptions& options, const RenewFlightEndpointRequest& request);
250
+ arrow::Result<FlightEndpoint> RenewFlightEndpoint(
251
+ const RenewFlightEndpointRequest& request) {
252
+ return RenewFlightEndpoint({}, request);
253
+ }
254
+
255
+ /// \brief Retrieve a list of available Action types
256
+ /// \param[in] options Per-RPC options
257
+ /// \return Arrow result with the available actions
258
+ arrow::Result<std::vector<ActionType>> ListActions(const FlightCallOptions& options);
259
+ arrow::Result<std::vector<ActionType>> ListActions() {
260
+ return ListActions(FlightCallOptions());
261
+ }
262
+
263
+ /// \brief Request access plan for a single flight, which may be an existing
264
+ /// dataset or a command to be executed
265
+ /// \param[in] options Per-RPC options
266
+ /// \param[in] descriptor the dataset request, whether a named dataset or
267
+ /// command
268
+ /// \return Arrow result with the FlightInfo describing where to access the dataset
269
+ arrow::Result<std::unique_ptr<FlightInfo>> GetFlightInfo(
270
+ const FlightCallOptions& options, const FlightDescriptor& descriptor);
271
+ arrow::Result<std::unique_ptr<FlightInfo>> GetFlightInfo(
272
+ const FlightDescriptor& descriptor) {
273
+ return GetFlightInfo({}, descriptor);
274
+ }
275
+
276
+ /// \brief Asynchronous GetFlightInfo.
277
+ /// \param[in] options Per-RPC options
278
+ /// \param[in] descriptor the dataset request
279
+ /// \param[in] listener Callbacks for response and RPC completion
280
+ void GetFlightInfoAsync(const FlightCallOptions& options,
281
+ const FlightDescriptor& descriptor,
282
+ std::shared_ptr<AsyncListener<FlightInfo>> listener);
283
+ void GetFlightInfoAsync(const FlightDescriptor& descriptor,
284
+ std::shared_ptr<AsyncListener<FlightInfo>> listener) {
285
+ return GetFlightInfoAsync({}, descriptor, std::move(listener));
286
+ }
287
+
288
+ /// \brief Asynchronous GetFlightInfo returning a Future.
289
+ /// \param[in] options Per-RPC options
290
+ /// \param[in] descriptor the dataset request
291
+ arrow::Future<FlightInfo> GetFlightInfoAsync(const FlightCallOptions& options,
292
+ const FlightDescriptor& descriptor);
293
+ arrow::Future<FlightInfo> GetFlightInfoAsync(const FlightDescriptor& descriptor) {
294
+ return GetFlightInfoAsync({}, descriptor);
295
+ }
296
+
297
+ /// \brief Request and poll a long running query
298
+ /// \param[in] options Per-RPC options
299
+ /// \param[in] descriptor the dataset request or a descriptor returned by a
300
+ /// prior PollFlightInfo call
301
+ /// \return Arrow result with the PollInfo describing the status of
302
+ /// the requested query
303
+ arrow::Result<std::unique_ptr<PollInfo>> PollFlightInfo(
304
+ const FlightCallOptions& options, const FlightDescriptor& descriptor);
305
+ arrow::Result<std::unique_ptr<PollInfo>> PollFlightInfo(
306
+ const FlightDescriptor& descriptor) {
307
+ return PollFlightInfo({}, descriptor);
308
+ }
309
+
310
+ /// \brief Request schema for a single flight, which may be an existing
311
+ /// dataset or a command to be executed
312
+ /// \param[in] options Per-RPC options
313
+ /// \param[in] descriptor the dataset request, whether a named dataset or
314
+ /// command
315
+ /// \return Arrow result with the SchemaResult describing the dataset schema
316
+ arrow::Result<std::unique_ptr<SchemaResult>> GetSchema(
317
+ const FlightCallOptions& options, const FlightDescriptor& descriptor);
318
+
319
+ arrow::Result<std::unique_ptr<SchemaResult>> GetSchema(
320
+ const FlightDescriptor& descriptor) {
321
+ return GetSchema({}, descriptor);
322
+ }
323
+
324
+ /// \brief List all available flights known to the server
325
+ /// \return Arrow result with an iterator that returns a FlightInfo for each flight
326
+ arrow::Result<std::unique_ptr<FlightListing>> ListFlights();
327
+
328
+ /// \brief List available flights given indicated filter criteria
329
+ /// \param[in] options Per-RPC options
330
+ /// \param[in] criteria the filter criteria (opaque)
331
+ /// \return Arrow result with an iterator that returns a FlightInfo for each flight
332
+ arrow::Result<std::unique_ptr<FlightListing>> ListFlights(
333
+ const FlightCallOptions& options, const Criteria& criteria);
334
+
335
+ /// \brief Given a flight ticket and schema, request to be sent the
336
+ /// stream. Returns record batch stream reader
337
+ /// \param[in] options Per-RPC options
338
+ /// \param[in] ticket The flight ticket to use
339
+ /// \return Arrow result with the returned RecordBatchReader
340
+ arrow::Result<std::unique_ptr<FlightStreamReader>> DoGet(
341
+ const FlightCallOptions& options, const Ticket& ticket);
342
+ arrow::Result<std::unique_ptr<FlightStreamReader>> DoGet(const Ticket& ticket) {
343
+ return DoGet({}, ticket);
344
+ }
345
+
346
+ /// \brief DoPut return value
347
+ struct DoPutResult {
348
+ /// \brief a writer to write record batches to
349
+ std::unique_ptr<FlightStreamWriter> writer;
350
+ /// \brief a reader for application metadata from the server
351
+ std::unique_ptr<FlightMetadataReader> reader;
352
+ };
353
+ /// \brief Upload data to a Flight described by the given
354
+ /// descriptor. The caller must call Close() on the returned stream
355
+ /// once they are done writing.
356
+ ///
357
+ /// The reader and writer are linked; closing the writer will also
358
+ /// close the reader. Use \a DoneWriting to only close the write
359
+ /// side of the channel.
360
+ ///
361
+ /// \param[in] options Per-RPC options
362
+ /// \param[in] descriptor the descriptor of the stream
363
+ /// \param[in] schema the schema for the data to upload
364
+ /// \return Arrow result with a DoPutResult struct holding a reader and a writer
365
+ arrow::Result<DoPutResult> DoPut(const FlightCallOptions& options,
366
+ const FlightDescriptor& descriptor,
367
+ const std::shared_ptr<Schema>& schema);
368
+
369
+ arrow::Result<DoPutResult> DoPut(const FlightDescriptor& descriptor,
370
+ const std::shared_ptr<Schema>& schema) {
371
+ return DoPut({}, descriptor, schema);
372
+ }
373
+
374
+ struct DoExchangeResult {
375
+ std::unique_ptr<FlightStreamWriter> writer;
376
+ std::unique_ptr<FlightStreamReader> reader;
377
+ };
378
+ arrow::Result<DoExchangeResult> DoExchange(const FlightCallOptions& options,
379
+ const FlightDescriptor& descriptor);
380
+ arrow::Result<DoExchangeResult> DoExchange(const FlightDescriptor& descriptor) {
381
+ return DoExchange({}, descriptor);
382
+ }
383
+
384
+ /// \brief Set server session option(s) by name/value. Sessions are generally
385
+ /// persisted via HTTP cookies.
386
+ /// \param[in] options Per-RPC options
387
+ /// \param[in] request The server session options to set
388
+ ::arrow::Result<SetSessionOptionsResult> SetSessionOptions(
389
+ const FlightCallOptions& options, const SetSessionOptionsRequest& request);
390
+
391
+ /// \brief Get the current server session options. The session is generally
392
+ /// accessed via an HTTP cookie.
393
+ /// \param[in] options Per-RPC options
394
+ /// \param[in] request The (empty) GetSessionOptions request object.
395
+ ::arrow::Result<GetSessionOptionsResult> GetSessionOptions(
396
+ const FlightCallOptions& options, const GetSessionOptionsRequest& request);
397
+
398
+ /// \brief Close/invalidate the current server session. The session is generally
399
+ /// accessed via an HTTP cookie.
400
+ /// \param[in] options Per-RPC options
401
+ /// \param[in] request The (empty) CloseSession request object.
402
+ ::arrow::Result<CloseSessionResult> CloseSession(const FlightCallOptions& options,
403
+ const CloseSessionRequest& request);
404
+
405
+ /// \brief Explicitly shut down and clean up the client.
406
+ ///
407
+ /// For backwards compatibility, this will be implicitly called by
408
+ /// the destructor if not already called, but this gives the
409
+ /// application no chance to handle errors, so it is recommended to
410
+ /// explicitly close the client.
411
+ ///
412
+ /// \since 8.0.0
413
+ Status Close();
414
+
415
+ /// \brief Whether this client supports asynchronous methods.
416
+ bool supports_async() const;
417
+
418
+ /// \brief Check whether this client supports asynchronous methods.
419
+ ///
420
+ /// This is like supports_async(), except that a detailed error message
421
+ /// is returned if async support is not available. If async support is
422
+ /// available, this function returns successfully.
423
+ Status CheckAsyncSupport() const;
424
+
425
+ private:
426
+ FlightClient();
427
+ Status CheckOpen() const;
428
+ std::unique_ptr<internal::ClientTransport> transport_;
429
+ bool closed_;
430
+ int64_t write_size_limit_bytes_;
431
+ };
432
+
433
+ } // namespace flight
434
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_auth.h ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <string>
21
+
22
+ #include "arrow/flight/visibility.h"
23
+ #include "arrow/status.h"
24
+
25
+ namespace arrow {
26
+
27
+ namespace flight {
28
+
29
+ /// \brief A reader for messages from the server during an
30
+ /// authentication handshake.
31
+ class ARROW_FLIGHT_EXPORT ClientAuthReader {
32
+ public:
33
+ virtual ~ClientAuthReader() = default;
34
+ virtual Status Read(std::string* response) = 0;
35
+ };
36
+
37
+ /// \brief A writer for messages to the server during an
38
+ /// authentication handshake.
39
+ class ARROW_FLIGHT_EXPORT ClientAuthSender {
40
+ public:
41
+ virtual ~ClientAuthSender() = default;
42
+ virtual Status Write(const std::string& token) = 0;
43
+ };
44
+
45
+ /// \brief An authentication implementation for a Flight service.
46
+ /// Authentication includes both an initial negotiation and a per-call
47
+ /// token validation. Implementations may choose to use either or both
48
+ /// mechanisms.
49
+ class ARROW_FLIGHT_EXPORT ClientAuthHandler {
50
+ public:
51
+ virtual ~ClientAuthHandler() = default;
52
+ /// \brief Authenticate the client on initial connection. The client
53
+ /// can send messages to/read responses from the server at any time.
54
+ /// \return Status OK if authenticated successfully
55
+ virtual Status Authenticate(ClientAuthSender* outgoing, ClientAuthReader* incoming) = 0;
56
+ /// \brief Get a per-call token.
57
+ /// \param[out] token The token to send to the server.
58
+ virtual Status GetToken(std::string* token) = 0;
59
+ };
60
+
61
+ } // namespace flight
62
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_cookie_middleware.h ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ // Middleware implementation for sending and receiving HTTP cookies.
19
+
20
+ #pragma once
21
+
22
+ #include <memory>
23
+
24
+ #include "arrow/flight/client_middleware.h"
25
+
26
+ namespace arrow {
27
+ namespace flight {
28
+
29
+ /// \brief Returns a ClientMiddlewareFactory that handles sending and receiving cookies.
30
+ ARROW_FLIGHT_EXPORT std::shared_ptr<ClientMiddlewareFactory> GetCookieFactory();
31
+
32
+ } // namespace flight
33
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_middleware.h ADDED
@@ -0,0 +1,78 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ // Interfaces for defining middleware for Flight clients. Currently
19
+ // experimental.
20
+
21
+ #pragma once
22
+
23
+ #include <memory>
24
+
25
+ #include "arrow/flight/middleware.h"
26
+ #include "arrow/flight/visibility.h" // IWYU pragma: keep
27
+ #include "arrow/status.h"
28
+
29
+ namespace arrow {
30
+ namespace flight {
31
+
32
+ /// \brief Client-side middleware for a call, instantiated per RPC.
33
+ ///
34
+ /// Middleware should be fast and must be infallible: there is no way
35
+ /// to reject the call or report errors from the middleware instance.
36
+ class ARROW_FLIGHT_EXPORT ClientMiddleware {
37
+ public:
38
+ virtual ~ClientMiddleware() = default;
39
+
40
+ /// \brief A callback before headers are sent. Extra headers can be
41
+ /// added, but existing ones cannot be read.
42
+ virtual void SendingHeaders(AddCallHeaders* outgoing_headers) = 0;
43
+
44
+ /// \brief A callback when headers are received from the server.
45
+ ///
46
+ /// This may be called more than once, since servers send both
47
+ /// headers and trailers. Some implementations (e.g. gRPC-Java, and
48
+ /// hence Arrow Flight in Java) may consolidate headers into
49
+ /// trailers if the RPC errored.
50
+ virtual void ReceivedHeaders(const CallHeaders& incoming_headers) = 0;
51
+
52
+ /// \brief A callback after the call has completed.
53
+ virtual void CallCompleted(const Status& status) = 0;
54
+ };
55
+
56
+ /// \brief A factory for new middleware instances.
57
+ ///
58
+ /// If added to a client, this will be called for each RPC (including
59
+ /// Handshake) to give the opportunity to intercept the call.
60
+ ///
61
+ /// It is guaranteed that all client middleware methods are called
62
+ /// from the same thread that calls the RPC method implementation.
63
+ class ARROW_FLIGHT_EXPORT ClientMiddlewareFactory {
64
+ public:
65
+ virtual ~ClientMiddlewareFactory() = default;
66
+
67
+ /// \brief A callback for the start of a new call.
68
+ ///
69
+ /// \param info Information about the call.
70
+ /// \param[out] middleware The middleware instance for this call. If
71
+ /// unset, will not add middleware to this call instance from
72
+ /// this factory.
73
+ virtual void StartCall(const CallInfo& info,
74
+ std::unique_ptr<ClientMiddleware>* middleware) = 0;
75
+ };
76
+
77
+ } // namespace flight
78
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_tracing_middleware.h ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ // Middleware implementation for propagating OpenTelemetry spans.
19
+
20
+ #pragma once
21
+
22
+ #include <memory>
23
+
24
+ #include "arrow/flight/client_middleware.h"
25
+
26
+ namespace arrow {
27
+ namespace flight {
28
+
29
+ /// \brief Returns a ClientMiddlewareFactory that handles sending OpenTelemetry spans.
30
+ ARROW_FLIGHT_EXPORT std::shared_ptr<ClientMiddlewareFactory>
31
+ MakeTracingClientMiddlewareFactory();
32
+
33
+ } // namespace flight
34
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/middleware.h ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ // Interfaces for defining middleware for Flight clients and
19
+ // servers.
20
+
21
+ #pragma once
22
+
23
+ #include <memory>
24
+ #include <string>
25
+ #include <string_view>
26
+ #include <utility>
27
+
28
+ #include "arrow/flight/types.h"
29
+ #include "arrow/status.h"
30
+
31
+ namespace arrow {
32
+ namespace flight {
33
+
34
+ /// \brief A write-only wrapper around headers for an RPC call.
35
+ class ARROW_FLIGHT_EXPORT AddCallHeaders {
36
+ public:
37
+ virtual ~AddCallHeaders() = default;
38
+
39
+ /// \brief Add a header to be sent to the client.
40
+ ///
41
+ /// \param[in] key The header name. Must be lowercase ASCII; some
42
+ /// transports may reject invalid header names.
43
+ /// \param[in] value The header value. Some transports may only
44
+ /// accept binary header values if the header name ends in "-bin".
45
+ virtual void AddHeader(const std::string& key, const std::string& value) = 0;
46
+ };
47
+
48
+ /// \brief An enumeration of the RPC methods Flight implements.
49
+ enum class FlightMethod : char {
50
+ Invalid = 0,
51
+ Handshake = 1,
52
+ ListFlights = 2,
53
+ GetFlightInfo = 3,
54
+ GetSchema = 4,
55
+ DoGet = 5,
56
+ DoPut = 6,
57
+ DoAction = 7,
58
+ ListActions = 8,
59
+ DoExchange = 9,
60
+ PollFlightInfo = 10,
61
+ };
62
+
63
+ /// \brief Get a human-readable name for a Flight method.
64
+ ARROW_FLIGHT_EXPORT
65
+ std::string ToString(FlightMethod method);
66
+
67
+ /// \brief Information about an instance of a Flight RPC.
68
+ struct ARROW_FLIGHT_EXPORT CallInfo {
69
+ public:
70
+ /// \brief The RPC method of this call.
71
+ FlightMethod method;
72
+ };
73
+
74
+ } // namespace flight
75
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/otel_logging.h ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include "arrow/util/config.h"
21
+
22
+ #ifdef ARROW_WITH_OPENTELEMETRY
23
+ # include "arrow/status.h"
24
+ # include "arrow/telemetry/logging.h"
25
+ # include "arrow/util/macros.h"
26
+
27
+ namespace arrow::flight {
28
+
29
+ ARROW_EXPORT Status
30
+ RegisterFlightOtelLoggers(const telemetry::OtelLoggingOptions& options);
31
+
32
+ } // namespace arrow::flight
33
+ #endif
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/platform.h ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ // Internal header. Platform-specific definitions for Flight.
19
+
20
+ #pragma once
21
+
22
+ #ifdef _MSC_VER
23
+
24
+ // The protobuf documentation says that C4251 warnings when using the
25
+ // library are spurious and suppressed when the build the library and
26
+ // compiler, but must be also suppressed in downstream projects
27
+ # pragma warning(disable : 4251)
28
+
29
+ #endif // _MSC_VER
30
+
31
+ #include "arrow/util/config.h" // IWYU pragma: keep
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server.h ADDED
@@ -0,0 +1,327 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ // Interfaces to use for defining Flight RPC servers.
19
+
20
+ #pragma once
21
+
22
+ #include <chrono>
23
+ #include <functional>
24
+ #include <memory>
25
+ #include <string>
26
+ #include <utility>
27
+ #include <vector>
28
+
29
+ #include "arrow/flight/server_auth.h"
30
+ #include "arrow/flight/type_fwd.h"
31
+ #include "arrow/flight/types.h" // IWYU pragma: keep
32
+ #include "arrow/flight/visibility.h" // IWYU pragma: keep
33
+ #include "arrow/ipc/dictionary.h"
34
+ #include "arrow/ipc/options.h"
35
+ #include "arrow/record_batch.h"
36
+
37
+ namespace arrow {
38
+
39
+ class Schema;
40
+ class Status;
41
+
42
+ namespace flight {
43
+
44
+ /// \brief Interface that produces a sequence of IPC payloads to be sent in
45
+ /// FlightData protobuf messages
46
+ class ARROW_FLIGHT_EXPORT FlightDataStream {
47
+ public:
48
+ virtual ~FlightDataStream();
49
+
50
+ virtual std::shared_ptr<Schema> schema() = 0;
51
+
52
+ /// \brief Compute FlightPayload containing serialized RecordBatch schema
53
+ virtual arrow::Result<FlightPayload> GetSchemaPayload() = 0;
54
+
55
+ // When the stream is completed, the last payload written will have null
56
+ // metadata
57
+ virtual arrow::Result<FlightPayload> Next() = 0;
58
+
59
+ virtual Status Close();
60
+ };
61
+
62
+ /// \brief A basic implementation of FlightDataStream that will provide
63
+ /// a sequence of FlightData messages to be written to a stream
64
+ class ARROW_FLIGHT_EXPORT RecordBatchStream : public FlightDataStream {
65
+ public:
66
+ /// \param[in] reader produces a sequence of record batches
67
+ /// \param[in] options IPC options for writing
68
+ explicit RecordBatchStream(
69
+ const std::shared_ptr<RecordBatchReader>& reader,
70
+ const ipc::IpcWriteOptions& options = ipc::IpcWriteOptions::Defaults());
71
+ ~RecordBatchStream() override;
72
+
73
+ // inherit deprecated API
74
+ using FlightDataStream::GetSchemaPayload;
75
+ using FlightDataStream::Next;
76
+
77
+ std::shared_ptr<Schema> schema() override;
78
+ arrow::Result<FlightPayload> GetSchemaPayload() override;
79
+
80
+ arrow::Result<FlightPayload> Next() override;
81
+ Status Close() override;
82
+
83
+ private:
84
+ class RecordBatchStreamImpl;
85
+ std::unique_ptr<RecordBatchStreamImpl> impl_;
86
+ };
87
+
88
+ /// \brief A reader for IPC payloads uploaded by a client. Also allows
89
+ /// reading application-defined metadata via the Flight protocol.
90
+ class ARROW_FLIGHT_EXPORT FlightMessageReader : public MetadataRecordBatchReader {
91
+ public:
92
+ /// \brief Get the descriptor for this upload.
93
+ virtual const FlightDescriptor& descriptor() const = 0;
94
+ };
95
+
96
+ /// \brief A writer for application-specific metadata sent back to the
97
+ /// client during an upload.
98
+ class ARROW_FLIGHT_EXPORT FlightMetadataWriter {
99
+ public:
100
+ virtual ~FlightMetadataWriter();
101
+ /// \brief Send a message to the client.
102
+ virtual Status WriteMetadata(const Buffer& app_metadata) = 0;
103
+ };
104
+
105
+ /// \brief A writer for IPC payloads to a client. Also allows sending
106
+ /// application-defined metadata via the Flight protocol.
107
+ ///
108
+ /// This class offers more control compared to FlightDataStream,
109
+ /// including the option to write metadata without data and the
110
+ /// ability to interleave reading and writing.
111
+ class ARROW_FLIGHT_EXPORT FlightMessageWriter : public MetadataRecordBatchWriter {
112
+ public:
113
+ virtual ~FlightMessageWriter() = default;
114
+ };
115
+
116
+ /// \brief Call state/contextual data.
117
+ class ARROW_FLIGHT_EXPORT ServerCallContext {
118
+ public:
119
+ virtual ~ServerCallContext() = default;
120
+ /// \brief The name of the authenticated peer (may be the empty string)
121
+ virtual const std::string& peer_identity() const = 0;
122
+ /// \brief The peer address (not validated)
123
+ virtual const std::string& peer() const = 0;
124
+ /// \brief Add a response header. This is only valid before the server
125
+ /// starts sending the response; generally this isn't an issue unless you
126
+ /// are implementing FlightDataStream, ResultStream, or similar interfaces
127
+ /// yourself, or during a DoExchange or DoPut.
128
+ virtual void AddHeader(const std::string& key, const std::string& value) const = 0;
129
+ /// \brief Add a response trailer. This is only valid before the server
130
+ /// sends the final status; generally this isn't an issue unless your RPC
131
+ /// handler launches a thread or similar.
132
+ virtual void AddTrailer(const std::string& key, const std::string& value) const = 0;
133
+ /// \brief Look up a middleware by key. Do not maintain a reference
134
+ /// to the object beyond the request body.
135
+ /// \return The middleware, or nullptr if not found.
136
+ virtual ServerMiddleware* GetMiddleware(const std::string& key) const = 0;
137
+ /// \brief Check if the current RPC has been cancelled (by the client, by
138
+ /// a network error, etc.).
139
+ virtual bool is_cancelled() const = 0;
140
+ /// \brief The headers sent by the client for this call.
141
+ virtual const CallHeaders& incoming_headers() const = 0;
142
+ };
143
+
144
+ class ARROW_FLIGHT_EXPORT FlightServerOptions {
145
+ public:
146
+ explicit FlightServerOptions(const Location& location_);
147
+
148
+ ~FlightServerOptions();
149
+
150
+ /// \brief The host & port (or domain socket path) to listen on.
151
+ /// Use port 0 to bind to an available port.
152
+ Location location;
153
+ /// \brief The authentication handler to use.
154
+ std::shared_ptr<ServerAuthHandler> auth_handler;
155
+ /// \brief A list of TLS certificate+key pairs to use.
156
+ std::vector<CertKeyPair> tls_certificates;
157
+ /// \brief Enable mTLS and require that the client present a certificate.
158
+ bool verify_client;
159
+ /// \brief If using mTLS, the PEM-encoded root certificate to use.
160
+ std::string root_certificates;
161
+ /// \brief A list of server middleware to apply, along with a key to
162
+ /// identify them by.
163
+ ///
164
+ /// Middleware are always applied in the order provided. Duplicate
165
+ /// keys are an error.
166
+ std::vector<std::pair<std::string, std::shared_ptr<ServerMiddlewareFactory>>>
167
+ middleware;
168
+
169
+ /// \brief An optional memory manager to control where to allocate incoming data.
170
+ std::shared_ptr<MemoryManager> memory_manager;
171
+
172
+ /// \brief A Flight implementation-specific callback to customize
173
+ /// transport-specific options.
174
+ ///
175
+ /// Not guaranteed to be called. The type of the parameter is
176
+ /// specific to the Flight implementation. Users should take care to
177
+ /// link to the same transport implementation as Flight to avoid
178
+ /// runtime problems. See "Using Arrow C++ in your own project" in
179
+ /// the documentation for more details.
180
+ std::function<void(void*)> builder_hook;
181
+ };
182
+
183
+ /// \brief Skeleton RPC server implementation which can be used to create
184
+ /// custom servers by implementing its abstract methods
185
+ class ARROW_FLIGHT_EXPORT FlightServerBase {
186
+ public:
187
+ FlightServerBase();
188
+ virtual ~FlightServerBase();
189
+
190
+ // Lifecycle methods.
191
+
192
+ /// \brief Initialize a Flight server listening at the given location.
193
+ /// This method must be called before any other method.
194
+ /// \param[in] options The configuration for this server.
195
+ Status Init(const FlightServerOptions& options);
196
+
197
+ /// \brief Get the port that the Flight server is listening on.
198
+ /// This method must only be called after Init(). Will return a
199
+ /// non-positive value if no port exists (e.g. when listening on a
200
+ /// domain socket).
201
+ int port() const;
202
+
203
+ /// \brief Get the address that the Flight server is listening on.
204
+ /// This method must only be called after Init().
205
+ Location location() const;
206
+
207
+ /// \brief Set the server to stop when receiving any of the given signal
208
+ /// numbers.
209
+ /// This method must be called before Serve().
210
+ Status SetShutdownOnSignals(const std::vector<int> sigs);
211
+
212
+ /// \brief Start serving.
213
+ /// This method blocks until the server shuts down.
214
+ ///
215
+ /// The server will start to shut down when either Shutdown() is called
216
+ /// or one of the signals registered in SetShutdownOnSignals() is received.
217
+ Status Serve();
218
+
219
+ /// \brief Query whether Serve() was interrupted by a signal.
220
+ /// This method must be called after Serve() has returned.
221
+ ///
222
+ /// \return int the signal number that interrupted Serve(), if any, otherwise 0
223
+ int GotSignal() const;
224
+
225
+ /// \brief Shut down the server, blocking until current requests finish.
226
+ ///
227
+ /// Can be called from a signal handler or another thread while Serve()
228
+ /// blocks. Optionally a deadline can be set. Once the deadline expires
229
+ /// server will wait until remaining running calls complete.
230
+ ///
231
+ /// Should only be called once.
232
+ Status Shutdown(const std::chrono::system_clock::time_point* deadline = NULLPTR);
233
+
234
+ /// \brief Block until server shuts down with Shutdown.
235
+ ///
236
+ /// Does not respond to signals like Serve().
237
+ Status Wait();
238
+
239
+ // Implement these methods to create your own server. The default
240
+ // implementations will return a not-implemented result to the client
241
+
242
+ /// \brief Retrieve a list of available fields given an optional opaque
243
+ /// criteria
244
+ /// \param[in] context The call context.
245
+ /// \param[in] criteria may be null
246
+ /// \param[out] listings the returned listings iterator
247
+ /// \return Status
248
+ virtual Status ListFlights(const ServerCallContext& context, const Criteria* criteria,
249
+ std::unique_ptr<FlightListing>* listings);
250
+
251
+ /// \brief Retrieve the schema and an access plan for the indicated
252
+ /// descriptor
253
+ /// \param[in] context The call context.
254
+ /// \param[in] request the dataset request, whether a named dataset or command
255
+ /// \param[out] info the returned flight info provider
256
+ /// \return Status
257
+ virtual Status GetFlightInfo(const ServerCallContext& context,
258
+ const FlightDescriptor& request,
259
+ std::unique_ptr<FlightInfo>* info);
260
+
261
+ /// \brief Retrieve the current status of the target query
262
+ /// \param[in] context The call context.
263
+ /// \param[in] request the dataset request or a descriptor returned by a
264
+ /// prior PollFlightInfo call
265
+ /// \param[out] info the returned retry info provider
266
+ /// \return Status
267
+ virtual Status PollFlightInfo(const ServerCallContext& context,
268
+ const FlightDescriptor& request,
269
+ std::unique_ptr<PollInfo>* info);
270
+
271
+ /// \brief Retrieve the schema for the indicated descriptor
272
+ /// \param[in] context The call context.
273
+ /// \param[in] request the dataset request, whether a named dataset or command
274
+ /// \param[out] schema the returned flight schema provider
275
+ /// \return Status
276
+ virtual Status GetSchema(const ServerCallContext& context,
277
+ const FlightDescriptor& request,
278
+ std::unique_ptr<SchemaResult>* schema);
279
+
280
+ /// \brief Get a stream of IPC payloads to put on the wire
281
+ /// \param[in] context The call context.
282
+ /// \param[in] request an opaque ticket
283
+ /// \param[out] stream the returned stream provider
284
+ /// \return Status
285
+ virtual Status DoGet(const ServerCallContext& context, const Ticket& request,
286
+ std::unique_ptr<FlightDataStream>* stream);
287
+
288
+ /// \brief Process a stream of IPC payloads sent from a client
289
+ /// \param[in] context The call context.
290
+ /// \param[in] reader a sequence of uploaded record batches
291
+ /// \param[in] writer send metadata back to the client
292
+ /// \return Status
293
+ virtual Status DoPut(const ServerCallContext& context,
294
+ std::unique_ptr<FlightMessageReader> reader,
295
+ std::unique_ptr<FlightMetadataWriter> writer);
296
+
297
+ /// \brief Process a bidirectional stream of IPC payloads
298
+ /// \param[in] context The call context.
299
+ /// \param[in] reader a sequence of uploaded record batches
300
+ /// \param[in] writer send data back to the client
301
+ /// \return Status
302
+ virtual Status DoExchange(const ServerCallContext& context,
303
+ std::unique_ptr<FlightMessageReader> reader,
304
+ std::unique_ptr<FlightMessageWriter> writer);
305
+
306
+ /// \brief Execute an action, return stream of zero or more results
307
+ /// \param[in] context The call context.
308
+ /// \param[in] action the action to execute, with type and body
309
+ /// \param[out] result the result iterator
310
+ /// \return Status
311
+ virtual Status DoAction(const ServerCallContext& context, const Action& action,
312
+ std::unique_ptr<ResultStream>* result);
313
+
314
+ /// \brief Retrieve the list of available actions
315
+ /// \param[in] context The call context.
316
+ /// \param[out] actions a vector of available action types
317
+ /// \return Status
318
+ virtual Status ListActions(const ServerCallContext& context,
319
+ std::vector<ActionType>* actions);
320
+
321
+ private:
322
+ struct Impl;
323
+ std::unique_ptr<Impl> impl_;
324
+ };
325
+
326
+ } // namespace flight
327
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server_auth.h ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ /// \brief Server-side APIs to implement authentication for Flight.
19
+
20
+ #pragma once
21
+
22
+ #include <string>
23
+
24
+ #include "arrow/flight/type_fwd.h"
25
+ #include "arrow/flight/visibility.h"
26
+ #include "arrow/status.h"
27
+
28
+ namespace arrow {
29
+
30
+ namespace flight {
31
+
32
+ /// \brief A reader for messages from the client during an
33
+ /// authentication handshake.
34
+ class ARROW_FLIGHT_EXPORT ServerAuthReader {
35
+ public:
36
+ virtual ~ServerAuthReader() = default;
37
+ virtual Status Read(std::string* token) = 0;
38
+ };
39
+
40
+ /// \brief A writer for messages to the client during an
41
+ /// authentication handshake.
42
+ class ARROW_FLIGHT_EXPORT ServerAuthSender {
43
+ public:
44
+ virtual ~ServerAuthSender() = default;
45
+ virtual Status Write(const std::string& message) = 0;
46
+ };
47
+
48
+ /// \brief An authentication implementation for a Flight service.
49
+ /// Authentication includes both an initial negotiation and a per-call
50
+ /// token validation. Implementations may choose to use either or both
51
+ /// mechanisms.
52
+ /// An implementation may need to track some state, e.g. a mapping of
53
+ /// client tokens to authenticated identities.
54
+ class ARROW_FLIGHT_EXPORT ServerAuthHandler {
55
+ public:
56
+ virtual ~ServerAuthHandler();
57
+ /// \brief Authenticate the client on initial connection. The server
58
+ /// can send and read responses from the client at any time.
59
+ /// \param[in] context The call context.
60
+ /// \param[in] outgoing The writer for messages to the client.
61
+ /// \param[in] incoming The reader for messages from the client.
62
+ /// \return Status OK if this authentication is succeeded.
63
+ virtual Status Authenticate(const ServerCallContext& context,
64
+ ServerAuthSender* outgoing, ServerAuthReader* incoming) = 0;
65
+ /// \brief Validate a per-call client token.
66
+ /// \param[in] context The call context.
67
+ /// \param[in] token The client token. May be the empty string if
68
+ /// the client does not provide a token.
69
+ /// \param[out] peer_identity The identity of the peer, if this
70
+ /// authentication method supports it.
71
+ /// \return Status OK if the token is valid, any other status if
72
+ /// validation failed
73
+ virtual Status IsValid(const ServerCallContext& context, const std::string& token,
74
+ std::string* peer_identity) {
75
+ // TODO: We can make this pure virtual function when we remove
76
+ // the deprecated version.
77
+ ARROW_SUPPRESS_DEPRECATION_WARNING
78
+ return IsValid(token, peer_identity);
79
+ ARROW_UNSUPPRESS_DEPRECATION_WARNING
80
+ }
81
+ /// \brief Validate a per-call client token.
82
+ /// \param[in] token The client token. May be the empty string if
83
+ /// the client does not provide a token.
84
+ /// \param[out] peer_identity The identity of the peer, if this
85
+ /// authentication method supports it.
86
+ /// \return Status OK if the token is valid, any other status if
87
+ /// validation failed
88
+ /// \deprecated Deprecated in 13.0.0. Implement the IsValid()
89
+ /// with ServerCallContext version instead.
90
+ ARROW_DEPRECATED("Deprecated in 13.0.0. Use ServerCallContext overload instead.")
91
+ virtual Status IsValid(const std::string& token, std::string* peer_identity) {
92
+ return Status::NotImplemented(typeid(this).name(), "::IsValid() isn't implemented");
93
+ }
94
+ };
95
+
96
+ /// \brief An authentication mechanism that does nothing.
97
+ class ARROW_FLIGHT_EXPORT NoOpAuthHandler : public ServerAuthHandler {
98
+ public:
99
+ ~NoOpAuthHandler() override;
100
+ Status Authenticate(const ServerCallContext& context, ServerAuthSender* outgoing,
101
+ ServerAuthReader* incoming) override;
102
+ Status IsValid(const ServerCallContext& context, const std::string& token,
103
+ std::string* peer_identity) override;
104
+ };
105
+
106
+ } // namespace flight
107
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server_middleware.h ADDED
@@ -0,0 +1,82 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ // Interfaces for defining middleware for Flight servers.
19
+
20
+ #pragma once
21
+
22
+ #include <memory>
23
+ #include <string>
24
+
25
+ #include "arrow/flight/middleware.h"
26
+ #include "arrow/flight/type_fwd.h"
27
+ #include "arrow/flight/visibility.h" // IWYU pragma: keep
28
+ #include "arrow/status.h"
29
+
30
+ namespace arrow {
31
+ namespace flight {
32
+
33
+ /// \brief Server-side middleware for a call, instantiated per RPC.
34
+ ///
35
+ /// Middleware should be fast and must be infallible: there is no way
36
+ /// to reject the call or report errors from the middleware instance.
37
+ class ARROW_FLIGHT_EXPORT ServerMiddleware {
38
+ public:
39
+ virtual ~ServerMiddleware() = default;
40
+
41
+ /// \brief Unique name of middleware, used as alternative to RTTI
42
+ /// \return the string name of the middleware
43
+ virtual std::string name() const = 0;
44
+
45
+ /// \brief A callback before headers are sent. Extra headers can be
46
+ /// added, but existing ones cannot be read.
47
+ virtual void SendingHeaders(AddCallHeaders* outgoing_headers) = 0;
48
+
49
+ /// \brief A callback after the call has completed.
50
+ virtual void CallCompleted(const Status& status) = 0;
51
+ };
52
+
53
+ /// \brief A factory for new middleware instances.
54
+ ///
55
+ /// If added to a server, this will be called for each RPC (including
56
+ /// Handshake) to give the opportunity to intercept the call.
57
+ ///
58
+ /// It is guaranteed that all server middleware methods are called
59
+ /// from the same thread that calls the RPC method implementation.
60
+ class ARROW_FLIGHT_EXPORT ServerMiddlewareFactory {
61
+ public:
62
+ virtual ~ServerMiddlewareFactory() = default;
63
+
64
+ /// \brief A callback for the start of a new call.
65
+ ///
66
+ /// Return a non-OK status to reject the call with the given status.
67
+ ///
68
+ /// \param[in] info Information about the call.
69
+ /// \param[in] context The call context.
70
+ /// \param[out] middleware The middleware instance for this call. If
71
+ /// null, no middleware will be added to this call instance from
72
+ /// this factory.
73
+ /// \return Status A non-OK status will reject the call with the
74
+ /// given status. Middleware previously in the chain will have
75
+ /// their CallCompleted callback called. Other middleware
76
+ /// factories will not be called.
77
+ virtual Status StartCall(const CallInfo& info, const ServerCallContext& context,
78
+ std::shared_ptr<ServerMiddleware>* middleware) = 0;
79
+ };
80
+
81
+ } // namespace flight
82
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server_tracing_middleware.h ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ // Middleware implementation for propagating OpenTelemetry spans.
19
+
20
+ #pragma once
21
+
22
+ #include <memory>
23
+ #include <string>
24
+ #include <vector>
25
+
26
+ #include "arrow/flight/server_middleware.h"
27
+ #include "arrow/flight/visibility.h"
28
+ #include "arrow/status.h"
29
+
30
+ namespace arrow {
31
+ namespace flight {
32
+
33
+ /// \brief Returns a ServerMiddlewareFactory that handles receiving OpenTelemetry spans.
34
+ ARROW_FLIGHT_EXPORT std::shared_ptr<ServerMiddlewareFactory>
35
+ MakeTracingServerMiddlewareFactory();
36
+
37
+ /// \brief A server middleware that provides access to the
38
+ /// OpenTelemetry context, if present.
39
+ ///
40
+ /// Used to make the OpenTelemetry span available in Python.
41
+ class ARROW_FLIGHT_EXPORT TracingServerMiddleware : public ServerMiddleware {
42
+ public:
43
+ ~TracingServerMiddleware();
44
+
45
+ static constexpr const char kMiddlewareName[] =
46
+ "arrow::flight::TracingServerMiddleware";
47
+
48
+ std::string name() const override { return kMiddlewareName; }
49
+ void SendingHeaders(AddCallHeaders*) override;
50
+ void CallCompleted(const Status&) override;
51
+
52
+ struct TraceKey {
53
+ std::string key;
54
+ std::string value;
55
+ };
56
+ /// \brief Get the trace context.
57
+ std::vector<TraceKey> GetTraceContext() const;
58
+
59
+ private:
60
+ class Impl;
61
+ friend class TracingServerMiddlewareFactory;
62
+
63
+ explicit TracingServerMiddleware(std::unique_ptr<Impl> impl);
64
+ std::unique_ptr<Impl> impl_;
65
+ };
66
+
67
+ } // namespace flight
68
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_auth_handlers.h ADDED
@@ -0,0 +1,89 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <string>
21
+
22
+ #include "arrow/flight/client_auth.h"
23
+ #include "arrow/flight/server.h"
24
+ #include "arrow/flight/server_auth.h"
25
+ #include "arrow/flight/types.h"
26
+ #include "arrow/flight/visibility.h"
27
+ #include "arrow/status.h"
28
+
29
+ // A pair of authentication handlers that check for a predefined password
30
+ // and set the peer identity to a predefined username.
31
+
32
+ namespace arrow::flight {
33
+
34
+ class ARROW_FLIGHT_EXPORT TestServerAuthHandler : public ServerAuthHandler {
35
+ public:
36
+ explicit TestServerAuthHandler(const std::string& username,
37
+ const std::string& password);
38
+ ~TestServerAuthHandler() override;
39
+ Status Authenticate(const ServerCallContext& context, ServerAuthSender* outgoing,
40
+ ServerAuthReader* incoming) override;
41
+ Status IsValid(const ServerCallContext& context, const std::string& token,
42
+ std::string* peer_identity) override;
43
+
44
+ private:
45
+ std::string username_;
46
+ std::string password_;
47
+ };
48
+
49
+ class ARROW_FLIGHT_EXPORT TestServerBasicAuthHandler : public ServerAuthHandler {
50
+ public:
51
+ explicit TestServerBasicAuthHandler(const std::string& username,
52
+ const std::string& password);
53
+ ~TestServerBasicAuthHandler() override;
54
+ Status Authenticate(const ServerCallContext& context, ServerAuthSender* outgoing,
55
+ ServerAuthReader* incoming) override;
56
+ Status IsValid(const ServerCallContext& context, const std::string& token,
57
+ std::string* peer_identity) override;
58
+
59
+ private:
60
+ BasicAuth basic_auth_;
61
+ };
62
+
63
+ class ARROW_FLIGHT_EXPORT TestClientAuthHandler : public ClientAuthHandler {
64
+ public:
65
+ explicit TestClientAuthHandler(const std::string& username,
66
+ const std::string& password);
67
+ ~TestClientAuthHandler() override;
68
+ Status Authenticate(ClientAuthSender* outgoing, ClientAuthReader* incoming) override;
69
+ Status GetToken(std::string* token) override;
70
+
71
+ private:
72
+ std::string username_;
73
+ std::string password_;
74
+ };
75
+
76
+ class ARROW_FLIGHT_EXPORT TestClientBasicAuthHandler : public ClientAuthHandler {
77
+ public:
78
+ explicit TestClientBasicAuthHandler(const std::string& username,
79
+ const std::string& password);
80
+ ~TestClientBasicAuthHandler() override;
81
+ Status Authenticate(ClientAuthSender* outgoing, ClientAuthReader* incoming) override;
82
+ Status GetToken(std::string* token) override;
83
+
84
+ private:
85
+ BasicAuth basic_auth_;
86
+ std::string token_;
87
+ };
88
+
89
+ } // namespace arrow::flight
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_definitions.h ADDED
@@ -0,0 +1,318 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ // Common test definitions for Flight. Individual transport
19
+ // implementations can instantiate these tests.
20
+ //
21
+ // While Googletest's value-parameterized tests would be a more
22
+ // natural way to do this, they cause runtime issues on MinGW/MSVC
23
+ // (Googletest thinks the test suite has been defined twice).
24
+
25
+ #pragma once
26
+
27
+ #include <functional>
28
+ #include <memory>
29
+ #include <string>
30
+ #include <type_traits>
31
+ #include <vector>
32
+
33
+ #include "arrow/flight/server.h"
34
+ #include "arrow/flight/types.h"
35
+ #include "arrow/util/macros.h"
36
+
37
+ namespace arrow {
38
+ namespace flight {
39
+
40
+ class ARROW_FLIGHT_EXPORT FlightTest {
41
+ protected:
42
+ virtual std::string transport() const = 0;
43
+ virtual bool supports_async() const { return false; }
44
+ virtual void SetUpTest() {}
45
+ virtual void TearDownTest() {}
46
+ };
47
+
48
+ /// Common tests of startup/shutdown
49
+ class ARROW_FLIGHT_EXPORT ConnectivityTest : public FlightTest {
50
+ public:
51
+ // Test methods
52
+ void TestGetPort();
53
+ void TestBuilderHook();
54
+ void TestShutdown();
55
+ void TestShutdownWithDeadline();
56
+ void TestBrokenConnection();
57
+ };
58
+
59
+ #define ARROW_FLIGHT_TEST_CONNECTIVITY(FIXTURE) \
60
+ static_assert(std::is_base_of<ConnectivityTest, FIXTURE>::value, \
61
+ ARROW_STRINGIFY(FIXTURE) " must inherit from ConnectivityTest"); \
62
+ TEST_F(FIXTURE, GetPort) { TestGetPort(); } \
63
+ TEST_F(FIXTURE, BuilderHook) { TestBuilderHook(); } \
64
+ TEST_F(FIXTURE, Shutdown) { TestShutdown(); } \
65
+ TEST_F(FIXTURE, ShutdownWithDeadline) { TestShutdownWithDeadline(); } \
66
+ TEST_F(FIXTURE, BrokenConnection) { TestBrokenConnection(); }
67
+
68
+ /// Common tests of data plane methods
69
+ class ARROW_FLIGHT_EXPORT DataTest : public FlightTest {
70
+ public:
71
+ void SetUpTest() override;
72
+ void TearDownTest() override;
73
+ Status ConnectClient();
74
+
75
+ // Test methods
76
+ void TestDoGetInts();
77
+ void TestDoGetFloats();
78
+ void TestDoGetDicts();
79
+ void TestDoGetLargeBatch();
80
+ void TestFlightDataStreamError();
81
+ void TestOverflowServerBatch();
82
+ void TestOverflowClientBatch();
83
+ void TestDoExchange();
84
+ void TestDoExchangeNoData();
85
+ void TestDoExchangeWriteOnlySchema();
86
+ void TestDoExchangeGet();
87
+ void TestDoExchangePut();
88
+ void TestDoExchangeEcho();
89
+ void TestDoExchangeTotal();
90
+ void TestDoExchangeError();
91
+ void TestDoExchangeConcurrency();
92
+ void TestDoExchangeUndrained();
93
+ void TestIssue5095();
94
+
95
+ private:
96
+ void CheckDoGet(
97
+ const FlightDescriptor& descr, const RecordBatchVector& expected_batches,
98
+ std::function<void(const std::vector<FlightEndpoint>&)> check_endpoints);
99
+ void CheckDoGet(const Ticket& ticket, const RecordBatchVector& expected_batches);
100
+
101
+ std::unique_ptr<FlightClient> client_;
102
+ std::unique_ptr<FlightServerBase> server_;
103
+ };
104
+
105
+ #define ARROW_FLIGHT_TEST_DATA(FIXTURE) \
106
+ static_assert(std::is_base_of<DataTest, FIXTURE>::value, \
107
+ ARROW_STRINGIFY(FIXTURE) " must inherit from DataTest"); \
108
+ TEST_F(FIXTURE, TestDoGetInts) { TestDoGetInts(); } \
109
+ TEST_F(FIXTURE, TestDoGetFloats) { TestDoGetFloats(); } \
110
+ TEST_F(FIXTURE, TestDoGetDicts) { TestDoGetDicts(); } \
111
+ TEST_F(FIXTURE, TestDoGetLargeBatch) { TestDoGetLargeBatch(); } \
112
+ TEST_F(FIXTURE, TestFlightDataStreamError) { TestFlightDataStreamError(); } \
113
+ TEST_F(FIXTURE, TestOverflowServerBatch) { TestOverflowServerBatch(); } \
114
+ TEST_F(FIXTURE, TestOverflowClientBatch) { TestOverflowClientBatch(); } \
115
+ TEST_F(FIXTURE, TestDoExchange) { TestDoExchange(); } \
116
+ TEST_F(FIXTURE, TestDoExchangeNoData) { TestDoExchangeNoData(); } \
117
+ TEST_F(FIXTURE, TestDoExchangeWriteOnlySchema) { TestDoExchangeWriteOnlySchema(); } \
118
+ TEST_F(FIXTURE, TestDoExchangeGet) { TestDoExchangeGet(); } \
119
+ TEST_F(FIXTURE, TestDoExchangePut) { TestDoExchangePut(); } \
120
+ TEST_F(FIXTURE, TestDoExchangeEcho) { TestDoExchangeEcho(); } \
121
+ TEST_F(FIXTURE, TestDoExchangeTotal) { TestDoExchangeTotal(); } \
122
+ TEST_F(FIXTURE, TestDoExchangeError) { TestDoExchangeError(); } \
123
+ TEST_F(FIXTURE, TestDoExchangeConcurrency) { TestDoExchangeConcurrency(); } \
124
+ TEST_F(FIXTURE, TestDoExchangeUndrained) { TestDoExchangeUndrained(); } \
125
+ TEST_F(FIXTURE, TestIssue5095) { TestIssue5095(); }
126
+
127
+ /// \brief Specific tests of DoPut.
128
+ class ARROW_FLIGHT_EXPORT DoPutTest : public FlightTest {
129
+ public:
130
+ void SetUpTest() override;
131
+ void TearDownTest() override;
132
+ void CheckBatches(const FlightDescriptor& expected_descriptor,
133
+ const RecordBatchVector& expected_batches);
134
+ void CheckDoPut(const FlightDescriptor& descr, const std::shared_ptr<Schema>& schema,
135
+ const RecordBatchVector& batches);
136
+
137
+ // Test methods
138
+ void TestInts();
139
+ void TestFloats();
140
+ void TestEmptyBatch();
141
+ void TestDicts();
142
+ void TestLargeBatch();
143
+ void TestSizeLimit();
144
+ void TestUndrained();
145
+
146
+ private:
147
+ std::unique_ptr<FlightClient> client_;
148
+ std::unique_ptr<FlightServerBase> server_;
149
+ };
150
+
151
+ #define ARROW_FLIGHT_TEST_DO_PUT(FIXTURE) \
152
+ static_assert(std::is_base_of<DoPutTest, FIXTURE>::value, \
153
+ ARROW_STRINGIFY(FIXTURE) " must inherit from DoPutTest"); \
154
+ TEST_F(FIXTURE, TestInts) { TestInts(); } \
155
+ TEST_F(FIXTURE, TestFloats) { TestFloats(); } \
156
+ TEST_F(FIXTURE, TestEmptyBatch) { TestEmptyBatch(); } \
157
+ TEST_F(FIXTURE, TestDicts) { TestDicts(); } \
158
+ TEST_F(FIXTURE, TestLargeBatch) { TestLargeBatch(); } \
159
+ TEST_F(FIXTURE, TestSizeLimit) { TestSizeLimit(); } \
160
+ TEST_F(FIXTURE, TestUndrained) { TestUndrained(); }
161
+
162
+ class ARROW_FLIGHT_EXPORT AppMetadataTestServer : public FlightServerBase {
163
+ public:
164
+ virtual ~AppMetadataTestServer() = default;
165
+
166
+ Status DoGet(const ServerCallContext& context, const Ticket& request,
167
+ std::unique_ptr<FlightDataStream>* data_stream) override;
168
+
169
+ Status DoPut(const ServerCallContext& context,
170
+ std::unique_ptr<FlightMessageReader> reader,
171
+ std::unique_ptr<FlightMetadataWriter> writer) override;
172
+ };
173
+
174
+ /// \brief Tests of app_metadata in data plane methods.
175
+ class ARROW_FLIGHT_EXPORT AppMetadataTest : public FlightTest {
176
+ public:
177
+ void SetUpTest() override;
178
+ void TearDownTest() override;
179
+
180
+ // Test methods
181
+ void TestDoGet();
182
+ void TestDoGetDictionaries();
183
+ void TestDoPut();
184
+ void TestDoPutDictionaries();
185
+ void TestDoPutReadMetadata();
186
+
187
+ private:
188
+ std::unique_ptr<FlightClient> client_;
189
+ std::unique_ptr<FlightServerBase> server_;
190
+ };
191
+
192
+ #define ARROW_FLIGHT_TEST_APP_METADATA(FIXTURE) \
193
+ static_assert(std::is_base_of<AppMetadataTest, FIXTURE>::value, \
194
+ ARROW_STRINGIFY(FIXTURE) " must inherit from AppMetadataTest"); \
195
+ TEST_F(FIXTURE, TestDoGet) { TestDoGet(); } \
196
+ TEST_F(FIXTURE, TestDoGetDictionaries) { TestDoGetDictionaries(); } \
197
+ TEST_F(FIXTURE, TestDoPut) { TestDoPut(); } \
198
+ TEST_F(FIXTURE, TestDoPutDictionaries) { TestDoPutDictionaries(); } \
199
+ TEST_F(FIXTURE, TestDoPutReadMetadata) { TestDoPutReadMetadata(); }
200
+
201
+ /// \brief Tests of IPC options in data plane methods.
202
+ class ARROW_FLIGHT_EXPORT IpcOptionsTest : public FlightTest {
203
+ public:
204
+ void SetUpTest() override;
205
+ void TearDownTest() override;
206
+
207
+ // Test methods
208
+ void TestDoGetReadOptions();
209
+ void TestDoPutWriteOptions();
210
+ void TestDoExchangeClientWriteOptions();
211
+ void TestDoExchangeClientWriteOptionsBegin();
212
+ void TestDoExchangeServerWriteOptions();
213
+
214
+ private:
215
+ std::unique_ptr<FlightClient> client_;
216
+ std::unique_ptr<FlightServerBase> server_;
217
+ };
218
+
219
+ #define ARROW_FLIGHT_TEST_IPC_OPTIONS(FIXTURE) \
220
+ static_assert(std::is_base_of<IpcOptionsTest, FIXTURE>::value, \
221
+ ARROW_STRINGIFY(FIXTURE) " must inherit from IpcOptionsTest"); \
222
+ TEST_F(FIXTURE, TestDoGetReadOptions) { TestDoGetReadOptions(); } \
223
+ TEST_F(FIXTURE, TestDoPutWriteOptions) { TestDoPutWriteOptions(); } \
224
+ TEST_F(FIXTURE, TestDoExchangeClientWriteOptions) { \
225
+ TestDoExchangeClientWriteOptions(); \
226
+ } \
227
+ TEST_F(FIXTURE, TestDoExchangeClientWriteOptionsBegin) { \
228
+ TestDoExchangeClientWriteOptionsBegin(); \
229
+ } \
230
+ TEST_F(FIXTURE, TestDoExchangeServerWriteOptions) { \
231
+ TestDoExchangeServerWriteOptions(); \
232
+ }
233
+
234
+ /// \brief Tests of data plane methods with CUDA memory.
235
+ ///
236
+ /// If not built with ARROW_CUDA, tests are no-ops.
237
+ class ARROW_FLIGHT_EXPORT CudaDataTest : public FlightTest {
238
+ public:
239
+ void SetUpTest() override;
240
+ void TearDownTest() override;
241
+
242
+ // Test methods
243
+ void TestDoGet();
244
+ void TestDoPut();
245
+ void TestDoExchange();
246
+
247
+ private:
248
+ class Impl;
249
+ std::unique_ptr<FlightClient> client_;
250
+ std::unique_ptr<FlightServerBase> server_;
251
+ std::shared_ptr<Impl> impl_;
252
+ };
253
+
254
+ #define ARROW_FLIGHT_TEST_CUDA_DATA(FIXTURE) \
255
+ static_assert(std::is_base_of<CudaDataTest, FIXTURE>::value, \
256
+ ARROW_STRINGIFY(FIXTURE) " must inherit from CudaDataTest"); \
257
+ TEST_F(FIXTURE, TestDoGet) { TestDoGet(); } \
258
+ TEST_F(FIXTURE, TestDoPut) { TestDoPut(); } \
259
+ TEST_F(FIXTURE, TestDoExchange) { TestDoExchange(); }
260
+
261
+ /// \brief Tests of error handling.
262
+ class ARROW_FLIGHT_EXPORT ErrorHandlingTest : public FlightTest {
263
+ public:
264
+ void SetUpTest() override;
265
+ void TearDownTest() override;
266
+
267
+ // Test methods
268
+ void TestGetFlightInfo();
269
+ void TestGetFlightInfoMetadata();
270
+ void TestAsyncGetFlightInfo();
271
+ void TestDoPut();
272
+ void TestDoExchange();
273
+
274
+ protected:
275
+ struct Impl;
276
+
277
+ std::vector<std::pair<std::string, std::string>> GetHeaders();
278
+
279
+ std::shared_ptr<Impl> impl_;
280
+ std::unique_ptr<FlightClient> client_;
281
+ std::unique_ptr<FlightServerBase> server_;
282
+ };
283
+
284
+ #define ARROW_FLIGHT_TEST_ERROR_HANDLING(FIXTURE) \
285
+ static_assert(std::is_base_of<ErrorHandlingTest, FIXTURE>::value, \
286
+ ARROW_STRINGIFY(FIXTURE) " must inherit from ErrorHandlingTest"); \
287
+ TEST_F(FIXTURE, TestAsyncGetFlightInfo) { TestAsyncGetFlightInfo(); } \
288
+ TEST_F(FIXTURE, TestGetFlightInfo) { TestGetFlightInfo(); } \
289
+ TEST_F(FIXTURE, TestGetFlightInfoMetadata) { TestGetFlightInfoMetadata(); } \
290
+ TEST_F(FIXTURE, TestDoPut) { TestDoPut(); } \
291
+ TEST_F(FIXTURE, TestDoExchange) { TestDoExchange(); }
292
+
293
+ /// \brief Tests of the async client.
294
+ class ARROW_FLIGHT_EXPORT AsyncClientTest : public FlightTest {
295
+ public:
296
+ void SetUpTest() override;
297
+ void TearDownTest() override;
298
+
299
+ // Test methods
300
+ void TestGetFlightInfo();
301
+ void TestGetFlightInfoFuture();
302
+ void TestListenerLifetime();
303
+
304
+ private:
305
+ std::unique_ptr<FlightClient> client_;
306
+ std::unique_ptr<FlightServerBase> server_;
307
+ };
308
+
309
+ // DISABLED TestListenerLifetime: https://github.com/apache/arrow/issues/45120
310
+ #define ARROW_FLIGHT_TEST_ASYNC_CLIENT(FIXTURE) \
311
+ static_assert(std::is_base_of<AsyncClientTest, FIXTURE>::value, \
312
+ ARROW_STRINGIFY(FIXTURE) " must inherit from AsyncClientTest"); \
313
+ TEST_F(FIXTURE, TestGetFlightInfo) { TestGetFlightInfo(); } \
314
+ TEST_F(FIXTURE, TestGetFlightInfoFuture) { TestGetFlightInfoFuture(); } \
315
+ TEST_F(FIXTURE, DISABLED_TestListenerLifetime) { TestListenerLifetime(); }
316
+
317
+ } // namespace flight
318
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_flight_server.h ADDED
@@ -0,0 +1,92 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <memory>
21
+
22
+ #include "arrow/flight/server.h"
23
+ #include "arrow/flight/type_fwd.h"
24
+ #include "arrow/flight/visibility.h"
25
+ #include "arrow/status.h"
26
+
27
+ namespace arrow::flight {
28
+
29
+ class ARROW_FLIGHT_EXPORT TestFlightServer : public FlightServerBase {
30
+ public:
31
+ static std::unique_ptr<FlightServerBase> Make();
32
+
33
+ Status ListFlights(const ServerCallContext& context, const Criteria* criteria,
34
+ std::unique_ptr<FlightListing>* listings) override;
35
+
36
+ Status GetFlightInfo(const ServerCallContext& context, const FlightDescriptor& request,
37
+ std::unique_ptr<FlightInfo>* out) override;
38
+
39
+ Status DoGet(const ServerCallContext& context, const Ticket& request,
40
+ std::unique_ptr<FlightDataStream>* data_stream) override;
41
+
42
+ Status DoPut(const ServerCallContext&, std::unique_ptr<FlightMessageReader> reader,
43
+ std::unique_ptr<FlightMetadataWriter> writer) override;
44
+
45
+ Status DoExchange(const ServerCallContext& context,
46
+ std::unique_ptr<FlightMessageReader> reader,
47
+ std::unique_ptr<FlightMessageWriter> writer) override;
48
+
49
+ // A simple example - act like DoGet.
50
+ Status RunExchangeGet(std::unique_ptr<FlightMessageReader> reader,
51
+ std::unique_ptr<FlightMessageWriter> writer);
52
+
53
+ // A simple example - act like DoPut
54
+ Status RunExchangePut(std::unique_ptr<FlightMessageReader> reader,
55
+ std::unique_ptr<FlightMessageWriter> writer);
56
+
57
+ // Read some number of record batches from the client, send a
58
+ // metadata message back with the count, then echo the batches back.
59
+ Status RunExchangeCounter(std::unique_ptr<FlightMessageReader> reader,
60
+ std::unique_ptr<FlightMessageWriter> writer);
61
+
62
+ // Read int64 batches from the client, each time sending back a
63
+ // batch with a running sum of columns.
64
+ Status RunExchangeTotal(std::unique_ptr<FlightMessageReader> reader,
65
+ std::unique_ptr<FlightMessageWriter> writer);
66
+
67
+ // Echo the client's messages back.
68
+ Status RunExchangeEcho(std::unique_ptr<FlightMessageReader> reader,
69
+ std::unique_ptr<FlightMessageWriter> writer);
70
+
71
+ // Regression test for ARROW-13253
72
+ Status RunExchangeLargeBatch(std::unique_ptr<FlightMessageReader>,
73
+ std::unique_ptr<FlightMessageWriter> writer);
74
+
75
+ Status RunAction1(const Action& action, std::unique_ptr<ResultStream>* out);
76
+
77
+ Status RunAction2(std::unique_ptr<ResultStream>* out);
78
+
79
+ Status ListIncomingHeaders(const ServerCallContext& context, const Action& action,
80
+ std::unique_ptr<ResultStream>* out);
81
+
82
+ Status DoAction(const ServerCallContext& context, const Action& action,
83
+ std::unique_ptr<ResultStream>* out) override;
84
+
85
+ Status ListActions(const ServerCallContext& context,
86
+ std::vector<ActionType>* out) override;
87
+
88
+ Status GetSchema(const ServerCallContext& context, const FlightDescriptor& request,
89
+ std::unique_ptr<SchemaResult>* schema) override;
90
+ };
91
+
92
+ } // namespace arrow::flight
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_util.h ADDED
@@ -0,0 +1,188 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <gtest/gtest.h>
21
+
22
+ #include <cstdint>
23
+ #include <memory>
24
+ #include <string>
25
+ #include <thread>
26
+ #include <utility>
27
+ #include <vector>
28
+
29
+ #include "arrow/status.h"
30
+ #include "arrow/testing/gtest_util.h"
31
+ #include "arrow/testing/process.h"
32
+ #include "arrow/testing/util.h"
33
+
34
+ #include "arrow/flight/client.h"
35
+ #include "arrow/flight/server.h"
36
+ #include "arrow/flight/types.h"
37
+ #include "arrow/flight/visibility.h"
38
+
39
+ namespace arrow {
40
+ namespace flight {
41
+
42
+ // ----------------------------------------------------------------------
43
+ // Helpers to compare values for equality
44
+ ARROW_FLIGHT_EXPORT
45
+ void AssertEqual(const FlightInfo& expected, const FlightInfo& actual);
46
+
47
+ // ----------------------------------------------------------------------
48
+ // Fixture to use for running test servers
49
+
50
+ class ARROW_FLIGHT_EXPORT TestServer {
51
+ public:
52
+ explicit TestServer(const std::string& executable_name)
53
+ : executable_name_(executable_name), port_(::arrow::GetListenPort()) {}
54
+ TestServer(const std::string& executable_name, int port)
55
+ : executable_name_(executable_name), port_(port) {}
56
+ TestServer(const std::string& executable_name, const std::string& unix_sock)
57
+ : executable_name_(executable_name), unix_sock_(unix_sock) {}
58
+
59
+ Status Start(const std::vector<std::string>& extra_args);
60
+ Status Start() { return Start({}); }
61
+
62
+ void Stop();
63
+
64
+ bool IsRunning();
65
+
66
+ int port() const;
67
+ const std::string& unix_sock() const;
68
+
69
+ private:
70
+ std::string executable_name_;
71
+ int port_;
72
+ std::string unix_sock_;
73
+ std::unique_ptr<util::Process> server_process_;
74
+ };
75
+
76
+ // Helper to initialize a server and matching client with callbacks to
77
+ // populate options.
78
+ template <typename T, typename... Args>
79
+ Status MakeServer(const Location& location, std::unique_ptr<FlightServerBase>* server,
80
+ std::unique_ptr<FlightClient>* client,
81
+ std::function<Status(FlightServerOptions*)> make_server_options,
82
+ std::function<Status(FlightClientOptions*)> make_client_options,
83
+ Args&&... server_args) {
84
+ *server = std::make_unique<T>(std::forward<Args>(server_args)...);
85
+ FlightServerOptions server_options(location);
86
+ RETURN_NOT_OK(make_server_options(&server_options));
87
+ RETURN_NOT_OK((*server)->Init(server_options));
88
+ std::string uri =
89
+ location.scheme() + "://127.0.0.1:" + std::to_string((*server)->port());
90
+ ARROW_ASSIGN_OR_RAISE(auto real_location, Location::Parse(uri));
91
+ FlightClientOptions client_options = FlightClientOptions::Defaults();
92
+ RETURN_NOT_OK(make_client_options(&client_options));
93
+ return FlightClient::Connect(real_location, client_options).Value(client);
94
+ }
95
+
96
+ // Helper to initialize a server and matching client with callbacks to
97
+ // populate options.
98
+ template <typename T, typename... Args>
99
+ Status MakeServer(std::unique_ptr<FlightServerBase>* server,
100
+ std::unique_ptr<FlightClient>* client,
101
+ std::function<Status(FlightServerOptions*)> make_server_options,
102
+ std::function<Status(FlightClientOptions*)> make_client_options,
103
+ Args&&... server_args) {
104
+ ARROW_ASSIGN_OR_RAISE(auto location, Location::ForGrpcTcp("localhost", 0));
105
+ return MakeServer<T>(location, server, client, std::move(make_server_options),
106
+ std::move(make_client_options),
107
+ std::forward<Args>(server_args)...);
108
+ }
109
+
110
+ // ----------------------------------------------------------------------
111
+ // A FlightDataStream that numbers the record batches
112
+ /// \brief A basic implementation of FlightDataStream that will provide
113
+ /// a sequence of FlightData messages to be written to a stream
114
+ class ARROW_FLIGHT_EXPORT NumberingStream : public FlightDataStream {
115
+ public:
116
+ explicit NumberingStream(std::unique_ptr<FlightDataStream> stream);
117
+
118
+ std::shared_ptr<Schema> schema() override;
119
+ arrow::Result<FlightPayload> GetSchemaPayload() override;
120
+ arrow::Result<FlightPayload> Next() override;
121
+
122
+ private:
123
+ int counter_;
124
+ std::shared_ptr<FlightDataStream> stream_;
125
+ };
126
+
127
+ // ----------------------------------------------------------------------
128
+ // Example data for test-server and unit tests
129
+
130
+ ARROW_FLIGHT_EXPORT
131
+ std::shared_ptr<Schema> ExampleIntSchema();
132
+
133
+ ARROW_FLIGHT_EXPORT
134
+ std::shared_ptr<Schema> ExampleFloatSchema();
135
+
136
+ ARROW_FLIGHT_EXPORT
137
+ std::shared_ptr<Schema> ExampleStringSchema();
138
+
139
+ ARROW_FLIGHT_EXPORT
140
+ std::shared_ptr<Schema> ExampleDictSchema();
141
+
142
+ ARROW_FLIGHT_EXPORT
143
+ std::shared_ptr<Schema> ExampleLargeSchema();
144
+
145
+ ARROW_FLIGHT_EXPORT
146
+ Status ExampleIntBatches(RecordBatchVector* out);
147
+
148
+ ARROW_FLIGHT_EXPORT
149
+ Status ExampleFloatBatches(RecordBatchVector* out);
150
+
151
+ ARROW_FLIGHT_EXPORT
152
+ Status ExampleDictBatches(RecordBatchVector* out);
153
+
154
+ ARROW_FLIGHT_EXPORT
155
+ Status ExampleNestedBatches(RecordBatchVector* out);
156
+
157
+ ARROW_FLIGHT_EXPORT
158
+ Status ExampleLargeBatches(RecordBatchVector* out);
159
+
160
+ ARROW_FLIGHT_EXPORT
161
+ arrow::Result<std::shared_ptr<RecordBatch>> VeryLargeBatch();
162
+
163
+ ARROW_FLIGHT_EXPORT
164
+ std::vector<FlightInfo> ExampleFlightInfo();
165
+
166
+ ARROW_FLIGHT_EXPORT
167
+ std::vector<ActionType> ExampleActionTypes();
168
+
169
+ ARROW_FLIGHT_EXPORT
170
+ FlightInfo MakeFlightInfo(const Schema& schema, const FlightDescriptor& descriptor,
171
+ const std::vector<FlightEndpoint>& endpoints,
172
+ int64_t total_records, int64_t total_bytes, bool ordered,
173
+ std::string app_metadata);
174
+
175
+ ARROW_FLIGHT_EXPORT
176
+ FlightInfo MakeFlightInfo(const FlightDescriptor& descriptor,
177
+ const std::vector<FlightEndpoint>& endpoints,
178
+ int64_t total_records, int64_t total_bytes, bool ordered,
179
+ std::string app_metadata);
180
+
181
+ ARROW_FLIGHT_EXPORT
182
+ Status ExampleTlsCertificates(std::vector<CertKeyPair>* out);
183
+
184
+ ARROW_FLIGHT_EXPORT
185
+ Status ExampleTlsCertificateRoot(CertKeyPair* out);
186
+
187
+ } // namespace flight
188
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/transport.h ADDED
@@ -0,0 +1,298 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ /// \file
19
+ /// Internal (but not private) interface for implementing
20
+ /// alternate network transports in Flight.
21
+ ///
22
+ /// To implement a transport, implement ServerTransport and
23
+ /// ClientTransport, and register the desired URI schemes with
24
+ /// TransportRegistry. Flight takes care of most of the per-RPC
25
+ /// details; transports only handle connections and providing a I/O
26
+ /// stream implementation (TransportDataStream).
27
+ ///
28
+ /// On the server side:
29
+ ///
30
+ /// 1. Applications subclass FlightServerBase and override RPC handlers.
31
+ /// 2. FlightServerBase::Init will look up and create a ServerTransport
32
+ /// based on the scheme of the Location given to it.
33
+ /// 3. The ServerTransport will start the actual server. (For instance,
34
+ /// for gRPC, it creates a gRPC server and registers a gRPC service.)
35
+ /// That server will handle connections.
36
+ /// 4. The transport should forward incoming calls to the server to the RPC
37
+ /// handlers defined on ServerTransport, which implements the actual
38
+ /// RPC handler using the interfaces here. Any I/O the RPC handler needs
39
+ /// to do is managed by transport-specific implementations of
40
+ /// TransportDataStream.
41
+ /// 5. ServerTransport calls FlightServerBase for the actual application
42
+ /// logic.
43
+ ///
44
+ /// On the client side:
45
+ ///
46
+ /// 1. Applications create a FlightClient with a Location.
47
+ /// 2. FlightClient will look up and create a ClientTransport based on
48
+ /// the scheme of the Location given to it.
49
+ /// 3. When calling a method on FlightClient, FlightClient will delegate to
50
+ /// the ClientTransport. There is some indirection, e.g. for DoGet,
51
+ /// FlightClient only requests that the ClientTransport start the
52
+ /// call and provide it with an I/O stream. The "Flight implementation"
53
+ /// itself still lives in FlightClient.
54
+
55
+ #pragma once
56
+
57
+ #include <functional>
58
+ #include <memory>
59
+ #include <optional>
60
+ #include <string>
61
+ #include <utility>
62
+ #include <vector>
63
+
64
+ #include "arrow/flight/type_fwd.h"
65
+ #include "arrow/flight/types.h"
66
+ #include "arrow/flight/visibility.h"
67
+ #include "arrow/ipc/options.h"
68
+ #include "arrow/type_fwd.h"
69
+
70
+ namespace arrow {
71
+ namespace ipc {
72
+ class Message;
73
+ }
74
+ namespace flight {
75
+ class FlightStatusDetail;
76
+ namespace internal {
77
+
78
+ /// Internal, not user-visible type used for memory-efficient reads
79
+ struct ARROW_FLIGHT_EXPORT FlightData {
80
+ /// Used only for puts, may be null
81
+ std::unique_ptr<FlightDescriptor> descriptor;
82
+
83
+ /// Non-length-prefixed Message header as described in format/Message.fbs
84
+ std::shared_ptr<Buffer> metadata;
85
+
86
+ /// Application-defined metadata
87
+ std::shared_ptr<Buffer> app_metadata;
88
+
89
+ /// Message body
90
+ std::shared_ptr<Buffer> body;
91
+
92
+ /// Open IPC message from the metadata and body
93
+ ::arrow::Result<std::unique_ptr<ipc::Message>> OpenMessage();
94
+ };
95
+
96
+ /// \brief A transport-specific interface for reading/writing Arrow data.
97
+ ///
98
+ /// New transports will implement this to read/write IPC payloads to
99
+ /// the underlying stream.
100
+ class ARROW_FLIGHT_EXPORT TransportDataStream {
101
+ public:
102
+ virtual ~TransportDataStream() = default;
103
+ /// \brief Attempt to read the next FlightData message.
104
+ ///
105
+ /// \return success true if data was populated, false if there was
106
+ /// an error. For clients, the error can be retrieved from
107
+ /// Finish(Status).
108
+ virtual bool ReadData(FlightData* data);
109
+ /// \brief Attempt to write a FlightPayload.
110
+ ///
111
+ /// \param[in] payload The data to write.
112
+ /// \return true if the message was accepted by the transport, false
113
+ /// if not (e.g. due to client/server disconnect), Status if there
114
+ /// was an error (e.g. with the payload itself).
115
+ virtual arrow::Result<bool> WriteData(const FlightPayload& payload);
116
+ /// \brief Indicate that there are no more writes on this stream.
117
+ ///
118
+ /// This is only a hint for the underlying transport and may not
119
+ /// actually do anything.
120
+ virtual Status WritesDone();
121
+ };
122
+
123
+ /// \brief A transport-specific interface for reading/writing Arrow
124
+ /// data for a client.
125
+ class ARROW_FLIGHT_EXPORT ClientDataStream : public TransportDataStream {
126
+ public:
127
+ /// \brief Attempt to read a non-data message.
128
+ ///
129
+ /// Only implemented for DoPut; mutually exclusive with
130
+ /// ReadData(FlightData*).
131
+ virtual bool ReadPutMetadata(std::shared_ptr<Buffer>* out);
132
+ /// \brief Attempt to cancel the call.
133
+ ///
134
+ /// This is only a hint and may not take effect immediately. The
135
+ /// client should still finish the call with Finish(Status) as usual.
136
+ virtual void TryCancel() {}
137
+ /// \brief Finish the call, reporting the server-sent status and/or
138
+ /// any client-side errors as appropriate.
139
+ ///
140
+ /// Implies WritesDone() and DoFinish().
141
+ ///
142
+ /// \param[in] st A client-side status to combine with the
143
+ /// server-side error. That is, if an error occurs on the
144
+ /// client-side, call Finish(Status) to finish the server-side
145
+ /// call, get the server-side status, and merge the statuses
146
+ /// together so context is not lost.
147
+ Status Finish(Status st);
148
+
149
+ protected:
150
+ /// \brief End the call, returning the final server status.
151
+ ///
152
+ /// For implementors: should imply WritesDone() (even if it does not
153
+ /// directly call it).
154
+ ///
155
+ /// Implies WritesDone().
156
+ virtual Status DoFinish() = 0;
157
+ };
158
+
159
+ /// An implementation of a Flight client for a particular transport.
160
+ ///
161
+ /// Transports should override the methods they are capable of
162
+ /// supporting. The default method implementations return an error.
163
+ class ARROW_FLIGHT_EXPORT ClientTransport {
164
+ public:
165
+ virtual ~ClientTransport() = default;
166
+
167
+ /// Initialize the client.
168
+ virtual Status Init(const FlightClientOptions& options, const Location& location,
169
+ const arrow::util::Uri& uri) = 0;
170
+ /// Close the client. Once this returns, the client is no longer usable.
171
+ virtual Status Close() = 0;
172
+
173
+ virtual Status Authenticate(const FlightCallOptions& options,
174
+ std::unique_ptr<ClientAuthHandler> auth_handler);
175
+ virtual arrow::Result<std::pair<std::string, std::string>> AuthenticateBasicToken(
176
+ const FlightCallOptions& options, const std::string& username,
177
+ const std::string& password);
178
+ virtual Status DoAction(const FlightCallOptions& options, const Action& action,
179
+ std::unique_ptr<ResultStream>* results);
180
+ virtual Status ListActions(const FlightCallOptions& options,
181
+ std::vector<ActionType>* actions);
182
+ virtual Status GetFlightInfo(const FlightCallOptions& options,
183
+ const FlightDescriptor& descriptor,
184
+ std::unique_ptr<FlightInfo>* info);
185
+ virtual void GetFlightInfoAsync(const FlightCallOptions& options,
186
+ const FlightDescriptor& descriptor,
187
+ std::shared_ptr<AsyncListener<FlightInfo>> listener);
188
+ virtual Status PollFlightInfo(const FlightCallOptions& options,
189
+ const FlightDescriptor& descriptor,
190
+ std::unique_ptr<PollInfo>* info);
191
+ virtual arrow::Result<std::unique_ptr<SchemaResult>> GetSchema(
192
+ const FlightCallOptions& options, const FlightDescriptor& descriptor);
193
+ virtual Status ListFlights(const FlightCallOptions& options, const Criteria& criteria,
194
+ std::unique_ptr<FlightListing>* listing);
195
+ virtual Status DoGet(const FlightCallOptions& options, const Ticket& ticket,
196
+ std::unique_ptr<ClientDataStream>* stream);
197
+ virtual Status DoPut(const FlightCallOptions& options,
198
+ std::unique_ptr<ClientDataStream>* stream);
199
+ virtual Status DoExchange(const FlightCallOptions& options,
200
+ std::unique_ptr<ClientDataStream>* stream);
201
+
202
+ bool supports_async() const { return CheckAsyncSupport().ok(); }
203
+ virtual Status CheckAsyncSupport() const {
204
+ return Status::NotImplemented(
205
+ "this Flight transport does not support async operations");
206
+ }
207
+
208
+ static void SetAsyncRpc(AsyncListenerBase* listener, std::unique_ptr<AsyncRpc>&& rpc);
209
+ static AsyncRpc* GetAsyncRpc(AsyncListenerBase* listener);
210
+ static std::unique_ptr<AsyncRpc> ReleaseAsyncRpc(AsyncListenerBase* listener);
211
+ };
212
+
213
+ /// A registry of transport implementations.
214
+ class ARROW_FLIGHT_EXPORT TransportRegistry {
215
+ public:
216
+ using ClientFactory = std::function<arrow::Result<std::unique_ptr<ClientTransport>>()>;
217
+ using ServerFactory = std::function<arrow::Result<std::unique_ptr<ServerTransport>>(
218
+ FlightServerBase*, std::shared_ptr<MemoryManager> memory_manager)>;
219
+
220
+ TransportRegistry();
221
+ ~TransportRegistry();
222
+
223
+ arrow::Result<std::unique_ptr<ClientTransport>> MakeClient(
224
+ const std::string& scheme) const;
225
+ arrow::Result<std::unique_ptr<ServerTransport>> MakeServer(
226
+ const std::string& scheme, FlightServerBase* base,
227
+ std::shared_ptr<MemoryManager> memory_manager) const;
228
+
229
+ Status RegisterClient(const std::string& scheme, ClientFactory factory);
230
+ Status RegisterServer(const std::string& scheme, ServerFactory factory);
231
+
232
+ private:
233
+ class Impl;
234
+ std::unique_ptr<Impl> impl_;
235
+ };
236
+
237
+ /// \brief Get the registry of transport implementations.
238
+ ARROW_FLIGHT_EXPORT
239
+ TransportRegistry* GetDefaultTransportRegistry();
240
+
241
+ //------------------------------------------------------------
242
+ // Async APIs
243
+
244
+ /// \brief Transport-specific state for an async RPC.
245
+ ///
246
+ /// Transport implementations may subclass this to store their own
247
+ /// state, and stash an instance in a user-supplied AsyncListener via
248
+ /// ClientTransport::GetAsyncRpc and ClientTransport::SetAsyncRpc.
249
+ class ARROW_FLIGHT_EXPORT AsyncRpc {
250
+ public:
251
+ virtual ~AsyncRpc() = default;
252
+ /// \brief Request cancellation of the RPC.
253
+ virtual void TryCancel() {}
254
+
255
+ /// Only needed for DoPut/DoExchange
256
+ virtual void Begin(const FlightDescriptor& descriptor, std::shared_ptr<Schema> schema) {
257
+ }
258
+ /// Only needed for DoPut/DoExchange
259
+ virtual void Write(arrow::flight::FlightStreamChunk chunk) {}
260
+ /// Only needed for DoPut/DoExchange
261
+ virtual void DoneWriting() {}
262
+ };
263
+
264
+ //------------------------------------------------------------
265
+ // Error propagation helpers
266
+
267
+ /// \brief Abstract error status.
268
+ ///
269
+ /// Transport implementations may use side channels (e.g. HTTP
270
+ /// trailers) to convey additional information to reconstruct the
271
+ /// original C++ status for implementations that can use it.
272
+ struct ARROW_FLIGHT_EXPORT TransportStatus {
273
+ TransportStatusCode code;
274
+ std::string message;
275
+
276
+ /// \brief Convert a C++ status to an abstract transport status.
277
+ static TransportStatus FromStatus(const Status& arrow_status);
278
+
279
+ /// \brief Reconstruct a string-encoded TransportStatus.
280
+ static TransportStatus FromCodeStringAndMessage(const std::string& code_str,
281
+ std::string message);
282
+
283
+ /// \brief Convert an abstract transport status to a C++ status.
284
+ Status ToStatus() const;
285
+ };
286
+
287
+ /// \brief Convert the string representation of an Arrow status code
288
+ /// back to an Arrow status.
289
+ ARROW_FLIGHT_EXPORT
290
+ Status ReconstructStatus(const std::string& code_str, const Status& current_status,
291
+ std::optional<std::string> message,
292
+ std::optional<std::string> detail_message,
293
+ std::optional<std::string> detail_bin,
294
+ std::shared_ptr<FlightStatusDetail> detail);
295
+
296
+ } // namespace internal
297
+ } // namespace flight
298
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/transport_server.h ADDED
@@ -0,0 +1,133 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <chrono>
21
+ #include <memory>
22
+
23
+ #include "arrow/flight/transport.h"
24
+ #include "arrow/flight/type_fwd.h"
25
+ #include "arrow/flight/visibility.h"
26
+ #include "arrow/type_fwd.h"
27
+
28
+ namespace arrow {
29
+ namespace ipc {
30
+ class Message;
31
+ }
32
+ namespace flight {
33
+ namespace internal {
34
+
35
+ /// \brief A transport-specific interface for reading/writing Arrow
36
+ /// data for a server.
37
+ class ARROW_FLIGHT_EXPORT ServerDataStream : public TransportDataStream {
38
+ public:
39
+ /// \brief Attempt to write a non-data message.
40
+ ///
41
+ /// Only implemented for DoPut; mutually exclusive with
42
+ /// WriteData(const FlightPayload&).
43
+ virtual Status WritePutMetadata(const Buffer& payload);
44
+ };
45
+
46
+ /// \brief An implementation of a Flight server for a particular
47
+ /// transport.
48
+ ///
49
+ /// This class (the transport implementation) implements the underlying
50
+ /// server and handles connections/incoming RPC calls. It should forward RPC
51
+ /// calls to the RPC handlers defined on this class, which work in terms of
52
+ /// the generic interfaces above. The RPC handlers here then forward calls
53
+ /// to the underlying FlightServerBase instance that contains the actual
54
+ /// application RPC method handlers.
55
+ ///
56
+ /// Used by FlightServerBase to manage the server lifecycle.
57
+ class ARROW_FLIGHT_EXPORT ServerTransport {
58
+ public:
59
+ ServerTransport(FlightServerBase* base, std::shared_ptr<MemoryManager> memory_manager)
60
+ : base_(base), memory_manager_(std::move(memory_manager)) {}
61
+ virtual ~ServerTransport() = default;
62
+
63
+ /// \name Server Lifecycle Methods
64
+ /// Transports implement these methods to start/shutdown the underlying
65
+ /// server.
66
+ /// @{
67
+ /// \brief Initialize the server.
68
+ ///
69
+ /// This method should launch the server in a background thread, i.e. it
70
+ /// should not block. Once this returns, the server should be active.
71
+ virtual Status Init(const FlightServerOptions& options,
72
+ const arrow::util::Uri& uri) = 0;
73
+ /// \brief Shutdown the server.
74
+ ///
75
+ /// This should wait for active RPCs to finish. Once this returns, the
76
+ /// server is no longer listening.
77
+ virtual Status Shutdown() = 0;
78
+ /// \brief Shutdown the server with a deadline.
79
+ ///
80
+ /// This should wait for active RPCs to finish, or for the deadline to
81
+ /// expire. Once this returns, the server is no longer listening.
82
+ virtual Status Shutdown(const std::chrono::system_clock::time_point& deadline) = 0;
83
+ /// \brief Wait for the server to shutdown (but do not shut down the server).
84
+ ///
85
+ /// Once this returns, the server is no longer listening.
86
+ virtual Status Wait() = 0;
87
+ /// \brief Get the address the server is listening on, else an empty Location.
88
+ virtual Location location() const = 0;
89
+ ///@}
90
+
91
+ /// \name RPC Handlers
92
+ /// Implementations of RPC handlers for Flight methods using the common
93
+ /// interfaces here. Transports should call these methods from their
94
+ /// server implementation to handle the actual RPC calls.
95
+ ///@{
96
+ /// \brief Get the FlightServerBase.
97
+ ///
98
+ /// Intended as an escape hatch for now since not all methods have been
99
+ /// factored into a transport-agnostic interface.
100
+ FlightServerBase* base() const { return base_; }
101
+ /// \brief Implement DoGet in terms of a transport-level stream.
102
+ ///
103
+ /// \param[in] context The server context.
104
+ /// \param[in] request The request payload.
105
+ /// \param[in] stream The transport-specific data stream
106
+ /// implementation. Must implement WriteData(const
107
+ /// FlightPayload&).
108
+ Status DoGet(const ServerCallContext& context, const Ticket& request,
109
+ ServerDataStream* stream);
110
+ /// \brief Implement DoPut in terms of a transport-level stream.
111
+ ///
112
+ /// \param[in] context The server context.
113
+ /// \param[in] stream The transport-specific data stream
114
+ /// implementation. Must implement ReadData(FlightData*)
115
+ /// and WritePutMetadata(const Buffer&).
116
+ Status DoPut(const ServerCallContext& context, ServerDataStream* stream);
117
+ /// \brief Implement DoExchange in terms of a transport-level stream.
118
+ ///
119
+ /// \param[in] context The server context.
120
+ /// \param[in] stream The transport-specific data stream
121
+ /// implementation. Must implement ReadData(FlightData*)
122
+ /// and WriteData(const FlightPayload&).
123
+ Status DoExchange(const ServerCallContext& context, ServerDataStream* stream);
124
+ ///@}
125
+
126
+ protected:
127
+ FlightServerBase* base_;
128
+ std::shared_ptr<MemoryManager> memory_manager_;
129
+ };
130
+
131
+ } // namespace internal
132
+ } // namespace flight
133
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/type_fwd.h ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ namespace arrow {
21
+ namespace internal {
22
+ class Uri;
23
+ }
24
+ namespace flight {
25
+ struct Action;
26
+ struct ActionType;
27
+ template <typename T>
28
+ class AsyncListener;
29
+ class AsyncListenerBase;
30
+ class AsyncRpc;
31
+ struct BasicAuth;
32
+ class ClientAuthHandler;
33
+ class ClientMiddleware;
34
+ class ClientMiddlewareFactory;
35
+ struct Criteria;
36
+ class FlightCallOptions;
37
+ struct FlightClientOptions;
38
+ struct FlightDescriptor;
39
+ struct FlightEndpoint;
40
+ class FlightInfo;
41
+ class PollInfo;
42
+ class FlightListing;
43
+ class FlightMetadataReader;
44
+ class FlightMetadataWriter;
45
+ struct FlightPayload;
46
+ class FlightServerBase;
47
+ class FlightServerOptions;
48
+ class FlightStreamReader;
49
+ class FlightStreamWriter;
50
+ struct Location;
51
+ struct Result;
52
+ class ResultStream;
53
+ struct SchemaResult;
54
+ class ServerCallContext;
55
+ class ServerMiddleware;
56
+ class ServerMiddlewareFactory;
57
+ struct Ticket;
58
+ namespace internal {
59
+ class AsyncRpc;
60
+ class ClientTransport;
61
+ struct FlightData;
62
+ class ServerTransport;
63
+ } // namespace internal
64
+ } // namespace flight
65
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/types.h ADDED
@@ -0,0 +1,1314 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ // Data structure for Flight RPC.
19
+
20
+ #pragma once
21
+
22
+ #include <chrono>
23
+ #include <cstddef>
24
+ #include <cstdint>
25
+ #include <map>
26
+ #include <memory>
27
+ #include <optional>
28
+ #include <string>
29
+ #include <string_view>
30
+ #include <utility>
31
+ #include <variant>
32
+ #include <vector>
33
+
34
+ #include "arrow/buffer.h"
35
+ #include "arrow/flight/type_fwd.h"
36
+ #include "arrow/flight/visibility.h"
37
+ #include "arrow/ipc/options.h"
38
+ #include "arrow/ipc/writer.h"
39
+ #include "arrow/result.h"
40
+ #include "arrow/status.h"
41
+
42
+ namespace arrow {
43
+
44
+ class RecordBatch;
45
+ class Schema;
46
+ class Table;
47
+
48
+ namespace ipc {
49
+ class DictionaryMemo;
50
+ struct ReadStats;
51
+ } // namespace ipc
52
+
53
+ namespace util {
54
+ class Uri;
55
+ } // namespace util
56
+
57
+ namespace flight {
58
+
59
+ ARROW_FLIGHT_EXPORT
60
+ extern const char* kSchemeGrpc;
61
+ ARROW_FLIGHT_EXPORT
62
+ extern const char* kSchemeGrpcTcp;
63
+ ARROW_FLIGHT_EXPORT
64
+ extern const char* kSchemeGrpcUnix;
65
+ ARROW_FLIGHT_EXPORT
66
+ extern const char* kSchemeGrpcTls;
67
+
68
+ class FlightClient;
69
+ class FlightServerBase;
70
+
71
+ /// \brief A timestamp compatible with Protocol Buffer's
72
+ /// google.protobuf.Timestamp:
73
+ ///
74
+ /// https://protobuf.dev/reference/protobuf/google.protobuf/#timestamp
75
+ ///
76
+ /// > A Timestamp represents a point in time independent of any time
77
+ /// > zone or calendar, represented as seconds and fractions of
78
+ /// > seconds at nanosecond resolution in UTC Epoch time. It is
79
+ /// > encoded using the Proleptic Gregorian Calendar which extends the
80
+ /// > Gregorian calendar backwards to year one. It is encoded assuming
81
+ /// > all minutes are 60 seconds long, i.e. leap seconds are "smeared"
82
+ /// > so that no leap second table is needed for interpretation. Range
83
+ /// > is from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z.
84
+ using Timestamp =
85
+ std::chrono::time_point<std::chrono::system_clock, std::chrono::nanoseconds>;
86
+
87
+ /// \brief A Flight-specific status code. Used to encode some
88
+ /// additional status codes into an Arrow Status.
89
+ enum class FlightStatusCode : int8_t {
90
+ /// An implementation error has occurred.
91
+ Internal,
92
+ /// A request timed out.
93
+ TimedOut,
94
+ /// A request was cancelled.
95
+ Cancelled,
96
+ /// We are not authenticated to the remote service.
97
+ Unauthenticated,
98
+ /// We do not have permission to make this request.
99
+ Unauthorized,
100
+ /// The remote service cannot handle this request at the moment.
101
+ Unavailable,
102
+ /// A request failed for some other reason
103
+ Failed
104
+ };
105
+
106
+ // Silence warning
107
+ // "non dll-interface class RecordBatchReader used as base for dll-interface class"
108
+ #ifdef _MSC_VER
109
+ # pragma warning(push)
110
+ # pragma warning(disable : 4275)
111
+ #endif
112
+
113
+ /// \brief Flight-specific error information in a Status.
114
+ class ARROW_FLIGHT_EXPORT FlightStatusDetail : public arrow::StatusDetail {
115
+ public:
116
+ explicit FlightStatusDetail(FlightStatusCode code) : code_{code} {}
117
+ explicit FlightStatusDetail(FlightStatusCode code, std::string extra_info)
118
+ : code_{code}, extra_info_(std::move(extra_info)) {}
119
+ const char* type_id() const override;
120
+ std::string ToString() const override;
121
+
122
+ /// \brief Get the Flight status code.
123
+ FlightStatusCode code() const;
124
+ /// \brief Get the extra error info
125
+ std::string extra_info() const;
126
+ /// \brief Get the human-readable name of the status code.
127
+ std::string CodeAsString() const;
128
+ /// \brief Set the extra error info
129
+ void set_extra_info(std::string extra_info);
130
+
131
+ /// \brief Try to extract a \a FlightStatusDetail from any Arrow
132
+ /// status.
133
+ ///
134
+ /// \return a \a FlightStatusDetail if it could be unwrapped, \a
135
+ /// nullptr otherwise
136
+ static std::shared_ptr<FlightStatusDetail> UnwrapStatus(const arrow::Status& status);
137
+
138
+ private:
139
+ FlightStatusCode code_;
140
+ std::string extra_info_;
141
+ };
142
+
143
+ #ifdef _MSC_VER
144
+ # pragma warning(pop)
145
+ #endif
146
+
147
+ /// \brief Make an appropriate Arrow status for the given
148
+ /// Flight-specific status.
149
+ ///
150
+ /// \param code The Flight status code.
151
+ /// \param message The message for the error.
152
+ /// \param extra_info Optional extra binary info for the error (eg protobuf)
153
+ ARROW_FLIGHT_EXPORT
154
+ Status MakeFlightError(FlightStatusCode code, std::string message,
155
+ std::string extra_info = {});
156
+
157
+ /// \brief Headers sent from the client or server.
158
+ ///
159
+ /// Header values are ordered.
160
+ using CallHeaders = std::multimap<std::string_view, std::string_view>;
161
+
162
+ /// \brief A TLS certificate plus key.
163
+ struct ARROW_FLIGHT_EXPORT CertKeyPair {
164
+ /// \brief The certificate in PEM format.
165
+ std::string pem_cert;
166
+
167
+ /// \brief The key in PEM format.
168
+ std::string pem_key;
169
+ };
170
+
171
+ namespace internal {
172
+
173
+ template <typename T>
174
+ struct remove_unique_ptr {
175
+ using type = T;
176
+ };
177
+
178
+ template <typename T>
179
+ struct remove_unique_ptr<std::unique_ptr<T>> {
180
+ using type = T;
181
+ };
182
+
183
+ // Base CRTP type
184
+ template <class T>
185
+ struct BaseType {
186
+ protected:
187
+ using SuperT = BaseType<T>;
188
+ using SelfT = typename remove_unique_ptr<T>::type;
189
+
190
+ const SelfT& self() const { return static_cast<const SelfT&>(*this); }
191
+ SelfT& self() { return static_cast<SelfT&>(*this); }
192
+
193
+ public:
194
+ BaseType() = default;
195
+
196
+ friend bool operator==(const SelfT& left, const SelfT& right) {
197
+ return left.Equals(right);
198
+ }
199
+ friend bool operator!=(const SelfT& left, const SelfT& right) {
200
+ return !left.Equals(right);
201
+ }
202
+
203
+ /// \brief Serialize this message to its wire-format representation.
204
+ inline arrow::Result<std::string> SerializeToString() const {
205
+ std::string out;
206
+ ARROW_RETURN_NOT_OK(self().SelfT::SerializeToString(&out));
207
+ return out;
208
+ }
209
+
210
+ inline static arrow::Result<T> Deserialize(std::string_view serialized) {
211
+ T out;
212
+ ARROW_RETURN_NOT_OK(SelfT::Deserialize(serialized, &out));
213
+ return out;
214
+ }
215
+
216
+ inline arrow::Result<std::shared_ptr<Buffer>> SerializeToBuffer() const {
217
+ std::string out;
218
+ ARROW_RETURN_NOT_OK(self().SelfT::SerializeToString(&out));
219
+ return Buffer::FromString(std::move(out));
220
+ }
221
+ };
222
+
223
+ } // namespace internal
224
+
225
+ //------------------------------------------------------------
226
+ // Wrapper types for Flight RPC protobuf messages
227
+
228
+ // A wrapper around arrow.flight.protocol.HandshakeRequest is not defined
229
+ // A wrapper around arrow.flight.protocol.HandshakeResponse is not defined
230
+
231
+ /// \brief message for simple auth
232
+ struct ARROW_FLIGHT_EXPORT BasicAuth : public internal::BaseType<BasicAuth> {
233
+ std::string username;
234
+ std::string password;
235
+
236
+ BasicAuth() = default;
237
+ BasicAuth(std::string username, std::string password)
238
+ : username(std::move(username)), password(std::move(password)) {}
239
+
240
+ std::string ToString() const;
241
+ bool Equals(const BasicAuth& other) const;
242
+
243
+ using SuperT::Deserialize;
244
+ using SuperT::SerializeToString;
245
+
246
+ /// \brief Serialize this message to its wire-format representation.
247
+ ///
248
+ /// Use `SerializeToString()` if you want a Result-returning version.
249
+ arrow::Status SerializeToString(std::string* out) const;
250
+
251
+ /// \brief Deserialize this message from its wire-format representation.
252
+ ///
253
+ /// Use `Deserialize(serialized)` if you want a Result-returning version.
254
+ static arrow::Status Deserialize(std::string_view serialized, BasicAuth* out);
255
+ };
256
+
257
+ // A wrapper around arrow.flight.protocol.Empty is not defined
258
+
259
+ /// \brief A type of action that can be performed with the DoAction RPC.
260
+ struct ARROW_FLIGHT_EXPORT ActionType : public internal::BaseType<ActionType> {
261
+ /// \brief The name of the action.
262
+ std::string type;
263
+
264
+ /// \brief A human-readable description of the action.
265
+ std::string description;
266
+
267
+ ActionType() = default;
268
+
269
+ ActionType(std::string type, std::string description)
270
+ : type(std::move(type)), description(std::move(description)) {}
271
+
272
+ std::string ToString() const;
273
+ bool Equals(const ActionType& other) const;
274
+
275
+ using SuperT::Deserialize;
276
+ using SuperT::SerializeToString;
277
+
278
+ /// \brief Serialize this message to its wire-format representation.
279
+ ///
280
+ /// Use `SerializeToString()` if you want a Result-returning version.
281
+ arrow::Status SerializeToString(std::string* out) const;
282
+
283
+ /// \brief Deserialize this message from its wire-format representation.
284
+ ///
285
+ /// Use `Deserialize(serialized)` if you want a Result-returning version.
286
+ static arrow::Status Deserialize(std::string_view serialized, ActionType* out);
287
+
288
+ static const ActionType kCancelFlightInfo;
289
+ static const ActionType kRenewFlightEndpoint;
290
+ static const ActionType kSetSessionOptions;
291
+ static const ActionType kGetSessionOptions;
292
+ static const ActionType kCloseSession;
293
+ };
294
+
295
+ /// \brief Opaque selection criteria for ListFlights RPC
296
+ struct ARROW_FLIGHT_EXPORT Criteria : public internal::BaseType<Criteria> {
297
+ /// Opaque criteria expression, dependent on server implementation
298
+ std::string expression;
299
+
300
+ Criteria() = default;
301
+ Criteria(std::string expression) // NOLINT runtime/explicit
302
+ : expression(std::move(expression)) {}
303
+
304
+ std::string ToString() const;
305
+ bool Equals(const Criteria& other) const;
306
+
307
+ using SuperT::Deserialize;
308
+ using SuperT::SerializeToString;
309
+
310
+ /// \brief Serialize this message to its wire-format representation.
311
+ ///
312
+ /// Use `SerializeToString()` if you want a Result-returning version.
313
+ arrow::Status SerializeToString(std::string* out) const;
314
+
315
+ /// \brief Deserialize this message from its wire-format representation.
316
+ ///
317
+ /// Use `Deserialize(serialized)` if you want a Result-returning version.
318
+ static arrow::Status Deserialize(std::string_view serialized, Criteria* out);
319
+ };
320
+
321
+ /// \brief An action to perform with the DoAction RPC
322
+ struct ARROW_FLIGHT_EXPORT Action : public internal::BaseType<Action> {
323
+ /// The action type
324
+ std::string type;
325
+
326
+ /// The action content as a Buffer
327
+ std::shared_ptr<Buffer> body;
328
+
329
+ Action() = default;
330
+ Action(std::string type, std::shared_ptr<Buffer> body)
331
+ : type(std::move(type)), body(std::move(body)) {}
332
+
333
+ std::string ToString() const;
334
+ bool Equals(const Action& other) const;
335
+
336
+ using SuperT::Deserialize;
337
+ using SuperT::SerializeToString;
338
+
339
+ /// \brief Serialize this message to its wire-format representation.
340
+ ///
341
+ /// Use `SerializeToString()` if you want a Result-returning version.
342
+ arrow::Status SerializeToString(std::string* out) const;
343
+
344
+ /// \brief Deserialize this message from its wire-format representation.
345
+ ///
346
+ /// Use `Deserialize(serialized)` if you want a Result-returning version.
347
+ static arrow::Status Deserialize(std::string_view serialized, Action* out);
348
+ };
349
+
350
+ /// \brief Opaque result returned after executing an action
351
+ struct ARROW_FLIGHT_EXPORT Result : public internal::BaseType<Result> {
352
+ std::shared_ptr<Buffer> body;
353
+
354
+ Result() = default;
355
+ Result(std::shared_ptr<Buffer> body) // NOLINT runtime/explicit
356
+ : body(std::move(body)) {}
357
+
358
+ std::string ToString() const;
359
+ bool Equals(const Result& other) const;
360
+
361
+ using SuperT::Deserialize;
362
+ using SuperT::SerializeToString;
363
+
364
+ /// \brief Serialize this message to its wire-format representation.
365
+ ///
366
+ /// Use `SerializeToString()` if you want a Result-returning version.
367
+ arrow::Status SerializeToString(std::string* out) const;
368
+
369
+ /// \brief Deserialize this message from its wire-format representation.
370
+ ///
371
+ /// Use `Deserialize(serialized)` if you want a Result-returning version.
372
+ static arrow::Status Deserialize(std::string_view serialized, Result* out);
373
+ };
374
+
375
+ /// \brief Schema result returned after a schema request RPC
376
+ struct ARROW_FLIGHT_EXPORT SchemaResult : public internal::BaseType<SchemaResult> {
377
+ public:
378
+ SchemaResult() = default;
379
+ explicit SchemaResult(std::string schema) : raw_schema_(std::move(schema)) {}
380
+
381
+ /// \brief Factory method to construct a SchemaResult.
382
+ static arrow::Result<std::unique_ptr<SchemaResult>> Make(const Schema& schema);
383
+
384
+ /// \brief return schema
385
+ /// \param[in,out] dictionary_memo for dictionary bookkeeping, will
386
+ /// be modified
387
+ /// \return Arrow result with the reconstructed Schema
388
+ arrow::Result<std::shared_ptr<Schema>> GetSchema(
389
+ ipc::DictionaryMemo* dictionary_memo) const;
390
+
391
+ const std::string& serialized_schema() const { return raw_schema_; }
392
+
393
+ std::string ToString() const;
394
+ bool Equals(const SchemaResult& other) const;
395
+
396
+ using SuperT::Deserialize;
397
+ using SuperT::SerializeToString;
398
+
399
+ /// \brief Serialize this message to its wire-format representation.
400
+ ///
401
+ /// Use `SerializeToString()` if you want a Result-returning version.
402
+ arrow::Status SerializeToString(std::string* out) const;
403
+
404
+ /// \brief Deserialize this message from its wire-format representation.
405
+ ///
406
+ /// Use `Deserialize(serialized)` if you want a Result-returning version.
407
+ static arrow::Status Deserialize(std::string_view serialized, SchemaResult* out);
408
+
409
+ private:
410
+ std::string raw_schema_;
411
+ };
412
+
413
+ /// \brief A request to retrieve or generate a dataset
414
+ struct ARROW_FLIGHT_EXPORT FlightDescriptor
415
+ : public internal::BaseType<FlightDescriptor> {
416
+ enum DescriptorType {
417
+ UNKNOWN = 0, /// Unused
418
+ PATH = 1, /// Named path identifying a dataset
419
+ CMD = 2 /// Opaque command to generate a dataset
420
+ };
421
+
422
+ /// The descriptor type
423
+ DescriptorType type = UNKNOWN;
424
+
425
+ /// Opaque value used to express a command. Should only be defined when type
426
+ /// is CMD
427
+ std::string cmd;
428
+
429
+ /// List of strings identifying a particular dataset. Should only be defined
430
+ /// when type is PATH
431
+ std::vector<std::string> path;
432
+
433
+ FlightDescriptor();
434
+ FlightDescriptor(DescriptorType type, std::string cmd,
435
+ std::vector<std::string> path) noexcept;
436
+ ~FlightDescriptor();
437
+
438
+ /// \brief Get a human-readable form of this descriptor.
439
+ std::string ToString() const;
440
+ bool Equals(const FlightDescriptor& other) const;
441
+
442
+ using SuperT::Deserialize;
443
+ using SuperT::SerializeToString;
444
+
445
+ /// \brief Get the wire-format representation of this type.
446
+ ///
447
+ /// Useful when interoperating with non-Flight systems (e.g. REST
448
+ /// services) that may want to return Flight types.
449
+ ///
450
+ /// Use `SerializeToString()` if you want a Result-returning version.
451
+ arrow::Status SerializeToString(std::string* out) const;
452
+
453
+ /// \brief Parse the wire-format representation of this type.
454
+ ///
455
+ /// Useful when interoperating with non-Flight systems (e.g. REST
456
+ /// services) that may want to return Flight types.
457
+ ///
458
+ /// Use `Deserialize(serialized)` if you want a Result-returning version.
459
+ static arrow::Status Deserialize(std::string_view serialized, FlightDescriptor* out);
460
+
461
+ // Convenience factory functions
462
+
463
+ static FlightDescriptor Command(std::string cmd) {
464
+ return FlightDescriptor{CMD, std::move(cmd), {}};
465
+ }
466
+
467
+ static FlightDescriptor Path(std::vector<std::string> path) {
468
+ return FlightDescriptor{PATH, "", std::move(path)};
469
+ }
470
+ };
471
+
472
+ /// \brief Data structure providing an opaque identifier or credential to use
473
+ /// when requesting a data stream with the DoGet RPC
474
+ struct ARROW_FLIGHT_EXPORT Ticket : public internal::BaseType<Ticket> {
475
+ std::string ticket;
476
+
477
+ Ticket() = default;
478
+ Ticket(std::string ticket) // NOLINT runtime/explicit
479
+ : ticket(std::move(ticket)) {}
480
+
481
+ std::string ToString() const;
482
+ bool Equals(const Ticket& other) const;
483
+
484
+ using SuperT::Deserialize;
485
+ using SuperT::SerializeToString;
486
+
487
+ /// \brief Get the wire-format representation of this type.
488
+ ///
489
+ /// Useful when interoperating with non-Flight systems (e.g. REST
490
+ /// services) that may want to return Flight types.
491
+ ///
492
+ /// Use `SerializeToString()` if you want a Result-returning version.
493
+ arrow::Status SerializeToString(std::string* out) const;
494
+
495
+ /// \brief Parse the wire-format representation of this type.
496
+ ///
497
+ /// Useful when interoperating with non-Flight systems (e.g. REST
498
+ /// services) that may want to return Flight types.
499
+ ///
500
+ /// Use `Deserialize(serialized)` if you want a Result-returning version.
501
+ static arrow::Status Deserialize(std::string_view serialized, Ticket* out);
502
+ };
503
+
504
+ /// \brief A host location (a URI)
505
+ struct ARROW_FLIGHT_EXPORT Location : public internal::BaseType<Location> {
506
+ public:
507
+ /// \brief Initialize a blank location.
508
+ Location();
509
+
510
+ ~Location();
511
+
512
+ /// \brief Initialize a location by parsing a URI string
513
+ static arrow::Result<Location> Parse(const std::string& uri_string);
514
+
515
+ /// \brief Get the fallback URI.
516
+ ///
517
+ /// arrow-flight-reuse-connection://? means that a client may attempt to
518
+ /// reuse an existing connection to a Flight service to fetch data instead
519
+ /// of creating a new connection to one of the other locations listed in a
520
+ /// FlightEndpoint response.
521
+ static const Location& ReuseConnection();
522
+
523
+ /// \brief Initialize a location for a non-TLS, gRPC-based Flight
524
+ /// service from a host and port
525
+ /// \param[in] host The hostname to connect to
526
+ /// \param[in] port The port
527
+ /// \return Arrow result with the resulting location
528
+ static arrow::Result<Location> ForGrpcTcp(const std::string& host, const int port);
529
+
530
+ /// \brief Initialize a location for a TLS-enabled, gRPC-based Flight
531
+ /// service from a host and port
532
+ /// \param[in] host The hostname to connect to
533
+ /// \param[in] port The port
534
+ /// \return Arrow result with the resulting location
535
+ static arrow::Result<Location> ForGrpcTls(const std::string& host, const int port);
536
+
537
+ /// \brief Initialize a location for a domain socket-based Flight
538
+ /// service
539
+ /// \param[in] path The path to the domain socket
540
+ /// \return Arrow result with the resulting location
541
+ static arrow::Result<Location> ForGrpcUnix(const std::string& path);
542
+
543
+ /// \brief Initialize a location based on a URI scheme
544
+ static arrow::Result<Location> ForScheme(const std::string& scheme,
545
+ const std::string& host, const int port);
546
+
547
+ /// \brief Get the scheme of this URI.
548
+ std::string scheme() const;
549
+
550
+ /// \brief Get a representation of this URI as a string.
551
+ std::string ToString() const;
552
+ bool Equals(const Location& other) const;
553
+
554
+ using SuperT::Deserialize;
555
+ using SuperT::SerializeToString;
556
+
557
+ /// \brief Serialize this message to its wire-format representation.
558
+ ///
559
+ /// Use `SerializeToString()` if you want a Result-returning version.
560
+ arrow::Status SerializeToString(std::string* out) const;
561
+
562
+ /// \brief Deserialize this message from its wire-format representation.
563
+ ///
564
+ /// Use `Deserialize(serialized)` if you want a Result-returning version.
565
+ static arrow::Status Deserialize(std::string_view serialized, Location* out);
566
+
567
+ private:
568
+ friend class FlightClient;
569
+ friend class FlightServerBase;
570
+ std::shared_ptr<arrow::util::Uri> uri_;
571
+ };
572
+
573
+ /// \brief A flight ticket and list of locations where the ticket can be
574
+ /// redeemed
575
+ struct ARROW_FLIGHT_EXPORT FlightEndpoint : public internal::BaseType<FlightEndpoint> {
576
+ /// Opaque ticket identify; use with DoGet RPC
577
+ Ticket ticket;
578
+
579
+ /// List of locations where ticket can be redeemed. If the list is empty, the
580
+ /// ticket can only be redeemed on the current service where the ticket was
581
+ /// generated
582
+ std::vector<Location> locations;
583
+
584
+ /// Expiration time of this stream. If present, clients may assume
585
+ /// they can retry DoGet requests. Otherwise, clients should avoid
586
+ /// retrying DoGet requests.
587
+ std::optional<Timestamp> expiration_time;
588
+
589
+ /// Opaque Application-defined metadata
590
+ std::string app_metadata;
591
+
592
+ FlightEndpoint() = default;
593
+ FlightEndpoint(Ticket ticket, std::vector<Location> locations,
594
+ std::optional<Timestamp> expiration_time, std::string app_metadata)
595
+ : ticket(std::move(ticket)),
596
+ locations(std::move(locations)),
597
+ expiration_time(expiration_time),
598
+ app_metadata(std::move(app_metadata)) {}
599
+
600
+ std::string ToString() const;
601
+ bool Equals(const FlightEndpoint& other) const;
602
+
603
+ using SuperT::Deserialize;
604
+ using SuperT::SerializeToString;
605
+
606
+ /// \brief Serialize this message to its wire-format representation.
607
+ ///
608
+ /// Use `SerializeToString()` if you want a Result-returning version.
609
+ arrow::Status SerializeToString(std::string* out) const;
610
+
611
+ /// \brief Deserialize this message from its wire-format representation.
612
+ ///
613
+ /// Use `Deserialize(serialized)` if you want a Result-returning version.
614
+ static arrow::Status Deserialize(std::string_view serialized, FlightEndpoint* out);
615
+ };
616
+
617
+ /// \brief The access coordinates for retrieval of a dataset, returned by
618
+ /// GetFlightInfo
619
+ class ARROW_FLIGHT_EXPORT FlightInfo
620
+ : public internal::BaseType<std::unique_ptr<FlightInfo>> {
621
+ public:
622
+ struct Data {
623
+ std::string schema;
624
+ FlightDescriptor descriptor;
625
+ std::vector<FlightEndpoint> endpoints;
626
+ int64_t total_records = -1;
627
+ int64_t total_bytes = -1;
628
+ bool ordered = false;
629
+ std::string app_metadata;
630
+ };
631
+
632
+ explicit FlightInfo(Data data) : data_(std::move(data)), reconstructed_schema_(false) {}
633
+
634
+ /// \brief Factory method to construct a FlightInfo.
635
+ static arrow::Result<FlightInfo> Make(const Schema& schema,
636
+ const FlightDescriptor& descriptor,
637
+ const std::vector<FlightEndpoint>& endpoints,
638
+ int64_t total_records, int64_t total_bytes,
639
+ bool ordered = false,
640
+ std::string app_metadata = "");
641
+
642
+ /// \brief Factory method to construct a FlightInfo.
643
+ static arrow::Result<FlightInfo> Make(const std::shared_ptr<Schema>& schema,
644
+ const FlightDescriptor& descriptor,
645
+ const std::vector<FlightEndpoint>& endpoints,
646
+ int64_t total_records, int64_t total_bytes,
647
+ bool ordered = false,
648
+ std::string app_metadata = "");
649
+
650
+ /// \brief Deserialize the Arrow schema of the dataset. Populate any
651
+ /// dictionary encoded fields into a DictionaryMemo for
652
+ /// bookkeeping
653
+ /// \param[in,out] dictionary_memo for dictionary bookkeeping, will
654
+ /// be modified
655
+ /// \return Arrow result with the reconstructed Schema. Note that the schema
656
+ /// may be nullptr, as the schema is optional.
657
+ arrow::Result<std::shared_ptr<Schema>> GetSchema(
658
+ ipc::DictionaryMemo* dictionary_memo) const;
659
+
660
+ const std::string& serialized_schema() const { return data_.schema; }
661
+
662
+ /// The descriptor associated with this flight, may not be set
663
+ const FlightDescriptor& descriptor() const { return data_.descriptor; }
664
+
665
+ /// A list of endpoints associated with the flight (dataset). To consume the
666
+ /// whole flight, all endpoints must be consumed
667
+ const std::vector<FlightEndpoint>& endpoints() const { return data_.endpoints; }
668
+
669
+ /// The total number of records (rows) in the dataset. If unknown, set to -1
670
+ int64_t total_records() const { return data_.total_records; }
671
+
672
+ /// The total number of bytes in the dataset. If unknown, set to -1
673
+ int64_t total_bytes() const { return data_.total_bytes; }
674
+
675
+ /// Whether endpoints are in the same order as the data.
676
+ bool ordered() const { return data_.ordered; }
677
+
678
+ /// Application-defined opaque metadata
679
+ const std::string& app_metadata() const { return data_.app_metadata; }
680
+
681
+ using SuperT::Deserialize;
682
+ using SuperT::SerializeToString;
683
+
684
+ /// \brief Get the wire-format representation of this type.
685
+ ///
686
+ /// Useful when interoperating with non-Flight systems (e.g. REST
687
+ /// services) that may want to return Flight types.
688
+ ///
689
+ /// Use `SerializeToString()` if you want a Result-returning version.
690
+ arrow::Status SerializeToString(std::string* out) const;
691
+
692
+ /// \brief Parse the wire-format representation of this type.
693
+ ///
694
+ /// Useful when interoperating with non-Flight systems (e.g. REST
695
+ /// services) that may want to return Flight types.
696
+ ///
697
+ /// Use `Deserialize(serialized)` if you want a Result-returning version.
698
+ static arrow::Status Deserialize(std::string_view serialized,
699
+ std::unique_ptr<FlightInfo>* out);
700
+
701
+ std::string ToString() const;
702
+
703
+ /// Compare two FlightInfo for equality. This will compare the
704
+ /// serialized schema representations, NOT the logical equality of
705
+ /// the schemas.
706
+ bool Equals(const FlightInfo& other) const;
707
+
708
+ private:
709
+ Data data_;
710
+ mutable std::shared_ptr<Schema> schema_;
711
+ mutable bool reconstructed_schema_;
712
+ };
713
+
714
+ /// \brief The information to process a long-running query.
715
+ class ARROW_FLIGHT_EXPORT PollInfo
716
+ : public internal::BaseType<std::unique_ptr<PollInfo>> {
717
+ public:
718
+ /// The currently available results so far.
719
+ std::unique_ptr<FlightInfo> info = NULLPTR;
720
+ /// The descriptor the client should use on the next try. If unset,
721
+ /// the query is complete.
722
+ std::optional<FlightDescriptor> descriptor = std::nullopt;
723
+ /// Query progress. Must be in [0.0, 1.0] but need not be
724
+ /// monotonic or nondecreasing. If unknown, do not set.
725
+ std::optional<double> progress = std::nullopt;
726
+ /// Expiration time for this request. After this passes, the server
727
+ /// might not accept the poll descriptor anymore (and the query may
728
+ /// be cancelled). This may be updated on a call to PollFlightInfo.
729
+ std::optional<Timestamp> expiration_time = std::nullopt;
730
+
731
+ PollInfo()
732
+ : info(NULLPTR),
733
+ descriptor(std::nullopt),
734
+ progress(std::nullopt),
735
+ expiration_time(std::nullopt) {}
736
+
737
+ PollInfo(std::unique_ptr<FlightInfo> info, std::optional<FlightDescriptor> descriptor,
738
+ std::optional<double> progress, std::optional<Timestamp> expiration_time)
739
+ : info(std::move(info)),
740
+ descriptor(std::move(descriptor)),
741
+ progress(progress),
742
+ expiration_time(expiration_time) {}
743
+
744
+ PollInfo(const PollInfo& other)
745
+ : info(other.info ? std::make_unique<FlightInfo>(*other.info) : NULLPTR),
746
+ descriptor(other.descriptor),
747
+ progress(other.progress),
748
+ expiration_time(other.expiration_time) {}
749
+ PollInfo(PollInfo&& other) noexcept = default;
750
+ ~PollInfo() = default;
751
+ PollInfo& operator=(const PollInfo& other) {
752
+ info = other.info ? std::make_unique<FlightInfo>(*other.info) : NULLPTR;
753
+ descriptor = other.descriptor;
754
+ progress = other.progress;
755
+ expiration_time = other.expiration_time;
756
+ return *this;
757
+ }
758
+ PollInfo& operator=(PollInfo&& other) = default;
759
+
760
+ using SuperT::Deserialize;
761
+ using SuperT::SerializeToString;
762
+
763
+ /// \brief Get the wire-format representation of this type.
764
+ ///
765
+ /// Useful when interoperating with non-Flight systems (e.g. REST
766
+ /// services) that may want to return Flight types.
767
+ ///
768
+ /// Use `SerializeToString()` if you want a Result-returning version.
769
+ arrow::Status SerializeToString(std::string* out) const;
770
+
771
+ /// \brief Parse the wire-format representation of this type.
772
+ ///
773
+ /// Useful when interoperating with non-Flight systems (e.g. REST
774
+ /// services) that may want to return Flight types.
775
+ ///
776
+ /// Use `Deserialize(serialized)` if you want a Result-returning version.
777
+ static arrow::Status Deserialize(std::string_view serialized,
778
+ std::unique_ptr<PollInfo>* out);
779
+
780
+ std::string ToString() const;
781
+
782
+ /// Compare two PollInfo for equality. This will compare the
783
+ /// serialized schema representations, NOT the logical equality of
784
+ /// the schemas.
785
+ bool Equals(const PollInfo& other) const;
786
+ };
787
+
788
+ /// \brief The request of the CancelFlightInfoRequest action.
789
+ struct ARROW_FLIGHT_EXPORT CancelFlightInfoRequest
790
+ : public internal::BaseType<CancelFlightInfoRequest> {
791
+ std::unique_ptr<FlightInfo> info;
792
+
793
+ CancelFlightInfoRequest() = default;
794
+ CancelFlightInfoRequest(std::unique_ptr<FlightInfo> info) // NOLINT runtime/explicit
795
+ : info(std::move(info)) {}
796
+
797
+ std::string ToString() const;
798
+ bool Equals(const CancelFlightInfoRequest& other) const;
799
+
800
+ using SuperT::Deserialize;
801
+ using SuperT::SerializeToString;
802
+
803
+ /// \brief Serialize this message to its wire-format representation.
804
+ ///
805
+ /// Use `SerializeToString()` if you want a Result-returning version.
806
+ arrow::Status SerializeToString(std::string* out) const;
807
+
808
+ /// \brief Deserialize this message from its wire-format representation.
809
+ ///
810
+ /// Use `Deserialize(serialized)` if you want a Result-returning version.
811
+ static arrow::Status Deserialize(std::string_view serialized,
812
+ CancelFlightInfoRequest* out);
813
+ };
814
+
815
+ enum class CancelStatus {
816
+ /// The cancellation status is unknown. Servers should avoid using
817
+ /// this value (send a kNotCancellable if the requested FlightInfo
818
+ /// is not known). Clients can retry the request.
819
+ kUnspecified = 0,
820
+ /// The cancellation request is complete. Subsequent requests with
821
+ /// the same payload may return kCancelled or a kNotCancellable error.
822
+ kCancelled = 1,
823
+ /// The cancellation request is in progress. The client may retry
824
+ /// the cancellation request.
825
+ kCancelling = 2,
826
+ // The FlightInfo is not cancellable. The client should not retry the
827
+ // cancellation request.
828
+ kNotCancellable = 3,
829
+ };
830
+
831
+ /// \brief The result of the CancelFlightInfo action.
832
+ struct ARROW_FLIGHT_EXPORT CancelFlightInfoResult
833
+ : public internal::BaseType<CancelFlightInfoResult> {
834
+ CancelStatus status = CancelStatus::kUnspecified;
835
+
836
+ CancelFlightInfoResult() = default;
837
+ CancelFlightInfoResult(CancelStatus status) // NOLINT runtime/explicit
838
+ : status(status) {}
839
+
840
+ std::string ToString() const;
841
+ bool Equals(const CancelFlightInfoResult& other) const;
842
+
843
+ using SuperT::Deserialize;
844
+ using SuperT::SerializeToString;
845
+
846
+ /// \brief Serialize this message to its wire-format representation.
847
+ ///
848
+ /// Use `SerializeToString()` if you want a Result-returning version.
849
+ arrow::Status SerializeToString(std::string* out) const;
850
+
851
+ /// \brief Deserialize this message from its wire-format representation.
852
+ ///
853
+ /// Use `Deserialize(serialized)` if you want a Result-returning version.
854
+ static arrow::Status Deserialize(std::string_view serialized,
855
+ CancelFlightInfoResult* out);
856
+ };
857
+
858
+ ARROW_FLIGHT_EXPORT
859
+ std::ostream& operator<<(std::ostream& os, CancelStatus status);
860
+
861
+ /// \brief The request of the RenewFlightEndpoint action.
862
+ struct ARROW_FLIGHT_EXPORT RenewFlightEndpointRequest
863
+ : public internal::BaseType<RenewFlightEndpointRequest> {
864
+ FlightEndpoint endpoint;
865
+
866
+ RenewFlightEndpointRequest() = default;
867
+ explicit RenewFlightEndpointRequest(FlightEndpoint endpoint)
868
+ : endpoint(std::move(endpoint)) {}
869
+
870
+ std::string ToString() const;
871
+ bool Equals(const RenewFlightEndpointRequest& other) const;
872
+
873
+ using SuperT::Deserialize;
874
+ using SuperT::SerializeToString;
875
+
876
+ /// \brief Serialize this message to its wire-format representation.
877
+ ///
878
+ /// Use `SerializeToString()` if you want a Result-returning version.
879
+ arrow::Status SerializeToString(std::string* out) const;
880
+
881
+ /// \brief Deserialize this message from its wire-format representation.
882
+ ///
883
+ /// Use `Deserialize(serialized)` if you want a Result-returning version.
884
+ static arrow::Status Deserialize(std::string_view serialized,
885
+ RenewFlightEndpointRequest* out);
886
+ };
887
+
888
+ // FlightData in Flight.proto maps to FlightPayload here.
889
+
890
+ /// \brief Staging data structure for messages about to be put on the wire
891
+ ///
892
+ /// This structure corresponds to FlightData in the protocol.
893
+ struct ARROW_FLIGHT_EXPORT FlightPayload {
894
+ std::shared_ptr<Buffer> descriptor;
895
+ std::shared_ptr<Buffer> app_metadata;
896
+ ipc::IpcPayload ipc_message;
897
+
898
+ FlightPayload() = default;
899
+ FlightPayload(std::shared_ptr<Buffer> descriptor, std::shared_ptr<Buffer> app_metadata,
900
+ ipc::IpcPayload ipc_message)
901
+ : descriptor(std::move(descriptor)),
902
+ app_metadata(std::move(app_metadata)),
903
+ ipc_message(std::move(ipc_message)) {}
904
+
905
+ /// \brief Check that the payload can be written to the wire.
906
+ Status Validate() const;
907
+
908
+ /// \brief Serialize this payload to a vector of buffers.
909
+ arrow::Result<BufferVector> SerializeToBuffers() const;
910
+ };
911
+
912
+ // A wrapper around arrow.flight.protocol.PutResult is not defined
913
+
914
+ // Session management messages
915
+
916
+ /// \brief Variant supporting all possible value types for {Set,Get}SessionOptions
917
+ ///
918
+ /// By convention, an attempt to set a valueless (std::monostate) SessionOptionValue
919
+ /// should attempt to unset or clear the named option value on the server.
920
+ using SessionOptionValue = std::variant<std::monostate, std::string, bool, int64_t,
921
+ double, std::vector<std::string>>;
922
+ std::ostream& operator<<(std::ostream& os, const SessionOptionValue& v);
923
+
924
+ /// \brief A request to set a set of session options by name/value.
925
+ struct ARROW_FLIGHT_EXPORT SetSessionOptionsRequest
926
+ : public internal::BaseType<SetSessionOptionsRequest> {
927
+ std::map<std::string, SessionOptionValue> session_options;
928
+
929
+ SetSessionOptionsRequest() = default;
930
+ explicit SetSessionOptionsRequest(
931
+ std::map<std::string, SessionOptionValue> session_options)
932
+ : session_options(std::move(session_options)) {}
933
+
934
+ std::string ToString() const;
935
+ bool Equals(const SetSessionOptionsRequest& other) const;
936
+
937
+ using SuperT::Deserialize;
938
+ using SuperT::SerializeToString;
939
+
940
+ /// \brief Serialize this message to its wire-format representation.
941
+ ///
942
+ /// Use `SerializeToString()` if you want a Result-returning version.
943
+ arrow::Status SerializeToString(std::string* out) const;
944
+
945
+ /// \brief Deserialize this message from its wire-format representation.
946
+ ///
947
+ /// Use `Deserialize(serialized)` if you want a Result-returning version.
948
+ static arrow::Status Deserialize(std::string_view serialized,
949
+ SetSessionOptionsRequest* out);
950
+ };
951
+
952
+ /// \brief The result of setting a session option.
953
+ enum class SetSessionOptionErrorValue : int8_t {
954
+ /// \brief The status of setting the option is unknown.
955
+ ///
956
+ /// Servers should avoid using this value (send a NOT_FOUND error if the requested
957
+ /// session is not known). Clients can retry the request.
958
+ kUnspecified,
959
+ /// \brief The given session option name is invalid.
960
+ kInvalidName,
961
+ /// \brief The session option value or type is invalid.
962
+ kInvalidValue,
963
+ /// \brief The session option cannot be set.
964
+ kError
965
+ };
966
+ std::string ToString(const SetSessionOptionErrorValue& error_value);
967
+ std::ostream& operator<<(std::ostream& os, const SetSessionOptionErrorValue& error_value);
968
+
969
+ /// \brief The result(s) of setting session option(s).
970
+ struct ARROW_FLIGHT_EXPORT SetSessionOptionsResult
971
+ : public internal::BaseType<SetSessionOptionsResult> {
972
+ struct Error {
973
+ SetSessionOptionErrorValue value;
974
+
975
+ bool Equals(const Error& other) const { return value == other.value; }
976
+ friend bool operator==(const Error& left, const Error& right) {
977
+ return left.Equals(right);
978
+ }
979
+ friend bool operator!=(const Error& left, const Error& right) {
980
+ return !(left == right);
981
+ }
982
+ };
983
+
984
+ std::map<std::string, Error> errors;
985
+
986
+ SetSessionOptionsResult() = default;
987
+ SetSessionOptionsResult(std::map<std::string, Error> errors) // NOLINT runtime/explicit
988
+ : errors(std::move(errors)) {}
989
+
990
+ std::string ToString() const;
991
+ bool Equals(const SetSessionOptionsResult& other) const;
992
+
993
+ using SuperT::Deserialize;
994
+ using SuperT::SerializeToString;
995
+
996
+ /// \brief Serialize this message to its wire-format representation.
997
+ ///
998
+ /// Use `SerializeToString()` if you want a Result-returning version.
999
+ arrow::Status SerializeToString(std::string* out) const;
1000
+
1001
+ /// \brief Deserialize this message from its wire-format representation.
1002
+ ///
1003
+ /// Use `Deserialize(serialized)` if you want a Result-returning version.
1004
+ static arrow::Status Deserialize(std::string_view serialized,
1005
+ SetSessionOptionsResult* out);
1006
+ };
1007
+
1008
+ /// \brief A request to get current session options.
1009
+ struct ARROW_FLIGHT_EXPORT GetSessionOptionsRequest
1010
+ : public internal::BaseType<GetSessionOptionsRequest> {
1011
+ GetSessionOptionsRequest() = default;
1012
+
1013
+ std::string ToString() const;
1014
+ bool Equals(const GetSessionOptionsRequest& other) const;
1015
+
1016
+ using SuperT::Deserialize;
1017
+ using SuperT::SerializeToString;
1018
+
1019
+ /// \brief Serialize this message to its wire-format representation.
1020
+ ///
1021
+ /// Use `SerializeToString()` if you want a Result-returning version.
1022
+ arrow::Status SerializeToString(std::string* out) const;
1023
+
1024
+ /// \brief Deserialize this message from its wire-format representation.
1025
+ ///
1026
+ /// Use `Deserialize(serialized)` if you want a Result-returning version.
1027
+ static arrow::Status Deserialize(std::string_view serialized,
1028
+ GetSessionOptionsRequest* out);
1029
+ };
1030
+
1031
+ /// \brief The current session options.
1032
+ struct ARROW_FLIGHT_EXPORT GetSessionOptionsResult
1033
+ : public internal::BaseType<GetSessionOptionsResult> {
1034
+ std::map<std::string, SessionOptionValue> session_options;
1035
+
1036
+ GetSessionOptionsResult() = default;
1037
+ GetSessionOptionsResult( // NOLINT runtime/explicit
1038
+ std::map<std::string, SessionOptionValue> session_options)
1039
+ : session_options(std::move(session_options)) {}
1040
+
1041
+ std::string ToString() const;
1042
+ bool Equals(const GetSessionOptionsResult& other) const;
1043
+
1044
+ using SuperT::Deserialize;
1045
+ using SuperT::SerializeToString;
1046
+
1047
+ /// \brief Serialize this message to its wire-format representation.
1048
+ ///
1049
+ /// Use `SerializeToString()` if you want a Result-returning version.
1050
+ arrow::Status SerializeToString(std::string* out) const;
1051
+
1052
+ /// \brief Deserialize this message from its wire-format representation.
1053
+ ///
1054
+ /// Use `Deserialize(serialized)` if you want a Result-returning version.
1055
+ static arrow::Status Deserialize(std::string_view serialized,
1056
+ GetSessionOptionsResult* out);
1057
+ };
1058
+
1059
+ /// \brief A request to close the open client session.
1060
+ struct ARROW_FLIGHT_EXPORT CloseSessionRequest
1061
+ : public internal::BaseType<CloseSessionRequest> {
1062
+ CloseSessionRequest() = default;
1063
+
1064
+ std::string ToString() const;
1065
+ bool Equals(const CloseSessionRequest& other) const;
1066
+
1067
+ using SuperT::Deserialize;
1068
+ using SuperT::SerializeToString;
1069
+
1070
+ /// \brief Serialize this message to its wire-format representation.
1071
+ ///
1072
+ /// Use `SerializeToString()` if you want a Result-returning version.
1073
+ arrow::Status SerializeToString(std::string* out) const;
1074
+
1075
+ /// \brief Deserialize this message from its wire-format representation.
1076
+ ///
1077
+ /// Use `Deserialize(serialized)` if you want a Result-returning version.
1078
+ static arrow::Status Deserialize(std::string_view serialized, CloseSessionRequest* out);
1079
+ };
1080
+
1081
+ /// \brief The result of closing a session.
1082
+ enum class CloseSessionStatus : int8_t {
1083
+ // \brief The session close status is unknown.
1084
+ //
1085
+ // Servers should avoid using this value (send a NOT_FOUND error if the requested
1086
+ // session is not known). Clients can retry the request.
1087
+ kUnspecified,
1088
+ // \brief The session close request is complete.
1089
+ //
1090
+ // Subsequent requests with the same session produce a NOT_FOUND error.
1091
+ kClosed,
1092
+ // \brief The session close request is in progress.
1093
+ //
1094
+ // The client may retry the request.
1095
+ kClosing,
1096
+ // \brief The session is not closeable.
1097
+ //
1098
+ // The client should not retry the request.
1099
+ kNotClosable
1100
+ };
1101
+ std::string ToString(const CloseSessionStatus& status);
1102
+ std::ostream& operator<<(std::ostream& os, const CloseSessionStatus& status);
1103
+
1104
+ /// \brief The result of attempting to close the client session.
1105
+ struct ARROW_FLIGHT_EXPORT CloseSessionResult
1106
+ : public internal::BaseType<CloseSessionResult> {
1107
+ CloseSessionStatus status;
1108
+
1109
+ CloseSessionResult() = default;
1110
+ CloseSessionResult(CloseSessionStatus status) // NOLINT runtime/explicit
1111
+ : status(status) {}
1112
+
1113
+ std::string ToString() const;
1114
+ bool Equals(const CloseSessionResult& other) const;
1115
+
1116
+ using SuperT::Deserialize;
1117
+ using SuperT::SerializeToString;
1118
+
1119
+ /// \brief Serialize this message to its wire-format representation.
1120
+ ///
1121
+ /// Use `SerializeToString()` if you want a Result-returning version.
1122
+ arrow::Status SerializeToString(std::string* out) const;
1123
+
1124
+ /// \brief Deserialize this message from its wire-format representation.
1125
+ ///
1126
+ /// Use `Deserialize(serialized)` if you want a Result-returning version.
1127
+ static arrow::Status Deserialize(std::string_view serialized, CloseSessionResult* out);
1128
+ };
1129
+
1130
+ //------------------------------------------------------------
1131
+
1132
+ /// \brief An iterator to FlightInfo instances returned by ListFlights.
1133
+ class ARROW_FLIGHT_EXPORT FlightListing {
1134
+ public:
1135
+ virtual ~FlightListing() = default;
1136
+
1137
+ /// \brief Retrieve the next FlightInfo from the iterator.
1138
+ /// \return Arrow result with a single FlightInfo. Set to \a nullptr if there
1139
+ /// are none left.
1140
+ virtual arrow::Result<std::unique_ptr<FlightInfo>> Next() = 0;
1141
+ };
1142
+
1143
+ /// \brief An iterator to Result instances returned by DoAction.
1144
+ class ARROW_FLIGHT_EXPORT ResultStream {
1145
+ public:
1146
+ virtual ~ResultStream() = default;
1147
+
1148
+ /// \brief Retrieve the next Result from the iterator.
1149
+ /// \return Arrow result with a single Result. Set to \a nullptr if there are none left.
1150
+ virtual arrow::Result<std::unique_ptr<Result>> Next() = 0;
1151
+
1152
+ /// \brief Read and drop the remaining messages to get the error (if any) from a server.
1153
+ /// \return Status OK if this is no error from a server, any other status if a
1154
+ /// server returns an error.
1155
+ Status Drain();
1156
+ };
1157
+
1158
+ /// \brief A holder for a RecordBatch with associated Flight metadata.
1159
+ struct ARROW_FLIGHT_EXPORT FlightStreamChunk {
1160
+ public:
1161
+ FlightStreamChunk() noexcept;
1162
+ ~FlightStreamChunk();
1163
+
1164
+ std::shared_ptr<RecordBatch> data;
1165
+ std::shared_ptr<Buffer> app_metadata;
1166
+ };
1167
+
1168
+ /// \brief An interface to read Flight data with metadata.
1169
+ class ARROW_FLIGHT_EXPORT MetadataRecordBatchReader {
1170
+ public:
1171
+ virtual ~MetadataRecordBatchReader() = default;
1172
+
1173
+ /// \brief Get the schema for this stream.
1174
+ virtual arrow::Result<std::shared_ptr<Schema>> GetSchema() = 0;
1175
+
1176
+ /// \brief Get the next message from Flight. If the stream is
1177
+ /// finished, then the members of \a FlightStreamChunk will be
1178
+ /// nullptr.
1179
+ virtual arrow::Result<FlightStreamChunk> Next() = 0;
1180
+
1181
+ /// \brief Consume entire stream as a vector of record batches
1182
+ virtual arrow::Result<std::vector<std::shared_ptr<RecordBatch>>> ToRecordBatches();
1183
+
1184
+ /// \brief Consume entire stream as a Table
1185
+ virtual arrow::Result<std::shared_ptr<Table>> ToTable();
1186
+
1187
+ /// \brief Return current read statistics
1188
+ virtual arrow::ipc::ReadStats stats() const = 0;
1189
+ };
1190
+
1191
+ /// \brief Convert a MetadataRecordBatchReader to a regular RecordBatchReader.
1192
+ ARROW_FLIGHT_EXPORT
1193
+ arrow::Result<std::shared_ptr<RecordBatchReader>> MakeRecordBatchReader(
1194
+ std::shared_ptr<MetadataRecordBatchReader> reader);
1195
+
1196
+ /// \brief An interface to write IPC payloads with metadata.
1197
+ class ARROW_FLIGHT_EXPORT MetadataRecordBatchWriter : public ipc::RecordBatchWriter {
1198
+ public:
1199
+ virtual ~MetadataRecordBatchWriter() = default;
1200
+ /// \brief Begin writing data with the given schema. Only used with \a DoExchange.
1201
+ virtual Status Begin(const std::shared_ptr<Schema>& schema,
1202
+ const ipc::IpcWriteOptions& options) = 0;
1203
+ virtual Status Begin(const std::shared_ptr<Schema>& schema);
1204
+ virtual Status WriteMetadata(std::shared_ptr<Buffer> app_metadata) = 0;
1205
+ virtual Status WriteWithMetadata(const RecordBatch& batch,
1206
+ std::shared_ptr<Buffer> app_metadata) = 0;
1207
+ };
1208
+
1209
+ /// \brief A FlightListing implementation based on a vector of
1210
+ /// FlightInfo objects.
1211
+ ///
1212
+ /// This can be iterated once, then it is consumed.
1213
+ class ARROW_FLIGHT_EXPORT SimpleFlightListing : public FlightListing {
1214
+ public:
1215
+ explicit SimpleFlightListing(const std::vector<FlightInfo>& flights);
1216
+ explicit SimpleFlightListing(std::vector<FlightInfo>&& flights);
1217
+
1218
+ arrow::Result<std::unique_ptr<FlightInfo>> Next() override;
1219
+
1220
+ private:
1221
+ int position_;
1222
+ std::vector<FlightInfo> flights_;
1223
+ };
1224
+
1225
+ /// \brief A ResultStream implementation based on a vector of
1226
+ /// Result objects.
1227
+ ///
1228
+ /// This can be iterated once, then it is consumed.
1229
+ class ARROW_FLIGHT_EXPORT SimpleResultStream : public ResultStream {
1230
+ public:
1231
+ explicit SimpleResultStream(std::vector<Result>&& results);
1232
+ arrow::Result<std::unique_ptr<Result>> Next() override;
1233
+
1234
+ private:
1235
+ std::vector<Result> results_;
1236
+ size_t position_;
1237
+ };
1238
+
1239
+ /// \defgroup flight-error Error Handling
1240
+ /// Types for handling errors from RPCs. Flight uses a set of status
1241
+ /// codes standardized across Flight implementations, so these types
1242
+ /// let applications work directly with those codes instead of having
1243
+ /// to translate to and from Arrow Status.
1244
+ /// @{
1245
+
1246
+ /// \brief Abstract status code for an RPC as per the Flight
1247
+ /// specification.
1248
+ enum class TransportStatusCode {
1249
+ /// \brief No error.
1250
+ kOk = 0,
1251
+ /// \brief An unknown error occurred.
1252
+ kUnknown = 1,
1253
+ /// \brief An error occurred in the transport implementation, or an
1254
+ /// error internal to the service implementation occurred.
1255
+ kInternal = 2,
1256
+ /// \brief An argument is invalid.
1257
+ kInvalidArgument = 3,
1258
+ /// \brief The request timed out.
1259
+ kTimedOut = 4,
1260
+ /// \brief An argument is not necessarily invalid, but references
1261
+ /// some resource that does not exist. Prefer over
1262
+ /// kInvalidArgument where applicable.
1263
+ kNotFound = 5,
1264
+ /// \brief The request attempted to create some resource that does
1265
+ /// not exist.
1266
+ kAlreadyExists = 6,
1267
+ /// \brief The request was explicitly cancelled.
1268
+ kCancelled = 7,
1269
+ /// \brief The client is not authenticated.
1270
+ kUnauthenticated = 8,
1271
+ /// \brief The client is not authorized to perform this request.
1272
+ kUnauthorized = 9,
1273
+ /// \brief The request is not implemented
1274
+ kUnimplemented = 10,
1275
+ /// \brief There is a network connectivity error, or some resource
1276
+ /// is otherwise unavailable. Most likely a temporary condition.
1277
+ kUnavailable = 11,
1278
+ };
1279
+
1280
+ /// \brief Convert a code to a string.
1281
+ std::string ToString(TransportStatusCode code);
1282
+
1283
+ /// \brief An error from an RPC call, using Flight error codes directly
1284
+ /// instead of trying to translate to Arrow Status.
1285
+ ///
1286
+ /// Currently, only attached to the Status passed to AsyncListener::OnFinish.
1287
+ class ARROW_FLIGHT_EXPORT TransportStatusDetail : public StatusDetail {
1288
+ public:
1289
+ constexpr static const char* kTypeId = "flight::TransportStatusDetail";
1290
+ explicit TransportStatusDetail(TransportStatusCode code, std::string message,
1291
+ std::vector<std::pair<std::string, std::string>> details)
1292
+ : code_(code), message_(std::move(message)), details_(std::move(details)) {}
1293
+ const char* type_id() const override { return kTypeId; }
1294
+ std::string ToString() const override;
1295
+
1296
+ static std::optional<std::reference_wrapper<const TransportStatusDetail>> Unwrap(
1297
+ const Status& status);
1298
+
1299
+ TransportStatusCode code() const { return code_; }
1300
+ std::string_view message() const { return message_; }
1301
+ const std::vector<std::pair<std::string, std::string>>& details() const {
1302
+ return details_;
1303
+ }
1304
+
1305
+ private:
1306
+ TransportStatusCode code_;
1307
+ std::string message_;
1308
+ std::vector<std::pair<std::string, std::string>> details_;
1309
+ };
1310
+
1311
+ /// @}
1312
+
1313
+ } // namespace flight
1314
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/types_async.h ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <memory>
21
+
22
+ #include "arrow/flight/type_fwd.h"
23
+ #include "arrow/flight/types.h"
24
+ #include "arrow/ipc/options.h"
25
+ #include "arrow/type_fwd.h"
26
+
27
+ namespace arrow::flight {
28
+
29
+ /// \defgroup flight-async Async Flight Types
30
+ /// Common types used for asynchronous Flight APIs.
31
+ /// @{
32
+
33
+ /// \brief Non-templated state for an async RPC.
34
+ class ARROW_FLIGHT_EXPORT AsyncListenerBase {
35
+ public:
36
+ AsyncListenerBase();
37
+ virtual ~AsyncListenerBase();
38
+
39
+ /// \brief Request cancellation of the RPC.
40
+ ///
41
+ /// The RPC is not cancelled until AsyncListener::OnFinish is called.
42
+ void TryCancel();
43
+
44
+ private:
45
+ friend class arrow::flight::internal::ClientTransport;
46
+
47
+ /// Transport-specific state for this RPC. Transport
48
+ /// implementations may store and retrieve state here via
49
+ /// ClientTransport::SetAsyncRpc and ClientTransport::GetAsyncRpc.
50
+ std::unique_ptr<internal::AsyncRpc> rpc_state_;
51
+ };
52
+
53
+ /// \brief Callbacks for results from async RPCs.
54
+ ///
55
+ /// A single listener may not be used for multiple concurrent RPC
56
+ /// calls. The application MUST hold the listener alive until
57
+ /// OnFinish() is called and has finished.
58
+ template <typename T>
59
+ class ARROW_FLIGHT_EXPORT AsyncListener : public AsyncListenerBase {
60
+ public:
61
+ /// \brief Get the next server result.
62
+ ///
63
+ /// This will never be called concurrently with itself or OnFinish.
64
+ virtual void OnNext(T message) = 0;
65
+ /// \brief Get the final status.
66
+ ///
67
+ /// This will never be called concurrently with itself or OnNext. If the
68
+ /// error comes from the remote server, then a TransportStatusDetail will be
69
+ /// attached. Otherwise, the error is generated by the client-side
70
+ /// transport and will not have a TransportStatusDetail.
71
+ virtual void OnFinish(Status status) = 0;
72
+ };
73
+
74
+ /// @}
75
+
76
+ } // namespace arrow::flight
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/visibility.h ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #if defined(_WIN32) || defined(__CYGWIN__)
21
+ # if defined(_MSC_VER)
22
+ # pragma warning(push)
23
+ # pragma warning(disable : 4251)
24
+ # else
25
+ # pragma GCC diagnostic ignored "-Wattributes"
26
+ # endif
27
+
28
+ # ifdef ARROW_FLIGHT_STATIC
29
+ # define ARROW_FLIGHT_EXPORT
30
+ # elif defined(ARROW_FLIGHT_EXPORTING)
31
+ # define ARROW_FLIGHT_EXPORT __declspec(dllexport)
32
+ # else
33
+ # define ARROW_FLIGHT_EXPORT __declspec(dllimport)
34
+ # endif
35
+
36
+ # define ARROW_FLIGHT_NO_EXPORT
37
+ #else // Not Windows
38
+ # ifndef ARROW_FLIGHT_EXPORT
39
+ # define ARROW_FLIGHT_EXPORT __attribute__((visibility("default")))
40
+ # endif
41
+ # ifndef ARROW_FLIGHT_NO_EXPORT
42
+ # define ARROW_FLIGHT_NO_EXPORT __attribute__((visibility("hidden")))
43
+ # endif
44
+ #endif // Non-Windows
45
+
46
+ #if defined(_MSC_VER)
47
+ # pragma warning(pop)
48
+ #endif
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/api.h ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include "arrow/io/buffered.h"
21
+ #include "arrow/io/compressed.h"
22
+ #include "arrow/io/file.h"
23
+ #include "arrow/io/hdfs.h"
24
+ #include "arrow/io/interfaces.h"
25
+ #include "arrow/io/memory.h"
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/buffered.h ADDED
@@ -0,0 +1,168 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ // Buffered stream implementations
19
+
20
+ #pragma once
21
+
22
+ #include <cstdint>
23
+ #include <memory>
24
+ #include <string_view>
25
+
26
+ #include "arrow/io/concurrency.h"
27
+ #include "arrow/io/interfaces.h"
28
+ #include "arrow/util/visibility.h"
29
+
30
+ namespace arrow {
31
+
32
+ class Buffer;
33
+ class MemoryPool;
34
+ class Status;
35
+
36
+ namespace io {
37
+
38
+ class ARROW_EXPORT BufferedOutputStream : public OutputStream {
39
+ public:
40
+ ~BufferedOutputStream() override;
41
+
42
+ /// \brief Create a buffered output stream wrapping the given output stream.
43
+ /// \param[in] buffer_size the size of the temporary write buffer
44
+ /// \param[in] pool a MemoryPool to use for allocations
45
+ /// \param[in] raw another OutputStream
46
+ /// \return the created BufferedOutputStream
47
+ static Result<std::shared_ptr<BufferedOutputStream>> Create(
48
+ int64_t buffer_size, MemoryPool* pool, std::shared_ptr<OutputStream> raw);
49
+
50
+ /// \brief Resize internal buffer
51
+ /// \param[in] new_buffer_size the new buffer size
52
+ /// \return Status
53
+ Status SetBufferSize(int64_t new_buffer_size);
54
+
55
+ /// \brief Return the current size of the internal buffer
56
+ int64_t buffer_size() const;
57
+
58
+ /// \brief Return the number of remaining bytes that have not been flushed to
59
+ /// the raw OutputStream
60
+ int64_t bytes_buffered() const;
61
+
62
+ /// \brief Flush any buffered writes and release the raw
63
+ /// OutputStream. Further operations on this object are invalid
64
+ /// \return the underlying OutputStream
65
+ Result<std::shared_ptr<OutputStream>> Detach();
66
+
67
+ // OutputStream interface
68
+
69
+ /// \brief Close the buffered output stream. This implicitly closes the
70
+ /// underlying raw output stream.
71
+ Status Close() override;
72
+ Status Abort() override;
73
+ bool closed() const override;
74
+
75
+ Result<int64_t> Tell() const override;
76
+ // Write bytes to the stream. Thread-safe
77
+ Status Write(const void* data, int64_t nbytes) override;
78
+ Status Write(const std::shared_ptr<Buffer>& data) override;
79
+
80
+ Status Flush() override;
81
+
82
+ /// \brief Return the underlying raw output stream.
83
+ std::shared_ptr<OutputStream> raw() const;
84
+
85
+ private:
86
+ explicit BufferedOutputStream(std::shared_ptr<OutputStream> raw, MemoryPool* pool);
87
+
88
+ class ARROW_NO_EXPORT Impl;
89
+ std::unique_ptr<Impl> impl_;
90
+ };
91
+
92
+ /// \class BufferedInputStream
93
+ /// \brief An InputStream that performs buffered reads from an unbuffered
94
+ /// InputStream, which can mitigate the overhead of many small reads in some
95
+ /// cases
96
+ class ARROW_EXPORT BufferedInputStream
97
+ : public internal::InputStreamConcurrencyWrapper<BufferedInputStream> {
98
+ public:
99
+ ~BufferedInputStream() override;
100
+
101
+ /// \brief Create a BufferedInputStream from a raw InputStream
102
+ /// \param[in] buffer_size the size of the temporary read buffer
103
+ /// \param[in] pool a MemoryPool to use for allocations
104
+ /// \param[in] raw a raw InputStream
105
+ /// \param[in] raw_read_bound a bound on the maximum number of bytes
106
+ /// to read from the raw input stream. The default -1 indicates that
107
+ /// it is unbounded
108
+ /// \return the created BufferedInputStream
109
+ static Result<std::shared_ptr<BufferedInputStream>> Create(
110
+ int64_t buffer_size, MemoryPool* pool, std::shared_ptr<InputStream> raw,
111
+ int64_t raw_read_bound = -1);
112
+
113
+ /// \brief Resize internal read buffer; calls to Read(...) will read at least
114
+ /// this many bytes from the raw InputStream if possible.
115
+ /// \param[in] new_buffer_size the new read buffer size
116
+ /// \return Status
117
+ Status SetBufferSize(int64_t new_buffer_size);
118
+
119
+ /// \brief Return the number of remaining bytes in the read buffer
120
+ int64_t bytes_buffered() const;
121
+
122
+ /// \brief Return the current size of the internal buffer
123
+ int64_t buffer_size() const;
124
+
125
+ /// \brief Release the raw InputStream. Any data buffered will be
126
+ /// discarded. Further operations on this object are invalid
127
+ /// \return raw the underlying InputStream
128
+ std::shared_ptr<InputStream> Detach();
129
+
130
+ /// \brief Return the unbuffered InputStream
131
+ std::shared_ptr<InputStream> raw() const;
132
+
133
+ // InputStream APIs
134
+
135
+ bool closed() const override;
136
+ Result<std::shared_ptr<const KeyValueMetadata>> ReadMetadata() override;
137
+ Future<std::shared_ptr<const KeyValueMetadata>> ReadMetadataAsync(
138
+ const IOContext& io_context) override;
139
+
140
+ private:
141
+ friend InputStreamConcurrencyWrapper<BufferedInputStream>;
142
+
143
+ explicit BufferedInputStream(std::shared_ptr<InputStream> raw, MemoryPool* pool,
144
+ int64_t raw_total_bytes_bound);
145
+
146
+ Status DoClose();
147
+ Status DoAbort() override;
148
+
149
+ /// \brief Returns the position of the buffered stream, though the position
150
+ /// of the unbuffered stream may be further advanced.
151
+ Result<int64_t> DoTell() const;
152
+
153
+ Result<int64_t> DoRead(int64_t nbytes, void* out);
154
+
155
+ /// \brief Read into buffer.
156
+ Result<std::shared_ptr<Buffer>> DoRead(int64_t nbytes);
157
+
158
+ /// \brief Return a zero-copy string view referencing buffered data,
159
+ /// but do not advance the position of the stream. Buffers data and
160
+ /// expands the buffer size if necessary
161
+ Result<std::string_view> DoPeek(int64_t nbytes) override;
162
+
163
+ class ARROW_NO_EXPORT Impl;
164
+ std::unique_ptr<Impl> impl_;
165
+ };
166
+
167
+ } // namespace io
168
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/caching.h ADDED
@@ -0,0 +1,157 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <cstdint>
21
+ #include <memory>
22
+ #include <string>
23
+ #include <utility>
24
+ #include <vector>
25
+
26
+ #include "arrow/io/interfaces.h"
27
+ #include "arrow/util/type_fwd.h"
28
+ #include "arrow/util/visibility.h"
29
+
30
+ namespace arrow {
31
+ namespace io {
32
+
33
+ struct ARROW_EXPORT CacheOptions {
34
+ static constexpr double kDefaultIdealBandwidthUtilizationFrac = 0.9;
35
+ static constexpr int64_t kDefaultMaxIdealRequestSizeMib = 64;
36
+
37
+ /// \brief The maximum distance in bytes between two consecutive
38
+ /// ranges; beyond this value, ranges are not combined
39
+ int64_t hole_size_limit;
40
+ /// \brief The maximum size in bytes of a combined range; if
41
+ /// combining two consecutive ranges would produce a range of a
42
+ /// size greater than this, they are not combined
43
+ int64_t range_size_limit;
44
+ /// \brief A lazy cache does not perform any I/O until requested.
45
+ /// lazy = false: request all byte ranges when PreBuffer or WillNeed is called.
46
+ /// lazy = True, prefetch_limit = 0: request merged byte ranges only after the reader
47
+ /// needs them.
48
+ /// lazy = True, prefetch_limit = k: prefetch up to k merged byte ranges ahead of the
49
+ /// range that is currently being read.
50
+ bool lazy;
51
+ /// \brief The maximum number of ranges to be prefetched. This is only used
52
+ /// for lazy cache to asynchronously read some ranges after reading the target range.
53
+ int64_t prefetch_limit = 0;
54
+
55
+ bool operator==(const CacheOptions& other) const {
56
+ return hole_size_limit == other.hole_size_limit &&
57
+ range_size_limit == other.range_size_limit && lazy == other.lazy &&
58
+ prefetch_limit == other.prefetch_limit;
59
+ }
60
+
61
+ /// \brief Construct CacheOptions from network storage metrics (e.g. S3).
62
+ ///
63
+ /// \param[in] time_to_first_byte_millis Seek-time or Time-To-First-Byte (TTFB) in
64
+ /// milliseconds, also called call setup latency of a new read request.
65
+ /// The value is a positive integer.
66
+ /// \param[in] transfer_bandwidth_mib_per_sec Data transfer Bandwidth (BW) in MiB/sec
67
+ /// (per connection).
68
+ /// The value is a positive integer.
69
+ /// \param[in] ideal_bandwidth_utilization_frac Transfer bandwidth utilization fraction
70
+ /// (per connection) to maximize the net data load.
71
+ /// The value is a positive double precision number less than 1.
72
+ /// \param[in] max_ideal_request_size_mib The maximum single data request size (in MiB)
73
+ /// to maximize the net data load.
74
+ /// The value is a positive integer.
75
+ /// \return A new instance of CacheOptions.
76
+ static CacheOptions MakeFromNetworkMetrics(
77
+ int64_t time_to_first_byte_millis, int64_t transfer_bandwidth_mib_per_sec,
78
+ double ideal_bandwidth_utilization_frac = kDefaultIdealBandwidthUtilizationFrac,
79
+ int64_t max_ideal_request_size_mib = kDefaultMaxIdealRequestSizeMib);
80
+
81
+ static CacheOptions Defaults();
82
+ static CacheOptions LazyDefaults();
83
+ };
84
+
85
+ namespace internal {
86
+
87
+ /// \brief A read cache designed to hide IO latencies when reading.
88
+ ///
89
+ /// This class takes multiple byte ranges that an application expects to read, and
90
+ /// coalesces them into fewer, larger read requests, which benefits performance on some
91
+ /// filesystems, particularly remote ones like Amazon S3. By default, it also issues
92
+ /// these read requests in parallel up front.
93
+ ///
94
+ /// To use:
95
+ /// 1. Cache() the ranges you expect to read in the future. Ideally, these ranges have
96
+ /// the exact offset and length that will later be read. The cache will combine those
97
+ /// ranges according to parameters (see constructor).
98
+ ///
99
+ /// By default, the cache will also start fetching the combined ranges in parallel in
100
+ /// the background, unless CacheOptions.lazy is set.
101
+ ///
102
+ /// 2. Call WaitFor() to be notified when the given ranges have been read. If
103
+ /// CacheOptions.lazy is set, I/O will be triggered in the background here instead.
104
+ /// This can be done in parallel (e.g. if parsing a file, call WaitFor() for each
105
+ /// chunk of the file that can be parsed in parallel).
106
+ ///
107
+ /// 3. Call Read() to retrieve the actual data for the given ranges.
108
+ /// A synchronous application may skip WaitFor() and just call Read() - it will still
109
+ /// benefit from coalescing and parallel fetching.
110
+ class ARROW_EXPORT ReadRangeCache {
111
+ public:
112
+ static constexpr int64_t kDefaultHoleSizeLimit = 8192;
113
+ static constexpr int64_t kDefaultRangeSizeLimit = 32 * 1024 * 1024;
114
+
115
+ /// Construct a read cache with default
116
+ explicit ReadRangeCache(std::shared_ptr<RandomAccessFile> file, IOContext ctx)
117
+ : ReadRangeCache(file, file.get(), std::move(ctx), CacheOptions::Defaults()) {}
118
+
119
+ /// Construct a read cache with given options
120
+ explicit ReadRangeCache(std::shared_ptr<RandomAccessFile> file, IOContext ctx,
121
+ CacheOptions options)
122
+ : ReadRangeCache(file, file.get(), std::move(ctx), options) {}
123
+
124
+ /// Construct a read cache with an unowned file
125
+ ReadRangeCache(RandomAccessFile* file, IOContext ctx, CacheOptions options)
126
+ : ReadRangeCache(NULLPTR, file, std::move(ctx), options) {}
127
+
128
+ ~ReadRangeCache();
129
+
130
+ /// \brief Cache the given ranges in the background.
131
+ ///
132
+ /// The caller must ensure that the ranges do not overlap with each other,
133
+ /// nor with previously cached ranges. Otherwise, behaviour will be undefined.
134
+ Status Cache(std::vector<ReadRange> ranges);
135
+
136
+ /// \brief Read a range previously given to Cache().
137
+ Result<std::shared_ptr<Buffer>> Read(ReadRange range);
138
+
139
+ /// \brief Wait until all ranges added so far have been cached.
140
+ Future<> Wait();
141
+
142
+ /// \brief Wait until all given ranges have been cached.
143
+ Future<> WaitFor(std::vector<ReadRange> ranges);
144
+
145
+ protected:
146
+ struct Impl;
147
+ struct LazyImpl;
148
+
149
+ ReadRangeCache(std::shared_ptr<RandomAccessFile> owned_file, RandomAccessFile* file,
150
+ IOContext ctx, CacheOptions options);
151
+
152
+ std::unique_ptr<Impl> impl_;
153
+ };
154
+
155
+ } // namespace internal
156
+ } // namespace io
157
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/compressed.h ADDED
@@ -0,0 +1,124 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ // Compressed stream implementations
19
+
20
+ #pragma once
21
+
22
+ #include <memory>
23
+ #include <string>
24
+
25
+ #include "arrow/io/concurrency.h"
26
+ #include "arrow/io/interfaces.h"
27
+ #include "arrow/util/visibility.h"
28
+
29
+ namespace arrow {
30
+
31
+ class MemoryPool;
32
+ class Status;
33
+
34
+ namespace util {
35
+
36
+ class Codec;
37
+
38
+ } // namespace util
39
+
40
+ namespace io {
41
+
42
+ class ARROW_EXPORT CompressedOutputStream : public OutputStream {
43
+ public:
44
+ ~CompressedOutputStream() override;
45
+
46
+ /// \brief Create a compressed output stream wrapping the given output stream.
47
+ ///
48
+ /// The codec must be capable of streaming compression. Some codecs,
49
+ /// like Snappy, are not able to do so.
50
+ static Result<std::shared_ptr<CompressedOutputStream>> Make(
51
+ util::Codec* codec, const std::shared_ptr<OutputStream>& raw,
52
+ MemoryPool* pool = default_memory_pool());
53
+
54
+ // OutputStream interface
55
+
56
+ /// \brief Close the compressed output stream. This implicitly closes the
57
+ /// underlying raw output stream.
58
+ Status Close() override;
59
+ Status Abort() override;
60
+ bool closed() const override;
61
+
62
+ Result<int64_t> Tell() const override;
63
+
64
+ Status Write(const void* data, int64_t nbytes) override;
65
+ /// \cond FALSE
66
+ using Writable::Write;
67
+ /// \endcond
68
+ Status Flush() override;
69
+
70
+ /// \brief Return the underlying raw output stream.
71
+ std::shared_ptr<OutputStream> raw() const;
72
+
73
+ private:
74
+ ARROW_DISALLOW_COPY_AND_ASSIGN(CompressedOutputStream);
75
+
76
+ CompressedOutputStream() = default;
77
+
78
+ class ARROW_NO_EXPORT Impl;
79
+ std::unique_ptr<Impl> impl_;
80
+ };
81
+
82
+ class ARROW_EXPORT CompressedInputStream
83
+ : public internal::InputStreamConcurrencyWrapper<CompressedInputStream> {
84
+ public:
85
+ ~CompressedInputStream() override;
86
+
87
+ /// \brief Create a compressed input stream wrapping the given input stream.
88
+ ///
89
+ /// The codec must be capable of streaming decompression. Some codecs,
90
+ /// like Snappy, are not able to do so.
91
+ static Result<std::shared_ptr<CompressedInputStream>> Make(
92
+ util::Codec* codec, const std::shared_ptr<InputStream>& raw,
93
+ MemoryPool* pool = default_memory_pool());
94
+
95
+ // InputStream interface
96
+
97
+ bool closed() const override;
98
+ Result<std::shared_ptr<const KeyValueMetadata>> ReadMetadata() override;
99
+ Future<std::shared_ptr<const KeyValueMetadata>> ReadMetadataAsync(
100
+ const IOContext& io_context) override;
101
+
102
+ /// \brief Return the underlying raw input stream.
103
+ std::shared_ptr<InputStream> raw() const;
104
+
105
+ private:
106
+ friend InputStreamConcurrencyWrapper<CompressedInputStream>;
107
+ ARROW_DISALLOW_COPY_AND_ASSIGN(CompressedInputStream);
108
+
109
+ CompressedInputStream() = default;
110
+
111
+ /// \brief Close the compressed input stream. This implicitly closes the
112
+ /// underlying raw input stream.
113
+ Status DoClose();
114
+ Status DoAbort() override;
115
+ Result<int64_t> DoTell() const;
116
+ Result<int64_t> DoRead(int64_t nbytes, void* out);
117
+ Result<std::shared_ptr<Buffer>> DoRead(int64_t nbytes);
118
+
119
+ class ARROW_NO_EXPORT Impl;
120
+ std::unique_ptr<Impl> impl_;
121
+ };
122
+
123
+ } // namespace io
124
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/concurrency.h ADDED
@@ -0,0 +1,263 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <memory>
21
+
22
+ #include "arrow/io/interfaces.h"
23
+ #include "arrow/result.h"
24
+ #include "arrow/status.h"
25
+ #include "arrow/util/checked_cast.h"
26
+ #include "arrow/util/macros.h"
27
+ #include "arrow/util/visibility.h"
28
+
29
+ namespace arrow {
30
+ namespace io {
31
+ namespace internal {
32
+
33
+ template <class LockType>
34
+ class SharedLockGuard {
35
+ public:
36
+ explicit SharedLockGuard(LockType* lock) : lock_(lock) { lock_->LockShared(); }
37
+
38
+ ~SharedLockGuard() { lock_->UnlockShared(); }
39
+
40
+ protected:
41
+ LockType* lock_;
42
+ };
43
+
44
+ template <class LockType>
45
+ class ExclusiveLockGuard {
46
+ public:
47
+ explicit ExclusiveLockGuard(LockType* lock) : lock_(lock) { lock_->LockExclusive(); }
48
+
49
+ ~ExclusiveLockGuard() { lock_->UnlockExclusive(); }
50
+
51
+ protected:
52
+ LockType* lock_;
53
+ };
54
+
55
+ // Debug concurrency checker that marks "shared" and "exclusive" code sections,
56
+ // aborting if the concurrency rules get violated. Does nothing in release mode.
57
+ // Note that we intentionally use the same class declaration in debug and
58
+ // release builds in order to avoid runtime failures when e.g. loading a
59
+ // release-built DLL with a debug-built application, or the reverse.
60
+
61
+ class ARROW_EXPORT SharedExclusiveChecker {
62
+ public:
63
+ SharedExclusiveChecker();
64
+ void LockShared();
65
+ void UnlockShared();
66
+ void LockExclusive();
67
+ void UnlockExclusive();
68
+
69
+ SharedLockGuard<SharedExclusiveChecker> shared_guard() {
70
+ return SharedLockGuard<SharedExclusiveChecker>(this);
71
+ }
72
+
73
+ ExclusiveLockGuard<SharedExclusiveChecker> exclusive_guard() {
74
+ return ExclusiveLockGuard<SharedExclusiveChecker>(this);
75
+ }
76
+
77
+ protected:
78
+ struct Impl;
79
+ std::shared_ptr<Impl> impl_;
80
+ };
81
+
82
+ // Concurrency wrappers for IO classes that check the correctness of
83
+ // concurrent calls to various methods. It is not necessary to wrap all
84
+ // IO classes with these, only a few core classes that get used in tests.
85
+ //
86
+ // We're not using virtual inheritance here as virtual bases have poorly
87
+ // understood semantic overhead which we'd be passing on to implementers
88
+ // and users of these interfaces. Instead, we just duplicate the method
89
+ // wrappers between those two classes.
90
+
91
+ template <class Derived>
92
+ class InputStreamConcurrencyWrapper : public InputStream {
93
+ public:
94
+ Status Close() final {
95
+ auto guard = lock_.exclusive_guard();
96
+ return derived()->DoClose();
97
+ }
98
+
99
+ Status Abort() final {
100
+ auto guard = lock_.exclusive_guard();
101
+ return derived()->DoAbort();
102
+ }
103
+
104
+ Result<int64_t> Tell() const final {
105
+ auto guard = lock_.exclusive_guard();
106
+ return derived()->DoTell();
107
+ }
108
+
109
+ Result<int64_t> Read(int64_t nbytes, void* out) final {
110
+ auto guard = lock_.exclusive_guard();
111
+ return derived()->DoRead(nbytes, out);
112
+ }
113
+
114
+ Result<std::shared_ptr<Buffer>> Read(int64_t nbytes) final {
115
+ auto guard = lock_.exclusive_guard();
116
+ return derived()->DoRead(nbytes);
117
+ }
118
+
119
+ Result<std::string_view> Peek(int64_t nbytes) final {
120
+ auto guard = lock_.exclusive_guard();
121
+ return derived()->DoPeek(nbytes);
122
+ }
123
+
124
+ /*
125
+ Methods to implement in derived class:
126
+
127
+ Status DoClose();
128
+ Result<int64_t> DoTell() const;
129
+ Result<int64_t> DoRead(int64_t nbytes, void* out);
130
+ Result<std::shared_ptr<Buffer>> DoRead(int64_t nbytes);
131
+
132
+ And optionally:
133
+
134
+ Status DoAbort() override;
135
+ Result<std::string_view> DoPeek(int64_t nbytes) override;
136
+
137
+ These methods should be protected in the derived class and
138
+ InputStreamConcurrencyWrapper declared as a friend with
139
+
140
+ friend InputStreamConcurrencyWrapper<derived>;
141
+ */
142
+
143
+ protected:
144
+ // Default implementations. They are virtual because the derived class may
145
+ // have derived classes itself.
146
+ virtual Status DoAbort() { return derived()->DoClose(); }
147
+
148
+ virtual Result<std::string_view> DoPeek(int64_t ARROW_ARG_UNUSED(nbytes)) {
149
+ return Status::NotImplemented("Peek not implemented");
150
+ }
151
+
152
+ Derived* derived() { return ::arrow::internal::checked_cast<Derived*>(this); }
153
+
154
+ const Derived* derived() const {
155
+ return ::arrow::internal::checked_cast<const Derived*>(this);
156
+ }
157
+
158
+ mutable SharedExclusiveChecker lock_;
159
+ };
160
+
161
+ template <class Derived>
162
+ class RandomAccessFileConcurrencyWrapper : public RandomAccessFile {
163
+ public:
164
+ Status Close() final {
165
+ auto guard = lock_.exclusive_guard();
166
+ return derived()->DoClose();
167
+ }
168
+
169
+ Status Abort() final {
170
+ auto guard = lock_.exclusive_guard();
171
+ return derived()->DoAbort();
172
+ }
173
+
174
+ Result<int64_t> Tell() const final {
175
+ auto guard = lock_.exclusive_guard();
176
+ return derived()->DoTell();
177
+ }
178
+
179
+ Result<int64_t> Read(int64_t nbytes, void* out) final {
180
+ auto guard = lock_.exclusive_guard();
181
+ return derived()->DoRead(nbytes, out);
182
+ }
183
+
184
+ Result<std::shared_ptr<Buffer>> Read(int64_t nbytes) final {
185
+ auto guard = lock_.exclusive_guard();
186
+ return derived()->DoRead(nbytes);
187
+ }
188
+
189
+ Result<std::string_view> Peek(int64_t nbytes) final {
190
+ auto guard = lock_.exclusive_guard();
191
+ return derived()->DoPeek(nbytes);
192
+ }
193
+
194
+ Status Seek(int64_t position) final {
195
+ auto guard = lock_.exclusive_guard();
196
+ return derived()->DoSeek(position);
197
+ }
198
+
199
+ Result<int64_t> GetSize() final {
200
+ auto guard = lock_.shared_guard();
201
+ return derived()->DoGetSize();
202
+ }
203
+
204
+ // NOTE: ReadAt doesn't use stream pointer, but it is allowed to update it
205
+ // (it's the case on Windows when using ReadFileEx).
206
+ // So any method that relies on the current position (even if it doesn't
207
+ // update it, such as Peek) cannot run in parallel with ReadAt and has
208
+ // to use the exclusive_guard.
209
+
210
+ Result<int64_t> ReadAt(int64_t position, int64_t nbytes, void* out) final {
211
+ auto guard = lock_.shared_guard();
212
+ return derived()->DoReadAt(position, nbytes, out);
213
+ }
214
+
215
+ Result<std::shared_ptr<Buffer>> ReadAt(int64_t position, int64_t nbytes) final {
216
+ auto guard = lock_.shared_guard();
217
+ return derived()->DoReadAt(position, nbytes);
218
+ }
219
+
220
+ /*
221
+ Methods to implement in derived class:
222
+
223
+ Status DoClose();
224
+ Result<int64_t> DoTell() const;
225
+ Result<int64_t> DoRead(int64_t nbytes, void* out);
226
+ Result<std::shared_ptr<Buffer>> DoRead(int64_t nbytes);
227
+ Status DoSeek(int64_t position);
228
+ Result<int64_t> DoGetSize()
229
+ Result<int64_t> DoReadAt(int64_t position, int64_t nbytes, void* out);
230
+ Result<std::shared_ptr<Buffer>> DoReadAt(int64_t position, int64_t nbytes);
231
+
232
+ And optionally:
233
+
234
+ Status DoAbort() override;
235
+ Result<std::string_view> DoPeek(int64_t nbytes) override;
236
+
237
+ These methods should be protected in the derived class and
238
+ RandomAccessFileConcurrencyWrapper declared as a friend with
239
+
240
+ friend RandomAccessFileConcurrencyWrapper<derived>;
241
+ */
242
+
243
+ protected:
244
+ // Default implementations. They are virtual because the derived class may
245
+ // have derived classes itself.
246
+ virtual Status DoAbort() { return derived()->DoClose(); }
247
+
248
+ virtual Result<std::string_view> DoPeek(int64_t ARROW_ARG_UNUSED(nbytes)) {
249
+ return Status::NotImplemented("Peek not implemented");
250
+ }
251
+
252
+ Derived* derived() { return ::arrow::internal::checked_cast<Derived*>(this); }
253
+
254
+ const Derived* derived() const {
255
+ return ::arrow::internal::checked_cast<const Derived*>(this);
256
+ }
257
+
258
+ mutable SharedExclusiveChecker lock_;
259
+ };
260
+
261
+ } // namespace internal
262
+ } // namespace io
263
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/file.h ADDED
@@ -0,0 +1,221 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ // IO interface implementations for OS files
19
+
20
+ #pragma once
21
+
22
+ #include <cstdint>
23
+ #include <memory>
24
+ #include <string>
25
+ #include <vector>
26
+
27
+ #include "arrow/io/concurrency.h"
28
+ #include "arrow/io/interfaces.h"
29
+ #include "arrow/util/visibility.h"
30
+
31
+ namespace arrow {
32
+
33
+ class Buffer;
34
+ class MemoryPool;
35
+ class Status;
36
+
37
+ namespace io {
38
+
39
+ /// \brief An operating system file open in write-only mode.
40
+ class ARROW_EXPORT FileOutputStream : public OutputStream {
41
+ public:
42
+ ~FileOutputStream() override;
43
+
44
+ /// \brief Open a local file for writing, truncating any existing file
45
+ /// \param[in] path with UTF8 encoding
46
+ /// \param[in] append append to existing file, otherwise truncate to 0 bytes
47
+ /// \return an open FileOutputStream
48
+ ///
49
+ /// When opening a new file, any existing file with the indicated path is
50
+ /// truncated to 0 bytes, deleting any existing data
51
+ static Result<std::shared_ptr<FileOutputStream>> Open(const std::string& path,
52
+ bool append = false);
53
+
54
+ /// \brief Open a file descriptor for writing. The underlying file isn't
55
+ /// truncated.
56
+ /// \param[in] fd file descriptor
57
+ /// \return an open FileOutputStream
58
+ ///
59
+ /// The file descriptor becomes owned by the OutputStream, and will be closed
60
+ /// on Close() or destruction.
61
+ static Result<std::shared_ptr<FileOutputStream>> Open(int fd);
62
+
63
+ // OutputStream interface
64
+ Status Close() override;
65
+ bool closed() const override;
66
+ Result<int64_t> Tell() const override;
67
+
68
+ // Write bytes to the stream. Thread-safe
69
+ Status Write(const void* data, int64_t nbytes) override;
70
+ /// \cond FALSE
71
+ using Writable::Write;
72
+ /// \endcond
73
+
74
+ int file_descriptor() const;
75
+
76
+ private:
77
+ FileOutputStream();
78
+
79
+ class ARROW_NO_EXPORT FileOutputStreamImpl;
80
+ std::unique_ptr<FileOutputStreamImpl> impl_;
81
+ };
82
+
83
+ /// \brief An operating system file open in read-only mode.
84
+ ///
85
+ /// Reads through this implementation are unbuffered. If many small reads
86
+ /// need to be issued, it is recommended to use a buffering layer for good
87
+ /// performance.
88
+ class ARROW_EXPORT ReadableFile
89
+ : public internal::RandomAccessFileConcurrencyWrapper<ReadableFile> {
90
+ public:
91
+ ~ReadableFile() override;
92
+
93
+ /// \brief Open a local file for reading
94
+ /// \param[in] path with UTF8 encoding
95
+ /// \param[in] pool a MemoryPool for memory allocations
96
+ /// \return ReadableFile instance
97
+ static Result<std::shared_ptr<ReadableFile>> Open(
98
+ const std::string& path, MemoryPool* pool = default_memory_pool());
99
+
100
+ /// \brief Open a local file for reading
101
+ /// \param[in] fd file descriptor
102
+ /// \param[in] pool a MemoryPool for memory allocations
103
+ /// \return ReadableFile instance
104
+ ///
105
+ /// The file descriptor becomes owned by the ReadableFile, and will be closed
106
+ /// on Close() or destruction.
107
+ static Result<std::shared_ptr<ReadableFile>> Open(
108
+ int fd, MemoryPool* pool = default_memory_pool());
109
+
110
+ bool closed() const override;
111
+
112
+ int file_descriptor() const;
113
+
114
+ Status WillNeed(const std::vector<ReadRange>& ranges) override;
115
+
116
+ private:
117
+ friend RandomAccessFileConcurrencyWrapper<ReadableFile>;
118
+
119
+ explicit ReadableFile(MemoryPool* pool);
120
+
121
+ Status DoClose();
122
+ Result<int64_t> DoTell() const;
123
+ Result<int64_t> DoRead(int64_t nbytes, void* buffer);
124
+ Result<std::shared_ptr<Buffer>> DoRead(int64_t nbytes);
125
+
126
+ /// \brief Thread-safe implementation of ReadAt
127
+ Result<int64_t> DoReadAt(int64_t position, int64_t nbytes, void* out);
128
+
129
+ /// \brief Thread-safe implementation of ReadAt
130
+ Result<std::shared_ptr<Buffer>> DoReadAt(int64_t position, int64_t nbytes);
131
+
132
+ Result<int64_t> DoGetSize();
133
+ Status DoSeek(int64_t position);
134
+
135
+ class ARROW_NO_EXPORT ReadableFileImpl;
136
+ std::unique_ptr<ReadableFileImpl> impl_;
137
+ };
138
+
139
+ /// \brief A file interface that uses memory-mapped files for memory interactions
140
+ ///
141
+ /// This implementation supports zero-copy reads. The same class is used
142
+ /// for both reading and writing.
143
+ ///
144
+ /// If opening a file in a writable mode, it is not truncated first as with
145
+ /// FileOutputStream.
146
+ class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
147
+ public:
148
+ ~MemoryMappedFile() override;
149
+
150
+ /// Create new file with indicated size, return in read/write mode
151
+ static Result<std::shared_ptr<MemoryMappedFile>> Create(const std::string& path,
152
+ int64_t size);
153
+
154
+ // mmap() with whole file
155
+ static Result<std::shared_ptr<MemoryMappedFile>> Open(const std::string& path,
156
+ FileMode::type mode);
157
+
158
+ // mmap() with a region of file, the offset must be a multiple of the page size
159
+ static Result<std::shared_ptr<MemoryMappedFile>> Open(const std::string& path,
160
+ FileMode::type mode,
161
+ const int64_t offset,
162
+ const int64_t length);
163
+
164
+ Status Close() override;
165
+
166
+ bool closed() const override;
167
+
168
+ Result<int64_t> Tell() const override;
169
+
170
+ Status Seek(int64_t position) override;
171
+
172
+ // Required by RandomAccessFile, copies memory into out. Not thread-safe
173
+ Result<int64_t> Read(int64_t nbytes, void* out) override;
174
+
175
+ // Zero copy read, moves position pointer. Not thread-safe
176
+ Result<std::shared_ptr<Buffer>> Read(int64_t nbytes) override;
177
+
178
+ // Zero-copy read, leaves position unchanged. Acquires a reader lock
179
+ // for the duration of slice creation (typically very short). Is thread-safe.
180
+ Result<std::shared_ptr<Buffer>> ReadAt(int64_t position, int64_t nbytes) override;
181
+
182
+ // Raw copy of the memory at specified position. Thread-safe, but
183
+ // locks out other readers for the duration of memcpy. Prefer the
184
+ // zero copy method
185
+ Result<int64_t> ReadAt(int64_t position, int64_t nbytes, void* out) override;
186
+
187
+ // Synchronous ReadAsync override
188
+ Future<std::shared_ptr<Buffer>> ReadAsync(const IOContext&, int64_t position,
189
+ int64_t nbytes) override;
190
+
191
+ Status WillNeed(const std::vector<ReadRange>& ranges) override;
192
+
193
+ bool supports_zero_copy() const override;
194
+
195
+ /// Write data at the current position in the file. Thread-safe
196
+ Status Write(const void* data, int64_t nbytes) override;
197
+ /// \cond FALSE
198
+ using Writable::Write;
199
+ /// \endcond
200
+
201
+ /// Set the size of the map to new_size.
202
+ Status Resize(int64_t new_size);
203
+
204
+ /// Write data at a particular position in the file. Thread-safe
205
+ Status WriteAt(int64_t position, const void* data, int64_t nbytes) override;
206
+
207
+ Result<int64_t> GetSize() override;
208
+
209
+ int file_descriptor() const;
210
+
211
+ private:
212
+ MemoryMappedFile();
213
+
214
+ Status WriteInternal(const void* data, int64_t nbytes);
215
+
216
+ class ARROW_NO_EXPORT MemoryMap;
217
+ std::shared_ptr<MemoryMap> memory_map_;
218
+ };
219
+
220
+ } // namespace io
221
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/hdfs.h ADDED
@@ -0,0 +1,284 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <cstdint>
21
+ #include <memory>
22
+ #include <string>
23
+ #include <unordered_map>
24
+ #include <vector>
25
+
26
+ #include "arrow/io/interfaces.h"
27
+ #include "arrow/util/macros.h"
28
+ #include "arrow/util/visibility.h"
29
+
30
+ namespace arrow {
31
+
32
+ class Buffer;
33
+ class MemoryPool;
34
+ class Status;
35
+
36
+ namespace io {
37
+
38
+ class HdfsReadableFile;
39
+ class HdfsOutputStream;
40
+
41
+ /// DEPRECATED. Use the FileSystem API in arrow::fs instead.
42
+ struct ObjectType {
43
+ enum type { FILE, DIRECTORY };
44
+ };
45
+
46
+ /// DEPRECATED. Use the FileSystem API in arrow::fs instead.
47
+ struct ARROW_EXPORT FileStatistics {
48
+ /// Size of file, -1 if finding length is unsupported
49
+ int64_t size;
50
+ ObjectType::type kind;
51
+ };
52
+
53
+ class ARROW_EXPORT FileSystem {
54
+ public:
55
+ virtual ~FileSystem() = default;
56
+
57
+ virtual Status MakeDirectory(const std::string& path) = 0;
58
+
59
+ virtual Status DeleteDirectory(const std::string& path) = 0;
60
+
61
+ virtual Status GetChildren(const std::string& path,
62
+ std::vector<std::string>* listing) = 0;
63
+
64
+ virtual Status Rename(const std::string& src, const std::string& dst) = 0;
65
+
66
+ virtual Status Stat(const std::string& path, FileStatistics* stat) = 0;
67
+ };
68
+
69
+ struct HdfsPathInfo {
70
+ ObjectType::type kind;
71
+
72
+ std::string name;
73
+ std::string owner;
74
+ std::string group;
75
+
76
+ // Access times in UNIX timestamps (seconds)
77
+ int64_t size;
78
+ int64_t block_size;
79
+
80
+ int32_t last_modified_time;
81
+ int32_t last_access_time;
82
+
83
+ int16_t replication;
84
+ int16_t permissions;
85
+ };
86
+
87
+ struct HdfsConnectionConfig {
88
+ std::string host;
89
+ int port;
90
+ std::string user;
91
+ std::string kerb_ticket;
92
+ std::unordered_map<std::string, std::string> extra_conf;
93
+ };
94
+
95
+ class ARROW_EXPORT HadoopFileSystem : public FileSystem {
96
+ public:
97
+ ~HadoopFileSystem() override;
98
+
99
+ // Connect to an HDFS cluster given a configuration
100
+ //
101
+ // @param config (in): configuration for connecting
102
+ // @param fs (out): the created client
103
+ // @returns Status
104
+ static Status Connect(const HdfsConnectionConfig* config,
105
+ std::shared_ptr<HadoopFileSystem>* fs);
106
+
107
+ // Create directory and all parents
108
+ //
109
+ // @param path (in): absolute HDFS path
110
+ // @returns Status
111
+ Status MakeDirectory(const std::string& path) override;
112
+
113
+ // Delete file or directory
114
+ // @param path absolute path to data
115
+ // @param recursive if path is a directory, delete contents as well
116
+ // @returns error status on failure
117
+ Status Delete(const std::string& path, bool recursive = false);
118
+
119
+ Status DeleteDirectory(const std::string& path) override;
120
+
121
+ // Disconnect from cluster
122
+ //
123
+ // @returns Status
124
+ Status Disconnect();
125
+
126
+ // @param path (in): absolute HDFS path
127
+ // @returns bool, true if the path exists, false if not (or on error)
128
+ bool Exists(const std::string& path);
129
+
130
+ // @param path (in): absolute HDFS path
131
+ // @param info (out)
132
+ // @returns Status
133
+ Status GetPathInfo(const std::string& path, HdfsPathInfo* info);
134
+
135
+ // @param nbytes (out): total capacity of the filesystem
136
+ // @returns Status
137
+ Status GetCapacity(int64_t* nbytes);
138
+
139
+ // @param nbytes (out): total bytes used of the filesystem
140
+ // @returns Status
141
+ Status GetUsed(int64_t* nbytes);
142
+
143
+ Status GetChildren(const std::string& path, std::vector<std::string>* listing) override;
144
+
145
+ /// List directory contents
146
+ ///
147
+ /// If path is a relative path, returned values will be absolute paths or URIs
148
+ /// starting from the current working directory.
149
+ Status ListDirectory(const std::string& path, std::vector<HdfsPathInfo>* listing);
150
+
151
+ /// Return the filesystem's current working directory.
152
+ ///
153
+ /// The working directory is the base path for all relative paths given to
154
+ /// other APIs.
155
+ /// NOTE: this actually returns a URI.
156
+ Status GetWorkingDirectory(std::string* out);
157
+
158
+ /// Change
159
+ ///
160
+ /// @param path file path to change
161
+ /// @param owner pass null for no change
162
+ /// @param group pass null for no change
163
+ Status Chown(const std::string& path, const char* owner, const char* group);
164
+
165
+ /// Change path permissions
166
+ ///
167
+ /// \param path Absolute path in file system
168
+ /// \param mode Mode bitset
169
+ /// \return Status
170
+ Status Chmod(const std::string& path, int mode);
171
+
172
+ // Move file or directory from source path to destination path within the
173
+ // current filesystem
174
+ Status Rename(const std::string& src, const std::string& dst) override;
175
+
176
+ Status Copy(const std::string& src, const std::string& dst);
177
+
178
+ Status Move(const std::string& src, const std::string& dst);
179
+
180
+ Status Stat(const std::string& path, FileStatistics* stat) override;
181
+
182
+ // TODO(wesm): GetWorkingDirectory, SetWorkingDirectory
183
+
184
+ // Open an HDFS file in READ mode. Returns error
185
+ // status if the file is not found.
186
+ //
187
+ // @param path complete file path
188
+ Status OpenReadable(const std::string& path, int32_t buffer_size,
189
+ std::shared_ptr<HdfsReadableFile>* file);
190
+
191
+ Status OpenReadable(const std::string& path, int32_t buffer_size,
192
+ const io::IOContext& io_context,
193
+ std::shared_ptr<HdfsReadableFile>* file);
194
+
195
+ Status OpenReadable(const std::string& path, std::shared_ptr<HdfsReadableFile>* file);
196
+
197
+ Status OpenReadable(const std::string& path, const io::IOContext& io_context,
198
+ std::shared_ptr<HdfsReadableFile>* file);
199
+
200
+ // FileMode::WRITE options
201
+ // @param path complete file path
202
+ // @param buffer_size 0 by default
203
+ // @param replication 0 by default
204
+ // @param default_block_size 0 by default
205
+ Status OpenWritable(const std::string& path, bool append, int32_t buffer_size,
206
+ int16_t replication, int64_t default_block_size,
207
+ std::shared_ptr<HdfsOutputStream>* file);
208
+
209
+ Status OpenWritable(const std::string& path, bool append,
210
+ std::shared_ptr<HdfsOutputStream>* file);
211
+
212
+ private:
213
+ friend class HdfsReadableFile;
214
+ friend class HdfsOutputStream;
215
+
216
+ class ARROW_NO_EXPORT HadoopFileSystemImpl;
217
+ std::unique_ptr<HadoopFileSystemImpl> impl_;
218
+
219
+ HadoopFileSystem();
220
+ ARROW_DISALLOW_COPY_AND_ASSIGN(HadoopFileSystem);
221
+ };
222
+
223
+ class ARROW_EXPORT HdfsReadableFile : public RandomAccessFile {
224
+ public:
225
+ ~HdfsReadableFile() override;
226
+
227
+ Status Close() override;
228
+
229
+ bool closed() const override;
230
+
231
+ // NOTE: If you wish to read a particular range of a file in a multithreaded
232
+ // context, you may prefer to use ReadAt to avoid locking issues
233
+ Result<int64_t> Read(int64_t nbytes, void* out) override;
234
+ Result<std::shared_ptr<Buffer>> Read(int64_t nbytes) override;
235
+ Result<int64_t> ReadAt(int64_t position, int64_t nbytes, void* out) override;
236
+ Result<std::shared_ptr<Buffer>> ReadAt(int64_t position, int64_t nbytes) override;
237
+
238
+ Status Seek(int64_t position) override;
239
+ Result<int64_t> Tell() const override;
240
+ Result<int64_t> GetSize() override;
241
+
242
+ private:
243
+ explicit HdfsReadableFile(const io::IOContext&);
244
+
245
+ class ARROW_NO_EXPORT HdfsReadableFileImpl;
246
+ std::unique_ptr<HdfsReadableFileImpl> impl_;
247
+
248
+ friend class HadoopFileSystem::HadoopFileSystemImpl;
249
+
250
+ ARROW_DISALLOW_COPY_AND_ASSIGN(HdfsReadableFile);
251
+ };
252
+
253
+ // Naming this file OutputStream because it does not support seeking (like the
254
+ // WritableFile interface)
255
+ class ARROW_EXPORT HdfsOutputStream : public OutputStream {
256
+ public:
257
+ ~HdfsOutputStream() override;
258
+
259
+ Status Close() override;
260
+
261
+ bool closed() const override;
262
+
263
+ using OutputStream::Write;
264
+ Status Write(const void* buffer, int64_t nbytes) override;
265
+
266
+ Status Flush() override;
267
+
268
+ Result<int64_t> Tell() const override;
269
+
270
+ private:
271
+ class ARROW_NO_EXPORT HdfsOutputStreamImpl;
272
+ std::unique_ptr<HdfsOutputStreamImpl> impl_;
273
+
274
+ friend class HadoopFileSystem::HadoopFileSystemImpl;
275
+
276
+ HdfsOutputStream();
277
+
278
+ ARROW_DISALLOW_COPY_AND_ASSIGN(HdfsOutputStream);
279
+ };
280
+
281
+ ARROW_EXPORT Status HaveLibHdfs();
282
+
283
+ } // namespace io
284
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/interfaces.h ADDED
@@ -0,0 +1,362 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <cstdint>
21
+ #include <memory>
22
+ #include <string>
23
+ #include <string_view>
24
+ #include <vector>
25
+
26
+ #include "arrow/io/type_fwd.h"
27
+ #include "arrow/type_fwd.h"
28
+ #include "arrow/util/cancel.h"
29
+ #include "arrow/util/macros.h"
30
+ #include "arrow/util/type_fwd.h"
31
+ #include "arrow/util/visibility.h"
32
+
33
+ namespace arrow {
34
+ namespace io {
35
+
36
+ struct ReadRange {
37
+ int64_t offset;
38
+ int64_t length;
39
+
40
+ friend bool operator==(const ReadRange& left, const ReadRange& right) {
41
+ return (left.offset == right.offset && left.length == right.length);
42
+ }
43
+ friend bool operator!=(const ReadRange& left, const ReadRange& right) {
44
+ return !(left == right);
45
+ }
46
+
47
+ bool Contains(const ReadRange& other) const {
48
+ return (offset <= other.offset && offset + length >= other.offset + other.length);
49
+ }
50
+ };
51
+
52
+ /// EXPERIMENTAL: options provider for IO tasks
53
+ ///
54
+ /// Includes an Executor (which will be used to execute asynchronous reads),
55
+ /// a MemoryPool (which will be used to allocate buffers when zero copy reads
56
+ /// are not possible), and an external id (in case the executor receives tasks from
57
+ /// multiple sources and must distinguish tasks associated with this IOContext).
58
+ struct ARROW_EXPORT IOContext {
59
+ // No specified executor: will use a global IO thread pool
60
+ IOContext() : IOContext(default_memory_pool(), StopToken::Unstoppable()) {}
61
+
62
+ explicit IOContext(StopToken stop_token)
63
+ : IOContext(default_memory_pool(), std::move(stop_token)) {}
64
+
65
+ explicit IOContext(MemoryPool* pool, StopToken stop_token = StopToken::Unstoppable());
66
+
67
+ explicit IOContext(MemoryPool* pool, ::arrow::internal::Executor* executor,
68
+ StopToken stop_token = StopToken::Unstoppable(),
69
+ int64_t external_id = -1)
70
+ : pool_(pool),
71
+ executor_(executor),
72
+ external_id_(external_id),
73
+ stop_token_(std::move(stop_token)) {}
74
+
75
+ explicit IOContext(::arrow::internal::Executor* executor,
76
+ StopToken stop_token = StopToken::Unstoppable(),
77
+ int64_t external_id = -1)
78
+ : pool_(default_memory_pool()),
79
+ executor_(executor),
80
+ external_id_(external_id),
81
+ stop_token_(std::move(stop_token)) {}
82
+
83
+ MemoryPool* pool() const { return pool_; }
84
+
85
+ ::arrow::internal::Executor* executor() const { return executor_; }
86
+
87
+ // An application-specific ID, forwarded to executor task submissions
88
+ int64_t external_id() const { return external_id_; }
89
+
90
+ StopToken stop_token() const { return stop_token_; }
91
+
92
+ private:
93
+ MemoryPool* pool_;
94
+ ::arrow::internal::Executor* executor_;
95
+ int64_t external_id_;
96
+ StopToken stop_token_;
97
+ };
98
+
99
+ class ARROW_EXPORT FileInterface : public std::enable_shared_from_this<FileInterface> {
100
+ public:
101
+ virtual ~FileInterface() = 0;
102
+
103
+ /// \brief Close the stream cleanly
104
+ ///
105
+ /// For writable streams, this will attempt to flush any pending data
106
+ /// before releasing the underlying resource.
107
+ ///
108
+ /// After Close() is called, closed() returns true and the stream is not
109
+ /// available for further operations.
110
+ virtual Status Close() = 0;
111
+
112
+ /// \brief Close the stream asynchronously
113
+ ///
114
+ /// By default, this will just submit the synchronous Close() to the
115
+ /// default I/O thread pool. Subclasses may implement this in a more
116
+ /// efficient manner.
117
+ virtual Future<> CloseAsync();
118
+
119
+ /// \brief Close the stream abruptly
120
+ ///
121
+ /// This method does not guarantee that any pending data is flushed.
122
+ /// It merely releases any underlying resource used by the stream for
123
+ /// its operation.
124
+ ///
125
+ /// After Abort() is called, closed() returns true and the stream is not
126
+ /// available for further operations.
127
+ virtual Status Abort();
128
+
129
+ /// \brief Return the position in this stream
130
+ virtual Result<int64_t> Tell() const = 0;
131
+
132
+ /// \brief Return whether the stream is closed
133
+ virtual bool closed() const = 0;
134
+
135
+ FileMode::type mode() const { return mode_; }
136
+
137
+ protected:
138
+ FileInterface() : mode_(FileMode::READ) {}
139
+ FileMode::type mode_;
140
+ void set_mode(FileMode::type mode) { mode_ = mode; }
141
+
142
+ private:
143
+ ARROW_DISALLOW_COPY_AND_ASSIGN(FileInterface);
144
+ };
145
+
146
+ class ARROW_EXPORT Seekable {
147
+ public:
148
+ virtual ~Seekable() = default;
149
+ virtual Status Seek(int64_t position) = 0;
150
+ };
151
+
152
+ class ARROW_EXPORT Writable {
153
+ public:
154
+ virtual ~Writable() = default;
155
+
156
+ /// \brief Write the given data to the stream
157
+ ///
158
+ /// This method always processes the bytes in full. Depending on the
159
+ /// semantics of the stream, the data may be written out immediately,
160
+ /// held in a buffer, or written asynchronously. In the case where
161
+ /// the stream buffers the data, it will be copied. To avoid potentially
162
+ /// large copies, use the Write variant that takes an owned Buffer.
163
+ virtual Status Write(const void* data, int64_t nbytes) = 0;
164
+
165
+ /// \brief Write the given data to the stream
166
+ ///
167
+ /// Since the Buffer owns its memory, this method can avoid a copy if
168
+ /// buffering is required. See Write(const void*, int64_t) for details.
169
+ virtual Status Write(const std::shared_ptr<Buffer>& data);
170
+
171
+ /// \brief Flush buffered bytes, if any
172
+ virtual Status Flush();
173
+
174
+ Status Write(std::string_view data);
175
+ };
176
+
177
+ class ARROW_EXPORT Readable {
178
+ public:
179
+ virtual ~Readable() = default;
180
+
181
+ /// \brief Read data from current file position.
182
+ ///
183
+ /// Read at most `nbytes` from the current file position into `out`.
184
+ /// The number of bytes read is returned.
185
+ virtual Result<int64_t> Read(int64_t nbytes, void* out) = 0;
186
+
187
+ /// \brief Read data from current file position.
188
+ ///
189
+ /// Read at most `nbytes` from the current file position. Less bytes may
190
+ /// be read if EOF is reached. This method updates the current file position.
191
+ ///
192
+ /// In some cases (e.g. a memory-mapped file), this method may avoid a
193
+ /// memory copy.
194
+ virtual Result<std::shared_ptr<Buffer>> Read(int64_t nbytes) = 0;
195
+
196
+ /// EXPERIMENTAL: The IOContext associated with this file.
197
+ ///
198
+ /// By default, this is the same as default_io_context(), but it may be
199
+ /// overridden by subclasses.
200
+ virtual const IOContext& io_context() const;
201
+ };
202
+
203
+ class ARROW_EXPORT OutputStream : virtual public FileInterface, public Writable {
204
+ protected:
205
+ OutputStream() = default;
206
+ };
207
+
208
+ class ARROW_EXPORT InputStream : virtual public FileInterface, virtual public Readable {
209
+ public:
210
+ /// \brief Advance or skip stream indicated number of bytes
211
+ /// \param[in] nbytes the number to move forward
212
+ /// \return Status
213
+ Status Advance(int64_t nbytes);
214
+
215
+ /// \brief Return zero-copy string_view to upcoming bytes.
216
+ ///
217
+ /// Do not modify the stream position. The view becomes invalid after
218
+ /// any operation on the stream. May trigger buffering if the requested
219
+ /// size is larger than the number of buffered bytes.
220
+ ///
221
+ /// May return NotImplemented on streams that don't support it.
222
+ ///
223
+ /// \param[in] nbytes the maximum number of bytes to see
224
+ virtual Result<std::string_view> Peek(int64_t nbytes);
225
+
226
+ /// \brief Return true if InputStream is capable of zero copy Buffer reads
227
+ ///
228
+ /// Zero copy reads imply the use of Buffer-returning Read() overloads.
229
+ virtual bool supports_zero_copy() const;
230
+
231
+ /// \brief Read and return stream metadata
232
+ ///
233
+ /// If the stream implementation doesn't support metadata, empty metadata
234
+ /// is returned. Note that it is allowed to return a null pointer rather
235
+ /// than an allocated empty metadata.
236
+ virtual Result<std::shared_ptr<const KeyValueMetadata>> ReadMetadata();
237
+
238
+ /// \brief Read stream metadata asynchronously
239
+ virtual Future<std::shared_ptr<const KeyValueMetadata>> ReadMetadataAsync(
240
+ const IOContext& io_context);
241
+ Future<std::shared_ptr<const KeyValueMetadata>> ReadMetadataAsync();
242
+
243
+ protected:
244
+ InputStream() = default;
245
+ };
246
+
247
+ class ARROW_EXPORT RandomAccessFile : public InputStream, public Seekable {
248
+ public:
249
+ /// Necessary because we hold a std::unique_ptr
250
+ ~RandomAccessFile() override;
251
+
252
+ /// \brief Create an isolated InputStream that reads a segment of a
253
+ /// RandomAccessFile. Multiple such stream can be created and used
254
+ /// independently without interference
255
+ /// \param[in] file a file instance
256
+ /// \param[in] file_offset the starting position in the file
257
+ /// \param[in] nbytes the extent of bytes to read. The file should have
258
+ /// sufficient bytes available
259
+ static Result<std::shared_ptr<InputStream>> GetStream(
260
+ std::shared_ptr<RandomAccessFile> file, int64_t file_offset, int64_t nbytes);
261
+
262
+ /// \brief Return the total file size in bytes.
263
+ ///
264
+ /// This method does not read or move the current file position, so is safe
265
+ /// to call concurrently with e.g. ReadAt().
266
+ virtual Result<int64_t> GetSize() = 0;
267
+
268
+ /// \brief Read data from given file position.
269
+ ///
270
+ /// At most `nbytes` bytes are read. The number of bytes read is returned
271
+ /// (it can be less than `nbytes` if EOF is reached).
272
+ ///
273
+ /// This method can be safely called from multiple threads concurrently.
274
+ /// It is unspecified whether this method updates the file position or not.
275
+ ///
276
+ /// The default RandomAccessFile-provided implementation uses Seek() and Read(),
277
+ /// but subclasses may override it with a more efficient implementation
278
+ /// that doesn't depend on implicit file positioning.
279
+ ///
280
+ /// \param[in] position Where to read bytes from
281
+ /// \param[in] nbytes The number of bytes to read
282
+ /// \param[out] out The buffer to read bytes into
283
+ /// \return The number of bytes read, or an error
284
+ virtual Result<int64_t> ReadAt(int64_t position, int64_t nbytes, void* out);
285
+
286
+ /// \brief Read data from given file position.
287
+ ///
288
+ /// At most `nbytes` bytes are read, but it can be less if EOF is reached.
289
+ ///
290
+ /// \param[in] position Where to read bytes from
291
+ /// \param[in] nbytes The number of bytes to read
292
+ /// \return A buffer containing the bytes read, or an error
293
+ virtual Result<std::shared_ptr<Buffer>> ReadAt(int64_t position, int64_t nbytes);
294
+
295
+ /// EXPERIMENTAL: Read data asynchronously.
296
+ virtual Future<std::shared_ptr<Buffer>> ReadAsync(const IOContext&, int64_t position,
297
+ int64_t nbytes);
298
+
299
+ /// EXPERIMENTAL: Read data asynchronously, using the file's IOContext.
300
+ Future<std::shared_ptr<Buffer>> ReadAsync(int64_t position, int64_t nbytes);
301
+
302
+ /// EXPERIMENTAL: Explicit multi-read.
303
+ /// \brief Request multiple reads at once
304
+ ///
305
+ /// The underlying filesystem may optimize these reads by coalescing small reads into
306
+ /// large reads or by breaking up large reads into multiple parallel smaller reads. The
307
+ /// reads should be issued in parallel if it makes sense for the filesystem.
308
+ ///
309
+ /// One future will be returned for each input read range. Multiple returned futures
310
+ /// may correspond to a single read. Or, a single returned future may be a combined
311
+ /// result of several individual reads.
312
+ ///
313
+ /// \param[in] ranges The ranges to read
314
+ /// \return A future that will complete with the data from the requested range is
315
+ /// available
316
+ virtual std::vector<Future<std::shared_ptr<Buffer>>> ReadManyAsync(
317
+ const IOContext&, const std::vector<ReadRange>& ranges);
318
+
319
+ /// EXPERIMENTAL: Explicit multi-read, using the file's IOContext.
320
+ std::vector<Future<std::shared_ptr<Buffer>>> ReadManyAsync(
321
+ const std::vector<ReadRange>& ranges);
322
+
323
+ /// EXPERIMENTAL: Inform that the given ranges may be read soon.
324
+ ///
325
+ /// Some implementations might arrange to prefetch some of the data.
326
+ /// However, no guarantee is made and the default implementation does nothing.
327
+ /// For robust prefetching, use ReadAt() or ReadAsync().
328
+ virtual Status WillNeed(const std::vector<ReadRange>& ranges);
329
+
330
+ protected:
331
+ RandomAccessFile();
332
+
333
+ private:
334
+ struct ARROW_NO_EXPORT Impl;
335
+ std::unique_ptr<Impl> interface_impl_;
336
+ };
337
+
338
+ class ARROW_EXPORT WritableFile : public OutputStream, public Seekable {
339
+ public:
340
+ virtual Status WriteAt(int64_t position, const void* data, int64_t nbytes) = 0;
341
+
342
+ protected:
343
+ WritableFile() = default;
344
+ };
345
+
346
+ class ARROW_EXPORT ReadWriteFileInterface : public RandomAccessFile, public WritableFile {
347
+ protected:
348
+ ReadWriteFileInterface() { RandomAccessFile::set_mode(FileMode::READWRITE); }
349
+ };
350
+
351
+ /// \brief Return an iterator on an input stream
352
+ ///
353
+ /// The iterator yields a fixed-size block on each Next() call, except the
354
+ /// last block in the stream which may be smaller.
355
+ /// Once the end of stream is reached, Next() returns nullptr
356
+ /// (unlike InputStream::Read() which returns an empty buffer).
357
+ ARROW_EXPORT
358
+ Result<Iterator<std::shared_ptr<Buffer>>> MakeInputStreamIterator(
359
+ std::shared_ptr<InputStream> stream, int64_t block_size);
360
+
361
+ } // namespace io
362
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/memory.h ADDED
@@ -0,0 +1,197 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ // Public API for different memory sharing / IO mechanisms
19
+
20
+ #pragma once
21
+
22
+ #include <cstdint>
23
+ #include <memory>
24
+ #include <string_view>
25
+ #include <vector>
26
+
27
+ #include "arrow/io/concurrency.h"
28
+ #include "arrow/io/interfaces.h"
29
+ #include "arrow/type_fwd.h"
30
+ #include "arrow/util/visibility.h"
31
+
32
+ namespace arrow {
33
+
34
+ class Status;
35
+
36
+ namespace io {
37
+
38
+ /// \brief An output stream that writes to a resizable buffer
39
+ class ARROW_EXPORT BufferOutputStream : public OutputStream {
40
+ public:
41
+ explicit BufferOutputStream(const std::shared_ptr<ResizableBuffer>& buffer);
42
+
43
+ /// \brief Create in-memory output stream with indicated capacity using a
44
+ /// memory pool
45
+ /// \param[in] initial_capacity the initial allocated internal capacity of
46
+ /// the OutputStream
47
+ /// \param[in,out] pool a MemoryPool to use for allocations
48
+ /// \return the created stream
49
+ static Result<std::shared_ptr<BufferOutputStream>> Create(
50
+ int64_t initial_capacity = 4096, MemoryPool* pool = default_memory_pool());
51
+
52
+ ~BufferOutputStream() override;
53
+
54
+ // Implement the OutputStream interface
55
+
56
+ /// Close the stream, preserving the buffer (retrieve it with Finish()).
57
+ Status Close() override;
58
+ bool closed() const override;
59
+ Result<int64_t> Tell() const override;
60
+ Status Write(const void* data, int64_t nbytes) override;
61
+
62
+ /// \cond FALSE
63
+ using OutputStream::Write;
64
+ /// \endcond
65
+
66
+ /// Close the stream and return the buffer
67
+ Result<std::shared_ptr<Buffer>> Finish();
68
+
69
+ /// \brief Initialize state of OutputStream with newly allocated memory and
70
+ /// set position to 0
71
+ /// \param[in] initial_capacity the starting allocated capacity
72
+ /// \param[in,out] pool the memory pool to use for allocations
73
+ /// \return Status
74
+ Status Reset(int64_t initial_capacity = 1024, MemoryPool* pool = default_memory_pool());
75
+
76
+ int64_t capacity() const { return capacity_; }
77
+
78
+ private:
79
+ BufferOutputStream();
80
+
81
+ // Ensures there is sufficient space available to write nbytes
82
+ Status Reserve(int64_t nbytes);
83
+
84
+ std::shared_ptr<ResizableBuffer> buffer_;
85
+ bool is_open_;
86
+ int64_t capacity_;
87
+ int64_t position_;
88
+ uint8_t* mutable_data_;
89
+ };
90
+
91
+ /// \brief A helper class to track the size of allocations
92
+ ///
93
+ /// Writes to this stream do not copy or retain any data, they just bump
94
+ /// a size counter that can be later used to know exactly which data size
95
+ /// needs to be allocated for actual writing.
96
+ class ARROW_EXPORT MockOutputStream : public OutputStream {
97
+ public:
98
+ MockOutputStream() : extent_bytes_written_(0), is_open_(true) {}
99
+
100
+ // Implement the OutputStream interface
101
+ Status Close() override;
102
+ bool closed() const override;
103
+ Result<int64_t> Tell() const override;
104
+ Status Write(const void* data, int64_t nbytes) override;
105
+ /// \cond FALSE
106
+ using Writable::Write;
107
+ /// \endcond
108
+
109
+ int64_t GetExtentBytesWritten() const { return extent_bytes_written_; }
110
+
111
+ private:
112
+ int64_t extent_bytes_written_;
113
+ bool is_open_;
114
+ };
115
+
116
+ /// \brief An output stream that writes into a fixed-size mutable buffer
117
+ class ARROW_EXPORT FixedSizeBufferWriter : public WritableFile {
118
+ public:
119
+ /// Input buffer must be mutable, will abort if not
120
+ explicit FixedSizeBufferWriter(const std::shared_ptr<Buffer>& buffer);
121
+ ~FixedSizeBufferWriter() override;
122
+
123
+ Status Close() override;
124
+ bool closed() const override;
125
+ Status Seek(int64_t position) override;
126
+ Result<int64_t> Tell() const override;
127
+ Status Write(const void* data, int64_t nbytes) override;
128
+ /// \cond FALSE
129
+ using Writable::Write;
130
+ /// \endcond
131
+
132
+ Status WriteAt(int64_t position, const void* data, int64_t nbytes) override;
133
+
134
+ void set_memcopy_threads(int num_threads);
135
+ void set_memcopy_blocksize(int64_t blocksize);
136
+ void set_memcopy_threshold(int64_t threshold);
137
+
138
+ protected:
139
+ class FixedSizeBufferWriterImpl;
140
+ std::unique_ptr<FixedSizeBufferWriterImpl> impl_;
141
+ };
142
+
143
+ /// \class BufferReader
144
+ /// \brief Random access zero-copy reads on an arrow::Buffer
145
+ class ARROW_EXPORT BufferReader
146
+ : public internal::RandomAccessFileConcurrencyWrapper<BufferReader> {
147
+ public:
148
+ /// \brief Instantiate from std::shared_ptr<Buffer>.
149
+ ///
150
+ /// This is a zero-copy constructor.
151
+ explicit BufferReader(std::shared_ptr<Buffer> buffer);
152
+
153
+ /// \brief Instantiate from std::string. Owns data.
154
+ static std::unique_ptr<BufferReader> FromString(std::string data);
155
+
156
+ bool closed() const override;
157
+
158
+ bool supports_zero_copy() const override;
159
+
160
+ std::shared_ptr<Buffer> buffer() const { return buffer_; }
161
+
162
+ // Synchronous ReadAsync override
163
+ Future<std::shared_ptr<Buffer>> ReadAsync(const IOContext&, int64_t position,
164
+ int64_t nbytes) override;
165
+ Status WillNeed(const std::vector<ReadRange>& ranges) override;
166
+
167
+ protected:
168
+ friend RandomAccessFileConcurrencyWrapper<BufferReader>;
169
+
170
+ Status DoClose();
171
+
172
+ Result<int64_t> DoRead(int64_t nbytes, void* buffer);
173
+ Result<std::shared_ptr<Buffer>> DoRead(int64_t nbytes);
174
+ Result<int64_t> DoReadAt(int64_t position, int64_t nbytes, void* out);
175
+ Result<std::shared_ptr<Buffer>> DoReadAt(int64_t position, int64_t nbytes);
176
+ Result<std::string_view> DoPeek(int64_t nbytes) override;
177
+
178
+ Result<int64_t> DoTell() const;
179
+ Status DoSeek(int64_t position);
180
+ Result<int64_t> DoGetSize();
181
+
182
+ Status CheckClosed() const {
183
+ if (!is_open_) {
184
+ return Status::Invalid("Operation forbidden on closed BufferReader");
185
+ }
186
+ return Status::OK();
187
+ }
188
+
189
+ std::shared_ptr<Buffer> buffer_;
190
+ const uint8_t* data_;
191
+ int64_t size_;
192
+ int64_t position_;
193
+ bool is_open_;
194
+ };
195
+
196
+ } // namespace io
197
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/mman.h ADDED
@@ -0,0 +1,169 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Copyright https://code.google.com/p/mman-win32/
2
+ //
3
+ // Licensed under the MIT License;
4
+ // You may obtain a copy of the License at
5
+ //
6
+ // https://opensource.org/licenses/MIT
7
+
8
+ #pragma once
9
+
10
+ #include "arrow/util/windows_compatibility.h"
11
+
12
+ #include <errno.h>
13
+ #include <io.h>
14
+ #include <sys/types.h>
15
+
16
+ #include <cstdint>
17
+
18
+ #define PROT_NONE 0
19
+ #define PROT_READ 1
20
+ #define PROT_WRITE 2
21
+ #define PROT_EXEC 4
22
+
23
+ #define MAP_FILE 0
24
+ #define MAP_SHARED 1
25
+ #define MAP_PRIVATE 2
26
+ #define MAP_TYPE 0xf
27
+ #define MAP_FIXED 0x10
28
+ #define MAP_ANONYMOUS 0x20
29
+ #define MAP_ANON MAP_ANONYMOUS
30
+
31
+ #define MAP_FAILED ((void*)-1)
32
+
33
+ /* Flags for msync. */
34
+ #define MS_ASYNC 1
35
+ #define MS_SYNC 2
36
+ #define MS_INVALIDATE 4
37
+
38
+ #ifndef FILE_MAP_EXECUTE
39
+ # define FILE_MAP_EXECUTE 0x0020
40
+ #endif
41
+
42
+ static inline int __map_mman_error(const DWORD err, const int deferr) {
43
+ if (err == 0) return 0;
44
+ // TODO: implement
45
+ return err;
46
+ }
47
+
48
+ static inline DWORD __map_mmap_prot_page(const int prot) {
49
+ DWORD protect = 0;
50
+
51
+ if (prot == PROT_NONE) return protect;
52
+
53
+ if ((prot & PROT_EXEC) != 0) {
54
+ protect = ((prot & PROT_WRITE) != 0) ? PAGE_EXECUTE_READWRITE : PAGE_EXECUTE_READ;
55
+ } else {
56
+ protect = ((prot & PROT_WRITE) != 0) ? PAGE_READWRITE : PAGE_READONLY;
57
+ }
58
+
59
+ return protect;
60
+ }
61
+
62
+ static inline DWORD __map_mmap_prot_file(const int prot) {
63
+ DWORD desiredAccess = 0;
64
+
65
+ if (prot == PROT_NONE) return desiredAccess;
66
+
67
+ if ((prot & PROT_READ) != 0) desiredAccess |= FILE_MAP_READ;
68
+ if ((prot & PROT_WRITE) != 0) desiredAccess |= FILE_MAP_WRITE;
69
+ if ((prot & PROT_EXEC) != 0) desiredAccess |= FILE_MAP_EXECUTE;
70
+
71
+ return desiredAccess;
72
+ }
73
+
74
+ static inline void* mmap(void* addr, size_t len, int prot, int flags, int fildes,
75
+ off_t off) {
76
+ HANDLE fm, h;
77
+
78
+ void* map = MAP_FAILED;
79
+ const uint64_t off64 = static_cast<uint64_t>(off);
80
+ const uint64_t maxSize = off64 + len;
81
+
82
+ const DWORD dwFileOffsetLow = static_cast<DWORD>(off64 & 0xFFFFFFFFUL);
83
+ const DWORD dwFileOffsetHigh = static_cast<DWORD>((off64 >> 32) & 0xFFFFFFFFUL);
84
+ const DWORD dwMaxSizeLow = static_cast<DWORD>(maxSize & 0xFFFFFFFFUL);
85
+ const DWORD dwMaxSizeHigh = static_cast<DWORD>((maxSize >> 32) & 0xFFFFFFFFUL);
86
+
87
+ const DWORD protect = __map_mmap_prot_page(prot);
88
+ const DWORD desiredAccess = __map_mmap_prot_file(prot);
89
+
90
+ errno = 0;
91
+
92
+ if (len == 0
93
+ /* Unsupported flag combinations */
94
+ || (flags & MAP_FIXED) != 0
95
+ /* Unsupported protection combinations */
96
+ || prot == PROT_EXEC) {
97
+ errno = EINVAL;
98
+ return MAP_FAILED;
99
+ }
100
+
101
+ h = ((flags & MAP_ANONYMOUS) == 0) ? (HANDLE)_get_osfhandle(fildes)
102
+ : INVALID_HANDLE_VALUE;
103
+
104
+ if ((flags & MAP_ANONYMOUS) == 0 && h == INVALID_HANDLE_VALUE) {
105
+ errno = EBADF;
106
+ return MAP_FAILED;
107
+ }
108
+
109
+ fm = CreateFileMapping(h, NULL, protect, dwMaxSizeHigh, dwMaxSizeLow, NULL);
110
+
111
+ if (fm == NULL) {
112
+ errno = __map_mman_error(GetLastError(), EPERM);
113
+ return MAP_FAILED;
114
+ }
115
+
116
+ map = MapViewOfFile(fm, desiredAccess, dwFileOffsetHigh, dwFileOffsetLow, len);
117
+
118
+ CloseHandle(fm);
119
+
120
+ if (map == NULL) {
121
+ errno = __map_mman_error(GetLastError(), EPERM);
122
+ return MAP_FAILED;
123
+ }
124
+
125
+ return map;
126
+ }
127
+
128
+ static inline int munmap(void* addr, size_t len) {
129
+ if (UnmapViewOfFile(addr)) return 0;
130
+
131
+ errno = __map_mman_error(GetLastError(), EPERM);
132
+
133
+ return -1;
134
+ }
135
+
136
+ static inline int mprotect(void* addr, size_t len, int prot) {
137
+ DWORD newProtect = __map_mmap_prot_page(prot);
138
+ DWORD oldProtect = 0;
139
+
140
+ if (VirtualProtect(addr, len, newProtect, &oldProtect)) return 0;
141
+
142
+ errno = __map_mman_error(GetLastError(), EPERM);
143
+
144
+ return -1;
145
+ }
146
+
147
+ static inline int msync(void* addr, size_t len, int flags) {
148
+ if (FlushViewOfFile(addr, len)) return 0;
149
+
150
+ errno = __map_mman_error(GetLastError(), EPERM);
151
+
152
+ return -1;
153
+ }
154
+
155
+ static inline int mlock(const void* addr, size_t len) {
156
+ if (VirtualLock((LPVOID)addr, len)) return 0;
157
+
158
+ errno = __map_mman_error(GetLastError(), EPERM);
159
+
160
+ return -1;
161
+ }
162
+
163
+ static inline int munlock(const void* addr, size_t len) {
164
+ if (VirtualUnlock((LPVOID)addr, len)) return 0;
165
+
166
+ errno = __map_mman_error(GetLastError(), EPERM);
167
+
168
+ return -1;
169
+ }
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/slow.h ADDED
@@ -0,0 +1,118 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ // Slow stream implementations, mainly for testing and benchmarking
19
+
20
+ #pragma once
21
+
22
+ #include <cstdint>
23
+ #include <memory>
24
+ #include <utility>
25
+
26
+ #include "arrow/io/interfaces.h"
27
+ #include "arrow/util/visibility.h"
28
+
29
+ namespace arrow {
30
+
31
+ class Buffer;
32
+ class Status;
33
+
34
+ namespace io {
35
+
36
+ class ARROW_EXPORT LatencyGenerator {
37
+ public:
38
+ virtual ~LatencyGenerator();
39
+
40
+ void Sleep();
41
+
42
+ virtual double NextLatency() = 0;
43
+
44
+ static std::shared_ptr<LatencyGenerator> Make(double average_latency);
45
+ static std::shared_ptr<LatencyGenerator> Make(double average_latency, int32_t seed);
46
+ };
47
+
48
+ // XXX use ConcurrencyWrapper? It could increase chances of finding a race.
49
+
50
+ template <class StreamType>
51
+ class SlowInputStreamBase : public StreamType {
52
+ public:
53
+ SlowInputStreamBase(std::shared_ptr<StreamType> stream,
54
+ std::shared_ptr<LatencyGenerator> latencies)
55
+ : stream_(std::move(stream)), latencies_(std::move(latencies)) {}
56
+
57
+ SlowInputStreamBase(std::shared_ptr<StreamType> stream, double average_latency)
58
+ : stream_(std::move(stream)), latencies_(LatencyGenerator::Make(average_latency)) {}
59
+
60
+ SlowInputStreamBase(std::shared_ptr<StreamType> stream, double average_latency,
61
+ int32_t seed)
62
+ : stream_(std::move(stream)),
63
+ latencies_(LatencyGenerator::Make(average_latency, seed)) {}
64
+
65
+ protected:
66
+ std::shared_ptr<StreamType> stream_;
67
+ std::shared_ptr<LatencyGenerator> latencies_;
68
+ };
69
+
70
+ /// \brief An InputStream wrapper that makes reads slower.
71
+ ///
72
+ /// Read() calls are made slower by an average latency (in seconds).
73
+ /// Actual latencies form a normal distribution closely centered
74
+ /// on the average latency.
75
+ /// Other calls are forwarded directly.
76
+ class ARROW_EXPORT SlowInputStream : public SlowInputStreamBase<InputStream> {
77
+ public:
78
+ ~SlowInputStream() override;
79
+
80
+ using SlowInputStreamBase<InputStream>::SlowInputStreamBase;
81
+
82
+ Status Close() override;
83
+ Status Abort() override;
84
+ bool closed() const override;
85
+
86
+ Result<int64_t> Read(int64_t nbytes, void* out) override;
87
+ Result<std::shared_ptr<Buffer>> Read(int64_t nbytes) override;
88
+ Result<std::string_view> Peek(int64_t nbytes) override;
89
+
90
+ Result<int64_t> Tell() const override;
91
+ };
92
+
93
+ /// \brief A RandomAccessFile wrapper that makes reads slower.
94
+ ///
95
+ /// Similar to SlowInputStream, but allows random access and seeking.
96
+ class ARROW_EXPORT SlowRandomAccessFile : public SlowInputStreamBase<RandomAccessFile> {
97
+ public:
98
+ ~SlowRandomAccessFile() override;
99
+
100
+ using SlowInputStreamBase<RandomAccessFile>::SlowInputStreamBase;
101
+
102
+ Status Close() override;
103
+ Status Abort() override;
104
+ bool closed() const override;
105
+
106
+ Result<int64_t> Read(int64_t nbytes, void* out) override;
107
+ Result<std::shared_ptr<Buffer>> Read(int64_t nbytes) override;
108
+ Result<int64_t> ReadAt(int64_t position, int64_t nbytes, void* out) override;
109
+ Result<std::shared_ptr<Buffer>> ReadAt(int64_t position, int64_t nbytes) override;
110
+ Result<std::string_view> Peek(int64_t nbytes) override;
111
+
112
+ Result<int64_t> GetSize() override;
113
+ Status Seek(int64_t position) override;
114
+ Result<int64_t> Tell() const override;
115
+ };
116
+
117
+ } // namespace io
118
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/stdio.h ADDED
@@ -0,0 +1,82 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <cstdint>
21
+
22
+ #include "arrow/io/interfaces.h"
23
+ #include "arrow/util/visibility.h"
24
+
25
+ namespace arrow {
26
+ namespace io {
27
+
28
+ // Output stream that just writes to stdout.
29
+ class ARROW_EXPORT StdoutStream : public OutputStream {
30
+ public:
31
+ StdoutStream();
32
+ ~StdoutStream() override {}
33
+
34
+ Status Close() override;
35
+ bool closed() const override;
36
+
37
+ Result<int64_t> Tell() const override;
38
+
39
+ Status Write(const void* data, int64_t nbytes) override;
40
+
41
+ private:
42
+ int64_t pos_;
43
+ };
44
+
45
+ // Output stream that just writes to stderr.
46
+ class ARROW_EXPORT StderrStream : public OutputStream {
47
+ public:
48
+ StderrStream();
49
+ ~StderrStream() override {}
50
+
51
+ Status Close() override;
52
+ bool closed() const override;
53
+
54
+ Result<int64_t> Tell() const override;
55
+
56
+ Status Write(const void* data, int64_t nbytes) override;
57
+
58
+ private:
59
+ int64_t pos_;
60
+ };
61
+
62
+ // Input stream that just reads from stdin.
63
+ class ARROW_EXPORT StdinStream : public InputStream {
64
+ public:
65
+ StdinStream();
66
+ ~StdinStream() override {}
67
+
68
+ Status Close() override;
69
+ bool closed() const override;
70
+
71
+ Result<int64_t> Tell() const override;
72
+
73
+ Result<int64_t> Read(int64_t nbytes, void* out) override;
74
+
75
+ Result<std::shared_ptr<Buffer>> Read(int64_t nbytes) override;
76
+
77
+ private:
78
+ int64_t pos_;
79
+ };
80
+
81
+ } // namespace io
82
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/test_common.h ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <memory>
21
+ #include <string>
22
+ #include <vector>
23
+
24
+ #include "arrow/io/interfaces.h"
25
+ #include "arrow/testing/visibility.h"
26
+ #include "arrow/type_fwd.h"
27
+
28
+ namespace arrow {
29
+ namespace io {
30
+
31
+ class MemoryMappedFile;
32
+
33
+ ARROW_TESTING_EXPORT
34
+ void AssertFileContents(const std::string& path, const std::string& contents);
35
+
36
+ ARROW_TESTING_EXPORT bool FileExists(const std::string& path);
37
+
38
+ ARROW_TESTING_EXPORT Status PurgeLocalFileFromOsCache(const std::string& path);
39
+
40
+ ARROW_TESTING_EXPORT
41
+ Status ZeroMemoryMap(MemoryMappedFile* file);
42
+
43
+ class ARROW_TESTING_EXPORT MemoryMapFixture {
44
+ public:
45
+ void TearDown();
46
+
47
+ void CreateFile(const std::string& path, int64_t size);
48
+
49
+ Result<std::shared_ptr<MemoryMappedFile>> InitMemoryMap(int64_t size,
50
+ const std::string& path);
51
+
52
+ void AppendFile(const std::string& path);
53
+
54
+ private:
55
+ std::vector<std::string> tmp_files_;
56
+ };
57
+
58
+ class ARROW_TESTING_EXPORT TrackedRandomAccessFile : public io::RandomAccessFile {
59
+ public:
60
+ virtual int64_t num_reads() const = 0;
61
+ virtual int64_t bytes_read() const = 0;
62
+ virtual const std::vector<io::ReadRange>& get_read_ranges() const = 0;
63
+ virtual void ResetStats() = 0;
64
+
65
+ static std::unique_ptr<TrackedRandomAccessFile> Make(io::RandomAccessFile* target);
66
+ };
67
+
68
+ } // namespace io
69
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/transform.h ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ // Transform stream implementations
19
+
20
+ #pragma once
21
+
22
+ #include <cstdint>
23
+ #include <functional>
24
+ #include <memory>
25
+ #include <utility>
26
+
27
+ #include "arrow/io/interfaces.h"
28
+ #include "arrow/util/visibility.h"
29
+
30
+ namespace arrow {
31
+ namespace io {
32
+
33
+ class ARROW_EXPORT TransformInputStream : public InputStream {
34
+ public:
35
+ using TransformFunc =
36
+ std::function<Result<std::shared_ptr<Buffer>>(const std::shared_ptr<Buffer>&)>;
37
+
38
+ TransformInputStream(std::shared_ptr<InputStream> wrapped, TransformFunc transform);
39
+ ~TransformInputStream() override;
40
+
41
+ Status Close() override;
42
+ Status Abort() override;
43
+ bool closed() const override;
44
+
45
+ Result<int64_t> Read(int64_t nbytes, void* out) override;
46
+ Result<std::shared_ptr<Buffer>> Read(int64_t nbytes) override;
47
+
48
+ Result<std::shared_ptr<const KeyValueMetadata>> ReadMetadata() override;
49
+ Future<std::shared_ptr<const KeyValueMetadata>> ReadMetadataAsync(
50
+ const IOContext& io_context) override;
51
+
52
+ Result<int64_t> Tell() const override;
53
+
54
+ protected:
55
+ struct Impl;
56
+ std::unique_ptr<Impl> impl_;
57
+ };
58
+
59
+ } // namespace io
60
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/type_fwd.h ADDED
@@ -0,0 +1,77 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include "arrow/type_fwd.h"
21
+ #include "arrow/util/visibility.h"
22
+
23
+ namespace arrow {
24
+ namespace io {
25
+
26
+ struct FileMode {
27
+ enum type { READ, WRITE, READWRITE };
28
+ };
29
+
30
+ struct IOContext;
31
+ struct CacheOptions;
32
+
33
+ /// EXPERIMENTAL: convenience global singleton for default IOContext settings
34
+ ARROW_EXPORT
35
+ const IOContext& default_io_context();
36
+
37
+ /// \brief Get the capacity of the global I/O thread pool
38
+ ///
39
+ /// Return the number of worker threads in the thread pool to which
40
+ /// Arrow dispatches various I/O-bound tasks. This is an ideal number,
41
+ /// not necessarily the exact number of threads at a given point in time.
42
+ ///
43
+ /// You can change this number using SetIOThreadPoolCapacity().
44
+ ARROW_EXPORT int GetIOThreadPoolCapacity();
45
+
46
+ /// \brief Set the capacity of the global I/O thread pool
47
+ ///
48
+ /// Set the number of worker threads in the thread pool to which
49
+ /// Arrow dispatches various I/O-bound tasks.
50
+ ///
51
+ /// The current number is returned by GetIOThreadPoolCapacity().
52
+ ARROW_EXPORT Status SetIOThreadPoolCapacity(int threads);
53
+
54
+ class FileInterface;
55
+ class Seekable;
56
+ class Writable;
57
+ class Readable;
58
+ class OutputStream;
59
+ class FileOutputStream;
60
+ class InputStream;
61
+ class ReadableFile;
62
+ class RandomAccessFile;
63
+ class MemoryMappedFile;
64
+ class WritableFile;
65
+ class ReadWriteFileInterface;
66
+
67
+ class LatencyGenerator;
68
+
69
+ class BufferOutputStream;
70
+ class BufferReader;
71
+ class CompressedInputStream;
72
+ class CompressedOutputStream;
73
+ class BufferedInputStream;
74
+ class BufferedOutputStream;
75
+
76
+ } // namespace io
77
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/api.h ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include "arrow/ipc/dictionary.h"
21
+ #include "arrow/ipc/feather.h"
22
+ #include "arrow/ipc/message.h"
23
+ #include "arrow/ipc/reader.h"
24
+ #include "arrow/ipc/writer.h"
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/dictionary.h ADDED
@@ -0,0 +1,177 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ // Tools for dictionaries in IPC context
19
+
20
+ #pragma once
21
+
22
+ #include <cstdint>
23
+ #include <memory>
24
+ #include <utility>
25
+ #include <vector>
26
+
27
+ #include "arrow/result.h"
28
+ #include "arrow/status.h"
29
+ #include "arrow/type_fwd.h"
30
+ #include "arrow/util/macros.h"
31
+ #include "arrow/util/visibility.h"
32
+
33
+ namespace arrow {
34
+ namespace ipc {
35
+
36
+ namespace internal {
37
+
38
+ class FieldPosition {
39
+ public:
40
+ FieldPosition() : parent_(NULLPTR), index_(-1), depth_(0) {}
41
+
42
+ FieldPosition child(int index) const { return {this, index}; }
43
+
44
+ std::vector<int> path() const {
45
+ std::vector<int> path(depth_);
46
+ const FieldPosition* cur = this;
47
+ for (int i = depth_ - 1; i >= 0; --i) {
48
+ path[i] = cur->index_;
49
+ cur = cur->parent_;
50
+ }
51
+ return path;
52
+ }
53
+
54
+ protected:
55
+ FieldPosition(const FieldPosition* parent, int index)
56
+ : parent_(parent), index_(index), depth_(parent->depth_ + 1) {}
57
+
58
+ const FieldPosition* parent_;
59
+ int index_;
60
+ int depth_;
61
+ };
62
+
63
+ } // namespace internal
64
+
65
+ /// \brief Map fields in a schema to dictionary ids
66
+ ///
67
+ /// The mapping is structural, i.e. the field path (as a vector of indices)
68
+ /// is associated to the dictionary id. A dictionary id may be associated
69
+ /// to multiple fields.
70
+ class ARROW_EXPORT DictionaryFieldMapper {
71
+ public:
72
+ DictionaryFieldMapper();
73
+ explicit DictionaryFieldMapper(const Schema& schema);
74
+ ~DictionaryFieldMapper();
75
+
76
+ Status AddSchemaFields(const Schema& schema);
77
+ Status AddField(int64_t id, std::vector<int> field_path);
78
+
79
+ Result<int64_t> GetFieldId(std::vector<int> field_path) const;
80
+
81
+ int num_fields() const;
82
+
83
+ /// \brief Returns number of unique dictionaries, taking into
84
+ /// account that different fields can share the same dictionary.
85
+ int num_dicts() const;
86
+
87
+ private:
88
+ struct Impl;
89
+ std::unique_ptr<Impl> impl_;
90
+ };
91
+
92
+ using DictionaryVector = std::vector<std::pair<int64_t, std::shared_ptr<Array>>>;
93
+
94
+ /// \brief Memoization data structure for reading dictionaries from IPC streams
95
+ ///
96
+ /// This structure tracks the following associations:
97
+ /// - field position (structural) -> dictionary id
98
+ /// - dictionary id -> value type
99
+ /// - dictionary id -> dictionary (value) data
100
+ ///
101
+ /// Together, they allow resolving dictionary data when reading an IPC stream,
102
+ /// using metadata recorded in the schema message and data recorded in the
103
+ /// dictionary batch messages (see ResolveDictionaries).
104
+ ///
105
+ /// This structure isn't useful for writing an IPC stream, where only
106
+ /// DictionaryFieldMapper is necessary.
107
+ class ARROW_EXPORT DictionaryMemo {
108
+ public:
109
+ DictionaryMemo();
110
+ ~DictionaryMemo();
111
+
112
+ DictionaryFieldMapper& fields();
113
+ const DictionaryFieldMapper& fields() const;
114
+
115
+ /// \brief Return current dictionary corresponding to a particular
116
+ /// id. Returns KeyError if id not found
117
+ Result<std::shared_ptr<ArrayData>> GetDictionary(int64_t id, MemoryPool* pool) const;
118
+
119
+ /// \brief Return dictionary value type corresponding to a
120
+ /// particular dictionary id.
121
+ Result<std::shared_ptr<DataType>> GetDictionaryType(int64_t id) const;
122
+
123
+ /// \brief Return true if we have a dictionary for the input id
124
+ bool HasDictionary(int64_t id) const;
125
+
126
+ /// \brief Add a dictionary value type to the memo with a particular id.
127
+ /// Returns KeyError if a different type is already registered with the same id.
128
+ Status AddDictionaryType(int64_t id, const std::shared_ptr<DataType>& type);
129
+
130
+ /// \brief Add a dictionary to the memo with a particular id. Returns
131
+ /// KeyError if that dictionary already exists
132
+ Status AddDictionary(int64_t id, const std::shared_ptr<ArrayData>& dictionary);
133
+
134
+ /// \brief Append a dictionary delta to the memo with a particular id. Returns
135
+ /// KeyError if that dictionary does not exists
136
+ Status AddDictionaryDelta(int64_t id, const std::shared_ptr<ArrayData>& dictionary);
137
+
138
+ /// \brief Add a dictionary to the memo if it does not have one with the id,
139
+ /// otherwise, replace the dictionary with the new one.
140
+ ///
141
+ /// Return true if the dictionary was added, false if replaced.
142
+ Result<bool> AddOrReplaceDictionary(int64_t id,
143
+ const std::shared_ptr<ArrayData>& dictionary);
144
+
145
+ private:
146
+ struct Impl;
147
+ std::unique_ptr<Impl> impl_;
148
+ };
149
+
150
+ // For writing: collect dictionary entries to write to the IPC stream, in order
151
+ // (i.e. inner dictionaries before dependent outer dictionaries).
152
+ ARROW_EXPORT
153
+ Result<DictionaryVector> CollectDictionaries(const RecordBatch& batch,
154
+ const DictionaryFieldMapper& mapper);
155
+
156
+ // For reading: resolve all dictionaries in columns, according to the field
157
+ // mapping and dictionary arrays stored in memo.
158
+ // Columns may be sparse, i.e. some entries may be left null
159
+ // (e.g. if an inclusion mask was used).
160
+ ARROW_EXPORT
161
+ Status ResolveDictionaries(const ArrayDataVector& columns, const DictionaryMemo& memo,
162
+ MemoryPool* pool);
163
+
164
+ namespace internal {
165
+
166
+ // Like CollectDictionaries above, but uses the memo's DictionaryFieldMapper
167
+ // and all collected dictionaries are added to the memo using AddDictionary.
168
+ //
169
+ // This is used as a shortcut in some roundtripping tests (to avoid emitting
170
+ // any actual dictionary batches).
171
+ ARROW_EXPORT
172
+ Status CollectDictionaries(const RecordBatch& batch, DictionaryMemo* memo);
173
+
174
+ } // namespace internal
175
+
176
+ } // namespace ipc
177
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/feather.h ADDED
@@ -0,0 +1,150 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ // Public API for the "Feather" file format, originally created at
19
+ // http://github.com/wesm/feather
20
+
21
+ #pragma once
22
+
23
+ #include <cstdint>
24
+ #include <memory>
25
+ #include <string>
26
+ #include <vector>
27
+
28
+ #include "arrow/ipc/options.h"
29
+ #include "arrow/type_fwd.h"
30
+ #include "arrow/util/compression.h"
31
+ #include "arrow/util/visibility.h"
32
+
33
+ namespace arrow {
34
+
35
+ class Schema;
36
+ class Status;
37
+ class Table;
38
+
39
+ namespace io {
40
+
41
+ class OutputStream;
42
+ class RandomAccessFile;
43
+
44
+ } // namespace io
45
+
46
+ namespace ipc {
47
+ namespace feather {
48
+
49
+ static constexpr const int kFeatherV1Version = 2;
50
+ static constexpr const int kFeatherV2Version = 3;
51
+
52
+ // ----------------------------------------------------------------------
53
+ // Metadata accessor classes
54
+
55
+ /// \class Reader
56
+ /// \brief An interface for reading columns from Feather files
57
+ class ARROW_EXPORT Reader {
58
+ public:
59
+ virtual ~Reader() = default;
60
+
61
+ /// \brief Open a Feather file from a RandomAccessFile interface
62
+ ///
63
+ /// \param[in] source a RandomAccessFile instance
64
+ /// \return the table reader
65
+ static Result<std::shared_ptr<Reader>> Open(
66
+ const std::shared_ptr<io::RandomAccessFile>& source);
67
+
68
+ /// \brief Open a Feather file from a RandomAccessFile interface
69
+ /// with IPC Read options
70
+ ///
71
+ /// \param[in] source a RandomAccessFile instance
72
+ /// \param[in] options IPC Read options
73
+ /// \return the table reader
74
+ static Result<std::shared_ptr<Reader>> Open(
75
+ const std::shared_ptr<io::RandomAccessFile>& source, const IpcReadOptions& options);
76
+
77
+ /// \brief Return the version number of the Feather file
78
+ virtual int version() const = 0;
79
+
80
+ virtual std::shared_ptr<Schema> schema() const = 0;
81
+
82
+ /// \brief Read all columns from the file as an arrow::Table.
83
+ ///
84
+ /// \param[out] out the returned table
85
+ /// \return Status
86
+ ///
87
+ /// This function is zero-copy if the file source supports zero-copy reads
88
+ virtual Status Read(std::shared_ptr<Table>* out) = 0;
89
+
90
+ /// \brief Read only the specified columns from the file as an arrow::Table.
91
+ ///
92
+ /// \param[in] indices the column indices to read
93
+ /// \param[out] out the returned table
94
+ /// \return Status
95
+ ///
96
+ /// This function is zero-copy if the file source supports zero-copy reads
97
+ virtual Status Read(const std::vector<int>& indices, std::shared_ptr<Table>* out) = 0;
98
+
99
+ /// \brief Read only the specified columns from the file as an arrow::Table.
100
+ ///
101
+ /// \param[in] names the column names to read
102
+ /// \param[out] out the returned table
103
+ /// \return Status
104
+ ///
105
+ /// This function is zero-copy if the file source supports zero-copy reads
106
+ virtual Status Read(const std::vector<std::string>& names,
107
+ std::shared_ptr<Table>* out) = 0;
108
+ };
109
+
110
+ struct ARROW_EXPORT WriteProperties {
111
+ static WriteProperties Defaults();
112
+
113
+ static WriteProperties DefaultsV1() {
114
+ WriteProperties props = Defaults();
115
+ props.version = kFeatherV1Version;
116
+ return props;
117
+ }
118
+
119
+ /// Feather file version number
120
+ ///
121
+ /// version 2: "Feather V1" Apache Arrow <= 0.16.0
122
+ /// version 3: "Feather V2" Apache Arrow > 0.16.0
123
+ int version = kFeatherV2Version;
124
+
125
+ // Parameters for Feather V2 only
126
+
127
+ /// Number of rows per intra-file chunk. Use smaller chunksize when you need
128
+ /// faster random row access
129
+ int64_t chunksize = 1LL << 16;
130
+
131
+ /// Compression type to use. Only UNCOMPRESSED, LZ4_FRAME, and ZSTD are
132
+ /// supported. The default compression returned by Defaults() is LZ4 if the
133
+ /// project is built with support for it, otherwise
134
+ /// UNCOMPRESSED. UNCOMPRESSED is set as the object default here so that if
135
+ /// WriteProperties::Defaults() is not used, the default constructor for
136
+ /// WriteProperties will work regardless of the options used to build the C++
137
+ /// project.
138
+ Compression::type compression = Compression::UNCOMPRESSED;
139
+
140
+ /// Compressor-specific compression level
141
+ int compression_level = ::arrow::util::kUseDefaultCompressionLevel;
142
+ };
143
+
144
+ ARROW_EXPORT
145
+ Status WriteTable(const Table& table, io::OutputStream* dst,
146
+ const WriteProperties& properties = WriteProperties::Defaults());
147
+
148
+ } // namespace feather
149
+ } // namespace ipc
150
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/message.h ADDED
@@ -0,0 +1,585 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ // C++ object model and user API for interprocess schema messaging
19
+
20
+ #pragma once
21
+
22
+ #include <cstdint>
23
+ #include <functional>
24
+ #include <memory>
25
+ #include <string>
26
+ #include <utility>
27
+
28
+ #include "arrow/io/type_fwd.h"
29
+ #include "arrow/ipc/type_fwd.h"
30
+ #include "arrow/result.h"
31
+ #include "arrow/status.h"
32
+ #include "arrow/type_fwd.h"
33
+ #include "arrow/util/macros.h"
34
+ #include "arrow/util/visibility.h"
35
+
36
+ namespace arrow {
37
+ namespace ipc {
38
+
39
+ struct IpcWriteOptions;
40
+
41
+ // Read interface classes. We do not fully deserialize the flatbuffers so that
42
+ // individual fields metadata can be retrieved from very large schema without
43
+ //
44
+
45
+ /// \class Message
46
+ /// \brief An IPC message including metadata and body
47
+ class ARROW_EXPORT Message {
48
+ public:
49
+ /// \brief Construct message, but do not validate
50
+ ///
51
+ /// Use at your own risk; Message::Open has more metadata validation
52
+ Message(std::shared_ptr<Buffer> metadata, std::shared_ptr<Buffer> body);
53
+
54
+ ~Message();
55
+
56
+ /// \brief Create and validate a Message instance from two buffers
57
+ ///
58
+ /// \param[in] metadata a buffer containing the Flatbuffer metadata
59
+ /// \param[in] body a buffer containing the message body, which may be null
60
+ /// \return the created message
61
+ static Result<std::unique_ptr<Message>> Open(std::shared_ptr<Buffer> metadata,
62
+ std::shared_ptr<Buffer> body);
63
+
64
+ /// \brief Read message body and create Message given Flatbuffer metadata
65
+ /// \param[in] metadata containing a serialized Message flatbuffer
66
+ /// \param[in] stream an InputStream
67
+ /// \return the created Message
68
+ ///
69
+ /// \note If stream supports zero-copy, this is zero-copy
70
+ static Result<std::unique_ptr<Message>> ReadFrom(std::shared_ptr<Buffer> metadata,
71
+ io::InputStream* stream);
72
+
73
+ /// \brief Read message body from position in file, and create Message given
74
+ /// the Flatbuffer metadata
75
+ /// \param[in] offset the position in the file where the message body starts.
76
+ /// \param[in] metadata containing a serialized Message flatbuffer
77
+ /// \param[in] file the seekable file interface to read from
78
+ /// \return the created Message
79
+ ///
80
+ /// \note If file supports zero-copy, this is zero-copy
81
+ static Result<std::unique_ptr<Message>> ReadFrom(const int64_t offset,
82
+ std::shared_ptr<Buffer> metadata,
83
+ io::RandomAccessFile* file);
84
+
85
+ /// \brief Return true if message type and contents are equal
86
+ ///
87
+ /// \param other another message
88
+ /// \return true if contents equal
89
+ bool Equals(const Message& other) const;
90
+
91
+ /// \brief the Message metadata
92
+ ///
93
+ /// \return buffer
94
+ std::shared_ptr<Buffer> metadata() const;
95
+
96
+ /// \brief Custom metadata serialized in metadata Flatbuffer. Returns nullptr
97
+ /// when none set
98
+ const std::shared_ptr<const KeyValueMetadata>& custom_metadata() const;
99
+
100
+ /// \brief the Message body, if any
101
+ ///
102
+ /// \return buffer is null if no body
103
+ std::shared_ptr<Buffer> body() const;
104
+
105
+ /// \brief The expected body length according to the metadata, for
106
+ /// verification purposes
107
+ int64_t body_length() const;
108
+
109
+ /// \brief The Message type
110
+ MessageType type() const;
111
+
112
+ /// \brief The Message metadata version
113
+ MetadataVersion metadata_version() const;
114
+
115
+ const void* header() const;
116
+
117
+ /// \brief Write length-prefixed metadata and body to output stream
118
+ ///
119
+ /// \param[in] file output stream to write to
120
+ /// \param[in] options IPC writing options including alignment
121
+ /// \param[out] output_length the number of bytes written
122
+ /// \return Status
123
+ Status SerializeTo(io::OutputStream* file, const IpcWriteOptions& options,
124
+ int64_t* output_length) const;
125
+
126
+ /// \brief Return true if the Message metadata passes Flatbuffer validation
127
+ bool Verify() const;
128
+
129
+ /// \brief Whether a given message type needs a body.
130
+ static bool HasBody(MessageType type) {
131
+ return type != MessageType::NONE && type != MessageType::SCHEMA;
132
+ }
133
+
134
+ private:
135
+ // Hide serialization details from user API
136
+ class MessageImpl;
137
+ std::unique_ptr<MessageImpl> impl_;
138
+
139
+ ARROW_DISALLOW_COPY_AND_ASSIGN(Message);
140
+ };
141
+
142
+ ARROW_EXPORT std::string FormatMessageType(MessageType type);
143
+
144
+ /// \class MessageDecoderListener
145
+ /// \brief An abstract class to listen events from MessageDecoder.
146
+ ///
147
+ /// This API is EXPERIMENTAL.
148
+ ///
149
+ /// \since 0.17.0
150
+ class ARROW_EXPORT MessageDecoderListener {
151
+ public:
152
+ virtual ~MessageDecoderListener() = default;
153
+
154
+ /// \brief Called when a message is decoded.
155
+ ///
156
+ /// MessageDecoder calls this method when it decodes a message. This
157
+ /// method is called multiple times when the target stream has
158
+ /// multiple messages.
159
+ ///
160
+ /// \param[in] message a decoded message
161
+ /// \return Status
162
+ virtual Status OnMessageDecoded(std::unique_ptr<Message> message) = 0;
163
+
164
+ /// \brief Called when the decoder state is changed to
165
+ /// MessageDecoder::State::INITIAL.
166
+ ///
167
+ /// The default implementation just returns arrow::Status::OK().
168
+ ///
169
+ /// \return Status
170
+ virtual Status OnInitial();
171
+
172
+ /// \brief Called when the decoder state is changed to
173
+ /// MessageDecoder::State::METADATA_LENGTH.
174
+ ///
175
+ /// The default implementation just returns arrow::Status::OK().
176
+ ///
177
+ /// \return Status
178
+ virtual Status OnMetadataLength();
179
+
180
+ /// \brief Called when the decoder state is changed to
181
+ /// MessageDecoder::State::METADATA.
182
+ ///
183
+ /// The default implementation just returns arrow::Status::OK().
184
+ ///
185
+ /// \return Status
186
+ virtual Status OnMetadata();
187
+
188
+ /// \brief Called when the decoder state is changed to
189
+ /// MessageDecoder::State::BODY.
190
+ ///
191
+ /// The default implementation just returns arrow::Status::OK().
192
+ ///
193
+ /// \return Status
194
+ virtual Status OnBody();
195
+
196
+ /// \brief Called when the decoder state is changed to
197
+ /// MessageDecoder::State::EOS.
198
+ ///
199
+ /// The default implementation just returns arrow::Status::OK().
200
+ ///
201
+ /// \return Status
202
+ virtual Status OnEOS();
203
+ };
204
+
205
+ /// \class AssignMessageDecoderListener
206
+ /// \brief Assign a message decoded by MessageDecoder.
207
+ ///
208
+ /// This API is EXPERIMENTAL.
209
+ ///
210
+ /// \since 0.17.0
211
+ class ARROW_EXPORT AssignMessageDecoderListener : public MessageDecoderListener {
212
+ public:
213
+ /// \brief Construct a listener that assigns a decoded message to the
214
+ /// specified location.
215
+ ///
216
+ /// \param[in] message a location to store the received message
217
+ explicit AssignMessageDecoderListener(std::unique_ptr<Message>* message)
218
+ : message_(message) {}
219
+
220
+ virtual ~AssignMessageDecoderListener() = default;
221
+
222
+ Status OnMessageDecoded(std::unique_ptr<Message> message) override {
223
+ *message_ = std::move(message);
224
+ return Status::OK();
225
+ }
226
+
227
+ private:
228
+ std::unique_ptr<Message>* message_;
229
+
230
+ ARROW_DISALLOW_COPY_AND_ASSIGN(AssignMessageDecoderListener);
231
+ };
232
+
233
+ /// \class MessageDecoder
234
+ /// \brief Push style message decoder that receives data from user.
235
+ ///
236
+ /// This API is EXPERIMENTAL.
237
+ ///
238
+ /// \since 0.17.0
239
+ class ARROW_EXPORT MessageDecoder {
240
+ public:
241
+ /// \brief State for reading a message
242
+ enum State {
243
+ /// The initial state. It requires one of the followings as the next data:
244
+ ///
245
+ /// * int32_t continuation token
246
+ /// * int32_t end-of-stream mark (== 0)
247
+ /// * int32_t metadata length (backward compatibility for
248
+ /// reading old IPC messages produced prior to version 0.15.0
249
+ INITIAL,
250
+
251
+ /// It requires int32_t metadata length.
252
+ METADATA_LENGTH,
253
+
254
+ /// It requires metadata.
255
+ METADATA,
256
+
257
+ /// It requires message body.
258
+ BODY,
259
+
260
+ /// The end-of-stream state. No more data is processed.
261
+ EOS,
262
+ };
263
+
264
+ /// \brief Construct a message decoder.
265
+ ///
266
+ /// \param[in] listener a MessageDecoderListener that responds events from
267
+ /// the decoder
268
+ /// \param[in] pool an optional MemoryPool to copy metadata on the
269
+ /// \param[in] skip_body if true the body will be skipped even if the message has a body
270
+ /// CPU, if required
271
+ explicit MessageDecoder(std::shared_ptr<MessageDecoderListener> listener,
272
+ MemoryPool* pool = default_memory_pool(),
273
+ bool skip_body = false);
274
+
275
+ /// \brief Construct a message decoder with the specified state.
276
+ ///
277
+ /// This is a construct for advanced users that know how to decode
278
+ /// Message.
279
+ ///
280
+ /// \param[in] listener a MessageDecoderListener that responds events from
281
+ /// the decoder
282
+ /// \param[in] initial_state an initial state of the decode
283
+ /// \param[in] initial_next_required_size the number of bytes needed
284
+ /// to run the next action
285
+ /// \param[in] pool an optional MemoryPool to copy metadata on the
286
+ /// CPU, if required
287
+ /// \param[in] skip_body if true the body will be skipped even if the message has a body
288
+ MessageDecoder(std::shared_ptr<MessageDecoderListener> listener, State initial_state,
289
+ int64_t initial_next_required_size,
290
+ MemoryPool* pool = default_memory_pool(), bool skip_body = false);
291
+
292
+ virtual ~MessageDecoder();
293
+
294
+ /// \brief Feed data to the decoder as a raw data.
295
+ ///
296
+ /// If the decoder can decode one or more messages by the data, the
297
+ /// decoder calls listener->OnMessageDecoded() with a decoded
298
+ /// message multiple times.
299
+ ///
300
+ /// If the state of the decoder is changed, corresponding callbacks
301
+ /// on listener is called:
302
+ ///
303
+ /// * MessageDecoder::State::INITIAL: listener->OnInitial()
304
+ /// * MessageDecoder::State::METADATA_LENGTH: listener->OnMetadataLength()
305
+ /// * MessageDecoder::State::METADATA: listener->OnMetadata()
306
+ /// * MessageDecoder::State::BODY: listener->OnBody()
307
+ /// * MessageDecoder::State::EOS: listener->OnEOS()
308
+ ///
309
+ /// \param[in] data a raw data to be processed. This data isn't
310
+ /// copied. The passed memory must be kept alive through message
311
+ /// processing.
312
+ /// \param[in] size raw data size.
313
+ /// \return Status
314
+ Status Consume(const uint8_t* data, int64_t size);
315
+
316
+ /// \brief Feed data to the decoder as a Buffer.
317
+ ///
318
+ /// If the decoder can decode one or more messages by the Buffer,
319
+ /// the decoder calls listener->OnMessageDecoded() with a decoded
320
+ /// message multiple times.
321
+ ///
322
+ /// \param[in] buffer a Buffer to be processed.
323
+ /// \return Status
324
+ Status Consume(std::shared_ptr<Buffer> buffer);
325
+
326
+ /// \brief Return the number of bytes needed to advance the state of
327
+ /// the decoder.
328
+ ///
329
+ /// This method is provided for users who want to optimize performance.
330
+ /// Normal users don't need to use this method.
331
+ ///
332
+ /// Here is an example usage for normal users:
333
+ ///
334
+ /// ~~~{.cpp}
335
+ /// decoder.Consume(buffer1);
336
+ /// decoder.Consume(buffer2);
337
+ /// decoder.Consume(buffer3);
338
+ /// ~~~
339
+ ///
340
+ /// Decoder has internal buffer. If consumed data isn't enough to
341
+ /// advance the state of the decoder, consumed data is buffered to
342
+ /// the internal buffer. It causes performance overhead.
343
+ ///
344
+ /// If you pass next_required_size() size data to each Consume()
345
+ /// call, the decoder doesn't use its internal buffer. It improves
346
+ /// performance.
347
+ ///
348
+ /// Here is an example usage to avoid using internal buffer:
349
+ ///
350
+ /// ~~~{.cpp}
351
+ /// buffer1 = get_data(decoder.next_required_size());
352
+ /// decoder.Consume(buffer1);
353
+ /// buffer2 = get_data(decoder.next_required_size());
354
+ /// decoder.Consume(buffer2);
355
+ /// ~~~
356
+ ///
357
+ /// Users can use this method to avoid creating small
358
+ /// chunks. Message body must be contiguous data. If users pass
359
+ /// small chunks to the decoder, the decoder needs concatenate small
360
+ /// chunks internally. It causes performance overhead.
361
+ ///
362
+ /// Here is an example usage to reduce small chunks:
363
+ ///
364
+ /// ~~~{.cpp}
365
+ /// buffer = AllocateResizableBuffer();
366
+ /// while ((small_chunk = get_data(&small_chunk_size))) {
367
+ /// auto current_buffer_size = buffer->size();
368
+ /// buffer->Resize(current_buffer_size + small_chunk_size);
369
+ /// memcpy(buffer->mutable_data() + current_buffer_size,
370
+ /// small_chunk,
371
+ /// small_chunk_size);
372
+ /// if (buffer->size() < decoder.next_required_size()) {
373
+ /// continue;
374
+ /// }
375
+ /// std::shared_ptr<arrow::Buffer> chunk(buffer.release());
376
+ /// decoder.Consume(chunk);
377
+ /// buffer = AllocateResizableBuffer();
378
+ /// }
379
+ /// if (buffer->size() > 0) {
380
+ /// std::shared_ptr<arrow::Buffer> chunk(buffer.release());
381
+ /// decoder.Consume(chunk);
382
+ /// }
383
+ /// ~~~
384
+ ///
385
+ /// \return the number of bytes needed to advance the state of the
386
+ /// decoder
387
+ int64_t next_required_size() const;
388
+
389
+ /// \brief Return the current state of the decoder.
390
+ ///
391
+ /// This method is provided for users who want to optimize performance.
392
+ /// Normal users don't need to use this method.
393
+ ///
394
+ /// Decoder doesn't need Buffer to process data on the
395
+ /// MessageDecoder::State::INITIAL state and the
396
+ /// MessageDecoder::State::METADATA_LENGTH. Creating Buffer has
397
+ /// performance overhead. Advanced users can avoid creating Buffer
398
+ /// by checking the current state of the decoder:
399
+ ///
400
+ /// ~~~{.cpp}
401
+ /// switch (decoder.state()) {
402
+ /// MessageDecoder::State::INITIAL:
403
+ /// MessageDecoder::State::METADATA_LENGTH:
404
+ /// {
405
+ /// uint8_t data[sizeof(int32_t)];
406
+ /// auto data_size = input->Read(decoder.next_required_size(), data);
407
+ /// decoder.Consume(data, data_size);
408
+ /// }
409
+ /// break;
410
+ /// default:
411
+ /// {
412
+ /// auto buffer = input->Read(decoder.next_required_size());
413
+ /// decoder.Consume(buffer);
414
+ /// }
415
+ /// break;
416
+ /// }
417
+ /// ~~~
418
+ ///
419
+ /// \return the current state
420
+ State state() const;
421
+
422
+ private:
423
+ class MessageDecoderImpl;
424
+ std::unique_ptr<MessageDecoderImpl> impl_;
425
+
426
+ ARROW_DISALLOW_COPY_AND_ASSIGN(MessageDecoder);
427
+ };
428
+
429
+ /// \brief Abstract interface for a sequence of messages
430
+ /// \since 0.5.0
431
+ class ARROW_EXPORT MessageReader {
432
+ public:
433
+ virtual ~MessageReader() = default;
434
+
435
+ /// \brief Create MessageReader that reads from InputStream
436
+ static std::unique_ptr<MessageReader> Open(io::InputStream* stream);
437
+
438
+ /// \brief Create MessageReader that reads from owned InputStream
439
+ static std::unique_ptr<MessageReader> Open(
440
+ const std::shared_ptr<io::InputStream>& owned_stream);
441
+
442
+ /// \brief Read next Message from the interface
443
+ ///
444
+ /// \return an arrow::ipc::Message instance
445
+ virtual Result<std::unique_ptr<Message>> ReadNextMessage() = 0;
446
+ };
447
+
448
+ // the first parameter of the function should be a pointer to metadata (aka.
449
+ // org::apache::arrow::flatbuf::RecordBatch*)
450
+ using FieldsLoaderFunction = std::function<Status(const void*, io::RandomAccessFile*)>;
451
+
452
+ /// \brief Read encapsulated IPC message from position in file
453
+ ///
454
+ /// Read a length-prefixed message flatbuffer starting at the indicated file
455
+ /// offset. If the message has a body with non-zero length, it will also be
456
+ /// read
457
+ ///
458
+ /// The metadata_length includes at least the length prefix and the flatbuffer
459
+ ///
460
+ /// \param[in] offset the position in the file where the message starts. The
461
+ /// first 4 bytes after the offset are the message length
462
+ /// \param[in] metadata_length the total number of bytes to read from file
463
+ /// \param[in] file the seekable file interface to read from
464
+ /// \param[in] fields_loader the function for loading subset of fields from the given file
465
+ /// \return the message read
466
+
467
+ ARROW_EXPORT
468
+ Result<std::unique_ptr<Message>> ReadMessage(
469
+ const int64_t offset, const int32_t metadata_length, io::RandomAccessFile* file,
470
+ const FieldsLoaderFunction& fields_loader = {});
471
+
472
+ /// \brief Read encapsulated IPC message from position in file
473
+ ///
474
+ /// Read a length-prefixed message flatbuffer starting at the indicated file
475
+ /// offset.
476
+ ///
477
+ /// The metadata_length includes at least the length prefix and the flatbuffer
478
+ ///
479
+ /// \param[in] offset the position in the file where the message starts. The
480
+ /// first 4 bytes after the offset are the message length
481
+ /// \param[in] metadata_length the total number of bytes to read from file
482
+ /// \param[in] body_length the number of bytes for the message body
483
+ /// \param[in] file the seekable file interface to read from
484
+ /// \return the message read
485
+
486
+ ARROW_EXPORT
487
+ Result<std::unique_ptr<Message>> ReadMessage(const int64_t offset,
488
+ const int32_t metadata_length,
489
+ const int64_t body_length,
490
+ io::RandomAccessFile* file);
491
+
492
+ /// \brief Read encapsulated IPC message from cached buffers
493
+ ///
494
+ /// The buffers should contain an entire message. Partial reads are not handled.
495
+ ///
496
+ /// This method can be used to read just the metadata by passing in a nullptr for the
497
+ /// body. The body will then be skipped and the body size will not be validated.
498
+ ///
499
+ /// If the body buffer is provided then it must be the complete body buffer
500
+ ///
501
+ /// This is similar to Message::Open but performs slightly more validation (e.g. checks
502
+ /// to see that the metadata length is correct and that the body is the size the metadata
503
+ /// expected)
504
+ ///
505
+ /// \param metadata The bytes for the metadata
506
+ /// \param body The bytes for the body
507
+ /// \return The message represented by the buffers
508
+ ARROW_EXPORT Result<std::unique_ptr<Message>> ReadMessage(
509
+ std::shared_ptr<Buffer> metadata, std::shared_ptr<Buffer> body);
510
+
511
+ ARROW_EXPORT
512
+ Future<std::shared_ptr<Message>> ReadMessageAsync(
513
+ const int64_t offset, const int32_t metadata_length, const int64_t body_length,
514
+ io::RandomAccessFile* file, const io::IOContext& context = io::default_io_context());
515
+
516
+ /// \brief Advance stream to an 8-byte offset if its position is not a multiple
517
+ /// of 8 already
518
+ /// \param[in] stream an input stream
519
+ /// \param[in] alignment the byte multiple for the metadata prefix, usually 8
520
+ /// or 64, to ensure the body starts on a multiple of that alignment
521
+ /// \return Status
522
+ ARROW_EXPORT
523
+ Status AlignStream(io::InputStream* stream, int32_t alignment = 8);
524
+
525
+ /// \brief Advance stream to an 8-byte offset if its position is not a multiple
526
+ /// of 8 already
527
+ /// \param[in] stream an output stream
528
+ /// \param[in] alignment the byte multiple for the metadata prefix, usually 8
529
+ /// or 64, to ensure the body starts on a multiple of that alignment
530
+ /// \return Status
531
+ ARROW_EXPORT
532
+ Status AlignStream(io::OutputStream* stream, int32_t alignment = 8);
533
+
534
+ /// \brief Return error Status if file position is not a multiple of the
535
+ /// indicated alignment
536
+ ARROW_EXPORT
537
+ Status CheckAligned(io::FileInterface* stream, int32_t alignment = 8);
538
+
539
+ /// \brief Read encapsulated IPC message (metadata and body) from InputStream
540
+ ///
541
+ /// Returns null if there are not enough bytes available or the
542
+ /// message length is 0 (e.g. EOS in a stream)
543
+ ///
544
+ /// \param[in] stream an input stream
545
+ /// \param[in] pool an optional MemoryPool to copy metadata on the CPU, if required
546
+ /// \return Message
547
+ ARROW_EXPORT
548
+ Result<std::unique_ptr<Message>> ReadMessage(io::InputStream* stream,
549
+ MemoryPool* pool = default_memory_pool());
550
+
551
+ /// \brief Feed data from InputStream to MessageDecoder to decode an
552
+ /// encapsulated IPC message (metadata and body)
553
+ ///
554
+ /// This API is EXPERIMENTAL.
555
+ ///
556
+ /// \param[in] decoder a decoder
557
+ /// \param[in] stream an input stream
558
+ /// \return Status
559
+ ///
560
+ /// \since 0.17.0
561
+ ARROW_EXPORT
562
+ Status DecodeMessage(MessageDecoder* decoder, io::InputStream* stream);
563
+
564
+ /// Write encapsulated IPC message Does not make assumptions about
565
+ /// whether the stream is aligned already. Can write legacy (pre
566
+ /// version 0.15.0) IPC message if option set
567
+ ///
568
+ /// continuation: 0xFFFFFFFF
569
+ /// message_size: int32
570
+ /// message: const void*
571
+ /// padding
572
+ ///
573
+ ///
574
+ /// \param[in] message a buffer containing the metadata to write
575
+ /// \param[in] options IPC writing options, including alignment and
576
+ /// legacy message support
577
+ /// \param[in,out] file the OutputStream to write to
578
+ /// \param[out] message_length the total size of the payload written including
579
+ /// padding
580
+ /// \return Status
581
+ Status WriteMessage(const Buffer& message, const IpcWriteOptions& options,
582
+ io::OutputStream* file, int32_t* message_length);
583
+
584
+ } // namespace ipc
585
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/options.h ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <cstdint>
21
+ #include <optional>
22
+ #include <vector>
23
+
24
+ #include "arrow/io/caching.h"
25
+ #include "arrow/ipc/type_fwd.h"
26
+ #include "arrow/status.h"
27
+ #include "arrow/type_fwd.h"
28
+ #include "arrow/util/align_util.h"
29
+ #include "arrow/util/compression.h"
30
+ #include "arrow/util/visibility.h"
31
+
32
+ namespace arrow {
33
+
34
+ class MemoryPool;
35
+
36
+ namespace ipc {
37
+
38
+ // ARROW-109: We set this number arbitrarily to help catch user mistakes. For
39
+ // deeply nested schemas, it is expected the user will indicate explicitly the
40
+ // maximum allowed recursion depth
41
+ constexpr int kMaxNestingDepth = 64;
42
+
43
+ /// \brief Options for writing Arrow IPC messages
44
+ struct ARROW_EXPORT IpcWriteOptions {
45
+ /// \brief If true, allow field lengths that don't fit in a signed 32-bit int.
46
+ ///
47
+ /// Some implementations may not be able to parse streams created with this option.
48
+ bool allow_64bit = false;
49
+
50
+ /// \brief The maximum permitted schema nesting depth.
51
+ int max_recursion_depth = kMaxNestingDepth;
52
+
53
+ /// \brief Write padding after memory buffers up to this multiple of bytes.
54
+ int32_t alignment = 8;
55
+
56
+ /// \brief Write the pre-0.15.0 IPC message format
57
+ ///
58
+ /// This legacy format consists of a 4-byte prefix instead of 8-byte.
59
+ bool write_legacy_ipc_format = false;
60
+
61
+ /// \brief The memory pool to use for allocations made during IPC writing
62
+ ///
63
+ /// While Arrow IPC is predominantly zero-copy, it may have to allocate
64
+ /// memory in some cases (for example if compression is enabled).
65
+ MemoryPool* memory_pool = default_memory_pool();
66
+
67
+ /// \brief Compression codec to use for record batch body buffers
68
+ ///
69
+ /// May only be UNCOMPRESSED, LZ4_FRAME and ZSTD.
70
+ std::shared_ptr<util::Codec> codec;
71
+
72
+ /// \brief Minimum space savings percentage required for compression to be applied
73
+ ///
74
+ /// Space savings is calculated as (1.0 - compressed_size / uncompressed_size).
75
+ ///
76
+ /// For example, if min_space_savings = 0.1, a 100-byte body buffer won't undergo
77
+ /// compression if its expected compressed size exceeds 90 bytes. If this option is
78
+ /// unset, compression will be used indiscriminately. If no codec was supplied, this
79
+ /// option is ignored.
80
+ ///
81
+ /// Values outside of the range [0,1] are handled as errors.
82
+ ///
83
+ /// Note that enabling this option may result in unreadable data for Arrow C++ versions
84
+ /// prior to 12.0.0.
85
+ std::optional<double> min_space_savings;
86
+
87
+ /// \brief Use global CPU thread pool to parallelize any computational tasks
88
+ /// like compression
89
+ bool use_threads = true;
90
+
91
+ /// \brief Whether to emit dictionary deltas
92
+ ///
93
+ /// If false, a changed dictionary for a given field will emit a full
94
+ /// dictionary replacement.
95
+ /// If true, a changed dictionary will be compared against the previous
96
+ /// version. If possible, a dictionary delta will be emitted, otherwise
97
+ /// a full dictionary replacement.
98
+ ///
99
+ /// Default is false to maximize stream compatibility.
100
+ ///
101
+ /// Also, note that if a changed dictionary is a nested dictionary,
102
+ /// then a delta is never emitted, for compatibility with the read path.
103
+ bool emit_dictionary_deltas = false;
104
+
105
+ /// \brief Whether to unify dictionaries for the IPC file format
106
+ ///
107
+ /// The IPC file format doesn't support dictionary replacements.
108
+ /// Therefore, chunks of a column with a dictionary type must have the same
109
+ /// dictionary in each record batch (or an extended dictionary + delta).
110
+ ///
111
+ /// If this option is true, RecordBatchWriter::WriteTable will attempt
112
+ /// to unify dictionaries across each table column. If this option is
113
+ /// false, incompatible dictionaries across a table column will simply
114
+ /// raise an error.
115
+ ///
116
+ /// Note that enabling this option has a runtime cost. Also, not all types
117
+ /// currently support dictionary unification.
118
+ ///
119
+ /// This option is ignored for IPC streams, which support dictionary replacement
120
+ /// and deltas.
121
+ bool unify_dictionaries = false;
122
+
123
+ /// \brief Format version to use for IPC messages and their metadata.
124
+ ///
125
+ /// Presently using V5 version (readable by 1.0.0 and later).
126
+ /// V4 is also available (readable by 0.8.0 and later).
127
+ MetadataVersion metadata_version = MetadataVersion::V5;
128
+
129
+ static IpcWriteOptions Defaults();
130
+ };
131
+
132
+ /// \brief Alignment of data in memory
133
+ /// Alignment values larger than 0 are taken directly as byte alignment value
134
+ /// See util::EnsureAlignment(..., int64_t alignment, ...)
135
+ enum class Alignment : int64_t {
136
+ /// \brief data is aligned depending on the actual data type
137
+ kDataTypeSpecificAlignment = util::kValueAlignment,
138
+ /// \brief no particular alignment enforced
139
+ kAnyAlignment = 0,
140
+ /// \brief data is aligned to 64-byte boundary
141
+ k64ByteAlignment = 64
142
+ };
143
+
144
+ /// \brief Options for reading Arrow IPC messages
145
+ struct ARROW_EXPORT IpcReadOptions {
146
+ /// \brief The maximum permitted schema nesting depth.
147
+ int max_recursion_depth = kMaxNestingDepth;
148
+
149
+ /// \brief The memory pool to use for allocations made during IPC reading
150
+ ///
151
+ /// While Arrow IPC is predominantly zero-copy, it may have to allocate
152
+ /// memory in some cases (for example if compression is enabled).
153
+ MemoryPool* memory_pool = default_memory_pool();
154
+
155
+ /// \brief Top-level schema fields to include when deserializing RecordBatch.
156
+ ///
157
+ /// If empty (the default), return all deserialized fields.
158
+ /// If non-empty, the values are the indices of fields in the top-level schema.
159
+ std::vector<int> included_fields;
160
+
161
+ /// \brief Use global CPU thread pool to parallelize any computational tasks
162
+ /// like decompression
163
+ bool use_threads = true;
164
+
165
+ /// \brief Whether to convert incoming data to platform-native endianness
166
+ ///
167
+ /// If the endianness of the received schema is not equal to platform-native
168
+ /// endianness, then all buffers with endian-sensitive data will be byte-swapped.
169
+ /// This includes the value buffers of numeric types, temporal types, decimal
170
+ /// types, as well as the offset buffers of variable-sized binary and list-like
171
+ /// types.
172
+ ///
173
+ /// Endianness conversion is achieved by the RecordBatchFileReader,
174
+ /// RecordBatchStreamReader and StreamDecoder classes.
175
+ bool ensure_native_endian = true;
176
+
177
+ /// \brief How to align data if mis-aligned
178
+ ///
179
+ /// Data is copied to aligned memory locations allocated via the
180
+ /// MemoryPool configured as \ref arrow::ipc::IpcReadOptions::memory_pool.
181
+ /// Some use cases might require data to have a specific alignment, for example,
182
+ /// for the data buffer of an Int32 array to be aligned on a 4-byte boundary.
183
+ ///
184
+ /// Default (kAnyAlignment) keeps the alignment as is, so no copy of data occurs.
185
+ Alignment ensure_alignment = Alignment::kAnyAlignment;
186
+
187
+ /// \brief Options to control caching behavior when pre-buffering is requested
188
+ ///
189
+ /// The lazy property will always be reset to true to deliver the expected behavior
190
+ io::CacheOptions pre_buffer_cache_options = io::CacheOptions::LazyDefaults();
191
+
192
+ static IpcReadOptions Defaults();
193
+ };
194
+
195
+ namespace internal {
196
+
197
+ Status CheckCompressionSupported(Compression::type codec);
198
+
199
+ } // namespace internal
200
+ } // namespace ipc
201
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/reader.h ADDED
@@ -0,0 +1,638 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ // Read Arrow files and streams
19
+
20
+ #pragma once
21
+
22
+ #include <cstddef>
23
+ #include <cstdint>
24
+ #include <memory>
25
+ #include <utility>
26
+ #include <vector>
27
+
28
+ #include "arrow/io/caching.h"
29
+ #include "arrow/io/type_fwd.h"
30
+ #include "arrow/ipc/message.h"
31
+ #include "arrow/ipc/options.h"
32
+ #include "arrow/record_batch.h"
33
+ #include "arrow/result.h"
34
+ #include "arrow/type_fwd.h"
35
+ #include "arrow/util/async_generator.h"
36
+ #include "arrow/util/macros.h"
37
+ #include "arrow/util/visibility.h"
38
+
39
+ namespace arrow {
40
+ namespace ipc {
41
+
42
+ class DictionaryMemo;
43
+ struct IpcPayload;
44
+
45
+ using RecordBatchReader = ::arrow::RecordBatchReader;
46
+
47
+ struct ReadStats {
48
+ /// Number of IPC messages read.
49
+ int64_t num_messages = 0;
50
+ /// Number of record batches read.
51
+ int64_t num_record_batches = 0;
52
+ /// Number of dictionary batches read.
53
+ ///
54
+ /// Note: num_dictionary_batches >= num_dictionary_deltas + num_replaced_dictionaries
55
+ int64_t num_dictionary_batches = 0;
56
+
57
+ /// Number of dictionary deltas read.
58
+ int64_t num_dictionary_deltas = 0;
59
+ /// Number of replaced dictionaries (i.e. where a dictionary batch replaces
60
+ /// an existing dictionary with an unrelated new dictionary).
61
+ int64_t num_replaced_dictionaries = 0;
62
+ };
63
+
64
+ /// \brief Synchronous batch stream reader that reads from io::InputStream
65
+ ///
66
+ /// This class reads the schema (plus any dictionaries) as the first messages
67
+ /// in the stream, followed by record batches. For more granular zero-copy
68
+ /// reads see the ReadRecordBatch functions
69
+ class ARROW_EXPORT RecordBatchStreamReader : public RecordBatchReader {
70
+ public:
71
+ /// Create batch reader from generic MessageReader.
72
+ /// This will take ownership of the given MessageReader.
73
+ ///
74
+ /// \param[in] message_reader a MessageReader implementation
75
+ /// \param[in] options any IPC reading options (optional)
76
+ /// \return the created batch reader
77
+ static Result<std::shared_ptr<RecordBatchStreamReader>> Open(
78
+ std::unique_ptr<MessageReader> message_reader,
79
+ const IpcReadOptions& options = IpcReadOptions::Defaults());
80
+
81
+ /// \brief Record batch stream reader from InputStream
82
+ ///
83
+ /// \param[in] stream an input stream instance. Must stay alive throughout
84
+ /// lifetime of stream reader
85
+ /// \param[in] options any IPC reading options (optional)
86
+ /// \return the created batch reader
87
+ static Result<std::shared_ptr<RecordBatchStreamReader>> Open(
88
+ io::InputStream* stream,
89
+ const IpcReadOptions& options = IpcReadOptions::Defaults());
90
+
91
+ /// \brief Open stream and retain ownership of stream object
92
+ /// \param[in] stream the input stream
93
+ /// \param[in] options any IPC reading options (optional)
94
+ /// \return the created batch reader
95
+ static Result<std::shared_ptr<RecordBatchStreamReader>> Open(
96
+ const std::shared_ptr<io::InputStream>& stream,
97
+ const IpcReadOptions& options = IpcReadOptions::Defaults());
98
+
99
+ /// \brief Return current read statistics
100
+ virtual ReadStats stats() const = 0;
101
+ };
102
+
103
+ /// \brief Reads the record batch file format
104
+ class ARROW_EXPORT RecordBatchFileReader
105
+ : public std::enable_shared_from_this<RecordBatchFileReader> {
106
+ public:
107
+ virtual ~RecordBatchFileReader() = default;
108
+
109
+ /// \brief Open a RecordBatchFileReader
110
+ ///
111
+ /// Open a file-like object that is assumed to be self-contained; i.e., the
112
+ /// end of the file interface is the end of the Arrow file. Note that there
113
+ /// can be any amount of data preceding the Arrow-formatted data, because we
114
+ /// need only locate the end of the Arrow file stream to discover the metadata
115
+ /// and then proceed to read the data into memory.
116
+ static Result<std::shared_ptr<RecordBatchFileReader>> Open(
117
+ io::RandomAccessFile* file,
118
+ const IpcReadOptions& options = IpcReadOptions::Defaults());
119
+
120
+ /// \brief Open a RecordBatchFileReader
121
+ /// If the file is embedded within some larger file or memory region, you can
122
+ /// pass the absolute memory offset to the end of the file (which contains the
123
+ /// metadata footer). The metadata must have been written with memory offsets
124
+ /// relative to the start of the containing file
125
+ ///
126
+ /// \param[in] file the data source
127
+ /// \param[in] footer_offset the position of the end of the Arrow file
128
+ /// \param[in] options options for IPC reading
129
+ /// \return the returned reader
130
+ static Result<std::shared_ptr<RecordBatchFileReader>> Open(
131
+ io::RandomAccessFile* file, int64_t footer_offset,
132
+ const IpcReadOptions& options = IpcReadOptions::Defaults());
133
+
134
+ /// \brief Version of Open that retains ownership of file
135
+ ///
136
+ /// \param[in] file the data source
137
+ /// \param[in] options options for IPC reading
138
+ /// \return the returned reader
139
+ static Result<std::shared_ptr<RecordBatchFileReader>> Open(
140
+ const std::shared_ptr<io::RandomAccessFile>& file,
141
+ const IpcReadOptions& options = IpcReadOptions::Defaults());
142
+
143
+ /// \brief Version of Open that retains ownership of file
144
+ ///
145
+ /// \param[in] file the data source
146
+ /// \param[in] footer_offset the position of the end of the Arrow file
147
+ /// \param[in] options options for IPC reading
148
+ /// \return the returned reader
149
+ static Result<std::shared_ptr<RecordBatchFileReader>> Open(
150
+ const std::shared_ptr<io::RandomAccessFile>& file, int64_t footer_offset,
151
+ const IpcReadOptions& options = IpcReadOptions::Defaults());
152
+
153
+ /// \brief Open a file asynchronously (owns the file).
154
+ static Future<std::shared_ptr<RecordBatchFileReader>> OpenAsync(
155
+ const std::shared_ptr<io::RandomAccessFile>& file,
156
+ const IpcReadOptions& options = IpcReadOptions::Defaults());
157
+
158
+ /// \brief Open a file asynchronously (borrows the file).
159
+ static Future<std::shared_ptr<RecordBatchFileReader>> OpenAsync(
160
+ io::RandomAccessFile* file,
161
+ const IpcReadOptions& options = IpcReadOptions::Defaults());
162
+
163
+ /// \brief Open a file asynchronously (owns the file).
164
+ static Future<std::shared_ptr<RecordBatchFileReader>> OpenAsync(
165
+ const std::shared_ptr<io::RandomAccessFile>& file, int64_t footer_offset,
166
+ const IpcReadOptions& options = IpcReadOptions::Defaults());
167
+
168
+ /// \brief Open a file asynchronously (borrows the file).
169
+ static Future<std::shared_ptr<RecordBatchFileReader>> OpenAsync(
170
+ io::RandomAccessFile* file, int64_t footer_offset,
171
+ const IpcReadOptions& options = IpcReadOptions::Defaults());
172
+
173
+ /// \brief The schema read from the file
174
+ virtual std::shared_ptr<Schema> schema() const = 0;
175
+
176
+ /// \brief Returns the number of record batches in the file
177
+ virtual int num_record_batches() const = 0;
178
+
179
+ /// \brief Return the metadata version from the file metadata
180
+ virtual MetadataVersion version() const = 0;
181
+
182
+ /// \brief Return the contents of the custom_metadata field from the file's
183
+ /// Footer
184
+ virtual std::shared_ptr<const KeyValueMetadata> metadata() const = 0;
185
+
186
+ /// \brief Read a particular record batch from the file. Does not copy memory
187
+ /// if the input source supports zero-copy.
188
+ ///
189
+ /// \param[in] i the index of the record batch to return
190
+ /// \return the read batch
191
+ virtual Result<std::shared_ptr<RecordBatch>> ReadRecordBatch(int i) = 0;
192
+
193
+ /// \brief Read a particular record batch along with its custom metadata from the file.
194
+ /// Does not copy memory if the input source supports zero-copy.
195
+ ///
196
+ /// \param[in] i the index of the record batch to return
197
+ /// \return a struct containing the read batch and its custom metadata
198
+ virtual Result<RecordBatchWithMetadata> ReadRecordBatchWithCustomMetadata(int i) = 0;
199
+
200
+ /// \brief Return current read statistics
201
+ virtual ReadStats stats() const = 0;
202
+
203
+ /// \brief Computes the total number of rows in the file.
204
+ virtual Result<int64_t> CountRows() = 0;
205
+
206
+ /// \brief Begin loading metadata for the desired batches into memory.
207
+ ///
208
+ /// This method will also begin loading all dictionaries messages into memory.
209
+ ///
210
+ /// For a regular file this will immediately begin disk I/O in the background on a
211
+ /// thread on the IOContext's thread pool. If the file is memory mapped this will
212
+ /// ensure the memory needed for the metadata is paged from disk into memory
213
+ ///
214
+ /// \param indices Indices of the batches to prefetch
215
+ /// If empty then all batches will be prefetched.
216
+ virtual Status PreBufferMetadata(const std::vector<int>& indices) = 0;
217
+
218
+ /// \brief Get a reentrant generator of record batches.
219
+ ///
220
+ /// \param[in] coalesce If true, enable I/O coalescing.
221
+ /// \param[in] io_context The IOContext to use (controls which thread pool
222
+ /// is used for I/O).
223
+ /// \param[in] cache_options Options for coalescing (if enabled).
224
+ /// \param[in] executor Optionally, an executor to use for decoding record
225
+ /// batches. This is generally only a benefit for very wide and/or
226
+ /// compressed batches.
227
+ virtual Result<AsyncGenerator<std::shared_ptr<RecordBatch>>> GetRecordBatchGenerator(
228
+ const bool coalesce = false,
229
+ const io::IOContext& io_context = io::default_io_context(),
230
+ const io::CacheOptions cache_options = io::CacheOptions::LazyDefaults(),
231
+ arrow::internal::Executor* executor = NULLPTR) = 0;
232
+
233
+ /// \brief Collect all batches as a vector of record batches
234
+ Result<RecordBatchVector> ToRecordBatches();
235
+
236
+ /// \brief Collect all batches and concatenate as arrow::Table
237
+ Result<std::shared_ptr<Table>> ToTable();
238
+ };
239
+
240
+ /// \brief A general listener class to receive events.
241
+ ///
242
+ /// You must implement callback methods for interested events.
243
+ ///
244
+ /// This API is EXPERIMENTAL.
245
+ ///
246
+ /// \since 0.17.0
247
+ class ARROW_EXPORT Listener {
248
+ public:
249
+ virtual ~Listener() = default;
250
+
251
+ /// \brief Called when end-of-stream is received.
252
+ ///
253
+ /// The default implementation just returns arrow::Status::OK().
254
+ ///
255
+ /// \return Status
256
+ ///
257
+ /// \see StreamDecoder
258
+ virtual Status OnEOS();
259
+
260
+ /// \brief Called when a record batch is decoded and
261
+ /// OnRecordBatchWithMetadataDecoded() isn't overridden.
262
+ ///
263
+ /// The default implementation just returns
264
+ /// arrow::Status::NotImplemented().
265
+ ///
266
+ /// \param[in] record_batch a record batch decoded
267
+ /// \return Status
268
+ ///
269
+ /// \see StreamDecoder
270
+ virtual Status OnRecordBatchDecoded(std::shared_ptr<RecordBatch> record_batch);
271
+
272
+ /// \brief Called when a record batch with custom metadata is decoded.
273
+ ///
274
+ /// The default implementation just calls OnRecordBatchDecoded()
275
+ /// without custom metadata.
276
+ ///
277
+ /// \param[in] record_batch_with_metadata a record batch with custom
278
+ /// metadata decoded
279
+ /// \return Status
280
+ ///
281
+ /// \see StreamDecoder
282
+ ///
283
+ /// \since 13.0.0
284
+ virtual Status OnRecordBatchWithMetadataDecoded(
285
+ RecordBatchWithMetadata record_batch_with_metadata);
286
+
287
+ /// \brief Called when a schema is decoded.
288
+ ///
289
+ /// The default implementation just returns arrow::Status::OK().
290
+ ///
291
+ /// \param[in] schema a schema decoded
292
+ /// \return Status
293
+ ///
294
+ /// \see StreamDecoder
295
+ virtual Status OnSchemaDecoded(std::shared_ptr<Schema> schema);
296
+
297
+ /// \brief Called when a schema is decoded.
298
+ ///
299
+ /// The default implementation just calls OnSchemaDecoded(schema)
300
+ /// (without filtered_schema) to keep backward compatibility.
301
+ ///
302
+ /// \param[in] schema a schema decoded
303
+ /// \param[in] filtered_schema a filtered schema that only has read fields
304
+ /// \return Status
305
+ ///
306
+ /// \see StreamDecoder
307
+ ///
308
+ /// \since 13.0.0
309
+ virtual Status OnSchemaDecoded(std::shared_ptr<Schema> schema,
310
+ std::shared_ptr<Schema> filtered_schema);
311
+ };
312
+
313
+ /// \brief Collect schema and record batches decoded by StreamDecoder.
314
+ ///
315
+ /// This API is EXPERIMENTAL.
316
+ ///
317
+ /// \since 0.17.0
318
+ class ARROW_EXPORT CollectListener : public Listener {
319
+ public:
320
+ CollectListener() : schema_(), filtered_schema_(), record_batches_(), metadatas_() {}
321
+ virtual ~CollectListener() = default;
322
+
323
+ Status OnSchemaDecoded(std::shared_ptr<Schema> schema,
324
+ std::shared_ptr<Schema> filtered_schema) override {
325
+ schema_ = std::move(schema);
326
+ filtered_schema_ = std::move(filtered_schema);
327
+ return Status::OK();
328
+ }
329
+
330
+ Status OnRecordBatchWithMetadataDecoded(
331
+ RecordBatchWithMetadata record_batch_with_metadata) override {
332
+ record_batches_.push_back(std::move(record_batch_with_metadata.batch));
333
+ metadatas_.push_back(std::move(record_batch_with_metadata.custom_metadata));
334
+ return Status::OK();
335
+ }
336
+
337
+ /// \return the decoded schema
338
+ std::shared_ptr<Schema> schema() const { return schema_; }
339
+
340
+ /// \return the filtered schema
341
+ std::shared_ptr<Schema> filtered_schema() const { return filtered_schema_; }
342
+
343
+ /// \return the all decoded record batches
344
+ const std::vector<std::shared_ptr<RecordBatch>>& record_batches() const {
345
+ return record_batches_;
346
+ }
347
+
348
+ /// \return the all decoded metadatas
349
+ const std::vector<std::shared_ptr<KeyValueMetadata>>& metadatas() const {
350
+ return metadatas_;
351
+ }
352
+
353
+ /// \return the number of collected record batches
354
+ int64_t num_record_batches() const { return record_batches_.size(); }
355
+
356
+ /// \return the last decoded record batch and remove it from
357
+ /// record_batches
358
+ std::shared_ptr<RecordBatch> PopRecordBatch() {
359
+ auto record_batch_with_metadata = PopRecordBatchWithMetadata();
360
+ return std::move(record_batch_with_metadata.batch);
361
+ }
362
+
363
+ /// \return the last decoded record batch with custom metadata and
364
+ /// remove it from record_batches
365
+ RecordBatchWithMetadata PopRecordBatchWithMetadata() {
366
+ RecordBatchWithMetadata record_batch_with_metadata;
367
+ if (record_batches_.empty()) {
368
+ return record_batch_with_metadata;
369
+ }
370
+ record_batch_with_metadata.batch = std::move(record_batches_.back());
371
+ record_batch_with_metadata.custom_metadata = std::move(metadatas_.back());
372
+ record_batches_.pop_back();
373
+ metadatas_.pop_back();
374
+ return record_batch_with_metadata;
375
+ }
376
+
377
+ private:
378
+ std::shared_ptr<Schema> schema_;
379
+ std::shared_ptr<Schema> filtered_schema_;
380
+ std::vector<std::shared_ptr<RecordBatch>> record_batches_;
381
+ std::vector<std::shared_ptr<KeyValueMetadata>> metadatas_;
382
+ };
383
+
384
+ /// \brief Push style stream decoder that receives data from user.
385
+ ///
386
+ /// This class decodes the Apache Arrow IPC streaming format data.
387
+ ///
388
+ /// This API is EXPERIMENTAL.
389
+ ///
390
+ /// \see https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format
391
+ ///
392
+ /// \since 0.17.0
393
+ class ARROW_EXPORT StreamDecoder {
394
+ public:
395
+ /// \brief Construct a stream decoder.
396
+ ///
397
+ /// \param[in] listener a Listener that must implement
398
+ /// Listener::OnRecordBatchDecoded() to receive decoded record batches
399
+ /// \param[in] options any IPC reading options (optional)
400
+ StreamDecoder(std::shared_ptr<Listener> listener,
401
+ IpcReadOptions options = IpcReadOptions::Defaults());
402
+
403
+ virtual ~StreamDecoder();
404
+
405
+ /// \brief Feed data to the decoder as a raw data.
406
+ ///
407
+ /// If the decoder can read one or more record batches by the data,
408
+ /// the decoder calls listener->OnRecordBatchDecoded() with a
409
+ /// decoded record batch multiple times.
410
+ ///
411
+ /// \param[in] data a raw data to be processed. This data isn't
412
+ /// copied. The passed memory must be kept alive through record
413
+ /// batch processing.
414
+ /// \param[in] size raw data size.
415
+ /// \return Status
416
+ Status Consume(const uint8_t* data, int64_t size);
417
+
418
+ /// \brief Feed data to the decoder as a Buffer.
419
+ ///
420
+ /// If the decoder can read one or more record batches by the
421
+ /// Buffer, the decoder calls listener->RecordBatchReceived() with a
422
+ /// decoded record batch multiple times.
423
+ ///
424
+ /// \param[in] buffer a Buffer to be processed.
425
+ /// \return Status
426
+ Status Consume(std::shared_ptr<Buffer> buffer);
427
+
428
+ /// \brief Reset the internal status.
429
+ ///
430
+ /// You can reuse this decoder for new stream after calling
431
+ /// this.
432
+ ///
433
+ /// \return Status
434
+ Status Reset();
435
+
436
+ /// \return the shared schema of the record batches in the stream
437
+ std::shared_ptr<Schema> schema() const;
438
+
439
+ /// \brief Return the number of bytes needed to advance the state of
440
+ /// the decoder.
441
+ ///
442
+ /// This method is provided for users who want to optimize performance.
443
+ /// Normal users don't need to use this method.
444
+ ///
445
+ /// Here is an example usage for normal users:
446
+ ///
447
+ /// ~~~{.cpp}
448
+ /// decoder.Consume(buffer1);
449
+ /// decoder.Consume(buffer2);
450
+ /// decoder.Consume(buffer3);
451
+ /// ~~~
452
+ ///
453
+ /// Decoder has internal buffer. If consumed data isn't enough to
454
+ /// advance the state of the decoder, consumed data is buffered to
455
+ /// the internal buffer. It causes performance overhead.
456
+ ///
457
+ /// If you pass next_required_size() size data to each Consume()
458
+ /// call, the decoder doesn't use its internal buffer. It improves
459
+ /// performance.
460
+ ///
461
+ /// Here is an example usage to avoid using internal buffer:
462
+ ///
463
+ /// ~~~{.cpp}
464
+ /// buffer1 = get_data(decoder.next_required_size());
465
+ /// decoder.Consume(buffer1);
466
+ /// buffer2 = get_data(decoder.next_required_size());
467
+ /// decoder.Consume(buffer2);
468
+ /// ~~~
469
+ ///
470
+ /// Users can use this method to avoid creating small chunks. Record
471
+ /// batch data must be contiguous data. If users pass small chunks
472
+ /// to the decoder, the decoder needs concatenate small chunks
473
+ /// internally. It causes performance overhead.
474
+ ///
475
+ /// Here is an example usage to reduce small chunks:
476
+ ///
477
+ /// ~~~{.cpp}
478
+ /// buffer = AllocateResizableBuffer();
479
+ /// while ((small_chunk = get_data(&small_chunk_size))) {
480
+ /// auto current_buffer_size = buffer->size();
481
+ /// buffer->Resize(current_buffer_size + small_chunk_size);
482
+ /// memcpy(buffer->mutable_data() + current_buffer_size,
483
+ /// small_chunk,
484
+ /// small_chunk_size);
485
+ /// if (buffer->size() < decoder.next_required_size()) {
486
+ /// continue;
487
+ /// }
488
+ /// std::shared_ptr<arrow::Buffer> chunk(buffer.release());
489
+ /// decoder.Consume(chunk);
490
+ /// buffer = AllocateResizableBuffer();
491
+ /// }
492
+ /// if (buffer->size() > 0) {
493
+ /// std::shared_ptr<arrow::Buffer> chunk(buffer.release());
494
+ /// decoder.Consume(chunk);
495
+ /// }
496
+ /// ~~~
497
+ ///
498
+ /// \return the number of bytes needed to advance the state of the
499
+ /// decoder
500
+ int64_t next_required_size() const;
501
+
502
+ /// \brief Return current read statistics
503
+ ReadStats stats() const;
504
+
505
+ private:
506
+ class StreamDecoderImpl;
507
+ std::unique_ptr<StreamDecoderImpl> impl_;
508
+
509
+ ARROW_DISALLOW_COPY_AND_ASSIGN(StreamDecoder);
510
+ };
511
+
512
+ // Generic read functions; does not copy data if the input supports zero copy reads
513
+
514
+ /// \brief Read Schema from stream serialized as a single IPC message
515
+ /// and populate any dictionary-encoded fields into a DictionaryMemo
516
+ ///
517
+ /// \param[in] stream an InputStream
518
+ /// \param[in] dictionary_memo for recording dictionary-encoded fields
519
+ /// \return the output Schema
520
+ ///
521
+ /// If record batches follow the schema, it is better to use
522
+ /// RecordBatchStreamReader
523
+ ARROW_EXPORT
524
+ Result<std::shared_ptr<Schema>> ReadSchema(io::InputStream* stream,
525
+ DictionaryMemo* dictionary_memo);
526
+
527
+ /// \brief Read Schema from encapsulated Message
528
+ ///
529
+ /// \param[in] message the message containing the Schema IPC metadata
530
+ /// \param[in] dictionary_memo DictionaryMemo for recording dictionary-encoded
531
+ /// fields. Can be nullptr if you are sure there are no
532
+ /// dictionary-encoded fields
533
+ /// \return the resulting Schema
534
+ ARROW_EXPORT
535
+ Result<std::shared_ptr<Schema>> ReadSchema(const Message& message,
536
+ DictionaryMemo* dictionary_memo);
537
+
538
+ /// Read record batch as encapsulated IPC message with metadata size prefix and
539
+ /// header
540
+ ///
541
+ /// \param[in] schema the record batch schema
542
+ /// \param[in] dictionary_memo DictionaryMemo which has any
543
+ /// dictionaries. Can be nullptr if you are sure there are no
544
+ /// dictionary-encoded fields
545
+ /// \param[in] options IPC options for reading
546
+ /// \param[in] stream the file where the batch is located
547
+ /// \return the read record batch
548
+ ARROW_EXPORT
549
+ Result<std::shared_ptr<RecordBatch>> ReadRecordBatch(
550
+ const std::shared_ptr<Schema>& schema, const DictionaryMemo* dictionary_memo,
551
+ const IpcReadOptions& options, io::InputStream* stream);
552
+
553
+ /// \brief Read record batch from message
554
+ ///
555
+ /// \param[in] message a Message containing the record batch metadata
556
+ /// \param[in] schema the record batch schema
557
+ /// \param[in] dictionary_memo DictionaryMemo which has any
558
+ /// dictionaries. Can be nullptr if you are sure there are no
559
+ /// dictionary-encoded fields
560
+ /// \param[in] options IPC options for reading
561
+ /// \return the read record batch
562
+ ARROW_EXPORT
563
+ Result<std::shared_ptr<RecordBatch>> ReadRecordBatch(
564
+ const Message& message, const std::shared_ptr<Schema>& schema,
565
+ const DictionaryMemo* dictionary_memo, const IpcReadOptions& options);
566
+
567
+ /// Read record batch from file given metadata and schema
568
+ ///
569
+ /// \param[in] metadata a Message containing the record batch metadata
570
+ /// \param[in] schema the record batch schema
571
+ /// \param[in] dictionary_memo DictionaryMemo which has any
572
+ /// dictionaries. Can be nullptr if you are sure there are no
573
+ /// dictionary-encoded fields
574
+ /// \param[in] file a random access file
575
+ /// \param[in] options options for deserialization
576
+ /// \return the read record batch
577
+ ARROW_EXPORT
578
+ Result<std::shared_ptr<RecordBatch>> ReadRecordBatch(
579
+ const Buffer& metadata, const std::shared_ptr<Schema>& schema,
580
+ const DictionaryMemo* dictionary_memo, const IpcReadOptions& options,
581
+ io::RandomAccessFile* file);
582
+
583
+ /// \brief Read arrow::Tensor as encapsulated IPC message in file
584
+ ///
585
+ /// \param[in] file an InputStream pointed at the start of the message
586
+ /// \return the read tensor
587
+ ARROW_EXPORT
588
+ Result<std::shared_ptr<Tensor>> ReadTensor(io::InputStream* file);
589
+
590
+ /// \brief EXPERIMENTAL: Read arrow::Tensor from IPC message
591
+ ///
592
+ /// \param[in] message a Message containing the tensor metadata and body
593
+ /// \return the read tensor
594
+ ARROW_EXPORT
595
+ Result<std::shared_ptr<Tensor>> ReadTensor(const Message& message);
596
+
597
+ /// \brief EXPERIMENTAL: Read arrow::SparseTensor as encapsulated IPC message in file
598
+ ///
599
+ /// \param[in] file an InputStream pointed at the start of the message
600
+ /// \return the read sparse tensor
601
+ ARROW_EXPORT
602
+ Result<std::shared_ptr<SparseTensor>> ReadSparseTensor(io::InputStream* file);
603
+
604
+ /// \brief EXPERIMENTAL: Read arrow::SparseTensor from IPC message
605
+ ///
606
+ /// \param[in] message a Message containing the tensor metadata and body
607
+ /// \return the read sparse tensor
608
+ ARROW_EXPORT
609
+ Result<std::shared_ptr<SparseTensor>> ReadSparseTensor(const Message& message);
610
+
611
+ namespace internal {
612
+
613
+ // These internal APIs may change without warning or deprecation
614
+
615
+ /// \brief EXPERIMENTAL: Read arrow::SparseTensorFormat::type from a metadata
616
+ /// \param[in] metadata a Buffer containing the sparse tensor metadata
617
+ /// \return the count of the body buffers
618
+ ARROW_EXPORT
619
+ Result<size_t> ReadSparseTensorBodyBufferCount(const Buffer& metadata);
620
+
621
+ /// \brief EXPERIMENTAL: Read arrow::SparseTensor from an IpcPayload
622
+ /// \param[in] payload a IpcPayload contains a serialized SparseTensor
623
+ /// \return the read sparse tensor
624
+ ARROW_EXPORT
625
+ Result<std::shared_ptr<SparseTensor>> ReadSparseTensorPayload(const IpcPayload& payload);
626
+
627
+ // For fuzzing targets
628
+ ARROW_EXPORT
629
+ Status FuzzIpcStream(const uint8_t* data, int64_t size);
630
+ ARROW_EXPORT
631
+ Status FuzzIpcTensorStream(const uint8_t* data, int64_t size);
632
+ ARROW_EXPORT
633
+ Status FuzzIpcFile(const uint8_t* data, int64_t size);
634
+
635
+ } // namespace internal
636
+
637
+ } // namespace ipc
638
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/test_common.h ADDED
@@ -0,0 +1,198 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <cstdint>
21
+ #include <memory>
22
+ #include <vector>
23
+
24
+ #include "arrow/array.h"
25
+ #include "arrow/record_batch.h"
26
+ #include "arrow/status.h"
27
+ #include "arrow/testing/visibility.h"
28
+ #include "arrow/type.h"
29
+
30
+ namespace arrow {
31
+ namespace ipc {
32
+ namespace test {
33
+
34
+ // A typedef used for test parameterization
35
+ typedef Status MakeRecordBatch(std::shared_ptr<RecordBatch>* out);
36
+
37
+ ARROW_TESTING_EXPORT
38
+ void CompareArraysDetailed(int index, const Array& result, const Array& expected);
39
+
40
+ ARROW_TESTING_EXPORT
41
+ void CompareBatchColumnsDetailed(const RecordBatch& result, const RecordBatch& expected);
42
+
43
+ ARROW_TESTING_EXPORT
44
+ Status MakeRandomInt32Array(int64_t length, bool include_nulls, MemoryPool* pool,
45
+ std::shared_ptr<Array>* out, uint32_t seed = 0,
46
+ int32_t min = 0, int32_t max = 1000);
47
+
48
+ ARROW_TESTING_EXPORT
49
+ Status MakeRandomInt64Array(int64_t length, bool include_nulls, MemoryPool* pool,
50
+ std::shared_ptr<Array>* out, uint32_t seed = 0);
51
+
52
+ ARROW_TESTING_EXPORT
53
+ Status MakeRandomListArray(const std::shared_ptr<Array>& child_array, int num_lists,
54
+ bool include_nulls, MemoryPool* pool,
55
+ std::shared_ptr<Array>* out);
56
+
57
+ ARROW_TESTING_EXPORT
58
+ Status MakeRandomLargeListArray(const std::shared_ptr<Array>& child_array, int num_lists,
59
+ bool include_nulls, MemoryPool* pool,
60
+ std::shared_ptr<Array>* out);
61
+
62
+ ARROW_TESTING_EXPORT
63
+ Status MakeRandomBooleanArray(const int length, bool include_nulls,
64
+ std::shared_ptr<Array>* out);
65
+
66
+ ARROW_TESTING_EXPORT
67
+ Status MakeBooleanBatchSized(const int length, std::shared_ptr<RecordBatch>* out);
68
+
69
+ ARROW_TESTING_EXPORT
70
+ Status MakeBooleanBatch(std::shared_ptr<RecordBatch>* out);
71
+
72
+ ARROW_TESTING_EXPORT
73
+ Status MakeIntBatchSized(int length, std::shared_ptr<RecordBatch>* out,
74
+ uint32_t seed = 0);
75
+
76
+ ARROW_TESTING_EXPORT
77
+ Status MakeIntRecordBatch(std::shared_ptr<RecordBatch>* out);
78
+
79
+ ARROW_TESTING_EXPORT
80
+ Status MakeFloat3264BatchSized(int length, std::shared_ptr<RecordBatch>* out,
81
+ uint32_t seed = 0);
82
+
83
+ ARROW_TESTING_EXPORT
84
+ Status MakeFloat3264Batch(std::shared_ptr<RecordBatch>* out);
85
+
86
+ ARROW_TESTING_EXPORT
87
+ Status MakeFloatBatchSized(int length, std::shared_ptr<RecordBatch>* out,
88
+ uint32_t seed = 0);
89
+
90
+ ARROW_TESTING_EXPORT
91
+ Status MakeFloatBatch(std::shared_ptr<RecordBatch>* out);
92
+
93
+ ARROW_TESTING_EXPORT
94
+ Status MakeRandomStringArray(int64_t length, bool include_nulls, MemoryPool* pool,
95
+ std::shared_ptr<Array>* out);
96
+
97
+ ARROW_TESTING_EXPORT
98
+ Status MakeStringTypesRecordBatch(std::shared_ptr<RecordBatch>* out,
99
+ bool with_nulls = true, bool with_view_types = true);
100
+
101
+ ARROW_TESTING_EXPORT
102
+ Status MakeStringTypesRecordBatchWithNulls(std::shared_ptr<RecordBatch>* out);
103
+
104
+ ARROW_TESTING_EXPORT
105
+ Status MakeNullRecordBatch(std::shared_ptr<RecordBatch>* out);
106
+
107
+ ARROW_TESTING_EXPORT
108
+ Status MakeListRecordBatchSized(int length, std::shared_ptr<RecordBatch>* out);
109
+
110
+ ARROW_TESTING_EXPORT
111
+ Status MakeListRecordBatch(std::shared_ptr<RecordBatch>* out);
112
+
113
+ ARROW_TESTING_EXPORT
114
+ Status MakeListViewRecordBatchSized(int length, std::shared_ptr<RecordBatch>* out);
115
+
116
+ ARROW_TESTING_EXPORT
117
+ Status MakeListViewRecordBatch(std::shared_ptr<RecordBatch>* out);
118
+
119
+ ARROW_TESTING_EXPORT
120
+ Status MakeFixedSizeListRecordBatch(std::shared_ptr<RecordBatch>* out);
121
+
122
+ ARROW_TESTING_EXPORT
123
+ Status MakeZeroLengthRecordBatch(std::shared_ptr<RecordBatch>* out);
124
+
125
+ ARROW_TESTING_EXPORT
126
+ Status MakeNonNullRecordBatch(std::shared_ptr<RecordBatch>* out);
127
+
128
+ ARROW_TESTING_EXPORT
129
+ Status MakeDeeplyNestedList(std::shared_ptr<RecordBatch>* out);
130
+
131
+ ARROW_TESTING_EXPORT
132
+ Status MakeDeeplyNestedListView(std::shared_ptr<RecordBatch>* out);
133
+
134
+ ARROW_TESTING_EXPORT
135
+ Status MakeStruct(std::shared_ptr<RecordBatch>* out);
136
+
137
+ ARROW_TESTING_EXPORT
138
+ Status MakeRunEndEncoded(std::shared_ptr<RecordBatch>* out);
139
+
140
+ ARROW_TESTING_EXPORT
141
+ Status MakeUnion(std::shared_ptr<RecordBatch>* out);
142
+
143
+ ARROW_TESTING_EXPORT
144
+ Status MakeDictionary(std::shared_ptr<RecordBatch>* out);
145
+
146
+ ARROW_TESTING_EXPORT
147
+ Status MakeDictionaryFlat(std::shared_ptr<RecordBatch>* out);
148
+
149
+ ARROW_TESTING_EXPORT
150
+ Status MakeNestedDictionary(std::shared_ptr<RecordBatch>* out);
151
+
152
+ ARROW_TESTING_EXPORT
153
+ Status MakeMap(std::shared_ptr<RecordBatch>* out);
154
+
155
+ ARROW_TESTING_EXPORT
156
+ Status MakeMapOfDictionary(std::shared_ptr<RecordBatch>* out);
157
+
158
+ ARROW_TESTING_EXPORT
159
+ Status MakeDates(std::shared_ptr<RecordBatch>* out);
160
+
161
+ ARROW_TESTING_EXPORT
162
+ Status MakeTimestamps(std::shared_ptr<RecordBatch>* out);
163
+
164
+ ARROW_TESTING_EXPORT
165
+ Status MakeIntervals(std::shared_ptr<RecordBatch>* out);
166
+
167
+ ARROW_TESTING_EXPORT
168
+ Status MakeTimes(std::shared_ptr<RecordBatch>* out);
169
+
170
+ ARROW_TESTING_EXPORT
171
+ Status MakeFWBinary(std::shared_ptr<RecordBatch>* out);
172
+
173
+ ARROW_TESTING_EXPORT
174
+ Status MakeDecimal(std::shared_ptr<RecordBatch>* out);
175
+
176
+ ARROW_TESTING_EXPORT
177
+ Status MakeNull(std::shared_ptr<RecordBatch>* out);
178
+
179
+ ARROW_TESTING_EXPORT
180
+ Status MakeUuid(std::shared_ptr<RecordBatch>* out);
181
+
182
+ ARROW_TESTING_EXPORT
183
+ Status MakeComplex128(std::shared_ptr<RecordBatch>* out);
184
+
185
+ ARROW_TESTING_EXPORT
186
+ Status MakeDictExtension(std::shared_ptr<RecordBatch>* out);
187
+
188
+ ARROW_TESTING_EXPORT
189
+ Status MakeRandomTensor(const std::shared_ptr<DataType>& type,
190
+ const std::vector<int64_t>& shape, bool row_major_p,
191
+ std::shared_ptr<Tensor>* out, uint32_t seed = 0);
192
+
193
+ ARROW_TESTING_EXPORT Status RoundtripBatch(const std::shared_ptr<RecordBatch>& batch,
194
+ std::shared_ptr<RecordBatch>* out);
195
+
196
+ } // namespace test
197
+ } // namespace ipc
198
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/type_fwd.h ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ namespace arrow {
21
+ namespace ipc {
22
+
23
+ enum class MetadataVersion : char {
24
+ /// 0.1.0
25
+ V1,
26
+
27
+ /// 0.2.0
28
+ V2,
29
+
30
+ /// 0.3.0 to 0.7.1
31
+ V3,
32
+
33
+ /// 0.8.0 to 0.17.0
34
+ V4,
35
+
36
+ /// >= 1.0.0
37
+ V5
38
+ };
39
+
40
+ class Message;
41
+ enum class MessageType {
42
+ NONE,
43
+ SCHEMA,
44
+ DICTIONARY_BATCH,
45
+ RECORD_BATCH,
46
+ TENSOR,
47
+ SPARSE_TENSOR
48
+ };
49
+
50
+ struct IpcReadOptions;
51
+ struct IpcWriteOptions;
52
+
53
+ class MessageReader;
54
+
55
+ class RecordBatchStreamReader;
56
+ class RecordBatchFileReader;
57
+ class RecordBatchWriter;
58
+
59
+ class DictionaryFieldMapper;
60
+ class DictionaryMemo;
61
+
62
+ namespace feather {
63
+
64
+ class Reader;
65
+
66
+ } // namespace feather
67
+ } // namespace ipc
68
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/util.h ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <cstdint>
21
+
22
+ namespace arrow {
23
+ namespace ipc {
24
+
25
+ // Buffers are padded to 64-byte boundaries (for SIMD)
26
+ static constexpr int32_t kArrowAlignment = 64;
27
+
28
+ // Tensors are padded to 64-byte boundaries
29
+ static constexpr int32_t kTensorAlignment = 64;
30
+
31
+ // Align on 8-byte boundaries in IPC
32
+ static constexpr int32_t kArrowIpcAlignment = 8;
33
+
34
+ static constexpr uint8_t kPaddingBytes[kArrowAlignment] = {0};
35
+
36
+ static inline int64_t PaddedLength(int64_t nbytes, int32_t alignment = kArrowAlignment) {
37
+ return ((nbytes + alignment - 1) / alignment) * alignment;
38
+ }
39
+
40
+ } // namespace ipc
41
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/writer.h ADDED
@@ -0,0 +1,475 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ // Implement Arrow streaming binary format
19
+
20
+ #pragma once
21
+
22
+ #include <cstdint>
23
+ #include <memory>
24
+ #include <vector>
25
+
26
+ #include "arrow/ipc/dictionary.h" // IWYU pragma: export
27
+ #include "arrow/ipc/message.h"
28
+ #include "arrow/ipc/options.h"
29
+ #include "arrow/result.h"
30
+ #include "arrow/util/macros.h"
31
+ #include "arrow/util/visibility.h"
32
+
33
+ namespace arrow {
34
+
35
+ class Array;
36
+ class Buffer;
37
+ class MemoryManager;
38
+ class MemoryPool;
39
+ class RecordBatch;
40
+ class Schema;
41
+ class Status;
42
+ class Table;
43
+ class Tensor;
44
+ class SparseTensor;
45
+
46
+ namespace io {
47
+
48
+ class OutputStream;
49
+
50
+ } // namespace io
51
+
52
+ namespace ipc {
53
+
54
+ /// \brief Intermediate data structure with metadata header, and zero
55
+ /// or more buffers for the message body.
56
+ struct IpcPayload {
57
+ MessageType type = MessageType::NONE;
58
+ std::shared_ptr<Buffer> metadata;
59
+ std::vector<std::shared_ptr<Buffer>> body_buffers;
60
+ std::vector<int64_t> variadic_buffer_counts;
61
+ int64_t body_length = 0; // serialized body length (padded, maybe compressed)
62
+ int64_t raw_body_length = 0; // initial uncompressed body length
63
+ };
64
+
65
+ struct WriteStats {
66
+ /// Number of IPC messages written.
67
+ int64_t num_messages = 0;
68
+ /// Number of record batches written.
69
+ int64_t num_record_batches = 0;
70
+ /// Number of dictionary batches written.
71
+ ///
72
+ /// Note: num_dictionary_batches >= num_dictionary_deltas + num_replaced_dictionaries
73
+ int64_t num_dictionary_batches = 0;
74
+
75
+ /// Number of dictionary deltas written.
76
+ int64_t num_dictionary_deltas = 0;
77
+ /// Number of replaced dictionaries (i.e. where a dictionary batch replaces
78
+ /// an existing dictionary with an unrelated new dictionary).
79
+ int64_t num_replaced_dictionaries = 0;
80
+
81
+ /// Total size in bytes of record batches emitted.
82
+ /// The "raw" size counts the original buffer sizes, while the "serialized" size
83
+ /// includes padding and (optionally) compression.
84
+ int64_t total_raw_body_size = 0;
85
+ int64_t total_serialized_body_size = 0;
86
+ };
87
+
88
+ /// \class RecordBatchWriter
89
+ /// \brief Abstract interface for writing a stream of record batches
90
+ class ARROW_EXPORT RecordBatchWriter {
91
+ public:
92
+ virtual ~RecordBatchWriter();
93
+
94
+ /// \brief Write a record batch to the stream
95
+ ///
96
+ /// \param[in] batch the record batch to write to the stream
97
+ /// \return Status
98
+ virtual Status WriteRecordBatch(const RecordBatch& batch) = 0;
99
+
100
+ /// \brief Write a record batch with custom metadata to the stream
101
+ ///
102
+ /// \param[in] batch the record batch to write to the stream
103
+ /// \param[in] custom_metadata the record batch's custom metadata to write to the stream
104
+ /// \return Status
105
+ virtual Status WriteRecordBatch(
106
+ const RecordBatch& batch,
107
+ const std::shared_ptr<const KeyValueMetadata>& custom_metadata);
108
+
109
+ /// \brief Write possibly-chunked table by creating sequence of record batches
110
+ /// \param[in] table table to write
111
+ /// \return Status
112
+ Status WriteTable(const Table& table);
113
+
114
+ /// \brief Write Table with a particular chunksize
115
+ /// \param[in] table table to write
116
+ /// \param[in] max_chunksize maximum number of rows for table chunks. To
117
+ /// indicate that no maximum should be enforced, pass -1.
118
+ /// \return Status
119
+ virtual Status WriteTable(const Table& table, int64_t max_chunksize);
120
+
121
+ /// \brief Perform any logic necessary to finish the stream
122
+ ///
123
+ /// \return Status
124
+ virtual Status Close() = 0;
125
+
126
+ /// \brief Return current write statistics
127
+ virtual WriteStats stats() const = 0;
128
+ };
129
+
130
+ /// \defgroup record-batch-writer-factories Functions for creating RecordBatchWriter
131
+ /// instances
132
+ ///
133
+ /// @{
134
+
135
+ /// Create a new IPC stream writer from stream sink and schema. User is
136
+ /// responsible for closing the actual OutputStream.
137
+ ///
138
+ /// \param[in] sink output stream to write to
139
+ /// \param[in] schema the schema of the record batches to be written
140
+ /// \param[in] options options for serialization
141
+ /// \return Result<std::shared_ptr<RecordBatchWriter>>
142
+ ARROW_EXPORT
143
+ Result<std::shared_ptr<RecordBatchWriter>> MakeStreamWriter(
144
+ io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
145
+ const IpcWriteOptions& options = IpcWriteOptions::Defaults());
146
+
147
+ /// Create a new IPC stream writer from stream sink and schema. User is
148
+ /// responsible for closing the actual OutputStream.
149
+ ///
150
+ /// \param[in] sink output stream to write to
151
+ /// \param[in] schema the schema of the record batches to be written
152
+ /// \param[in] options options for serialization
153
+ /// \return Result<std::shared_ptr<RecordBatchWriter>>
154
+ ARROW_EXPORT
155
+ Result<std::shared_ptr<RecordBatchWriter>> MakeStreamWriter(
156
+ std::shared_ptr<io::OutputStream> sink, const std::shared_ptr<Schema>& schema,
157
+ const IpcWriteOptions& options = IpcWriteOptions::Defaults());
158
+
159
+ /// Create a new IPC file writer from stream sink and schema
160
+ ///
161
+ /// \param[in] sink output stream to write to
162
+ /// \param[in] schema the schema of the record batches to be written
163
+ /// \param[in] options options for serialization, optional
164
+ /// \param[in] metadata custom metadata for File Footer, optional
165
+ /// \return Result<std::shared_ptr<RecordBatchWriter>>
166
+ ARROW_EXPORT
167
+ Result<std::shared_ptr<RecordBatchWriter>> MakeFileWriter(
168
+ io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
169
+ const IpcWriteOptions& options = IpcWriteOptions::Defaults(),
170
+ const std::shared_ptr<const KeyValueMetadata>& metadata = NULLPTR);
171
+
172
+ /// Create a new IPC file writer from stream sink and schema
173
+ ///
174
+ /// \param[in] sink output stream to write to
175
+ /// \param[in] schema the schema of the record batches to be written
176
+ /// \param[in] options options for serialization, optional
177
+ /// \param[in] metadata custom metadata for File Footer, optional
178
+ /// \return Result<std::shared_ptr<RecordBatchWriter>>
179
+ ARROW_EXPORT
180
+ Result<std::shared_ptr<RecordBatchWriter>> MakeFileWriter(
181
+ std::shared_ptr<io::OutputStream> sink, const std::shared_ptr<Schema>& schema,
182
+ const IpcWriteOptions& options = IpcWriteOptions::Defaults(),
183
+ const std::shared_ptr<const KeyValueMetadata>& metadata = NULLPTR);
184
+
185
+ /// @}
186
+
187
+ /// \brief Low-level API for writing a record batch (without schema)
188
+ /// to an OutputStream as encapsulated IPC message. See Arrow format
189
+ /// documentation for more detail.
190
+ ///
191
+ /// \param[in] batch the record batch to write
192
+ /// \param[in] buffer_start_offset the start offset to use in the buffer metadata,
193
+ /// generally should be 0
194
+ /// \param[in] dst an OutputStream
195
+ /// \param[out] metadata_length the size of the length-prefixed flatbuffer
196
+ /// including padding to a 64-byte boundary
197
+ /// \param[out] body_length the size of the contiguous buffer block plus
198
+ /// \param[in] options options for serialization
199
+ /// \return Status
200
+ ARROW_EXPORT
201
+ Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
202
+ io::OutputStream* dst, int32_t* metadata_length,
203
+ int64_t* body_length, const IpcWriteOptions& options);
204
+
205
+ /// \brief Serialize record batch as encapsulated IPC message in a new buffer
206
+ ///
207
+ /// \param[in] batch the record batch
208
+ /// \param[in] options the IpcWriteOptions to use for serialization
209
+ /// \return the serialized message
210
+ ARROW_EXPORT
211
+ Result<std::shared_ptr<Buffer>> SerializeRecordBatch(const RecordBatch& batch,
212
+ const IpcWriteOptions& options);
213
+
214
+ /// \brief Serialize record batch as encapsulated IPC message in a new buffer
215
+ ///
216
+ /// \param[in] batch the record batch
217
+ /// \param[in] mm a MemoryManager to allocate memory from
218
+ /// \return the serialized message
219
+ ARROW_EXPORT
220
+ Result<std::shared_ptr<Buffer>> SerializeRecordBatch(const RecordBatch& batch,
221
+ std::shared_ptr<MemoryManager> mm);
222
+
223
+ /// \brief Write record batch to OutputStream
224
+ ///
225
+ /// \param[in] batch the record batch to write
226
+ /// \param[in] options the IpcWriteOptions to use for serialization
227
+ /// \param[in] out the OutputStream to write the output to
228
+ /// \return Status
229
+ ///
230
+ /// If writing to pre-allocated memory, you can use
231
+ /// arrow::ipc::GetRecordBatchSize to compute how much space is required
232
+ ARROW_EXPORT
233
+ Status SerializeRecordBatch(const RecordBatch& batch, const IpcWriteOptions& options,
234
+ io::OutputStream* out);
235
+
236
+ /// \brief Serialize schema as encapsulated IPC message
237
+ ///
238
+ /// \param[in] schema the schema to write
239
+ /// \param[in] pool a MemoryPool to allocate memory from
240
+ /// \return the serialized schema
241
+ ARROW_EXPORT
242
+ Result<std::shared_ptr<Buffer>> SerializeSchema(const Schema& schema,
243
+ MemoryPool* pool = default_memory_pool());
244
+
245
+ /// \brief Write multiple record batches to OutputStream, including schema
246
+ /// \param[in] batches a vector of batches. Must all have same schema
247
+ /// \param[in] options options for serialization
248
+ /// \param[out] dst an OutputStream
249
+ /// \return Status
250
+ ARROW_EXPORT
251
+ Status WriteRecordBatchStream(const std::vector<std::shared_ptr<RecordBatch>>& batches,
252
+ const IpcWriteOptions& options, io::OutputStream* dst);
253
+
254
+ /// \brief Compute the number of bytes needed to write an IPC payload
255
+ /// including metadata
256
+ ///
257
+ /// \param[in] payload the IPC payload to write
258
+ /// \param[in] options write options
259
+ /// \return the size of the complete encapsulated message
260
+ ARROW_EXPORT
261
+ int64_t GetPayloadSize(const IpcPayload& payload,
262
+ const IpcWriteOptions& options = IpcWriteOptions::Defaults());
263
+
264
+ /// \brief Compute the number of bytes needed to write a record batch including metadata
265
+ ///
266
+ /// \param[in] batch the record batch to write
267
+ /// \param[out] size the size of the complete encapsulated message
268
+ /// \return Status
269
+ ARROW_EXPORT
270
+ Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size);
271
+
272
+ /// \brief Compute the number of bytes needed to write a record batch including metadata
273
+ ///
274
+ /// \param[in] batch the record batch to write
275
+ /// \param[in] options options for serialization
276
+ /// \param[out] size the size of the complete encapsulated message
277
+ /// \return Status
278
+ ARROW_EXPORT
279
+ Status GetRecordBatchSize(const RecordBatch& batch, const IpcWriteOptions& options,
280
+ int64_t* size);
281
+
282
+ /// \brief Compute the number of bytes needed to write a tensor including metadata
283
+ ///
284
+ /// \param[in] tensor the tensor to write
285
+ /// \param[out] size the size of the complete encapsulated message
286
+ /// \return Status
287
+ ARROW_EXPORT
288
+ Status GetTensorSize(const Tensor& tensor, int64_t* size);
289
+
290
+ /// \brief EXPERIMENTAL: Convert arrow::Tensor to a Message with minimal memory
291
+ /// allocation
292
+ ///
293
+ /// \param[in] tensor the Tensor to write
294
+ /// \param[in] pool MemoryPool to allocate space for metadata
295
+ /// \return the resulting Message
296
+ ARROW_EXPORT
297
+ Result<std::unique_ptr<Message>> GetTensorMessage(const Tensor& tensor, MemoryPool* pool);
298
+
299
+ /// \brief Write arrow::Tensor as a contiguous message.
300
+ ///
301
+ /// The metadata and body are written assuming 64-byte alignment. It is the
302
+ /// user's responsibility to ensure that the OutputStream has been aligned
303
+ /// to a 64-byte multiple before writing the message.
304
+ ///
305
+ /// The message is written out as followed:
306
+ /// \code
307
+ /// <metadata size> <metadata> <tensor data>
308
+ /// \endcode
309
+ ///
310
+ /// \param[in] tensor the Tensor to write
311
+ /// \param[in] dst the OutputStream to write to
312
+ /// \param[out] metadata_length the actual metadata length, including padding
313
+ /// \param[out] body_length the actual message body length
314
+ /// \return Status
315
+ ARROW_EXPORT
316
+ Status WriteTensor(const Tensor& tensor, io::OutputStream* dst, int32_t* metadata_length,
317
+ int64_t* body_length);
318
+
319
+ /// \brief EXPERIMENTAL: Convert arrow::SparseTensor to a Message with minimal memory
320
+ /// allocation
321
+ ///
322
+ /// The message is written out as followed:
323
+ /// \code
324
+ /// <metadata size> <metadata> <sparse index> <sparse tensor body>
325
+ /// \endcode
326
+ ///
327
+ /// \param[in] sparse_tensor the SparseTensor to write
328
+ /// \param[in] pool MemoryPool to allocate space for metadata
329
+ /// \return the resulting Message
330
+ ARROW_EXPORT
331
+ Result<std::unique_ptr<Message>> GetSparseTensorMessage(const SparseTensor& sparse_tensor,
332
+ MemoryPool* pool);
333
+
334
+ /// \brief EXPERIMENTAL: Write arrow::SparseTensor as a contiguous message. The metadata,
335
+ /// sparse index, and body are written assuming 64-byte alignment. It is the
336
+ /// user's responsibility to ensure that the OutputStream has been aligned
337
+ /// to a 64-byte multiple before writing the message.
338
+ ///
339
+ /// \param[in] sparse_tensor the SparseTensor to write
340
+ /// \param[in] dst the OutputStream to write to
341
+ /// \param[out] metadata_length the actual metadata length, including padding
342
+ /// \param[out] body_length the actual message body length
343
+ /// \return Status
344
+ ARROW_EXPORT
345
+ Status WriteSparseTensor(const SparseTensor& sparse_tensor, io::OutputStream* dst,
346
+ int32_t* metadata_length, int64_t* body_length);
347
+
348
+ /// \brief Compute IpcPayload for the given schema
349
+ /// \param[in] schema the Schema that is being serialized
350
+ /// \param[in] options options for serialization
351
+ /// \param[in] mapper object mapping dictionary fields to dictionary ids
352
+ /// \param[out] out the returned vector of IpcPayloads
353
+ /// \return Status
354
+ ARROW_EXPORT
355
+ Status GetSchemaPayload(const Schema& schema, const IpcWriteOptions& options,
356
+ const DictionaryFieldMapper& mapper, IpcPayload* out);
357
+
358
+ /// \brief Compute IpcPayload for a dictionary
359
+ /// \param[in] id the dictionary id
360
+ /// \param[in] dictionary the dictionary values
361
+ /// \param[in] options options for serialization
362
+ /// \param[out] payload the output IpcPayload
363
+ /// \return Status
364
+ ARROW_EXPORT
365
+ Status GetDictionaryPayload(int64_t id, const std::shared_ptr<Array>& dictionary,
366
+ const IpcWriteOptions& options, IpcPayload* payload);
367
+
368
+ /// \brief Compute IpcPayload for a dictionary
369
+ /// \param[in] id the dictionary id
370
+ /// \param[in] is_delta whether the dictionary is a delta dictionary
371
+ /// \param[in] dictionary the dictionary values
372
+ /// \param[in] options options for serialization
373
+ /// \param[out] payload the output IpcPayload
374
+ /// \return Status
375
+ ARROW_EXPORT
376
+ Status GetDictionaryPayload(int64_t id, bool is_delta,
377
+ const std::shared_ptr<Array>& dictionary,
378
+ const IpcWriteOptions& options, IpcPayload* payload);
379
+
380
+ /// \brief Compute IpcPayload for the given record batch
381
+ /// \param[in] batch the RecordBatch that is being serialized
382
+ /// \param[in] options options for serialization
383
+ /// \param[out] out the returned IpcPayload
384
+ /// \return Status
385
+ ARROW_EXPORT
386
+ Status GetRecordBatchPayload(const RecordBatch& batch, const IpcWriteOptions& options,
387
+ IpcPayload* out);
388
+
389
+ /// \brief Compute IpcPayload for the given record batch and custom metadata
390
+ /// \param[in] batch the RecordBatch that is being serialized
391
+ /// \param[in] custom_metadata the custom metadata to be serialized with the record batch
392
+ /// \param[in] options options for serialization
393
+ /// \param[out] out the returned IpcPayload
394
+ /// \return Status
395
+ ARROW_EXPORT
396
+ Status GetRecordBatchPayload(
397
+ const RecordBatch& batch,
398
+ const std::shared_ptr<const KeyValueMetadata>& custom_metadata,
399
+ const IpcWriteOptions& options, IpcPayload* out);
400
+
401
+ /// \brief Write an IPC payload to the given stream.
402
+ /// \param[in] payload the payload to write
403
+ /// \param[in] options options for serialization
404
+ /// \param[in] dst The stream to write the payload to.
405
+ /// \param[out] metadata_length the length of the serialized metadata
406
+ /// \return Status
407
+ ARROW_EXPORT
408
+ Status WriteIpcPayload(const IpcPayload& payload, const IpcWriteOptions& options,
409
+ io::OutputStream* dst, int32_t* metadata_length);
410
+
411
+ /// \brief Compute IpcPayload for the given sparse tensor
412
+ /// \param[in] sparse_tensor the SparseTensor that is being serialized
413
+ /// \param[in,out] pool for any required temporary memory allocations
414
+ /// \param[out] out the returned IpcPayload
415
+ /// \return Status
416
+ ARROW_EXPORT
417
+ Status GetSparseTensorPayload(const SparseTensor& sparse_tensor, MemoryPool* pool,
418
+ IpcPayload* out);
419
+
420
+ namespace internal {
421
+
422
+ // These internal APIs may change without warning or deprecation
423
+
424
+ class ARROW_EXPORT IpcPayloadWriter {
425
+ public:
426
+ virtual ~IpcPayloadWriter();
427
+
428
+ // Default implementation is a no-op
429
+ virtual Status Start();
430
+
431
+ virtual Status WritePayload(const IpcPayload& payload) = 0;
432
+
433
+ virtual Status Close() = 0;
434
+ };
435
+
436
+ /// Create a new IPC payload stream writer from stream sink. User is
437
+ /// responsible for closing the actual OutputStream.
438
+ ///
439
+ /// \param[in] sink output stream to write to
440
+ /// \param[in] options options for serialization
441
+ /// \return Result<std::shared_ptr<IpcPayloadWriter>>
442
+ ARROW_EXPORT
443
+ Result<std::unique_ptr<IpcPayloadWriter>> MakePayloadStreamWriter(
444
+ io::OutputStream* sink, const IpcWriteOptions& options = IpcWriteOptions::Defaults());
445
+
446
+ /// Create a new IPC payload file writer from stream sink.
447
+ ///
448
+ /// \param[in] sink output stream to write to
449
+ /// \param[in] schema the schema of the record batches to be written
450
+ /// \param[in] options options for serialization, optional
451
+ /// \param[in] metadata custom metadata for File Footer, optional
452
+ /// \return Status
453
+ ARROW_EXPORT
454
+ Result<std::unique_ptr<IpcPayloadWriter>> MakePayloadFileWriter(
455
+ io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
456
+ const IpcWriteOptions& options = IpcWriteOptions::Defaults(),
457
+ const std::shared_ptr<const KeyValueMetadata>& metadata = NULLPTR);
458
+
459
+ /// Create a new RecordBatchWriter from IpcPayloadWriter and schema.
460
+ ///
461
+ /// The format is implicitly the IPC stream format (allowing dictionary
462
+ /// replacement and deltas).
463
+ ///
464
+ /// \param[in] sink the IpcPayloadWriter to write to
465
+ /// \param[in] schema the schema of the record batches to be written
466
+ /// \param[in] options options for serialization
467
+ /// \return Result<std::unique_ptr<RecordBatchWriter>>
468
+ ARROW_EXPORT
469
+ Result<std::unique_ptr<RecordBatchWriter>> OpenRecordBatchWriter(
470
+ std::unique_ptr<IpcPayloadWriter> sink, const std::shared_ptr<Schema>& schema,
471
+ const IpcWriteOptions& options = IpcWriteOptions::Defaults());
472
+
473
+ } // namespace internal
474
+ } // namespace ipc
475
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/api.h ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include "arrow/json/options.h"
21
+ #include "arrow/json/reader.h"
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/chunked_builder.h ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <memory>
21
+ #include <vector>
22
+
23
+ #include "arrow/status.h"
24
+ #include "arrow/type_fwd.h"
25
+ #include "arrow/util/type_fwd.h"
26
+ #include "arrow/util/visibility.h"
27
+
28
+ namespace arrow {
29
+ namespace json {
30
+
31
+ class PromotionGraph;
32
+
33
+ class ARROW_EXPORT ChunkedArrayBuilder {
34
+ public:
35
+ virtual ~ChunkedArrayBuilder() = default;
36
+
37
+ /// Spawn a task that will try to convert and insert the given JSON block
38
+ virtual void Insert(int64_t block_index,
39
+ const std::shared_ptr<Field>& unconverted_field,
40
+ const std::shared_ptr<Array>& unconverted) = 0;
41
+
42
+ /// Return the final chunked array.
43
+ /// Every chunk must be inserted before this is called!
44
+ virtual Status Finish(std::shared_ptr<ChunkedArray>* out) = 0;
45
+
46
+ /// Finish current task group and substitute a new one
47
+ virtual Status ReplaceTaskGroup(
48
+ const std::shared_ptr<arrow::internal::TaskGroup>& task_group) = 0;
49
+
50
+ protected:
51
+ explicit ChunkedArrayBuilder(
52
+ const std::shared_ptr<arrow::internal::TaskGroup>& task_group)
53
+ : task_group_(task_group) {}
54
+
55
+ std::shared_ptr<arrow::internal::TaskGroup> task_group_;
56
+ };
57
+
58
+ /// create a chunked builder
59
+ ///
60
+ /// if unexpected fields and promotion need to be handled, promotion_graph must be
61
+ /// non-null
62
+ ARROW_EXPORT Status MakeChunkedArrayBuilder(
63
+ const std::shared_ptr<arrow::internal::TaskGroup>& task_group, MemoryPool* pool,
64
+ const PromotionGraph* promotion_graph, const std::shared_ptr<DataType>& type,
65
+ std::shared_ptr<ChunkedArrayBuilder>* out);
66
+
67
+ } // namespace json
68
+ } // namespace arrow
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/chunker.h ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ #pragma once
19
+
20
+ #include <memory>
21
+
22
+ #include "arrow/util/delimiting.h"
23
+ #include "arrow/util/macros.h"
24
+ #include "arrow/util/visibility.h"
25
+
26
+ namespace arrow {
27
+ namespace json {
28
+
29
+ struct ParseOptions;
30
+
31
+ ARROW_EXPORT
32
+ std::unique_ptr<Chunker> MakeChunker(const ParseOptions& options);
33
+
34
+ } // namespace json
35
+ } // namespace arrow