Add loan-advisor-agent (part 2)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client.h +434 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_auth.h +62 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_cookie_middleware.h +33 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_middleware.h +78 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_tracing_middleware.h +34 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/middleware.h +75 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/otel_logging.h +33 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/platform.h +31 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server.h +327 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server_auth.h +107 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server_middleware.h +82 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server_tracing_middleware.h +68 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_auth_handlers.h +89 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_definitions.h +318 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_flight_server.h +92 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_util.h +188 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/transport.h +298 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/transport_server.h +133 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/type_fwd.h +65 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/types.h +1314 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/types_async.h +76 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/visibility.h +48 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/api.h +25 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/buffered.h +168 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/caching.h +157 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/compressed.h +124 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/concurrency.h +263 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/file.h +221 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/hdfs.h +284 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/interfaces.h +362 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/memory.h +197 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/mman.h +169 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/slow.h +118 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/stdio.h +82 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/test_common.h +69 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/transform.h +60 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/type_fwd.h +77 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/api.h +24 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/dictionary.h +177 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/feather.h +150 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/message.h +585 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/options.h +201 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/reader.h +638 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/test_common.h +198 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/type_fwd.h +68 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/util.h +41 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/writer.h +475 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/api.h +21 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/chunked_builder.h +68 -0
- loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/chunker.h +35 -0
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client.h
ADDED
|
@@ -0,0 +1,434 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
/// \brief Implementation of Flight RPC client.
|
| 19 |
+
|
| 20 |
+
#pragma once
|
| 21 |
+
|
| 22 |
+
#include <chrono>
|
| 23 |
+
#include <memory>
|
| 24 |
+
#include <string>
|
| 25 |
+
#include <utility>
|
| 26 |
+
#include <variant>
|
| 27 |
+
#include <vector>
|
| 28 |
+
|
| 29 |
+
#include "arrow/ipc/options.h"
|
| 30 |
+
#include "arrow/ipc/reader.h"
|
| 31 |
+
#include "arrow/ipc/writer.h"
|
| 32 |
+
#include "arrow/result.h"
|
| 33 |
+
#include "arrow/status.h"
|
| 34 |
+
#include "arrow/util/cancel.h"
|
| 35 |
+
|
| 36 |
+
#include "arrow/flight/type_fwd.h"
|
| 37 |
+
#include "arrow/flight/types.h" // IWYU pragma: keep
|
| 38 |
+
#include "arrow/flight/visibility.h"
|
| 39 |
+
|
| 40 |
+
namespace arrow {
|
| 41 |
+
|
| 42 |
+
class RecordBatch;
|
| 43 |
+
class Schema;
|
| 44 |
+
|
| 45 |
+
namespace flight {
|
| 46 |
+
|
| 47 |
+
/// \brief A duration type for Flight call timeouts.
|
| 48 |
+
typedef std::chrono::duration<double, std::chrono::seconds::period> TimeoutDuration;
|
| 49 |
+
|
| 50 |
+
/// \brief Hints to the underlying RPC layer for Arrow Flight calls.
|
| 51 |
+
class ARROW_FLIGHT_EXPORT FlightCallOptions {
|
| 52 |
+
public:
|
| 53 |
+
/// Create a default set of call options.
|
| 54 |
+
FlightCallOptions();
|
| 55 |
+
|
| 56 |
+
/// \brief An optional timeout for this call. Negative durations
|
| 57 |
+
/// mean an implementation-defined default behavior will be used
|
| 58 |
+
/// instead. This is the default value.
|
| 59 |
+
TimeoutDuration timeout;
|
| 60 |
+
|
| 61 |
+
/// \brief IPC reader options, if applicable for the call.
|
| 62 |
+
ipc::IpcReadOptions read_options;
|
| 63 |
+
|
| 64 |
+
/// \brief IPC writer options, if applicable for the call.
|
| 65 |
+
ipc::IpcWriteOptions write_options;
|
| 66 |
+
|
| 67 |
+
/// \brief Headers for client to add to context.
|
| 68 |
+
std::vector<std::pair<std::string, std::string>> headers;
|
| 69 |
+
|
| 70 |
+
/// \brief A token to enable interactive user cancellation of long-running requests.
|
| 71 |
+
StopToken stop_token;
|
| 72 |
+
|
| 73 |
+
/// \brief An optional memory manager to control where to allocate incoming data.
|
| 74 |
+
std::shared_ptr<MemoryManager> memory_manager;
|
| 75 |
+
};
|
| 76 |
+
|
| 77 |
+
/// \brief Indicate that the client attempted to write a message
|
| 78 |
+
/// larger than the soft limit set via write_size_limit_bytes.
|
| 79 |
+
class ARROW_FLIGHT_EXPORT FlightWriteSizeStatusDetail : public arrow::StatusDetail {
|
| 80 |
+
public:
|
| 81 |
+
explicit FlightWriteSizeStatusDetail(int64_t limit, int64_t actual)
|
| 82 |
+
: limit_(limit), actual_(actual) {}
|
| 83 |
+
const char* type_id() const override;
|
| 84 |
+
std::string ToString() const override;
|
| 85 |
+
int64_t limit() const { return limit_; }
|
| 86 |
+
int64_t actual() const { return actual_; }
|
| 87 |
+
|
| 88 |
+
/// \brief Extract this status detail from a status, or return
|
| 89 |
+
/// nullptr if the status doesn't contain this status detail.
|
| 90 |
+
static std::shared_ptr<FlightWriteSizeStatusDetail> UnwrapStatus(
|
| 91 |
+
const arrow::Status& status);
|
| 92 |
+
|
| 93 |
+
private:
|
| 94 |
+
int64_t limit_;
|
| 95 |
+
int64_t actual_;
|
| 96 |
+
};
|
| 97 |
+
|
| 98 |
+
struct ARROW_FLIGHT_EXPORT FlightClientOptions {
|
| 99 |
+
/// \brief Root certificates to use for validating server
|
| 100 |
+
/// certificates.
|
| 101 |
+
std::string tls_root_certs;
|
| 102 |
+
/// \brief Override the hostname checked by TLS. Use with caution.
|
| 103 |
+
std::string override_hostname;
|
| 104 |
+
/// \brief The client certificate to use if using Mutual TLS
|
| 105 |
+
std::string cert_chain;
|
| 106 |
+
/// \brief The private key associated with the client certificate for Mutual TLS
|
| 107 |
+
std::string private_key;
|
| 108 |
+
/// \brief A list of client middleware to apply.
|
| 109 |
+
std::vector<std::shared_ptr<ClientMiddlewareFactory>> middleware;
|
| 110 |
+
/// \brief A soft limit on the number of bytes to write in a single
|
| 111 |
+
/// batch when sending Arrow data to a server.
|
| 112 |
+
///
|
| 113 |
+
/// Used to help limit server memory consumption. Only enabled if
|
| 114 |
+
/// positive. When enabled, FlightStreamWriter.Write* may yield a
|
| 115 |
+
/// IOError with error detail FlightWriteSizeStatusDetail.
|
| 116 |
+
int64_t write_size_limit_bytes = 0;
|
| 117 |
+
|
| 118 |
+
/// \brief Generic connection options, passed to the underlying
|
| 119 |
+
/// transport; interpretation is implementation-dependent.
|
| 120 |
+
std::vector<std::pair<std::string, std::variant<int, std::string>>> generic_options;
|
| 121 |
+
|
| 122 |
+
/// \brief Use TLS without validating the server certificate. Use with caution.
|
| 123 |
+
bool disable_server_verification = false;
|
| 124 |
+
|
| 125 |
+
/// \brief Get default options.
|
| 126 |
+
static FlightClientOptions Defaults();
|
| 127 |
+
};
|
| 128 |
+
|
| 129 |
+
/// \brief A RecordBatchReader exposing Flight metadata and cancel
|
| 130 |
+
/// operations.
|
| 131 |
+
class ARROW_FLIGHT_EXPORT FlightStreamReader : public MetadataRecordBatchReader {
|
| 132 |
+
public:
|
| 133 |
+
/// \brief Try to cancel the call.
|
| 134 |
+
virtual void Cancel() = 0;
|
| 135 |
+
|
| 136 |
+
using MetadataRecordBatchReader::ToRecordBatches;
|
| 137 |
+
/// \brief Consume entire stream as a vector of record batches
|
| 138 |
+
virtual arrow::Result<std::vector<std::shared_ptr<RecordBatch>>> ToRecordBatches(
|
| 139 |
+
const StopToken& stop_token) = 0;
|
| 140 |
+
|
| 141 |
+
using MetadataRecordBatchReader::ToTable;
|
| 142 |
+
/// \brief Consume entire stream as a Table
|
| 143 |
+
arrow::Result<std::shared_ptr<Table>> ToTable(const StopToken& stop_token);
|
| 144 |
+
|
| 145 |
+
using MetadataRecordBatchReader::stats;
|
| 146 |
+
/// \brief Return current read statistics
|
| 147 |
+
virtual arrow::ipc::ReadStats stats() const = 0;
|
| 148 |
+
};
|
| 149 |
+
|
| 150 |
+
// Silence warning
|
| 151 |
+
// "non dll-interface class RecordBatchReader used as base for dll-interface class"
|
| 152 |
+
#ifdef _MSC_VER
|
| 153 |
+
# pragma warning(push)
|
| 154 |
+
# pragma warning(disable : 4275)
|
| 155 |
+
#endif
|
| 156 |
+
|
| 157 |
+
/// \brief A RecordBatchWriter that also allows sending
|
| 158 |
+
/// application-defined metadata via the Flight protocol.
|
| 159 |
+
class ARROW_FLIGHT_EXPORT FlightStreamWriter : public MetadataRecordBatchWriter {
|
| 160 |
+
public:
|
| 161 |
+
/// \brief Indicate that the application is done writing to this stream.
|
| 162 |
+
///
|
| 163 |
+
/// The application may not write to this stream after calling
|
| 164 |
+
/// this. This differs from closing the stream because this writer
|
| 165 |
+
/// may represent only one half of a readable and writable stream.
|
| 166 |
+
virtual Status DoneWriting() = 0;
|
| 167 |
+
};
|
| 168 |
+
|
| 169 |
+
#ifdef _MSC_VER
|
| 170 |
+
# pragma warning(pop)
|
| 171 |
+
#endif
|
| 172 |
+
|
| 173 |
+
/// \brief A reader for application-specific metadata sent back to the
|
| 174 |
+
/// client during an upload.
|
| 175 |
+
class ARROW_FLIGHT_EXPORT FlightMetadataReader {
|
| 176 |
+
public:
|
| 177 |
+
virtual ~FlightMetadataReader();
|
| 178 |
+
/// \brief Read a message from the server.
|
| 179 |
+
virtual Status ReadMetadata(std::shared_ptr<Buffer>* out) = 0;
|
| 180 |
+
};
|
| 181 |
+
|
| 182 |
+
/// \brief Client class for Arrow Flight RPC services.
|
| 183 |
+
class ARROW_FLIGHT_EXPORT FlightClient {
|
| 184 |
+
public:
|
| 185 |
+
~FlightClient();
|
| 186 |
+
|
| 187 |
+
/// \brief Connect to an unauthenticated flight service
|
| 188 |
+
/// \param[in] location the URI
|
| 189 |
+
/// \return Arrow result with the created FlightClient, OK status may not indicate that
|
| 190 |
+
/// the connection was successful
|
| 191 |
+
static arrow::Result<std::unique_ptr<FlightClient>> Connect(const Location& location);
|
| 192 |
+
|
| 193 |
+
/// \brief Connect to an unauthenticated flight service
|
| 194 |
+
/// \param[in] location the URI
|
| 195 |
+
/// \param[in] options Other options for setting up the client
|
| 196 |
+
/// \return Arrow result with the created FlightClient, OK status may not indicate that
|
| 197 |
+
/// the connection was successful
|
| 198 |
+
static arrow::Result<std::unique_ptr<FlightClient>> Connect(
|
| 199 |
+
const Location& location, const FlightClientOptions& options);
|
| 200 |
+
|
| 201 |
+
/// \brief Authenticate to the server using the given handler.
|
| 202 |
+
/// \param[in] options Per-RPC options
|
| 203 |
+
/// \param[in] auth_handler The authentication mechanism to use
|
| 204 |
+
/// \return Status OK if the client authenticated successfully
|
| 205 |
+
Status Authenticate(const FlightCallOptions& options,
|
| 206 |
+
std::unique_ptr<ClientAuthHandler> auth_handler);
|
| 207 |
+
|
| 208 |
+
/// \brief Authenticate to the server using basic HTTP style authentication.
|
| 209 |
+
/// \param[in] options Per-RPC options
|
| 210 |
+
/// \param[in] username Username to use
|
| 211 |
+
/// \param[in] password Password to use
|
| 212 |
+
/// \return Arrow result with bearer token and status OK if client authenticated
|
| 213 |
+
/// successfully
|
| 214 |
+
arrow::Result<std::pair<std::string, std::string>> AuthenticateBasicToken(
|
| 215 |
+
const FlightCallOptions& options, const std::string& username,
|
| 216 |
+
const std::string& password);
|
| 217 |
+
|
| 218 |
+
/// \brief Perform the indicated action, returning an iterator to the stream
|
| 219 |
+
/// of results, if any
|
| 220 |
+
/// \param[in] options Per-RPC options
|
| 221 |
+
/// \param[in] action the action to be performed
|
| 222 |
+
/// \return Arrow result with an iterator object for reading the returned results
|
| 223 |
+
arrow::Result<std::unique_ptr<ResultStream>> DoAction(const FlightCallOptions& options,
|
| 224 |
+
const Action& action);
|
| 225 |
+
arrow::Result<std::unique_ptr<ResultStream>> DoAction(const Action& action) {
|
| 226 |
+
return DoAction({}, action);
|
| 227 |
+
}
|
| 228 |
+
|
| 229 |
+
/// \brief Perform the CancelFlightInfo action, returning a
|
| 230 |
+
/// CancelFlightInfoResult
|
| 231 |
+
///
|
| 232 |
+
/// \param[in] options Per-RPC options
|
| 233 |
+
/// \param[in] request The CancelFlightInfoRequest
|
| 234 |
+
/// \return Arrow result with a CancelFlightInfoResult
|
| 235 |
+
arrow::Result<CancelFlightInfoResult> CancelFlightInfo(
|
| 236 |
+
const FlightCallOptions& options, const CancelFlightInfoRequest& request);
|
| 237 |
+
arrow::Result<CancelFlightInfoResult> CancelFlightInfo(
|
| 238 |
+
const CancelFlightInfoRequest& request) {
|
| 239 |
+
return CancelFlightInfo({}, request);
|
| 240 |
+
}
|
| 241 |
+
|
| 242 |
+
/// \brief Perform the RenewFlightEndpoint action, returning a renewed
|
| 243 |
+
/// FlightEndpoint
|
| 244 |
+
///
|
| 245 |
+
/// \param[in] options Per-RPC options
|
| 246 |
+
/// \param[in] request The RenewFlightEndpointRequest
|
| 247 |
+
/// \return Arrow result with a renewed FlightEndpoint
|
| 248 |
+
arrow::Result<FlightEndpoint> RenewFlightEndpoint(
|
| 249 |
+
const FlightCallOptions& options, const RenewFlightEndpointRequest& request);
|
| 250 |
+
arrow::Result<FlightEndpoint> RenewFlightEndpoint(
|
| 251 |
+
const RenewFlightEndpointRequest& request) {
|
| 252 |
+
return RenewFlightEndpoint({}, request);
|
| 253 |
+
}
|
| 254 |
+
|
| 255 |
+
/// \brief Retrieve a list of available Action types
|
| 256 |
+
/// \param[in] options Per-RPC options
|
| 257 |
+
/// \return Arrow result with the available actions
|
| 258 |
+
arrow::Result<std::vector<ActionType>> ListActions(const FlightCallOptions& options);
|
| 259 |
+
arrow::Result<std::vector<ActionType>> ListActions() {
|
| 260 |
+
return ListActions(FlightCallOptions());
|
| 261 |
+
}
|
| 262 |
+
|
| 263 |
+
/// \brief Request access plan for a single flight, which may be an existing
|
| 264 |
+
/// dataset or a command to be executed
|
| 265 |
+
/// \param[in] options Per-RPC options
|
| 266 |
+
/// \param[in] descriptor the dataset request, whether a named dataset or
|
| 267 |
+
/// command
|
| 268 |
+
/// \return Arrow result with the FlightInfo describing where to access the dataset
|
| 269 |
+
arrow::Result<std::unique_ptr<FlightInfo>> GetFlightInfo(
|
| 270 |
+
const FlightCallOptions& options, const FlightDescriptor& descriptor);
|
| 271 |
+
arrow::Result<std::unique_ptr<FlightInfo>> GetFlightInfo(
|
| 272 |
+
const FlightDescriptor& descriptor) {
|
| 273 |
+
return GetFlightInfo({}, descriptor);
|
| 274 |
+
}
|
| 275 |
+
|
| 276 |
+
/// \brief Asynchronous GetFlightInfo.
|
| 277 |
+
/// \param[in] options Per-RPC options
|
| 278 |
+
/// \param[in] descriptor the dataset request
|
| 279 |
+
/// \param[in] listener Callbacks for response and RPC completion
|
| 280 |
+
void GetFlightInfoAsync(const FlightCallOptions& options,
|
| 281 |
+
const FlightDescriptor& descriptor,
|
| 282 |
+
std::shared_ptr<AsyncListener<FlightInfo>> listener);
|
| 283 |
+
void GetFlightInfoAsync(const FlightDescriptor& descriptor,
|
| 284 |
+
std::shared_ptr<AsyncListener<FlightInfo>> listener) {
|
| 285 |
+
return GetFlightInfoAsync({}, descriptor, std::move(listener));
|
| 286 |
+
}
|
| 287 |
+
|
| 288 |
+
/// \brief Asynchronous GetFlightInfo returning a Future.
|
| 289 |
+
/// \param[in] options Per-RPC options
|
| 290 |
+
/// \param[in] descriptor the dataset request
|
| 291 |
+
arrow::Future<FlightInfo> GetFlightInfoAsync(const FlightCallOptions& options,
|
| 292 |
+
const FlightDescriptor& descriptor);
|
| 293 |
+
arrow::Future<FlightInfo> GetFlightInfoAsync(const FlightDescriptor& descriptor) {
|
| 294 |
+
return GetFlightInfoAsync({}, descriptor);
|
| 295 |
+
}
|
| 296 |
+
|
| 297 |
+
/// \brief Request and poll a long running query
|
| 298 |
+
/// \param[in] options Per-RPC options
|
| 299 |
+
/// \param[in] descriptor the dataset request or a descriptor returned by a
|
| 300 |
+
/// prior PollFlightInfo call
|
| 301 |
+
/// \return Arrow result with the PollInfo describing the status of
|
| 302 |
+
/// the requested query
|
| 303 |
+
arrow::Result<std::unique_ptr<PollInfo>> PollFlightInfo(
|
| 304 |
+
const FlightCallOptions& options, const FlightDescriptor& descriptor);
|
| 305 |
+
arrow::Result<std::unique_ptr<PollInfo>> PollFlightInfo(
|
| 306 |
+
const FlightDescriptor& descriptor) {
|
| 307 |
+
return PollFlightInfo({}, descriptor);
|
| 308 |
+
}
|
| 309 |
+
|
| 310 |
+
/// \brief Request schema for a single flight, which may be an existing
|
| 311 |
+
/// dataset or a command to be executed
|
| 312 |
+
/// \param[in] options Per-RPC options
|
| 313 |
+
/// \param[in] descriptor the dataset request, whether a named dataset or
|
| 314 |
+
/// command
|
| 315 |
+
/// \return Arrow result with the SchemaResult describing the dataset schema
|
| 316 |
+
arrow::Result<std::unique_ptr<SchemaResult>> GetSchema(
|
| 317 |
+
const FlightCallOptions& options, const FlightDescriptor& descriptor);
|
| 318 |
+
|
| 319 |
+
arrow::Result<std::unique_ptr<SchemaResult>> GetSchema(
|
| 320 |
+
const FlightDescriptor& descriptor) {
|
| 321 |
+
return GetSchema({}, descriptor);
|
| 322 |
+
}
|
| 323 |
+
|
| 324 |
+
/// \brief List all available flights known to the server
|
| 325 |
+
/// \return Arrow result with an iterator that returns a FlightInfo for each flight
|
| 326 |
+
arrow::Result<std::unique_ptr<FlightListing>> ListFlights();
|
| 327 |
+
|
| 328 |
+
/// \brief List available flights given indicated filter criteria
|
| 329 |
+
/// \param[in] options Per-RPC options
|
| 330 |
+
/// \param[in] criteria the filter criteria (opaque)
|
| 331 |
+
/// \return Arrow result with an iterator that returns a FlightInfo for each flight
|
| 332 |
+
arrow::Result<std::unique_ptr<FlightListing>> ListFlights(
|
| 333 |
+
const FlightCallOptions& options, const Criteria& criteria);
|
| 334 |
+
|
| 335 |
+
/// \brief Given a flight ticket and schema, request to be sent the
|
| 336 |
+
/// stream. Returns record batch stream reader
|
| 337 |
+
/// \param[in] options Per-RPC options
|
| 338 |
+
/// \param[in] ticket The flight ticket to use
|
| 339 |
+
/// \return Arrow result with the returned RecordBatchReader
|
| 340 |
+
arrow::Result<std::unique_ptr<FlightStreamReader>> DoGet(
|
| 341 |
+
const FlightCallOptions& options, const Ticket& ticket);
|
| 342 |
+
arrow::Result<std::unique_ptr<FlightStreamReader>> DoGet(const Ticket& ticket) {
|
| 343 |
+
return DoGet({}, ticket);
|
| 344 |
+
}
|
| 345 |
+
|
| 346 |
+
/// \brief DoPut return value
|
| 347 |
+
struct DoPutResult {
|
| 348 |
+
/// \brief a writer to write record batches to
|
| 349 |
+
std::unique_ptr<FlightStreamWriter> writer;
|
| 350 |
+
/// \brief a reader for application metadata from the server
|
| 351 |
+
std::unique_ptr<FlightMetadataReader> reader;
|
| 352 |
+
};
|
| 353 |
+
/// \brief Upload data to a Flight described by the given
|
| 354 |
+
/// descriptor. The caller must call Close() on the returned stream
|
| 355 |
+
/// once they are done writing.
|
| 356 |
+
///
|
| 357 |
+
/// The reader and writer are linked; closing the writer will also
|
| 358 |
+
/// close the reader. Use \a DoneWriting to only close the write
|
| 359 |
+
/// side of the channel.
|
| 360 |
+
///
|
| 361 |
+
/// \param[in] options Per-RPC options
|
| 362 |
+
/// \param[in] descriptor the descriptor of the stream
|
| 363 |
+
/// \param[in] schema the schema for the data to upload
|
| 364 |
+
/// \return Arrow result with a DoPutResult struct holding a reader and a writer
|
| 365 |
+
arrow::Result<DoPutResult> DoPut(const FlightCallOptions& options,
|
| 366 |
+
const FlightDescriptor& descriptor,
|
| 367 |
+
const std::shared_ptr<Schema>& schema);
|
| 368 |
+
|
| 369 |
+
arrow::Result<DoPutResult> DoPut(const FlightDescriptor& descriptor,
|
| 370 |
+
const std::shared_ptr<Schema>& schema) {
|
| 371 |
+
return DoPut({}, descriptor, schema);
|
| 372 |
+
}
|
| 373 |
+
|
| 374 |
+
struct DoExchangeResult {
|
| 375 |
+
std::unique_ptr<FlightStreamWriter> writer;
|
| 376 |
+
std::unique_ptr<FlightStreamReader> reader;
|
| 377 |
+
};
|
| 378 |
+
arrow::Result<DoExchangeResult> DoExchange(const FlightCallOptions& options,
|
| 379 |
+
const FlightDescriptor& descriptor);
|
| 380 |
+
arrow::Result<DoExchangeResult> DoExchange(const FlightDescriptor& descriptor) {
|
| 381 |
+
return DoExchange({}, descriptor);
|
| 382 |
+
}
|
| 383 |
+
|
| 384 |
+
/// \brief Set server session option(s) by name/value. Sessions are generally
|
| 385 |
+
/// persisted via HTTP cookies.
|
| 386 |
+
/// \param[in] options Per-RPC options
|
| 387 |
+
/// \param[in] request The server session options to set
|
| 388 |
+
::arrow::Result<SetSessionOptionsResult> SetSessionOptions(
|
| 389 |
+
const FlightCallOptions& options, const SetSessionOptionsRequest& request);
|
| 390 |
+
|
| 391 |
+
/// \brief Get the current server session options. The session is generally
|
| 392 |
+
/// accessed via an HTTP cookie.
|
| 393 |
+
/// \param[in] options Per-RPC options
|
| 394 |
+
/// \param[in] request The (empty) GetSessionOptions request object.
|
| 395 |
+
::arrow::Result<GetSessionOptionsResult> GetSessionOptions(
|
| 396 |
+
const FlightCallOptions& options, const GetSessionOptionsRequest& request);
|
| 397 |
+
|
| 398 |
+
/// \brief Close/invalidate the current server session. The session is generally
|
| 399 |
+
/// accessed via an HTTP cookie.
|
| 400 |
+
/// \param[in] options Per-RPC options
|
| 401 |
+
/// \param[in] request The (empty) CloseSession request object.
|
| 402 |
+
::arrow::Result<CloseSessionResult> CloseSession(const FlightCallOptions& options,
|
| 403 |
+
const CloseSessionRequest& request);
|
| 404 |
+
|
| 405 |
+
/// \brief Explicitly shut down and clean up the client.
|
| 406 |
+
///
|
| 407 |
+
/// For backwards compatibility, this will be implicitly called by
|
| 408 |
+
/// the destructor if not already called, but this gives the
|
| 409 |
+
/// application no chance to handle errors, so it is recommended to
|
| 410 |
+
/// explicitly close the client.
|
| 411 |
+
///
|
| 412 |
+
/// \since 8.0.0
|
| 413 |
+
Status Close();
|
| 414 |
+
|
| 415 |
+
/// \brief Whether this client supports asynchronous methods.
|
| 416 |
+
bool supports_async() const;
|
| 417 |
+
|
| 418 |
+
/// \brief Check whether this client supports asynchronous methods.
|
| 419 |
+
///
|
| 420 |
+
/// This is like supports_async(), except that a detailed error message
|
| 421 |
+
/// is returned if async support is not available. If async support is
|
| 422 |
+
/// available, this function returns successfully.
|
| 423 |
+
Status CheckAsyncSupport() const;
|
| 424 |
+
|
| 425 |
+
private:
|
| 426 |
+
FlightClient();
|
| 427 |
+
Status CheckOpen() const;
|
| 428 |
+
std::unique_ptr<internal::ClientTransport> transport_;
|
| 429 |
+
bool closed_;
|
| 430 |
+
int64_t write_size_limit_bytes_;
|
| 431 |
+
};
|
| 432 |
+
|
| 433 |
+
} // namespace flight
|
| 434 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_auth.h
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <string>
|
| 21 |
+
|
| 22 |
+
#include "arrow/flight/visibility.h"
|
| 23 |
+
#include "arrow/status.h"
|
| 24 |
+
|
| 25 |
+
namespace arrow {
|
| 26 |
+
|
| 27 |
+
namespace flight {
|
| 28 |
+
|
| 29 |
+
/// \brief A reader for messages from the server during an
|
| 30 |
+
/// authentication handshake.
|
| 31 |
+
class ARROW_FLIGHT_EXPORT ClientAuthReader {
|
| 32 |
+
public:
|
| 33 |
+
virtual ~ClientAuthReader() = default;
|
| 34 |
+
virtual Status Read(std::string* response) = 0;
|
| 35 |
+
};
|
| 36 |
+
|
| 37 |
+
/// \brief A writer for messages to the server during an
|
| 38 |
+
/// authentication handshake.
|
| 39 |
+
class ARROW_FLIGHT_EXPORT ClientAuthSender {
|
| 40 |
+
public:
|
| 41 |
+
virtual ~ClientAuthSender() = default;
|
| 42 |
+
virtual Status Write(const std::string& token) = 0;
|
| 43 |
+
};
|
| 44 |
+
|
| 45 |
+
/// \brief An authentication implementation for a Flight service.
|
| 46 |
+
/// Authentication includes both an initial negotiation and a per-call
|
| 47 |
+
/// token validation. Implementations may choose to use either or both
|
| 48 |
+
/// mechanisms.
|
| 49 |
+
class ARROW_FLIGHT_EXPORT ClientAuthHandler {
|
| 50 |
+
public:
|
| 51 |
+
virtual ~ClientAuthHandler() = default;
|
| 52 |
+
/// \brief Authenticate the client on initial connection. The client
|
| 53 |
+
/// can send messages to/read responses from the server at any time.
|
| 54 |
+
/// \return Status OK if authenticated successfully
|
| 55 |
+
virtual Status Authenticate(ClientAuthSender* outgoing, ClientAuthReader* incoming) = 0;
|
| 56 |
+
/// \brief Get a per-call token.
|
| 57 |
+
/// \param[out] token The token to send to the server.
|
| 58 |
+
virtual Status GetToken(std::string* token) = 0;
|
| 59 |
+
};
|
| 60 |
+
|
| 61 |
+
} // namespace flight
|
| 62 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_cookie_middleware.h
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
// Middleware implementation for sending and receiving HTTP cookies.
|
| 19 |
+
|
| 20 |
+
#pragma once
|
| 21 |
+
|
| 22 |
+
#include <memory>
|
| 23 |
+
|
| 24 |
+
#include "arrow/flight/client_middleware.h"
|
| 25 |
+
|
| 26 |
+
namespace arrow {
|
| 27 |
+
namespace flight {
|
| 28 |
+
|
| 29 |
+
/// \brief Returns a ClientMiddlewareFactory that handles sending and receiving cookies.
|
| 30 |
+
ARROW_FLIGHT_EXPORT std::shared_ptr<ClientMiddlewareFactory> GetCookieFactory();
|
| 31 |
+
|
| 32 |
+
} // namespace flight
|
| 33 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_middleware.h
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
// Interfaces for defining middleware for Flight clients. Currently
|
| 19 |
+
// experimental.
|
| 20 |
+
|
| 21 |
+
#pragma once
|
| 22 |
+
|
| 23 |
+
#include <memory>
|
| 24 |
+
|
| 25 |
+
#include "arrow/flight/middleware.h"
|
| 26 |
+
#include "arrow/flight/visibility.h" // IWYU pragma: keep
|
| 27 |
+
#include "arrow/status.h"
|
| 28 |
+
|
| 29 |
+
namespace arrow {
|
| 30 |
+
namespace flight {
|
| 31 |
+
|
| 32 |
+
/// \brief Client-side middleware for a call, instantiated per RPC.
|
| 33 |
+
///
|
| 34 |
+
/// Middleware should be fast and must be infallible: there is no way
|
| 35 |
+
/// to reject the call or report errors from the middleware instance.
|
| 36 |
+
class ARROW_FLIGHT_EXPORT ClientMiddleware {
|
| 37 |
+
public:
|
| 38 |
+
virtual ~ClientMiddleware() = default;
|
| 39 |
+
|
| 40 |
+
/// \brief A callback before headers are sent. Extra headers can be
|
| 41 |
+
/// added, but existing ones cannot be read.
|
| 42 |
+
virtual void SendingHeaders(AddCallHeaders* outgoing_headers) = 0;
|
| 43 |
+
|
| 44 |
+
/// \brief A callback when headers are received from the server.
|
| 45 |
+
///
|
| 46 |
+
/// This may be called more than once, since servers send both
|
| 47 |
+
/// headers and trailers. Some implementations (e.g. gRPC-Java, and
|
| 48 |
+
/// hence Arrow Flight in Java) may consolidate headers into
|
| 49 |
+
/// trailers if the RPC errored.
|
| 50 |
+
virtual void ReceivedHeaders(const CallHeaders& incoming_headers) = 0;
|
| 51 |
+
|
| 52 |
+
/// \brief A callback after the call has completed.
|
| 53 |
+
virtual void CallCompleted(const Status& status) = 0;
|
| 54 |
+
};
|
| 55 |
+
|
| 56 |
+
/// \brief A factory for new middleware instances.
|
| 57 |
+
///
|
| 58 |
+
/// If added to a client, this will be called for each RPC (including
|
| 59 |
+
/// Handshake) to give the opportunity to intercept the call.
|
| 60 |
+
///
|
| 61 |
+
/// It is guaranteed that all client middleware methods are called
|
| 62 |
+
/// from the same thread that calls the RPC method implementation.
|
| 63 |
+
class ARROW_FLIGHT_EXPORT ClientMiddlewareFactory {
|
| 64 |
+
public:
|
| 65 |
+
virtual ~ClientMiddlewareFactory() = default;
|
| 66 |
+
|
| 67 |
+
/// \brief A callback for the start of a new call.
|
| 68 |
+
///
|
| 69 |
+
/// \param info Information about the call.
|
| 70 |
+
/// \param[out] middleware The middleware instance for this call. If
|
| 71 |
+
/// unset, will not add middleware to this call instance from
|
| 72 |
+
/// this factory.
|
| 73 |
+
virtual void StartCall(const CallInfo& info,
|
| 74 |
+
std::unique_ptr<ClientMiddleware>* middleware) = 0;
|
| 75 |
+
};
|
| 76 |
+
|
| 77 |
+
} // namespace flight
|
| 78 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_tracing_middleware.h
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
// Middleware implementation for propagating OpenTelemetry spans.
|
| 19 |
+
|
| 20 |
+
#pragma once
|
| 21 |
+
|
| 22 |
+
#include <memory>
|
| 23 |
+
|
| 24 |
+
#include "arrow/flight/client_middleware.h"
|
| 25 |
+
|
| 26 |
+
namespace arrow {
|
| 27 |
+
namespace flight {
|
| 28 |
+
|
| 29 |
+
/// \brief Returns a ClientMiddlewareFactory that handles sending OpenTelemetry spans.
|
| 30 |
+
ARROW_FLIGHT_EXPORT std::shared_ptr<ClientMiddlewareFactory>
|
| 31 |
+
MakeTracingClientMiddlewareFactory();
|
| 32 |
+
|
| 33 |
+
} // namespace flight
|
| 34 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/middleware.h
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
// Interfaces for defining middleware for Flight clients and
|
| 19 |
+
// servers.
|
| 20 |
+
|
| 21 |
+
#pragma once
|
| 22 |
+
|
| 23 |
+
#include <memory>
|
| 24 |
+
#include <string>
|
| 25 |
+
#include <string_view>
|
| 26 |
+
#include <utility>
|
| 27 |
+
|
| 28 |
+
#include "arrow/flight/types.h"
|
| 29 |
+
#include "arrow/status.h"
|
| 30 |
+
|
| 31 |
+
namespace arrow {
|
| 32 |
+
namespace flight {
|
| 33 |
+
|
| 34 |
+
/// \brief A write-only wrapper around headers for an RPC call.
|
| 35 |
+
class ARROW_FLIGHT_EXPORT AddCallHeaders {
|
| 36 |
+
public:
|
| 37 |
+
virtual ~AddCallHeaders() = default;
|
| 38 |
+
|
| 39 |
+
/// \brief Add a header to be sent to the client.
|
| 40 |
+
///
|
| 41 |
+
/// \param[in] key The header name. Must be lowercase ASCII; some
|
| 42 |
+
/// transports may reject invalid header names.
|
| 43 |
+
/// \param[in] value The header value. Some transports may only
|
| 44 |
+
/// accept binary header values if the header name ends in "-bin".
|
| 45 |
+
virtual void AddHeader(const std::string& key, const std::string& value) = 0;
|
| 46 |
+
};
|
| 47 |
+
|
| 48 |
+
/// \brief An enumeration of the RPC methods Flight implements.
|
| 49 |
+
enum class FlightMethod : char {
|
| 50 |
+
Invalid = 0,
|
| 51 |
+
Handshake = 1,
|
| 52 |
+
ListFlights = 2,
|
| 53 |
+
GetFlightInfo = 3,
|
| 54 |
+
GetSchema = 4,
|
| 55 |
+
DoGet = 5,
|
| 56 |
+
DoPut = 6,
|
| 57 |
+
DoAction = 7,
|
| 58 |
+
ListActions = 8,
|
| 59 |
+
DoExchange = 9,
|
| 60 |
+
PollFlightInfo = 10,
|
| 61 |
+
};
|
| 62 |
+
|
| 63 |
+
/// \brief Get a human-readable name for a Flight method.
|
| 64 |
+
ARROW_FLIGHT_EXPORT
|
| 65 |
+
std::string ToString(FlightMethod method);
|
| 66 |
+
|
| 67 |
+
/// \brief Information about an instance of a Flight RPC.
|
| 68 |
+
struct ARROW_FLIGHT_EXPORT CallInfo {
|
| 69 |
+
public:
|
| 70 |
+
/// \brief The RPC method of this call.
|
| 71 |
+
FlightMethod method;
|
| 72 |
+
};
|
| 73 |
+
|
| 74 |
+
} // namespace flight
|
| 75 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/otel_logging.h
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include "arrow/util/config.h"
|
| 21 |
+
|
| 22 |
+
#ifdef ARROW_WITH_OPENTELEMETRY
|
| 23 |
+
# include "arrow/status.h"
|
| 24 |
+
# include "arrow/telemetry/logging.h"
|
| 25 |
+
# include "arrow/util/macros.h"
|
| 26 |
+
|
| 27 |
+
namespace arrow::flight {
|
| 28 |
+
|
| 29 |
+
ARROW_EXPORT Status
|
| 30 |
+
RegisterFlightOtelLoggers(const telemetry::OtelLoggingOptions& options);
|
| 31 |
+
|
| 32 |
+
} // namespace arrow::flight
|
| 33 |
+
#endif
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/platform.h
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
// Internal header. Platform-specific definitions for Flight.
|
| 19 |
+
|
| 20 |
+
#pragma once
|
| 21 |
+
|
| 22 |
+
#ifdef _MSC_VER
|
| 23 |
+
|
| 24 |
+
// The protobuf documentation says that C4251 warnings when using the
|
| 25 |
+
// library are spurious and suppressed when the build the library and
|
| 26 |
+
// compiler, but must be also suppressed in downstream projects
|
| 27 |
+
# pragma warning(disable : 4251)
|
| 28 |
+
|
| 29 |
+
#endif // _MSC_VER
|
| 30 |
+
|
| 31 |
+
#include "arrow/util/config.h" // IWYU pragma: keep
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server.h
ADDED
|
@@ -0,0 +1,327 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
// Interfaces to use for defining Flight RPC servers.
|
| 19 |
+
|
| 20 |
+
#pragma once
|
| 21 |
+
|
| 22 |
+
#include <chrono>
|
| 23 |
+
#include <functional>
|
| 24 |
+
#include <memory>
|
| 25 |
+
#include <string>
|
| 26 |
+
#include <utility>
|
| 27 |
+
#include <vector>
|
| 28 |
+
|
| 29 |
+
#include "arrow/flight/server_auth.h"
|
| 30 |
+
#include "arrow/flight/type_fwd.h"
|
| 31 |
+
#include "arrow/flight/types.h" // IWYU pragma: keep
|
| 32 |
+
#include "arrow/flight/visibility.h" // IWYU pragma: keep
|
| 33 |
+
#include "arrow/ipc/dictionary.h"
|
| 34 |
+
#include "arrow/ipc/options.h"
|
| 35 |
+
#include "arrow/record_batch.h"
|
| 36 |
+
|
| 37 |
+
namespace arrow {
|
| 38 |
+
|
| 39 |
+
class Schema;
|
| 40 |
+
class Status;
|
| 41 |
+
|
| 42 |
+
namespace flight {
|
| 43 |
+
|
| 44 |
+
/// \brief Interface that produces a sequence of IPC payloads to be sent in
|
| 45 |
+
/// FlightData protobuf messages
|
| 46 |
+
class ARROW_FLIGHT_EXPORT FlightDataStream {
|
| 47 |
+
public:
|
| 48 |
+
virtual ~FlightDataStream();
|
| 49 |
+
|
| 50 |
+
virtual std::shared_ptr<Schema> schema() = 0;
|
| 51 |
+
|
| 52 |
+
/// \brief Compute FlightPayload containing serialized RecordBatch schema
|
| 53 |
+
virtual arrow::Result<FlightPayload> GetSchemaPayload() = 0;
|
| 54 |
+
|
| 55 |
+
// When the stream is completed, the last payload written will have null
|
| 56 |
+
// metadata
|
| 57 |
+
virtual arrow::Result<FlightPayload> Next() = 0;
|
| 58 |
+
|
| 59 |
+
virtual Status Close();
|
| 60 |
+
};
|
| 61 |
+
|
| 62 |
+
/// \brief A basic implementation of FlightDataStream that will provide
|
| 63 |
+
/// a sequence of FlightData messages to be written to a stream
|
| 64 |
+
class ARROW_FLIGHT_EXPORT RecordBatchStream : public FlightDataStream {
|
| 65 |
+
public:
|
| 66 |
+
/// \param[in] reader produces a sequence of record batches
|
| 67 |
+
/// \param[in] options IPC options for writing
|
| 68 |
+
explicit RecordBatchStream(
|
| 69 |
+
const std::shared_ptr<RecordBatchReader>& reader,
|
| 70 |
+
const ipc::IpcWriteOptions& options = ipc::IpcWriteOptions::Defaults());
|
| 71 |
+
~RecordBatchStream() override;
|
| 72 |
+
|
| 73 |
+
// inherit deprecated API
|
| 74 |
+
using FlightDataStream::GetSchemaPayload;
|
| 75 |
+
using FlightDataStream::Next;
|
| 76 |
+
|
| 77 |
+
std::shared_ptr<Schema> schema() override;
|
| 78 |
+
arrow::Result<FlightPayload> GetSchemaPayload() override;
|
| 79 |
+
|
| 80 |
+
arrow::Result<FlightPayload> Next() override;
|
| 81 |
+
Status Close() override;
|
| 82 |
+
|
| 83 |
+
private:
|
| 84 |
+
class RecordBatchStreamImpl;
|
| 85 |
+
std::unique_ptr<RecordBatchStreamImpl> impl_;
|
| 86 |
+
};
|
| 87 |
+
|
| 88 |
+
/// \brief A reader for IPC payloads uploaded by a client. Also allows
|
| 89 |
+
/// reading application-defined metadata via the Flight protocol.
|
| 90 |
+
class ARROW_FLIGHT_EXPORT FlightMessageReader : public MetadataRecordBatchReader {
|
| 91 |
+
public:
|
| 92 |
+
/// \brief Get the descriptor for this upload.
|
| 93 |
+
virtual const FlightDescriptor& descriptor() const = 0;
|
| 94 |
+
};
|
| 95 |
+
|
| 96 |
+
/// \brief A writer for application-specific metadata sent back to the
|
| 97 |
+
/// client during an upload.
|
| 98 |
+
class ARROW_FLIGHT_EXPORT FlightMetadataWriter {
|
| 99 |
+
public:
|
| 100 |
+
virtual ~FlightMetadataWriter();
|
| 101 |
+
/// \brief Send a message to the client.
|
| 102 |
+
virtual Status WriteMetadata(const Buffer& app_metadata) = 0;
|
| 103 |
+
};
|
| 104 |
+
|
| 105 |
+
/// \brief A writer for IPC payloads to a client. Also allows sending
|
| 106 |
+
/// application-defined metadata via the Flight protocol.
|
| 107 |
+
///
|
| 108 |
+
/// This class offers more control compared to FlightDataStream,
|
| 109 |
+
/// including the option to write metadata without data and the
|
| 110 |
+
/// ability to interleave reading and writing.
|
| 111 |
+
class ARROW_FLIGHT_EXPORT FlightMessageWriter : public MetadataRecordBatchWriter {
|
| 112 |
+
public:
|
| 113 |
+
virtual ~FlightMessageWriter() = default;
|
| 114 |
+
};
|
| 115 |
+
|
| 116 |
+
/// \brief Call state/contextual data.
|
| 117 |
+
class ARROW_FLIGHT_EXPORT ServerCallContext {
|
| 118 |
+
public:
|
| 119 |
+
virtual ~ServerCallContext() = default;
|
| 120 |
+
/// \brief The name of the authenticated peer (may be the empty string)
|
| 121 |
+
virtual const std::string& peer_identity() const = 0;
|
| 122 |
+
/// \brief The peer address (not validated)
|
| 123 |
+
virtual const std::string& peer() const = 0;
|
| 124 |
+
/// \brief Add a response header. This is only valid before the server
|
| 125 |
+
/// starts sending the response; generally this isn't an issue unless you
|
| 126 |
+
/// are implementing FlightDataStream, ResultStream, or similar interfaces
|
| 127 |
+
/// yourself, or during a DoExchange or DoPut.
|
| 128 |
+
virtual void AddHeader(const std::string& key, const std::string& value) const = 0;
|
| 129 |
+
/// \brief Add a response trailer. This is only valid before the server
|
| 130 |
+
/// sends the final status; generally this isn't an issue unless your RPC
|
| 131 |
+
/// handler launches a thread or similar.
|
| 132 |
+
virtual void AddTrailer(const std::string& key, const std::string& value) const = 0;
|
| 133 |
+
/// \brief Look up a middleware by key. Do not maintain a reference
|
| 134 |
+
/// to the object beyond the request body.
|
| 135 |
+
/// \return The middleware, or nullptr if not found.
|
| 136 |
+
virtual ServerMiddleware* GetMiddleware(const std::string& key) const = 0;
|
| 137 |
+
/// \brief Check if the current RPC has been cancelled (by the client, by
|
| 138 |
+
/// a network error, etc.).
|
| 139 |
+
virtual bool is_cancelled() const = 0;
|
| 140 |
+
/// \brief The headers sent by the client for this call.
|
| 141 |
+
virtual const CallHeaders& incoming_headers() const = 0;
|
| 142 |
+
};
|
| 143 |
+
|
| 144 |
+
class ARROW_FLIGHT_EXPORT FlightServerOptions {
|
| 145 |
+
public:
|
| 146 |
+
explicit FlightServerOptions(const Location& location_);
|
| 147 |
+
|
| 148 |
+
~FlightServerOptions();
|
| 149 |
+
|
| 150 |
+
/// \brief The host & port (or domain socket path) to listen on.
|
| 151 |
+
/// Use port 0 to bind to an available port.
|
| 152 |
+
Location location;
|
| 153 |
+
/// \brief The authentication handler to use.
|
| 154 |
+
std::shared_ptr<ServerAuthHandler> auth_handler;
|
| 155 |
+
/// \brief A list of TLS certificate+key pairs to use.
|
| 156 |
+
std::vector<CertKeyPair> tls_certificates;
|
| 157 |
+
/// \brief Enable mTLS and require that the client present a certificate.
|
| 158 |
+
bool verify_client;
|
| 159 |
+
/// \brief If using mTLS, the PEM-encoded root certificate to use.
|
| 160 |
+
std::string root_certificates;
|
| 161 |
+
/// \brief A list of server middleware to apply, along with a key to
|
| 162 |
+
/// identify them by.
|
| 163 |
+
///
|
| 164 |
+
/// Middleware are always applied in the order provided. Duplicate
|
| 165 |
+
/// keys are an error.
|
| 166 |
+
std::vector<std::pair<std::string, std::shared_ptr<ServerMiddlewareFactory>>>
|
| 167 |
+
middleware;
|
| 168 |
+
|
| 169 |
+
/// \brief An optional memory manager to control where to allocate incoming data.
|
| 170 |
+
std::shared_ptr<MemoryManager> memory_manager;
|
| 171 |
+
|
| 172 |
+
/// \brief A Flight implementation-specific callback to customize
|
| 173 |
+
/// transport-specific options.
|
| 174 |
+
///
|
| 175 |
+
/// Not guaranteed to be called. The type of the parameter is
|
| 176 |
+
/// specific to the Flight implementation. Users should take care to
|
| 177 |
+
/// link to the same transport implementation as Flight to avoid
|
| 178 |
+
/// runtime problems. See "Using Arrow C++ in your own project" in
|
| 179 |
+
/// the documentation for more details.
|
| 180 |
+
std::function<void(void*)> builder_hook;
|
| 181 |
+
};
|
| 182 |
+
|
| 183 |
+
/// \brief Skeleton RPC server implementation which can be used to create
|
| 184 |
+
/// custom servers by implementing its abstract methods
|
| 185 |
+
class ARROW_FLIGHT_EXPORT FlightServerBase {
|
| 186 |
+
public:
|
| 187 |
+
FlightServerBase();
|
| 188 |
+
virtual ~FlightServerBase();
|
| 189 |
+
|
| 190 |
+
// Lifecycle methods.
|
| 191 |
+
|
| 192 |
+
/// \brief Initialize a Flight server listening at the given location.
|
| 193 |
+
/// This method must be called before any other method.
|
| 194 |
+
/// \param[in] options The configuration for this server.
|
| 195 |
+
Status Init(const FlightServerOptions& options);
|
| 196 |
+
|
| 197 |
+
/// \brief Get the port that the Flight server is listening on.
|
| 198 |
+
/// This method must only be called after Init(). Will return a
|
| 199 |
+
/// non-positive value if no port exists (e.g. when listening on a
|
| 200 |
+
/// domain socket).
|
| 201 |
+
int port() const;
|
| 202 |
+
|
| 203 |
+
/// \brief Get the address that the Flight server is listening on.
|
| 204 |
+
/// This method must only be called after Init().
|
| 205 |
+
Location location() const;
|
| 206 |
+
|
| 207 |
+
/// \brief Set the server to stop when receiving any of the given signal
|
| 208 |
+
/// numbers.
|
| 209 |
+
/// This method must be called before Serve().
|
| 210 |
+
Status SetShutdownOnSignals(const std::vector<int> sigs);
|
| 211 |
+
|
| 212 |
+
/// \brief Start serving.
|
| 213 |
+
/// This method blocks until the server shuts down.
|
| 214 |
+
///
|
| 215 |
+
/// The server will start to shut down when either Shutdown() is called
|
| 216 |
+
/// or one of the signals registered in SetShutdownOnSignals() is received.
|
| 217 |
+
Status Serve();
|
| 218 |
+
|
| 219 |
+
/// \brief Query whether Serve() was interrupted by a signal.
|
| 220 |
+
/// This method must be called after Serve() has returned.
|
| 221 |
+
///
|
| 222 |
+
/// \return int the signal number that interrupted Serve(), if any, otherwise 0
|
| 223 |
+
int GotSignal() const;
|
| 224 |
+
|
| 225 |
+
/// \brief Shut down the server, blocking until current requests finish.
|
| 226 |
+
///
|
| 227 |
+
/// Can be called from a signal handler or another thread while Serve()
|
| 228 |
+
/// blocks. Optionally a deadline can be set. Once the deadline expires
|
| 229 |
+
/// server will wait until remaining running calls complete.
|
| 230 |
+
///
|
| 231 |
+
/// Should only be called once.
|
| 232 |
+
Status Shutdown(const std::chrono::system_clock::time_point* deadline = NULLPTR);
|
| 233 |
+
|
| 234 |
+
/// \brief Block until server shuts down with Shutdown.
|
| 235 |
+
///
|
| 236 |
+
/// Does not respond to signals like Serve().
|
| 237 |
+
Status Wait();
|
| 238 |
+
|
| 239 |
+
// Implement these methods to create your own server. The default
|
| 240 |
+
// implementations will return a not-implemented result to the client
|
| 241 |
+
|
| 242 |
+
/// \brief Retrieve a list of available fields given an optional opaque
|
| 243 |
+
/// criteria
|
| 244 |
+
/// \param[in] context The call context.
|
| 245 |
+
/// \param[in] criteria may be null
|
| 246 |
+
/// \param[out] listings the returned listings iterator
|
| 247 |
+
/// \return Status
|
| 248 |
+
virtual Status ListFlights(const ServerCallContext& context, const Criteria* criteria,
|
| 249 |
+
std::unique_ptr<FlightListing>* listings);
|
| 250 |
+
|
| 251 |
+
/// \brief Retrieve the schema and an access plan for the indicated
|
| 252 |
+
/// descriptor
|
| 253 |
+
/// \param[in] context The call context.
|
| 254 |
+
/// \param[in] request the dataset request, whether a named dataset or command
|
| 255 |
+
/// \param[out] info the returned flight info provider
|
| 256 |
+
/// \return Status
|
| 257 |
+
virtual Status GetFlightInfo(const ServerCallContext& context,
|
| 258 |
+
const FlightDescriptor& request,
|
| 259 |
+
std::unique_ptr<FlightInfo>* info);
|
| 260 |
+
|
| 261 |
+
/// \brief Retrieve the current status of the target query
|
| 262 |
+
/// \param[in] context The call context.
|
| 263 |
+
/// \param[in] request the dataset request or a descriptor returned by a
|
| 264 |
+
/// prior PollFlightInfo call
|
| 265 |
+
/// \param[out] info the returned retry info provider
|
| 266 |
+
/// \return Status
|
| 267 |
+
virtual Status PollFlightInfo(const ServerCallContext& context,
|
| 268 |
+
const FlightDescriptor& request,
|
| 269 |
+
std::unique_ptr<PollInfo>* info);
|
| 270 |
+
|
| 271 |
+
/// \brief Retrieve the schema for the indicated descriptor
|
| 272 |
+
/// \param[in] context The call context.
|
| 273 |
+
/// \param[in] request the dataset request, whether a named dataset or command
|
| 274 |
+
/// \param[out] schema the returned flight schema provider
|
| 275 |
+
/// \return Status
|
| 276 |
+
virtual Status GetSchema(const ServerCallContext& context,
|
| 277 |
+
const FlightDescriptor& request,
|
| 278 |
+
std::unique_ptr<SchemaResult>* schema);
|
| 279 |
+
|
| 280 |
+
/// \brief Get a stream of IPC payloads to put on the wire
|
| 281 |
+
/// \param[in] context The call context.
|
| 282 |
+
/// \param[in] request an opaque ticket
|
| 283 |
+
/// \param[out] stream the returned stream provider
|
| 284 |
+
/// \return Status
|
| 285 |
+
virtual Status DoGet(const ServerCallContext& context, const Ticket& request,
|
| 286 |
+
std::unique_ptr<FlightDataStream>* stream);
|
| 287 |
+
|
| 288 |
+
/// \brief Process a stream of IPC payloads sent from a client
|
| 289 |
+
/// \param[in] context The call context.
|
| 290 |
+
/// \param[in] reader a sequence of uploaded record batches
|
| 291 |
+
/// \param[in] writer send metadata back to the client
|
| 292 |
+
/// \return Status
|
| 293 |
+
virtual Status DoPut(const ServerCallContext& context,
|
| 294 |
+
std::unique_ptr<FlightMessageReader> reader,
|
| 295 |
+
std::unique_ptr<FlightMetadataWriter> writer);
|
| 296 |
+
|
| 297 |
+
/// \brief Process a bidirectional stream of IPC payloads
|
| 298 |
+
/// \param[in] context The call context.
|
| 299 |
+
/// \param[in] reader a sequence of uploaded record batches
|
| 300 |
+
/// \param[in] writer send data back to the client
|
| 301 |
+
/// \return Status
|
| 302 |
+
virtual Status DoExchange(const ServerCallContext& context,
|
| 303 |
+
std::unique_ptr<FlightMessageReader> reader,
|
| 304 |
+
std::unique_ptr<FlightMessageWriter> writer);
|
| 305 |
+
|
| 306 |
+
/// \brief Execute an action, return stream of zero or more results
|
| 307 |
+
/// \param[in] context The call context.
|
| 308 |
+
/// \param[in] action the action to execute, with type and body
|
| 309 |
+
/// \param[out] result the result iterator
|
| 310 |
+
/// \return Status
|
| 311 |
+
virtual Status DoAction(const ServerCallContext& context, const Action& action,
|
| 312 |
+
std::unique_ptr<ResultStream>* result);
|
| 313 |
+
|
| 314 |
+
/// \brief Retrieve the list of available actions
|
| 315 |
+
/// \param[in] context The call context.
|
| 316 |
+
/// \param[out] actions a vector of available action types
|
| 317 |
+
/// \return Status
|
| 318 |
+
virtual Status ListActions(const ServerCallContext& context,
|
| 319 |
+
std::vector<ActionType>* actions);
|
| 320 |
+
|
| 321 |
+
private:
|
| 322 |
+
struct Impl;
|
| 323 |
+
std::unique_ptr<Impl> impl_;
|
| 324 |
+
};
|
| 325 |
+
|
| 326 |
+
} // namespace flight
|
| 327 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server_auth.h
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
/// \brief Server-side APIs to implement authentication for Flight.
|
| 19 |
+
|
| 20 |
+
#pragma once
|
| 21 |
+
|
| 22 |
+
#include <string>
|
| 23 |
+
|
| 24 |
+
#include "arrow/flight/type_fwd.h"
|
| 25 |
+
#include "arrow/flight/visibility.h"
|
| 26 |
+
#include "arrow/status.h"
|
| 27 |
+
|
| 28 |
+
namespace arrow {
|
| 29 |
+
|
| 30 |
+
namespace flight {
|
| 31 |
+
|
| 32 |
+
/// \brief A reader for messages from the client during an
|
| 33 |
+
/// authentication handshake.
|
| 34 |
+
class ARROW_FLIGHT_EXPORT ServerAuthReader {
|
| 35 |
+
public:
|
| 36 |
+
virtual ~ServerAuthReader() = default;
|
| 37 |
+
virtual Status Read(std::string* token) = 0;
|
| 38 |
+
};
|
| 39 |
+
|
| 40 |
+
/// \brief A writer for messages to the client during an
|
| 41 |
+
/// authentication handshake.
|
| 42 |
+
class ARROW_FLIGHT_EXPORT ServerAuthSender {
|
| 43 |
+
public:
|
| 44 |
+
virtual ~ServerAuthSender() = default;
|
| 45 |
+
virtual Status Write(const std::string& message) = 0;
|
| 46 |
+
};
|
| 47 |
+
|
| 48 |
+
/// \brief An authentication implementation for a Flight service.
|
| 49 |
+
/// Authentication includes both an initial negotiation and a per-call
|
| 50 |
+
/// token validation. Implementations may choose to use either or both
|
| 51 |
+
/// mechanisms.
|
| 52 |
+
/// An implementation may need to track some state, e.g. a mapping of
|
| 53 |
+
/// client tokens to authenticated identities.
|
| 54 |
+
class ARROW_FLIGHT_EXPORT ServerAuthHandler {
|
| 55 |
+
public:
|
| 56 |
+
virtual ~ServerAuthHandler();
|
| 57 |
+
/// \brief Authenticate the client on initial connection. The server
|
| 58 |
+
/// can send and read responses from the client at any time.
|
| 59 |
+
/// \param[in] context The call context.
|
| 60 |
+
/// \param[in] outgoing The writer for messages to the client.
|
| 61 |
+
/// \param[in] incoming The reader for messages from the client.
|
| 62 |
+
/// \return Status OK if this authentication is succeeded.
|
| 63 |
+
virtual Status Authenticate(const ServerCallContext& context,
|
| 64 |
+
ServerAuthSender* outgoing, ServerAuthReader* incoming) = 0;
|
| 65 |
+
/// \brief Validate a per-call client token.
|
| 66 |
+
/// \param[in] context The call context.
|
| 67 |
+
/// \param[in] token The client token. May be the empty string if
|
| 68 |
+
/// the client does not provide a token.
|
| 69 |
+
/// \param[out] peer_identity The identity of the peer, if this
|
| 70 |
+
/// authentication method supports it.
|
| 71 |
+
/// \return Status OK if the token is valid, any other status if
|
| 72 |
+
/// validation failed
|
| 73 |
+
virtual Status IsValid(const ServerCallContext& context, const std::string& token,
|
| 74 |
+
std::string* peer_identity) {
|
| 75 |
+
// TODO: We can make this pure virtual function when we remove
|
| 76 |
+
// the deprecated version.
|
| 77 |
+
ARROW_SUPPRESS_DEPRECATION_WARNING
|
| 78 |
+
return IsValid(token, peer_identity);
|
| 79 |
+
ARROW_UNSUPPRESS_DEPRECATION_WARNING
|
| 80 |
+
}
|
| 81 |
+
/// \brief Validate a per-call client token.
|
| 82 |
+
/// \param[in] token The client token. May be the empty string if
|
| 83 |
+
/// the client does not provide a token.
|
| 84 |
+
/// \param[out] peer_identity The identity of the peer, if this
|
| 85 |
+
/// authentication method supports it.
|
| 86 |
+
/// \return Status OK if the token is valid, any other status if
|
| 87 |
+
/// validation failed
|
| 88 |
+
/// \deprecated Deprecated in 13.0.0. Implement the IsValid()
|
| 89 |
+
/// with ServerCallContext version instead.
|
| 90 |
+
ARROW_DEPRECATED("Deprecated in 13.0.0. Use ServerCallContext overload instead.")
|
| 91 |
+
virtual Status IsValid(const std::string& token, std::string* peer_identity) {
|
| 92 |
+
return Status::NotImplemented(typeid(this).name(), "::IsValid() isn't implemented");
|
| 93 |
+
}
|
| 94 |
+
};
|
| 95 |
+
|
| 96 |
+
/// \brief An authentication mechanism that does nothing.
|
| 97 |
+
class ARROW_FLIGHT_EXPORT NoOpAuthHandler : public ServerAuthHandler {
|
| 98 |
+
public:
|
| 99 |
+
~NoOpAuthHandler() override;
|
| 100 |
+
Status Authenticate(const ServerCallContext& context, ServerAuthSender* outgoing,
|
| 101 |
+
ServerAuthReader* incoming) override;
|
| 102 |
+
Status IsValid(const ServerCallContext& context, const std::string& token,
|
| 103 |
+
std::string* peer_identity) override;
|
| 104 |
+
};
|
| 105 |
+
|
| 106 |
+
} // namespace flight
|
| 107 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server_middleware.h
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
// Interfaces for defining middleware for Flight servers.
|
| 19 |
+
|
| 20 |
+
#pragma once
|
| 21 |
+
|
| 22 |
+
#include <memory>
|
| 23 |
+
#include <string>
|
| 24 |
+
|
| 25 |
+
#include "arrow/flight/middleware.h"
|
| 26 |
+
#include "arrow/flight/type_fwd.h"
|
| 27 |
+
#include "arrow/flight/visibility.h" // IWYU pragma: keep
|
| 28 |
+
#include "arrow/status.h"
|
| 29 |
+
|
| 30 |
+
namespace arrow {
|
| 31 |
+
namespace flight {
|
| 32 |
+
|
| 33 |
+
/// \brief Server-side middleware for a call, instantiated per RPC.
|
| 34 |
+
///
|
| 35 |
+
/// Middleware should be fast and must be infallible: there is no way
|
| 36 |
+
/// to reject the call or report errors from the middleware instance.
|
| 37 |
+
class ARROW_FLIGHT_EXPORT ServerMiddleware {
|
| 38 |
+
public:
|
| 39 |
+
virtual ~ServerMiddleware() = default;
|
| 40 |
+
|
| 41 |
+
/// \brief Unique name of middleware, used as alternative to RTTI
|
| 42 |
+
/// \return the string name of the middleware
|
| 43 |
+
virtual std::string name() const = 0;
|
| 44 |
+
|
| 45 |
+
/// \brief A callback before headers are sent. Extra headers can be
|
| 46 |
+
/// added, but existing ones cannot be read.
|
| 47 |
+
virtual void SendingHeaders(AddCallHeaders* outgoing_headers) = 0;
|
| 48 |
+
|
| 49 |
+
/// \brief A callback after the call has completed.
|
| 50 |
+
virtual void CallCompleted(const Status& status) = 0;
|
| 51 |
+
};
|
| 52 |
+
|
| 53 |
+
/// \brief A factory for new middleware instances.
|
| 54 |
+
///
|
| 55 |
+
/// If added to a server, this will be called for each RPC (including
|
| 56 |
+
/// Handshake) to give the opportunity to intercept the call.
|
| 57 |
+
///
|
| 58 |
+
/// It is guaranteed that all server middleware methods are called
|
| 59 |
+
/// from the same thread that calls the RPC method implementation.
|
| 60 |
+
class ARROW_FLIGHT_EXPORT ServerMiddlewareFactory {
|
| 61 |
+
public:
|
| 62 |
+
virtual ~ServerMiddlewareFactory() = default;
|
| 63 |
+
|
| 64 |
+
/// \brief A callback for the start of a new call.
|
| 65 |
+
///
|
| 66 |
+
/// Return a non-OK status to reject the call with the given status.
|
| 67 |
+
///
|
| 68 |
+
/// \param[in] info Information about the call.
|
| 69 |
+
/// \param[in] context The call context.
|
| 70 |
+
/// \param[out] middleware The middleware instance for this call. If
|
| 71 |
+
/// null, no middleware will be added to this call instance from
|
| 72 |
+
/// this factory.
|
| 73 |
+
/// \return Status A non-OK status will reject the call with the
|
| 74 |
+
/// given status. Middleware previously in the chain will have
|
| 75 |
+
/// their CallCompleted callback called. Other middleware
|
| 76 |
+
/// factories will not be called.
|
| 77 |
+
virtual Status StartCall(const CallInfo& info, const ServerCallContext& context,
|
| 78 |
+
std::shared_ptr<ServerMiddleware>* middleware) = 0;
|
| 79 |
+
};
|
| 80 |
+
|
| 81 |
+
} // namespace flight
|
| 82 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server_tracing_middleware.h
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
// Middleware implementation for propagating OpenTelemetry spans.
|
| 19 |
+
|
| 20 |
+
#pragma once
|
| 21 |
+
|
| 22 |
+
#include <memory>
|
| 23 |
+
#include <string>
|
| 24 |
+
#include <vector>
|
| 25 |
+
|
| 26 |
+
#include "arrow/flight/server_middleware.h"
|
| 27 |
+
#include "arrow/flight/visibility.h"
|
| 28 |
+
#include "arrow/status.h"
|
| 29 |
+
|
| 30 |
+
namespace arrow {
|
| 31 |
+
namespace flight {
|
| 32 |
+
|
| 33 |
+
/// \brief Returns a ServerMiddlewareFactory that handles receiving OpenTelemetry spans.
|
| 34 |
+
ARROW_FLIGHT_EXPORT std::shared_ptr<ServerMiddlewareFactory>
|
| 35 |
+
MakeTracingServerMiddlewareFactory();
|
| 36 |
+
|
| 37 |
+
/// \brief A server middleware that provides access to the
|
| 38 |
+
/// OpenTelemetry context, if present.
|
| 39 |
+
///
|
| 40 |
+
/// Used to make the OpenTelemetry span available in Python.
|
| 41 |
+
class ARROW_FLIGHT_EXPORT TracingServerMiddleware : public ServerMiddleware {
|
| 42 |
+
public:
|
| 43 |
+
~TracingServerMiddleware();
|
| 44 |
+
|
| 45 |
+
static constexpr const char kMiddlewareName[] =
|
| 46 |
+
"arrow::flight::TracingServerMiddleware";
|
| 47 |
+
|
| 48 |
+
std::string name() const override { return kMiddlewareName; }
|
| 49 |
+
void SendingHeaders(AddCallHeaders*) override;
|
| 50 |
+
void CallCompleted(const Status&) override;
|
| 51 |
+
|
| 52 |
+
struct TraceKey {
|
| 53 |
+
std::string key;
|
| 54 |
+
std::string value;
|
| 55 |
+
};
|
| 56 |
+
/// \brief Get the trace context.
|
| 57 |
+
std::vector<TraceKey> GetTraceContext() const;
|
| 58 |
+
|
| 59 |
+
private:
|
| 60 |
+
class Impl;
|
| 61 |
+
friend class TracingServerMiddlewareFactory;
|
| 62 |
+
|
| 63 |
+
explicit TracingServerMiddleware(std::unique_ptr<Impl> impl);
|
| 64 |
+
std::unique_ptr<Impl> impl_;
|
| 65 |
+
};
|
| 66 |
+
|
| 67 |
+
} // namespace flight
|
| 68 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_auth_handlers.h
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <string>
|
| 21 |
+
|
| 22 |
+
#include "arrow/flight/client_auth.h"
|
| 23 |
+
#include "arrow/flight/server.h"
|
| 24 |
+
#include "arrow/flight/server_auth.h"
|
| 25 |
+
#include "arrow/flight/types.h"
|
| 26 |
+
#include "arrow/flight/visibility.h"
|
| 27 |
+
#include "arrow/status.h"
|
| 28 |
+
|
| 29 |
+
// A pair of authentication handlers that check for a predefined password
|
| 30 |
+
// and set the peer identity to a predefined username.
|
| 31 |
+
|
| 32 |
+
namespace arrow::flight {
|
| 33 |
+
|
| 34 |
+
class ARROW_FLIGHT_EXPORT TestServerAuthHandler : public ServerAuthHandler {
|
| 35 |
+
public:
|
| 36 |
+
explicit TestServerAuthHandler(const std::string& username,
|
| 37 |
+
const std::string& password);
|
| 38 |
+
~TestServerAuthHandler() override;
|
| 39 |
+
Status Authenticate(const ServerCallContext& context, ServerAuthSender* outgoing,
|
| 40 |
+
ServerAuthReader* incoming) override;
|
| 41 |
+
Status IsValid(const ServerCallContext& context, const std::string& token,
|
| 42 |
+
std::string* peer_identity) override;
|
| 43 |
+
|
| 44 |
+
private:
|
| 45 |
+
std::string username_;
|
| 46 |
+
std::string password_;
|
| 47 |
+
};
|
| 48 |
+
|
| 49 |
+
class ARROW_FLIGHT_EXPORT TestServerBasicAuthHandler : public ServerAuthHandler {
|
| 50 |
+
public:
|
| 51 |
+
explicit TestServerBasicAuthHandler(const std::string& username,
|
| 52 |
+
const std::string& password);
|
| 53 |
+
~TestServerBasicAuthHandler() override;
|
| 54 |
+
Status Authenticate(const ServerCallContext& context, ServerAuthSender* outgoing,
|
| 55 |
+
ServerAuthReader* incoming) override;
|
| 56 |
+
Status IsValid(const ServerCallContext& context, const std::string& token,
|
| 57 |
+
std::string* peer_identity) override;
|
| 58 |
+
|
| 59 |
+
private:
|
| 60 |
+
BasicAuth basic_auth_;
|
| 61 |
+
};
|
| 62 |
+
|
| 63 |
+
class ARROW_FLIGHT_EXPORT TestClientAuthHandler : public ClientAuthHandler {
|
| 64 |
+
public:
|
| 65 |
+
explicit TestClientAuthHandler(const std::string& username,
|
| 66 |
+
const std::string& password);
|
| 67 |
+
~TestClientAuthHandler() override;
|
| 68 |
+
Status Authenticate(ClientAuthSender* outgoing, ClientAuthReader* incoming) override;
|
| 69 |
+
Status GetToken(std::string* token) override;
|
| 70 |
+
|
| 71 |
+
private:
|
| 72 |
+
std::string username_;
|
| 73 |
+
std::string password_;
|
| 74 |
+
};
|
| 75 |
+
|
| 76 |
+
class ARROW_FLIGHT_EXPORT TestClientBasicAuthHandler : public ClientAuthHandler {
|
| 77 |
+
public:
|
| 78 |
+
explicit TestClientBasicAuthHandler(const std::string& username,
|
| 79 |
+
const std::string& password);
|
| 80 |
+
~TestClientBasicAuthHandler() override;
|
| 81 |
+
Status Authenticate(ClientAuthSender* outgoing, ClientAuthReader* incoming) override;
|
| 82 |
+
Status GetToken(std::string* token) override;
|
| 83 |
+
|
| 84 |
+
private:
|
| 85 |
+
BasicAuth basic_auth_;
|
| 86 |
+
std::string token_;
|
| 87 |
+
};
|
| 88 |
+
|
| 89 |
+
} // namespace arrow::flight
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_definitions.h
ADDED
|
@@ -0,0 +1,318 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
// Common test definitions for Flight. Individual transport
|
| 19 |
+
// implementations can instantiate these tests.
|
| 20 |
+
//
|
| 21 |
+
// While Googletest's value-parameterized tests would be a more
|
| 22 |
+
// natural way to do this, they cause runtime issues on MinGW/MSVC
|
| 23 |
+
// (Googletest thinks the test suite has been defined twice).
|
| 24 |
+
|
| 25 |
+
#pragma once
|
| 26 |
+
|
| 27 |
+
#include <functional>
|
| 28 |
+
#include <memory>
|
| 29 |
+
#include <string>
|
| 30 |
+
#include <type_traits>
|
| 31 |
+
#include <vector>
|
| 32 |
+
|
| 33 |
+
#include "arrow/flight/server.h"
|
| 34 |
+
#include "arrow/flight/types.h"
|
| 35 |
+
#include "arrow/util/macros.h"
|
| 36 |
+
|
| 37 |
+
namespace arrow {
|
| 38 |
+
namespace flight {
|
| 39 |
+
|
| 40 |
+
class ARROW_FLIGHT_EXPORT FlightTest {
|
| 41 |
+
protected:
|
| 42 |
+
virtual std::string transport() const = 0;
|
| 43 |
+
virtual bool supports_async() const { return false; }
|
| 44 |
+
virtual void SetUpTest() {}
|
| 45 |
+
virtual void TearDownTest() {}
|
| 46 |
+
};
|
| 47 |
+
|
| 48 |
+
/// Common tests of startup/shutdown
|
| 49 |
+
class ARROW_FLIGHT_EXPORT ConnectivityTest : public FlightTest {
|
| 50 |
+
public:
|
| 51 |
+
// Test methods
|
| 52 |
+
void TestGetPort();
|
| 53 |
+
void TestBuilderHook();
|
| 54 |
+
void TestShutdown();
|
| 55 |
+
void TestShutdownWithDeadline();
|
| 56 |
+
void TestBrokenConnection();
|
| 57 |
+
};
|
| 58 |
+
|
| 59 |
+
#define ARROW_FLIGHT_TEST_CONNECTIVITY(FIXTURE) \
|
| 60 |
+
static_assert(std::is_base_of<ConnectivityTest, FIXTURE>::value, \
|
| 61 |
+
ARROW_STRINGIFY(FIXTURE) " must inherit from ConnectivityTest"); \
|
| 62 |
+
TEST_F(FIXTURE, GetPort) { TestGetPort(); } \
|
| 63 |
+
TEST_F(FIXTURE, BuilderHook) { TestBuilderHook(); } \
|
| 64 |
+
TEST_F(FIXTURE, Shutdown) { TestShutdown(); } \
|
| 65 |
+
TEST_F(FIXTURE, ShutdownWithDeadline) { TestShutdownWithDeadline(); } \
|
| 66 |
+
TEST_F(FIXTURE, BrokenConnection) { TestBrokenConnection(); }
|
| 67 |
+
|
| 68 |
+
/// Common tests of data plane methods
|
| 69 |
+
class ARROW_FLIGHT_EXPORT DataTest : public FlightTest {
|
| 70 |
+
public:
|
| 71 |
+
void SetUpTest() override;
|
| 72 |
+
void TearDownTest() override;
|
| 73 |
+
Status ConnectClient();
|
| 74 |
+
|
| 75 |
+
// Test methods
|
| 76 |
+
void TestDoGetInts();
|
| 77 |
+
void TestDoGetFloats();
|
| 78 |
+
void TestDoGetDicts();
|
| 79 |
+
void TestDoGetLargeBatch();
|
| 80 |
+
void TestFlightDataStreamError();
|
| 81 |
+
void TestOverflowServerBatch();
|
| 82 |
+
void TestOverflowClientBatch();
|
| 83 |
+
void TestDoExchange();
|
| 84 |
+
void TestDoExchangeNoData();
|
| 85 |
+
void TestDoExchangeWriteOnlySchema();
|
| 86 |
+
void TestDoExchangeGet();
|
| 87 |
+
void TestDoExchangePut();
|
| 88 |
+
void TestDoExchangeEcho();
|
| 89 |
+
void TestDoExchangeTotal();
|
| 90 |
+
void TestDoExchangeError();
|
| 91 |
+
void TestDoExchangeConcurrency();
|
| 92 |
+
void TestDoExchangeUndrained();
|
| 93 |
+
void TestIssue5095();
|
| 94 |
+
|
| 95 |
+
private:
|
| 96 |
+
void CheckDoGet(
|
| 97 |
+
const FlightDescriptor& descr, const RecordBatchVector& expected_batches,
|
| 98 |
+
std::function<void(const std::vector<FlightEndpoint>&)> check_endpoints);
|
| 99 |
+
void CheckDoGet(const Ticket& ticket, const RecordBatchVector& expected_batches);
|
| 100 |
+
|
| 101 |
+
std::unique_ptr<FlightClient> client_;
|
| 102 |
+
std::unique_ptr<FlightServerBase> server_;
|
| 103 |
+
};
|
| 104 |
+
|
| 105 |
+
#define ARROW_FLIGHT_TEST_DATA(FIXTURE) \
|
| 106 |
+
static_assert(std::is_base_of<DataTest, FIXTURE>::value, \
|
| 107 |
+
ARROW_STRINGIFY(FIXTURE) " must inherit from DataTest"); \
|
| 108 |
+
TEST_F(FIXTURE, TestDoGetInts) { TestDoGetInts(); } \
|
| 109 |
+
TEST_F(FIXTURE, TestDoGetFloats) { TestDoGetFloats(); } \
|
| 110 |
+
TEST_F(FIXTURE, TestDoGetDicts) { TestDoGetDicts(); } \
|
| 111 |
+
TEST_F(FIXTURE, TestDoGetLargeBatch) { TestDoGetLargeBatch(); } \
|
| 112 |
+
TEST_F(FIXTURE, TestFlightDataStreamError) { TestFlightDataStreamError(); } \
|
| 113 |
+
TEST_F(FIXTURE, TestOverflowServerBatch) { TestOverflowServerBatch(); } \
|
| 114 |
+
TEST_F(FIXTURE, TestOverflowClientBatch) { TestOverflowClientBatch(); } \
|
| 115 |
+
TEST_F(FIXTURE, TestDoExchange) { TestDoExchange(); } \
|
| 116 |
+
TEST_F(FIXTURE, TestDoExchangeNoData) { TestDoExchangeNoData(); } \
|
| 117 |
+
TEST_F(FIXTURE, TestDoExchangeWriteOnlySchema) { TestDoExchangeWriteOnlySchema(); } \
|
| 118 |
+
TEST_F(FIXTURE, TestDoExchangeGet) { TestDoExchangeGet(); } \
|
| 119 |
+
TEST_F(FIXTURE, TestDoExchangePut) { TestDoExchangePut(); } \
|
| 120 |
+
TEST_F(FIXTURE, TestDoExchangeEcho) { TestDoExchangeEcho(); } \
|
| 121 |
+
TEST_F(FIXTURE, TestDoExchangeTotal) { TestDoExchangeTotal(); } \
|
| 122 |
+
TEST_F(FIXTURE, TestDoExchangeError) { TestDoExchangeError(); } \
|
| 123 |
+
TEST_F(FIXTURE, TestDoExchangeConcurrency) { TestDoExchangeConcurrency(); } \
|
| 124 |
+
TEST_F(FIXTURE, TestDoExchangeUndrained) { TestDoExchangeUndrained(); } \
|
| 125 |
+
TEST_F(FIXTURE, TestIssue5095) { TestIssue5095(); }
|
| 126 |
+
|
| 127 |
+
/// \brief Specific tests of DoPut.
|
| 128 |
+
class ARROW_FLIGHT_EXPORT DoPutTest : public FlightTest {
|
| 129 |
+
public:
|
| 130 |
+
void SetUpTest() override;
|
| 131 |
+
void TearDownTest() override;
|
| 132 |
+
void CheckBatches(const FlightDescriptor& expected_descriptor,
|
| 133 |
+
const RecordBatchVector& expected_batches);
|
| 134 |
+
void CheckDoPut(const FlightDescriptor& descr, const std::shared_ptr<Schema>& schema,
|
| 135 |
+
const RecordBatchVector& batches);
|
| 136 |
+
|
| 137 |
+
// Test methods
|
| 138 |
+
void TestInts();
|
| 139 |
+
void TestFloats();
|
| 140 |
+
void TestEmptyBatch();
|
| 141 |
+
void TestDicts();
|
| 142 |
+
void TestLargeBatch();
|
| 143 |
+
void TestSizeLimit();
|
| 144 |
+
void TestUndrained();
|
| 145 |
+
|
| 146 |
+
private:
|
| 147 |
+
std::unique_ptr<FlightClient> client_;
|
| 148 |
+
std::unique_ptr<FlightServerBase> server_;
|
| 149 |
+
};
|
| 150 |
+
|
| 151 |
+
#define ARROW_FLIGHT_TEST_DO_PUT(FIXTURE) \
|
| 152 |
+
static_assert(std::is_base_of<DoPutTest, FIXTURE>::value, \
|
| 153 |
+
ARROW_STRINGIFY(FIXTURE) " must inherit from DoPutTest"); \
|
| 154 |
+
TEST_F(FIXTURE, TestInts) { TestInts(); } \
|
| 155 |
+
TEST_F(FIXTURE, TestFloats) { TestFloats(); } \
|
| 156 |
+
TEST_F(FIXTURE, TestEmptyBatch) { TestEmptyBatch(); } \
|
| 157 |
+
TEST_F(FIXTURE, TestDicts) { TestDicts(); } \
|
| 158 |
+
TEST_F(FIXTURE, TestLargeBatch) { TestLargeBatch(); } \
|
| 159 |
+
TEST_F(FIXTURE, TestSizeLimit) { TestSizeLimit(); } \
|
| 160 |
+
TEST_F(FIXTURE, TestUndrained) { TestUndrained(); }
|
| 161 |
+
|
| 162 |
+
class ARROW_FLIGHT_EXPORT AppMetadataTestServer : public FlightServerBase {
|
| 163 |
+
public:
|
| 164 |
+
virtual ~AppMetadataTestServer() = default;
|
| 165 |
+
|
| 166 |
+
Status DoGet(const ServerCallContext& context, const Ticket& request,
|
| 167 |
+
std::unique_ptr<FlightDataStream>* data_stream) override;
|
| 168 |
+
|
| 169 |
+
Status DoPut(const ServerCallContext& context,
|
| 170 |
+
std::unique_ptr<FlightMessageReader> reader,
|
| 171 |
+
std::unique_ptr<FlightMetadataWriter> writer) override;
|
| 172 |
+
};
|
| 173 |
+
|
| 174 |
+
/// \brief Tests of app_metadata in data plane methods.
|
| 175 |
+
class ARROW_FLIGHT_EXPORT AppMetadataTest : public FlightTest {
|
| 176 |
+
public:
|
| 177 |
+
void SetUpTest() override;
|
| 178 |
+
void TearDownTest() override;
|
| 179 |
+
|
| 180 |
+
// Test methods
|
| 181 |
+
void TestDoGet();
|
| 182 |
+
void TestDoGetDictionaries();
|
| 183 |
+
void TestDoPut();
|
| 184 |
+
void TestDoPutDictionaries();
|
| 185 |
+
void TestDoPutReadMetadata();
|
| 186 |
+
|
| 187 |
+
private:
|
| 188 |
+
std::unique_ptr<FlightClient> client_;
|
| 189 |
+
std::unique_ptr<FlightServerBase> server_;
|
| 190 |
+
};
|
| 191 |
+
|
| 192 |
+
#define ARROW_FLIGHT_TEST_APP_METADATA(FIXTURE) \
|
| 193 |
+
static_assert(std::is_base_of<AppMetadataTest, FIXTURE>::value, \
|
| 194 |
+
ARROW_STRINGIFY(FIXTURE) " must inherit from AppMetadataTest"); \
|
| 195 |
+
TEST_F(FIXTURE, TestDoGet) { TestDoGet(); } \
|
| 196 |
+
TEST_F(FIXTURE, TestDoGetDictionaries) { TestDoGetDictionaries(); } \
|
| 197 |
+
TEST_F(FIXTURE, TestDoPut) { TestDoPut(); } \
|
| 198 |
+
TEST_F(FIXTURE, TestDoPutDictionaries) { TestDoPutDictionaries(); } \
|
| 199 |
+
TEST_F(FIXTURE, TestDoPutReadMetadata) { TestDoPutReadMetadata(); }
|
| 200 |
+
|
| 201 |
+
/// \brief Tests of IPC options in data plane methods.
|
| 202 |
+
class ARROW_FLIGHT_EXPORT IpcOptionsTest : public FlightTest {
|
| 203 |
+
public:
|
| 204 |
+
void SetUpTest() override;
|
| 205 |
+
void TearDownTest() override;
|
| 206 |
+
|
| 207 |
+
// Test methods
|
| 208 |
+
void TestDoGetReadOptions();
|
| 209 |
+
void TestDoPutWriteOptions();
|
| 210 |
+
void TestDoExchangeClientWriteOptions();
|
| 211 |
+
void TestDoExchangeClientWriteOptionsBegin();
|
| 212 |
+
void TestDoExchangeServerWriteOptions();
|
| 213 |
+
|
| 214 |
+
private:
|
| 215 |
+
std::unique_ptr<FlightClient> client_;
|
| 216 |
+
std::unique_ptr<FlightServerBase> server_;
|
| 217 |
+
};
|
| 218 |
+
|
| 219 |
+
#define ARROW_FLIGHT_TEST_IPC_OPTIONS(FIXTURE) \
|
| 220 |
+
static_assert(std::is_base_of<IpcOptionsTest, FIXTURE>::value, \
|
| 221 |
+
ARROW_STRINGIFY(FIXTURE) " must inherit from IpcOptionsTest"); \
|
| 222 |
+
TEST_F(FIXTURE, TestDoGetReadOptions) { TestDoGetReadOptions(); } \
|
| 223 |
+
TEST_F(FIXTURE, TestDoPutWriteOptions) { TestDoPutWriteOptions(); } \
|
| 224 |
+
TEST_F(FIXTURE, TestDoExchangeClientWriteOptions) { \
|
| 225 |
+
TestDoExchangeClientWriteOptions(); \
|
| 226 |
+
} \
|
| 227 |
+
TEST_F(FIXTURE, TestDoExchangeClientWriteOptionsBegin) { \
|
| 228 |
+
TestDoExchangeClientWriteOptionsBegin(); \
|
| 229 |
+
} \
|
| 230 |
+
TEST_F(FIXTURE, TestDoExchangeServerWriteOptions) { \
|
| 231 |
+
TestDoExchangeServerWriteOptions(); \
|
| 232 |
+
}
|
| 233 |
+
|
| 234 |
+
/// \brief Tests of data plane methods with CUDA memory.
|
| 235 |
+
///
|
| 236 |
+
/// If not built with ARROW_CUDA, tests are no-ops.
|
| 237 |
+
class ARROW_FLIGHT_EXPORT CudaDataTest : public FlightTest {
|
| 238 |
+
public:
|
| 239 |
+
void SetUpTest() override;
|
| 240 |
+
void TearDownTest() override;
|
| 241 |
+
|
| 242 |
+
// Test methods
|
| 243 |
+
void TestDoGet();
|
| 244 |
+
void TestDoPut();
|
| 245 |
+
void TestDoExchange();
|
| 246 |
+
|
| 247 |
+
private:
|
| 248 |
+
class Impl;
|
| 249 |
+
std::unique_ptr<FlightClient> client_;
|
| 250 |
+
std::unique_ptr<FlightServerBase> server_;
|
| 251 |
+
std::shared_ptr<Impl> impl_;
|
| 252 |
+
};
|
| 253 |
+
|
| 254 |
+
#define ARROW_FLIGHT_TEST_CUDA_DATA(FIXTURE) \
|
| 255 |
+
static_assert(std::is_base_of<CudaDataTest, FIXTURE>::value, \
|
| 256 |
+
ARROW_STRINGIFY(FIXTURE) " must inherit from CudaDataTest"); \
|
| 257 |
+
TEST_F(FIXTURE, TestDoGet) { TestDoGet(); } \
|
| 258 |
+
TEST_F(FIXTURE, TestDoPut) { TestDoPut(); } \
|
| 259 |
+
TEST_F(FIXTURE, TestDoExchange) { TestDoExchange(); }
|
| 260 |
+
|
| 261 |
+
/// \brief Tests of error handling.
|
| 262 |
+
class ARROW_FLIGHT_EXPORT ErrorHandlingTest : public FlightTest {
|
| 263 |
+
public:
|
| 264 |
+
void SetUpTest() override;
|
| 265 |
+
void TearDownTest() override;
|
| 266 |
+
|
| 267 |
+
// Test methods
|
| 268 |
+
void TestGetFlightInfo();
|
| 269 |
+
void TestGetFlightInfoMetadata();
|
| 270 |
+
void TestAsyncGetFlightInfo();
|
| 271 |
+
void TestDoPut();
|
| 272 |
+
void TestDoExchange();
|
| 273 |
+
|
| 274 |
+
protected:
|
| 275 |
+
struct Impl;
|
| 276 |
+
|
| 277 |
+
std::vector<std::pair<std::string, std::string>> GetHeaders();
|
| 278 |
+
|
| 279 |
+
std::shared_ptr<Impl> impl_;
|
| 280 |
+
std::unique_ptr<FlightClient> client_;
|
| 281 |
+
std::unique_ptr<FlightServerBase> server_;
|
| 282 |
+
};
|
| 283 |
+
|
| 284 |
+
#define ARROW_FLIGHT_TEST_ERROR_HANDLING(FIXTURE) \
|
| 285 |
+
static_assert(std::is_base_of<ErrorHandlingTest, FIXTURE>::value, \
|
| 286 |
+
ARROW_STRINGIFY(FIXTURE) " must inherit from ErrorHandlingTest"); \
|
| 287 |
+
TEST_F(FIXTURE, TestAsyncGetFlightInfo) { TestAsyncGetFlightInfo(); } \
|
| 288 |
+
TEST_F(FIXTURE, TestGetFlightInfo) { TestGetFlightInfo(); } \
|
| 289 |
+
TEST_F(FIXTURE, TestGetFlightInfoMetadata) { TestGetFlightInfoMetadata(); } \
|
| 290 |
+
TEST_F(FIXTURE, TestDoPut) { TestDoPut(); } \
|
| 291 |
+
TEST_F(FIXTURE, TestDoExchange) { TestDoExchange(); }
|
| 292 |
+
|
| 293 |
+
/// \brief Tests of the async client.
|
| 294 |
+
class ARROW_FLIGHT_EXPORT AsyncClientTest : public FlightTest {
|
| 295 |
+
public:
|
| 296 |
+
void SetUpTest() override;
|
| 297 |
+
void TearDownTest() override;
|
| 298 |
+
|
| 299 |
+
// Test methods
|
| 300 |
+
void TestGetFlightInfo();
|
| 301 |
+
void TestGetFlightInfoFuture();
|
| 302 |
+
void TestListenerLifetime();
|
| 303 |
+
|
| 304 |
+
private:
|
| 305 |
+
std::unique_ptr<FlightClient> client_;
|
| 306 |
+
std::unique_ptr<FlightServerBase> server_;
|
| 307 |
+
};
|
| 308 |
+
|
| 309 |
+
// DISABLED TestListenerLifetime: https://github.com/apache/arrow/issues/45120
|
| 310 |
+
#define ARROW_FLIGHT_TEST_ASYNC_CLIENT(FIXTURE) \
|
| 311 |
+
static_assert(std::is_base_of<AsyncClientTest, FIXTURE>::value, \
|
| 312 |
+
ARROW_STRINGIFY(FIXTURE) " must inherit from AsyncClientTest"); \
|
| 313 |
+
TEST_F(FIXTURE, TestGetFlightInfo) { TestGetFlightInfo(); } \
|
| 314 |
+
TEST_F(FIXTURE, TestGetFlightInfoFuture) { TestGetFlightInfoFuture(); } \
|
| 315 |
+
TEST_F(FIXTURE, DISABLED_TestListenerLifetime) { TestListenerLifetime(); }
|
| 316 |
+
|
| 317 |
+
} // namespace flight
|
| 318 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_flight_server.h
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <memory>
|
| 21 |
+
|
| 22 |
+
#include "arrow/flight/server.h"
|
| 23 |
+
#include "arrow/flight/type_fwd.h"
|
| 24 |
+
#include "arrow/flight/visibility.h"
|
| 25 |
+
#include "arrow/status.h"
|
| 26 |
+
|
| 27 |
+
namespace arrow::flight {
|
| 28 |
+
|
| 29 |
+
class ARROW_FLIGHT_EXPORT TestFlightServer : public FlightServerBase {
|
| 30 |
+
public:
|
| 31 |
+
static std::unique_ptr<FlightServerBase> Make();
|
| 32 |
+
|
| 33 |
+
Status ListFlights(const ServerCallContext& context, const Criteria* criteria,
|
| 34 |
+
std::unique_ptr<FlightListing>* listings) override;
|
| 35 |
+
|
| 36 |
+
Status GetFlightInfo(const ServerCallContext& context, const FlightDescriptor& request,
|
| 37 |
+
std::unique_ptr<FlightInfo>* out) override;
|
| 38 |
+
|
| 39 |
+
Status DoGet(const ServerCallContext& context, const Ticket& request,
|
| 40 |
+
std::unique_ptr<FlightDataStream>* data_stream) override;
|
| 41 |
+
|
| 42 |
+
Status DoPut(const ServerCallContext&, std::unique_ptr<FlightMessageReader> reader,
|
| 43 |
+
std::unique_ptr<FlightMetadataWriter> writer) override;
|
| 44 |
+
|
| 45 |
+
Status DoExchange(const ServerCallContext& context,
|
| 46 |
+
std::unique_ptr<FlightMessageReader> reader,
|
| 47 |
+
std::unique_ptr<FlightMessageWriter> writer) override;
|
| 48 |
+
|
| 49 |
+
// A simple example - act like DoGet.
|
| 50 |
+
Status RunExchangeGet(std::unique_ptr<FlightMessageReader> reader,
|
| 51 |
+
std::unique_ptr<FlightMessageWriter> writer);
|
| 52 |
+
|
| 53 |
+
// A simple example - act like DoPut
|
| 54 |
+
Status RunExchangePut(std::unique_ptr<FlightMessageReader> reader,
|
| 55 |
+
std::unique_ptr<FlightMessageWriter> writer);
|
| 56 |
+
|
| 57 |
+
// Read some number of record batches from the client, send a
|
| 58 |
+
// metadata message back with the count, then echo the batches back.
|
| 59 |
+
Status RunExchangeCounter(std::unique_ptr<FlightMessageReader> reader,
|
| 60 |
+
std::unique_ptr<FlightMessageWriter> writer);
|
| 61 |
+
|
| 62 |
+
// Read int64 batches from the client, each time sending back a
|
| 63 |
+
// batch with a running sum of columns.
|
| 64 |
+
Status RunExchangeTotal(std::unique_ptr<FlightMessageReader> reader,
|
| 65 |
+
std::unique_ptr<FlightMessageWriter> writer);
|
| 66 |
+
|
| 67 |
+
// Echo the client's messages back.
|
| 68 |
+
Status RunExchangeEcho(std::unique_ptr<FlightMessageReader> reader,
|
| 69 |
+
std::unique_ptr<FlightMessageWriter> writer);
|
| 70 |
+
|
| 71 |
+
// Regression test for ARROW-13253
|
| 72 |
+
Status RunExchangeLargeBatch(std::unique_ptr<FlightMessageReader>,
|
| 73 |
+
std::unique_ptr<FlightMessageWriter> writer);
|
| 74 |
+
|
| 75 |
+
Status RunAction1(const Action& action, std::unique_ptr<ResultStream>* out);
|
| 76 |
+
|
| 77 |
+
Status RunAction2(std::unique_ptr<ResultStream>* out);
|
| 78 |
+
|
| 79 |
+
Status ListIncomingHeaders(const ServerCallContext& context, const Action& action,
|
| 80 |
+
std::unique_ptr<ResultStream>* out);
|
| 81 |
+
|
| 82 |
+
Status DoAction(const ServerCallContext& context, const Action& action,
|
| 83 |
+
std::unique_ptr<ResultStream>* out) override;
|
| 84 |
+
|
| 85 |
+
Status ListActions(const ServerCallContext& context,
|
| 86 |
+
std::vector<ActionType>* out) override;
|
| 87 |
+
|
| 88 |
+
Status GetSchema(const ServerCallContext& context, const FlightDescriptor& request,
|
| 89 |
+
std::unique_ptr<SchemaResult>* schema) override;
|
| 90 |
+
};
|
| 91 |
+
|
| 92 |
+
} // namespace arrow::flight
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_util.h
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <gtest/gtest.h>
|
| 21 |
+
|
| 22 |
+
#include <cstdint>
|
| 23 |
+
#include <memory>
|
| 24 |
+
#include <string>
|
| 25 |
+
#include <thread>
|
| 26 |
+
#include <utility>
|
| 27 |
+
#include <vector>
|
| 28 |
+
|
| 29 |
+
#include "arrow/status.h"
|
| 30 |
+
#include "arrow/testing/gtest_util.h"
|
| 31 |
+
#include "arrow/testing/process.h"
|
| 32 |
+
#include "arrow/testing/util.h"
|
| 33 |
+
|
| 34 |
+
#include "arrow/flight/client.h"
|
| 35 |
+
#include "arrow/flight/server.h"
|
| 36 |
+
#include "arrow/flight/types.h"
|
| 37 |
+
#include "arrow/flight/visibility.h"
|
| 38 |
+
|
| 39 |
+
namespace arrow {
|
| 40 |
+
namespace flight {
|
| 41 |
+
|
| 42 |
+
// ----------------------------------------------------------------------
|
| 43 |
+
// Helpers to compare values for equality
|
| 44 |
+
ARROW_FLIGHT_EXPORT
|
| 45 |
+
void AssertEqual(const FlightInfo& expected, const FlightInfo& actual);
|
| 46 |
+
|
| 47 |
+
// ----------------------------------------------------------------------
|
| 48 |
+
// Fixture to use for running test servers
|
| 49 |
+
|
| 50 |
+
class ARROW_FLIGHT_EXPORT TestServer {
|
| 51 |
+
public:
|
| 52 |
+
explicit TestServer(const std::string& executable_name)
|
| 53 |
+
: executable_name_(executable_name), port_(::arrow::GetListenPort()) {}
|
| 54 |
+
TestServer(const std::string& executable_name, int port)
|
| 55 |
+
: executable_name_(executable_name), port_(port) {}
|
| 56 |
+
TestServer(const std::string& executable_name, const std::string& unix_sock)
|
| 57 |
+
: executable_name_(executable_name), unix_sock_(unix_sock) {}
|
| 58 |
+
|
| 59 |
+
Status Start(const std::vector<std::string>& extra_args);
|
| 60 |
+
Status Start() { return Start({}); }
|
| 61 |
+
|
| 62 |
+
void Stop();
|
| 63 |
+
|
| 64 |
+
bool IsRunning();
|
| 65 |
+
|
| 66 |
+
int port() const;
|
| 67 |
+
const std::string& unix_sock() const;
|
| 68 |
+
|
| 69 |
+
private:
|
| 70 |
+
std::string executable_name_;
|
| 71 |
+
int port_;
|
| 72 |
+
std::string unix_sock_;
|
| 73 |
+
std::unique_ptr<util::Process> server_process_;
|
| 74 |
+
};
|
| 75 |
+
|
| 76 |
+
// Helper to initialize a server and matching client with callbacks to
|
| 77 |
+
// populate options.
|
| 78 |
+
template <typename T, typename... Args>
|
| 79 |
+
Status MakeServer(const Location& location, std::unique_ptr<FlightServerBase>* server,
|
| 80 |
+
std::unique_ptr<FlightClient>* client,
|
| 81 |
+
std::function<Status(FlightServerOptions*)> make_server_options,
|
| 82 |
+
std::function<Status(FlightClientOptions*)> make_client_options,
|
| 83 |
+
Args&&... server_args) {
|
| 84 |
+
*server = std::make_unique<T>(std::forward<Args>(server_args)...);
|
| 85 |
+
FlightServerOptions server_options(location);
|
| 86 |
+
RETURN_NOT_OK(make_server_options(&server_options));
|
| 87 |
+
RETURN_NOT_OK((*server)->Init(server_options));
|
| 88 |
+
std::string uri =
|
| 89 |
+
location.scheme() + "://127.0.0.1:" + std::to_string((*server)->port());
|
| 90 |
+
ARROW_ASSIGN_OR_RAISE(auto real_location, Location::Parse(uri));
|
| 91 |
+
FlightClientOptions client_options = FlightClientOptions::Defaults();
|
| 92 |
+
RETURN_NOT_OK(make_client_options(&client_options));
|
| 93 |
+
return FlightClient::Connect(real_location, client_options).Value(client);
|
| 94 |
+
}
|
| 95 |
+
|
| 96 |
+
// Helper to initialize a server and matching client with callbacks to
|
| 97 |
+
// populate options.
|
| 98 |
+
template <typename T, typename... Args>
|
| 99 |
+
Status MakeServer(std::unique_ptr<FlightServerBase>* server,
|
| 100 |
+
std::unique_ptr<FlightClient>* client,
|
| 101 |
+
std::function<Status(FlightServerOptions*)> make_server_options,
|
| 102 |
+
std::function<Status(FlightClientOptions*)> make_client_options,
|
| 103 |
+
Args&&... server_args) {
|
| 104 |
+
ARROW_ASSIGN_OR_RAISE(auto location, Location::ForGrpcTcp("localhost", 0));
|
| 105 |
+
return MakeServer<T>(location, server, client, std::move(make_server_options),
|
| 106 |
+
std::move(make_client_options),
|
| 107 |
+
std::forward<Args>(server_args)...);
|
| 108 |
+
}
|
| 109 |
+
|
| 110 |
+
// ----------------------------------------------------------------------
|
| 111 |
+
// A FlightDataStream that numbers the record batches
|
| 112 |
+
/// \brief A basic implementation of FlightDataStream that will provide
|
| 113 |
+
/// a sequence of FlightData messages to be written to a stream
|
| 114 |
+
class ARROW_FLIGHT_EXPORT NumberingStream : public FlightDataStream {
|
| 115 |
+
public:
|
| 116 |
+
explicit NumberingStream(std::unique_ptr<FlightDataStream> stream);
|
| 117 |
+
|
| 118 |
+
std::shared_ptr<Schema> schema() override;
|
| 119 |
+
arrow::Result<FlightPayload> GetSchemaPayload() override;
|
| 120 |
+
arrow::Result<FlightPayload> Next() override;
|
| 121 |
+
|
| 122 |
+
private:
|
| 123 |
+
int counter_;
|
| 124 |
+
std::shared_ptr<FlightDataStream> stream_;
|
| 125 |
+
};
|
| 126 |
+
|
| 127 |
+
// ----------------------------------------------------------------------
|
| 128 |
+
// Example data for test-server and unit tests
|
| 129 |
+
|
| 130 |
+
ARROW_FLIGHT_EXPORT
|
| 131 |
+
std::shared_ptr<Schema> ExampleIntSchema();
|
| 132 |
+
|
| 133 |
+
ARROW_FLIGHT_EXPORT
|
| 134 |
+
std::shared_ptr<Schema> ExampleFloatSchema();
|
| 135 |
+
|
| 136 |
+
ARROW_FLIGHT_EXPORT
|
| 137 |
+
std::shared_ptr<Schema> ExampleStringSchema();
|
| 138 |
+
|
| 139 |
+
ARROW_FLIGHT_EXPORT
|
| 140 |
+
std::shared_ptr<Schema> ExampleDictSchema();
|
| 141 |
+
|
| 142 |
+
ARROW_FLIGHT_EXPORT
|
| 143 |
+
std::shared_ptr<Schema> ExampleLargeSchema();
|
| 144 |
+
|
| 145 |
+
ARROW_FLIGHT_EXPORT
|
| 146 |
+
Status ExampleIntBatches(RecordBatchVector* out);
|
| 147 |
+
|
| 148 |
+
ARROW_FLIGHT_EXPORT
|
| 149 |
+
Status ExampleFloatBatches(RecordBatchVector* out);
|
| 150 |
+
|
| 151 |
+
ARROW_FLIGHT_EXPORT
|
| 152 |
+
Status ExampleDictBatches(RecordBatchVector* out);
|
| 153 |
+
|
| 154 |
+
ARROW_FLIGHT_EXPORT
|
| 155 |
+
Status ExampleNestedBatches(RecordBatchVector* out);
|
| 156 |
+
|
| 157 |
+
ARROW_FLIGHT_EXPORT
|
| 158 |
+
Status ExampleLargeBatches(RecordBatchVector* out);
|
| 159 |
+
|
| 160 |
+
ARROW_FLIGHT_EXPORT
|
| 161 |
+
arrow::Result<std::shared_ptr<RecordBatch>> VeryLargeBatch();
|
| 162 |
+
|
| 163 |
+
ARROW_FLIGHT_EXPORT
|
| 164 |
+
std::vector<FlightInfo> ExampleFlightInfo();
|
| 165 |
+
|
| 166 |
+
ARROW_FLIGHT_EXPORT
|
| 167 |
+
std::vector<ActionType> ExampleActionTypes();
|
| 168 |
+
|
| 169 |
+
ARROW_FLIGHT_EXPORT
|
| 170 |
+
FlightInfo MakeFlightInfo(const Schema& schema, const FlightDescriptor& descriptor,
|
| 171 |
+
const std::vector<FlightEndpoint>& endpoints,
|
| 172 |
+
int64_t total_records, int64_t total_bytes, bool ordered,
|
| 173 |
+
std::string app_metadata);
|
| 174 |
+
|
| 175 |
+
ARROW_FLIGHT_EXPORT
|
| 176 |
+
FlightInfo MakeFlightInfo(const FlightDescriptor& descriptor,
|
| 177 |
+
const std::vector<FlightEndpoint>& endpoints,
|
| 178 |
+
int64_t total_records, int64_t total_bytes, bool ordered,
|
| 179 |
+
std::string app_metadata);
|
| 180 |
+
|
| 181 |
+
ARROW_FLIGHT_EXPORT
|
| 182 |
+
Status ExampleTlsCertificates(std::vector<CertKeyPair>* out);
|
| 183 |
+
|
| 184 |
+
ARROW_FLIGHT_EXPORT
|
| 185 |
+
Status ExampleTlsCertificateRoot(CertKeyPair* out);
|
| 186 |
+
|
| 187 |
+
} // namespace flight
|
| 188 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/transport.h
ADDED
|
@@ -0,0 +1,298 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
/// \file
|
| 19 |
+
/// Internal (but not private) interface for implementing
|
| 20 |
+
/// alternate network transports in Flight.
|
| 21 |
+
///
|
| 22 |
+
/// To implement a transport, implement ServerTransport and
|
| 23 |
+
/// ClientTransport, and register the desired URI schemes with
|
| 24 |
+
/// TransportRegistry. Flight takes care of most of the per-RPC
|
| 25 |
+
/// details; transports only handle connections and providing a I/O
|
| 26 |
+
/// stream implementation (TransportDataStream).
|
| 27 |
+
///
|
| 28 |
+
/// On the server side:
|
| 29 |
+
///
|
| 30 |
+
/// 1. Applications subclass FlightServerBase and override RPC handlers.
|
| 31 |
+
/// 2. FlightServerBase::Init will look up and create a ServerTransport
|
| 32 |
+
/// based on the scheme of the Location given to it.
|
| 33 |
+
/// 3. The ServerTransport will start the actual server. (For instance,
|
| 34 |
+
/// for gRPC, it creates a gRPC server and registers a gRPC service.)
|
| 35 |
+
/// That server will handle connections.
|
| 36 |
+
/// 4. The transport should forward incoming calls to the server to the RPC
|
| 37 |
+
/// handlers defined on ServerTransport, which implements the actual
|
| 38 |
+
/// RPC handler using the interfaces here. Any I/O the RPC handler needs
|
| 39 |
+
/// to do is managed by transport-specific implementations of
|
| 40 |
+
/// TransportDataStream.
|
| 41 |
+
/// 5. ServerTransport calls FlightServerBase for the actual application
|
| 42 |
+
/// logic.
|
| 43 |
+
///
|
| 44 |
+
/// On the client side:
|
| 45 |
+
///
|
| 46 |
+
/// 1. Applications create a FlightClient with a Location.
|
| 47 |
+
/// 2. FlightClient will look up and create a ClientTransport based on
|
| 48 |
+
/// the scheme of the Location given to it.
|
| 49 |
+
/// 3. When calling a method on FlightClient, FlightClient will delegate to
|
| 50 |
+
/// the ClientTransport. There is some indirection, e.g. for DoGet,
|
| 51 |
+
/// FlightClient only requests that the ClientTransport start the
|
| 52 |
+
/// call and provide it with an I/O stream. The "Flight implementation"
|
| 53 |
+
/// itself still lives in FlightClient.
|
| 54 |
+
|
| 55 |
+
#pragma once
|
| 56 |
+
|
| 57 |
+
#include <functional>
|
| 58 |
+
#include <memory>
|
| 59 |
+
#include <optional>
|
| 60 |
+
#include <string>
|
| 61 |
+
#include <utility>
|
| 62 |
+
#include <vector>
|
| 63 |
+
|
| 64 |
+
#include "arrow/flight/type_fwd.h"
|
| 65 |
+
#include "arrow/flight/types.h"
|
| 66 |
+
#include "arrow/flight/visibility.h"
|
| 67 |
+
#include "arrow/ipc/options.h"
|
| 68 |
+
#include "arrow/type_fwd.h"
|
| 69 |
+
|
| 70 |
+
namespace arrow {
|
| 71 |
+
namespace ipc {
|
| 72 |
+
class Message;
|
| 73 |
+
}
|
| 74 |
+
namespace flight {
|
| 75 |
+
class FlightStatusDetail;
|
| 76 |
+
namespace internal {
|
| 77 |
+
|
| 78 |
+
/// Internal, not user-visible type used for memory-efficient reads
|
| 79 |
+
struct ARROW_FLIGHT_EXPORT FlightData {
|
| 80 |
+
/// Used only for puts, may be null
|
| 81 |
+
std::unique_ptr<FlightDescriptor> descriptor;
|
| 82 |
+
|
| 83 |
+
/// Non-length-prefixed Message header as described in format/Message.fbs
|
| 84 |
+
std::shared_ptr<Buffer> metadata;
|
| 85 |
+
|
| 86 |
+
/// Application-defined metadata
|
| 87 |
+
std::shared_ptr<Buffer> app_metadata;
|
| 88 |
+
|
| 89 |
+
/// Message body
|
| 90 |
+
std::shared_ptr<Buffer> body;
|
| 91 |
+
|
| 92 |
+
/// Open IPC message from the metadata and body
|
| 93 |
+
::arrow::Result<std::unique_ptr<ipc::Message>> OpenMessage();
|
| 94 |
+
};
|
| 95 |
+
|
| 96 |
+
/// \brief A transport-specific interface for reading/writing Arrow data.
|
| 97 |
+
///
|
| 98 |
+
/// New transports will implement this to read/write IPC payloads to
|
| 99 |
+
/// the underlying stream.
|
| 100 |
+
class ARROW_FLIGHT_EXPORT TransportDataStream {
|
| 101 |
+
public:
|
| 102 |
+
virtual ~TransportDataStream() = default;
|
| 103 |
+
/// \brief Attempt to read the next FlightData message.
|
| 104 |
+
///
|
| 105 |
+
/// \return success true if data was populated, false if there was
|
| 106 |
+
/// an error. For clients, the error can be retrieved from
|
| 107 |
+
/// Finish(Status).
|
| 108 |
+
virtual bool ReadData(FlightData* data);
|
| 109 |
+
/// \brief Attempt to write a FlightPayload.
|
| 110 |
+
///
|
| 111 |
+
/// \param[in] payload The data to write.
|
| 112 |
+
/// \return true if the message was accepted by the transport, false
|
| 113 |
+
/// if not (e.g. due to client/server disconnect), Status if there
|
| 114 |
+
/// was an error (e.g. with the payload itself).
|
| 115 |
+
virtual arrow::Result<bool> WriteData(const FlightPayload& payload);
|
| 116 |
+
/// \brief Indicate that there are no more writes on this stream.
|
| 117 |
+
///
|
| 118 |
+
/// This is only a hint for the underlying transport and may not
|
| 119 |
+
/// actually do anything.
|
| 120 |
+
virtual Status WritesDone();
|
| 121 |
+
};
|
| 122 |
+
|
| 123 |
+
/// \brief A transport-specific interface for reading/writing Arrow
|
| 124 |
+
/// data for a client.
|
| 125 |
+
class ARROW_FLIGHT_EXPORT ClientDataStream : public TransportDataStream {
|
| 126 |
+
public:
|
| 127 |
+
/// \brief Attempt to read a non-data message.
|
| 128 |
+
///
|
| 129 |
+
/// Only implemented for DoPut; mutually exclusive with
|
| 130 |
+
/// ReadData(FlightData*).
|
| 131 |
+
virtual bool ReadPutMetadata(std::shared_ptr<Buffer>* out);
|
| 132 |
+
/// \brief Attempt to cancel the call.
|
| 133 |
+
///
|
| 134 |
+
/// This is only a hint and may not take effect immediately. The
|
| 135 |
+
/// client should still finish the call with Finish(Status) as usual.
|
| 136 |
+
virtual void TryCancel() {}
|
| 137 |
+
/// \brief Finish the call, reporting the server-sent status and/or
|
| 138 |
+
/// any client-side errors as appropriate.
|
| 139 |
+
///
|
| 140 |
+
/// Implies WritesDone() and DoFinish().
|
| 141 |
+
///
|
| 142 |
+
/// \param[in] st A client-side status to combine with the
|
| 143 |
+
/// server-side error. That is, if an error occurs on the
|
| 144 |
+
/// client-side, call Finish(Status) to finish the server-side
|
| 145 |
+
/// call, get the server-side status, and merge the statuses
|
| 146 |
+
/// together so context is not lost.
|
| 147 |
+
Status Finish(Status st);
|
| 148 |
+
|
| 149 |
+
protected:
|
| 150 |
+
/// \brief End the call, returning the final server status.
|
| 151 |
+
///
|
| 152 |
+
/// For implementors: should imply WritesDone() (even if it does not
|
| 153 |
+
/// directly call it).
|
| 154 |
+
///
|
| 155 |
+
/// Implies WritesDone().
|
| 156 |
+
virtual Status DoFinish() = 0;
|
| 157 |
+
};
|
| 158 |
+
|
| 159 |
+
/// An implementation of a Flight client for a particular transport.
|
| 160 |
+
///
|
| 161 |
+
/// Transports should override the methods they are capable of
|
| 162 |
+
/// supporting. The default method implementations return an error.
|
| 163 |
+
class ARROW_FLIGHT_EXPORT ClientTransport {
|
| 164 |
+
public:
|
| 165 |
+
virtual ~ClientTransport() = default;
|
| 166 |
+
|
| 167 |
+
/// Initialize the client.
|
| 168 |
+
virtual Status Init(const FlightClientOptions& options, const Location& location,
|
| 169 |
+
const arrow::util::Uri& uri) = 0;
|
| 170 |
+
/// Close the client. Once this returns, the client is no longer usable.
|
| 171 |
+
virtual Status Close() = 0;
|
| 172 |
+
|
| 173 |
+
virtual Status Authenticate(const FlightCallOptions& options,
|
| 174 |
+
std::unique_ptr<ClientAuthHandler> auth_handler);
|
| 175 |
+
virtual arrow::Result<std::pair<std::string, std::string>> AuthenticateBasicToken(
|
| 176 |
+
const FlightCallOptions& options, const std::string& username,
|
| 177 |
+
const std::string& password);
|
| 178 |
+
virtual Status DoAction(const FlightCallOptions& options, const Action& action,
|
| 179 |
+
std::unique_ptr<ResultStream>* results);
|
| 180 |
+
virtual Status ListActions(const FlightCallOptions& options,
|
| 181 |
+
std::vector<ActionType>* actions);
|
| 182 |
+
virtual Status GetFlightInfo(const FlightCallOptions& options,
|
| 183 |
+
const FlightDescriptor& descriptor,
|
| 184 |
+
std::unique_ptr<FlightInfo>* info);
|
| 185 |
+
virtual void GetFlightInfoAsync(const FlightCallOptions& options,
|
| 186 |
+
const FlightDescriptor& descriptor,
|
| 187 |
+
std::shared_ptr<AsyncListener<FlightInfo>> listener);
|
| 188 |
+
virtual Status PollFlightInfo(const FlightCallOptions& options,
|
| 189 |
+
const FlightDescriptor& descriptor,
|
| 190 |
+
std::unique_ptr<PollInfo>* info);
|
| 191 |
+
virtual arrow::Result<std::unique_ptr<SchemaResult>> GetSchema(
|
| 192 |
+
const FlightCallOptions& options, const FlightDescriptor& descriptor);
|
| 193 |
+
virtual Status ListFlights(const FlightCallOptions& options, const Criteria& criteria,
|
| 194 |
+
std::unique_ptr<FlightListing>* listing);
|
| 195 |
+
virtual Status DoGet(const FlightCallOptions& options, const Ticket& ticket,
|
| 196 |
+
std::unique_ptr<ClientDataStream>* stream);
|
| 197 |
+
virtual Status DoPut(const FlightCallOptions& options,
|
| 198 |
+
std::unique_ptr<ClientDataStream>* stream);
|
| 199 |
+
virtual Status DoExchange(const FlightCallOptions& options,
|
| 200 |
+
std::unique_ptr<ClientDataStream>* stream);
|
| 201 |
+
|
| 202 |
+
bool supports_async() const { return CheckAsyncSupport().ok(); }
|
| 203 |
+
virtual Status CheckAsyncSupport() const {
|
| 204 |
+
return Status::NotImplemented(
|
| 205 |
+
"this Flight transport does not support async operations");
|
| 206 |
+
}
|
| 207 |
+
|
| 208 |
+
static void SetAsyncRpc(AsyncListenerBase* listener, std::unique_ptr<AsyncRpc>&& rpc);
|
| 209 |
+
static AsyncRpc* GetAsyncRpc(AsyncListenerBase* listener);
|
| 210 |
+
static std::unique_ptr<AsyncRpc> ReleaseAsyncRpc(AsyncListenerBase* listener);
|
| 211 |
+
};
|
| 212 |
+
|
| 213 |
+
/// A registry of transport implementations.
|
| 214 |
+
class ARROW_FLIGHT_EXPORT TransportRegistry {
|
| 215 |
+
public:
|
| 216 |
+
using ClientFactory = std::function<arrow::Result<std::unique_ptr<ClientTransport>>()>;
|
| 217 |
+
using ServerFactory = std::function<arrow::Result<std::unique_ptr<ServerTransport>>(
|
| 218 |
+
FlightServerBase*, std::shared_ptr<MemoryManager> memory_manager)>;
|
| 219 |
+
|
| 220 |
+
TransportRegistry();
|
| 221 |
+
~TransportRegistry();
|
| 222 |
+
|
| 223 |
+
arrow::Result<std::unique_ptr<ClientTransport>> MakeClient(
|
| 224 |
+
const std::string& scheme) const;
|
| 225 |
+
arrow::Result<std::unique_ptr<ServerTransport>> MakeServer(
|
| 226 |
+
const std::string& scheme, FlightServerBase* base,
|
| 227 |
+
std::shared_ptr<MemoryManager> memory_manager) const;
|
| 228 |
+
|
| 229 |
+
Status RegisterClient(const std::string& scheme, ClientFactory factory);
|
| 230 |
+
Status RegisterServer(const std::string& scheme, ServerFactory factory);
|
| 231 |
+
|
| 232 |
+
private:
|
| 233 |
+
class Impl;
|
| 234 |
+
std::unique_ptr<Impl> impl_;
|
| 235 |
+
};
|
| 236 |
+
|
| 237 |
+
/// \brief Get the registry of transport implementations.
|
| 238 |
+
ARROW_FLIGHT_EXPORT
|
| 239 |
+
TransportRegistry* GetDefaultTransportRegistry();
|
| 240 |
+
|
| 241 |
+
//------------------------------------------------------------
|
| 242 |
+
// Async APIs
|
| 243 |
+
|
| 244 |
+
/// \brief Transport-specific state for an async RPC.
|
| 245 |
+
///
|
| 246 |
+
/// Transport implementations may subclass this to store their own
|
| 247 |
+
/// state, and stash an instance in a user-supplied AsyncListener via
|
| 248 |
+
/// ClientTransport::GetAsyncRpc and ClientTransport::SetAsyncRpc.
|
| 249 |
+
class ARROW_FLIGHT_EXPORT AsyncRpc {
|
| 250 |
+
public:
|
| 251 |
+
virtual ~AsyncRpc() = default;
|
| 252 |
+
/// \brief Request cancellation of the RPC.
|
| 253 |
+
virtual void TryCancel() {}
|
| 254 |
+
|
| 255 |
+
/// Only needed for DoPut/DoExchange
|
| 256 |
+
virtual void Begin(const FlightDescriptor& descriptor, std::shared_ptr<Schema> schema) {
|
| 257 |
+
}
|
| 258 |
+
/// Only needed for DoPut/DoExchange
|
| 259 |
+
virtual void Write(arrow::flight::FlightStreamChunk chunk) {}
|
| 260 |
+
/// Only needed for DoPut/DoExchange
|
| 261 |
+
virtual void DoneWriting() {}
|
| 262 |
+
};
|
| 263 |
+
|
| 264 |
+
//------------------------------------------------------------
|
| 265 |
+
// Error propagation helpers
|
| 266 |
+
|
| 267 |
+
/// \brief Abstract error status.
|
| 268 |
+
///
|
| 269 |
+
/// Transport implementations may use side channels (e.g. HTTP
|
| 270 |
+
/// trailers) to convey additional information to reconstruct the
|
| 271 |
+
/// original C++ status for implementations that can use it.
|
| 272 |
+
struct ARROW_FLIGHT_EXPORT TransportStatus {
|
| 273 |
+
TransportStatusCode code;
|
| 274 |
+
std::string message;
|
| 275 |
+
|
| 276 |
+
/// \brief Convert a C++ status to an abstract transport status.
|
| 277 |
+
static TransportStatus FromStatus(const Status& arrow_status);
|
| 278 |
+
|
| 279 |
+
/// \brief Reconstruct a string-encoded TransportStatus.
|
| 280 |
+
static TransportStatus FromCodeStringAndMessage(const std::string& code_str,
|
| 281 |
+
std::string message);
|
| 282 |
+
|
| 283 |
+
/// \brief Convert an abstract transport status to a C++ status.
|
| 284 |
+
Status ToStatus() const;
|
| 285 |
+
};
|
| 286 |
+
|
| 287 |
+
/// \brief Convert the string representation of an Arrow status code
|
| 288 |
+
/// back to an Arrow status.
|
| 289 |
+
ARROW_FLIGHT_EXPORT
|
| 290 |
+
Status ReconstructStatus(const std::string& code_str, const Status& current_status,
|
| 291 |
+
std::optional<std::string> message,
|
| 292 |
+
std::optional<std::string> detail_message,
|
| 293 |
+
std::optional<std::string> detail_bin,
|
| 294 |
+
std::shared_ptr<FlightStatusDetail> detail);
|
| 295 |
+
|
| 296 |
+
} // namespace internal
|
| 297 |
+
} // namespace flight
|
| 298 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/transport_server.h
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <chrono>
|
| 21 |
+
#include <memory>
|
| 22 |
+
|
| 23 |
+
#include "arrow/flight/transport.h"
|
| 24 |
+
#include "arrow/flight/type_fwd.h"
|
| 25 |
+
#include "arrow/flight/visibility.h"
|
| 26 |
+
#include "arrow/type_fwd.h"
|
| 27 |
+
|
| 28 |
+
namespace arrow {
|
| 29 |
+
namespace ipc {
|
| 30 |
+
class Message;
|
| 31 |
+
}
|
| 32 |
+
namespace flight {
|
| 33 |
+
namespace internal {
|
| 34 |
+
|
| 35 |
+
/// \brief A transport-specific interface for reading/writing Arrow
|
| 36 |
+
/// data for a server.
|
| 37 |
+
class ARROW_FLIGHT_EXPORT ServerDataStream : public TransportDataStream {
|
| 38 |
+
public:
|
| 39 |
+
/// \brief Attempt to write a non-data message.
|
| 40 |
+
///
|
| 41 |
+
/// Only implemented for DoPut; mutually exclusive with
|
| 42 |
+
/// WriteData(const FlightPayload&).
|
| 43 |
+
virtual Status WritePutMetadata(const Buffer& payload);
|
| 44 |
+
};
|
| 45 |
+
|
| 46 |
+
/// \brief An implementation of a Flight server for a particular
|
| 47 |
+
/// transport.
|
| 48 |
+
///
|
| 49 |
+
/// This class (the transport implementation) implements the underlying
|
| 50 |
+
/// server and handles connections/incoming RPC calls. It should forward RPC
|
| 51 |
+
/// calls to the RPC handlers defined on this class, which work in terms of
|
| 52 |
+
/// the generic interfaces above. The RPC handlers here then forward calls
|
| 53 |
+
/// to the underlying FlightServerBase instance that contains the actual
|
| 54 |
+
/// application RPC method handlers.
|
| 55 |
+
///
|
| 56 |
+
/// Used by FlightServerBase to manage the server lifecycle.
|
| 57 |
+
class ARROW_FLIGHT_EXPORT ServerTransport {
|
| 58 |
+
public:
|
| 59 |
+
ServerTransport(FlightServerBase* base, std::shared_ptr<MemoryManager> memory_manager)
|
| 60 |
+
: base_(base), memory_manager_(std::move(memory_manager)) {}
|
| 61 |
+
virtual ~ServerTransport() = default;
|
| 62 |
+
|
| 63 |
+
/// \name Server Lifecycle Methods
|
| 64 |
+
/// Transports implement these methods to start/shutdown the underlying
|
| 65 |
+
/// server.
|
| 66 |
+
/// @{
|
| 67 |
+
/// \brief Initialize the server.
|
| 68 |
+
///
|
| 69 |
+
/// This method should launch the server in a background thread, i.e. it
|
| 70 |
+
/// should not block. Once this returns, the server should be active.
|
| 71 |
+
virtual Status Init(const FlightServerOptions& options,
|
| 72 |
+
const arrow::util::Uri& uri) = 0;
|
| 73 |
+
/// \brief Shutdown the server.
|
| 74 |
+
///
|
| 75 |
+
/// This should wait for active RPCs to finish. Once this returns, the
|
| 76 |
+
/// server is no longer listening.
|
| 77 |
+
virtual Status Shutdown() = 0;
|
| 78 |
+
/// \brief Shutdown the server with a deadline.
|
| 79 |
+
///
|
| 80 |
+
/// This should wait for active RPCs to finish, or for the deadline to
|
| 81 |
+
/// expire. Once this returns, the server is no longer listening.
|
| 82 |
+
virtual Status Shutdown(const std::chrono::system_clock::time_point& deadline) = 0;
|
| 83 |
+
/// \brief Wait for the server to shutdown (but do not shut down the server).
|
| 84 |
+
///
|
| 85 |
+
/// Once this returns, the server is no longer listening.
|
| 86 |
+
virtual Status Wait() = 0;
|
| 87 |
+
/// \brief Get the address the server is listening on, else an empty Location.
|
| 88 |
+
virtual Location location() const = 0;
|
| 89 |
+
///@}
|
| 90 |
+
|
| 91 |
+
/// \name RPC Handlers
|
| 92 |
+
/// Implementations of RPC handlers for Flight methods using the common
|
| 93 |
+
/// interfaces here. Transports should call these methods from their
|
| 94 |
+
/// server implementation to handle the actual RPC calls.
|
| 95 |
+
///@{
|
| 96 |
+
/// \brief Get the FlightServerBase.
|
| 97 |
+
///
|
| 98 |
+
/// Intended as an escape hatch for now since not all methods have been
|
| 99 |
+
/// factored into a transport-agnostic interface.
|
| 100 |
+
FlightServerBase* base() const { return base_; }
|
| 101 |
+
/// \brief Implement DoGet in terms of a transport-level stream.
|
| 102 |
+
///
|
| 103 |
+
/// \param[in] context The server context.
|
| 104 |
+
/// \param[in] request The request payload.
|
| 105 |
+
/// \param[in] stream The transport-specific data stream
|
| 106 |
+
/// implementation. Must implement WriteData(const
|
| 107 |
+
/// FlightPayload&).
|
| 108 |
+
Status DoGet(const ServerCallContext& context, const Ticket& request,
|
| 109 |
+
ServerDataStream* stream);
|
| 110 |
+
/// \brief Implement DoPut in terms of a transport-level stream.
|
| 111 |
+
///
|
| 112 |
+
/// \param[in] context The server context.
|
| 113 |
+
/// \param[in] stream The transport-specific data stream
|
| 114 |
+
/// implementation. Must implement ReadData(FlightData*)
|
| 115 |
+
/// and WritePutMetadata(const Buffer&).
|
| 116 |
+
Status DoPut(const ServerCallContext& context, ServerDataStream* stream);
|
| 117 |
+
/// \brief Implement DoExchange in terms of a transport-level stream.
|
| 118 |
+
///
|
| 119 |
+
/// \param[in] context The server context.
|
| 120 |
+
/// \param[in] stream The transport-specific data stream
|
| 121 |
+
/// implementation. Must implement ReadData(FlightData*)
|
| 122 |
+
/// and WriteData(const FlightPayload&).
|
| 123 |
+
Status DoExchange(const ServerCallContext& context, ServerDataStream* stream);
|
| 124 |
+
///@}
|
| 125 |
+
|
| 126 |
+
protected:
|
| 127 |
+
FlightServerBase* base_;
|
| 128 |
+
std::shared_ptr<MemoryManager> memory_manager_;
|
| 129 |
+
};
|
| 130 |
+
|
| 131 |
+
} // namespace internal
|
| 132 |
+
} // namespace flight
|
| 133 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/type_fwd.h
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
namespace arrow {
|
| 21 |
+
namespace internal {
|
| 22 |
+
class Uri;
|
| 23 |
+
}
|
| 24 |
+
namespace flight {
|
| 25 |
+
struct Action;
|
| 26 |
+
struct ActionType;
|
| 27 |
+
template <typename T>
|
| 28 |
+
class AsyncListener;
|
| 29 |
+
class AsyncListenerBase;
|
| 30 |
+
class AsyncRpc;
|
| 31 |
+
struct BasicAuth;
|
| 32 |
+
class ClientAuthHandler;
|
| 33 |
+
class ClientMiddleware;
|
| 34 |
+
class ClientMiddlewareFactory;
|
| 35 |
+
struct Criteria;
|
| 36 |
+
class FlightCallOptions;
|
| 37 |
+
struct FlightClientOptions;
|
| 38 |
+
struct FlightDescriptor;
|
| 39 |
+
struct FlightEndpoint;
|
| 40 |
+
class FlightInfo;
|
| 41 |
+
class PollInfo;
|
| 42 |
+
class FlightListing;
|
| 43 |
+
class FlightMetadataReader;
|
| 44 |
+
class FlightMetadataWriter;
|
| 45 |
+
struct FlightPayload;
|
| 46 |
+
class FlightServerBase;
|
| 47 |
+
class FlightServerOptions;
|
| 48 |
+
class FlightStreamReader;
|
| 49 |
+
class FlightStreamWriter;
|
| 50 |
+
struct Location;
|
| 51 |
+
struct Result;
|
| 52 |
+
class ResultStream;
|
| 53 |
+
struct SchemaResult;
|
| 54 |
+
class ServerCallContext;
|
| 55 |
+
class ServerMiddleware;
|
| 56 |
+
class ServerMiddlewareFactory;
|
| 57 |
+
struct Ticket;
|
| 58 |
+
namespace internal {
|
| 59 |
+
class AsyncRpc;
|
| 60 |
+
class ClientTransport;
|
| 61 |
+
struct FlightData;
|
| 62 |
+
class ServerTransport;
|
| 63 |
+
} // namespace internal
|
| 64 |
+
} // namespace flight
|
| 65 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/types.h
ADDED
|
@@ -0,0 +1,1314 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
// Data structure for Flight RPC.
|
| 19 |
+
|
| 20 |
+
#pragma once
|
| 21 |
+
|
| 22 |
+
#include <chrono>
|
| 23 |
+
#include <cstddef>
|
| 24 |
+
#include <cstdint>
|
| 25 |
+
#include <map>
|
| 26 |
+
#include <memory>
|
| 27 |
+
#include <optional>
|
| 28 |
+
#include <string>
|
| 29 |
+
#include <string_view>
|
| 30 |
+
#include <utility>
|
| 31 |
+
#include <variant>
|
| 32 |
+
#include <vector>
|
| 33 |
+
|
| 34 |
+
#include "arrow/buffer.h"
|
| 35 |
+
#include "arrow/flight/type_fwd.h"
|
| 36 |
+
#include "arrow/flight/visibility.h"
|
| 37 |
+
#include "arrow/ipc/options.h"
|
| 38 |
+
#include "arrow/ipc/writer.h"
|
| 39 |
+
#include "arrow/result.h"
|
| 40 |
+
#include "arrow/status.h"
|
| 41 |
+
|
| 42 |
+
namespace arrow {
|
| 43 |
+
|
| 44 |
+
class RecordBatch;
|
| 45 |
+
class Schema;
|
| 46 |
+
class Table;
|
| 47 |
+
|
| 48 |
+
namespace ipc {
|
| 49 |
+
class DictionaryMemo;
|
| 50 |
+
struct ReadStats;
|
| 51 |
+
} // namespace ipc
|
| 52 |
+
|
| 53 |
+
namespace util {
|
| 54 |
+
class Uri;
|
| 55 |
+
} // namespace util
|
| 56 |
+
|
| 57 |
+
namespace flight {
|
| 58 |
+
|
| 59 |
+
ARROW_FLIGHT_EXPORT
|
| 60 |
+
extern const char* kSchemeGrpc;
|
| 61 |
+
ARROW_FLIGHT_EXPORT
|
| 62 |
+
extern const char* kSchemeGrpcTcp;
|
| 63 |
+
ARROW_FLIGHT_EXPORT
|
| 64 |
+
extern const char* kSchemeGrpcUnix;
|
| 65 |
+
ARROW_FLIGHT_EXPORT
|
| 66 |
+
extern const char* kSchemeGrpcTls;
|
| 67 |
+
|
| 68 |
+
class FlightClient;
|
| 69 |
+
class FlightServerBase;
|
| 70 |
+
|
| 71 |
+
/// \brief A timestamp compatible with Protocol Buffer's
|
| 72 |
+
/// google.protobuf.Timestamp:
|
| 73 |
+
///
|
| 74 |
+
/// https://protobuf.dev/reference/protobuf/google.protobuf/#timestamp
|
| 75 |
+
///
|
| 76 |
+
/// > A Timestamp represents a point in time independent of any time
|
| 77 |
+
/// > zone or calendar, represented as seconds and fractions of
|
| 78 |
+
/// > seconds at nanosecond resolution in UTC Epoch time. It is
|
| 79 |
+
/// > encoded using the Proleptic Gregorian Calendar which extends the
|
| 80 |
+
/// > Gregorian calendar backwards to year one. It is encoded assuming
|
| 81 |
+
/// > all minutes are 60 seconds long, i.e. leap seconds are "smeared"
|
| 82 |
+
/// > so that no leap second table is needed for interpretation. Range
|
| 83 |
+
/// > is from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z.
|
| 84 |
+
using Timestamp =
|
| 85 |
+
std::chrono::time_point<std::chrono::system_clock, std::chrono::nanoseconds>;
|
| 86 |
+
|
| 87 |
+
/// \brief A Flight-specific status code. Used to encode some
|
| 88 |
+
/// additional status codes into an Arrow Status.
|
| 89 |
+
enum class FlightStatusCode : int8_t {
|
| 90 |
+
/// An implementation error has occurred.
|
| 91 |
+
Internal,
|
| 92 |
+
/// A request timed out.
|
| 93 |
+
TimedOut,
|
| 94 |
+
/// A request was cancelled.
|
| 95 |
+
Cancelled,
|
| 96 |
+
/// We are not authenticated to the remote service.
|
| 97 |
+
Unauthenticated,
|
| 98 |
+
/// We do not have permission to make this request.
|
| 99 |
+
Unauthorized,
|
| 100 |
+
/// The remote service cannot handle this request at the moment.
|
| 101 |
+
Unavailable,
|
| 102 |
+
/// A request failed for some other reason
|
| 103 |
+
Failed
|
| 104 |
+
};
|
| 105 |
+
|
| 106 |
+
// Silence warning
|
| 107 |
+
// "non dll-interface class RecordBatchReader used as base for dll-interface class"
|
| 108 |
+
#ifdef _MSC_VER
|
| 109 |
+
# pragma warning(push)
|
| 110 |
+
# pragma warning(disable : 4275)
|
| 111 |
+
#endif
|
| 112 |
+
|
| 113 |
+
/// \brief Flight-specific error information in a Status.
|
| 114 |
+
class ARROW_FLIGHT_EXPORT FlightStatusDetail : public arrow::StatusDetail {
|
| 115 |
+
public:
|
| 116 |
+
explicit FlightStatusDetail(FlightStatusCode code) : code_{code} {}
|
| 117 |
+
explicit FlightStatusDetail(FlightStatusCode code, std::string extra_info)
|
| 118 |
+
: code_{code}, extra_info_(std::move(extra_info)) {}
|
| 119 |
+
const char* type_id() const override;
|
| 120 |
+
std::string ToString() const override;
|
| 121 |
+
|
| 122 |
+
/// \brief Get the Flight status code.
|
| 123 |
+
FlightStatusCode code() const;
|
| 124 |
+
/// \brief Get the extra error info
|
| 125 |
+
std::string extra_info() const;
|
| 126 |
+
/// \brief Get the human-readable name of the status code.
|
| 127 |
+
std::string CodeAsString() const;
|
| 128 |
+
/// \brief Set the extra error info
|
| 129 |
+
void set_extra_info(std::string extra_info);
|
| 130 |
+
|
| 131 |
+
/// \brief Try to extract a \a FlightStatusDetail from any Arrow
|
| 132 |
+
/// status.
|
| 133 |
+
///
|
| 134 |
+
/// \return a \a FlightStatusDetail if it could be unwrapped, \a
|
| 135 |
+
/// nullptr otherwise
|
| 136 |
+
static std::shared_ptr<FlightStatusDetail> UnwrapStatus(const arrow::Status& status);
|
| 137 |
+
|
| 138 |
+
private:
|
| 139 |
+
FlightStatusCode code_;
|
| 140 |
+
std::string extra_info_;
|
| 141 |
+
};
|
| 142 |
+
|
| 143 |
+
#ifdef _MSC_VER
|
| 144 |
+
# pragma warning(pop)
|
| 145 |
+
#endif
|
| 146 |
+
|
| 147 |
+
/// \brief Make an appropriate Arrow status for the given
|
| 148 |
+
/// Flight-specific status.
|
| 149 |
+
///
|
| 150 |
+
/// \param code The Flight status code.
|
| 151 |
+
/// \param message The message for the error.
|
| 152 |
+
/// \param extra_info Optional extra binary info for the error (eg protobuf)
|
| 153 |
+
ARROW_FLIGHT_EXPORT
|
| 154 |
+
Status MakeFlightError(FlightStatusCode code, std::string message,
|
| 155 |
+
std::string extra_info = {});
|
| 156 |
+
|
| 157 |
+
/// \brief Headers sent from the client or server.
|
| 158 |
+
///
|
| 159 |
+
/// Header values are ordered.
|
| 160 |
+
using CallHeaders = std::multimap<std::string_view, std::string_view>;
|
| 161 |
+
|
| 162 |
+
/// \brief A TLS certificate plus key.
|
| 163 |
+
struct ARROW_FLIGHT_EXPORT CertKeyPair {
|
| 164 |
+
/// \brief The certificate in PEM format.
|
| 165 |
+
std::string pem_cert;
|
| 166 |
+
|
| 167 |
+
/// \brief The key in PEM format.
|
| 168 |
+
std::string pem_key;
|
| 169 |
+
};
|
| 170 |
+
|
| 171 |
+
namespace internal {
|
| 172 |
+
|
| 173 |
+
template <typename T>
|
| 174 |
+
struct remove_unique_ptr {
|
| 175 |
+
using type = T;
|
| 176 |
+
};
|
| 177 |
+
|
| 178 |
+
template <typename T>
|
| 179 |
+
struct remove_unique_ptr<std::unique_ptr<T>> {
|
| 180 |
+
using type = T;
|
| 181 |
+
};
|
| 182 |
+
|
| 183 |
+
// Base CRTP type
|
| 184 |
+
template <class T>
|
| 185 |
+
struct BaseType {
|
| 186 |
+
protected:
|
| 187 |
+
using SuperT = BaseType<T>;
|
| 188 |
+
using SelfT = typename remove_unique_ptr<T>::type;
|
| 189 |
+
|
| 190 |
+
const SelfT& self() const { return static_cast<const SelfT&>(*this); }
|
| 191 |
+
SelfT& self() { return static_cast<SelfT&>(*this); }
|
| 192 |
+
|
| 193 |
+
public:
|
| 194 |
+
BaseType() = default;
|
| 195 |
+
|
| 196 |
+
friend bool operator==(const SelfT& left, const SelfT& right) {
|
| 197 |
+
return left.Equals(right);
|
| 198 |
+
}
|
| 199 |
+
friend bool operator!=(const SelfT& left, const SelfT& right) {
|
| 200 |
+
return !left.Equals(right);
|
| 201 |
+
}
|
| 202 |
+
|
| 203 |
+
/// \brief Serialize this message to its wire-format representation.
|
| 204 |
+
inline arrow::Result<std::string> SerializeToString() const {
|
| 205 |
+
std::string out;
|
| 206 |
+
ARROW_RETURN_NOT_OK(self().SelfT::SerializeToString(&out));
|
| 207 |
+
return out;
|
| 208 |
+
}
|
| 209 |
+
|
| 210 |
+
inline static arrow::Result<T> Deserialize(std::string_view serialized) {
|
| 211 |
+
T out;
|
| 212 |
+
ARROW_RETURN_NOT_OK(SelfT::Deserialize(serialized, &out));
|
| 213 |
+
return out;
|
| 214 |
+
}
|
| 215 |
+
|
| 216 |
+
inline arrow::Result<std::shared_ptr<Buffer>> SerializeToBuffer() const {
|
| 217 |
+
std::string out;
|
| 218 |
+
ARROW_RETURN_NOT_OK(self().SelfT::SerializeToString(&out));
|
| 219 |
+
return Buffer::FromString(std::move(out));
|
| 220 |
+
}
|
| 221 |
+
};
|
| 222 |
+
|
| 223 |
+
} // namespace internal
|
| 224 |
+
|
| 225 |
+
//------------------------------------------------------------
|
| 226 |
+
// Wrapper types for Flight RPC protobuf messages
|
| 227 |
+
|
| 228 |
+
// A wrapper around arrow.flight.protocol.HandshakeRequest is not defined
|
| 229 |
+
// A wrapper around arrow.flight.protocol.HandshakeResponse is not defined
|
| 230 |
+
|
| 231 |
+
/// \brief message for simple auth
|
| 232 |
+
struct ARROW_FLIGHT_EXPORT BasicAuth : public internal::BaseType<BasicAuth> {
|
| 233 |
+
std::string username;
|
| 234 |
+
std::string password;
|
| 235 |
+
|
| 236 |
+
BasicAuth() = default;
|
| 237 |
+
BasicAuth(std::string username, std::string password)
|
| 238 |
+
: username(std::move(username)), password(std::move(password)) {}
|
| 239 |
+
|
| 240 |
+
std::string ToString() const;
|
| 241 |
+
bool Equals(const BasicAuth& other) const;
|
| 242 |
+
|
| 243 |
+
using SuperT::Deserialize;
|
| 244 |
+
using SuperT::SerializeToString;
|
| 245 |
+
|
| 246 |
+
/// \brief Serialize this message to its wire-format representation.
|
| 247 |
+
///
|
| 248 |
+
/// Use `SerializeToString()` if you want a Result-returning version.
|
| 249 |
+
arrow::Status SerializeToString(std::string* out) const;
|
| 250 |
+
|
| 251 |
+
/// \brief Deserialize this message from its wire-format representation.
|
| 252 |
+
///
|
| 253 |
+
/// Use `Deserialize(serialized)` if you want a Result-returning version.
|
| 254 |
+
static arrow::Status Deserialize(std::string_view serialized, BasicAuth* out);
|
| 255 |
+
};
|
| 256 |
+
|
| 257 |
+
// A wrapper around arrow.flight.protocol.Empty is not defined
|
| 258 |
+
|
| 259 |
+
/// \brief A type of action that can be performed with the DoAction RPC.
|
| 260 |
+
struct ARROW_FLIGHT_EXPORT ActionType : public internal::BaseType<ActionType> {
|
| 261 |
+
/// \brief The name of the action.
|
| 262 |
+
std::string type;
|
| 263 |
+
|
| 264 |
+
/// \brief A human-readable description of the action.
|
| 265 |
+
std::string description;
|
| 266 |
+
|
| 267 |
+
ActionType() = default;
|
| 268 |
+
|
| 269 |
+
ActionType(std::string type, std::string description)
|
| 270 |
+
: type(std::move(type)), description(std::move(description)) {}
|
| 271 |
+
|
| 272 |
+
std::string ToString() const;
|
| 273 |
+
bool Equals(const ActionType& other) const;
|
| 274 |
+
|
| 275 |
+
using SuperT::Deserialize;
|
| 276 |
+
using SuperT::SerializeToString;
|
| 277 |
+
|
| 278 |
+
/// \brief Serialize this message to its wire-format representation.
|
| 279 |
+
///
|
| 280 |
+
/// Use `SerializeToString()` if you want a Result-returning version.
|
| 281 |
+
arrow::Status SerializeToString(std::string* out) const;
|
| 282 |
+
|
| 283 |
+
/// \brief Deserialize this message from its wire-format representation.
|
| 284 |
+
///
|
| 285 |
+
/// Use `Deserialize(serialized)` if you want a Result-returning version.
|
| 286 |
+
static arrow::Status Deserialize(std::string_view serialized, ActionType* out);
|
| 287 |
+
|
| 288 |
+
static const ActionType kCancelFlightInfo;
|
| 289 |
+
static const ActionType kRenewFlightEndpoint;
|
| 290 |
+
static const ActionType kSetSessionOptions;
|
| 291 |
+
static const ActionType kGetSessionOptions;
|
| 292 |
+
static const ActionType kCloseSession;
|
| 293 |
+
};
|
| 294 |
+
|
| 295 |
+
/// \brief Opaque selection criteria for ListFlights RPC
|
| 296 |
+
struct ARROW_FLIGHT_EXPORT Criteria : public internal::BaseType<Criteria> {
|
| 297 |
+
/// Opaque criteria expression, dependent on server implementation
|
| 298 |
+
std::string expression;
|
| 299 |
+
|
| 300 |
+
Criteria() = default;
|
| 301 |
+
Criteria(std::string expression) // NOLINT runtime/explicit
|
| 302 |
+
: expression(std::move(expression)) {}
|
| 303 |
+
|
| 304 |
+
std::string ToString() const;
|
| 305 |
+
bool Equals(const Criteria& other) const;
|
| 306 |
+
|
| 307 |
+
using SuperT::Deserialize;
|
| 308 |
+
using SuperT::SerializeToString;
|
| 309 |
+
|
| 310 |
+
/// \brief Serialize this message to its wire-format representation.
|
| 311 |
+
///
|
| 312 |
+
/// Use `SerializeToString()` if you want a Result-returning version.
|
| 313 |
+
arrow::Status SerializeToString(std::string* out) const;
|
| 314 |
+
|
| 315 |
+
/// \brief Deserialize this message from its wire-format representation.
|
| 316 |
+
///
|
| 317 |
+
/// Use `Deserialize(serialized)` if you want a Result-returning version.
|
| 318 |
+
static arrow::Status Deserialize(std::string_view serialized, Criteria* out);
|
| 319 |
+
};
|
| 320 |
+
|
| 321 |
+
/// \brief An action to perform with the DoAction RPC
|
| 322 |
+
struct ARROW_FLIGHT_EXPORT Action : public internal::BaseType<Action> {
|
| 323 |
+
/// The action type
|
| 324 |
+
std::string type;
|
| 325 |
+
|
| 326 |
+
/// The action content as a Buffer
|
| 327 |
+
std::shared_ptr<Buffer> body;
|
| 328 |
+
|
| 329 |
+
Action() = default;
|
| 330 |
+
Action(std::string type, std::shared_ptr<Buffer> body)
|
| 331 |
+
: type(std::move(type)), body(std::move(body)) {}
|
| 332 |
+
|
| 333 |
+
std::string ToString() const;
|
| 334 |
+
bool Equals(const Action& other) const;
|
| 335 |
+
|
| 336 |
+
using SuperT::Deserialize;
|
| 337 |
+
using SuperT::SerializeToString;
|
| 338 |
+
|
| 339 |
+
/// \brief Serialize this message to its wire-format representation.
|
| 340 |
+
///
|
| 341 |
+
/// Use `SerializeToString()` if you want a Result-returning version.
|
| 342 |
+
arrow::Status SerializeToString(std::string* out) const;
|
| 343 |
+
|
| 344 |
+
/// \brief Deserialize this message from its wire-format representation.
|
| 345 |
+
///
|
| 346 |
+
/// Use `Deserialize(serialized)` if you want a Result-returning version.
|
| 347 |
+
static arrow::Status Deserialize(std::string_view serialized, Action* out);
|
| 348 |
+
};
|
| 349 |
+
|
| 350 |
+
/// \brief Opaque result returned after executing an action
|
| 351 |
+
struct ARROW_FLIGHT_EXPORT Result : public internal::BaseType<Result> {
|
| 352 |
+
std::shared_ptr<Buffer> body;
|
| 353 |
+
|
| 354 |
+
Result() = default;
|
| 355 |
+
Result(std::shared_ptr<Buffer> body) // NOLINT runtime/explicit
|
| 356 |
+
: body(std::move(body)) {}
|
| 357 |
+
|
| 358 |
+
std::string ToString() const;
|
| 359 |
+
bool Equals(const Result& other) const;
|
| 360 |
+
|
| 361 |
+
using SuperT::Deserialize;
|
| 362 |
+
using SuperT::SerializeToString;
|
| 363 |
+
|
| 364 |
+
/// \brief Serialize this message to its wire-format representation.
|
| 365 |
+
///
|
| 366 |
+
/// Use `SerializeToString()` if you want a Result-returning version.
|
| 367 |
+
arrow::Status SerializeToString(std::string* out) const;
|
| 368 |
+
|
| 369 |
+
/// \brief Deserialize this message from its wire-format representation.
|
| 370 |
+
///
|
| 371 |
+
/// Use `Deserialize(serialized)` if you want a Result-returning version.
|
| 372 |
+
static arrow::Status Deserialize(std::string_view serialized, Result* out);
|
| 373 |
+
};
|
| 374 |
+
|
| 375 |
+
/// \brief Schema result returned after a schema request RPC
|
| 376 |
+
struct ARROW_FLIGHT_EXPORT SchemaResult : public internal::BaseType<SchemaResult> {
|
| 377 |
+
public:
|
| 378 |
+
SchemaResult() = default;
|
| 379 |
+
explicit SchemaResult(std::string schema) : raw_schema_(std::move(schema)) {}
|
| 380 |
+
|
| 381 |
+
/// \brief Factory method to construct a SchemaResult.
|
| 382 |
+
static arrow::Result<std::unique_ptr<SchemaResult>> Make(const Schema& schema);
|
| 383 |
+
|
| 384 |
+
/// \brief return schema
|
| 385 |
+
/// \param[in,out] dictionary_memo for dictionary bookkeeping, will
|
| 386 |
+
/// be modified
|
| 387 |
+
/// \return Arrow result with the reconstructed Schema
|
| 388 |
+
arrow::Result<std::shared_ptr<Schema>> GetSchema(
|
| 389 |
+
ipc::DictionaryMemo* dictionary_memo) const;
|
| 390 |
+
|
| 391 |
+
const std::string& serialized_schema() const { return raw_schema_; }
|
| 392 |
+
|
| 393 |
+
std::string ToString() const;
|
| 394 |
+
bool Equals(const SchemaResult& other) const;
|
| 395 |
+
|
| 396 |
+
using SuperT::Deserialize;
|
| 397 |
+
using SuperT::SerializeToString;
|
| 398 |
+
|
| 399 |
+
/// \brief Serialize this message to its wire-format representation.
|
| 400 |
+
///
|
| 401 |
+
/// Use `SerializeToString()` if you want a Result-returning version.
|
| 402 |
+
arrow::Status SerializeToString(std::string* out) const;
|
| 403 |
+
|
| 404 |
+
/// \brief Deserialize this message from its wire-format representation.
|
| 405 |
+
///
|
| 406 |
+
/// Use `Deserialize(serialized)` if you want a Result-returning version.
|
| 407 |
+
static arrow::Status Deserialize(std::string_view serialized, SchemaResult* out);
|
| 408 |
+
|
| 409 |
+
private:
|
| 410 |
+
std::string raw_schema_;
|
| 411 |
+
};
|
| 412 |
+
|
| 413 |
+
/// \brief A request to retrieve or generate a dataset
|
| 414 |
+
struct ARROW_FLIGHT_EXPORT FlightDescriptor
|
| 415 |
+
: public internal::BaseType<FlightDescriptor> {
|
| 416 |
+
enum DescriptorType {
|
| 417 |
+
UNKNOWN = 0, /// Unused
|
| 418 |
+
PATH = 1, /// Named path identifying a dataset
|
| 419 |
+
CMD = 2 /// Opaque command to generate a dataset
|
| 420 |
+
};
|
| 421 |
+
|
| 422 |
+
/// The descriptor type
|
| 423 |
+
DescriptorType type = UNKNOWN;
|
| 424 |
+
|
| 425 |
+
/// Opaque value used to express a command. Should only be defined when type
|
| 426 |
+
/// is CMD
|
| 427 |
+
std::string cmd;
|
| 428 |
+
|
| 429 |
+
/// List of strings identifying a particular dataset. Should only be defined
|
| 430 |
+
/// when type is PATH
|
| 431 |
+
std::vector<std::string> path;
|
| 432 |
+
|
| 433 |
+
FlightDescriptor();
|
| 434 |
+
FlightDescriptor(DescriptorType type, std::string cmd,
|
| 435 |
+
std::vector<std::string> path) noexcept;
|
| 436 |
+
~FlightDescriptor();
|
| 437 |
+
|
| 438 |
+
/// \brief Get a human-readable form of this descriptor.
|
| 439 |
+
std::string ToString() const;
|
| 440 |
+
bool Equals(const FlightDescriptor& other) const;
|
| 441 |
+
|
| 442 |
+
using SuperT::Deserialize;
|
| 443 |
+
using SuperT::SerializeToString;
|
| 444 |
+
|
| 445 |
+
/// \brief Get the wire-format representation of this type.
|
| 446 |
+
///
|
| 447 |
+
/// Useful when interoperating with non-Flight systems (e.g. REST
|
| 448 |
+
/// services) that may want to return Flight types.
|
| 449 |
+
///
|
| 450 |
+
/// Use `SerializeToString()` if you want a Result-returning version.
|
| 451 |
+
arrow::Status SerializeToString(std::string* out) const;
|
| 452 |
+
|
| 453 |
+
/// \brief Parse the wire-format representation of this type.
|
| 454 |
+
///
|
| 455 |
+
/// Useful when interoperating with non-Flight systems (e.g. REST
|
| 456 |
+
/// services) that may want to return Flight types.
|
| 457 |
+
///
|
| 458 |
+
/// Use `Deserialize(serialized)` if you want a Result-returning version.
|
| 459 |
+
static arrow::Status Deserialize(std::string_view serialized, FlightDescriptor* out);
|
| 460 |
+
|
| 461 |
+
// Convenience factory functions
|
| 462 |
+
|
| 463 |
+
static FlightDescriptor Command(std::string cmd) {
|
| 464 |
+
return FlightDescriptor{CMD, std::move(cmd), {}};
|
| 465 |
+
}
|
| 466 |
+
|
| 467 |
+
static FlightDescriptor Path(std::vector<std::string> path) {
|
| 468 |
+
return FlightDescriptor{PATH, "", std::move(path)};
|
| 469 |
+
}
|
| 470 |
+
};
|
| 471 |
+
|
| 472 |
+
/// \brief Data structure providing an opaque identifier or credential to use
|
| 473 |
+
/// when requesting a data stream with the DoGet RPC
|
| 474 |
+
struct ARROW_FLIGHT_EXPORT Ticket : public internal::BaseType<Ticket> {
|
| 475 |
+
std::string ticket;
|
| 476 |
+
|
| 477 |
+
Ticket() = default;
|
| 478 |
+
Ticket(std::string ticket) // NOLINT runtime/explicit
|
| 479 |
+
: ticket(std::move(ticket)) {}
|
| 480 |
+
|
| 481 |
+
std::string ToString() const;
|
| 482 |
+
bool Equals(const Ticket& other) const;
|
| 483 |
+
|
| 484 |
+
using SuperT::Deserialize;
|
| 485 |
+
using SuperT::SerializeToString;
|
| 486 |
+
|
| 487 |
+
/// \brief Get the wire-format representation of this type.
|
| 488 |
+
///
|
| 489 |
+
/// Useful when interoperating with non-Flight systems (e.g. REST
|
| 490 |
+
/// services) that may want to return Flight types.
|
| 491 |
+
///
|
| 492 |
+
/// Use `SerializeToString()` if you want a Result-returning version.
|
| 493 |
+
arrow::Status SerializeToString(std::string* out) const;
|
| 494 |
+
|
| 495 |
+
/// \brief Parse the wire-format representation of this type.
|
| 496 |
+
///
|
| 497 |
+
/// Useful when interoperating with non-Flight systems (e.g. REST
|
| 498 |
+
/// services) that may want to return Flight types.
|
| 499 |
+
///
|
| 500 |
+
/// Use `Deserialize(serialized)` if you want a Result-returning version.
|
| 501 |
+
static arrow::Status Deserialize(std::string_view serialized, Ticket* out);
|
| 502 |
+
};
|
| 503 |
+
|
| 504 |
+
/// \brief A host location (a URI)
|
| 505 |
+
struct ARROW_FLIGHT_EXPORT Location : public internal::BaseType<Location> {
|
| 506 |
+
public:
|
| 507 |
+
/// \brief Initialize a blank location.
|
| 508 |
+
Location();
|
| 509 |
+
|
| 510 |
+
~Location();
|
| 511 |
+
|
| 512 |
+
/// \brief Initialize a location by parsing a URI string
|
| 513 |
+
static arrow::Result<Location> Parse(const std::string& uri_string);
|
| 514 |
+
|
| 515 |
+
/// \brief Get the fallback URI.
|
| 516 |
+
///
|
| 517 |
+
/// arrow-flight-reuse-connection://? means that a client may attempt to
|
| 518 |
+
/// reuse an existing connection to a Flight service to fetch data instead
|
| 519 |
+
/// of creating a new connection to one of the other locations listed in a
|
| 520 |
+
/// FlightEndpoint response.
|
| 521 |
+
static const Location& ReuseConnection();
|
| 522 |
+
|
| 523 |
+
/// \brief Initialize a location for a non-TLS, gRPC-based Flight
|
| 524 |
+
/// service from a host and port
|
| 525 |
+
/// \param[in] host The hostname to connect to
|
| 526 |
+
/// \param[in] port The port
|
| 527 |
+
/// \return Arrow result with the resulting location
|
| 528 |
+
static arrow::Result<Location> ForGrpcTcp(const std::string& host, const int port);
|
| 529 |
+
|
| 530 |
+
/// \brief Initialize a location for a TLS-enabled, gRPC-based Flight
|
| 531 |
+
/// service from a host and port
|
| 532 |
+
/// \param[in] host The hostname to connect to
|
| 533 |
+
/// \param[in] port The port
|
| 534 |
+
/// \return Arrow result with the resulting location
|
| 535 |
+
static arrow::Result<Location> ForGrpcTls(const std::string& host, const int port);
|
| 536 |
+
|
| 537 |
+
/// \brief Initialize a location for a domain socket-based Flight
|
| 538 |
+
/// service
|
| 539 |
+
/// \param[in] path The path to the domain socket
|
| 540 |
+
/// \return Arrow result with the resulting location
|
| 541 |
+
static arrow::Result<Location> ForGrpcUnix(const std::string& path);
|
| 542 |
+
|
| 543 |
+
/// \brief Initialize a location based on a URI scheme
|
| 544 |
+
static arrow::Result<Location> ForScheme(const std::string& scheme,
|
| 545 |
+
const std::string& host, const int port);
|
| 546 |
+
|
| 547 |
+
/// \brief Get the scheme of this URI.
|
| 548 |
+
std::string scheme() const;
|
| 549 |
+
|
| 550 |
+
/// \brief Get a representation of this URI as a string.
|
| 551 |
+
std::string ToString() const;
|
| 552 |
+
bool Equals(const Location& other) const;
|
| 553 |
+
|
| 554 |
+
using SuperT::Deserialize;
|
| 555 |
+
using SuperT::SerializeToString;
|
| 556 |
+
|
| 557 |
+
/// \brief Serialize this message to its wire-format representation.
|
| 558 |
+
///
|
| 559 |
+
/// Use `SerializeToString()` if you want a Result-returning version.
|
| 560 |
+
arrow::Status SerializeToString(std::string* out) const;
|
| 561 |
+
|
| 562 |
+
/// \brief Deserialize this message from its wire-format representation.
|
| 563 |
+
///
|
| 564 |
+
/// Use `Deserialize(serialized)` if you want a Result-returning version.
|
| 565 |
+
static arrow::Status Deserialize(std::string_view serialized, Location* out);
|
| 566 |
+
|
| 567 |
+
private:
|
| 568 |
+
friend class FlightClient;
|
| 569 |
+
friend class FlightServerBase;
|
| 570 |
+
std::shared_ptr<arrow::util::Uri> uri_;
|
| 571 |
+
};
|
| 572 |
+
|
| 573 |
+
/// \brief A flight ticket and list of locations where the ticket can be
|
| 574 |
+
/// redeemed
|
| 575 |
+
struct ARROW_FLIGHT_EXPORT FlightEndpoint : public internal::BaseType<FlightEndpoint> {
|
| 576 |
+
/// Opaque ticket identify; use with DoGet RPC
|
| 577 |
+
Ticket ticket;
|
| 578 |
+
|
| 579 |
+
/// List of locations where ticket can be redeemed. If the list is empty, the
|
| 580 |
+
/// ticket can only be redeemed on the current service where the ticket was
|
| 581 |
+
/// generated
|
| 582 |
+
std::vector<Location> locations;
|
| 583 |
+
|
| 584 |
+
/// Expiration time of this stream. If present, clients may assume
|
| 585 |
+
/// they can retry DoGet requests. Otherwise, clients should avoid
|
| 586 |
+
/// retrying DoGet requests.
|
| 587 |
+
std::optional<Timestamp> expiration_time;
|
| 588 |
+
|
| 589 |
+
/// Opaque Application-defined metadata
|
| 590 |
+
std::string app_metadata;
|
| 591 |
+
|
| 592 |
+
FlightEndpoint() = default;
|
| 593 |
+
FlightEndpoint(Ticket ticket, std::vector<Location> locations,
|
| 594 |
+
std::optional<Timestamp> expiration_time, std::string app_metadata)
|
| 595 |
+
: ticket(std::move(ticket)),
|
| 596 |
+
locations(std::move(locations)),
|
| 597 |
+
expiration_time(expiration_time),
|
| 598 |
+
app_metadata(std::move(app_metadata)) {}
|
| 599 |
+
|
| 600 |
+
std::string ToString() const;
|
| 601 |
+
bool Equals(const FlightEndpoint& other) const;
|
| 602 |
+
|
| 603 |
+
using SuperT::Deserialize;
|
| 604 |
+
using SuperT::SerializeToString;
|
| 605 |
+
|
| 606 |
+
/// \brief Serialize this message to its wire-format representation.
|
| 607 |
+
///
|
| 608 |
+
/// Use `SerializeToString()` if you want a Result-returning version.
|
| 609 |
+
arrow::Status SerializeToString(std::string* out) const;
|
| 610 |
+
|
| 611 |
+
/// \brief Deserialize this message from its wire-format representation.
|
| 612 |
+
///
|
| 613 |
+
/// Use `Deserialize(serialized)` if you want a Result-returning version.
|
| 614 |
+
static arrow::Status Deserialize(std::string_view serialized, FlightEndpoint* out);
|
| 615 |
+
};
|
| 616 |
+
|
| 617 |
+
/// \brief The access coordinates for retrieval of a dataset, returned by
|
| 618 |
+
/// GetFlightInfo
|
| 619 |
+
class ARROW_FLIGHT_EXPORT FlightInfo
|
| 620 |
+
: public internal::BaseType<std::unique_ptr<FlightInfo>> {
|
| 621 |
+
public:
|
| 622 |
+
struct Data {
|
| 623 |
+
std::string schema;
|
| 624 |
+
FlightDescriptor descriptor;
|
| 625 |
+
std::vector<FlightEndpoint> endpoints;
|
| 626 |
+
int64_t total_records = -1;
|
| 627 |
+
int64_t total_bytes = -1;
|
| 628 |
+
bool ordered = false;
|
| 629 |
+
std::string app_metadata;
|
| 630 |
+
};
|
| 631 |
+
|
| 632 |
+
explicit FlightInfo(Data data) : data_(std::move(data)), reconstructed_schema_(false) {}
|
| 633 |
+
|
| 634 |
+
/// \brief Factory method to construct a FlightInfo.
|
| 635 |
+
static arrow::Result<FlightInfo> Make(const Schema& schema,
|
| 636 |
+
const FlightDescriptor& descriptor,
|
| 637 |
+
const std::vector<FlightEndpoint>& endpoints,
|
| 638 |
+
int64_t total_records, int64_t total_bytes,
|
| 639 |
+
bool ordered = false,
|
| 640 |
+
std::string app_metadata = "");
|
| 641 |
+
|
| 642 |
+
/// \brief Factory method to construct a FlightInfo.
|
| 643 |
+
static arrow::Result<FlightInfo> Make(const std::shared_ptr<Schema>& schema,
|
| 644 |
+
const FlightDescriptor& descriptor,
|
| 645 |
+
const std::vector<FlightEndpoint>& endpoints,
|
| 646 |
+
int64_t total_records, int64_t total_bytes,
|
| 647 |
+
bool ordered = false,
|
| 648 |
+
std::string app_metadata = "");
|
| 649 |
+
|
| 650 |
+
/// \brief Deserialize the Arrow schema of the dataset. Populate any
|
| 651 |
+
/// dictionary encoded fields into a DictionaryMemo for
|
| 652 |
+
/// bookkeeping
|
| 653 |
+
/// \param[in,out] dictionary_memo for dictionary bookkeeping, will
|
| 654 |
+
/// be modified
|
| 655 |
+
/// \return Arrow result with the reconstructed Schema. Note that the schema
|
| 656 |
+
/// may be nullptr, as the schema is optional.
|
| 657 |
+
arrow::Result<std::shared_ptr<Schema>> GetSchema(
|
| 658 |
+
ipc::DictionaryMemo* dictionary_memo) const;
|
| 659 |
+
|
| 660 |
+
const std::string& serialized_schema() const { return data_.schema; }
|
| 661 |
+
|
| 662 |
+
/// The descriptor associated with this flight, may not be set
|
| 663 |
+
const FlightDescriptor& descriptor() const { return data_.descriptor; }
|
| 664 |
+
|
| 665 |
+
/// A list of endpoints associated with the flight (dataset). To consume the
|
| 666 |
+
/// whole flight, all endpoints must be consumed
|
| 667 |
+
const std::vector<FlightEndpoint>& endpoints() const { return data_.endpoints; }
|
| 668 |
+
|
| 669 |
+
/// The total number of records (rows) in the dataset. If unknown, set to -1
|
| 670 |
+
int64_t total_records() const { return data_.total_records; }
|
| 671 |
+
|
| 672 |
+
/// The total number of bytes in the dataset. If unknown, set to -1
|
| 673 |
+
int64_t total_bytes() const { return data_.total_bytes; }
|
| 674 |
+
|
| 675 |
+
/// Whether endpoints are in the same order as the data.
|
| 676 |
+
bool ordered() const { return data_.ordered; }
|
| 677 |
+
|
| 678 |
+
/// Application-defined opaque metadata
|
| 679 |
+
const std::string& app_metadata() const { return data_.app_metadata; }
|
| 680 |
+
|
| 681 |
+
using SuperT::Deserialize;
|
| 682 |
+
using SuperT::SerializeToString;
|
| 683 |
+
|
| 684 |
+
/// \brief Get the wire-format representation of this type.
|
| 685 |
+
///
|
| 686 |
+
/// Useful when interoperating with non-Flight systems (e.g. REST
|
| 687 |
+
/// services) that may want to return Flight types.
|
| 688 |
+
///
|
| 689 |
+
/// Use `SerializeToString()` if you want a Result-returning version.
|
| 690 |
+
arrow::Status SerializeToString(std::string* out) const;
|
| 691 |
+
|
| 692 |
+
/// \brief Parse the wire-format representation of this type.
|
| 693 |
+
///
|
| 694 |
+
/// Useful when interoperating with non-Flight systems (e.g. REST
|
| 695 |
+
/// services) that may want to return Flight types.
|
| 696 |
+
///
|
| 697 |
+
/// Use `Deserialize(serialized)` if you want a Result-returning version.
|
| 698 |
+
static arrow::Status Deserialize(std::string_view serialized,
|
| 699 |
+
std::unique_ptr<FlightInfo>* out);
|
| 700 |
+
|
| 701 |
+
std::string ToString() const;
|
| 702 |
+
|
| 703 |
+
/// Compare two FlightInfo for equality. This will compare the
|
| 704 |
+
/// serialized schema representations, NOT the logical equality of
|
| 705 |
+
/// the schemas.
|
| 706 |
+
bool Equals(const FlightInfo& other) const;
|
| 707 |
+
|
| 708 |
+
private:
|
| 709 |
+
Data data_;
|
| 710 |
+
mutable std::shared_ptr<Schema> schema_;
|
| 711 |
+
mutable bool reconstructed_schema_;
|
| 712 |
+
};
|
| 713 |
+
|
| 714 |
+
/// \brief The information to process a long-running query.
|
| 715 |
+
class ARROW_FLIGHT_EXPORT PollInfo
|
| 716 |
+
: public internal::BaseType<std::unique_ptr<PollInfo>> {
|
| 717 |
+
public:
|
| 718 |
+
/// The currently available results so far.
|
| 719 |
+
std::unique_ptr<FlightInfo> info = NULLPTR;
|
| 720 |
+
/// The descriptor the client should use on the next try. If unset,
|
| 721 |
+
/// the query is complete.
|
| 722 |
+
std::optional<FlightDescriptor> descriptor = std::nullopt;
|
| 723 |
+
/// Query progress. Must be in [0.0, 1.0] but need not be
|
| 724 |
+
/// monotonic or nondecreasing. If unknown, do not set.
|
| 725 |
+
std::optional<double> progress = std::nullopt;
|
| 726 |
+
/// Expiration time for this request. After this passes, the server
|
| 727 |
+
/// might not accept the poll descriptor anymore (and the query may
|
| 728 |
+
/// be cancelled). This may be updated on a call to PollFlightInfo.
|
| 729 |
+
std::optional<Timestamp> expiration_time = std::nullopt;
|
| 730 |
+
|
| 731 |
+
PollInfo()
|
| 732 |
+
: info(NULLPTR),
|
| 733 |
+
descriptor(std::nullopt),
|
| 734 |
+
progress(std::nullopt),
|
| 735 |
+
expiration_time(std::nullopt) {}
|
| 736 |
+
|
| 737 |
+
PollInfo(std::unique_ptr<FlightInfo> info, std::optional<FlightDescriptor> descriptor,
|
| 738 |
+
std::optional<double> progress, std::optional<Timestamp> expiration_time)
|
| 739 |
+
: info(std::move(info)),
|
| 740 |
+
descriptor(std::move(descriptor)),
|
| 741 |
+
progress(progress),
|
| 742 |
+
expiration_time(expiration_time) {}
|
| 743 |
+
|
| 744 |
+
PollInfo(const PollInfo& other)
|
| 745 |
+
: info(other.info ? std::make_unique<FlightInfo>(*other.info) : NULLPTR),
|
| 746 |
+
descriptor(other.descriptor),
|
| 747 |
+
progress(other.progress),
|
| 748 |
+
expiration_time(other.expiration_time) {}
|
| 749 |
+
PollInfo(PollInfo&& other) noexcept = default;
|
| 750 |
+
~PollInfo() = default;
|
| 751 |
+
PollInfo& operator=(const PollInfo& other) {
|
| 752 |
+
info = other.info ? std::make_unique<FlightInfo>(*other.info) : NULLPTR;
|
| 753 |
+
descriptor = other.descriptor;
|
| 754 |
+
progress = other.progress;
|
| 755 |
+
expiration_time = other.expiration_time;
|
| 756 |
+
return *this;
|
| 757 |
+
}
|
| 758 |
+
PollInfo& operator=(PollInfo&& other) = default;
|
| 759 |
+
|
| 760 |
+
using SuperT::Deserialize;
|
| 761 |
+
using SuperT::SerializeToString;
|
| 762 |
+
|
| 763 |
+
/// \brief Get the wire-format representation of this type.
|
| 764 |
+
///
|
| 765 |
+
/// Useful when interoperating with non-Flight systems (e.g. REST
|
| 766 |
+
/// services) that may want to return Flight types.
|
| 767 |
+
///
|
| 768 |
+
/// Use `SerializeToString()` if you want a Result-returning version.
|
| 769 |
+
arrow::Status SerializeToString(std::string* out) const;
|
| 770 |
+
|
| 771 |
+
/// \brief Parse the wire-format representation of this type.
|
| 772 |
+
///
|
| 773 |
+
/// Useful when interoperating with non-Flight systems (e.g. REST
|
| 774 |
+
/// services) that may want to return Flight types.
|
| 775 |
+
///
|
| 776 |
+
/// Use `Deserialize(serialized)` if you want a Result-returning version.
|
| 777 |
+
static arrow::Status Deserialize(std::string_view serialized,
|
| 778 |
+
std::unique_ptr<PollInfo>* out);
|
| 779 |
+
|
| 780 |
+
std::string ToString() const;
|
| 781 |
+
|
| 782 |
+
/// Compare two PollInfo for equality. This will compare the
|
| 783 |
+
/// serialized schema representations, NOT the logical equality of
|
| 784 |
+
/// the schemas.
|
| 785 |
+
bool Equals(const PollInfo& other) const;
|
| 786 |
+
};
|
| 787 |
+
|
| 788 |
+
/// \brief The request of the CancelFlightInfoRequest action.
|
| 789 |
+
struct ARROW_FLIGHT_EXPORT CancelFlightInfoRequest
|
| 790 |
+
: public internal::BaseType<CancelFlightInfoRequest> {
|
| 791 |
+
std::unique_ptr<FlightInfo> info;
|
| 792 |
+
|
| 793 |
+
CancelFlightInfoRequest() = default;
|
| 794 |
+
CancelFlightInfoRequest(std::unique_ptr<FlightInfo> info) // NOLINT runtime/explicit
|
| 795 |
+
: info(std::move(info)) {}
|
| 796 |
+
|
| 797 |
+
std::string ToString() const;
|
| 798 |
+
bool Equals(const CancelFlightInfoRequest& other) const;
|
| 799 |
+
|
| 800 |
+
using SuperT::Deserialize;
|
| 801 |
+
using SuperT::SerializeToString;
|
| 802 |
+
|
| 803 |
+
/// \brief Serialize this message to its wire-format representation.
|
| 804 |
+
///
|
| 805 |
+
/// Use `SerializeToString()` if you want a Result-returning version.
|
| 806 |
+
arrow::Status SerializeToString(std::string* out) const;
|
| 807 |
+
|
| 808 |
+
/// \brief Deserialize this message from its wire-format representation.
|
| 809 |
+
///
|
| 810 |
+
/// Use `Deserialize(serialized)` if you want a Result-returning version.
|
| 811 |
+
static arrow::Status Deserialize(std::string_view serialized,
|
| 812 |
+
CancelFlightInfoRequest* out);
|
| 813 |
+
};
|
| 814 |
+
|
| 815 |
+
enum class CancelStatus {
|
| 816 |
+
/// The cancellation status is unknown. Servers should avoid using
|
| 817 |
+
/// this value (send a kNotCancellable if the requested FlightInfo
|
| 818 |
+
/// is not known). Clients can retry the request.
|
| 819 |
+
kUnspecified = 0,
|
| 820 |
+
/// The cancellation request is complete. Subsequent requests with
|
| 821 |
+
/// the same payload may return kCancelled or a kNotCancellable error.
|
| 822 |
+
kCancelled = 1,
|
| 823 |
+
/// The cancellation request is in progress. The client may retry
|
| 824 |
+
/// the cancellation request.
|
| 825 |
+
kCancelling = 2,
|
| 826 |
+
// The FlightInfo is not cancellable. The client should not retry the
|
| 827 |
+
// cancellation request.
|
| 828 |
+
kNotCancellable = 3,
|
| 829 |
+
};
|
| 830 |
+
|
| 831 |
+
/// \brief The result of the CancelFlightInfo action.
|
| 832 |
+
struct ARROW_FLIGHT_EXPORT CancelFlightInfoResult
|
| 833 |
+
: public internal::BaseType<CancelFlightInfoResult> {
|
| 834 |
+
CancelStatus status = CancelStatus::kUnspecified;
|
| 835 |
+
|
| 836 |
+
CancelFlightInfoResult() = default;
|
| 837 |
+
CancelFlightInfoResult(CancelStatus status) // NOLINT runtime/explicit
|
| 838 |
+
: status(status) {}
|
| 839 |
+
|
| 840 |
+
std::string ToString() const;
|
| 841 |
+
bool Equals(const CancelFlightInfoResult& other) const;
|
| 842 |
+
|
| 843 |
+
using SuperT::Deserialize;
|
| 844 |
+
using SuperT::SerializeToString;
|
| 845 |
+
|
| 846 |
+
/// \brief Serialize this message to its wire-format representation.
|
| 847 |
+
///
|
| 848 |
+
/// Use `SerializeToString()` if you want a Result-returning version.
|
| 849 |
+
arrow::Status SerializeToString(std::string* out) const;
|
| 850 |
+
|
| 851 |
+
/// \brief Deserialize this message from its wire-format representation.
|
| 852 |
+
///
|
| 853 |
+
/// Use `Deserialize(serialized)` if you want a Result-returning version.
|
| 854 |
+
static arrow::Status Deserialize(std::string_view serialized,
|
| 855 |
+
CancelFlightInfoResult* out);
|
| 856 |
+
};
|
| 857 |
+
|
| 858 |
+
ARROW_FLIGHT_EXPORT
|
| 859 |
+
std::ostream& operator<<(std::ostream& os, CancelStatus status);
|
| 860 |
+
|
| 861 |
+
/// \brief The request of the RenewFlightEndpoint action.
|
| 862 |
+
struct ARROW_FLIGHT_EXPORT RenewFlightEndpointRequest
|
| 863 |
+
: public internal::BaseType<RenewFlightEndpointRequest> {
|
| 864 |
+
FlightEndpoint endpoint;
|
| 865 |
+
|
| 866 |
+
RenewFlightEndpointRequest() = default;
|
| 867 |
+
explicit RenewFlightEndpointRequest(FlightEndpoint endpoint)
|
| 868 |
+
: endpoint(std::move(endpoint)) {}
|
| 869 |
+
|
| 870 |
+
std::string ToString() const;
|
| 871 |
+
bool Equals(const RenewFlightEndpointRequest& other) const;
|
| 872 |
+
|
| 873 |
+
using SuperT::Deserialize;
|
| 874 |
+
using SuperT::SerializeToString;
|
| 875 |
+
|
| 876 |
+
/// \brief Serialize this message to its wire-format representation.
|
| 877 |
+
///
|
| 878 |
+
/// Use `SerializeToString()` if you want a Result-returning version.
|
| 879 |
+
arrow::Status SerializeToString(std::string* out) const;
|
| 880 |
+
|
| 881 |
+
/// \brief Deserialize this message from its wire-format representation.
|
| 882 |
+
///
|
| 883 |
+
/// Use `Deserialize(serialized)` if you want a Result-returning version.
|
| 884 |
+
static arrow::Status Deserialize(std::string_view serialized,
|
| 885 |
+
RenewFlightEndpointRequest* out);
|
| 886 |
+
};
|
| 887 |
+
|
| 888 |
+
// FlightData in Flight.proto maps to FlightPayload here.
|
| 889 |
+
|
| 890 |
+
/// \brief Staging data structure for messages about to be put on the wire
|
| 891 |
+
///
|
| 892 |
+
/// This structure corresponds to FlightData in the protocol.
|
| 893 |
+
struct ARROW_FLIGHT_EXPORT FlightPayload {
|
| 894 |
+
std::shared_ptr<Buffer> descriptor;
|
| 895 |
+
std::shared_ptr<Buffer> app_metadata;
|
| 896 |
+
ipc::IpcPayload ipc_message;
|
| 897 |
+
|
| 898 |
+
FlightPayload() = default;
|
| 899 |
+
FlightPayload(std::shared_ptr<Buffer> descriptor, std::shared_ptr<Buffer> app_metadata,
|
| 900 |
+
ipc::IpcPayload ipc_message)
|
| 901 |
+
: descriptor(std::move(descriptor)),
|
| 902 |
+
app_metadata(std::move(app_metadata)),
|
| 903 |
+
ipc_message(std::move(ipc_message)) {}
|
| 904 |
+
|
| 905 |
+
/// \brief Check that the payload can be written to the wire.
|
| 906 |
+
Status Validate() const;
|
| 907 |
+
|
| 908 |
+
/// \brief Serialize this payload to a vector of buffers.
|
| 909 |
+
arrow::Result<BufferVector> SerializeToBuffers() const;
|
| 910 |
+
};
|
| 911 |
+
|
| 912 |
+
// A wrapper around arrow.flight.protocol.PutResult is not defined
|
| 913 |
+
|
| 914 |
+
// Session management messages
|
| 915 |
+
|
| 916 |
+
/// \brief Variant supporting all possible value types for {Set,Get}SessionOptions
|
| 917 |
+
///
|
| 918 |
+
/// By convention, an attempt to set a valueless (std::monostate) SessionOptionValue
|
| 919 |
+
/// should attempt to unset or clear the named option value on the server.
|
| 920 |
+
using SessionOptionValue = std::variant<std::monostate, std::string, bool, int64_t,
|
| 921 |
+
double, std::vector<std::string>>;
|
| 922 |
+
std::ostream& operator<<(std::ostream& os, const SessionOptionValue& v);
|
| 923 |
+
|
| 924 |
+
/// \brief A request to set a set of session options by name/value.
|
| 925 |
+
struct ARROW_FLIGHT_EXPORT SetSessionOptionsRequest
|
| 926 |
+
: public internal::BaseType<SetSessionOptionsRequest> {
|
| 927 |
+
std::map<std::string, SessionOptionValue> session_options;
|
| 928 |
+
|
| 929 |
+
SetSessionOptionsRequest() = default;
|
| 930 |
+
explicit SetSessionOptionsRequest(
|
| 931 |
+
std::map<std::string, SessionOptionValue> session_options)
|
| 932 |
+
: session_options(std::move(session_options)) {}
|
| 933 |
+
|
| 934 |
+
std::string ToString() const;
|
| 935 |
+
bool Equals(const SetSessionOptionsRequest& other) const;
|
| 936 |
+
|
| 937 |
+
using SuperT::Deserialize;
|
| 938 |
+
using SuperT::SerializeToString;
|
| 939 |
+
|
| 940 |
+
/// \brief Serialize this message to its wire-format representation.
|
| 941 |
+
///
|
| 942 |
+
/// Use `SerializeToString()` if you want a Result-returning version.
|
| 943 |
+
arrow::Status SerializeToString(std::string* out) const;
|
| 944 |
+
|
| 945 |
+
/// \brief Deserialize this message from its wire-format representation.
|
| 946 |
+
///
|
| 947 |
+
/// Use `Deserialize(serialized)` if you want a Result-returning version.
|
| 948 |
+
static arrow::Status Deserialize(std::string_view serialized,
|
| 949 |
+
SetSessionOptionsRequest* out);
|
| 950 |
+
};
|
| 951 |
+
|
| 952 |
+
/// \brief The result of setting a session option.
|
| 953 |
+
enum class SetSessionOptionErrorValue : int8_t {
|
| 954 |
+
/// \brief The status of setting the option is unknown.
|
| 955 |
+
///
|
| 956 |
+
/// Servers should avoid using this value (send a NOT_FOUND error if the requested
|
| 957 |
+
/// session is not known). Clients can retry the request.
|
| 958 |
+
kUnspecified,
|
| 959 |
+
/// \brief The given session option name is invalid.
|
| 960 |
+
kInvalidName,
|
| 961 |
+
/// \brief The session option value or type is invalid.
|
| 962 |
+
kInvalidValue,
|
| 963 |
+
/// \brief The session option cannot be set.
|
| 964 |
+
kError
|
| 965 |
+
};
|
| 966 |
+
std::string ToString(const SetSessionOptionErrorValue& error_value);
|
| 967 |
+
std::ostream& operator<<(std::ostream& os, const SetSessionOptionErrorValue& error_value);
|
| 968 |
+
|
| 969 |
+
/// \brief The result(s) of setting session option(s).
|
| 970 |
+
struct ARROW_FLIGHT_EXPORT SetSessionOptionsResult
|
| 971 |
+
: public internal::BaseType<SetSessionOptionsResult> {
|
| 972 |
+
struct Error {
|
| 973 |
+
SetSessionOptionErrorValue value;
|
| 974 |
+
|
| 975 |
+
bool Equals(const Error& other) const { return value == other.value; }
|
| 976 |
+
friend bool operator==(const Error& left, const Error& right) {
|
| 977 |
+
return left.Equals(right);
|
| 978 |
+
}
|
| 979 |
+
friend bool operator!=(const Error& left, const Error& right) {
|
| 980 |
+
return !(left == right);
|
| 981 |
+
}
|
| 982 |
+
};
|
| 983 |
+
|
| 984 |
+
std::map<std::string, Error> errors;
|
| 985 |
+
|
| 986 |
+
SetSessionOptionsResult() = default;
|
| 987 |
+
SetSessionOptionsResult(std::map<std::string, Error> errors) // NOLINT runtime/explicit
|
| 988 |
+
: errors(std::move(errors)) {}
|
| 989 |
+
|
| 990 |
+
std::string ToString() const;
|
| 991 |
+
bool Equals(const SetSessionOptionsResult& other) const;
|
| 992 |
+
|
| 993 |
+
using SuperT::Deserialize;
|
| 994 |
+
using SuperT::SerializeToString;
|
| 995 |
+
|
| 996 |
+
/// \brief Serialize this message to its wire-format representation.
|
| 997 |
+
///
|
| 998 |
+
/// Use `SerializeToString()` if you want a Result-returning version.
|
| 999 |
+
arrow::Status SerializeToString(std::string* out) const;
|
| 1000 |
+
|
| 1001 |
+
/// \brief Deserialize this message from its wire-format representation.
|
| 1002 |
+
///
|
| 1003 |
+
/// Use `Deserialize(serialized)` if you want a Result-returning version.
|
| 1004 |
+
static arrow::Status Deserialize(std::string_view serialized,
|
| 1005 |
+
SetSessionOptionsResult* out);
|
| 1006 |
+
};
|
| 1007 |
+
|
| 1008 |
+
/// \brief A request to get current session options.
|
| 1009 |
+
struct ARROW_FLIGHT_EXPORT GetSessionOptionsRequest
|
| 1010 |
+
: public internal::BaseType<GetSessionOptionsRequest> {
|
| 1011 |
+
GetSessionOptionsRequest() = default;
|
| 1012 |
+
|
| 1013 |
+
std::string ToString() const;
|
| 1014 |
+
bool Equals(const GetSessionOptionsRequest& other) const;
|
| 1015 |
+
|
| 1016 |
+
using SuperT::Deserialize;
|
| 1017 |
+
using SuperT::SerializeToString;
|
| 1018 |
+
|
| 1019 |
+
/// \brief Serialize this message to its wire-format representation.
|
| 1020 |
+
///
|
| 1021 |
+
/// Use `SerializeToString()` if you want a Result-returning version.
|
| 1022 |
+
arrow::Status SerializeToString(std::string* out) const;
|
| 1023 |
+
|
| 1024 |
+
/// \brief Deserialize this message from its wire-format representation.
|
| 1025 |
+
///
|
| 1026 |
+
/// Use `Deserialize(serialized)` if you want a Result-returning version.
|
| 1027 |
+
static arrow::Status Deserialize(std::string_view serialized,
|
| 1028 |
+
GetSessionOptionsRequest* out);
|
| 1029 |
+
};
|
| 1030 |
+
|
| 1031 |
+
/// \brief The current session options.
|
| 1032 |
+
struct ARROW_FLIGHT_EXPORT GetSessionOptionsResult
|
| 1033 |
+
: public internal::BaseType<GetSessionOptionsResult> {
|
| 1034 |
+
std::map<std::string, SessionOptionValue> session_options;
|
| 1035 |
+
|
| 1036 |
+
GetSessionOptionsResult() = default;
|
| 1037 |
+
GetSessionOptionsResult( // NOLINT runtime/explicit
|
| 1038 |
+
std::map<std::string, SessionOptionValue> session_options)
|
| 1039 |
+
: session_options(std::move(session_options)) {}
|
| 1040 |
+
|
| 1041 |
+
std::string ToString() const;
|
| 1042 |
+
bool Equals(const GetSessionOptionsResult& other) const;
|
| 1043 |
+
|
| 1044 |
+
using SuperT::Deserialize;
|
| 1045 |
+
using SuperT::SerializeToString;
|
| 1046 |
+
|
| 1047 |
+
/// \brief Serialize this message to its wire-format representation.
|
| 1048 |
+
///
|
| 1049 |
+
/// Use `SerializeToString()` if you want a Result-returning version.
|
| 1050 |
+
arrow::Status SerializeToString(std::string* out) const;
|
| 1051 |
+
|
| 1052 |
+
/// \brief Deserialize this message from its wire-format representation.
|
| 1053 |
+
///
|
| 1054 |
+
/// Use `Deserialize(serialized)` if you want a Result-returning version.
|
| 1055 |
+
static arrow::Status Deserialize(std::string_view serialized,
|
| 1056 |
+
GetSessionOptionsResult* out);
|
| 1057 |
+
};
|
| 1058 |
+
|
| 1059 |
+
/// \brief A request to close the open client session.
|
| 1060 |
+
struct ARROW_FLIGHT_EXPORT CloseSessionRequest
|
| 1061 |
+
: public internal::BaseType<CloseSessionRequest> {
|
| 1062 |
+
CloseSessionRequest() = default;
|
| 1063 |
+
|
| 1064 |
+
std::string ToString() const;
|
| 1065 |
+
bool Equals(const CloseSessionRequest& other) const;
|
| 1066 |
+
|
| 1067 |
+
using SuperT::Deserialize;
|
| 1068 |
+
using SuperT::SerializeToString;
|
| 1069 |
+
|
| 1070 |
+
/// \brief Serialize this message to its wire-format representation.
|
| 1071 |
+
///
|
| 1072 |
+
/// Use `SerializeToString()` if you want a Result-returning version.
|
| 1073 |
+
arrow::Status SerializeToString(std::string* out) const;
|
| 1074 |
+
|
| 1075 |
+
/// \brief Deserialize this message from its wire-format representation.
|
| 1076 |
+
///
|
| 1077 |
+
/// Use `Deserialize(serialized)` if you want a Result-returning version.
|
| 1078 |
+
static arrow::Status Deserialize(std::string_view serialized, CloseSessionRequest* out);
|
| 1079 |
+
};
|
| 1080 |
+
|
| 1081 |
+
/// \brief The result of closing a session.
|
| 1082 |
+
enum class CloseSessionStatus : int8_t {
|
| 1083 |
+
// \brief The session close status is unknown.
|
| 1084 |
+
//
|
| 1085 |
+
// Servers should avoid using this value (send a NOT_FOUND error if the requested
|
| 1086 |
+
// session is not known). Clients can retry the request.
|
| 1087 |
+
kUnspecified,
|
| 1088 |
+
// \brief The session close request is complete.
|
| 1089 |
+
//
|
| 1090 |
+
// Subsequent requests with the same session produce a NOT_FOUND error.
|
| 1091 |
+
kClosed,
|
| 1092 |
+
// \brief The session close request is in progress.
|
| 1093 |
+
//
|
| 1094 |
+
// The client may retry the request.
|
| 1095 |
+
kClosing,
|
| 1096 |
+
// \brief The session is not closeable.
|
| 1097 |
+
//
|
| 1098 |
+
// The client should not retry the request.
|
| 1099 |
+
kNotClosable
|
| 1100 |
+
};
|
| 1101 |
+
std::string ToString(const CloseSessionStatus& status);
|
| 1102 |
+
std::ostream& operator<<(std::ostream& os, const CloseSessionStatus& status);
|
| 1103 |
+
|
| 1104 |
+
/// \brief The result of attempting to close the client session.
|
| 1105 |
+
struct ARROW_FLIGHT_EXPORT CloseSessionResult
|
| 1106 |
+
: public internal::BaseType<CloseSessionResult> {
|
| 1107 |
+
CloseSessionStatus status;
|
| 1108 |
+
|
| 1109 |
+
CloseSessionResult() = default;
|
| 1110 |
+
CloseSessionResult(CloseSessionStatus status) // NOLINT runtime/explicit
|
| 1111 |
+
: status(status) {}
|
| 1112 |
+
|
| 1113 |
+
std::string ToString() const;
|
| 1114 |
+
bool Equals(const CloseSessionResult& other) const;
|
| 1115 |
+
|
| 1116 |
+
using SuperT::Deserialize;
|
| 1117 |
+
using SuperT::SerializeToString;
|
| 1118 |
+
|
| 1119 |
+
/// \brief Serialize this message to its wire-format representation.
|
| 1120 |
+
///
|
| 1121 |
+
/// Use `SerializeToString()` if you want a Result-returning version.
|
| 1122 |
+
arrow::Status SerializeToString(std::string* out) const;
|
| 1123 |
+
|
| 1124 |
+
/// \brief Deserialize this message from its wire-format representation.
|
| 1125 |
+
///
|
| 1126 |
+
/// Use `Deserialize(serialized)` if you want a Result-returning version.
|
| 1127 |
+
static arrow::Status Deserialize(std::string_view serialized, CloseSessionResult* out);
|
| 1128 |
+
};
|
| 1129 |
+
|
| 1130 |
+
//------------------------------------------------------------
|
| 1131 |
+
|
| 1132 |
+
/// \brief An iterator to FlightInfo instances returned by ListFlights.
|
| 1133 |
+
class ARROW_FLIGHT_EXPORT FlightListing {
|
| 1134 |
+
public:
|
| 1135 |
+
virtual ~FlightListing() = default;
|
| 1136 |
+
|
| 1137 |
+
/// \brief Retrieve the next FlightInfo from the iterator.
|
| 1138 |
+
/// \return Arrow result with a single FlightInfo. Set to \a nullptr if there
|
| 1139 |
+
/// are none left.
|
| 1140 |
+
virtual arrow::Result<std::unique_ptr<FlightInfo>> Next() = 0;
|
| 1141 |
+
};
|
| 1142 |
+
|
| 1143 |
+
/// \brief An iterator to Result instances returned by DoAction.
|
| 1144 |
+
class ARROW_FLIGHT_EXPORT ResultStream {
|
| 1145 |
+
public:
|
| 1146 |
+
virtual ~ResultStream() = default;
|
| 1147 |
+
|
| 1148 |
+
/// \brief Retrieve the next Result from the iterator.
|
| 1149 |
+
/// \return Arrow result with a single Result. Set to \a nullptr if there are none left.
|
| 1150 |
+
virtual arrow::Result<std::unique_ptr<Result>> Next() = 0;
|
| 1151 |
+
|
| 1152 |
+
/// \brief Read and drop the remaining messages to get the error (if any) from a server.
|
| 1153 |
+
/// \return Status OK if this is no error from a server, any other status if a
|
| 1154 |
+
/// server returns an error.
|
| 1155 |
+
Status Drain();
|
| 1156 |
+
};
|
| 1157 |
+
|
| 1158 |
+
/// \brief A holder for a RecordBatch with associated Flight metadata.
|
| 1159 |
+
struct ARROW_FLIGHT_EXPORT FlightStreamChunk {
|
| 1160 |
+
public:
|
| 1161 |
+
FlightStreamChunk() noexcept;
|
| 1162 |
+
~FlightStreamChunk();
|
| 1163 |
+
|
| 1164 |
+
std::shared_ptr<RecordBatch> data;
|
| 1165 |
+
std::shared_ptr<Buffer> app_metadata;
|
| 1166 |
+
};
|
| 1167 |
+
|
| 1168 |
+
/// \brief An interface to read Flight data with metadata.
|
| 1169 |
+
class ARROW_FLIGHT_EXPORT MetadataRecordBatchReader {
|
| 1170 |
+
public:
|
| 1171 |
+
virtual ~MetadataRecordBatchReader() = default;
|
| 1172 |
+
|
| 1173 |
+
/// \brief Get the schema for this stream.
|
| 1174 |
+
virtual arrow::Result<std::shared_ptr<Schema>> GetSchema() = 0;
|
| 1175 |
+
|
| 1176 |
+
/// \brief Get the next message from Flight. If the stream is
|
| 1177 |
+
/// finished, then the members of \a FlightStreamChunk will be
|
| 1178 |
+
/// nullptr.
|
| 1179 |
+
virtual arrow::Result<FlightStreamChunk> Next() = 0;
|
| 1180 |
+
|
| 1181 |
+
/// \brief Consume entire stream as a vector of record batches
|
| 1182 |
+
virtual arrow::Result<std::vector<std::shared_ptr<RecordBatch>>> ToRecordBatches();
|
| 1183 |
+
|
| 1184 |
+
/// \brief Consume entire stream as a Table
|
| 1185 |
+
virtual arrow::Result<std::shared_ptr<Table>> ToTable();
|
| 1186 |
+
|
| 1187 |
+
/// \brief Return current read statistics
|
| 1188 |
+
virtual arrow::ipc::ReadStats stats() const = 0;
|
| 1189 |
+
};
|
| 1190 |
+
|
| 1191 |
+
/// \brief Convert a MetadataRecordBatchReader to a regular RecordBatchReader.
|
| 1192 |
+
ARROW_FLIGHT_EXPORT
|
| 1193 |
+
arrow::Result<std::shared_ptr<RecordBatchReader>> MakeRecordBatchReader(
|
| 1194 |
+
std::shared_ptr<MetadataRecordBatchReader> reader);
|
| 1195 |
+
|
| 1196 |
+
/// \brief An interface to write IPC payloads with metadata.
|
| 1197 |
+
class ARROW_FLIGHT_EXPORT MetadataRecordBatchWriter : public ipc::RecordBatchWriter {
|
| 1198 |
+
public:
|
| 1199 |
+
virtual ~MetadataRecordBatchWriter() = default;
|
| 1200 |
+
/// \brief Begin writing data with the given schema. Only used with \a DoExchange.
|
| 1201 |
+
virtual Status Begin(const std::shared_ptr<Schema>& schema,
|
| 1202 |
+
const ipc::IpcWriteOptions& options) = 0;
|
| 1203 |
+
virtual Status Begin(const std::shared_ptr<Schema>& schema);
|
| 1204 |
+
virtual Status WriteMetadata(std::shared_ptr<Buffer> app_metadata) = 0;
|
| 1205 |
+
virtual Status WriteWithMetadata(const RecordBatch& batch,
|
| 1206 |
+
std::shared_ptr<Buffer> app_metadata) = 0;
|
| 1207 |
+
};
|
| 1208 |
+
|
| 1209 |
+
/// \brief A FlightListing implementation based on a vector of
|
| 1210 |
+
/// FlightInfo objects.
|
| 1211 |
+
///
|
| 1212 |
+
/// This can be iterated once, then it is consumed.
|
| 1213 |
+
class ARROW_FLIGHT_EXPORT SimpleFlightListing : public FlightListing {
|
| 1214 |
+
public:
|
| 1215 |
+
explicit SimpleFlightListing(const std::vector<FlightInfo>& flights);
|
| 1216 |
+
explicit SimpleFlightListing(std::vector<FlightInfo>&& flights);
|
| 1217 |
+
|
| 1218 |
+
arrow::Result<std::unique_ptr<FlightInfo>> Next() override;
|
| 1219 |
+
|
| 1220 |
+
private:
|
| 1221 |
+
int position_;
|
| 1222 |
+
std::vector<FlightInfo> flights_;
|
| 1223 |
+
};
|
| 1224 |
+
|
| 1225 |
+
/// \brief A ResultStream implementation based on a vector of
|
| 1226 |
+
/// Result objects.
|
| 1227 |
+
///
|
| 1228 |
+
/// This can be iterated once, then it is consumed.
|
| 1229 |
+
class ARROW_FLIGHT_EXPORT SimpleResultStream : public ResultStream {
|
| 1230 |
+
public:
|
| 1231 |
+
explicit SimpleResultStream(std::vector<Result>&& results);
|
| 1232 |
+
arrow::Result<std::unique_ptr<Result>> Next() override;
|
| 1233 |
+
|
| 1234 |
+
private:
|
| 1235 |
+
std::vector<Result> results_;
|
| 1236 |
+
size_t position_;
|
| 1237 |
+
};
|
| 1238 |
+
|
| 1239 |
+
/// \defgroup flight-error Error Handling
|
| 1240 |
+
/// Types for handling errors from RPCs. Flight uses a set of status
|
| 1241 |
+
/// codes standardized across Flight implementations, so these types
|
| 1242 |
+
/// let applications work directly with those codes instead of having
|
| 1243 |
+
/// to translate to and from Arrow Status.
|
| 1244 |
+
/// @{
|
| 1245 |
+
|
| 1246 |
+
/// \brief Abstract status code for an RPC as per the Flight
|
| 1247 |
+
/// specification.
|
| 1248 |
+
enum class TransportStatusCode {
|
| 1249 |
+
/// \brief No error.
|
| 1250 |
+
kOk = 0,
|
| 1251 |
+
/// \brief An unknown error occurred.
|
| 1252 |
+
kUnknown = 1,
|
| 1253 |
+
/// \brief An error occurred in the transport implementation, or an
|
| 1254 |
+
/// error internal to the service implementation occurred.
|
| 1255 |
+
kInternal = 2,
|
| 1256 |
+
/// \brief An argument is invalid.
|
| 1257 |
+
kInvalidArgument = 3,
|
| 1258 |
+
/// \brief The request timed out.
|
| 1259 |
+
kTimedOut = 4,
|
| 1260 |
+
/// \brief An argument is not necessarily invalid, but references
|
| 1261 |
+
/// some resource that does not exist. Prefer over
|
| 1262 |
+
/// kInvalidArgument where applicable.
|
| 1263 |
+
kNotFound = 5,
|
| 1264 |
+
/// \brief The request attempted to create some resource that does
|
| 1265 |
+
/// not exist.
|
| 1266 |
+
kAlreadyExists = 6,
|
| 1267 |
+
/// \brief The request was explicitly cancelled.
|
| 1268 |
+
kCancelled = 7,
|
| 1269 |
+
/// \brief The client is not authenticated.
|
| 1270 |
+
kUnauthenticated = 8,
|
| 1271 |
+
/// \brief The client is not authorized to perform this request.
|
| 1272 |
+
kUnauthorized = 9,
|
| 1273 |
+
/// \brief The request is not implemented
|
| 1274 |
+
kUnimplemented = 10,
|
| 1275 |
+
/// \brief There is a network connectivity error, or some resource
|
| 1276 |
+
/// is otherwise unavailable. Most likely a temporary condition.
|
| 1277 |
+
kUnavailable = 11,
|
| 1278 |
+
};
|
| 1279 |
+
|
| 1280 |
+
/// \brief Convert a code to a string.
|
| 1281 |
+
std::string ToString(TransportStatusCode code);
|
| 1282 |
+
|
| 1283 |
+
/// \brief An error from an RPC call, using Flight error codes directly
|
| 1284 |
+
/// instead of trying to translate to Arrow Status.
|
| 1285 |
+
///
|
| 1286 |
+
/// Currently, only attached to the Status passed to AsyncListener::OnFinish.
|
| 1287 |
+
class ARROW_FLIGHT_EXPORT TransportStatusDetail : public StatusDetail {
|
| 1288 |
+
public:
|
| 1289 |
+
constexpr static const char* kTypeId = "flight::TransportStatusDetail";
|
| 1290 |
+
explicit TransportStatusDetail(TransportStatusCode code, std::string message,
|
| 1291 |
+
std::vector<std::pair<std::string, std::string>> details)
|
| 1292 |
+
: code_(code), message_(std::move(message)), details_(std::move(details)) {}
|
| 1293 |
+
const char* type_id() const override { return kTypeId; }
|
| 1294 |
+
std::string ToString() const override;
|
| 1295 |
+
|
| 1296 |
+
static std::optional<std::reference_wrapper<const TransportStatusDetail>> Unwrap(
|
| 1297 |
+
const Status& status);
|
| 1298 |
+
|
| 1299 |
+
TransportStatusCode code() const { return code_; }
|
| 1300 |
+
std::string_view message() const { return message_; }
|
| 1301 |
+
const std::vector<std::pair<std::string, std::string>>& details() const {
|
| 1302 |
+
return details_;
|
| 1303 |
+
}
|
| 1304 |
+
|
| 1305 |
+
private:
|
| 1306 |
+
TransportStatusCode code_;
|
| 1307 |
+
std::string message_;
|
| 1308 |
+
std::vector<std::pair<std::string, std::string>> details_;
|
| 1309 |
+
};
|
| 1310 |
+
|
| 1311 |
+
/// @}
|
| 1312 |
+
|
| 1313 |
+
} // namespace flight
|
| 1314 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/types_async.h
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <memory>
|
| 21 |
+
|
| 22 |
+
#include "arrow/flight/type_fwd.h"
|
| 23 |
+
#include "arrow/flight/types.h"
|
| 24 |
+
#include "arrow/ipc/options.h"
|
| 25 |
+
#include "arrow/type_fwd.h"
|
| 26 |
+
|
| 27 |
+
namespace arrow::flight {
|
| 28 |
+
|
| 29 |
+
/// \defgroup flight-async Async Flight Types
|
| 30 |
+
/// Common types used for asynchronous Flight APIs.
|
| 31 |
+
/// @{
|
| 32 |
+
|
| 33 |
+
/// \brief Non-templated state for an async RPC.
|
| 34 |
+
class ARROW_FLIGHT_EXPORT AsyncListenerBase {
|
| 35 |
+
public:
|
| 36 |
+
AsyncListenerBase();
|
| 37 |
+
virtual ~AsyncListenerBase();
|
| 38 |
+
|
| 39 |
+
/// \brief Request cancellation of the RPC.
|
| 40 |
+
///
|
| 41 |
+
/// The RPC is not cancelled until AsyncListener::OnFinish is called.
|
| 42 |
+
void TryCancel();
|
| 43 |
+
|
| 44 |
+
private:
|
| 45 |
+
friend class arrow::flight::internal::ClientTransport;
|
| 46 |
+
|
| 47 |
+
/// Transport-specific state for this RPC. Transport
|
| 48 |
+
/// implementations may store and retrieve state here via
|
| 49 |
+
/// ClientTransport::SetAsyncRpc and ClientTransport::GetAsyncRpc.
|
| 50 |
+
std::unique_ptr<internal::AsyncRpc> rpc_state_;
|
| 51 |
+
};
|
| 52 |
+
|
| 53 |
+
/// \brief Callbacks for results from async RPCs.
|
| 54 |
+
///
|
| 55 |
+
/// A single listener may not be used for multiple concurrent RPC
|
| 56 |
+
/// calls. The application MUST hold the listener alive until
|
| 57 |
+
/// OnFinish() is called and has finished.
|
| 58 |
+
template <typename T>
|
| 59 |
+
class ARROW_FLIGHT_EXPORT AsyncListener : public AsyncListenerBase {
|
| 60 |
+
public:
|
| 61 |
+
/// \brief Get the next server result.
|
| 62 |
+
///
|
| 63 |
+
/// This will never be called concurrently with itself or OnFinish.
|
| 64 |
+
virtual void OnNext(T message) = 0;
|
| 65 |
+
/// \brief Get the final status.
|
| 66 |
+
///
|
| 67 |
+
/// This will never be called concurrently with itself or OnNext. If the
|
| 68 |
+
/// error comes from the remote server, then a TransportStatusDetail will be
|
| 69 |
+
/// attached. Otherwise, the error is generated by the client-side
|
| 70 |
+
/// transport and will not have a TransportStatusDetail.
|
| 71 |
+
virtual void OnFinish(Status status) = 0;
|
| 72 |
+
};
|
| 73 |
+
|
| 74 |
+
/// @}
|
| 75 |
+
|
| 76 |
+
} // namespace arrow::flight
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/visibility.h
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#if defined(_WIN32) || defined(__CYGWIN__)
|
| 21 |
+
# if defined(_MSC_VER)
|
| 22 |
+
# pragma warning(push)
|
| 23 |
+
# pragma warning(disable : 4251)
|
| 24 |
+
# else
|
| 25 |
+
# pragma GCC diagnostic ignored "-Wattributes"
|
| 26 |
+
# endif
|
| 27 |
+
|
| 28 |
+
# ifdef ARROW_FLIGHT_STATIC
|
| 29 |
+
# define ARROW_FLIGHT_EXPORT
|
| 30 |
+
# elif defined(ARROW_FLIGHT_EXPORTING)
|
| 31 |
+
# define ARROW_FLIGHT_EXPORT __declspec(dllexport)
|
| 32 |
+
# else
|
| 33 |
+
# define ARROW_FLIGHT_EXPORT __declspec(dllimport)
|
| 34 |
+
# endif
|
| 35 |
+
|
| 36 |
+
# define ARROW_FLIGHT_NO_EXPORT
|
| 37 |
+
#else // Not Windows
|
| 38 |
+
# ifndef ARROW_FLIGHT_EXPORT
|
| 39 |
+
# define ARROW_FLIGHT_EXPORT __attribute__((visibility("default")))
|
| 40 |
+
# endif
|
| 41 |
+
# ifndef ARROW_FLIGHT_NO_EXPORT
|
| 42 |
+
# define ARROW_FLIGHT_NO_EXPORT __attribute__((visibility("hidden")))
|
| 43 |
+
# endif
|
| 44 |
+
#endif // Non-Windows
|
| 45 |
+
|
| 46 |
+
#if defined(_MSC_VER)
|
| 47 |
+
# pragma warning(pop)
|
| 48 |
+
#endif
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/api.h
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include "arrow/io/buffered.h"
|
| 21 |
+
#include "arrow/io/compressed.h"
|
| 22 |
+
#include "arrow/io/file.h"
|
| 23 |
+
#include "arrow/io/hdfs.h"
|
| 24 |
+
#include "arrow/io/interfaces.h"
|
| 25 |
+
#include "arrow/io/memory.h"
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/buffered.h
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
// Buffered stream implementations
|
| 19 |
+
|
| 20 |
+
#pragma once
|
| 21 |
+
|
| 22 |
+
#include <cstdint>
|
| 23 |
+
#include <memory>
|
| 24 |
+
#include <string_view>
|
| 25 |
+
|
| 26 |
+
#include "arrow/io/concurrency.h"
|
| 27 |
+
#include "arrow/io/interfaces.h"
|
| 28 |
+
#include "arrow/util/visibility.h"
|
| 29 |
+
|
| 30 |
+
namespace arrow {
|
| 31 |
+
|
| 32 |
+
class Buffer;
|
| 33 |
+
class MemoryPool;
|
| 34 |
+
class Status;
|
| 35 |
+
|
| 36 |
+
namespace io {
|
| 37 |
+
|
| 38 |
+
class ARROW_EXPORT BufferedOutputStream : public OutputStream {
|
| 39 |
+
public:
|
| 40 |
+
~BufferedOutputStream() override;
|
| 41 |
+
|
| 42 |
+
/// \brief Create a buffered output stream wrapping the given output stream.
|
| 43 |
+
/// \param[in] buffer_size the size of the temporary write buffer
|
| 44 |
+
/// \param[in] pool a MemoryPool to use for allocations
|
| 45 |
+
/// \param[in] raw another OutputStream
|
| 46 |
+
/// \return the created BufferedOutputStream
|
| 47 |
+
static Result<std::shared_ptr<BufferedOutputStream>> Create(
|
| 48 |
+
int64_t buffer_size, MemoryPool* pool, std::shared_ptr<OutputStream> raw);
|
| 49 |
+
|
| 50 |
+
/// \brief Resize internal buffer
|
| 51 |
+
/// \param[in] new_buffer_size the new buffer size
|
| 52 |
+
/// \return Status
|
| 53 |
+
Status SetBufferSize(int64_t new_buffer_size);
|
| 54 |
+
|
| 55 |
+
/// \brief Return the current size of the internal buffer
|
| 56 |
+
int64_t buffer_size() const;
|
| 57 |
+
|
| 58 |
+
/// \brief Return the number of remaining bytes that have not been flushed to
|
| 59 |
+
/// the raw OutputStream
|
| 60 |
+
int64_t bytes_buffered() const;
|
| 61 |
+
|
| 62 |
+
/// \brief Flush any buffered writes and release the raw
|
| 63 |
+
/// OutputStream. Further operations on this object are invalid
|
| 64 |
+
/// \return the underlying OutputStream
|
| 65 |
+
Result<std::shared_ptr<OutputStream>> Detach();
|
| 66 |
+
|
| 67 |
+
// OutputStream interface
|
| 68 |
+
|
| 69 |
+
/// \brief Close the buffered output stream. This implicitly closes the
|
| 70 |
+
/// underlying raw output stream.
|
| 71 |
+
Status Close() override;
|
| 72 |
+
Status Abort() override;
|
| 73 |
+
bool closed() const override;
|
| 74 |
+
|
| 75 |
+
Result<int64_t> Tell() const override;
|
| 76 |
+
// Write bytes to the stream. Thread-safe
|
| 77 |
+
Status Write(const void* data, int64_t nbytes) override;
|
| 78 |
+
Status Write(const std::shared_ptr<Buffer>& data) override;
|
| 79 |
+
|
| 80 |
+
Status Flush() override;
|
| 81 |
+
|
| 82 |
+
/// \brief Return the underlying raw output stream.
|
| 83 |
+
std::shared_ptr<OutputStream> raw() const;
|
| 84 |
+
|
| 85 |
+
private:
|
| 86 |
+
explicit BufferedOutputStream(std::shared_ptr<OutputStream> raw, MemoryPool* pool);
|
| 87 |
+
|
| 88 |
+
class ARROW_NO_EXPORT Impl;
|
| 89 |
+
std::unique_ptr<Impl> impl_;
|
| 90 |
+
};
|
| 91 |
+
|
| 92 |
+
/// \class BufferedInputStream
|
| 93 |
+
/// \brief An InputStream that performs buffered reads from an unbuffered
|
| 94 |
+
/// InputStream, which can mitigate the overhead of many small reads in some
|
| 95 |
+
/// cases
|
| 96 |
+
class ARROW_EXPORT BufferedInputStream
|
| 97 |
+
: public internal::InputStreamConcurrencyWrapper<BufferedInputStream> {
|
| 98 |
+
public:
|
| 99 |
+
~BufferedInputStream() override;
|
| 100 |
+
|
| 101 |
+
/// \brief Create a BufferedInputStream from a raw InputStream
|
| 102 |
+
/// \param[in] buffer_size the size of the temporary read buffer
|
| 103 |
+
/// \param[in] pool a MemoryPool to use for allocations
|
| 104 |
+
/// \param[in] raw a raw InputStream
|
| 105 |
+
/// \param[in] raw_read_bound a bound on the maximum number of bytes
|
| 106 |
+
/// to read from the raw input stream. The default -1 indicates that
|
| 107 |
+
/// it is unbounded
|
| 108 |
+
/// \return the created BufferedInputStream
|
| 109 |
+
static Result<std::shared_ptr<BufferedInputStream>> Create(
|
| 110 |
+
int64_t buffer_size, MemoryPool* pool, std::shared_ptr<InputStream> raw,
|
| 111 |
+
int64_t raw_read_bound = -1);
|
| 112 |
+
|
| 113 |
+
/// \brief Resize internal read buffer; calls to Read(...) will read at least
|
| 114 |
+
/// this many bytes from the raw InputStream if possible.
|
| 115 |
+
/// \param[in] new_buffer_size the new read buffer size
|
| 116 |
+
/// \return Status
|
| 117 |
+
Status SetBufferSize(int64_t new_buffer_size);
|
| 118 |
+
|
| 119 |
+
/// \brief Return the number of remaining bytes in the read buffer
|
| 120 |
+
int64_t bytes_buffered() const;
|
| 121 |
+
|
| 122 |
+
/// \brief Return the current size of the internal buffer
|
| 123 |
+
int64_t buffer_size() const;
|
| 124 |
+
|
| 125 |
+
/// \brief Release the raw InputStream. Any data buffered will be
|
| 126 |
+
/// discarded. Further operations on this object are invalid
|
| 127 |
+
/// \return raw the underlying InputStream
|
| 128 |
+
std::shared_ptr<InputStream> Detach();
|
| 129 |
+
|
| 130 |
+
/// \brief Return the unbuffered InputStream
|
| 131 |
+
std::shared_ptr<InputStream> raw() const;
|
| 132 |
+
|
| 133 |
+
// InputStream APIs
|
| 134 |
+
|
| 135 |
+
bool closed() const override;
|
| 136 |
+
Result<std::shared_ptr<const KeyValueMetadata>> ReadMetadata() override;
|
| 137 |
+
Future<std::shared_ptr<const KeyValueMetadata>> ReadMetadataAsync(
|
| 138 |
+
const IOContext& io_context) override;
|
| 139 |
+
|
| 140 |
+
private:
|
| 141 |
+
friend InputStreamConcurrencyWrapper<BufferedInputStream>;
|
| 142 |
+
|
| 143 |
+
explicit BufferedInputStream(std::shared_ptr<InputStream> raw, MemoryPool* pool,
|
| 144 |
+
int64_t raw_total_bytes_bound);
|
| 145 |
+
|
| 146 |
+
Status DoClose();
|
| 147 |
+
Status DoAbort() override;
|
| 148 |
+
|
| 149 |
+
/// \brief Returns the position of the buffered stream, though the position
|
| 150 |
+
/// of the unbuffered stream may be further advanced.
|
| 151 |
+
Result<int64_t> DoTell() const;
|
| 152 |
+
|
| 153 |
+
Result<int64_t> DoRead(int64_t nbytes, void* out);
|
| 154 |
+
|
| 155 |
+
/// \brief Read into buffer.
|
| 156 |
+
Result<std::shared_ptr<Buffer>> DoRead(int64_t nbytes);
|
| 157 |
+
|
| 158 |
+
/// \brief Return a zero-copy string view referencing buffered data,
|
| 159 |
+
/// but do not advance the position of the stream. Buffers data and
|
| 160 |
+
/// expands the buffer size if necessary
|
| 161 |
+
Result<std::string_view> DoPeek(int64_t nbytes) override;
|
| 162 |
+
|
| 163 |
+
class ARROW_NO_EXPORT Impl;
|
| 164 |
+
std::unique_ptr<Impl> impl_;
|
| 165 |
+
};
|
| 166 |
+
|
| 167 |
+
} // namespace io
|
| 168 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/caching.h
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <cstdint>
|
| 21 |
+
#include <memory>
|
| 22 |
+
#include <string>
|
| 23 |
+
#include <utility>
|
| 24 |
+
#include <vector>
|
| 25 |
+
|
| 26 |
+
#include "arrow/io/interfaces.h"
|
| 27 |
+
#include "arrow/util/type_fwd.h"
|
| 28 |
+
#include "arrow/util/visibility.h"
|
| 29 |
+
|
| 30 |
+
namespace arrow {
|
| 31 |
+
namespace io {
|
| 32 |
+
|
| 33 |
+
struct ARROW_EXPORT CacheOptions {
|
| 34 |
+
static constexpr double kDefaultIdealBandwidthUtilizationFrac = 0.9;
|
| 35 |
+
static constexpr int64_t kDefaultMaxIdealRequestSizeMib = 64;
|
| 36 |
+
|
| 37 |
+
/// \brief The maximum distance in bytes between two consecutive
|
| 38 |
+
/// ranges; beyond this value, ranges are not combined
|
| 39 |
+
int64_t hole_size_limit;
|
| 40 |
+
/// \brief The maximum size in bytes of a combined range; if
|
| 41 |
+
/// combining two consecutive ranges would produce a range of a
|
| 42 |
+
/// size greater than this, they are not combined
|
| 43 |
+
int64_t range_size_limit;
|
| 44 |
+
/// \brief A lazy cache does not perform any I/O until requested.
|
| 45 |
+
/// lazy = false: request all byte ranges when PreBuffer or WillNeed is called.
|
| 46 |
+
/// lazy = True, prefetch_limit = 0: request merged byte ranges only after the reader
|
| 47 |
+
/// needs them.
|
| 48 |
+
/// lazy = True, prefetch_limit = k: prefetch up to k merged byte ranges ahead of the
|
| 49 |
+
/// range that is currently being read.
|
| 50 |
+
bool lazy;
|
| 51 |
+
/// \brief The maximum number of ranges to be prefetched. This is only used
|
| 52 |
+
/// for lazy cache to asynchronously read some ranges after reading the target range.
|
| 53 |
+
int64_t prefetch_limit = 0;
|
| 54 |
+
|
| 55 |
+
bool operator==(const CacheOptions& other) const {
|
| 56 |
+
return hole_size_limit == other.hole_size_limit &&
|
| 57 |
+
range_size_limit == other.range_size_limit && lazy == other.lazy &&
|
| 58 |
+
prefetch_limit == other.prefetch_limit;
|
| 59 |
+
}
|
| 60 |
+
|
| 61 |
+
/// \brief Construct CacheOptions from network storage metrics (e.g. S3).
|
| 62 |
+
///
|
| 63 |
+
/// \param[in] time_to_first_byte_millis Seek-time or Time-To-First-Byte (TTFB) in
|
| 64 |
+
/// milliseconds, also called call setup latency of a new read request.
|
| 65 |
+
/// The value is a positive integer.
|
| 66 |
+
/// \param[in] transfer_bandwidth_mib_per_sec Data transfer Bandwidth (BW) in MiB/sec
|
| 67 |
+
/// (per connection).
|
| 68 |
+
/// The value is a positive integer.
|
| 69 |
+
/// \param[in] ideal_bandwidth_utilization_frac Transfer bandwidth utilization fraction
|
| 70 |
+
/// (per connection) to maximize the net data load.
|
| 71 |
+
/// The value is a positive double precision number less than 1.
|
| 72 |
+
/// \param[in] max_ideal_request_size_mib The maximum single data request size (in MiB)
|
| 73 |
+
/// to maximize the net data load.
|
| 74 |
+
/// The value is a positive integer.
|
| 75 |
+
/// \return A new instance of CacheOptions.
|
| 76 |
+
static CacheOptions MakeFromNetworkMetrics(
|
| 77 |
+
int64_t time_to_first_byte_millis, int64_t transfer_bandwidth_mib_per_sec,
|
| 78 |
+
double ideal_bandwidth_utilization_frac = kDefaultIdealBandwidthUtilizationFrac,
|
| 79 |
+
int64_t max_ideal_request_size_mib = kDefaultMaxIdealRequestSizeMib);
|
| 80 |
+
|
| 81 |
+
static CacheOptions Defaults();
|
| 82 |
+
static CacheOptions LazyDefaults();
|
| 83 |
+
};
|
| 84 |
+
|
| 85 |
+
namespace internal {
|
| 86 |
+
|
| 87 |
+
/// \brief A read cache designed to hide IO latencies when reading.
|
| 88 |
+
///
|
| 89 |
+
/// This class takes multiple byte ranges that an application expects to read, and
|
| 90 |
+
/// coalesces them into fewer, larger read requests, which benefits performance on some
|
| 91 |
+
/// filesystems, particularly remote ones like Amazon S3. By default, it also issues
|
| 92 |
+
/// these read requests in parallel up front.
|
| 93 |
+
///
|
| 94 |
+
/// To use:
|
| 95 |
+
/// 1. Cache() the ranges you expect to read in the future. Ideally, these ranges have
|
| 96 |
+
/// the exact offset and length that will later be read. The cache will combine those
|
| 97 |
+
/// ranges according to parameters (see constructor).
|
| 98 |
+
///
|
| 99 |
+
/// By default, the cache will also start fetching the combined ranges in parallel in
|
| 100 |
+
/// the background, unless CacheOptions.lazy is set.
|
| 101 |
+
///
|
| 102 |
+
/// 2. Call WaitFor() to be notified when the given ranges have been read. If
|
| 103 |
+
/// CacheOptions.lazy is set, I/O will be triggered in the background here instead.
|
| 104 |
+
/// This can be done in parallel (e.g. if parsing a file, call WaitFor() for each
|
| 105 |
+
/// chunk of the file that can be parsed in parallel).
|
| 106 |
+
///
|
| 107 |
+
/// 3. Call Read() to retrieve the actual data for the given ranges.
|
| 108 |
+
/// A synchronous application may skip WaitFor() and just call Read() - it will still
|
| 109 |
+
/// benefit from coalescing and parallel fetching.
|
| 110 |
+
class ARROW_EXPORT ReadRangeCache {
|
| 111 |
+
public:
|
| 112 |
+
static constexpr int64_t kDefaultHoleSizeLimit = 8192;
|
| 113 |
+
static constexpr int64_t kDefaultRangeSizeLimit = 32 * 1024 * 1024;
|
| 114 |
+
|
| 115 |
+
/// Construct a read cache with default
|
| 116 |
+
explicit ReadRangeCache(std::shared_ptr<RandomAccessFile> file, IOContext ctx)
|
| 117 |
+
: ReadRangeCache(file, file.get(), std::move(ctx), CacheOptions::Defaults()) {}
|
| 118 |
+
|
| 119 |
+
/// Construct a read cache with given options
|
| 120 |
+
explicit ReadRangeCache(std::shared_ptr<RandomAccessFile> file, IOContext ctx,
|
| 121 |
+
CacheOptions options)
|
| 122 |
+
: ReadRangeCache(file, file.get(), std::move(ctx), options) {}
|
| 123 |
+
|
| 124 |
+
/// Construct a read cache with an unowned file
|
| 125 |
+
ReadRangeCache(RandomAccessFile* file, IOContext ctx, CacheOptions options)
|
| 126 |
+
: ReadRangeCache(NULLPTR, file, std::move(ctx), options) {}
|
| 127 |
+
|
| 128 |
+
~ReadRangeCache();
|
| 129 |
+
|
| 130 |
+
/// \brief Cache the given ranges in the background.
|
| 131 |
+
///
|
| 132 |
+
/// The caller must ensure that the ranges do not overlap with each other,
|
| 133 |
+
/// nor with previously cached ranges. Otherwise, behaviour will be undefined.
|
| 134 |
+
Status Cache(std::vector<ReadRange> ranges);
|
| 135 |
+
|
| 136 |
+
/// \brief Read a range previously given to Cache().
|
| 137 |
+
Result<std::shared_ptr<Buffer>> Read(ReadRange range);
|
| 138 |
+
|
| 139 |
+
/// \brief Wait until all ranges added so far have been cached.
|
| 140 |
+
Future<> Wait();
|
| 141 |
+
|
| 142 |
+
/// \brief Wait until all given ranges have been cached.
|
| 143 |
+
Future<> WaitFor(std::vector<ReadRange> ranges);
|
| 144 |
+
|
| 145 |
+
protected:
|
| 146 |
+
struct Impl;
|
| 147 |
+
struct LazyImpl;
|
| 148 |
+
|
| 149 |
+
ReadRangeCache(std::shared_ptr<RandomAccessFile> owned_file, RandomAccessFile* file,
|
| 150 |
+
IOContext ctx, CacheOptions options);
|
| 151 |
+
|
| 152 |
+
std::unique_ptr<Impl> impl_;
|
| 153 |
+
};
|
| 154 |
+
|
| 155 |
+
} // namespace internal
|
| 156 |
+
} // namespace io
|
| 157 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/compressed.h
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
// Compressed stream implementations
|
| 19 |
+
|
| 20 |
+
#pragma once
|
| 21 |
+
|
| 22 |
+
#include <memory>
|
| 23 |
+
#include <string>
|
| 24 |
+
|
| 25 |
+
#include "arrow/io/concurrency.h"
|
| 26 |
+
#include "arrow/io/interfaces.h"
|
| 27 |
+
#include "arrow/util/visibility.h"
|
| 28 |
+
|
| 29 |
+
namespace arrow {
|
| 30 |
+
|
| 31 |
+
class MemoryPool;
|
| 32 |
+
class Status;
|
| 33 |
+
|
| 34 |
+
namespace util {
|
| 35 |
+
|
| 36 |
+
class Codec;
|
| 37 |
+
|
| 38 |
+
} // namespace util
|
| 39 |
+
|
| 40 |
+
namespace io {
|
| 41 |
+
|
| 42 |
+
class ARROW_EXPORT CompressedOutputStream : public OutputStream {
|
| 43 |
+
public:
|
| 44 |
+
~CompressedOutputStream() override;
|
| 45 |
+
|
| 46 |
+
/// \brief Create a compressed output stream wrapping the given output stream.
|
| 47 |
+
///
|
| 48 |
+
/// The codec must be capable of streaming compression. Some codecs,
|
| 49 |
+
/// like Snappy, are not able to do so.
|
| 50 |
+
static Result<std::shared_ptr<CompressedOutputStream>> Make(
|
| 51 |
+
util::Codec* codec, const std::shared_ptr<OutputStream>& raw,
|
| 52 |
+
MemoryPool* pool = default_memory_pool());
|
| 53 |
+
|
| 54 |
+
// OutputStream interface
|
| 55 |
+
|
| 56 |
+
/// \brief Close the compressed output stream. This implicitly closes the
|
| 57 |
+
/// underlying raw output stream.
|
| 58 |
+
Status Close() override;
|
| 59 |
+
Status Abort() override;
|
| 60 |
+
bool closed() const override;
|
| 61 |
+
|
| 62 |
+
Result<int64_t> Tell() const override;
|
| 63 |
+
|
| 64 |
+
Status Write(const void* data, int64_t nbytes) override;
|
| 65 |
+
/// \cond FALSE
|
| 66 |
+
using Writable::Write;
|
| 67 |
+
/// \endcond
|
| 68 |
+
Status Flush() override;
|
| 69 |
+
|
| 70 |
+
/// \brief Return the underlying raw output stream.
|
| 71 |
+
std::shared_ptr<OutputStream> raw() const;
|
| 72 |
+
|
| 73 |
+
private:
|
| 74 |
+
ARROW_DISALLOW_COPY_AND_ASSIGN(CompressedOutputStream);
|
| 75 |
+
|
| 76 |
+
CompressedOutputStream() = default;
|
| 77 |
+
|
| 78 |
+
class ARROW_NO_EXPORT Impl;
|
| 79 |
+
std::unique_ptr<Impl> impl_;
|
| 80 |
+
};
|
| 81 |
+
|
| 82 |
+
class ARROW_EXPORT CompressedInputStream
|
| 83 |
+
: public internal::InputStreamConcurrencyWrapper<CompressedInputStream> {
|
| 84 |
+
public:
|
| 85 |
+
~CompressedInputStream() override;
|
| 86 |
+
|
| 87 |
+
/// \brief Create a compressed input stream wrapping the given input stream.
|
| 88 |
+
///
|
| 89 |
+
/// The codec must be capable of streaming decompression. Some codecs,
|
| 90 |
+
/// like Snappy, are not able to do so.
|
| 91 |
+
static Result<std::shared_ptr<CompressedInputStream>> Make(
|
| 92 |
+
util::Codec* codec, const std::shared_ptr<InputStream>& raw,
|
| 93 |
+
MemoryPool* pool = default_memory_pool());
|
| 94 |
+
|
| 95 |
+
// InputStream interface
|
| 96 |
+
|
| 97 |
+
bool closed() const override;
|
| 98 |
+
Result<std::shared_ptr<const KeyValueMetadata>> ReadMetadata() override;
|
| 99 |
+
Future<std::shared_ptr<const KeyValueMetadata>> ReadMetadataAsync(
|
| 100 |
+
const IOContext& io_context) override;
|
| 101 |
+
|
| 102 |
+
/// \brief Return the underlying raw input stream.
|
| 103 |
+
std::shared_ptr<InputStream> raw() const;
|
| 104 |
+
|
| 105 |
+
private:
|
| 106 |
+
friend InputStreamConcurrencyWrapper<CompressedInputStream>;
|
| 107 |
+
ARROW_DISALLOW_COPY_AND_ASSIGN(CompressedInputStream);
|
| 108 |
+
|
| 109 |
+
CompressedInputStream() = default;
|
| 110 |
+
|
| 111 |
+
/// \brief Close the compressed input stream. This implicitly closes the
|
| 112 |
+
/// underlying raw input stream.
|
| 113 |
+
Status DoClose();
|
| 114 |
+
Status DoAbort() override;
|
| 115 |
+
Result<int64_t> DoTell() const;
|
| 116 |
+
Result<int64_t> DoRead(int64_t nbytes, void* out);
|
| 117 |
+
Result<std::shared_ptr<Buffer>> DoRead(int64_t nbytes);
|
| 118 |
+
|
| 119 |
+
class ARROW_NO_EXPORT Impl;
|
| 120 |
+
std::unique_ptr<Impl> impl_;
|
| 121 |
+
};
|
| 122 |
+
|
| 123 |
+
} // namespace io
|
| 124 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/concurrency.h
ADDED
|
@@ -0,0 +1,263 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <memory>
|
| 21 |
+
|
| 22 |
+
#include "arrow/io/interfaces.h"
|
| 23 |
+
#include "arrow/result.h"
|
| 24 |
+
#include "arrow/status.h"
|
| 25 |
+
#include "arrow/util/checked_cast.h"
|
| 26 |
+
#include "arrow/util/macros.h"
|
| 27 |
+
#include "arrow/util/visibility.h"
|
| 28 |
+
|
| 29 |
+
namespace arrow {
|
| 30 |
+
namespace io {
|
| 31 |
+
namespace internal {
|
| 32 |
+
|
| 33 |
+
template <class LockType>
|
| 34 |
+
class SharedLockGuard {
|
| 35 |
+
public:
|
| 36 |
+
explicit SharedLockGuard(LockType* lock) : lock_(lock) { lock_->LockShared(); }
|
| 37 |
+
|
| 38 |
+
~SharedLockGuard() { lock_->UnlockShared(); }
|
| 39 |
+
|
| 40 |
+
protected:
|
| 41 |
+
LockType* lock_;
|
| 42 |
+
};
|
| 43 |
+
|
| 44 |
+
template <class LockType>
|
| 45 |
+
class ExclusiveLockGuard {
|
| 46 |
+
public:
|
| 47 |
+
explicit ExclusiveLockGuard(LockType* lock) : lock_(lock) { lock_->LockExclusive(); }
|
| 48 |
+
|
| 49 |
+
~ExclusiveLockGuard() { lock_->UnlockExclusive(); }
|
| 50 |
+
|
| 51 |
+
protected:
|
| 52 |
+
LockType* lock_;
|
| 53 |
+
};
|
| 54 |
+
|
| 55 |
+
// Debug concurrency checker that marks "shared" and "exclusive" code sections,
|
| 56 |
+
// aborting if the concurrency rules get violated. Does nothing in release mode.
|
| 57 |
+
// Note that we intentionally use the same class declaration in debug and
|
| 58 |
+
// release builds in order to avoid runtime failures when e.g. loading a
|
| 59 |
+
// release-built DLL with a debug-built application, or the reverse.
|
| 60 |
+
|
| 61 |
+
class ARROW_EXPORT SharedExclusiveChecker {
|
| 62 |
+
public:
|
| 63 |
+
SharedExclusiveChecker();
|
| 64 |
+
void LockShared();
|
| 65 |
+
void UnlockShared();
|
| 66 |
+
void LockExclusive();
|
| 67 |
+
void UnlockExclusive();
|
| 68 |
+
|
| 69 |
+
SharedLockGuard<SharedExclusiveChecker> shared_guard() {
|
| 70 |
+
return SharedLockGuard<SharedExclusiveChecker>(this);
|
| 71 |
+
}
|
| 72 |
+
|
| 73 |
+
ExclusiveLockGuard<SharedExclusiveChecker> exclusive_guard() {
|
| 74 |
+
return ExclusiveLockGuard<SharedExclusiveChecker>(this);
|
| 75 |
+
}
|
| 76 |
+
|
| 77 |
+
protected:
|
| 78 |
+
struct Impl;
|
| 79 |
+
std::shared_ptr<Impl> impl_;
|
| 80 |
+
};
|
| 81 |
+
|
| 82 |
+
// Concurrency wrappers for IO classes that check the correctness of
|
| 83 |
+
// concurrent calls to various methods. It is not necessary to wrap all
|
| 84 |
+
// IO classes with these, only a few core classes that get used in tests.
|
| 85 |
+
//
|
| 86 |
+
// We're not using virtual inheritance here as virtual bases have poorly
|
| 87 |
+
// understood semantic overhead which we'd be passing on to implementers
|
| 88 |
+
// and users of these interfaces. Instead, we just duplicate the method
|
| 89 |
+
// wrappers between those two classes.
|
| 90 |
+
|
| 91 |
+
template <class Derived>
|
| 92 |
+
class InputStreamConcurrencyWrapper : public InputStream {
|
| 93 |
+
public:
|
| 94 |
+
Status Close() final {
|
| 95 |
+
auto guard = lock_.exclusive_guard();
|
| 96 |
+
return derived()->DoClose();
|
| 97 |
+
}
|
| 98 |
+
|
| 99 |
+
Status Abort() final {
|
| 100 |
+
auto guard = lock_.exclusive_guard();
|
| 101 |
+
return derived()->DoAbort();
|
| 102 |
+
}
|
| 103 |
+
|
| 104 |
+
Result<int64_t> Tell() const final {
|
| 105 |
+
auto guard = lock_.exclusive_guard();
|
| 106 |
+
return derived()->DoTell();
|
| 107 |
+
}
|
| 108 |
+
|
| 109 |
+
Result<int64_t> Read(int64_t nbytes, void* out) final {
|
| 110 |
+
auto guard = lock_.exclusive_guard();
|
| 111 |
+
return derived()->DoRead(nbytes, out);
|
| 112 |
+
}
|
| 113 |
+
|
| 114 |
+
Result<std::shared_ptr<Buffer>> Read(int64_t nbytes) final {
|
| 115 |
+
auto guard = lock_.exclusive_guard();
|
| 116 |
+
return derived()->DoRead(nbytes);
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
Result<std::string_view> Peek(int64_t nbytes) final {
|
| 120 |
+
auto guard = lock_.exclusive_guard();
|
| 121 |
+
return derived()->DoPeek(nbytes);
|
| 122 |
+
}
|
| 123 |
+
|
| 124 |
+
/*
|
| 125 |
+
Methods to implement in derived class:
|
| 126 |
+
|
| 127 |
+
Status DoClose();
|
| 128 |
+
Result<int64_t> DoTell() const;
|
| 129 |
+
Result<int64_t> DoRead(int64_t nbytes, void* out);
|
| 130 |
+
Result<std::shared_ptr<Buffer>> DoRead(int64_t nbytes);
|
| 131 |
+
|
| 132 |
+
And optionally:
|
| 133 |
+
|
| 134 |
+
Status DoAbort() override;
|
| 135 |
+
Result<std::string_view> DoPeek(int64_t nbytes) override;
|
| 136 |
+
|
| 137 |
+
These methods should be protected in the derived class and
|
| 138 |
+
InputStreamConcurrencyWrapper declared as a friend with
|
| 139 |
+
|
| 140 |
+
friend InputStreamConcurrencyWrapper<derived>;
|
| 141 |
+
*/
|
| 142 |
+
|
| 143 |
+
protected:
|
| 144 |
+
// Default implementations. They are virtual because the derived class may
|
| 145 |
+
// have derived classes itself.
|
| 146 |
+
virtual Status DoAbort() { return derived()->DoClose(); }
|
| 147 |
+
|
| 148 |
+
virtual Result<std::string_view> DoPeek(int64_t ARROW_ARG_UNUSED(nbytes)) {
|
| 149 |
+
return Status::NotImplemented("Peek not implemented");
|
| 150 |
+
}
|
| 151 |
+
|
| 152 |
+
Derived* derived() { return ::arrow::internal::checked_cast<Derived*>(this); }
|
| 153 |
+
|
| 154 |
+
const Derived* derived() const {
|
| 155 |
+
return ::arrow::internal::checked_cast<const Derived*>(this);
|
| 156 |
+
}
|
| 157 |
+
|
| 158 |
+
mutable SharedExclusiveChecker lock_;
|
| 159 |
+
};
|
| 160 |
+
|
| 161 |
+
template <class Derived>
|
| 162 |
+
class RandomAccessFileConcurrencyWrapper : public RandomAccessFile {
|
| 163 |
+
public:
|
| 164 |
+
Status Close() final {
|
| 165 |
+
auto guard = lock_.exclusive_guard();
|
| 166 |
+
return derived()->DoClose();
|
| 167 |
+
}
|
| 168 |
+
|
| 169 |
+
Status Abort() final {
|
| 170 |
+
auto guard = lock_.exclusive_guard();
|
| 171 |
+
return derived()->DoAbort();
|
| 172 |
+
}
|
| 173 |
+
|
| 174 |
+
Result<int64_t> Tell() const final {
|
| 175 |
+
auto guard = lock_.exclusive_guard();
|
| 176 |
+
return derived()->DoTell();
|
| 177 |
+
}
|
| 178 |
+
|
| 179 |
+
Result<int64_t> Read(int64_t nbytes, void* out) final {
|
| 180 |
+
auto guard = lock_.exclusive_guard();
|
| 181 |
+
return derived()->DoRead(nbytes, out);
|
| 182 |
+
}
|
| 183 |
+
|
| 184 |
+
Result<std::shared_ptr<Buffer>> Read(int64_t nbytes) final {
|
| 185 |
+
auto guard = lock_.exclusive_guard();
|
| 186 |
+
return derived()->DoRead(nbytes);
|
| 187 |
+
}
|
| 188 |
+
|
| 189 |
+
Result<std::string_view> Peek(int64_t nbytes) final {
|
| 190 |
+
auto guard = lock_.exclusive_guard();
|
| 191 |
+
return derived()->DoPeek(nbytes);
|
| 192 |
+
}
|
| 193 |
+
|
| 194 |
+
Status Seek(int64_t position) final {
|
| 195 |
+
auto guard = lock_.exclusive_guard();
|
| 196 |
+
return derived()->DoSeek(position);
|
| 197 |
+
}
|
| 198 |
+
|
| 199 |
+
Result<int64_t> GetSize() final {
|
| 200 |
+
auto guard = lock_.shared_guard();
|
| 201 |
+
return derived()->DoGetSize();
|
| 202 |
+
}
|
| 203 |
+
|
| 204 |
+
// NOTE: ReadAt doesn't use stream pointer, but it is allowed to update it
|
| 205 |
+
// (it's the case on Windows when using ReadFileEx).
|
| 206 |
+
// So any method that relies on the current position (even if it doesn't
|
| 207 |
+
// update it, such as Peek) cannot run in parallel with ReadAt and has
|
| 208 |
+
// to use the exclusive_guard.
|
| 209 |
+
|
| 210 |
+
Result<int64_t> ReadAt(int64_t position, int64_t nbytes, void* out) final {
|
| 211 |
+
auto guard = lock_.shared_guard();
|
| 212 |
+
return derived()->DoReadAt(position, nbytes, out);
|
| 213 |
+
}
|
| 214 |
+
|
| 215 |
+
Result<std::shared_ptr<Buffer>> ReadAt(int64_t position, int64_t nbytes) final {
|
| 216 |
+
auto guard = lock_.shared_guard();
|
| 217 |
+
return derived()->DoReadAt(position, nbytes);
|
| 218 |
+
}
|
| 219 |
+
|
| 220 |
+
/*
|
| 221 |
+
Methods to implement in derived class:
|
| 222 |
+
|
| 223 |
+
Status DoClose();
|
| 224 |
+
Result<int64_t> DoTell() const;
|
| 225 |
+
Result<int64_t> DoRead(int64_t nbytes, void* out);
|
| 226 |
+
Result<std::shared_ptr<Buffer>> DoRead(int64_t nbytes);
|
| 227 |
+
Status DoSeek(int64_t position);
|
| 228 |
+
Result<int64_t> DoGetSize()
|
| 229 |
+
Result<int64_t> DoReadAt(int64_t position, int64_t nbytes, void* out);
|
| 230 |
+
Result<std::shared_ptr<Buffer>> DoReadAt(int64_t position, int64_t nbytes);
|
| 231 |
+
|
| 232 |
+
And optionally:
|
| 233 |
+
|
| 234 |
+
Status DoAbort() override;
|
| 235 |
+
Result<std::string_view> DoPeek(int64_t nbytes) override;
|
| 236 |
+
|
| 237 |
+
These methods should be protected in the derived class and
|
| 238 |
+
RandomAccessFileConcurrencyWrapper declared as a friend with
|
| 239 |
+
|
| 240 |
+
friend RandomAccessFileConcurrencyWrapper<derived>;
|
| 241 |
+
*/
|
| 242 |
+
|
| 243 |
+
protected:
|
| 244 |
+
// Default implementations. They are virtual because the derived class may
|
| 245 |
+
// have derived classes itself.
|
| 246 |
+
virtual Status DoAbort() { return derived()->DoClose(); }
|
| 247 |
+
|
| 248 |
+
virtual Result<std::string_view> DoPeek(int64_t ARROW_ARG_UNUSED(nbytes)) {
|
| 249 |
+
return Status::NotImplemented("Peek not implemented");
|
| 250 |
+
}
|
| 251 |
+
|
| 252 |
+
Derived* derived() { return ::arrow::internal::checked_cast<Derived*>(this); }
|
| 253 |
+
|
| 254 |
+
const Derived* derived() const {
|
| 255 |
+
return ::arrow::internal::checked_cast<const Derived*>(this);
|
| 256 |
+
}
|
| 257 |
+
|
| 258 |
+
mutable SharedExclusiveChecker lock_;
|
| 259 |
+
};
|
| 260 |
+
|
| 261 |
+
} // namespace internal
|
| 262 |
+
} // namespace io
|
| 263 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/file.h
ADDED
|
@@ -0,0 +1,221 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
// IO interface implementations for OS files
|
| 19 |
+
|
| 20 |
+
#pragma once
|
| 21 |
+
|
| 22 |
+
#include <cstdint>
|
| 23 |
+
#include <memory>
|
| 24 |
+
#include <string>
|
| 25 |
+
#include <vector>
|
| 26 |
+
|
| 27 |
+
#include "arrow/io/concurrency.h"
|
| 28 |
+
#include "arrow/io/interfaces.h"
|
| 29 |
+
#include "arrow/util/visibility.h"
|
| 30 |
+
|
| 31 |
+
namespace arrow {
|
| 32 |
+
|
| 33 |
+
class Buffer;
|
| 34 |
+
class MemoryPool;
|
| 35 |
+
class Status;
|
| 36 |
+
|
| 37 |
+
namespace io {
|
| 38 |
+
|
| 39 |
+
/// \brief An operating system file open in write-only mode.
|
| 40 |
+
class ARROW_EXPORT FileOutputStream : public OutputStream {
|
| 41 |
+
public:
|
| 42 |
+
~FileOutputStream() override;
|
| 43 |
+
|
| 44 |
+
/// \brief Open a local file for writing, truncating any existing file
|
| 45 |
+
/// \param[in] path with UTF8 encoding
|
| 46 |
+
/// \param[in] append append to existing file, otherwise truncate to 0 bytes
|
| 47 |
+
/// \return an open FileOutputStream
|
| 48 |
+
///
|
| 49 |
+
/// When opening a new file, any existing file with the indicated path is
|
| 50 |
+
/// truncated to 0 bytes, deleting any existing data
|
| 51 |
+
static Result<std::shared_ptr<FileOutputStream>> Open(const std::string& path,
|
| 52 |
+
bool append = false);
|
| 53 |
+
|
| 54 |
+
/// \brief Open a file descriptor for writing. The underlying file isn't
|
| 55 |
+
/// truncated.
|
| 56 |
+
/// \param[in] fd file descriptor
|
| 57 |
+
/// \return an open FileOutputStream
|
| 58 |
+
///
|
| 59 |
+
/// The file descriptor becomes owned by the OutputStream, and will be closed
|
| 60 |
+
/// on Close() or destruction.
|
| 61 |
+
static Result<std::shared_ptr<FileOutputStream>> Open(int fd);
|
| 62 |
+
|
| 63 |
+
// OutputStream interface
|
| 64 |
+
Status Close() override;
|
| 65 |
+
bool closed() const override;
|
| 66 |
+
Result<int64_t> Tell() const override;
|
| 67 |
+
|
| 68 |
+
// Write bytes to the stream. Thread-safe
|
| 69 |
+
Status Write(const void* data, int64_t nbytes) override;
|
| 70 |
+
/// \cond FALSE
|
| 71 |
+
using Writable::Write;
|
| 72 |
+
/// \endcond
|
| 73 |
+
|
| 74 |
+
int file_descriptor() const;
|
| 75 |
+
|
| 76 |
+
private:
|
| 77 |
+
FileOutputStream();
|
| 78 |
+
|
| 79 |
+
class ARROW_NO_EXPORT FileOutputStreamImpl;
|
| 80 |
+
std::unique_ptr<FileOutputStreamImpl> impl_;
|
| 81 |
+
};
|
| 82 |
+
|
| 83 |
+
/// \brief An operating system file open in read-only mode.
|
| 84 |
+
///
|
| 85 |
+
/// Reads through this implementation are unbuffered. If many small reads
|
| 86 |
+
/// need to be issued, it is recommended to use a buffering layer for good
|
| 87 |
+
/// performance.
|
| 88 |
+
class ARROW_EXPORT ReadableFile
|
| 89 |
+
: public internal::RandomAccessFileConcurrencyWrapper<ReadableFile> {
|
| 90 |
+
public:
|
| 91 |
+
~ReadableFile() override;
|
| 92 |
+
|
| 93 |
+
/// \brief Open a local file for reading
|
| 94 |
+
/// \param[in] path with UTF8 encoding
|
| 95 |
+
/// \param[in] pool a MemoryPool for memory allocations
|
| 96 |
+
/// \return ReadableFile instance
|
| 97 |
+
static Result<std::shared_ptr<ReadableFile>> Open(
|
| 98 |
+
const std::string& path, MemoryPool* pool = default_memory_pool());
|
| 99 |
+
|
| 100 |
+
/// \brief Open a local file for reading
|
| 101 |
+
/// \param[in] fd file descriptor
|
| 102 |
+
/// \param[in] pool a MemoryPool for memory allocations
|
| 103 |
+
/// \return ReadableFile instance
|
| 104 |
+
///
|
| 105 |
+
/// The file descriptor becomes owned by the ReadableFile, and will be closed
|
| 106 |
+
/// on Close() or destruction.
|
| 107 |
+
static Result<std::shared_ptr<ReadableFile>> Open(
|
| 108 |
+
int fd, MemoryPool* pool = default_memory_pool());
|
| 109 |
+
|
| 110 |
+
bool closed() const override;
|
| 111 |
+
|
| 112 |
+
int file_descriptor() const;
|
| 113 |
+
|
| 114 |
+
Status WillNeed(const std::vector<ReadRange>& ranges) override;
|
| 115 |
+
|
| 116 |
+
private:
|
| 117 |
+
friend RandomAccessFileConcurrencyWrapper<ReadableFile>;
|
| 118 |
+
|
| 119 |
+
explicit ReadableFile(MemoryPool* pool);
|
| 120 |
+
|
| 121 |
+
Status DoClose();
|
| 122 |
+
Result<int64_t> DoTell() const;
|
| 123 |
+
Result<int64_t> DoRead(int64_t nbytes, void* buffer);
|
| 124 |
+
Result<std::shared_ptr<Buffer>> DoRead(int64_t nbytes);
|
| 125 |
+
|
| 126 |
+
/// \brief Thread-safe implementation of ReadAt
|
| 127 |
+
Result<int64_t> DoReadAt(int64_t position, int64_t nbytes, void* out);
|
| 128 |
+
|
| 129 |
+
/// \brief Thread-safe implementation of ReadAt
|
| 130 |
+
Result<std::shared_ptr<Buffer>> DoReadAt(int64_t position, int64_t nbytes);
|
| 131 |
+
|
| 132 |
+
Result<int64_t> DoGetSize();
|
| 133 |
+
Status DoSeek(int64_t position);
|
| 134 |
+
|
| 135 |
+
class ARROW_NO_EXPORT ReadableFileImpl;
|
| 136 |
+
std::unique_ptr<ReadableFileImpl> impl_;
|
| 137 |
+
};
|
| 138 |
+
|
| 139 |
+
/// \brief A file interface that uses memory-mapped files for memory interactions
|
| 140 |
+
///
|
| 141 |
+
/// This implementation supports zero-copy reads. The same class is used
|
| 142 |
+
/// for both reading and writing.
|
| 143 |
+
///
|
| 144 |
+
/// If opening a file in a writable mode, it is not truncated first as with
|
| 145 |
+
/// FileOutputStream.
|
| 146 |
+
class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
|
| 147 |
+
public:
|
| 148 |
+
~MemoryMappedFile() override;
|
| 149 |
+
|
| 150 |
+
/// Create new file with indicated size, return in read/write mode
|
| 151 |
+
static Result<std::shared_ptr<MemoryMappedFile>> Create(const std::string& path,
|
| 152 |
+
int64_t size);
|
| 153 |
+
|
| 154 |
+
// mmap() with whole file
|
| 155 |
+
static Result<std::shared_ptr<MemoryMappedFile>> Open(const std::string& path,
|
| 156 |
+
FileMode::type mode);
|
| 157 |
+
|
| 158 |
+
// mmap() with a region of file, the offset must be a multiple of the page size
|
| 159 |
+
static Result<std::shared_ptr<MemoryMappedFile>> Open(const std::string& path,
|
| 160 |
+
FileMode::type mode,
|
| 161 |
+
const int64_t offset,
|
| 162 |
+
const int64_t length);
|
| 163 |
+
|
| 164 |
+
Status Close() override;
|
| 165 |
+
|
| 166 |
+
bool closed() const override;
|
| 167 |
+
|
| 168 |
+
Result<int64_t> Tell() const override;
|
| 169 |
+
|
| 170 |
+
Status Seek(int64_t position) override;
|
| 171 |
+
|
| 172 |
+
// Required by RandomAccessFile, copies memory into out. Not thread-safe
|
| 173 |
+
Result<int64_t> Read(int64_t nbytes, void* out) override;
|
| 174 |
+
|
| 175 |
+
// Zero copy read, moves position pointer. Not thread-safe
|
| 176 |
+
Result<std::shared_ptr<Buffer>> Read(int64_t nbytes) override;
|
| 177 |
+
|
| 178 |
+
// Zero-copy read, leaves position unchanged. Acquires a reader lock
|
| 179 |
+
// for the duration of slice creation (typically very short). Is thread-safe.
|
| 180 |
+
Result<std::shared_ptr<Buffer>> ReadAt(int64_t position, int64_t nbytes) override;
|
| 181 |
+
|
| 182 |
+
// Raw copy of the memory at specified position. Thread-safe, but
|
| 183 |
+
// locks out other readers for the duration of memcpy. Prefer the
|
| 184 |
+
// zero copy method
|
| 185 |
+
Result<int64_t> ReadAt(int64_t position, int64_t nbytes, void* out) override;
|
| 186 |
+
|
| 187 |
+
// Synchronous ReadAsync override
|
| 188 |
+
Future<std::shared_ptr<Buffer>> ReadAsync(const IOContext&, int64_t position,
|
| 189 |
+
int64_t nbytes) override;
|
| 190 |
+
|
| 191 |
+
Status WillNeed(const std::vector<ReadRange>& ranges) override;
|
| 192 |
+
|
| 193 |
+
bool supports_zero_copy() const override;
|
| 194 |
+
|
| 195 |
+
/// Write data at the current position in the file. Thread-safe
|
| 196 |
+
Status Write(const void* data, int64_t nbytes) override;
|
| 197 |
+
/// \cond FALSE
|
| 198 |
+
using Writable::Write;
|
| 199 |
+
/// \endcond
|
| 200 |
+
|
| 201 |
+
/// Set the size of the map to new_size.
|
| 202 |
+
Status Resize(int64_t new_size);
|
| 203 |
+
|
| 204 |
+
/// Write data at a particular position in the file. Thread-safe
|
| 205 |
+
Status WriteAt(int64_t position, const void* data, int64_t nbytes) override;
|
| 206 |
+
|
| 207 |
+
Result<int64_t> GetSize() override;
|
| 208 |
+
|
| 209 |
+
int file_descriptor() const;
|
| 210 |
+
|
| 211 |
+
private:
|
| 212 |
+
MemoryMappedFile();
|
| 213 |
+
|
| 214 |
+
Status WriteInternal(const void* data, int64_t nbytes);
|
| 215 |
+
|
| 216 |
+
class ARROW_NO_EXPORT MemoryMap;
|
| 217 |
+
std::shared_ptr<MemoryMap> memory_map_;
|
| 218 |
+
};
|
| 219 |
+
|
| 220 |
+
} // namespace io
|
| 221 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/hdfs.h
ADDED
|
@@ -0,0 +1,284 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <cstdint>
|
| 21 |
+
#include <memory>
|
| 22 |
+
#include <string>
|
| 23 |
+
#include <unordered_map>
|
| 24 |
+
#include <vector>
|
| 25 |
+
|
| 26 |
+
#include "arrow/io/interfaces.h"
|
| 27 |
+
#include "arrow/util/macros.h"
|
| 28 |
+
#include "arrow/util/visibility.h"
|
| 29 |
+
|
| 30 |
+
namespace arrow {
|
| 31 |
+
|
| 32 |
+
class Buffer;
|
| 33 |
+
class MemoryPool;
|
| 34 |
+
class Status;
|
| 35 |
+
|
| 36 |
+
namespace io {
|
| 37 |
+
|
| 38 |
+
class HdfsReadableFile;
|
| 39 |
+
class HdfsOutputStream;
|
| 40 |
+
|
| 41 |
+
/// DEPRECATED. Use the FileSystem API in arrow::fs instead.
|
| 42 |
+
struct ObjectType {
|
| 43 |
+
enum type { FILE, DIRECTORY };
|
| 44 |
+
};
|
| 45 |
+
|
| 46 |
+
/// DEPRECATED. Use the FileSystem API in arrow::fs instead.
|
| 47 |
+
struct ARROW_EXPORT FileStatistics {
|
| 48 |
+
/// Size of file, -1 if finding length is unsupported
|
| 49 |
+
int64_t size;
|
| 50 |
+
ObjectType::type kind;
|
| 51 |
+
};
|
| 52 |
+
|
| 53 |
+
class ARROW_EXPORT FileSystem {
|
| 54 |
+
public:
|
| 55 |
+
virtual ~FileSystem() = default;
|
| 56 |
+
|
| 57 |
+
virtual Status MakeDirectory(const std::string& path) = 0;
|
| 58 |
+
|
| 59 |
+
virtual Status DeleteDirectory(const std::string& path) = 0;
|
| 60 |
+
|
| 61 |
+
virtual Status GetChildren(const std::string& path,
|
| 62 |
+
std::vector<std::string>* listing) = 0;
|
| 63 |
+
|
| 64 |
+
virtual Status Rename(const std::string& src, const std::string& dst) = 0;
|
| 65 |
+
|
| 66 |
+
virtual Status Stat(const std::string& path, FileStatistics* stat) = 0;
|
| 67 |
+
};
|
| 68 |
+
|
| 69 |
+
struct HdfsPathInfo {
|
| 70 |
+
ObjectType::type kind;
|
| 71 |
+
|
| 72 |
+
std::string name;
|
| 73 |
+
std::string owner;
|
| 74 |
+
std::string group;
|
| 75 |
+
|
| 76 |
+
// Access times in UNIX timestamps (seconds)
|
| 77 |
+
int64_t size;
|
| 78 |
+
int64_t block_size;
|
| 79 |
+
|
| 80 |
+
int32_t last_modified_time;
|
| 81 |
+
int32_t last_access_time;
|
| 82 |
+
|
| 83 |
+
int16_t replication;
|
| 84 |
+
int16_t permissions;
|
| 85 |
+
};
|
| 86 |
+
|
| 87 |
+
struct HdfsConnectionConfig {
|
| 88 |
+
std::string host;
|
| 89 |
+
int port;
|
| 90 |
+
std::string user;
|
| 91 |
+
std::string kerb_ticket;
|
| 92 |
+
std::unordered_map<std::string, std::string> extra_conf;
|
| 93 |
+
};
|
| 94 |
+
|
| 95 |
+
class ARROW_EXPORT HadoopFileSystem : public FileSystem {
|
| 96 |
+
public:
|
| 97 |
+
~HadoopFileSystem() override;
|
| 98 |
+
|
| 99 |
+
// Connect to an HDFS cluster given a configuration
|
| 100 |
+
//
|
| 101 |
+
// @param config (in): configuration for connecting
|
| 102 |
+
// @param fs (out): the created client
|
| 103 |
+
// @returns Status
|
| 104 |
+
static Status Connect(const HdfsConnectionConfig* config,
|
| 105 |
+
std::shared_ptr<HadoopFileSystem>* fs);
|
| 106 |
+
|
| 107 |
+
// Create directory and all parents
|
| 108 |
+
//
|
| 109 |
+
// @param path (in): absolute HDFS path
|
| 110 |
+
// @returns Status
|
| 111 |
+
Status MakeDirectory(const std::string& path) override;
|
| 112 |
+
|
| 113 |
+
// Delete file or directory
|
| 114 |
+
// @param path absolute path to data
|
| 115 |
+
// @param recursive if path is a directory, delete contents as well
|
| 116 |
+
// @returns error status on failure
|
| 117 |
+
Status Delete(const std::string& path, bool recursive = false);
|
| 118 |
+
|
| 119 |
+
Status DeleteDirectory(const std::string& path) override;
|
| 120 |
+
|
| 121 |
+
// Disconnect from cluster
|
| 122 |
+
//
|
| 123 |
+
// @returns Status
|
| 124 |
+
Status Disconnect();
|
| 125 |
+
|
| 126 |
+
// @param path (in): absolute HDFS path
|
| 127 |
+
// @returns bool, true if the path exists, false if not (or on error)
|
| 128 |
+
bool Exists(const std::string& path);
|
| 129 |
+
|
| 130 |
+
// @param path (in): absolute HDFS path
|
| 131 |
+
// @param info (out)
|
| 132 |
+
// @returns Status
|
| 133 |
+
Status GetPathInfo(const std::string& path, HdfsPathInfo* info);
|
| 134 |
+
|
| 135 |
+
// @param nbytes (out): total capacity of the filesystem
|
| 136 |
+
// @returns Status
|
| 137 |
+
Status GetCapacity(int64_t* nbytes);
|
| 138 |
+
|
| 139 |
+
// @param nbytes (out): total bytes used of the filesystem
|
| 140 |
+
// @returns Status
|
| 141 |
+
Status GetUsed(int64_t* nbytes);
|
| 142 |
+
|
| 143 |
+
Status GetChildren(const std::string& path, std::vector<std::string>* listing) override;
|
| 144 |
+
|
| 145 |
+
/// List directory contents
|
| 146 |
+
///
|
| 147 |
+
/// If path is a relative path, returned values will be absolute paths or URIs
|
| 148 |
+
/// starting from the current working directory.
|
| 149 |
+
Status ListDirectory(const std::string& path, std::vector<HdfsPathInfo>* listing);
|
| 150 |
+
|
| 151 |
+
/// Return the filesystem's current working directory.
|
| 152 |
+
///
|
| 153 |
+
/// The working directory is the base path for all relative paths given to
|
| 154 |
+
/// other APIs.
|
| 155 |
+
/// NOTE: this actually returns a URI.
|
| 156 |
+
Status GetWorkingDirectory(std::string* out);
|
| 157 |
+
|
| 158 |
+
/// Change
|
| 159 |
+
///
|
| 160 |
+
/// @param path file path to change
|
| 161 |
+
/// @param owner pass null for no change
|
| 162 |
+
/// @param group pass null for no change
|
| 163 |
+
Status Chown(const std::string& path, const char* owner, const char* group);
|
| 164 |
+
|
| 165 |
+
/// Change path permissions
|
| 166 |
+
///
|
| 167 |
+
/// \param path Absolute path in file system
|
| 168 |
+
/// \param mode Mode bitset
|
| 169 |
+
/// \return Status
|
| 170 |
+
Status Chmod(const std::string& path, int mode);
|
| 171 |
+
|
| 172 |
+
// Move file or directory from source path to destination path within the
|
| 173 |
+
// current filesystem
|
| 174 |
+
Status Rename(const std::string& src, const std::string& dst) override;
|
| 175 |
+
|
| 176 |
+
Status Copy(const std::string& src, const std::string& dst);
|
| 177 |
+
|
| 178 |
+
Status Move(const std::string& src, const std::string& dst);
|
| 179 |
+
|
| 180 |
+
Status Stat(const std::string& path, FileStatistics* stat) override;
|
| 181 |
+
|
| 182 |
+
// TODO(wesm): GetWorkingDirectory, SetWorkingDirectory
|
| 183 |
+
|
| 184 |
+
// Open an HDFS file in READ mode. Returns error
|
| 185 |
+
// status if the file is not found.
|
| 186 |
+
//
|
| 187 |
+
// @param path complete file path
|
| 188 |
+
Status OpenReadable(const std::string& path, int32_t buffer_size,
|
| 189 |
+
std::shared_ptr<HdfsReadableFile>* file);
|
| 190 |
+
|
| 191 |
+
Status OpenReadable(const std::string& path, int32_t buffer_size,
|
| 192 |
+
const io::IOContext& io_context,
|
| 193 |
+
std::shared_ptr<HdfsReadableFile>* file);
|
| 194 |
+
|
| 195 |
+
Status OpenReadable(const std::string& path, std::shared_ptr<HdfsReadableFile>* file);
|
| 196 |
+
|
| 197 |
+
Status OpenReadable(const std::string& path, const io::IOContext& io_context,
|
| 198 |
+
std::shared_ptr<HdfsReadableFile>* file);
|
| 199 |
+
|
| 200 |
+
// FileMode::WRITE options
|
| 201 |
+
// @param path complete file path
|
| 202 |
+
// @param buffer_size 0 by default
|
| 203 |
+
// @param replication 0 by default
|
| 204 |
+
// @param default_block_size 0 by default
|
| 205 |
+
Status OpenWritable(const std::string& path, bool append, int32_t buffer_size,
|
| 206 |
+
int16_t replication, int64_t default_block_size,
|
| 207 |
+
std::shared_ptr<HdfsOutputStream>* file);
|
| 208 |
+
|
| 209 |
+
Status OpenWritable(const std::string& path, bool append,
|
| 210 |
+
std::shared_ptr<HdfsOutputStream>* file);
|
| 211 |
+
|
| 212 |
+
private:
|
| 213 |
+
friend class HdfsReadableFile;
|
| 214 |
+
friend class HdfsOutputStream;
|
| 215 |
+
|
| 216 |
+
class ARROW_NO_EXPORT HadoopFileSystemImpl;
|
| 217 |
+
std::unique_ptr<HadoopFileSystemImpl> impl_;
|
| 218 |
+
|
| 219 |
+
HadoopFileSystem();
|
| 220 |
+
ARROW_DISALLOW_COPY_AND_ASSIGN(HadoopFileSystem);
|
| 221 |
+
};
|
| 222 |
+
|
| 223 |
+
class ARROW_EXPORT HdfsReadableFile : public RandomAccessFile {
|
| 224 |
+
public:
|
| 225 |
+
~HdfsReadableFile() override;
|
| 226 |
+
|
| 227 |
+
Status Close() override;
|
| 228 |
+
|
| 229 |
+
bool closed() const override;
|
| 230 |
+
|
| 231 |
+
// NOTE: If you wish to read a particular range of a file in a multithreaded
|
| 232 |
+
// context, you may prefer to use ReadAt to avoid locking issues
|
| 233 |
+
Result<int64_t> Read(int64_t nbytes, void* out) override;
|
| 234 |
+
Result<std::shared_ptr<Buffer>> Read(int64_t nbytes) override;
|
| 235 |
+
Result<int64_t> ReadAt(int64_t position, int64_t nbytes, void* out) override;
|
| 236 |
+
Result<std::shared_ptr<Buffer>> ReadAt(int64_t position, int64_t nbytes) override;
|
| 237 |
+
|
| 238 |
+
Status Seek(int64_t position) override;
|
| 239 |
+
Result<int64_t> Tell() const override;
|
| 240 |
+
Result<int64_t> GetSize() override;
|
| 241 |
+
|
| 242 |
+
private:
|
| 243 |
+
explicit HdfsReadableFile(const io::IOContext&);
|
| 244 |
+
|
| 245 |
+
class ARROW_NO_EXPORT HdfsReadableFileImpl;
|
| 246 |
+
std::unique_ptr<HdfsReadableFileImpl> impl_;
|
| 247 |
+
|
| 248 |
+
friend class HadoopFileSystem::HadoopFileSystemImpl;
|
| 249 |
+
|
| 250 |
+
ARROW_DISALLOW_COPY_AND_ASSIGN(HdfsReadableFile);
|
| 251 |
+
};
|
| 252 |
+
|
| 253 |
+
// Naming this file OutputStream because it does not support seeking (like the
|
| 254 |
+
// WritableFile interface)
|
| 255 |
+
class ARROW_EXPORT HdfsOutputStream : public OutputStream {
|
| 256 |
+
public:
|
| 257 |
+
~HdfsOutputStream() override;
|
| 258 |
+
|
| 259 |
+
Status Close() override;
|
| 260 |
+
|
| 261 |
+
bool closed() const override;
|
| 262 |
+
|
| 263 |
+
using OutputStream::Write;
|
| 264 |
+
Status Write(const void* buffer, int64_t nbytes) override;
|
| 265 |
+
|
| 266 |
+
Status Flush() override;
|
| 267 |
+
|
| 268 |
+
Result<int64_t> Tell() const override;
|
| 269 |
+
|
| 270 |
+
private:
|
| 271 |
+
class ARROW_NO_EXPORT HdfsOutputStreamImpl;
|
| 272 |
+
std::unique_ptr<HdfsOutputStreamImpl> impl_;
|
| 273 |
+
|
| 274 |
+
friend class HadoopFileSystem::HadoopFileSystemImpl;
|
| 275 |
+
|
| 276 |
+
HdfsOutputStream();
|
| 277 |
+
|
| 278 |
+
ARROW_DISALLOW_COPY_AND_ASSIGN(HdfsOutputStream);
|
| 279 |
+
};
|
| 280 |
+
|
| 281 |
+
ARROW_EXPORT Status HaveLibHdfs();
|
| 282 |
+
|
| 283 |
+
} // namespace io
|
| 284 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/interfaces.h
ADDED
|
@@ -0,0 +1,362 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <cstdint>
|
| 21 |
+
#include <memory>
|
| 22 |
+
#include <string>
|
| 23 |
+
#include <string_view>
|
| 24 |
+
#include <vector>
|
| 25 |
+
|
| 26 |
+
#include "arrow/io/type_fwd.h"
|
| 27 |
+
#include "arrow/type_fwd.h"
|
| 28 |
+
#include "arrow/util/cancel.h"
|
| 29 |
+
#include "arrow/util/macros.h"
|
| 30 |
+
#include "arrow/util/type_fwd.h"
|
| 31 |
+
#include "arrow/util/visibility.h"
|
| 32 |
+
|
| 33 |
+
namespace arrow {
|
| 34 |
+
namespace io {
|
| 35 |
+
|
| 36 |
+
struct ReadRange {
|
| 37 |
+
int64_t offset;
|
| 38 |
+
int64_t length;
|
| 39 |
+
|
| 40 |
+
friend bool operator==(const ReadRange& left, const ReadRange& right) {
|
| 41 |
+
return (left.offset == right.offset && left.length == right.length);
|
| 42 |
+
}
|
| 43 |
+
friend bool operator!=(const ReadRange& left, const ReadRange& right) {
|
| 44 |
+
return !(left == right);
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
bool Contains(const ReadRange& other) const {
|
| 48 |
+
return (offset <= other.offset && offset + length >= other.offset + other.length);
|
| 49 |
+
}
|
| 50 |
+
};
|
| 51 |
+
|
| 52 |
+
/// EXPERIMENTAL: options provider for IO tasks
|
| 53 |
+
///
|
| 54 |
+
/// Includes an Executor (which will be used to execute asynchronous reads),
|
| 55 |
+
/// a MemoryPool (which will be used to allocate buffers when zero copy reads
|
| 56 |
+
/// are not possible), and an external id (in case the executor receives tasks from
|
| 57 |
+
/// multiple sources and must distinguish tasks associated with this IOContext).
|
| 58 |
+
struct ARROW_EXPORT IOContext {
|
| 59 |
+
// No specified executor: will use a global IO thread pool
|
| 60 |
+
IOContext() : IOContext(default_memory_pool(), StopToken::Unstoppable()) {}
|
| 61 |
+
|
| 62 |
+
explicit IOContext(StopToken stop_token)
|
| 63 |
+
: IOContext(default_memory_pool(), std::move(stop_token)) {}
|
| 64 |
+
|
| 65 |
+
explicit IOContext(MemoryPool* pool, StopToken stop_token = StopToken::Unstoppable());
|
| 66 |
+
|
| 67 |
+
explicit IOContext(MemoryPool* pool, ::arrow::internal::Executor* executor,
|
| 68 |
+
StopToken stop_token = StopToken::Unstoppable(),
|
| 69 |
+
int64_t external_id = -1)
|
| 70 |
+
: pool_(pool),
|
| 71 |
+
executor_(executor),
|
| 72 |
+
external_id_(external_id),
|
| 73 |
+
stop_token_(std::move(stop_token)) {}
|
| 74 |
+
|
| 75 |
+
explicit IOContext(::arrow::internal::Executor* executor,
|
| 76 |
+
StopToken stop_token = StopToken::Unstoppable(),
|
| 77 |
+
int64_t external_id = -1)
|
| 78 |
+
: pool_(default_memory_pool()),
|
| 79 |
+
executor_(executor),
|
| 80 |
+
external_id_(external_id),
|
| 81 |
+
stop_token_(std::move(stop_token)) {}
|
| 82 |
+
|
| 83 |
+
MemoryPool* pool() const { return pool_; }
|
| 84 |
+
|
| 85 |
+
::arrow::internal::Executor* executor() const { return executor_; }
|
| 86 |
+
|
| 87 |
+
// An application-specific ID, forwarded to executor task submissions
|
| 88 |
+
int64_t external_id() const { return external_id_; }
|
| 89 |
+
|
| 90 |
+
StopToken stop_token() const { return stop_token_; }
|
| 91 |
+
|
| 92 |
+
private:
|
| 93 |
+
MemoryPool* pool_;
|
| 94 |
+
::arrow::internal::Executor* executor_;
|
| 95 |
+
int64_t external_id_;
|
| 96 |
+
StopToken stop_token_;
|
| 97 |
+
};
|
| 98 |
+
|
| 99 |
+
class ARROW_EXPORT FileInterface : public std::enable_shared_from_this<FileInterface> {
|
| 100 |
+
public:
|
| 101 |
+
virtual ~FileInterface() = 0;
|
| 102 |
+
|
| 103 |
+
/// \brief Close the stream cleanly
|
| 104 |
+
///
|
| 105 |
+
/// For writable streams, this will attempt to flush any pending data
|
| 106 |
+
/// before releasing the underlying resource.
|
| 107 |
+
///
|
| 108 |
+
/// After Close() is called, closed() returns true and the stream is not
|
| 109 |
+
/// available for further operations.
|
| 110 |
+
virtual Status Close() = 0;
|
| 111 |
+
|
| 112 |
+
/// \brief Close the stream asynchronously
|
| 113 |
+
///
|
| 114 |
+
/// By default, this will just submit the synchronous Close() to the
|
| 115 |
+
/// default I/O thread pool. Subclasses may implement this in a more
|
| 116 |
+
/// efficient manner.
|
| 117 |
+
virtual Future<> CloseAsync();
|
| 118 |
+
|
| 119 |
+
/// \brief Close the stream abruptly
|
| 120 |
+
///
|
| 121 |
+
/// This method does not guarantee that any pending data is flushed.
|
| 122 |
+
/// It merely releases any underlying resource used by the stream for
|
| 123 |
+
/// its operation.
|
| 124 |
+
///
|
| 125 |
+
/// After Abort() is called, closed() returns true and the stream is not
|
| 126 |
+
/// available for further operations.
|
| 127 |
+
virtual Status Abort();
|
| 128 |
+
|
| 129 |
+
/// \brief Return the position in this stream
|
| 130 |
+
virtual Result<int64_t> Tell() const = 0;
|
| 131 |
+
|
| 132 |
+
/// \brief Return whether the stream is closed
|
| 133 |
+
virtual bool closed() const = 0;
|
| 134 |
+
|
| 135 |
+
FileMode::type mode() const { return mode_; }
|
| 136 |
+
|
| 137 |
+
protected:
|
| 138 |
+
FileInterface() : mode_(FileMode::READ) {}
|
| 139 |
+
FileMode::type mode_;
|
| 140 |
+
void set_mode(FileMode::type mode) { mode_ = mode; }
|
| 141 |
+
|
| 142 |
+
private:
|
| 143 |
+
ARROW_DISALLOW_COPY_AND_ASSIGN(FileInterface);
|
| 144 |
+
};
|
| 145 |
+
|
| 146 |
+
class ARROW_EXPORT Seekable {
|
| 147 |
+
public:
|
| 148 |
+
virtual ~Seekable() = default;
|
| 149 |
+
virtual Status Seek(int64_t position) = 0;
|
| 150 |
+
};
|
| 151 |
+
|
| 152 |
+
class ARROW_EXPORT Writable {
|
| 153 |
+
public:
|
| 154 |
+
virtual ~Writable() = default;
|
| 155 |
+
|
| 156 |
+
/// \brief Write the given data to the stream
|
| 157 |
+
///
|
| 158 |
+
/// This method always processes the bytes in full. Depending on the
|
| 159 |
+
/// semantics of the stream, the data may be written out immediately,
|
| 160 |
+
/// held in a buffer, or written asynchronously. In the case where
|
| 161 |
+
/// the stream buffers the data, it will be copied. To avoid potentially
|
| 162 |
+
/// large copies, use the Write variant that takes an owned Buffer.
|
| 163 |
+
virtual Status Write(const void* data, int64_t nbytes) = 0;
|
| 164 |
+
|
| 165 |
+
/// \brief Write the given data to the stream
|
| 166 |
+
///
|
| 167 |
+
/// Since the Buffer owns its memory, this method can avoid a copy if
|
| 168 |
+
/// buffering is required. See Write(const void*, int64_t) for details.
|
| 169 |
+
virtual Status Write(const std::shared_ptr<Buffer>& data);
|
| 170 |
+
|
| 171 |
+
/// \brief Flush buffered bytes, if any
|
| 172 |
+
virtual Status Flush();
|
| 173 |
+
|
| 174 |
+
Status Write(std::string_view data);
|
| 175 |
+
};
|
| 176 |
+
|
| 177 |
+
class ARROW_EXPORT Readable {
|
| 178 |
+
public:
|
| 179 |
+
virtual ~Readable() = default;
|
| 180 |
+
|
| 181 |
+
/// \brief Read data from current file position.
|
| 182 |
+
///
|
| 183 |
+
/// Read at most `nbytes` from the current file position into `out`.
|
| 184 |
+
/// The number of bytes read is returned.
|
| 185 |
+
virtual Result<int64_t> Read(int64_t nbytes, void* out) = 0;
|
| 186 |
+
|
| 187 |
+
/// \brief Read data from current file position.
|
| 188 |
+
///
|
| 189 |
+
/// Read at most `nbytes` from the current file position. Less bytes may
|
| 190 |
+
/// be read if EOF is reached. This method updates the current file position.
|
| 191 |
+
///
|
| 192 |
+
/// In some cases (e.g. a memory-mapped file), this method may avoid a
|
| 193 |
+
/// memory copy.
|
| 194 |
+
virtual Result<std::shared_ptr<Buffer>> Read(int64_t nbytes) = 0;
|
| 195 |
+
|
| 196 |
+
/// EXPERIMENTAL: The IOContext associated with this file.
|
| 197 |
+
///
|
| 198 |
+
/// By default, this is the same as default_io_context(), but it may be
|
| 199 |
+
/// overridden by subclasses.
|
| 200 |
+
virtual const IOContext& io_context() const;
|
| 201 |
+
};
|
| 202 |
+
|
| 203 |
+
class ARROW_EXPORT OutputStream : virtual public FileInterface, public Writable {
|
| 204 |
+
protected:
|
| 205 |
+
OutputStream() = default;
|
| 206 |
+
};
|
| 207 |
+
|
| 208 |
+
class ARROW_EXPORT InputStream : virtual public FileInterface, virtual public Readable {
|
| 209 |
+
public:
|
| 210 |
+
/// \brief Advance or skip stream indicated number of bytes
|
| 211 |
+
/// \param[in] nbytes the number to move forward
|
| 212 |
+
/// \return Status
|
| 213 |
+
Status Advance(int64_t nbytes);
|
| 214 |
+
|
| 215 |
+
/// \brief Return zero-copy string_view to upcoming bytes.
|
| 216 |
+
///
|
| 217 |
+
/// Do not modify the stream position. The view becomes invalid after
|
| 218 |
+
/// any operation on the stream. May trigger buffering if the requested
|
| 219 |
+
/// size is larger than the number of buffered bytes.
|
| 220 |
+
///
|
| 221 |
+
/// May return NotImplemented on streams that don't support it.
|
| 222 |
+
///
|
| 223 |
+
/// \param[in] nbytes the maximum number of bytes to see
|
| 224 |
+
virtual Result<std::string_view> Peek(int64_t nbytes);
|
| 225 |
+
|
| 226 |
+
/// \brief Return true if InputStream is capable of zero copy Buffer reads
|
| 227 |
+
///
|
| 228 |
+
/// Zero copy reads imply the use of Buffer-returning Read() overloads.
|
| 229 |
+
virtual bool supports_zero_copy() const;
|
| 230 |
+
|
| 231 |
+
/// \brief Read and return stream metadata
|
| 232 |
+
///
|
| 233 |
+
/// If the stream implementation doesn't support metadata, empty metadata
|
| 234 |
+
/// is returned. Note that it is allowed to return a null pointer rather
|
| 235 |
+
/// than an allocated empty metadata.
|
| 236 |
+
virtual Result<std::shared_ptr<const KeyValueMetadata>> ReadMetadata();
|
| 237 |
+
|
| 238 |
+
/// \brief Read stream metadata asynchronously
|
| 239 |
+
virtual Future<std::shared_ptr<const KeyValueMetadata>> ReadMetadataAsync(
|
| 240 |
+
const IOContext& io_context);
|
| 241 |
+
Future<std::shared_ptr<const KeyValueMetadata>> ReadMetadataAsync();
|
| 242 |
+
|
| 243 |
+
protected:
|
| 244 |
+
InputStream() = default;
|
| 245 |
+
};
|
| 246 |
+
|
| 247 |
+
class ARROW_EXPORT RandomAccessFile : public InputStream, public Seekable {
|
| 248 |
+
public:
|
| 249 |
+
/// Necessary because we hold a std::unique_ptr
|
| 250 |
+
~RandomAccessFile() override;
|
| 251 |
+
|
| 252 |
+
/// \brief Create an isolated InputStream that reads a segment of a
|
| 253 |
+
/// RandomAccessFile. Multiple such stream can be created and used
|
| 254 |
+
/// independently without interference
|
| 255 |
+
/// \param[in] file a file instance
|
| 256 |
+
/// \param[in] file_offset the starting position in the file
|
| 257 |
+
/// \param[in] nbytes the extent of bytes to read. The file should have
|
| 258 |
+
/// sufficient bytes available
|
| 259 |
+
static Result<std::shared_ptr<InputStream>> GetStream(
|
| 260 |
+
std::shared_ptr<RandomAccessFile> file, int64_t file_offset, int64_t nbytes);
|
| 261 |
+
|
| 262 |
+
/// \brief Return the total file size in bytes.
|
| 263 |
+
///
|
| 264 |
+
/// This method does not read or move the current file position, so is safe
|
| 265 |
+
/// to call concurrently with e.g. ReadAt().
|
| 266 |
+
virtual Result<int64_t> GetSize() = 0;
|
| 267 |
+
|
| 268 |
+
/// \brief Read data from given file position.
|
| 269 |
+
///
|
| 270 |
+
/// At most `nbytes` bytes are read. The number of bytes read is returned
|
| 271 |
+
/// (it can be less than `nbytes` if EOF is reached).
|
| 272 |
+
///
|
| 273 |
+
/// This method can be safely called from multiple threads concurrently.
|
| 274 |
+
/// It is unspecified whether this method updates the file position or not.
|
| 275 |
+
///
|
| 276 |
+
/// The default RandomAccessFile-provided implementation uses Seek() and Read(),
|
| 277 |
+
/// but subclasses may override it with a more efficient implementation
|
| 278 |
+
/// that doesn't depend on implicit file positioning.
|
| 279 |
+
///
|
| 280 |
+
/// \param[in] position Where to read bytes from
|
| 281 |
+
/// \param[in] nbytes The number of bytes to read
|
| 282 |
+
/// \param[out] out The buffer to read bytes into
|
| 283 |
+
/// \return The number of bytes read, or an error
|
| 284 |
+
virtual Result<int64_t> ReadAt(int64_t position, int64_t nbytes, void* out);
|
| 285 |
+
|
| 286 |
+
/// \brief Read data from given file position.
|
| 287 |
+
///
|
| 288 |
+
/// At most `nbytes` bytes are read, but it can be less if EOF is reached.
|
| 289 |
+
///
|
| 290 |
+
/// \param[in] position Where to read bytes from
|
| 291 |
+
/// \param[in] nbytes The number of bytes to read
|
| 292 |
+
/// \return A buffer containing the bytes read, or an error
|
| 293 |
+
virtual Result<std::shared_ptr<Buffer>> ReadAt(int64_t position, int64_t nbytes);
|
| 294 |
+
|
| 295 |
+
/// EXPERIMENTAL: Read data asynchronously.
|
| 296 |
+
virtual Future<std::shared_ptr<Buffer>> ReadAsync(const IOContext&, int64_t position,
|
| 297 |
+
int64_t nbytes);
|
| 298 |
+
|
| 299 |
+
/// EXPERIMENTAL: Read data asynchronously, using the file's IOContext.
|
| 300 |
+
Future<std::shared_ptr<Buffer>> ReadAsync(int64_t position, int64_t nbytes);
|
| 301 |
+
|
| 302 |
+
/// EXPERIMENTAL: Explicit multi-read.
|
| 303 |
+
/// \brief Request multiple reads at once
|
| 304 |
+
///
|
| 305 |
+
/// The underlying filesystem may optimize these reads by coalescing small reads into
|
| 306 |
+
/// large reads or by breaking up large reads into multiple parallel smaller reads. The
|
| 307 |
+
/// reads should be issued in parallel if it makes sense for the filesystem.
|
| 308 |
+
///
|
| 309 |
+
/// One future will be returned for each input read range. Multiple returned futures
|
| 310 |
+
/// may correspond to a single read. Or, a single returned future may be a combined
|
| 311 |
+
/// result of several individual reads.
|
| 312 |
+
///
|
| 313 |
+
/// \param[in] ranges The ranges to read
|
| 314 |
+
/// \return A future that will complete with the data from the requested range is
|
| 315 |
+
/// available
|
| 316 |
+
virtual std::vector<Future<std::shared_ptr<Buffer>>> ReadManyAsync(
|
| 317 |
+
const IOContext&, const std::vector<ReadRange>& ranges);
|
| 318 |
+
|
| 319 |
+
/// EXPERIMENTAL: Explicit multi-read, using the file's IOContext.
|
| 320 |
+
std::vector<Future<std::shared_ptr<Buffer>>> ReadManyAsync(
|
| 321 |
+
const std::vector<ReadRange>& ranges);
|
| 322 |
+
|
| 323 |
+
/// EXPERIMENTAL: Inform that the given ranges may be read soon.
|
| 324 |
+
///
|
| 325 |
+
/// Some implementations might arrange to prefetch some of the data.
|
| 326 |
+
/// However, no guarantee is made and the default implementation does nothing.
|
| 327 |
+
/// For robust prefetching, use ReadAt() or ReadAsync().
|
| 328 |
+
virtual Status WillNeed(const std::vector<ReadRange>& ranges);
|
| 329 |
+
|
| 330 |
+
protected:
|
| 331 |
+
RandomAccessFile();
|
| 332 |
+
|
| 333 |
+
private:
|
| 334 |
+
struct ARROW_NO_EXPORT Impl;
|
| 335 |
+
std::unique_ptr<Impl> interface_impl_;
|
| 336 |
+
};
|
| 337 |
+
|
| 338 |
+
class ARROW_EXPORT WritableFile : public OutputStream, public Seekable {
|
| 339 |
+
public:
|
| 340 |
+
virtual Status WriteAt(int64_t position, const void* data, int64_t nbytes) = 0;
|
| 341 |
+
|
| 342 |
+
protected:
|
| 343 |
+
WritableFile() = default;
|
| 344 |
+
};
|
| 345 |
+
|
| 346 |
+
class ARROW_EXPORT ReadWriteFileInterface : public RandomAccessFile, public WritableFile {
|
| 347 |
+
protected:
|
| 348 |
+
ReadWriteFileInterface() { RandomAccessFile::set_mode(FileMode::READWRITE); }
|
| 349 |
+
};
|
| 350 |
+
|
| 351 |
+
/// \brief Return an iterator on an input stream
|
| 352 |
+
///
|
| 353 |
+
/// The iterator yields a fixed-size block on each Next() call, except the
|
| 354 |
+
/// last block in the stream which may be smaller.
|
| 355 |
+
/// Once the end of stream is reached, Next() returns nullptr
|
| 356 |
+
/// (unlike InputStream::Read() which returns an empty buffer).
|
| 357 |
+
ARROW_EXPORT
|
| 358 |
+
Result<Iterator<std::shared_ptr<Buffer>>> MakeInputStreamIterator(
|
| 359 |
+
std::shared_ptr<InputStream> stream, int64_t block_size);
|
| 360 |
+
|
| 361 |
+
} // namespace io
|
| 362 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/memory.h
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
// Public API for different memory sharing / IO mechanisms
|
| 19 |
+
|
| 20 |
+
#pragma once
|
| 21 |
+
|
| 22 |
+
#include <cstdint>
|
| 23 |
+
#include <memory>
|
| 24 |
+
#include <string_view>
|
| 25 |
+
#include <vector>
|
| 26 |
+
|
| 27 |
+
#include "arrow/io/concurrency.h"
|
| 28 |
+
#include "arrow/io/interfaces.h"
|
| 29 |
+
#include "arrow/type_fwd.h"
|
| 30 |
+
#include "arrow/util/visibility.h"
|
| 31 |
+
|
| 32 |
+
namespace arrow {
|
| 33 |
+
|
| 34 |
+
class Status;
|
| 35 |
+
|
| 36 |
+
namespace io {
|
| 37 |
+
|
| 38 |
+
/// \brief An output stream that writes to a resizable buffer
|
| 39 |
+
class ARROW_EXPORT BufferOutputStream : public OutputStream {
|
| 40 |
+
public:
|
| 41 |
+
explicit BufferOutputStream(const std::shared_ptr<ResizableBuffer>& buffer);
|
| 42 |
+
|
| 43 |
+
/// \brief Create in-memory output stream with indicated capacity using a
|
| 44 |
+
/// memory pool
|
| 45 |
+
/// \param[in] initial_capacity the initial allocated internal capacity of
|
| 46 |
+
/// the OutputStream
|
| 47 |
+
/// \param[in,out] pool a MemoryPool to use for allocations
|
| 48 |
+
/// \return the created stream
|
| 49 |
+
static Result<std::shared_ptr<BufferOutputStream>> Create(
|
| 50 |
+
int64_t initial_capacity = 4096, MemoryPool* pool = default_memory_pool());
|
| 51 |
+
|
| 52 |
+
~BufferOutputStream() override;
|
| 53 |
+
|
| 54 |
+
// Implement the OutputStream interface
|
| 55 |
+
|
| 56 |
+
/// Close the stream, preserving the buffer (retrieve it with Finish()).
|
| 57 |
+
Status Close() override;
|
| 58 |
+
bool closed() const override;
|
| 59 |
+
Result<int64_t> Tell() const override;
|
| 60 |
+
Status Write(const void* data, int64_t nbytes) override;
|
| 61 |
+
|
| 62 |
+
/// \cond FALSE
|
| 63 |
+
using OutputStream::Write;
|
| 64 |
+
/// \endcond
|
| 65 |
+
|
| 66 |
+
/// Close the stream and return the buffer
|
| 67 |
+
Result<std::shared_ptr<Buffer>> Finish();
|
| 68 |
+
|
| 69 |
+
/// \brief Initialize state of OutputStream with newly allocated memory and
|
| 70 |
+
/// set position to 0
|
| 71 |
+
/// \param[in] initial_capacity the starting allocated capacity
|
| 72 |
+
/// \param[in,out] pool the memory pool to use for allocations
|
| 73 |
+
/// \return Status
|
| 74 |
+
Status Reset(int64_t initial_capacity = 1024, MemoryPool* pool = default_memory_pool());
|
| 75 |
+
|
| 76 |
+
int64_t capacity() const { return capacity_; }
|
| 77 |
+
|
| 78 |
+
private:
|
| 79 |
+
BufferOutputStream();
|
| 80 |
+
|
| 81 |
+
// Ensures there is sufficient space available to write nbytes
|
| 82 |
+
Status Reserve(int64_t nbytes);
|
| 83 |
+
|
| 84 |
+
std::shared_ptr<ResizableBuffer> buffer_;
|
| 85 |
+
bool is_open_;
|
| 86 |
+
int64_t capacity_;
|
| 87 |
+
int64_t position_;
|
| 88 |
+
uint8_t* mutable_data_;
|
| 89 |
+
};
|
| 90 |
+
|
| 91 |
+
/// \brief A helper class to track the size of allocations
|
| 92 |
+
///
|
| 93 |
+
/// Writes to this stream do not copy or retain any data, they just bump
|
| 94 |
+
/// a size counter that can be later used to know exactly which data size
|
| 95 |
+
/// needs to be allocated for actual writing.
|
| 96 |
+
class ARROW_EXPORT MockOutputStream : public OutputStream {
|
| 97 |
+
public:
|
| 98 |
+
MockOutputStream() : extent_bytes_written_(0), is_open_(true) {}
|
| 99 |
+
|
| 100 |
+
// Implement the OutputStream interface
|
| 101 |
+
Status Close() override;
|
| 102 |
+
bool closed() const override;
|
| 103 |
+
Result<int64_t> Tell() const override;
|
| 104 |
+
Status Write(const void* data, int64_t nbytes) override;
|
| 105 |
+
/// \cond FALSE
|
| 106 |
+
using Writable::Write;
|
| 107 |
+
/// \endcond
|
| 108 |
+
|
| 109 |
+
int64_t GetExtentBytesWritten() const { return extent_bytes_written_; }
|
| 110 |
+
|
| 111 |
+
private:
|
| 112 |
+
int64_t extent_bytes_written_;
|
| 113 |
+
bool is_open_;
|
| 114 |
+
};
|
| 115 |
+
|
| 116 |
+
/// \brief An output stream that writes into a fixed-size mutable buffer
|
| 117 |
+
class ARROW_EXPORT FixedSizeBufferWriter : public WritableFile {
|
| 118 |
+
public:
|
| 119 |
+
/// Input buffer must be mutable, will abort if not
|
| 120 |
+
explicit FixedSizeBufferWriter(const std::shared_ptr<Buffer>& buffer);
|
| 121 |
+
~FixedSizeBufferWriter() override;
|
| 122 |
+
|
| 123 |
+
Status Close() override;
|
| 124 |
+
bool closed() const override;
|
| 125 |
+
Status Seek(int64_t position) override;
|
| 126 |
+
Result<int64_t> Tell() const override;
|
| 127 |
+
Status Write(const void* data, int64_t nbytes) override;
|
| 128 |
+
/// \cond FALSE
|
| 129 |
+
using Writable::Write;
|
| 130 |
+
/// \endcond
|
| 131 |
+
|
| 132 |
+
Status WriteAt(int64_t position, const void* data, int64_t nbytes) override;
|
| 133 |
+
|
| 134 |
+
void set_memcopy_threads(int num_threads);
|
| 135 |
+
void set_memcopy_blocksize(int64_t blocksize);
|
| 136 |
+
void set_memcopy_threshold(int64_t threshold);
|
| 137 |
+
|
| 138 |
+
protected:
|
| 139 |
+
class FixedSizeBufferWriterImpl;
|
| 140 |
+
std::unique_ptr<FixedSizeBufferWriterImpl> impl_;
|
| 141 |
+
};
|
| 142 |
+
|
| 143 |
+
/// \class BufferReader
|
| 144 |
+
/// \brief Random access zero-copy reads on an arrow::Buffer
|
| 145 |
+
class ARROW_EXPORT BufferReader
|
| 146 |
+
: public internal::RandomAccessFileConcurrencyWrapper<BufferReader> {
|
| 147 |
+
public:
|
| 148 |
+
/// \brief Instantiate from std::shared_ptr<Buffer>.
|
| 149 |
+
///
|
| 150 |
+
/// This is a zero-copy constructor.
|
| 151 |
+
explicit BufferReader(std::shared_ptr<Buffer> buffer);
|
| 152 |
+
|
| 153 |
+
/// \brief Instantiate from std::string. Owns data.
|
| 154 |
+
static std::unique_ptr<BufferReader> FromString(std::string data);
|
| 155 |
+
|
| 156 |
+
bool closed() const override;
|
| 157 |
+
|
| 158 |
+
bool supports_zero_copy() const override;
|
| 159 |
+
|
| 160 |
+
std::shared_ptr<Buffer> buffer() const { return buffer_; }
|
| 161 |
+
|
| 162 |
+
// Synchronous ReadAsync override
|
| 163 |
+
Future<std::shared_ptr<Buffer>> ReadAsync(const IOContext&, int64_t position,
|
| 164 |
+
int64_t nbytes) override;
|
| 165 |
+
Status WillNeed(const std::vector<ReadRange>& ranges) override;
|
| 166 |
+
|
| 167 |
+
protected:
|
| 168 |
+
friend RandomAccessFileConcurrencyWrapper<BufferReader>;
|
| 169 |
+
|
| 170 |
+
Status DoClose();
|
| 171 |
+
|
| 172 |
+
Result<int64_t> DoRead(int64_t nbytes, void* buffer);
|
| 173 |
+
Result<std::shared_ptr<Buffer>> DoRead(int64_t nbytes);
|
| 174 |
+
Result<int64_t> DoReadAt(int64_t position, int64_t nbytes, void* out);
|
| 175 |
+
Result<std::shared_ptr<Buffer>> DoReadAt(int64_t position, int64_t nbytes);
|
| 176 |
+
Result<std::string_view> DoPeek(int64_t nbytes) override;
|
| 177 |
+
|
| 178 |
+
Result<int64_t> DoTell() const;
|
| 179 |
+
Status DoSeek(int64_t position);
|
| 180 |
+
Result<int64_t> DoGetSize();
|
| 181 |
+
|
| 182 |
+
Status CheckClosed() const {
|
| 183 |
+
if (!is_open_) {
|
| 184 |
+
return Status::Invalid("Operation forbidden on closed BufferReader");
|
| 185 |
+
}
|
| 186 |
+
return Status::OK();
|
| 187 |
+
}
|
| 188 |
+
|
| 189 |
+
std::shared_ptr<Buffer> buffer_;
|
| 190 |
+
const uint8_t* data_;
|
| 191 |
+
int64_t size_;
|
| 192 |
+
int64_t position_;
|
| 193 |
+
bool is_open_;
|
| 194 |
+
};
|
| 195 |
+
|
| 196 |
+
} // namespace io
|
| 197 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/mman.h
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Copyright https://code.google.com/p/mman-win32/
|
| 2 |
+
//
|
| 3 |
+
// Licensed under the MIT License;
|
| 4 |
+
// You may obtain a copy of the License at
|
| 5 |
+
//
|
| 6 |
+
// https://opensource.org/licenses/MIT
|
| 7 |
+
|
| 8 |
+
#pragma once
|
| 9 |
+
|
| 10 |
+
#include "arrow/util/windows_compatibility.h"
|
| 11 |
+
|
| 12 |
+
#include <errno.h>
|
| 13 |
+
#include <io.h>
|
| 14 |
+
#include <sys/types.h>
|
| 15 |
+
|
| 16 |
+
#include <cstdint>
|
| 17 |
+
|
| 18 |
+
#define PROT_NONE 0
|
| 19 |
+
#define PROT_READ 1
|
| 20 |
+
#define PROT_WRITE 2
|
| 21 |
+
#define PROT_EXEC 4
|
| 22 |
+
|
| 23 |
+
#define MAP_FILE 0
|
| 24 |
+
#define MAP_SHARED 1
|
| 25 |
+
#define MAP_PRIVATE 2
|
| 26 |
+
#define MAP_TYPE 0xf
|
| 27 |
+
#define MAP_FIXED 0x10
|
| 28 |
+
#define MAP_ANONYMOUS 0x20
|
| 29 |
+
#define MAP_ANON MAP_ANONYMOUS
|
| 30 |
+
|
| 31 |
+
#define MAP_FAILED ((void*)-1)
|
| 32 |
+
|
| 33 |
+
/* Flags for msync. */
|
| 34 |
+
#define MS_ASYNC 1
|
| 35 |
+
#define MS_SYNC 2
|
| 36 |
+
#define MS_INVALIDATE 4
|
| 37 |
+
|
| 38 |
+
#ifndef FILE_MAP_EXECUTE
|
| 39 |
+
# define FILE_MAP_EXECUTE 0x0020
|
| 40 |
+
#endif
|
| 41 |
+
|
| 42 |
+
static inline int __map_mman_error(const DWORD err, const int deferr) {
|
| 43 |
+
if (err == 0) return 0;
|
| 44 |
+
// TODO: implement
|
| 45 |
+
return err;
|
| 46 |
+
}
|
| 47 |
+
|
| 48 |
+
static inline DWORD __map_mmap_prot_page(const int prot) {
|
| 49 |
+
DWORD protect = 0;
|
| 50 |
+
|
| 51 |
+
if (prot == PROT_NONE) return protect;
|
| 52 |
+
|
| 53 |
+
if ((prot & PROT_EXEC) != 0) {
|
| 54 |
+
protect = ((prot & PROT_WRITE) != 0) ? PAGE_EXECUTE_READWRITE : PAGE_EXECUTE_READ;
|
| 55 |
+
} else {
|
| 56 |
+
protect = ((prot & PROT_WRITE) != 0) ? PAGE_READWRITE : PAGE_READONLY;
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
return protect;
|
| 60 |
+
}
|
| 61 |
+
|
| 62 |
+
static inline DWORD __map_mmap_prot_file(const int prot) {
|
| 63 |
+
DWORD desiredAccess = 0;
|
| 64 |
+
|
| 65 |
+
if (prot == PROT_NONE) return desiredAccess;
|
| 66 |
+
|
| 67 |
+
if ((prot & PROT_READ) != 0) desiredAccess |= FILE_MAP_READ;
|
| 68 |
+
if ((prot & PROT_WRITE) != 0) desiredAccess |= FILE_MAP_WRITE;
|
| 69 |
+
if ((prot & PROT_EXEC) != 0) desiredAccess |= FILE_MAP_EXECUTE;
|
| 70 |
+
|
| 71 |
+
return desiredAccess;
|
| 72 |
+
}
|
| 73 |
+
|
| 74 |
+
static inline void* mmap(void* addr, size_t len, int prot, int flags, int fildes,
|
| 75 |
+
off_t off) {
|
| 76 |
+
HANDLE fm, h;
|
| 77 |
+
|
| 78 |
+
void* map = MAP_FAILED;
|
| 79 |
+
const uint64_t off64 = static_cast<uint64_t>(off);
|
| 80 |
+
const uint64_t maxSize = off64 + len;
|
| 81 |
+
|
| 82 |
+
const DWORD dwFileOffsetLow = static_cast<DWORD>(off64 & 0xFFFFFFFFUL);
|
| 83 |
+
const DWORD dwFileOffsetHigh = static_cast<DWORD>((off64 >> 32) & 0xFFFFFFFFUL);
|
| 84 |
+
const DWORD dwMaxSizeLow = static_cast<DWORD>(maxSize & 0xFFFFFFFFUL);
|
| 85 |
+
const DWORD dwMaxSizeHigh = static_cast<DWORD>((maxSize >> 32) & 0xFFFFFFFFUL);
|
| 86 |
+
|
| 87 |
+
const DWORD protect = __map_mmap_prot_page(prot);
|
| 88 |
+
const DWORD desiredAccess = __map_mmap_prot_file(prot);
|
| 89 |
+
|
| 90 |
+
errno = 0;
|
| 91 |
+
|
| 92 |
+
if (len == 0
|
| 93 |
+
/* Unsupported flag combinations */
|
| 94 |
+
|| (flags & MAP_FIXED) != 0
|
| 95 |
+
/* Unsupported protection combinations */
|
| 96 |
+
|| prot == PROT_EXEC) {
|
| 97 |
+
errno = EINVAL;
|
| 98 |
+
return MAP_FAILED;
|
| 99 |
+
}
|
| 100 |
+
|
| 101 |
+
h = ((flags & MAP_ANONYMOUS) == 0) ? (HANDLE)_get_osfhandle(fildes)
|
| 102 |
+
: INVALID_HANDLE_VALUE;
|
| 103 |
+
|
| 104 |
+
if ((flags & MAP_ANONYMOUS) == 0 && h == INVALID_HANDLE_VALUE) {
|
| 105 |
+
errno = EBADF;
|
| 106 |
+
return MAP_FAILED;
|
| 107 |
+
}
|
| 108 |
+
|
| 109 |
+
fm = CreateFileMapping(h, NULL, protect, dwMaxSizeHigh, dwMaxSizeLow, NULL);
|
| 110 |
+
|
| 111 |
+
if (fm == NULL) {
|
| 112 |
+
errno = __map_mman_error(GetLastError(), EPERM);
|
| 113 |
+
return MAP_FAILED;
|
| 114 |
+
}
|
| 115 |
+
|
| 116 |
+
map = MapViewOfFile(fm, desiredAccess, dwFileOffsetHigh, dwFileOffsetLow, len);
|
| 117 |
+
|
| 118 |
+
CloseHandle(fm);
|
| 119 |
+
|
| 120 |
+
if (map == NULL) {
|
| 121 |
+
errno = __map_mman_error(GetLastError(), EPERM);
|
| 122 |
+
return MAP_FAILED;
|
| 123 |
+
}
|
| 124 |
+
|
| 125 |
+
return map;
|
| 126 |
+
}
|
| 127 |
+
|
| 128 |
+
static inline int munmap(void* addr, size_t len) {
|
| 129 |
+
if (UnmapViewOfFile(addr)) return 0;
|
| 130 |
+
|
| 131 |
+
errno = __map_mman_error(GetLastError(), EPERM);
|
| 132 |
+
|
| 133 |
+
return -1;
|
| 134 |
+
}
|
| 135 |
+
|
| 136 |
+
static inline int mprotect(void* addr, size_t len, int prot) {
|
| 137 |
+
DWORD newProtect = __map_mmap_prot_page(prot);
|
| 138 |
+
DWORD oldProtect = 0;
|
| 139 |
+
|
| 140 |
+
if (VirtualProtect(addr, len, newProtect, &oldProtect)) return 0;
|
| 141 |
+
|
| 142 |
+
errno = __map_mman_error(GetLastError(), EPERM);
|
| 143 |
+
|
| 144 |
+
return -1;
|
| 145 |
+
}
|
| 146 |
+
|
| 147 |
+
static inline int msync(void* addr, size_t len, int flags) {
|
| 148 |
+
if (FlushViewOfFile(addr, len)) return 0;
|
| 149 |
+
|
| 150 |
+
errno = __map_mman_error(GetLastError(), EPERM);
|
| 151 |
+
|
| 152 |
+
return -1;
|
| 153 |
+
}
|
| 154 |
+
|
| 155 |
+
static inline int mlock(const void* addr, size_t len) {
|
| 156 |
+
if (VirtualLock((LPVOID)addr, len)) return 0;
|
| 157 |
+
|
| 158 |
+
errno = __map_mman_error(GetLastError(), EPERM);
|
| 159 |
+
|
| 160 |
+
return -1;
|
| 161 |
+
}
|
| 162 |
+
|
| 163 |
+
static inline int munlock(const void* addr, size_t len) {
|
| 164 |
+
if (VirtualUnlock((LPVOID)addr, len)) return 0;
|
| 165 |
+
|
| 166 |
+
errno = __map_mman_error(GetLastError(), EPERM);
|
| 167 |
+
|
| 168 |
+
return -1;
|
| 169 |
+
}
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/slow.h
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
// Slow stream implementations, mainly for testing and benchmarking
|
| 19 |
+
|
| 20 |
+
#pragma once
|
| 21 |
+
|
| 22 |
+
#include <cstdint>
|
| 23 |
+
#include <memory>
|
| 24 |
+
#include <utility>
|
| 25 |
+
|
| 26 |
+
#include "arrow/io/interfaces.h"
|
| 27 |
+
#include "arrow/util/visibility.h"
|
| 28 |
+
|
| 29 |
+
namespace arrow {
|
| 30 |
+
|
| 31 |
+
class Buffer;
|
| 32 |
+
class Status;
|
| 33 |
+
|
| 34 |
+
namespace io {
|
| 35 |
+
|
| 36 |
+
class ARROW_EXPORT LatencyGenerator {
|
| 37 |
+
public:
|
| 38 |
+
virtual ~LatencyGenerator();
|
| 39 |
+
|
| 40 |
+
void Sleep();
|
| 41 |
+
|
| 42 |
+
virtual double NextLatency() = 0;
|
| 43 |
+
|
| 44 |
+
static std::shared_ptr<LatencyGenerator> Make(double average_latency);
|
| 45 |
+
static std::shared_ptr<LatencyGenerator> Make(double average_latency, int32_t seed);
|
| 46 |
+
};
|
| 47 |
+
|
| 48 |
+
// XXX use ConcurrencyWrapper? It could increase chances of finding a race.
|
| 49 |
+
|
| 50 |
+
template <class StreamType>
|
| 51 |
+
class SlowInputStreamBase : public StreamType {
|
| 52 |
+
public:
|
| 53 |
+
SlowInputStreamBase(std::shared_ptr<StreamType> stream,
|
| 54 |
+
std::shared_ptr<LatencyGenerator> latencies)
|
| 55 |
+
: stream_(std::move(stream)), latencies_(std::move(latencies)) {}
|
| 56 |
+
|
| 57 |
+
SlowInputStreamBase(std::shared_ptr<StreamType> stream, double average_latency)
|
| 58 |
+
: stream_(std::move(stream)), latencies_(LatencyGenerator::Make(average_latency)) {}
|
| 59 |
+
|
| 60 |
+
SlowInputStreamBase(std::shared_ptr<StreamType> stream, double average_latency,
|
| 61 |
+
int32_t seed)
|
| 62 |
+
: stream_(std::move(stream)),
|
| 63 |
+
latencies_(LatencyGenerator::Make(average_latency, seed)) {}
|
| 64 |
+
|
| 65 |
+
protected:
|
| 66 |
+
std::shared_ptr<StreamType> stream_;
|
| 67 |
+
std::shared_ptr<LatencyGenerator> latencies_;
|
| 68 |
+
};
|
| 69 |
+
|
| 70 |
+
/// \brief An InputStream wrapper that makes reads slower.
|
| 71 |
+
///
|
| 72 |
+
/// Read() calls are made slower by an average latency (in seconds).
|
| 73 |
+
/// Actual latencies form a normal distribution closely centered
|
| 74 |
+
/// on the average latency.
|
| 75 |
+
/// Other calls are forwarded directly.
|
| 76 |
+
class ARROW_EXPORT SlowInputStream : public SlowInputStreamBase<InputStream> {
|
| 77 |
+
public:
|
| 78 |
+
~SlowInputStream() override;
|
| 79 |
+
|
| 80 |
+
using SlowInputStreamBase<InputStream>::SlowInputStreamBase;
|
| 81 |
+
|
| 82 |
+
Status Close() override;
|
| 83 |
+
Status Abort() override;
|
| 84 |
+
bool closed() const override;
|
| 85 |
+
|
| 86 |
+
Result<int64_t> Read(int64_t nbytes, void* out) override;
|
| 87 |
+
Result<std::shared_ptr<Buffer>> Read(int64_t nbytes) override;
|
| 88 |
+
Result<std::string_view> Peek(int64_t nbytes) override;
|
| 89 |
+
|
| 90 |
+
Result<int64_t> Tell() const override;
|
| 91 |
+
};
|
| 92 |
+
|
| 93 |
+
/// \brief A RandomAccessFile wrapper that makes reads slower.
|
| 94 |
+
///
|
| 95 |
+
/// Similar to SlowInputStream, but allows random access and seeking.
|
| 96 |
+
class ARROW_EXPORT SlowRandomAccessFile : public SlowInputStreamBase<RandomAccessFile> {
|
| 97 |
+
public:
|
| 98 |
+
~SlowRandomAccessFile() override;
|
| 99 |
+
|
| 100 |
+
using SlowInputStreamBase<RandomAccessFile>::SlowInputStreamBase;
|
| 101 |
+
|
| 102 |
+
Status Close() override;
|
| 103 |
+
Status Abort() override;
|
| 104 |
+
bool closed() const override;
|
| 105 |
+
|
| 106 |
+
Result<int64_t> Read(int64_t nbytes, void* out) override;
|
| 107 |
+
Result<std::shared_ptr<Buffer>> Read(int64_t nbytes) override;
|
| 108 |
+
Result<int64_t> ReadAt(int64_t position, int64_t nbytes, void* out) override;
|
| 109 |
+
Result<std::shared_ptr<Buffer>> ReadAt(int64_t position, int64_t nbytes) override;
|
| 110 |
+
Result<std::string_view> Peek(int64_t nbytes) override;
|
| 111 |
+
|
| 112 |
+
Result<int64_t> GetSize() override;
|
| 113 |
+
Status Seek(int64_t position) override;
|
| 114 |
+
Result<int64_t> Tell() const override;
|
| 115 |
+
};
|
| 116 |
+
|
| 117 |
+
} // namespace io
|
| 118 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/stdio.h
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <cstdint>
|
| 21 |
+
|
| 22 |
+
#include "arrow/io/interfaces.h"
|
| 23 |
+
#include "arrow/util/visibility.h"
|
| 24 |
+
|
| 25 |
+
namespace arrow {
|
| 26 |
+
namespace io {
|
| 27 |
+
|
| 28 |
+
// Output stream that just writes to stdout.
|
| 29 |
+
class ARROW_EXPORT StdoutStream : public OutputStream {
|
| 30 |
+
public:
|
| 31 |
+
StdoutStream();
|
| 32 |
+
~StdoutStream() override {}
|
| 33 |
+
|
| 34 |
+
Status Close() override;
|
| 35 |
+
bool closed() const override;
|
| 36 |
+
|
| 37 |
+
Result<int64_t> Tell() const override;
|
| 38 |
+
|
| 39 |
+
Status Write(const void* data, int64_t nbytes) override;
|
| 40 |
+
|
| 41 |
+
private:
|
| 42 |
+
int64_t pos_;
|
| 43 |
+
};
|
| 44 |
+
|
| 45 |
+
// Output stream that just writes to stderr.
|
| 46 |
+
class ARROW_EXPORT StderrStream : public OutputStream {
|
| 47 |
+
public:
|
| 48 |
+
StderrStream();
|
| 49 |
+
~StderrStream() override {}
|
| 50 |
+
|
| 51 |
+
Status Close() override;
|
| 52 |
+
bool closed() const override;
|
| 53 |
+
|
| 54 |
+
Result<int64_t> Tell() const override;
|
| 55 |
+
|
| 56 |
+
Status Write(const void* data, int64_t nbytes) override;
|
| 57 |
+
|
| 58 |
+
private:
|
| 59 |
+
int64_t pos_;
|
| 60 |
+
};
|
| 61 |
+
|
| 62 |
+
// Input stream that just reads from stdin.
|
| 63 |
+
class ARROW_EXPORT StdinStream : public InputStream {
|
| 64 |
+
public:
|
| 65 |
+
StdinStream();
|
| 66 |
+
~StdinStream() override {}
|
| 67 |
+
|
| 68 |
+
Status Close() override;
|
| 69 |
+
bool closed() const override;
|
| 70 |
+
|
| 71 |
+
Result<int64_t> Tell() const override;
|
| 72 |
+
|
| 73 |
+
Result<int64_t> Read(int64_t nbytes, void* out) override;
|
| 74 |
+
|
| 75 |
+
Result<std::shared_ptr<Buffer>> Read(int64_t nbytes) override;
|
| 76 |
+
|
| 77 |
+
private:
|
| 78 |
+
int64_t pos_;
|
| 79 |
+
};
|
| 80 |
+
|
| 81 |
+
} // namespace io
|
| 82 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/test_common.h
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <memory>
|
| 21 |
+
#include <string>
|
| 22 |
+
#include <vector>
|
| 23 |
+
|
| 24 |
+
#include "arrow/io/interfaces.h"
|
| 25 |
+
#include "arrow/testing/visibility.h"
|
| 26 |
+
#include "arrow/type_fwd.h"
|
| 27 |
+
|
| 28 |
+
namespace arrow {
|
| 29 |
+
namespace io {
|
| 30 |
+
|
| 31 |
+
class MemoryMappedFile;
|
| 32 |
+
|
| 33 |
+
ARROW_TESTING_EXPORT
|
| 34 |
+
void AssertFileContents(const std::string& path, const std::string& contents);
|
| 35 |
+
|
| 36 |
+
ARROW_TESTING_EXPORT bool FileExists(const std::string& path);
|
| 37 |
+
|
| 38 |
+
ARROW_TESTING_EXPORT Status PurgeLocalFileFromOsCache(const std::string& path);
|
| 39 |
+
|
| 40 |
+
ARROW_TESTING_EXPORT
|
| 41 |
+
Status ZeroMemoryMap(MemoryMappedFile* file);
|
| 42 |
+
|
| 43 |
+
class ARROW_TESTING_EXPORT MemoryMapFixture {
|
| 44 |
+
public:
|
| 45 |
+
void TearDown();
|
| 46 |
+
|
| 47 |
+
void CreateFile(const std::string& path, int64_t size);
|
| 48 |
+
|
| 49 |
+
Result<std::shared_ptr<MemoryMappedFile>> InitMemoryMap(int64_t size,
|
| 50 |
+
const std::string& path);
|
| 51 |
+
|
| 52 |
+
void AppendFile(const std::string& path);
|
| 53 |
+
|
| 54 |
+
private:
|
| 55 |
+
std::vector<std::string> tmp_files_;
|
| 56 |
+
};
|
| 57 |
+
|
| 58 |
+
class ARROW_TESTING_EXPORT TrackedRandomAccessFile : public io::RandomAccessFile {
|
| 59 |
+
public:
|
| 60 |
+
virtual int64_t num_reads() const = 0;
|
| 61 |
+
virtual int64_t bytes_read() const = 0;
|
| 62 |
+
virtual const std::vector<io::ReadRange>& get_read_ranges() const = 0;
|
| 63 |
+
virtual void ResetStats() = 0;
|
| 64 |
+
|
| 65 |
+
static std::unique_ptr<TrackedRandomAccessFile> Make(io::RandomAccessFile* target);
|
| 66 |
+
};
|
| 67 |
+
|
| 68 |
+
} // namespace io
|
| 69 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/transform.h
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
// Transform stream implementations
|
| 19 |
+
|
| 20 |
+
#pragma once
|
| 21 |
+
|
| 22 |
+
#include <cstdint>
|
| 23 |
+
#include <functional>
|
| 24 |
+
#include <memory>
|
| 25 |
+
#include <utility>
|
| 26 |
+
|
| 27 |
+
#include "arrow/io/interfaces.h"
|
| 28 |
+
#include "arrow/util/visibility.h"
|
| 29 |
+
|
| 30 |
+
namespace arrow {
|
| 31 |
+
namespace io {
|
| 32 |
+
|
| 33 |
+
class ARROW_EXPORT TransformInputStream : public InputStream {
|
| 34 |
+
public:
|
| 35 |
+
using TransformFunc =
|
| 36 |
+
std::function<Result<std::shared_ptr<Buffer>>(const std::shared_ptr<Buffer>&)>;
|
| 37 |
+
|
| 38 |
+
TransformInputStream(std::shared_ptr<InputStream> wrapped, TransformFunc transform);
|
| 39 |
+
~TransformInputStream() override;
|
| 40 |
+
|
| 41 |
+
Status Close() override;
|
| 42 |
+
Status Abort() override;
|
| 43 |
+
bool closed() const override;
|
| 44 |
+
|
| 45 |
+
Result<int64_t> Read(int64_t nbytes, void* out) override;
|
| 46 |
+
Result<std::shared_ptr<Buffer>> Read(int64_t nbytes) override;
|
| 47 |
+
|
| 48 |
+
Result<std::shared_ptr<const KeyValueMetadata>> ReadMetadata() override;
|
| 49 |
+
Future<std::shared_ptr<const KeyValueMetadata>> ReadMetadataAsync(
|
| 50 |
+
const IOContext& io_context) override;
|
| 51 |
+
|
| 52 |
+
Result<int64_t> Tell() const override;
|
| 53 |
+
|
| 54 |
+
protected:
|
| 55 |
+
struct Impl;
|
| 56 |
+
std::unique_ptr<Impl> impl_;
|
| 57 |
+
};
|
| 58 |
+
|
| 59 |
+
} // namespace io
|
| 60 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/type_fwd.h
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include "arrow/type_fwd.h"
|
| 21 |
+
#include "arrow/util/visibility.h"
|
| 22 |
+
|
| 23 |
+
namespace arrow {
|
| 24 |
+
namespace io {
|
| 25 |
+
|
| 26 |
+
struct FileMode {
|
| 27 |
+
enum type { READ, WRITE, READWRITE };
|
| 28 |
+
};
|
| 29 |
+
|
| 30 |
+
struct IOContext;
|
| 31 |
+
struct CacheOptions;
|
| 32 |
+
|
| 33 |
+
/// EXPERIMENTAL: convenience global singleton for default IOContext settings
|
| 34 |
+
ARROW_EXPORT
|
| 35 |
+
const IOContext& default_io_context();
|
| 36 |
+
|
| 37 |
+
/// \brief Get the capacity of the global I/O thread pool
|
| 38 |
+
///
|
| 39 |
+
/// Return the number of worker threads in the thread pool to which
|
| 40 |
+
/// Arrow dispatches various I/O-bound tasks. This is an ideal number,
|
| 41 |
+
/// not necessarily the exact number of threads at a given point in time.
|
| 42 |
+
///
|
| 43 |
+
/// You can change this number using SetIOThreadPoolCapacity().
|
| 44 |
+
ARROW_EXPORT int GetIOThreadPoolCapacity();
|
| 45 |
+
|
| 46 |
+
/// \brief Set the capacity of the global I/O thread pool
|
| 47 |
+
///
|
| 48 |
+
/// Set the number of worker threads in the thread pool to which
|
| 49 |
+
/// Arrow dispatches various I/O-bound tasks.
|
| 50 |
+
///
|
| 51 |
+
/// The current number is returned by GetIOThreadPoolCapacity().
|
| 52 |
+
ARROW_EXPORT Status SetIOThreadPoolCapacity(int threads);
|
| 53 |
+
|
| 54 |
+
class FileInterface;
|
| 55 |
+
class Seekable;
|
| 56 |
+
class Writable;
|
| 57 |
+
class Readable;
|
| 58 |
+
class OutputStream;
|
| 59 |
+
class FileOutputStream;
|
| 60 |
+
class InputStream;
|
| 61 |
+
class ReadableFile;
|
| 62 |
+
class RandomAccessFile;
|
| 63 |
+
class MemoryMappedFile;
|
| 64 |
+
class WritableFile;
|
| 65 |
+
class ReadWriteFileInterface;
|
| 66 |
+
|
| 67 |
+
class LatencyGenerator;
|
| 68 |
+
|
| 69 |
+
class BufferOutputStream;
|
| 70 |
+
class BufferReader;
|
| 71 |
+
class CompressedInputStream;
|
| 72 |
+
class CompressedOutputStream;
|
| 73 |
+
class BufferedInputStream;
|
| 74 |
+
class BufferedOutputStream;
|
| 75 |
+
|
| 76 |
+
} // namespace io
|
| 77 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/api.h
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include "arrow/ipc/dictionary.h"
|
| 21 |
+
#include "arrow/ipc/feather.h"
|
| 22 |
+
#include "arrow/ipc/message.h"
|
| 23 |
+
#include "arrow/ipc/reader.h"
|
| 24 |
+
#include "arrow/ipc/writer.h"
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/dictionary.h
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
// Tools for dictionaries in IPC context
|
| 19 |
+
|
| 20 |
+
#pragma once
|
| 21 |
+
|
| 22 |
+
#include <cstdint>
|
| 23 |
+
#include <memory>
|
| 24 |
+
#include <utility>
|
| 25 |
+
#include <vector>
|
| 26 |
+
|
| 27 |
+
#include "arrow/result.h"
|
| 28 |
+
#include "arrow/status.h"
|
| 29 |
+
#include "arrow/type_fwd.h"
|
| 30 |
+
#include "arrow/util/macros.h"
|
| 31 |
+
#include "arrow/util/visibility.h"
|
| 32 |
+
|
| 33 |
+
namespace arrow {
|
| 34 |
+
namespace ipc {
|
| 35 |
+
|
| 36 |
+
namespace internal {
|
| 37 |
+
|
| 38 |
+
class FieldPosition {
|
| 39 |
+
public:
|
| 40 |
+
FieldPosition() : parent_(NULLPTR), index_(-1), depth_(0) {}
|
| 41 |
+
|
| 42 |
+
FieldPosition child(int index) const { return {this, index}; }
|
| 43 |
+
|
| 44 |
+
std::vector<int> path() const {
|
| 45 |
+
std::vector<int> path(depth_);
|
| 46 |
+
const FieldPosition* cur = this;
|
| 47 |
+
for (int i = depth_ - 1; i >= 0; --i) {
|
| 48 |
+
path[i] = cur->index_;
|
| 49 |
+
cur = cur->parent_;
|
| 50 |
+
}
|
| 51 |
+
return path;
|
| 52 |
+
}
|
| 53 |
+
|
| 54 |
+
protected:
|
| 55 |
+
FieldPosition(const FieldPosition* parent, int index)
|
| 56 |
+
: parent_(parent), index_(index), depth_(parent->depth_ + 1) {}
|
| 57 |
+
|
| 58 |
+
const FieldPosition* parent_;
|
| 59 |
+
int index_;
|
| 60 |
+
int depth_;
|
| 61 |
+
};
|
| 62 |
+
|
| 63 |
+
} // namespace internal
|
| 64 |
+
|
| 65 |
+
/// \brief Map fields in a schema to dictionary ids
|
| 66 |
+
///
|
| 67 |
+
/// The mapping is structural, i.e. the field path (as a vector of indices)
|
| 68 |
+
/// is associated to the dictionary id. A dictionary id may be associated
|
| 69 |
+
/// to multiple fields.
|
| 70 |
+
class ARROW_EXPORT DictionaryFieldMapper {
|
| 71 |
+
public:
|
| 72 |
+
DictionaryFieldMapper();
|
| 73 |
+
explicit DictionaryFieldMapper(const Schema& schema);
|
| 74 |
+
~DictionaryFieldMapper();
|
| 75 |
+
|
| 76 |
+
Status AddSchemaFields(const Schema& schema);
|
| 77 |
+
Status AddField(int64_t id, std::vector<int> field_path);
|
| 78 |
+
|
| 79 |
+
Result<int64_t> GetFieldId(std::vector<int> field_path) const;
|
| 80 |
+
|
| 81 |
+
int num_fields() const;
|
| 82 |
+
|
| 83 |
+
/// \brief Returns number of unique dictionaries, taking into
|
| 84 |
+
/// account that different fields can share the same dictionary.
|
| 85 |
+
int num_dicts() const;
|
| 86 |
+
|
| 87 |
+
private:
|
| 88 |
+
struct Impl;
|
| 89 |
+
std::unique_ptr<Impl> impl_;
|
| 90 |
+
};
|
| 91 |
+
|
| 92 |
+
using DictionaryVector = std::vector<std::pair<int64_t, std::shared_ptr<Array>>>;
|
| 93 |
+
|
| 94 |
+
/// \brief Memoization data structure for reading dictionaries from IPC streams
|
| 95 |
+
///
|
| 96 |
+
/// This structure tracks the following associations:
|
| 97 |
+
/// - field position (structural) -> dictionary id
|
| 98 |
+
/// - dictionary id -> value type
|
| 99 |
+
/// - dictionary id -> dictionary (value) data
|
| 100 |
+
///
|
| 101 |
+
/// Together, they allow resolving dictionary data when reading an IPC stream,
|
| 102 |
+
/// using metadata recorded in the schema message and data recorded in the
|
| 103 |
+
/// dictionary batch messages (see ResolveDictionaries).
|
| 104 |
+
///
|
| 105 |
+
/// This structure isn't useful for writing an IPC stream, where only
|
| 106 |
+
/// DictionaryFieldMapper is necessary.
|
| 107 |
+
class ARROW_EXPORT DictionaryMemo {
|
| 108 |
+
public:
|
| 109 |
+
DictionaryMemo();
|
| 110 |
+
~DictionaryMemo();
|
| 111 |
+
|
| 112 |
+
DictionaryFieldMapper& fields();
|
| 113 |
+
const DictionaryFieldMapper& fields() const;
|
| 114 |
+
|
| 115 |
+
/// \brief Return current dictionary corresponding to a particular
|
| 116 |
+
/// id. Returns KeyError if id not found
|
| 117 |
+
Result<std::shared_ptr<ArrayData>> GetDictionary(int64_t id, MemoryPool* pool) const;
|
| 118 |
+
|
| 119 |
+
/// \brief Return dictionary value type corresponding to a
|
| 120 |
+
/// particular dictionary id.
|
| 121 |
+
Result<std::shared_ptr<DataType>> GetDictionaryType(int64_t id) const;
|
| 122 |
+
|
| 123 |
+
/// \brief Return true if we have a dictionary for the input id
|
| 124 |
+
bool HasDictionary(int64_t id) const;
|
| 125 |
+
|
| 126 |
+
/// \brief Add a dictionary value type to the memo with a particular id.
|
| 127 |
+
/// Returns KeyError if a different type is already registered with the same id.
|
| 128 |
+
Status AddDictionaryType(int64_t id, const std::shared_ptr<DataType>& type);
|
| 129 |
+
|
| 130 |
+
/// \brief Add a dictionary to the memo with a particular id. Returns
|
| 131 |
+
/// KeyError if that dictionary already exists
|
| 132 |
+
Status AddDictionary(int64_t id, const std::shared_ptr<ArrayData>& dictionary);
|
| 133 |
+
|
| 134 |
+
/// \brief Append a dictionary delta to the memo with a particular id. Returns
|
| 135 |
+
/// KeyError if that dictionary does not exists
|
| 136 |
+
Status AddDictionaryDelta(int64_t id, const std::shared_ptr<ArrayData>& dictionary);
|
| 137 |
+
|
| 138 |
+
/// \brief Add a dictionary to the memo if it does not have one with the id,
|
| 139 |
+
/// otherwise, replace the dictionary with the new one.
|
| 140 |
+
///
|
| 141 |
+
/// Return true if the dictionary was added, false if replaced.
|
| 142 |
+
Result<bool> AddOrReplaceDictionary(int64_t id,
|
| 143 |
+
const std::shared_ptr<ArrayData>& dictionary);
|
| 144 |
+
|
| 145 |
+
private:
|
| 146 |
+
struct Impl;
|
| 147 |
+
std::unique_ptr<Impl> impl_;
|
| 148 |
+
};
|
| 149 |
+
|
| 150 |
+
// For writing: collect dictionary entries to write to the IPC stream, in order
|
| 151 |
+
// (i.e. inner dictionaries before dependent outer dictionaries).
|
| 152 |
+
ARROW_EXPORT
|
| 153 |
+
Result<DictionaryVector> CollectDictionaries(const RecordBatch& batch,
|
| 154 |
+
const DictionaryFieldMapper& mapper);
|
| 155 |
+
|
| 156 |
+
// For reading: resolve all dictionaries in columns, according to the field
|
| 157 |
+
// mapping and dictionary arrays stored in memo.
|
| 158 |
+
// Columns may be sparse, i.e. some entries may be left null
|
| 159 |
+
// (e.g. if an inclusion mask was used).
|
| 160 |
+
ARROW_EXPORT
|
| 161 |
+
Status ResolveDictionaries(const ArrayDataVector& columns, const DictionaryMemo& memo,
|
| 162 |
+
MemoryPool* pool);
|
| 163 |
+
|
| 164 |
+
namespace internal {
|
| 165 |
+
|
| 166 |
+
// Like CollectDictionaries above, but uses the memo's DictionaryFieldMapper
|
| 167 |
+
// and all collected dictionaries are added to the memo using AddDictionary.
|
| 168 |
+
//
|
| 169 |
+
// This is used as a shortcut in some roundtripping tests (to avoid emitting
|
| 170 |
+
// any actual dictionary batches).
|
| 171 |
+
ARROW_EXPORT
|
| 172 |
+
Status CollectDictionaries(const RecordBatch& batch, DictionaryMemo* memo);
|
| 173 |
+
|
| 174 |
+
} // namespace internal
|
| 175 |
+
|
| 176 |
+
} // namespace ipc
|
| 177 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/feather.h
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
// Public API for the "Feather" file format, originally created at
|
| 19 |
+
// http://github.com/wesm/feather
|
| 20 |
+
|
| 21 |
+
#pragma once
|
| 22 |
+
|
| 23 |
+
#include <cstdint>
|
| 24 |
+
#include <memory>
|
| 25 |
+
#include <string>
|
| 26 |
+
#include <vector>
|
| 27 |
+
|
| 28 |
+
#include "arrow/ipc/options.h"
|
| 29 |
+
#include "arrow/type_fwd.h"
|
| 30 |
+
#include "arrow/util/compression.h"
|
| 31 |
+
#include "arrow/util/visibility.h"
|
| 32 |
+
|
| 33 |
+
namespace arrow {
|
| 34 |
+
|
| 35 |
+
class Schema;
|
| 36 |
+
class Status;
|
| 37 |
+
class Table;
|
| 38 |
+
|
| 39 |
+
namespace io {
|
| 40 |
+
|
| 41 |
+
class OutputStream;
|
| 42 |
+
class RandomAccessFile;
|
| 43 |
+
|
| 44 |
+
} // namespace io
|
| 45 |
+
|
| 46 |
+
namespace ipc {
|
| 47 |
+
namespace feather {
|
| 48 |
+
|
| 49 |
+
static constexpr const int kFeatherV1Version = 2;
|
| 50 |
+
static constexpr const int kFeatherV2Version = 3;
|
| 51 |
+
|
| 52 |
+
// ----------------------------------------------------------------------
|
| 53 |
+
// Metadata accessor classes
|
| 54 |
+
|
| 55 |
+
/// \class Reader
|
| 56 |
+
/// \brief An interface for reading columns from Feather files
|
| 57 |
+
class ARROW_EXPORT Reader {
|
| 58 |
+
public:
|
| 59 |
+
virtual ~Reader() = default;
|
| 60 |
+
|
| 61 |
+
/// \brief Open a Feather file from a RandomAccessFile interface
|
| 62 |
+
///
|
| 63 |
+
/// \param[in] source a RandomAccessFile instance
|
| 64 |
+
/// \return the table reader
|
| 65 |
+
static Result<std::shared_ptr<Reader>> Open(
|
| 66 |
+
const std::shared_ptr<io::RandomAccessFile>& source);
|
| 67 |
+
|
| 68 |
+
/// \brief Open a Feather file from a RandomAccessFile interface
|
| 69 |
+
/// with IPC Read options
|
| 70 |
+
///
|
| 71 |
+
/// \param[in] source a RandomAccessFile instance
|
| 72 |
+
/// \param[in] options IPC Read options
|
| 73 |
+
/// \return the table reader
|
| 74 |
+
static Result<std::shared_ptr<Reader>> Open(
|
| 75 |
+
const std::shared_ptr<io::RandomAccessFile>& source, const IpcReadOptions& options);
|
| 76 |
+
|
| 77 |
+
/// \brief Return the version number of the Feather file
|
| 78 |
+
virtual int version() const = 0;
|
| 79 |
+
|
| 80 |
+
virtual std::shared_ptr<Schema> schema() const = 0;
|
| 81 |
+
|
| 82 |
+
/// \brief Read all columns from the file as an arrow::Table.
|
| 83 |
+
///
|
| 84 |
+
/// \param[out] out the returned table
|
| 85 |
+
/// \return Status
|
| 86 |
+
///
|
| 87 |
+
/// This function is zero-copy if the file source supports zero-copy reads
|
| 88 |
+
virtual Status Read(std::shared_ptr<Table>* out) = 0;
|
| 89 |
+
|
| 90 |
+
/// \brief Read only the specified columns from the file as an arrow::Table.
|
| 91 |
+
///
|
| 92 |
+
/// \param[in] indices the column indices to read
|
| 93 |
+
/// \param[out] out the returned table
|
| 94 |
+
/// \return Status
|
| 95 |
+
///
|
| 96 |
+
/// This function is zero-copy if the file source supports zero-copy reads
|
| 97 |
+
virtual Status Read(const std::vector<int>& indices, std::shared_ptr<Table>* out) = 0;
|
| 98 |
+
|
| 99 |
+
/// \brief Read only the specified columns from the file as an arrow::Table.
|
| 100 |
+
///
|
| 101 |
+
/// \param[in] names the column names to read
|
| 102 |
+
/// \param[out] out the returned table
|
| 103 |
+
/// \return Status
|
| 104 |
+
///
|
| 105 |
+
/// This function is zero-copy if the file source supports zero-copy reads
|
| 106 |
+
virtual Status Read(const std::vector<std::string>& names,
|
| 107 |
+
std::shared_ptr<Table>* out) = 0;
|
| 108 |
+
};
|
| 109 |
+
|
| 110 |
+
struct ARROW_EXPORT WriteProperties {
|
| 111 |
+
static WriteProperties Defaults();
|
| 112 |
+
|
| 113 |
+
static WriteProperties DefaultsV1() {
|
| 114 |
+
WriteProperties props = Defaults();
|
| 115 |
+
props.version = kFeatherV1Version;
|
| 116 |
+
return props;
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
/// Feather file version number
|
| 120 |
+
///
|
| 121 |
+
/// version 2: "Feather V1" Apache Arrow <= 0.16.0
|
| 122 |
+
/// version 3: "Feather V2" Apache Arrow > 0.16.0
|
| 123 |
+
int version = kFeatherV2Version;
|
| 124 |
+
|
| 125 |
+
// Parameters for Feather V2 only
|
| 126 |
+
|
| 127 |
+
/// Number of rows per intra-file chunk. Use smaller chunksize when you need
|
| 128 |
+
/// faster random row access
|
| 129 |
+
int64_t chunksize = 1LL << 16;
|
| 130 |
+
|
| 131 |
+
/// Compression type to use. Only UNCOMPRESSED, LZ4_FRAME, and ZSTD are
|
| 132 |
+
/// supported. The default compression returned by Defaults() is LZ4 if the
|
| 133 |
+
/// project is built with support for it, otherwise
|
| 134 |
+
/// UNCOMPRESSED. UNCOMPRESSED is set as the object default here so that if
|
| 135 |
+
/// WriteProperties::Defaults() is not used, the default constructor for
|
| 136 |
+
/// WriteProperties will work regardless of the options used to build the C++
|
| 137 |
+
/// project.
|
| 138 |
+
Compression::type compression = Compression::UNCOMPRESSED;
|
| 139 |
+
|
| 140 |
+
/// Compressor-specific compression level
|
| 141 |
+
int compression_level = ::arrow::util::kUseDefaultCompressionLevel;
|
| 142 |
+
};
|
| 143 |
+
|
| 144 |
+
ARROW_EXPORT
|
| 145 |
+
Status WriteTable(const Table& table, io::OutputStream* dst,
|
| 146 |
+
const WriteProperties& properties = WriteProperties::Defaults());
|
| 147 |
+
|
| 148 |
+
} // namespace feather
|
| 149 |
+
} // namespace ipc
|
| 150 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/message.h
ADDED
|
@@ -0,0 +1,585 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
// C++ object model and user API for interprocess schema messaging
|
| 19 |
+
|
| 20 |
+
#pragma once
|
| 21 |
+
|
| 22 |
+
#include <cstdint>
|
| 23 |
+
#include <functional>
|
| 24 |
+
#include <memory>
|
| 25 |
+
#include <string>
|
| 26 |
+
#include <utility>
|
| 27 |
+
|
| 28 |
+
#include "arrow/io/type_fwd.h"
|
| 29 |
+
#include "arrow/ipc/type_fwd.h"
|
| 30 |
+
#include "arrow/result.h"
|
| 31 |
+
#include "arrow/status.h"
|
| 32 |
+
#include "arrow/type_fwd.h"
|
| 33 |
+
#include "arrow/util/macros.h"
|
| 34 |
+
#include "arrow/util/visibility.h"
|
| 35 |
+
|
| 36 |
+
namespace arrow {
|
| 37 |
+
namespace ipc {
|
| 38 |
+
|
| 39 |
+
struct IpcWriteOptions;
|
| 40 |
+
|
| 41 |
+
// Read interface classes. We do not fully deserialize the flatbuffers so that
|
| 42 |
+
// individual fields metadata can be retrieved from very large schema without
|
| 43 |
+
//
|
| 44 |
+
|
| 45 |
+
/// \class Message
|
| 46 |
+
/// \brief An IPC message including metadata and body
|
| 47 |
+
class ARROW_EXPORT Message {
|
| 48 |
+
public:
|
| 49 |
+
/// \brief Construct message, but do not validate
|
| 50 |
+
///
|
| 51 |
+
/// Use at your own risk; Message::Open has more metadata validation
|
| 52 |
+
Message(std::shared_ptr<Buffer> metadata, std::shared_ptr<Buffer> body);
|
| 53 |
+
|
| 54 |
+
~Message();
|
| 55 |
+
|
| 56 |
+
/// \brief Create and validate a Message instance from two buffers
|
| 57 |
+
///
|
| 58 |
+
/// \param[in] metadata a buffer containing the Flatbuffer metadata
|
| 59 |
+
/// \param[in] body a buffer containing the message body, which may be null
|
| 60 |
+
/// \return the created message
|
| 61 |
+
static Result<std::unique_ptr<Message>> Open(std::shared_ptr<Buffer> metadata,
|
| 62 |
+
std::shared_ptr<Buffer> body);
|
| 63 |
+
|
| 64 |
+
/// \brief Read message body and create Message given Flatbuffer metadata
|
| 65 |
+
/// \param[in] metadata containing a serialized Message flatbuffer
|
| 66 |
+
/// \param[in] stream an InputStream
|
| 67 |
+
/// \return the created Message
|
| 68 |
+
///
|
| 69 |
+
/// \note If stream supports zero-copy, this is zero-copy
|
| 70 |
+
static Result<std::unique_ptr<Message>> ReadFrom(std::shared_ptr<Buffer> metadata,
|
| 71 |
+
io::InputStream* stream);
|
| 72 |
+
|
| 73 |
+
/// \brief Read message body from position in file, and create Message given
|
| 74 |
+
/// the Flatbuffer metadata
|
| 75 |
+
/// \param[in] offset the position in the file where the message body starts.
|
| 76 |
+
/// \param[in] metadata containing a serialized Message flatbuffer
|
| 77 |
+
/// \param[in] file the seekable file interface to read from
|
| 78 |
+
/// \return the created Message
|
| 79 |
+
///
|
| 80 |
+
/// \note If file supports zero-copy, this is zero-copy
|
| 81 |
+
static Result<std::unique_ptr<Message>> ReadFrom(const int64_t offset,
|
| 82 |
+
std::shared_ptr<Buffer> metadata,
|
| 83 |
+
io::RandomAccessFile* file);
|
| 84 |
+
|
| 85 |
+
/// \brief Return true if message type and contents are equal
|
| 86 |
+
///
|
| 87 |
+
/// \param other another message
|
| 88 |
+
/// \return true if contents equal
|
| 89 |
+
bool Equals(const Message& other) const;
|
| 90 |
+
|
| 91 |
+
/// \brief the Message metadata
|
| 92 |
+
///
|
| 93 |
+
/// \return buffer
|
| 94 |
+
std::shared_ptr<Buffer> metadata() const;
|
| 95 |
+
|
| 96 |
+
/// \brief Custom metadata serialized in metadata Flatbuffer. Returns nullptr
|
| 97 |
+
/// when none set
|
| 98 |
+
const std::shared_ptr<const KeyValueMetadata>& custom_metadata() const;
|
| 99 |
+
|
| 100 |
+
/// \brief the Message body, if any
|
| 101 |
+
///
|
| 102 |
+
/// \return buffer is null if no body
|
| 103 |
+
std::shared_ptr<Buffer> body() const;
|
| 104 |
+
|
| 105 |
+
/// \brief The expected body length according to the metadata, for
|
| 106 |
+
/// verification purposes
|
| 107 |
+
int64_t body_length() const;
|
| 108 |
+
|
| 109 |
+
/// \brief The Message type
|
| 110 |
+
MessageType type() const;
|
| 111 |
+
|
| 112 |
+
/// \brief The Message metadata version
|
| 113 |
+
MetadataVersion metadata_version() const;
|
| 114 |
+
|
| 115 |
+
const void* header() const;
|
| 116 |
+
|
| 117 |
+
/// \brief Write length-prefixed metadata and body to output stream
|
| 118 |
+
///
|
| 119 |
+
/// \param[in] file output stream to write to
|
| 120 |
+
/// \param[in] options IPC writing options including alignment
|
| 121 |
+
/// \param[out] output_length the number of bytes written
|
| 122 |
+
/// \return Status
|
| 123 |
+
Status SerializeTo(io::OutputStream* file, const IpcWriteOptions& options,
|
| 124 |
+
int64_t* output_length) const;
|
| 125 |
+
|
| 126 |
+
/// \brief Return true if the Message metadata passes Flatbuffer validation
|
| 127 |
+
bool Verify() const;
|
| 128 |
+
|
| 129 |
+
/// \brief Whether a given message type needs a body.
|
| 130 |
+
static bool HasBody(MessageType type) {
|
| 131 |
+
return type != MessageType::NONE && type != MessageType::SCHEMA;
|
| 132 |
+
}
|
| 133 |
+
|
| 134 |
+
private:
|
| 135 |
+
// Hide serialization details from user API
|
| 136 |
+
class MessageImpl;
|
| 137 |
+
std::unique_ptr<MessageImpl> impl_;
|
| 138 |
+
|
| 139 |
+
ARROW_DISALLOW_COPY_AND_ASSIGN(Message);
|
| 140 |
+
};
|
| 141 |
+
|
| 142 |
+
ARROW_EXPORT std::string FormatMessageType(MessageType type);
|
| 143 |
+
|
| 144 |
+
/// \class MessageDecoderListener
|
| 145 |
+
/// \brief An abstract class to listen events from MessageDecoder.
|
| 146 |
+
///
|
| 147 |
+
/// This API is EXPERIMENTAL.
|
| 148 |
+
///
|
| 149 |
+
/// \since 0.17.0
|
| 150 |
+
class ARROW_EXPORT MessageDecoderListener {
|
| 151 |
+
public:
|
| 152 |
+
virtual ~MessageDecoderListener() = default;
|
| 153 |
+
|
| 154 |
+
/// \brief Called when a message is decoded.
|
| 155 |
+
///
|
| 156 |
+
/// MessageDecoder calls this method when it decodes a message. This
|
| 157 |
+
/// method is called multiple times when the target stream has
|
| 158 |
+
/// multiple messages.
|
| 159 |
+
///
|
| 160 |
+
/// \param[in] message a decoded message
|
| 161 |
+
/// \return Status
|
| 162 |
+
virtual Status OnMessageDecoded(std::unique_ptr<Message> message) = 0;
|
| 163 |
+
|
| 164 |
+
/// \brief Called when the decoder state is changed to
|
| 165 |
+
/// MessageDecoder::State::INITIAL.
|
| 166 |
+
///
|
| 167 |
+
/// The default implementation just returns arrow::Status::OK().
|
| 168 |
+
///
|
| 169 |
+
/// \return Status
|
| 170 |
+
virtual Status OnInitial();
|
| 171 |
+
|
| 172 |
+
/// \brief Called when the decoder state is changed to
|
| 173 |
+
/// MessageDecoder::State::METADATA_LENGTH.
|
| 174 |
+
///
|
| 175 |
+
/// The default implementation just returns arrow::Status::OK().
|
| 176 |
+
///
|
| 177 |
+
/// \return Status
|
| 178 |
+
virtual Status OnMetadataLength();
|
| 179 |
+
|
| 180 |
+
/// \brief Called when the decoder state is changed to
|
| 181 |
+
/// MessageDecoder::State::METADATA.
|
| 182 |
+
///
|
| 183 |
+
/// The default implementation just returns arrow::Status::OK().
|
| 184 |
+
///
|
| 185 |
+
/// \return Status
|
| 186 |
+
virtual Status OnMetadata();
|
| 187 |
+
|
| 188 |
+
/// \brief Called when the decoder state is changed to
|
| 189 |
+
/// MessageDecoder::State::BODY.
|
| 190 |
+
///
|
| 191 |
+
/// The default implementation just returns arrow::Status::OK().
|
| 192 |
+
///
|
| 193 |
+
/// \return Status
|
| 194 |
+
virtual Status OnBody();
|
| 195 |
+
|
| 196 |
+
/// \brief Called when the decoder state is changed to
|
| 197 |
+
/// MessageDecoder::State::EOS.
|
| 198 |
+
///
|
| 199 |
+
/// The default implementation just returns arrow::Status::OK().
|
| 200 |
+
///
|
| 201 |
+
/// \return Status
|
| 202 |
+
virtual Status OnEOS();
|
| 203 |
+
};
|
| 204 |
+
|
| 205 |
+
/// \class AssignMessageDecoderListener
|
| 206 |
+
/// \brief Assign a message decoded by MessageDecoder.
|
| 207 |
+
///
|
| 208 |
+
/// This API is EXPERIMENTAL.
|
| 209 |
+
///
|
| 210 |
+
/// \since 0.17.0
|
| 211 |
+
class ARROW_EXPORT AssignMessageDecoderListener : public MessageDecoderListener {
|
| 212 |
+
public:
|
| 213 |
+
/// \brief Construct a listener that assigns a decoded message to the
|
| 214 |
+
/// specified location.
|
| 215 |
+
///
|
| 216 |
+
/// \param[in] message a location to store the received message
|
| 217 |
+
explicit AssignMessageDecoderListener(std::unique_ptr<Message>* message)
|
| 218 |
+
: message_(message) {}
|
| 219 |
+
|
| 220 |
+
virtual ~AssignMessageDecoderListener() = default;
|
| 221 |
+
|
| 222 |
+
Status OnMessageDecoded(std::unique_ptr<Message> message) override {
|
| 223 |
+
*message_ = std::move(message);
|
| 224 |
+
return Status::OK();
|
| 225 |
+
}
|
| 226 |
+
|
| 227 |
+
private:
|
| 228 |
+
std::unique_ptr<Message>* message_;
|
| 229 |
+
|
| 230 |
+
ARROW_DISALLOW_COPY_AND_ASSIGN(AssignMessageDecoderListener);
|
| 231 |
+
};
|
| 232 |
+
|
| 233 |
+
/// \class MessageDecoder
|
| 234 |
+
/// \brief Push style message decoder that receives data from user.
|
| 235 |
+
///
|
| 236 |
+
/// This API is EXPERIMENTAL.
|
| 237 |
+
///
|
| 238 |
+
/// \since 0.17.0
|
| 239 |
+
class ARROW_EXPORT MessageDecoder {
|
| 240 |
+
public:
|
| 241 |
+
/// \brief State for reading a message
|
| 242 |
+
enum State {
|
| 243 |
+
/// The initial state. It requires one of the followings as the next data:
|
| 244 |
+
///
|
| 245 |
+
/// * int32_t continuation token
|
| 246 |
+
/// * int32_t end-of-stream mark (== 0)
|
| 247 |
+
/// * int32_t metadata length (backward compatibility for
|
| 248 |
+
/// reading old IPC messages produced prior to version 0.15.0
|
| 249 |
+
INITIAL,
|
| 250 |
+
|
| 251 |
+
/// It requires int32_t metadata length.
|
| 252 |
+
METADATA_LENGTH,
|
| 253 |
+
|
| 254 |
+
/// It requires metadata.
|
| 255 |
+
METADATA,
|
| 256 |
+
|
| 257 |
+
/// It requires message body.
|
| 258 |
+
BODY,
|
| 259 |
+
|
| 260 |
+
/// The end-of-stream state. No more data is processed.
|
| 261 |
+
EOS,
|
| 262 |
+
};
|
| 263 |
+
|
| 264 |
+
/// \brief Construct a message decoder.
|
| 265 |
+
///
|
| 266 |
+
/// \param[in] listener a MessageDecoderListener that responds events from
|
| 267 |
+
/// the decoder
|
| 268 |
+
/// \param[in] pool an optional MemoryPool to copy metadata on the
|
| 269 |
+
/// \param[in] skip_body if true the body will be skipped even if the message has a body
|
| 270 |
+
/// CPU, if required
|
| 271 |
+
explicit MessageDecoder(std::shared_ptr<MessageDecoderListener> listener,
|
| 272 |
+
MemoryPool* pool = default_memory_pool(),
|
| 273 |
+
bool skip_body = false);
|
| 274 |
+
|
| 275 |
+
/// \brief Construct a message decoder with the specified state.
|
| 276 |
+
///
|
| 277 |
+
/// This is a construct for advanced users that know how to decode
|
| 278 |
+
/// Message.
|
| 279 |
+
///
|
| 280 |
+
/// \param[in] listener a MessageDecoderListener that responds events from
|
| 281 |
+
/// the decoder
|
| 282 |
+
/// \param[in] initial_state an initial state of the decode
|
| 283 |
+
/// \param[in] initial_next_required_size the number of bytes needed
|
| 284 |
+
/// to run the next action
|
| 285 |
+
/// \param[in] pool an optional MemoryPool to copy metadata on the
|
| 286 |
+
/// CPU, if required
|
| 287 |
+
/// \param[in] skip_body if true the body will be skipped even if the message has a body
|
| 288 |
+
MessageDecoder(std::shared_ptr<MessageDecoderListener> listener, State initial_state,
|
| 289 |
+
int64_t initial_next_required_size,
|
| 290 |
+
MemoryPool* pool = default_memory_pool(), bool skip_body = false);
|
| 291 |
+
|
| 292 |
+
virtual ~MessageDecoder();
|
| 293 |
+
|
| 294 |
+
/// \brief Feed data to the decoder as a raw data.
|
| 295 |
+
///
|
| 296 |
+
/// If the decoder can decode one or more messages by the data, the
|
| 297 |
+
/// decoder calls listener->OnMessageDecoded() with a decoded
|
| 298 |
+
/// message multiple times.
|
| 299 |
+
///
|
| 300 |
+
/// If the state of the decoder is changed, corresponding callbacks
|
| 301 |
+
/// on listener is called:
|
| 302 |
+
///
|
| 303 |
+
/// * MessageDecoder::State::INITIAL: listener->OnInitial()
|
| 304 |
+
/// * MessageDecoder::State::METADATA_LENGTH: listener->OnMetadataLength()
|
| 305 |
+
/// * MessageDecoder::State::METADATA: listener->OnMetadata()
|
| 306 |
+
/// * MessageDecoder::State::BODY: listener->OnBody()
|
| 307 |
+
/// * MessageDecoder::State::EOS: listener->OnEOS()
|
| 308 |
+
///
|
| 309 |
+
/// \param[in] data a raw data to be processed. This data isn't
|
| 310 |
+
/// copied. The passed memory must be kept alive through message
|
| 311 |
+
/// processing.
|
| 312 |
+
/// \param[in] size raw data size.
|
| 313 |
+
/// \return Status
|
| 314 |
+
Status Consume(const uint8_t* data, int64_t size);
|
| 315 |
+
|
| 316 |
+
/// \brief Feed data to the decoder as a Buffer.
|
| 317 |
+
///
|
| 318 |
+
/// If the decoder can decode one or more messages by the Buffer,
|
| 319 |
+
/// the decoder calls listener->OnMessageDecoded() with a decoded
|
| 320 |
+
/// message multiple times.
|
| 321 |
+
///
|
| 322 |
+
/// \param[in] buffer a Buffer to be processed.
|
| 323 |
+
/// \return Status
|
| 324 |
+
Status Consume(std::shared_ptr<Buffer> buffer);
|
| 325 |
+
|
| 326 |
+
/// \brief Return the number of bytes needed to advance the state of
|
| 327 |
+
/// the decoder.
|
| 328 |
+
///
|
| 329 |
+
/// This method is provided for users who want to optimize performance.
|
| 330 |
+
/// Normal users don't need to use this method.
|
| 331 |
+
///
|
| 332 |
+
/// Here is an example usage for normal users:
|
| 333 |
+
///
|
| 334 |
+
/// ~~~{.cpp}
|
| 335 |
+
/// decoder.Consume(buffer1);
|
| 336 |
+
/// decoder.Consume(buffer2);
|
| 337 |
+
/// decoder.Consume(buffer3);
|
| 338 |
+
/// ~~~
|
| 339 |
+
///
|
| 340 |
+
/// Decoder has internal buffer. If consumed data isn't enough to
|
| 341 |
+
/// advance the state of the decoder, consumed data is buffered to
|
| 342 |
+
/// the internal buffer. It causes performance overhead.
|
| 343 |
+
///
|
| 344 |
+
/// If you pass next_required_size() size data to each Consume()
|
| 345 |
+
/// call, the decoder doesn't use its internal buffer. It improves
|
| 346 |
+
/// performance.
|
| 347 |
+
///
|
| 348 |
+
/// Here is an example usage to avoid using internal buffer:
|
| 349 |
+
///
|
| 350 |
+
/// ~~~{.cpp}
|
| 351 |
+
/// buffer1 = get_data(decoder.next_required_size());
|
| 352 |
+
/// decoder.Consume(buffer1);
|
| 353 |
+
/// buffer2 = get_data(decoder.next_required_size());
|
| 354 |
+
/// decoder.Consume(buffer2);
|
| 355 |
+
/// ~~~
|
| 356 |
+
///
|
| 357 |
+
/// Users can use this method to avoid creating small
|
| 358 |
+
/// chunks. Message body must be contiguous data. If users pass
|
| 359 |
+
/// small chunks to the decoder, the decoder needs concatenate small
|
| 360 |
+
/// chunks internally. It causes performance overhead.
|
| 361 |
+
///
|
| 362 |
+
/// Here is an example usage to reduce small chunks:
|
| 363 |
+
///
|
| 364 |
+
/// ~~~{.cpp}
|
| 365 |
+
/// buffer = AllocateResizableBuffer();
|
| 366 |
+
/// while ((small_chunk = get_data(&small_chunk_size))) {
|
| 367 |
+
/// auto current_buffer_size = buffer->size();
|
| 368 |
+
/// buffer->Resize(current_buffer_size + small_chunk_size);
|
| 369 |
+
/// memcpy(buffer->mutable_data() + current_buffer_size,
|
| 370 |
+
/// small_chunk,
|
| 371 |
+
/// small_chunk_size);
|
| 372 |
+
/// if (buffer->size() < decoder.next_required_size()) {
|
| 373 |
+
/// continue;
|
| 374 |
+
/// }
|
| 375 |
+
/// std::shared_ptr<arrow::Buffer> chunk(buffer.release());
|
| 376 |
+
/// decoder.Consume(chunk);
|
| 377 |
+
/// buffer = AllocateResizableBuffer();
|
| 378 |
+
/// }
|
| 379 |
+
/// if (buffer->size() > 0) {
|
| 380 |
+
/// std::shared_ptr<arrow::Buffer> chunk(buffer.release());
|
| 381 |
+
/// decoder.Consume(chunk);
|
| 382 |
+
/// }
|
| 383 |
+
/// ~~~
|
| 384 |
+
///
|
| 385 |
+
/// \return the number of bytes needed to advance the state of the
|
| 386 |
+
/// decoder
|
| 387 |
+
int64_t next_required_size() const;
|
| 388 |
+
|
| 389 |
+
/// \brief Return the current state of the decoder.
|
| 390 |
+
///
|
| 391 |
+
/// This method is provided for users who want to optimize performance.
|
| 392 |
+
/// Normal users don't need to use this method.
|
| 393 |
+
///
|
| 394 |
+
/// Decoder doesn't need Buffer to process data on the
|
| 395 |
+
/// MessageDecoder::State::INITIAL state and the
|
| 396 |
+
/// MessageDecoder::State::METADATA_LENGTH. Creating Buffer has
|
| 397 |
+
/// performance overhead. Advanced users can avoid creating Buffer
|
| 398 |
+
/// by checking the current state of the decoder:
|
| 399 |
+
///
|
| 400 |
+
/// ~~~{.cpp}
|
| 401 |
+
/// switch (decoder.state()) {
|
| 402 |
+
/// MessageDecoder::State::INITIAL:
|
| 403 |
+
/// MessageDecoder::State::METADATA_LENGTH:
|
| 404 |
+
/// {
|
| 405 |
+
/// uint8_t data[sizeof(int32_t)];
|
| 406 |
+
/// auto data_size = input->Read(decoder.next_required_size(), data);
|
| 407 |
+
/// decoder.Consume(data, data_size);
|
| 408 |
+
/// }
|
| 409 |
+
/// break;
|
| 410 |
+
/// default:
|
| 411 |
+
/// {
|
| 412 |
+
/// auto buffer = input->Read(decoder.next_required_size());
|
| 413 |
+
/// decoder.Consume(buffer);
|
| 414 |
+
/// }
|
| 415 |
+
/// break;
|
| 416 |
+
/// }
|
| 417 |
+
/// ~~~
|
| 418 |
+
///
|
| 419 |
+
/// \return the current state
|
| 420 |
+
State state() const;
|
| 421 |
+
|
| 422 |
+
private:
|
| 423 |
+
class MessageDecoderImpl;
|
| 424 |
+
std::unique_ptr<MessageDecoderImpl> impl_;
|
| 425 |
+
|
| 426 |
+
ARROW_DISALLOW_COPY_AND_ASSIGN(MessageDecoder);
|
| 427 |
+
};
|
| 428 |
+
|
| 429 |
+
/// \brief Abstract interface for a sequence of messages
|
| 430 |
+
/// \since 0.5.0
|
| 431 |
+
class ARROW_EXPORT MessageReader {
|
| 432 |
+
public:
|
| 433 |
+
virtual ~MessageReader() = default;
|
| 434 |
+
|
| 435 |
+
/// \brief Create MessageReader that reads from InputStream
|
| 436 |
+
static std::unique_ptr<MessageReader> Open(io::InputStream* stream);
|
| 437 |
+
|
| 438 |
+
/// \brief Create MessageReader that reads from owned InputStream
|
| 439 |
+
static std::unique_ptr<MessageReader> Open(
|
| 440 |
+
const std::shared_ptr<io::InputStream>& owned_stream);
|
| 441 |
+
|
| 442 |
+
/// \brief Read next Message from the interface
|
| 443 |
+
///
|
| 444 |
+
/// \return an arrow::ipc::Message instance
|
| 445 |
+
virtual Result<std::unique_ptr<Message>> ReadNextMessage() = 0;
|
| 446 |
+
};
|
| 447 |
+
|
| 448 |
+
// the first parameter of the function should be a pointer to metadata (aka.
|
| 449 |
+
// org::apache::arrow::flatbuf::RecordBatch*)
|
| 450 |
+
using FieldsLoaderFunction = std::function<Status(const void*, io::RandomAccessFile*)>;
|
| 451 |
+
|
| 452 |
+
/// \brief Read encapsulated IPC message from position in file
|
| 453 |
+
///
|
| 454 |
+
/// Read a length-prefixed message flatbuffer starting at the indicated file
|
| 455 |
+
/// offset. If the message has a body with non-zero length, it will also be
|
| 456 |
+
/// read
|
| 457 |
+
///
|
| 458 |
+
/// The metadata_length includes at least the length prefix and the flatbuffer
|
| 459 |
+
///
|
| 460 |
+
/// \param[in] offset the position in the file where the message starts. The
|
| 461 |
+
/// first 4 bytes after the offset are the message length
|
| 462 |
+
/// \param[in] metadata_length the total number of bytes to read from file
|
| 463 |
+
/// \param[in] file the seekable file interface to read from
|
| 464 |
+
/// \param[in] fields_loader the function for loading subset of fields from the given file
|
| 465 |
+
/// \return the message read
|
| 466 |
+
|
| 467 |
+
ARROW_EXPORT
|
| 468 |
+
Result<std::unique_ptr<Message>> ReadMessage(
|
| 469 |
+
const int64_t offset, const int32_t metadata_length, io::RandomAccessFile* file,
|
| 470 |
+
const FieldsLoaderFunction& fields_loader = {});
|
| 471 |
+
|
| 472 |
+
/// \brief Read encapsulated IPC message from position in file
|
| 473 |
+
///
|
| 474 |
+
/// Read a length-prefixed message flatbuffer starting at the indicated file
|
| 475 |
+
/// offset.
|
| 476 |
+
///
|
| 477 |
+
/// The metadata_length includes at least the length prefix and the flatbuffer
|
| 478 |
+
///
|
| 479 |
+
/// \param[in] offset the position in the file where the message starts. The
|
| 480 |
+
/// first 4 bytes after the offset are the message length
|
| 481 |
+
/// \param[in] metadata_length the total number of bytes to read from file
|
| 482 |
+
/// \param[in] body_length the number of bytes for the message body
|
| 483 |
+
/// \param[in] file the seekable file interface to read from
|
| 484 |
+
/// \return the message read
|
| 485 |
+
|
| 486 |
+
ARROW_EXPORT
|
| 487 |
+
Result<std::unique_ptr<Message>> ReadMessage(const int64_t offset,
|
| 488 |
+
const int32_t metadata_length,
|
| 489 |
+
const int64_t body_length,
|
| 490 |
+
io::RandomAccessFile* file);
|
| 491 |
+
|
| 492 |
+
/// \brief Read encapsulated IPC message from cached buffers
|
| 493 |
+
///
|
| 494 |
+
/// The buffers should contain an entire message. Partial reads are not handled.
|
| 495 |
+
///
|
| 496 |
+
/// This method can be used to read just the metadata by passing in a nullptr for the
|
| 497 |
+
/// body. The body will then be skipped and the body size will not be validated.
|
| 498 |
+
///
|
| 499 |
+
/// If the body buffer is provided then it must be the complete body buffer
|
| 500 |
+
///
|
| 501 |
+
/// This is similar to Message::Open but performs slightly more validation (e.g. checks
|
| 502 |
+
/// to see that the metadata length is correct and that the body is the size the metadata
|
| 503 |
+
/// expected)
|
| 504 |
+
///
|
| 505 |
+
/// \param metadata The bytes for the metadata
|
| 506 |
+
/// \param body The bytes for the body
|
| 507 |
+
/// \return The message represented by the buffers
|
| 508 |
+
ARROW_EXPORT Result<std::unique_ptr<Message>> ReadMessage(
|
| 509 |
+
std::shared_ptr<Buffer> metadata, std::shared_ptr<Buffer> body);
|
| 510 |
+
|
| 511 |
+
ARROW_EXPORT
|
| 512 |
+
Future<std::shared_ptr<Message>> ReadMessageAsync(
|
| 513 |
+
const int64_t offset, const int32_t metadata_length, const int64_t body_length,
|
| 514 |
+
io::RandomAccessFile* file, const io::IOContext& context = io::default_io_context());
|
| 515 |
+
|
| 516 |
+
/// \brief Advance stream to an 8-byte offset if its position is not a multiple
|
| 517 |
+
/// of 8 already
|
| 518 |
+
/// \param[in] stream an input stream
|
| 519 |
+
/// \param[in] alignment the byte multiple for the metadata prefix, usually 8
|
| 520 |
+
/// or 64, to ensure the body starts on a multiple of that alignment
|
| 521 |
+
/// \return Status
|
| 522 |
+
ARROW_EXPORT
|
| 523 |
+
Status AlignStream(io::InputStream* stream, int32_t alignment = 8);
|
| 524 |
+
|
| 525 |
+
/// \brief Advance stream to an 8-byte offset if its position is not a multiple
|
| 526 |
+
/// of 8 already
|
| 527 |
+
/// \param[in] stream an output stream
|
| 528 |
+
/// \param[in] alignment the byte multiple for the metadata prefix, usually 8
|
| 529 |
+
/// or 64, to ensure the body starts on a multiple of that alignment
|
| 530 |
+
/// \return Status
|
| 531 |
+
ARROW_EXPORT
|
| 532 |
+
Status AlignStream(io::OutputStream* stream, int32_t alignment = 8);
|
| 533 |
+
|
| 534 |
+
/// \brief Return error Status if file position is not a multiple of the
|
| 535 |
+
/// indicated alignment
|
| 536 |
+
ARROW_EXPORT
|
| 537 |
+
Status CheckAligned(io::FileInterface* stream, int32_t alignment = 8);
|
| 538 |
+
|
| 539 |
+
/// \brief Read encapsulated IPC message (metadata and body) from InputStream
|
| 540 |
+
///
|
| 541 |
+
/// Returns null if there are not enough bytes available or the
|
| 542 |
+
/// message length is 0 (e.g. EOS in a stream)
|
| 543 |
+
///
|
| 544 |
+
/// \param[in] stream an input stream
|
| 545 |
+
/// \param[in] pool an optional MemoryPool to copy metadata on the CPU, if required
|
| 546 |
+
/// \return Message
|
| 547 |
+
ARROW_EXPORT
|
| 548 |
+
Result<std::unique_ptr<Message>> ReadMessage(io::InputStream* stream,
|
| 549 |
+
MemoryPool* pool = default_memory_pool());
|
| 550 |
+
|
| 551 |
+
/// \brief Feed data from InputStream to MessageDecoder to decode an
|
| 552 |
+
/// encapsulated IPC message (metadata and body)
|
| 553 |
+
///
|
| 554 |
+
/// This API is EXPERIMENTAL.
|
| 555 |
+
///
|
| 556 |
+
/// \param[in] decoder a decoder
|
| 557 |
+
/// \param[in] stream an input stream
|
| 558 |
+
/// \return Status
|
| 559 |
+
///
|
| 560 |
+
/// \since 0.17.0
|
| 561 |
+
ARROW_EXPORT
|
| 562 |
+
Status DecodeMessage(MessageDecoder* decoder, io::InputStream* stream);
|
| 563 |
+
|
| 564 |
+
/// Write encapsulated IPC message Does not make assumptions about
|
| 565 |
+
/// whether the stream is aligned already. Can write legacy (pre
|
| 566 |
+
/// version 0.15.0) IPC message if option set
|
| 567 |
+
///
|
| 568 |
+
/// continuation: 0xFFFFFFFF
|
| 569 |
+
/// message_size: int32
|
| 570 |
+
/// message: const void*
|
| 571 |
+
/// padding
|
| 572 |
+
///
|
| 573 |
+
///
|
| 574 |
+
/// \param[in] message a buffer containing the metadata to write
|
| 575 |
+
/// \param[in] options IPC writing options, including alignment and
|
| 576 |
+
/// legacy message support
|
| 577 |
+
/// \param[in,out] file the OutputStream to write to
|
| 578 |
+
/// \param[out] message_length the total size of the payload written including
|
| 579 |
+
/// padding
|
| 580 |
+
/// \return Status
|
| 581 |
+
Status WriteMessage(const Buffer& message, const IpcWriteOptions& options,
|
| 582 |
+
io::OutputStream* file, int32_t* message_length);
|
| 583 |
+
|
| 584 |
+
} // namespace ipc
|
| 585 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/options.h
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <cstdint>
|
| 21 |
+
#include <optional>
|
| 22 |
+
#include <vector>
|
| 23 |
+
|
| 24 |
+
#include "arrow/io/caching.h"
|
| 25 |
+
#include "arrow/ipc/type_fwd.h"
|
| 26 |
+
#include "arrow/status.h"
|
| 27 |
+
#include "arrow/type_fwd.h"
|
| 28 |
+
#include "arrow/util/align_util.h"
|
| 29 |
+
#include "arrow/util/compression.h"
|
| 30 |
+
#include "arrow/util/visibility.h"
|
| 31 |
+
|
| 32 |
+
namespace arrow {
|
| 33 |
+
|
| 34 |
+
class MemoryPool;
|
| 35 |
+
|
| 36 |
+
namespace ipc {
|
| 37 |
+
|
| 38 |
+
// ARROW-109: We set this number arbitrarily to help catch user mistakes. For
|
| 39 |
+
// deeply nested schemas, it is expected the user will indicate explicitly the
|
| 40 |
+
// maximum allowed recursion depth
|
| 41 |
+
constexpr int kMaxNestingDepth = 64;
|
| 42 |
+
|
| 43 |
+
/// \brief Options for writing Arrow IPC messages
|
| 44 |
+
struct ARROW_EXPORT IpcWriteOptions {
|
| 45 |
+
/// \brief If true, allow field lengths that don't fit in a signed 32-bit int.
|
| 46 |
+
///
|
| 47 |
+
/// Some implementations may not be able to parse streams created with this option.
|
| 48 |
+
bool allow_64bit = false;
|
| 49 |
+
|
| 50 |
+
/// \brief The maximum permitted schema nesting depth.
|
| 51 |
+
int max_recursion_depth = kMaxNestingDepth;
|
| 52 |
+
|
| 53 |
+
/// \brief Write padding after memory buffers up to this multiple of bytes.
|
| 54 |
+
int32_t alignment = 8;
|
| 55 |
+
|
| 56 |
+
/// \brief Write the pre-0.15.0 IPC message format
|
| 57 |
+
///
|
| 58 |
+
/// This legacy format consists of a 4-byte prefix instead of 8-byte.
|
| 59 |
+
bool write_legacy_ipc_format = false;
|
| 60 |
+
|
| 61 |
+
/// \brief The memory pool to use for allocations made during IPC writing
|
| 62 |
+
///
|
| 63 |
+
/// While Arrow IPC is predominantly zero-copy, it may have to allocate
|
| 64 |
+
/// memory in some cases (for example if compression is enabled).
|
| 65 |
+
MemoryPool* memory_pool = default_memory_pool();
|
| 66 |
+
|
| 67 |
+
/// \brief Compression codec to use for record batch body buffers
|
| 68 |
+
///
|
| 69 |
+
/// May only be UNCOMPRESSED, LZ4_FRAME and ZSTD.
|
| 70 |
+
std::shared_ptr<util::Codec> codec;
|
| 71 |
+
|
| 72 |
+
/// \brief Minimum space savings percentage required for compression to be applied
|
| 73 |
+
///
|
| 74 |
+
/// Space savings is calculated as (1.0 - compressed_size / uncompressed_size).
|
| 75 |
+
///
|
| 76 |
+
/// For example, if min_space_savings = 0.1, a 100-byte body buffer won't undergo
|
| 77 |
+
/// compression if its expected compressed size exceeds 90 bytes. If this option is
|
| 78 |
+
/// unset, compression will be used indiscriminately. If no codec was supplied, this
|
| 79 |
+
/// option is ignored.
|
| 80 |
+
///
|
| 81 |
+
/// Values outside of the range [0,1] are handled as errors.
|
| 82 |
+
///
|
| 83 |
+
/// Note that enabling this option may result in unreadable data for Arrow C++ versions
|
| 84 |
+
/// prior to 12.0.0.
|
| 85 |
+
std::optional<double> min_space_savings;
|
| 86 |
+
|
| 87 |
+
/// \brief Use global CPU thread pool to parallelize any computational tasks
|
| 88 |
+
/// like compression
|
| 89 |
+
bool use_threads = true;
|
| 90 |
+
|
| 91 |
+
/// \brief Whether to emit dictionary deltas
|
| 92 |
+
///
|
| 93 |
+
/// If false, a changed dictionary for a given field will emit a full
|
| 94 |
+
/// dictionary replacement.
|
| 95 |
+
/// If true, a changed dictionary will be compared against the previous
|
| 96 |
+
/// version. If possible, a dictionary delta will be emitted, otherwise
|
| 97 |
+
/// a full dictionary replacement.
|
| 98 |
+
///
|
| 99 |
+
/// Default is false to maximize stream compatibility.
|
| 100 |
+
///
|
| 101 |
+
/// Also, note that if a changed dictionary is a nested dictionary,
|
| 102 |
+
/// then a delta is never emitted, for compatibility with the read path.
|
| 103 |
+
bool emit_dictionary_deltas = false;
|
| 104 |
+
|
| 105 |
+
/// \brief Whether to unify dictionaries for the IPC file format
|
| 106 |
+
///
|
| 107 |
+
/// The IPC file format doesn't support dictionary replacements.
|
| 108 |
+
/// Therefore, chunks of a column with a dictionary type must have the same
|
| 109 |
+
/// dictionary in each record batch (or an extended dictionary + delta).
|
| 110 |
+
///
|
| 111 |
+
/// If this option is true, RecordBatchWriter::WriteTable will attempt
|
| 112 |
+
/// to unify dictionaries across each table column. If this option is
|
| 113 |
+
/// false, incompatible dictionaries across a table column will simply
|
| 114 |
+
/// raise an error.
|
| 115 |
+
///
|
| 116 |
+
/// Note that enabling this option has a runtime cost. Also, not all types
|
| 117 |
+
/// currently support dictionary unification.
|
| 118 |
+
///
|
| 119 |
+
/// This option is ignored for IPC streams, which support dictionary replacement
|
| 120 |
+
/// and deltas.
|
| 121 |
+
bool unify_dictionaries = false;
|
| 122 |
+
|
| 123 |
+
/// \brief Format version to use for IPC messages and their metadata.
|
| 124 |
+
///
|
| 125 |
+
/// Presently using V5 version (readable by 1.0.0 and later).
|
| 126 |
+
/// V4 is also available (readable by 0.8.0 and later).
|
| 127 |
+
MetadataVersion metadata_version = MetadataVersion::V5;
|
| 128 |
+
|
| 129 |
+
static IpcWriteOptions Defaults();
|
| 130 |
+
};
|
| 131 |
+
|
| 132 |
+
/// \brief Alignment of data in memory
|
| 133 |
+
/// Alignment values larger than 0 are taken directly as byte alignment value
|
| 134 |
+
/// See util::EnsureAlignment(..., int64_t alignment, ...)
|
| 135 |
+
enum class Alignment : int64_t {
|
| 136 |
+
/// \brief data is aligned depending on the actual data type
|
| 137 |
+
kDataTypeSpecificAlignment = util::kValueAlignment,
|
| 138 |
+
/// \brief no particular alignment enforced
|
| 139 |
+
kAnyAlignment = 0,
|
| 140 |
+
/// \brief data is aligned to 64-byte boundary
|
| 141 |
+
k64ByteAlignment = 64
|
| 142 |
+
};
|
| 143 |
+
|
| 144 |
+
/// \brief Options for reading Arrow IPC messages
|
| 145 |
+
struct ARROW_EXPORT IpcReadOptions {
|
| 146 |
+
/// \brief The maximum permitted schema nesting depth.
|
| 147 |
+
int max_recursion_depth = kMaxNestingDepth;
|
| 148 |
+
|
| 149 |
+
/// \brief The memory pool to use for allocations made during IPC reading
|
| 150 |
+
///
|
| 151 |
+
/// While Arrow IPC is predominantly zero-copy, it may have to allocate
|
| 152 |
+
/// memory in some cases (for example if compression is enabled).
|
| 153 |
+
MemoryPool* memory_pool = default_memory_pool();
|
| 154 |
+
|
| 155 |
+
/// \brief Top-level schema fields to include when deserializing RecordBatch.
|
| 156 |
+
///
|
| 157 |
+
/// If empty (the default), return all deserialized fields.
|
| 158 |
+
/// If non-empty, the values are the indices of fields in the top-level schema.
|
| 159 |
+
std::vector<int> included_fields;
|
| 160 |
+
|
| 161 |
+
/// \brief Use global CPU thread pool to parallelize any computational tasks
|
| 162 |
+
/// like decompression
|
| 163 |
+
bool use_threads = true;
|
| 164 |
+
|
| 165 |
+
/// \brief Whether to convert incoming data to platform-native endianness
|
| 166 |
+
///
|
| 167 |
+
/// If the endianness of the received schema is not equal to platform-native
|
| 168 |
+
/// endianness, then all buffers with endian-sensitive data will be byte-swapped.
|
| 169 |
+
/// This includes the value buffers of numeric types, temporal types, decimal
|
| 170 |
+
/// types, as well as the offset buffers of variable-sized binary and list-like
|
| 171 |
+
/// types.
|
| 172 |
+
///
|
| 173 |
+
/// Endianness conversion is achieved by the RecordBatchFileReader,
|
| 174 |
+
/// RecordBatchStreamReader and StreamDecoder classes.
|
| 175 |
+
bool ensure_native_endian = true;
|
| 176 |
+
|
| 177 |
+
/// \brief How to align data if mis-aligned
|
| 178 |
+
///
|
| 179 |
+
/// Data is copied to aligned memory locations allocated via the
|
| 180 |
+
/// MemoryPool configured as \ref arrow::ipc::IpcReadOptions::memory_pool.
|
| 181 |
+
/// Some use cases might require data to have a specific alignment, for example,
|
| 182 |
+
/// for the data buffer of an Int32 array to be aligned on a 4-byte boundary.
|
| 183 |
+
///
|
| 184 |
+
/// Default (kAnyAlignment) keeps the alignment as is, so no copy of data occurs.
|
| 185 |
+
Alignment ensure_alignment = Alignment::kAnyAlignment;
|
| 186 |
+
|
| 187 |
+
/// \brief Options to control caching behavior when pre-buffering is requested
|
| 188 |
+
///
|
| 189 |
+
/// The lazy property will always be reset to true to deliver the expected behavior
|
| 190 |
+
io::CacheOptions pre_buffer_cache_options = io::CacheOptions::LazyDefaults();
|
| 191 |
+
|
| 192 |
+
static IpcReadOptions Defaults();
|
| 193 |
+
};
|
| 194 |
+
|
| 195 |
+
namespace internal {
|
| 196 |
+
|
| 197 |
+
Status CheckCompressionSupported(Compression::type codec);
|
| 198 |
+
|
| 199 |
+
} // namespace internal
|
| 200 |
+
} // namespace ipc
|
| 201 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/reader.h
ADDED
|
@@ -0,0 +1,638 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
// Read Arrow files and streams
|
| 19 |
+
|
| 20 |
+
#pragma once
|
| 21 |
+
|
| 22 |
+
#include <cstddef>
|
| 23 |
+
#include <cstdint>
|
| 24 |
+
#include <memory>
|
| 25 |
+
#include <utility>
|
| 26 |
+
#include <vector>
|
| 27 |
+
|
| 28 |
+
#include "arrow/io/caching.h"
|
| 29 |
+
#include "arrow/io/type_fwd.h"
|
| 30 |
+
#include "arrow/ipc/message.h"
|
| 31 |
+
#include "arrow/ipc/options.h"
|
| 32 |
+
#include "arrow/record_batch.h"
|
| 33 |
+
#include "arrow/result.h"
|
| 34 |
+
#include "arrow/type_fwd.h"
|
| 35 |
+
#include "arrow/util/async_generator.h"
|
| 36 |
+
#include "arrow/util/macros.h"
|
| 37 |
+
#include "arrow/util/visibility.h"
|
| 38 |
+
|
| 39 |
+
namespace arrow {
|
| 40 |
+
namespace ipc {
|
| 41 |
+
|
| 42 |
+
class DictionaryMemo;
|
| 43 |
+
struct IpcPayload;
|
| 44 |
+
|
| 45 |
+
using RecordBatchReader = ::arrow::RecordBatchReader;
|
| 46 |
+
|
| 47 |
+
struct ReadStats {
|
| 48 |
+
/// Number of IPC messages read.
|
| 49 |
+
int64_t num_messages = 0;
|
| 50 |
+
/// Number of record batches read.
|
| 51 |
+
int64_t num_record_batches = 0;
|
| 52 |
+
/// Number of dictionary batches read.
|
| 53 |
+
///
|
| 54 |
+
/// Note: num_dictionary_batches >= num_dictionary_deltas + num_replaced_dictionaries
|
| 55 |
+
int64_t num_dictionary_batches = 0;
|
| 56 |
+
|
| 57 |
+
/// Number of dictionary deltas read.
|
| 58 |
+
int64_t num_dictionary_deltas = 0;
|
| 59 |
+
/// Number of replaced dictionaries (i.e. where a dictionary batch replaces
|
| 60 |
+
/// an existing dictionary with an unrelated new dictionary).
|
| 61 |
+
int64_t num_replaced_dictionaries = 0;
|
| 62 |
+
};
|
| 63 |
+
|
| 64 |
+
/// \brief Synchronous batch stream reader that reads from io::InputStream
|
| 65 |
+
///
|
| 66 |
+
/// This class reads the schema (plus any dictionaries) as the first messages
|
| 67 |
+
/// in the stream, followed by record batches. For more granular zero-copy
|
| 68 |
+
/// reads see the ReadRecordBatch functions
|
| 69 |
+
class ARROW_EXPORT RecordBatchStreamReader : public RecordBatchReader {
|
| 70 |
+
public:
|
| 71 |
+
/// Create batch reader from generic MessageReader.
|
| 72 |
+
/// This will take ownership of the given MessageReader.
|
| 73 |
+
///
|
| 74 |
+
/// \param[in] message_reader a MessageReader implementation
|
| 75 |
+
/// \param[in] options any IPC reading options (optional)
|
| 76 |
+
/// \return the created batch reader
|
| 77 |
+
static Result<std::shared_ptr<RecordBatchStreamReader>> Open(
|
| 78 |
+
std::unique_ptr<MessageReader> message_reader,
|
| 79 |
+
const IpcReadOptions& options = IpcReadOptions::Defaults());
|
| 80 |
+
|
| 81 |
+
/// \brief Record batch stream reader from InputStream
|
| 82 |
+
///
|
| 83 |
+
/// \param[in] stream an input stream instance. Must stay alive throughout
|
| 84 |
+
/// lifetime of stream reader
|
| 85 |
+
/// \param[in] options any IPC reading options (optional)
|
| 86 |
+
/// \return the created batch reader
|
| 87 |
+
static Result<std::shared_ptr<RecordBatchStreamReader>> Open(
|
| 88 |
+
io::InputStream* stream,
|
| 89 |
+
const IpcReadOptions& options = IpcReadOptions::Defaults());
|
| 90 |
+
|
| 91 |
+
/// \brief Open stream and retain ownership of stream object
|
| 92 |
+
/// \param[in] stream the input stream
|
| 93 |
+
/// \param[in] options any IPC reading options (optional)
|
| 94 |
+
/// \return the created batch reader
|
| 95 |
+
static Result<std::shared_ptr<RecordBatchStreamReader>> Open(
|
| 96 |
+
const std::shared_ptr<io::InputStream>& stream,
|
| 97 |
+
const IpcReadOptions& options = IpcReadOptions::Defaults());
|
| 98 |
+
|
| 99 |
+
/// \brief Return current read statistics
|
| 100 |
+
virtual ReadStats stats() const = 0;
|
| 101 |
+
};
|
| 102 |
+
|
| 103 |
+
/// \brief Reads the record batch file format
|
| 104 |
+
class ARROW_EXPORT RecordBatchFileReader
|
| 105 |
+
: public std::enable_shared_from_this<RecordBatchFileReader> {
|
| 106 |
+
public:
|
| 107 |
+
virtual ~RecordBatchFileReader() = default;
|
| 108 |
+
|
| 109 |
+
/// \brief Open a RecordBatchFileReader
|
| 110 |
+
///
|
| 111 |
+
/// Open a file-like object that is assumed to be self-contained; i.e., the
|
| 112 |
+
/// end of the file interface is the end of the Arrow file. Note that there
|
| 113 |
+
/// can be any amount of data preceding the Arrow-formatted data, because we
|
| 114 |
+
/// need only locate the end of the Arrow file stream to discover the metadata
|
| 115 |
+
/// and then proceed to read the data into memory.
|
| 116 |
+
static Result<std::shared_ptr<RecordBatchFileReader>> Open(
|
| 117 |
+
io::RandomAccessFile* file,
|
| 118 |
+
const IpcReadOptions& options = IpcReadOptions::Defaults());
|
| 119 |
+
|
| 120 |
+
/// \brief Open a RecordBatchFileReader
|
| 121 |
+
/// If the file is embedded within some larger file or memory region, you can
|
| 122 |
+
/// pass the absolute memory offset to the end of the file (which contains the
|
| 123 |
+
/// metadata footer). The metadata must have been written with memory offsets
|
| 124 |
+
/// relative to the start of the containing file
|
| 125 |
+
///
|
| 126 |
+
/// \param[in] file the data source
|
| 127 |
+
/// \param[in] footer_offset the position of the end of the Arrow file
|
| 128 |
+
/// \param[in] options options for IPC reading
|
| 129 |
+
/// \return the returned reader
|
| 130 |
+
static Result<std::shared_ptr<RecordBatchFileReader>> Open(
|
| 131 |
+
io::RandomAccessFile* file, int64_t footer_offset,
|
| 132 |
+
const IpcReadOptions& options = IpcReadOptions::Defaults());
|
| 133 |
+
|
| 134 |
+
/// \brief Version of Open that retains ownership of file
|
| 135 |
+
///
|
| 136 |
+
/// \param[in] file the data source
|
| 137 |
+
/// \param[in] options options for IPC reading
|
| 138 |
+
/// \return the returned reader
|
| 139 |
+
static Result<std::shared_ptr<RecordBatchFileReader>> Open(
|
| 140 |
+
const std::shared_ptr<io::RandomAccessFile>& file,
|
| 141 |
+
const IpcReadOptions& options = IpcReadOptions::Defaults());
|
| 142 |
+
|
| 143 |
+
/// \brief Version of Open that retains ownership of file
|
| 144 |
+
///
|
| 145 |
+
/// \param[in] file the data source
|
| 146 |
+
/// \param[in] footer_offset the position of the end of the Arrow file
|
| 147 |
+
/// \param[in] options options for IPC reading
|
| 148 |
+
/// \return the returned reader
|
| 149 |
+
static Result<std::shared_ptr<RecordBatchFileReader>> Open(
|
| 150 |
+
const std::shared_ptr<io::RandomAccessFile>& file, int64_t footer_offset,
|
| 151 |
+
const IpcReadOptions& options = IpcReadOptions::Defaults());
|
| 152 |
+
|
| 153 |
+
/// \brief Open a file asynchronously (owns the file).
|
| 154 |
+
static Future<std::shared_ptr<RecordBatchFileReader>> OpenAsync(
|
| 155 |
+
const std::shared_ptr<io::RandomAccessFile>& file,
|
| 156 |
+
const IpcReadOptions& options = IpcReadOptions::Defaults());
|
| 157 |
+
|
| 158 |
+
/// \brief Open a file asynchronously (borrows the file).
|
| 159 |
+
static Future<std::shared_ptr<RecordBatchFileReader>> OpenAsync(
|
| 160 |
+
io::RandomAccessFile* file,
|
| 161 |
+
const IpcReadOptions& options = IpcReadOptions::Defaults());
|
| 162 |
+
|
| 163 |
+
/// \brief Open a file asynchronously (owns the file).
|
| 164 |
+
static Future<std::shared_ptr<RecordBatchFileReader>> OpenAsync(
|
| 165 |
+
const std::shared_ptr<io::RandomAccessFile>& file, int64_t footer_offset,
|
| 166 |
+
const IpcReadOptions& options = IpcReadOptions::Defaults());
|
| 167 |
+
|
| 168 |
+
/// \brief Open a file asynchronously (borrows the file).
|
| 169 |
+
static Future<std::shared_ptr<RecordBatchFileReader>> OpenAsync(
|
| 170 |
+
io::RandomAccessFile* file, int64_t footer_offset,
|
| 171 |
+
const IpcReadOptions& options = IpcReadOptions::Defaults());
|
| 172 |
+
|
| 173 |
+
/// \brief The schema read from the file
|
| 174 |
+
virtual std::shared_ptr<Schema> schema() const = 0;
|
| 175 |
+
|
| 176 |
+
/// \brief Returns the number of record batches in the file
|
| 177 |
+
virtual int num_record_batches() const = 0;
|
| 178 |
+
|
| 179 |
+
/// \brief Return the metadata version from the file metadata
|
| 180 |
+
virtual MetadataVersion version() const = 0;
|
| 181 |
+
|
| 182 |
+
/// \brief Return the contents of the custom_metadata field from the file's
|
| 183 |
+
/// Footer
|
| 184 |
+
virtual std::shared_ptr<const KeyValueMetadata> metadata() const = 0;
|
| 185 |
+
|
| 186 |
+
/// \brief Read a particular record batch from the file. Does not copy memory
|
| 187 |
+
/// if the input source supports zero-copy.
|
| 188 |
+
///
|
| 189 |
+
/// \param[in] i the index of the record batch to return
|
| 190 |
+
/// \return the read batch
|
| 191 |
+
virtual Result<std::shared_ptr<RecordBatch>> ReadRecordBatch(int i) = 0;
|
| 192 |
+
|
| 193 |
+
/// \brief Read a particular record batch along with its custom metadata from the file.
|
| 194 |
+
/// Does not copy memory if the input source supports zero-copy.
|
| 195 |
+
///
|
| 196 |
+
/// \param[in] i the index of the record batch to return
|
| 197 |
+
/// \return a struct containing the read batch and its custom metadata
|
| 198 |
+
virtual Result<RecordBatchWithMetadata> ReadRecordBatchWithCustomMetadata(int i) = 0;
|
| 199 |
+
|
| 200 |
+
/// \brief Return current read statistics
|
| 201 |
+
virtual ReadStats stats() const = 0;
|
| 202 |
+
|
| 203 |
+
/// \brief Computes the total number of rows in the file.
|
| 204 |
+
virtual Result<int64_t> CountRows() = 0;
|
| 205 |
+
|
| 206 |
+
/// \brief Begin loading metadata for the desired batches into memory.
|
| 207 |
+
///
|
| 208 |
+
/// This method will also begin loading all dictionaries messages into memory.
|
| 209 |
+
///
|
| 210 |
+
/// For a regular file this will immediately begin disk I/O in the background on a
|
| 211 |
+
/// thread on the IOContext's thread pool. If the file is memory mapped this will
|
| 212 |
+
/// ensure the memory needed for the metadata is paged from disk into memory
|
| 213 |
+
///
|
| 214 |
+
/// \param indices Indices of the batches to prefetch
|
| 215 |
+
/// If empty then all batches will be prefetched.
|
| 216 |
+
virtual Status PreBufferMetadata(const std::vector<int>& indices) = 0;
|
| 217 |
+
|
| 218 |
+
/// \brief Get a reentrant generator of record batches.
|
| 219 |
+
///
|
| 220 |
+
/// \param[in] coalesce If true, enable I/O coalescing.
|
| 221 |
+
/// \param[in] io_context The IOContext to use (controls which thread pool
|
| 222 |
+
/// is used for I/O).
|
| 223 |
+
/// \param[in] cache_options Options for coalescing (if enabled).
|
| 224 |
+
/// \param[in] executor Optionally, an executor to use for decoding record
|
| 225 |
+
/// batches. This is generally only a benefit for very wide and/or
|
| 226 |
+
/// compressed batches.
|
| 227 |
+
virtual Result<AsyncGenerator<std::shared_ptr<RecordBatch>>> GetRecordBatchGenerator(
|
| 228 |
+
const bool coalesce = false,
|
| 229 |
+
const io::IOContext& io_context = io::default_io_context(),
|
| 230 |
+
const io::CacheOptions cache_options = io::CacheOptions::LazyDefaults(),
|
| 231 |
+
arrow::internal::Executor* executor = NULLPTR) = 0;
|
| 232 |
+
|
| 233 |
+
/// \brief Collect all batches as a vector of record batches
|
| 234 |
+
Result<RecordBatchVector> ToRecordBatches();
|
| 235 |
+
|
| 236 |
+
/// \brief Collect all batches and concatenate as arrow::Table
|
| 237 |
+
Result<std::shared_ptr<Table>> ToTable();
|
| 238 |
+
};
|
| 239 |
+
|
| 240 |
+
/// \brief A general listener class to receive events.
|
| 241 |
+
///
|
| 242 |
+
/// You must implement callback methods for interested events.
|
| 243 |
+
///
|
| 244 |
+
/// This API is EXPERIMENTAL.
|
| 245 |
+
///
|
| 246 |
+
/// \since 0.17.0
|
| 247 |
+
class ARROW_EXPORT Listener {
|
| 248 |
+
public:
|
| 249 |
+
virtual ~Listener() = default;
|
| 250 |
+
|
| 251 |
+
/// \brief Called when end-of-stream is received.
|
| 252 |
+
///
|
| 253 |
+
/// The default implementation just returns arrow::Status::OK().
|
| 254 |
+
///
|
| 255 |
+
/// \return Status
|
| 256 |
+
///
|
| 257 |
+
/// \see StreamDecoder
|
| 258 |
+
virtual Status OnEOS();
|
| 259 |
+
|
| 260 |
+
/// \brief Called when a record batch is decoded and
|
| 261 |
+
/// OnRecordBatchWithMetadataDecoded() isn't overridden.
|
| 262 |
+
///
|
| 263 |
+
/// The default implementation just returns
|
| 264 |
+
/// arrow::Status::NotImplemented().
|
| 265 |
+
///
|
| 266 |
+
/// \param[in] record_batch a record batch decoded
|
| 267 |
+
/// \return Status
|
| 268 |
+
///
|
| 269 |
+
/// \see StreamDecoder
|
| 270 |
+
virtual Status OnRecordBatchDecoded(std::shared_ptr<RecordBatch> record_batch);
|
| 271 |
+
|
| 272 |
+
/// \brief Called when a record batch with custom metadata is decoded.
|
| 273 |
+
///
|
| 274 |
+
/// The default implementation just calls OnRecordBatchDecoded()
|
| 275 |
+
/// without custom metadata.
|
| 276 |
+
///
|
| 277 |
+
/// \param[in] record_batch_with_metadata a record batch with custom
|
| 278 |
+
/// metadata decoded
|
| 279 |
+
/// \return Status
|
| 280 |
+
///
|
| 281 |
+
/// \see StreamDecoder
|
| 282 |
+
///
|
| 283 |
+
/// \since 13.0.0
|
| 284 |
+
virtual Status OnRecordBatchWithMetadataDecoded(
|
| 285 |
+
RecordBatchWithMetadata record_batch_with_metadata);
|
| 286 |
+
|
| 287 |
+
/// \brief Called when a schema is decoded.
|
| 288 |
+
///
|
| 289 |
+
/// The default implementation just returns arrow::Status::OK().
|
| 290 |
+
///
|
| 291 |
+
/// \param[in] schema a schema decoded
|
| 292 |
+
/// \return Status
|
| 293 |
+
///
|
| 294 |
+
/// \see StreamDecoder
|
| 295 |
+
virtual Status OnSchemaDecoded(std::shared_ptr<Schema> schema);
|
| 296 |
+
|
| 297 |
+
/// \brief Called when a schema is decoded.
|
| 298 |
+
///
|
| 299 |
+
/// The default implementation just calls OnSchemaDecoded(schema)
|
| 300 |
+
/// (without filtered_schema) to keep backward compatibility.
|
| 301 |
+
///
|
| 302 |
+
/// \param[in] schema a schema decoded
|
| 303 |
+
/// \param[in] filtered_schema a filtered schema that only has read fields
|
| 304 |
+
/// \return Status
|
| 305 |
+
///
|
| 306 |
+
/// \see StreamDecoder
|
| 307 |
+
///
|
| 308 |
+
/// \since 13.0.0
|
| 309 |
+
virtual Status OnSchemaDecoded(std::shared_ptr<Schema> schema,
|
| 310 |
+
std::shared_ptr<Schema> filtered_schema);
|
| 311 |
+
};
|
| 312 |
+
|
| 313 |
+
/// \brief Collect schema and record batches decoded by StreamDecoder.
|
| 314 |
+
///
|
| 315 |
+
/// This API is EXPERIMENTAL.
|
| 316 |
+
///
|
| 317 |
+
/// \since 0.17.0
|
| 318 |
+
class ARROW_EXPORT CollectListener : public Listener {
|
| 319 |
+
public:
|
| 320 |
+
CollectListener() : schema_(), filtered_schema_(), record_batches_(), metadatas_() {}
|
| 321 |
+
virtual ~CollectListener() = default;
|
| 322 |
+
|
| 323 |
+
Status OnSchemaDecoded(std::shared_ptr<Schema> schema,
|
| 324 |
+
std::shared_ptr<Schema> filtered_schema) override {
|
| 325 |
+
schema_ = std::move(schema);
|
| 326 |
+
filtered_schema_ = std::move(filtered_schema);
|
| 327 |
+
return Status::OK();
|
| 328 |
+
}
|
| 329 |
+
|
| 330 |
+
Status OnRecordBatchWithMetadataDecoded(
|
| 331 |
+
RecordBatchWithMetadata record_batch_with_metadata) override {
|
| 332 |
+
record_batches_.push_back(std::move(record_batch_with_metadata.batch));
|
| 333 |
+
metadatas_.push_back(std::move(record_batch_with_metadata.custom_metadata));
|
| 334 |
+
return Status::OK();
|
| 335 |
+
}
|
| 336 |
+
|
| 337 |
+
/// \return the decoded schema
|
| 338 |
+
std::shared_ptr<Schema> schema() const { return schema_; }
|
| 339 |
+
|
| 340 |
+
/// \return the filtered schema
|
| 341 |
+
std::shared_ptr<Schema> filtered_schema() const { return filtered_schema_; }
|
| 342 |
+
|
| 343 |
+
/// \return the all decoded record batches
|
| 344 |
+
const std::vector<std::shared_ptr<RecordBatch>>& record_batches() const {
|
| 345 |
+
return record_batches_;
|
| 346 |
+
}
|
| 347 |
+
|
| 348 |
+
/// \return the all decoded metadatas
|
| 349 |
+
const std::vector<std::shared_ptr<KeyValueMetadata>>& metadatas() const {
|
| 350 |
+
return metadatas_;
|
| 351 |
+
}
|
| 352 |
+
|
| 353 |
+
/// \return the number of collected record batches
|
| 354 |
+
int64_t num_record_batches() const { return record_batches_.size(); }
|
| 355 |
+
|
| 356 |
+
/// \return the last decoded record batch and remove it from
|
| 357 |
+
/// record_batches
|
| 358 |
+
std::shared_ptr<RecordBatch> PopRecordBatch() {
|
| 359 |
+
auto record_batch_with_metadata = PopRecordBatchWithMetadata();
|
| 360 |
+
return std::move(record_batch_with_metadata.batch);
|
| 361 |
+
}
|
| 362 |
+
|
| 363 |
+
/// \return the last decoded record batch with custom metadata and
|
| 364 |
+
/// remove it from record_batches
|
| 365 |
+
RecordBatchWithMetadata PopRecordBatchWithMetadata() {
|
| 366 |
+
RecordBatchWithMetadata record_batch_with_metadata;
|
| 367 |
+
if (record_batches_.empty()) {
|
| 368 |
+
return record_batch_with_metadata;
|
| 369 |
+
}
|
| 370 |
+
record_batch_with_metadata.batch = std::move(record_batches_.back());
|
| 371 |
+
record_batch_with_metadata.custom_metadata = std::move(metadatas_.back());
|
| 372 |
+
record_batches_.pop_back();
|
| 373 |
+
metadatas_.pop_back();
|
| 374 |
+
return record_batch_with_metadata;
|
| 375 |
+
}
|
| 376 |
+
|
| 377 |
+
private:
|
| 378 |
+
std::shared_ptr<Schema> schema_;
|
| 379 |
+
std::shared_ptr<Schema> filtered_schema_;
|
| 380 |
+
std::vector<std::shared_ptr<RecordBatch>> record_batches_;
|
| 381 |
+
std::vector<std::shared_ptr<KeyValueMetadata>> metadatas_;
|
| 382 |
+
};
|
| 383 |
+
|
| 384 |
+
/// \brief Push style stream decoder that receives data from user.
|
| 385 |
+
///
|
| 386 |
+
/// This class decodes the Apache Arrow IPC streaming format data.
|
| 387 |
+
///
|
| 388 |
+
/// This API is EXPERIMENTAL.
|
| 389 |
+
///
|
| 390 |
+
/// \see https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format
|
| 391 |
+
///
|
| 392 |
+
/// \since 0.17.0
|
| 393 |
+
class ARROW_EXPORT StreamDecoder {
|
| 394 |
+
public:
|
| 395 |
+
/// \brief Construct a stream decoder.
|
| 396 |
+
///
|
| 397 |
+
/// \param[in] listener a Listener that must implement
|
| 398 |
+
/// Listener::OnRecordBatchDecoded() to receive decoded record batches
|
| 399 |
+
/// \param[in] options any IPC reading options (optional)
|
| 400 |
+
StreamDecoder(std::shared_ptr<Listener> listener,
|
| 401 |
+
IpcReadOptions options = IpcReadOptions::Defaults());
|
| 402 |
+
|
| 403 |
+
virtual ~StreamDecoder();
|
| 404 |
+
|
| 405 |
+
/// \brief Feed data to the decoder as a raw data.
|
| 406 |
+
///
|
| 407 |
+
/// If the decoder can read one or more record batches by the data,
|
| 408 |
+
/// the decoder calls listener->OnRecordBatchDecoded() with a
|
| 409 |
+
/// decoded record batch multiple times.
|
| 410 |
+
///
|
| 411 |
+
/// \param[in] data a raw data to be processed. This data isn't
|
| 412 |
+
/// copied. The passed memory must be kept alive through record
|
| 413 |
+
/// batch processing.
|
| 414 |
+
/// \param[in] size raw data size.
|
| 415 |
+
/// \return Status
|
| 416 |
+
Status Consume(const uint8_t* data, int64_t size);
|
| 417 |
+
|
| 418 |
+
/// \brief Feed data to the decoder as a Buffer.
|
| 419 |
+
///
|
| 420 |
+
/// If the decoder can read one or more record batches by the
|
| 421 |
+
/// Buffer, the decoder calls listener->RecordBatchReceived() with a
|
| 422 |
+
/// decoded record batch multiple times.
|
| 423 |
+
///
|
| 424 |
+
/// \param[in] buffer a Buffer to be processed.
|
| 425 |
+
/// \return Status
|
| 426 |
+
Status Consume(std::shared_ptr<Buffer> buffer);
|
| 427 |
+
|
| 428 |
+
/// \brief Reset the internal status.
|
| 429 |
+
///
|
| 430 |
+
/// You can reuse this decoder for new stream after calling
|
| 431 |
+
/// this.
|
| 432 |
+
///
|
| 433 |
+
/// \return Status
|
| 434 |
+
Status Reset();
|
| 435 |
+
|
| 436 |
+
/// \return the shared schema of the record batches in the stream
|
| 437 |
+
std::shared_ptr<Schema> schema() const;
|
| 438 |
+
|
| 439 |
+
/// \brief Return the number of bytes needed to advance the state of
|
| 440 |
+
/// the decoder.
|
| 441 |
+
///
|
| 442 |
+
/// This method is provided for users who want to optimize performance.
|
| 443 |
+
/// Normal users don't need to use this method.
|
| 444 |
+
///
|
| 445 |
+
/// Here is an example usage for normal users:
|
| 446 |
+
///
|
| 447 |
+
/// ~~~{.cpp}
|
| 448 |
+
/// decoder.Consume(buffer1);
|
| 449 |
+
/// decoder.Consume(buffer2);
|
| 450 |
+
/// decoder.Consume(buffer3);
|
| 451 |
+
/// ~~~
|
| 452 |
+
///
|
| 453 |
+
/// Decoder has internal buffer. If consumed data isn't enough to
|
| 454 |
+
/// advance the state of the decoder, consumed data is buffered to
|
| 455 |
+
/// the internal buffer. It causes performance overhead.
|
| 456 |
+
///
|
| 457 |
+
/// If you pass next_required_size() size data to each Consume()
|
| 458 |
+
/// call, the decoder doesn't use its internal buffer. It improves
|
| 459 |
+
/// performance.
|
| 460 |
+
///
|
| 461 |
+
/// Here is an example usage to avoid using internal buffer:
|
| 462 |
+
///
|
| 463 |
+
/// ~~~{.cpp}
|
| 464 |
+
/// buffer1 = get_data(decoder.next_required_size());
|
| 465 |
+
/// decoder.Consume(buffer1);
|
| 466 |
+
/// buffer2 = get_data(decoder.next_required_size());
|
| 467 |
+
/// decoder.Consume(buffer2);
|
| 468 |
+
/// ~~~
|
| 469 |
+
///
|
| 470 |
+
/// Users can use this method to avoid creating small chunks. Record
|
| 471 |
+
/// batch data must be contiguous data. If users pass small chunks
|
| 472 |
+
/// to the decoder, the decoder needs concatenate small chunks
|
| 473 |
+
/// internally. It causes performance overhead.
|
| 474 |
+
///
|
| 475 |
+
/// Here is an example usage to reduce small chunks:
|
| 476 |
+
///
|
| 477 |
+
/// ~~~{.cpp}
|
| 478 |
+
/// buffer = AllocateResizableBuffer();
|
| 479 |
+
/// while ((small_chunk = get_data(&small_chunk_size))) {
|
| 480 |
+
/// auto current_buffer_size = buffer->size();
|
| 481 |
+
/// buffer->Resize(current_buffer_size + small_chunk_size);
|
| 482 |
+
/// memcpy(buffer->mutable_data() + current_buffer_size,
|
| 483 |
+
/// small_chunk,
|
| 484 |
+
/// small_chunk_size);
|
| 485 |
+
/// if (buffer->size() < decoder.next_required_size()) {
|
| 486 |
+
/// continue;
|
| 487 |
+
/// }
|
| 488 |
+
/// std::shared_ptr<arrow::Buffer> chunk(buffer.release());
|
| 489 |
+
/// decoder.Consume(chunk);
|
| 490 |
+
/// buffer = AllocateResizableBuffer();
|
| 491 |
+
/// }
|
| 492 |
+
/// if (buffer->size() > 0) {
|
| 493 |
+
/// std::shared_ptr<arrow::Buffer> chunk(buffer.release());
|
| 494 |
+
/// decoder.Consume(chunk);
|
| 495 |
+
/// }
|
| 496 |
+
/// ~~~
|
| 497 |
+
///
|
| 498 |
+
/// \return the number of bytes needed to advance the state of the
|
| 499 |
+
/// decoder
|
| 500 |
+
int64_t next_required_size() const;
|
| 501 |
+
|
| 502 |
+
/// \brief Return current read statistics
|
| 503 |
+
ReadStats stats() const;
|
| 504 |
+
|
| 505 |
+
private:
|
| 506 |
+
class StreamDecoderImpl;
|
| 507 |
+
std::unique_ptr<StreamDecoderImpl> impl_;
|
| 508 |
+
|
| 509 |
+
ARROW_DISALLOW_COPY_AND_ASSIGN(StreamDecoder);
|
| 510 |
+
};
|
| 511 |
+
|
| 512 |
+
// Generic read functions; does not copy data if the input supports zero copy reads
|
| 513 |
+
|
| 514 |
+
/// \brief Read Schema from stream serialized as a single IPC message
|
| 515 |
+
/// and populate any dictionary-encoded fields into a DictionaryMemo
|
| 516 |
+
///
|
| 517 |
+
/// \param[in] stream an InputStream
|
| 518 |
+
/// \param[in] dictionary_memo for recording dictionary-encoded fields
|
| 519 |
+
/// \return the output Schema
|
| 520 |
+
///
|
| 521 |
+
/// If record batches follow the schema, it is better to use
|
| 522 |
+
/// RecordBatchStreamReader
|
| 523 |
+
ARROW_EXPORT
|
| 524 |
+
Result<std::shared_ptr<Schema>> ReadSchema(io::InputStream* stream,
|
| 525 |
+
DictionaryMemo* dictionary_memo);
|
| 526 |
+
|
| 527 |
+
/// \brief Read Schema from encapsulated Message
|
| 528 |
+
///
|
| 529 |
+
/// \param[in] message the message containing the Schema IPC metadata
|
| 530 |
+
/// \param[in] dictionary_memo DictionaryMemo for recording dictionary-encoded
|
| 531 |
+
/// fields. Can be nullptr if you are sure there are no
|
| 532 |
+
/// dictionary-encoded fields
|
| 533 |
+
/// \return the resulting Schema
|
| 534 |
+
ARROW_EXPORT
|
| 535 |
+
Result<std::shared_ptr<Schema>> ReadSchema(const Message& message,
|
| 536 |
+
DictionaryMemo* dictionary_memo);
|
| 537 |
+
|
| 538 |
+
/// Read record batch as encapsulated IPC message with metadata size prefix and
|
| 539 |
+
/// header
|
| 540 |
+
///
|
| 541 |
+
/// \param[in] schema the record batch schema
|
| 542 |
+
/// \param[in] dictionary_memo DictionaryMemo which has any
|
| 543 |
+
/// dictionaries. Can be nullptr if you are sure there are no
|
| 544 |
+
/// dictionary-encoded fields
|
| 545 |
+
/// \param[in] options IPC options for reading
|
| 546 |
+
/// \param[in] stream the file where the batch is located
|
| 547 |
+
/// \return the read record batch
|
| 548 |
+
ARROW_EXPORT
|
| 549 |
+
Result<std::shared_ptr<RecordBatch>> ReadRecordBatch(
|
| 550 |
+
const std::shared_ptr<Schema>& schema, const DictionaryMemo* dictionary_memo,
|
| 551 |
+
const IpcReadOptions& options, io::InputStream* stream);
|
| 552 |
+
|
| 553 |
+
/// \brief Read record batch from message
|
| 554 |
+
///
|
| 555 |
+
/// \param[in] message a Message containing the record batch metadata
|
| 556 |
+
/// \param[in] schema the record batch schema
|
| 557 |
+
/// \param[in] dictionary_memo DictionaryMemo which has any
|
| 558 |
+
/// dictionaries. Can be nullptr if you are sure there are no
|
| 559 |
+
/// dictionary-encoded fields
|
| 560 |
+
/// \param[in] options IPC options for reading
|
| 561 |
+
/// \return the read record batch
|
| 562 |
+
ARROW_EXPORT
|
| 563 |
+
Result<std::shared_ptr<RecordBatch>> ReadRecordBatch(
|
| 564 |
+
const Message& message, const std::shared_ptr<Schema>& schema,
|
| 565 |
+
const DictionaryMemo* dictionary_memo, const IpcReadOptions& options);
|
| 566 |
+
|
| 567 |
+
/// Read record batch from file given metadata and schema
|
| 568 |
+
///
|
| 569 |
+
/// \param[in] metadata a Message containing the record batch metadata
|
| 570 |
+
/// \param[in] schema the record batch schema
|
| 571 |
+
/// \param[in] dictionary_memo DictionaryMemo which has any
|
| 572 |
+
/// dictionaries. Can be nullptr if you are sure there are no
|
| 573 |
+
/// dictionary-encoded fields
|
| 574 |
+
/// \param[in] file a random access file
|
| 575 |
+
/// \param[in] options options for deserialization
|
| 576 |
+
/// \return the read record batch
|
| 577 |
+
ARROW_EXPORT
|
| 578 |
+
Result<std::shared_ptr<RecordBatch>> ReadRecordBatch(
|
| 579 |
+
const Buffer& metadata, const std::shared_ptr<Schema>& schema,
|
| 580 |
+
const DictionaryMemo* dictionary_memo, const IpcReadOptions& options,
|
| 581 |
+
io::RandomAccessFile* file);
|
| 582 |
+
|
| 583 |
+
/// \brief Read arrow::Tensor as encapsulated IPC message in file
|
| 584 |
+
///
|
| 585 |
+
/// \param[in] file an InputStream pointed at the start of the message
|
| 586 |
+
/// \return the read tensor
|
| 587 |
+
ARROW_EXPORT
|
| 588 |
+
Result<std::shared_ptr<Tensor>> ReadTensor(io::InputStream* file);
|
| 589 |
+
|
| 590 |
+
/// \brief EXPERIMENTAL: Read arrow::Tensor from IPC message
|
| 591 |
+
///
|
| 592 |
+
/// \param[in] message a Message containing the tensor metadata and body
|
| 593 |
+
/// \return the read tensor
|
| 594 |
+
ARROW_EXPORT
|
| 595 |
+
Result<std::shared_ptr<Tensor>> ReadTensor(const Message& message);
|
| 596 |
+
|
| 597 |
+
/// \brief EXPERIMENTAL: Read arrow::SparseTensor as encapsulated IPC message in file
|
| 598 |
+
///
|
| 599 |
+
/// \param[in] file an InputStream pointed at the start of the message
|
| 600 |
+
/// \return the read sparse tensor
|
| 601 |
+
ARROW_EXPORT
|
| 602 |
+
Result<std::shared_ptr<SparseTensor>> ReadSparseTensor(io::InputStream* file);
|
| 603 |
+
|
| 604 |
+
/// \brief EXPERIMENTAL: Read arrow::SparseTensor from IPC message
|
| 605 |
+
///
|
| 606 |
+
/// \param[in] message a Message containing the tensor metadata and body
|
| 607 |
+
/// \return the read sparse tensor
|
| 608 |
+
ARROW_EXPORT
|
| 609 |
+
Result<std::shared_ptr<SparseTensor>> ReadSparseTensor(const Message& message);
|
| 610 |
+
|
| 611 |
+
namespace internal {
|
| 612 |
+
|
| 613 |
+
// These internal APIs may change without warning or deprecation
|
| 614 |
+
|
| 615 |
+
/// \brief EXPERIMENTAL: Read arrow::SparseTensorFormat::type from a metadata
|
| 616 |
+
/// \param[in] metadata a Buffer containing the sparse tensor metadata
|
| 617 |
+
/// \return the count of the body buffers
|
| 618 |
+
ARROW_EXPORT
|
| 619 |
+
Result<size_t> ReadSparseTensorBodyBufferCount(const Buffer& metadata);
|
| 620 |
+
|
| 621 |
+
/// \brief EXPERIMENTAL: Read arrow::SparseTensor from an IpcPayload
|
| 622 |
+
/// \param[in] payload a IpcPayload contains a serialized SparseTensor
|
| 623 |
+
/// \return the read sparse tensor
|
| 624 |
+
ARROW_EXPORT
|
| 625 |
+
Result<std::shared_ptr<SparseTensor>> ReadSparseTensorPayload(const IpcPayload& payload);
|
| 626 |
+
|
| 627 |
+
// For fuzzing targets
|
| 628 |
+
ARROW_EXPORT
|
| 629 |
+
Status FuzzIpcStream(const uint8_t* data, int64_t size);
|
| 630 |
+
ARROW_EXPORT
|
| 631 |
+
Status FuzzIpcTensorStream(const uint8_t* data, int64_t size);
|
| 632 |
+
ARROW_EXPORT
|
| 633 |
+
Status FuzzIpcFile(const uint8_t* data, int64_t size);
|
| 634 |
+
|
| 635 |
+
} // namespace internal
|
| 636 |
+
|
| 637 |
+
} // namespace ipc
|
| 638 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/test_common.h
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <cstdint>
|
| 21 |
+
#include <memory>
|
| 22 |
+
#include <vector>
|
| 23 |
+
|
| 24 |
+
#include "arrow/array.h"
|
| 25 |
+
#include "arrow/record_batch.h"
|
| 26 |
+
#include "arrow/status.h"
|
| 27 |
+
#include "arrow/testing/visibility.h"
|
| 28 |
+
#include "arrow/type.h"
|
| 29 |
+
|
| 30 |
+
namespace arrow {
|
| 31 |
+
namespace ipc {
|
| 32 |
+
namespace test {
|
| 33 |
+
|
| 34 |
+
// A typedef used for test parameterization
|
| 35 |
+
typedef Status MakeRecordBatch(std::shared_ptr<RecordBatch>* out);
|
| 36 |
+
|
| 37 |
+
ARROW_TESTING_EXPORT
|
| 38 |
+
void CompareArraysDetailed(int index, const Array& result, const Array& expected);
|
| 39 |
+
|
| 40 |
+
ARROW_TESTING_EXPORT
|
| 41 |
+
void CompareBatchColumnsDetailed(const RecordBatch& result, const RecordBatch& expected);
|
| 42 |
+
|
| 43 |
+
ARROW_TESTING_EXPORT
|
| 44 |
+
Status MakeRandomInt32Array(int64_t length, bool include_nulls, MemoryPool* pool,
|
| 45 |
+
std::shared_ptr<Array>* out, uint32_t seed = 0,
|
| 46 |
+
int32_t min = 0, int32_t max = 1000);
|
| 47 |
+
|
| 48 |
+
ARROW_TESTING_EXPORT
|
| 49 |
+
Status MakeRandomInt64Array(int64_t length, bool include_nulls, MemoryPool* pool,
|
| 50 |
+
std::shared_ptr<Array>* out, uint32_t seed = 0);
|
| 51 |
+
|
| 52 |
+
ARROW_TESTING_EXPORT
|
| 53 |
+
Status MakeRandomListArray(const std::shared_ptr<Array>& child_array, int num_lists,
|
| 54 |
+
bool include_nulls, MemoryPool* pool,
|
| 55 |
+
std::shared_ptr<Array>* out);
|
| 56 |
+
|
| 57 |
+
ARROW_TESTING_EXPORT
|
| 58 |
+
Status MakeRandomLargeListArray(const std::shared_ptr<Array>& child_array, int num_lists,
|
| 59 |
+
bool include_nulls, MemoryPool* pool,
|
| 60 |
+
std::shared_ptr<Array>* out);
|
| 61 |
+
|
| 62 |
+
ARROW_TESTING_EXPORT
|
| 63 |
+
Status MakeRandomBooleanArray(const int length, bool include_nulls,
|
| 64 |
+
std::shared_ptr<Array>* out);
|
| 65 |
+
|
| 66 |
+
ARROW_TESTING_EXPORT
|
| 67 |
+
Status MakeBooleanBatchSized(const int length, std::shared_ptr<RecordBatch>* out);
|
| 68 |
+
|
| 69 |
+
ARROW_TESTING_EXPORT
|
| 70 |
+
Status MakeBooleanBatch(std::shared_ptr<RecordBatch>* out);
|
| 71 |
+
|
| 72 |
+
ARROW_TESTING_EXPORT
|
| 73 |
+
Status MakeIntBatchSized(int length, std::shared_ptr<RecordBatch>* out,
|
| 74 |
+
uint32_t seed = 0);
|
| 75 |
+
|
| 76 |
+
ARROW_TESTING_EXPORT
|
| 77 |
+
Status MakeIntRecordBatch(std::shared_ptr<RecordBatch>* out);
|
| 78 |
+
|
| 79 |
+
ARROW_TESTING_EXPORT
|
| 80 |
+
Status MakeFloat3264BatchSized(int length, std::shared_ptr<RecordBatch>* out,
|
| 81 |
+
uint32_t seed = 0);
|
| 82 |
+
|
| 83 |
+
ARROW_TESTING_EXPORT
|
| 84 |
+
Status MakeFloat3264Batch(std::shared_ptr<RecordBatch>* out);
|
| 85 |
+
|
| 86 |
+
ARROW_TESTING_EXPORT
|
| 87 |
+
Status MakeFloatBatchSized(int length, std::shared_ptr<RecordBatch>* out,
|
| 88 |
+
uint32_t seed = 0);
|
| 89 |
+
|
| 90 |
+
ARROW_TESTING_EXPORT
|
| 91 |
+
Status MakeFloatBatch(std::shared_ptr<RecordBatch>* out);
|
| 92 |
+
|
| 93 |
+
ARROW_TESTING_EXPORT
|
| 94 |
+
Status MakeRandomStringArray(int64_t length, bool include_nulls, MemoryPool* pool,
|
| 95 |
+
std::shared_ptr<Array>* out);
|
| 96 |
+
|
| 97 |
+
ARROW_TESTING_EXPORT
|
| 98 |
+
Status MakeStringTypesRecordBatch(std::shared_ptr<RecordBatch>* out,
|
| 99 |
+
bool with_nulls = true, bool with_view_types = true);
|
| 100 |
+
|
| 101 |
+
ARROW_TESTING_EXPORT
|
| 102 |
+
Status MakeStringTypesRecordBatchWithNulls(std::shared_ptr<RecordBatch>* out);
|
| 103 |
+
|
| 104 |
+
ARROW_TESTING_EXPORT
|
| 105 |
+
Status MakeNullRecordBatch(std::shared_ptr<RecordBatch>* out);
|
| 106 |
+
|
| 107 |
+
ARROW_TESTING_EXPORT
|
| 108 |
+
Status MakeListRecordBatchSized(int length, std::shared_ptr<RecordBatch>* out);
|
| 109 |
+
|
| 110 |
+
ARROW_TESTING_EXPORT
|
| 111 |
+
Status MakeListRecordBatch(std::shared_ptr<RecordBatch>* out);
|
| 112 |
+
|
| 113 |
+
ARROW_TESTING_EXPORT
|
| 114 |
+
Status MakeListViewRecordBatchSized(int length, std::shared_ptr<RecordBatch>* out);
|
| 115 |
+
|
| 116 |
+
ARROW_TESTING_EXPORT
|
| 117 |
+
Status MakeListViewRecordBatch(std::shared_ptr<RecordBatch>* out);
|
| 118 |
+
|
| 119 |
+
ARROW_TESTING_EXPORT
|
| 120 |
+
Status MakeFixedSizeListRecordBatch(std::shared_ptr<RecordBatch>* out);
|
| 121 |
+
|
| 122 |
+
ARROW_TESTING_EXPORT
|
| 123 |
+
Status MakeZeroLengthRecordBatch(std::shared_ptr<RecordBatch>* out);
|
| 124 |
+
|
| 125 |
+
ARROW_TESTING_EXPORT
|
| 126 |
+
Status MakeNonNullRecordBatch(std::shared_ptr<RecordBatch>* out);
|
| 127 |
+
|
| 128 |
+
ARROW_TESTING_EXPORT
|
| 129 |
+
Status MakeDeeplyNestedList(std::shared_ptr<RecordBatch>* out);
|
| 130 |
+
|
| 131 |
+
ARROW_TESTING_EXPORT
|
| 132 |
+
Status MakeDeeplyNestedListView(std::shared_ptr<RecordBatch>* out);
|
| 133 |
+
|
| 134 |
+
ARROW_TESTING_EXPORT
|
| 135 |
+
Status MakeStruct(std::shared_ptr<RecordBatch>* out);
|
| 136 |
+
|
| 137 |
+
ARROW_TESTING_EXPORT
|
| 138 |
+
Status MakeRunEndEncoded(std::shared_ptr<RecordBatch>* out);
|
| 139 |
+
|
| 140 |
+
ARROW_TESTING_EXPORT
|
| 141 |
+
Status MakeUnion(std::shared_ptr<RecordBatch>* out);
|
| 142 |
+
|
| 143 |
+
ARROW_TESTING_EXPORT
|
| 144 |
+
Status MakeDictionary(std::shared_ptr<RecordBatch>* out);
|
| 145 |
+
|
| 146 |
+
ARROW_TESTING_EXPORT
|
| 147 |
+
Status MakeDictionaryFlat(std::shared_ptr<RecordBatch>* out);
|
| 148 |
+
|
| 149 |
+
ARROW_TESTING_EXPORT
|
| 150 |
+
Status MakeNestedDictionary(std::shared_ptr<RecordBatch>* out);
|
| 151 |
+
|
| 152 |
+
ARROW_TESTING_EXPORT
|
| 153 |
+
Status MakeMap(std::shared_ptr<RecordBatch>* out);
|
| 154 |
+
|
| 155 |
+
ARROW_TESTING_EXPORT
|
| 156 |
+
Status MakeMapOfDictionary(std::shared_ptr<RecordBatch>* out);
|
| 157 |
+
|
| 158 |
+
ARROW_TESTING_EXPORT
|
| 159 |
+
Status MakeDates(std::shared_ptr<RecordBatch>* out);
|
| 160 |
+
|
| 161 |
+
ARROW_TESTING_EXPORT
|
| 162 |
+
Status MakeTimestamps(std::shared_ptr<RecordBatch>* out);
|
| 163 |
+
|
| 164 |
+
ARROW_TESTING_EXPORT
|
| 165 |
+
Status MakeIntervals(std::shared_ptr<RecordBatch>* out);
|
| 166 |
+
|
| 167 |
+
ARROW_TESTING_EXPORT
|
| 168 |
+
Status MakeTimes(std::shared_ptr<RecordBatch>* out);
|
| 169 |
+
|
| 170 |
+
ARROW_TESTING_EXPORT
|
| 171 |
+
Status MakeFWBinary(std::shared_ptr<RecordBatch>* out);
|
| 172 |
+
|
| 173 |
+
ARROW_TESTING_EXPORT
|
| 174 |
+
Status MakeDecimal(std::shared_ptr<RecordBatch>* out);
|
| 175 |
+
|
| 176 |
+
ARROW_TESTING_EXPORT
|
| 177 |
+
Status MakeNull(std::shared_ptr<RecordBatch>* out);
|
| 178 |
+
|
| 179 |
+
ARROW_TESTING_EXPORT
|
| 180 |
+
Status MakeUuid(std::shared_ptr<RecordBatch>* out);
|
| 181 |
+
|
| 182 |
+
ARROW_TESTING_EXPORT
|
| 183 |
+
Status MakeComplex128(std::shared_ptr<RecordBatch>* out);
|
| 184 |
+
|
| 185 |
+
ARROW_TESTING_EXPORT
|
| 186 |
+
Status MakeDictExtension(std::shared_ptr<RecordBatch>* out);
|
| 187 |
+
|
| 188 |
+
ARROW_TESTING_EXPORT
|
| 189 |
+
Status MakeRandomTensor(const std::shared_ptr<DataType>& type,
|
| 190 |
+
const std::vector<int64_t>& shape, bool row_major_p,
|
| 191 |
+
std::shared_ptr<Tensor>* out, uint32_t seed = 0);
|
| 192 |
+
|
| 193 |
+
ARROW_TESTING_EXPORT Status RoundtripBatch(const std::shared_ptr<RecordBatch>& batch,
|
| 194 |
+
std::shared_ptr<RecordBatch>* out);
|
| 195 |
+
|
| 196 |
+
} // namespace test
|
| 197 |
+
} // namespace ipc
|
| 198 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/type_fwd.h
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
namespace arrow {
|
| 21 |
+
namespace ipc {
|
| 22 |
+
|
| 23 |
+
enum class MetadataVersion : char {
|
| 24 |
+
/// 0.1.0
|
| 25 |
+
V1,
|
| 26 |
+
|
| 27 |
+
/// 0.2.0
|
| 28 |
+
V2,
|
| 29 |
+
|
| 30 |
+
/// 0.3.0 to 0.7.1
|
| 31 |
+
V3,
|
| 32 |
+
|
| 33 |
+
/// 0.8.0 to 0.17.0
|
| 34 |
+
V4,
|
| 35 |
+
|
| 36 |
+
/// >= 1.0.0
|
| 37 |
+
V5
|
| 38 |
+
};
|
| 39 |
+
|
| 40 |
+
class Message;
|
| 41 |
+
enum class MessageType {
|
| 42 |
+
NONE,
|
| 43 |
+
SCHEMA,
|
| 44 |
+
DICTIONARY_BATCH,
|
| 45 |
+
RECORD_BATCH,
|
| 46 |
+
TENSOR,
|
| 47 |
+
SPARSE_TENSOR
|
| 48 |
+
};
|
| 49 |
+
|
| 50 |
+
struct IpcReadOptions;
|
| 51 |
+
struct IpcWriteOptions;
|
| 52 |
+
|
| 53 |
+
class MessageReader;
|
| 54 |
+
|
| 55 |
+
class RecordBatchStreamReader;
|
| 56 |
+
class RecordBatchFileReader;
|
| 57 |
+
class RecordBatchWriter;
|
| 58 |
+
|
| 59 |
+
class DictionaryFieldMapper;
|
| 60 |
+
class DictionaryMemo;
|
| 61 |
+
|
| 62 |
+
namespace feather {
|
| 63 |
+
|
| 64 |
+
class Reader;
|
| 65 |
+
|
| 66 |
+
} // namespace feather
|
| 67 |
+
} // namespace ipc
|
| 68 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/util.h
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <cstdint>
|
| 21 |
+
|
| 22 |
+
namespace arrow {
|
| 23 |
+
namespace ipc {
|
| 24 |
+
|
| 25 |
+
// Buffers are padded to 64-byte boundaries (for SIMD)
|
| 26 |
+
static constexpr int32_t kArrowAlignment = 64;
|
| 27 |
+
|
| 28 |
+
// Tensors are padded to 64-byte boundaries
|
| 29 |
+
static constexpr int32_t kTensorAlignment = 64;
|
| 30 |
+
|
| 31 |
+
// Align on 8-byte boundaries in IPC
|
| 32 |
+
static constexpr int32_t kArrowIpcAlignment = 8;
|
| 33 |
+
|
| 34 |
+
static constexpr uint8_t kPaddingBytes[kArrowAlignment] = {0};
|
| 35 |
+
|
| 36 |
+
static inline int64_t PaddedLength(int64_t nbytes, int32_t alignment = kArrowAlignment) {
|
| 37 |
+
return ((nbytes + alignment - 1) / alignment) * alignment;
|
| 38 |
+
}
|
| 39 |
+
|
| 40 |
+
} // namespace ipc
|
| 41 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/writer.h
ADDED
|
@@ -0,0 +1,475 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
// Implement Arrow streaming binary format
|
| 19 |
+
|
| 20 |
+
#pragma once
|
| 21 |
+
|
| 22 |
+
#include <cstdint>
|
| 23 |
+
#include <memory>
|
| 24 |
+
#include <vector>
|
| 25 |
+
|
| 26 |
+
#include "arrow/ipc/dictionary.h" // IWYU pragma: export
|
| 27 |
+
#include "arrow/ipc/message.h"
|
| 28 |
+
#include "arrow/ipc/options.h"
|
| 29 |
+
#include "arrow/result.h"
|
| 30 |
+
#include "arrow/util/macros.h"
|
| 31 |
+
#include "arrow/util/visibility.h"
|
| 32 |
+
|
| 33 |
+
namespace arrow {
|
| 34 |
+
|
| 35 |
+
class Array;
|
| 36 |
+
class Buffer;
|
| 37 |
+
class MemoryManager;
|
| 38 |
+
class MemoryPool;
|
| 39 |
+
class RecordBatch;
|
| 40 |
+
class Schema;
|
| 41 |
+
class Status;
|
| 42 |
+
class Table;
|
| 43 |
+
class Tensor;
|
| 44 |
+
class SparseTensor;
|
| 45 |
+
|
| 46 |
+
namespace io {
|
| 47 |
+
|
| 48 |
+
class OutputStream;
|
| 49 |
+
|
| 50 |
+
} // namespace io
|
| 51 |
+
|
| 52 |
+
namespace ipc {
|
| 53 |
+
|
| 54 |
+
/// \brief Intermediate data structure with metadata header, and zero
|
| 55 |
+
/// or more buffers for the message body.
|
| 56 |
+
struct IpcPayload {
|
| 57 |
+
MessageType type = MessageType::NONE;
|
| 58 |
+
std::shared_ptr<Buffer> metadata;
|
| 59 |
+
std::vector<std::shared_ptr<Buffer>> body_buffers;
|
| 60 |
+
std::vector<int64_t> variadic_buffer_counts;
|
| 61 |
+
int64_t body_length = 0; // serialized body length (padded, maybe compressed)
|
| 62 |
+
int64_t raw_body_length = 0; // initial uncompressed body length
|
| 63 |
+
};
|
| 64 |
+
|
| 65 |
+
struct WriteStats {
|
| 66 |
+
/// Number of IPC messages written.
|
| 67 |
+
int64_t num_messages = 0;
|
| 68 |
+
/// Number of record batches written.
|
| 69 |
+
int64_t num_record_batches = 0;
|
| 70 |
+
/// Number of dictionary batches written.
|
| 71 |
+
///
|
| 72 |
+
/// Note: num_dictionary_batches >= num_dictionary_deltas + num_replaced_dictionaries
|
| 73 |
+
int64_t num_dictionary_batches = 0;
|
| 74 |
+
|
| 75 |
+
/// Number of dictionary deltas written.
|
| 76 |
+
int64_t num_dictionary_deltas = 0;
|
| 77 |
+
/// Number of replaced dictionaries (i.e. where a dictionary batch replaces
|
| 78 |
+
/// an existing dictionary with an unrelated new dictionary).
|
| 79 |
+
int64_t num_replaced_dictionaries = 0;
|
| 80 |
+
|
| 81 |
+
/// Total size in bytes of record batches emitted.
|
| 82 |
+
/// The "raw" size counts the original buffer sizes, while the "serialized" size
|
| 83 |
+
/// includes padding and (optionally) compression.
|
| 84 |
+
int64_t total_raw_body_size = 0;
|
| 85 |
+
int64_t total_serialized_body_size = 0;
|
| 86 |
+
};
|
| 87 |
+
|
| 88 |
+
/// \class RecordBatchWriter
|
| 89 |
+
/// \brief Abstract interface for writing a stream of record batches
|
| 90 |
+
class ARROW_EXPORT RecordBatchWriter {
|
| 91 |
+
public:
|
| 92 |
+
virtual ~RecordBatchWriter();
|
| 93 |
+
|
| 94 |
+
/// \brief Write a record batch to the stream
|
| 95 |
+
///
|
| 96 |
+
/// \param[in] batch the record batch to write to the stream
|
| 97 |
+
/// \return Status
|
| 98 |
+
virtual Status WriteRecordBatch(const RecordBatch& batch) = 0;
|
| 99 |
+
|
| 100 |
+
/// \brief Write a record batch with custom metadata to the stream
|
| 101 |
+
///
|
| 102 |
+
/// \param[in] batch the record batch to write to the stream
|
| 103 |
+
/// \param[in] custom_metadata the record batch's custom metadata to write to the stream
|
| 104 |
+
/// \return Status
|
| 105 |
+
virtual Status WriteRecordBatch(
|
| 106 |
+
const RecordBatch& batch,
|
| 107 |
+
const std::shared_ptr<const KeyValueMetadata>& custom_metadata);
|
| 108 |
+
|
| 109 |
+
/// \brief Write possibly-chunked table by creating sequence of record batches
|
| 110 |
+
/// \param[in] table table to write
|
| 111 |
+
/// \return Status
|
| 112 |
+
Status WriteTable(const Table& table);
|
| 113 |
+
|
| 114 |
+
/// \brief Write Table with a particular chunksize
|
| 115 |
+
/// \param[in] table table to write
|
| 116 |
+
/// \param[in] max_chunksize maximum number of rows for table chunks. To
|
| 117 |
+
/// indicate that no maximum should be enforced, pass -1.
|
| 118 |
+
/// \return Status
|
| 119 |
+
virtual Status WriteTable(const Table& table, int64_t max_chunksize);
|
| 120 |
+
|
| 121 |
+
/// \brief Perform any logic necessary to finish the stream
|
| 122 |
+
///
|
| 123 |
+
/// \return Status
|
| 124 |
+
virtual Status Close() = 0;
|
| 125 |
+
|
| 126 |
+
/// \brief Return current write statistics
|
| 127 |
+
virtual WriteStats stats() const = 0;
|
| 128 |
+
};
|
| 129 |
+
|
| 130 |
+
/// \defgroup record-batch-writer-factories Functions for creating RecordBatchWriter
|
| 131 |
+
/// instances
|
| 132 |
+
///
|
| 133 |
+
/// @{
|
| 134 |
+
|
| 135 |
+
/// Create a new IPC stream writer from stream sink and schema. User is
|
| 136 |
+
/// responsible for closing the actual OutputStream.
|
| 137 |
+
///
|
| 138 |
+
/// \param[in] sink output stream to write to
|
| 139 |
+
/// \param[in] schema the schema of the record batches to be written
|
| 140 |
+
/// \param[in] options options for serialization
|
| 141 |
+
/// \return Result<std::shared_ptr<RecordBatchWriter>>
|
| 142 |
+
ARROW_EXPORT
|
| 143 |
+
Result<std::shared_ptr<RecordBatchWriter>> MakeStreamWriter(
|
| 144 |
+
io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
|
| 145 |
+
const IpcWriteOptions& options = IpcWriteOptions::Defaults());
|
| 146 |
+
|
| 147 |
+
/// Create a new IPC stream writer from stream sink and schema. User is
|
| 148 |
+
/// responsible for closing the actual OutputStream.
|
| 149 |
+
///
|
| 150 |
+
/// \param[in] sink output stream to write to
|
| 151 |
+
/// \param[in] schema the schema of the record batches to be written
|
| 152 |
+
/// \param[in] options options for serialization
|
| 153 |
+
/// \return Result<std::shared_ptr<RecordBatchWriter>>
|
| 154 |
+
ARROW_EXPORT
|
| 155 |
+
Result<std::shared_ptr<RecordBatchWriter>> MakeStreamWriter(
|
| 156 |
+
std::shared_ptr<io::OutputStream> sink, const std::shared_ptr<Schema>& schema,
|
| 157 |
+
const IpcWriteOptions& options = IpcWriteOptions::Defaults());
|
| 158 |
+
|
| 159 |
+
/// Create a new IPC file writer from stream sink and schema
|
| 160 |
+
///
|
| 161 |
+
/// \param[in] sink output stream to write to
|
| 162 |
+
/// \param[in] schema the schema of the record batches to be written
|
| 163 |
+
/// \param[in] options options for serialization, optional
|
| 164 |
+
/// \param[in] metadata custom metadata for File Footer, optional
|
| 165 |
+
/// \return Result<std::shared_ptr<RecordBatchWriter>>
|
| 166 |
+
ARROW_EXPORT
|
| 167 |
+
Result<std::shared_ptr<RecordBatchWriter>> MakeFileWriter(
|
| 168 |
+
io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
|
| 169 |
+
const IpcWriteOptions& options = IpcWriteOptions::Defaults(),
|
| 170 |
+
const std::shared_ptr<const KeyValueMetadata>& metadata = NULLPTR);
|
| 171 |
+
|
| 172 |
+
/// Create a new IPC file writer from stream sink and schema
|
| 173 |
+
///
|
| 174 |
+
/// \param[in] sink output stream to write to
|
| 175 |
+
/// \param[in] schema the schema of the record batches to be written
|
| 176 |
+
/// \param[in] options options for serialization, optional
|
| 177 |
+
/// \param[in] metadata custom metadata for File Footer, optional
|
| 178 |
+
/// \return Result<std::shared_ptr<RecordBatchWriter>>
|
| 179 |
+
ARROW_EXPORT
|
| 180 |
+
Result<std::shared_ptr<RecordBatchWriter>> MakeFileWriter(
|
| 181 |
+
std::shared_ptr<io::OutputStream> sink, const std::shared_ptr<Schema>& schema,
|
| 182 |
+
const IpcWriteOptions& options = IpcWriteOptions::Defaults(),
|
| 183 |
+
const std::shared_ptr<const KeyValueMetadata>& metadata = NULLPTR);
|
| 184 |
+
|
| 185 |
+
/// @}
|
| 186 |
+
|
| 187 |
+
/// \brief Low-level API for writing a record batch (without schema)
|
| 188 |
+
/// to an OutputStream as encapsulated IPC message. See Arrow format
|
| 189 |
+
/// documentation for more detail.
|
| 190 |
+
///
|
| 191 |
+
/// \param[in] batch the record batch to write
|
| 192 |
+
/// \param[in] buffer_start_offset the start offset to use in the buffer metadata,
|
| 193 |
+
/// generally should be 0
|
| 194 |
+
/// \param[in] dst an OutputStream
|
| 195 |
+
/// \param[out] metadata_length the size of the length-prefixed flatbuffer
|
| 196 |
+
/// including padding to a 64-byte boundary
|
| 197 |
+
/// \param[out] body_length the size of the contiguous buffer block plus
|
| 198 |
+
/// \param[in] options options for serialization
|
| 199 |
+
/// \return Status
|
| 200 |
+
ARROW_EXPORT
|
| 201 |
+
Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
|
| 202 |
+
io::OutputStream* dst, int32_t* metadata_length,
|
| 203 |
+
int64_t* body_length, const IpcWriteOptions& options);
|
| 204 |
+
|
| 205 |
+
/// \brief Serialize record batch as encapsulated IPC message in a new buffer
|
| 206 |
+
///
|
| 207 |
+
/// \param[in] batch the record batch
|
| 208 |
+
/// \param[in] options the IpcWriteOptions to use for serialization
|
| 209 |
+
/// \return the serialized message
|
| 210 |
+
ARROW_EXPORT
|
| 211 |
+
Result<std::shared_ptr<Buffer>> SerializeRecordBatch(const RecordBatch& batch,
|
| 212 |
+
const IpcWriteOptions& options);
|
| 213 |
+
|
| 214 |
+
/// \brief Serialize record batch as encapsulated IPC message in a new buffer
|
| 215 |
+
///
|
| 216 |
+
/// \param[in] batch the record batch
|
| 217 |
+
/// \param[in] mm a MemoryManager to allocate memory from
|
| 218 |
+
/// \return the serialized message
|
| 219 |
+
ARROW_EXPORT
|
| 220 |
+
Result<std::shared_ptr<Buffer>> SerializeRecordBatch(const RecordBatch& batch,
|
| 221 |
+
std::shared_ptr<MemoryManager> mm);
|
| 222 |
+
|
| 223 |
+
/// \brief Write record batch to OutputStream
|
| 224 |
+
///
|
| 225 |
+
/// \param[in] batch the record batch to write
|
| 226 |
+
/// \param[in] options the IpcWriteOptions to use for serialization
|
| 227 |
+
/// \param[in] out the OutputStream to write the output to
|
| 228 |
+
/// \return Status
|
| 229 |
+
///
|
| 230 |
+
/// If writing to pre-allocated memory, you can use
|
| 231 |
+
/// arrow::ipc::GetRecordBatchSize to compute how much space is required
|
| 232 |
+
ARROW_EXPORT
|
| 233 |
+
Status SerializeRecordBatch(const RecordBatch& batch, const IpcWriteOptions& options,
|
| 234 |
+
io::OutputStream* out);
|
| 235 |
+
|
| 236 |
+
/// \brief Serialize schema as encapsulated IPC message
|
| 237 |
+
///
|
| 238 |
+
/// \param[in] schema the schema to write
|
| 239 |
+
/// \param[in] pool a MemoryPool to allocate memory from
|
| 240 |
+
/// \return the serialized schema
|
| 241 |
+
ARROW_EXPORT
|
| 242 |
+
Result<std::shared_ptr<Buffer>> SerializeSchema(const Schema& schema,
|
| 243 |
+
MemoryPool* pool = default_memory_pool());
|
| 244 |
+
|
| 245 |
+
/// \brief Write multiple record batches to OutputStream, including schema
|
| 246 |
+
/// \param[in] batches a vector of batches. Must all have same schema
|
| 247 |
+
/// \param[in] options options for serialization
|
| 248 |
+
/// \param[out] dst an OutputStream
|
| 249 |
+
/// \return Status
|
| 250 |
+
ARROW_EXPORT
|
| 251 |
+
Status WriteRecordBatchStream(const std::vector<std::shared_ptr<RecordBatch>>& batches,
|
| 252 |
+
const IpcWriteOptions& options, io::OutputStream* dst);
|
| 253 |
+
|
| 254 |
+
/// \brief Compute the number of bytes needed to write an IPC payload
|
| 255 |
+
/// including metadata
|
| 256 |
+
///
|
| 257 |
+
/// \param[in] payload the IPC payload to write
|
| 258 |
+
/// \param[in] options write options
|
| 259 |
+
/// \return the size of the complete encapsulated message
|
| 260 |
+
ARROW_EXPORT
|
| 261 |
+
int64_t GetPayloadSize(const IpcPayload& payload,
|
| 262 |
+
const IpcWriteOptions& options = IpcWriteOptions::Defaults());
|
| 263 |
+
|
| 264 |
+
/// \brief Compute the number of bytes needed to write a record batch including metadata
|
| 265 |
+
///
|
| 266 |
+
/// \param[in] batch the record batch to write
|
| 267 |
+
/// \param[out] size the size of the complete encapsulated message
|
| 268 |
+
/// \return Status
|
| 269 |
+
ARROW_EXPORT
|
| 270 |
+
Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size);
|
| 271 |
+
|
| 272 |
+
/// \brief Compute the number of bytes needed to write a record batch including metadata
|
| 273 |
+
///
|
| 274 |
+
/// \param[in] batch the record batch to write
|
| 275 |
+
/// \param[in] options options for serialization
|
| 276 |
+
/// \param[out] size the size of the complete encapsulated message
|
| 277 |
+
/// \return Status
|
| 278 |
+
ARROW_EXPORT
|
| 279 |
+
Status GetRecordBatchSize(const RecordBatch& batch, const IpcWriteOptions& options,
|
| 280 |
+
int64_t* size);
|
| 281 |
+
|
| 282 |
+
/// \brief Compute the number of bytes needed to write a tensor including metadata
|
| 283 |
+
///
|
| 284 |
+
/// \param[in] tensor the tensor to write
|
| 285 |
+
/// \param[out] size the size of the complete encapsulated message
|
| 286 |
+
/// \return Status
|
| 287 |
+
ARROW_EXPORT
|
| 288 |
+
Status GetTensorSize(const Tensor& tensor, int64_t* size);
|
| 289 |
+
|
| 290 |
+
/// \brief EXPERIMENTAL: Convert arrow::Tensor to a Message with minimal memory
|
| 291 |
+
/// allocation
|
| 292 |
+
///
|
| 293 |
+
/// \param[in] tensor the Tensor to write
|
| 294 |
+
/// \param[in] pool MemoryPool to allocate space for metadata
|
| 295 |
+
/// \return the resulting Message
|
| 296 |
+
ARROW_EXPORT
|
| 297 |
+
Result<std::unique_ptr<Message>> GetTensorMessage(const Tensor& tensor, MemoryPool* pool);
|
| 298 |
+
|
| 299 |
+
/// \brief Write arrow::Tensor as a contiguous message.
|
| 300 |
+
///
|
| 301 |
+
/// The metadata and body are written assuming 64-byte alignment. It is the
|
| 302 |
+
/// user's responsibility to ensure that the OutputStream has been aligned
|
| 303 |
+
/// to a 64-byte multiple before writing the message.
|
| 304 |
+
///
|
| 305 |
+
/// The message is written out as followed:
|
| 306 |
+
/// \code
|
| 307 |
+
/// <metadata size> <metadata> <tensor data>
|
| 308 |
+
/// \endcode
|
| 309 |
+
///
|
| 310 |
+
/// \param[in] tensor the Tensor to write
|
| 311 |
+
/// \param[in] dst the OutputStream to write to
|
| 312 |
+
/// \param[out] metadata_length the actual metadata length, including padding
|
| 313 |
+
/// \param[out] body_length the actual message body length
|
| 314 |
+
/// \return Status
|
| 315 |
+
ARROW_EXPORT
|
| 316 |
+
Status WriteTensor(const Tensor& tensor, io::OutputStream* dst, int32_t* metadata_length,
|
| 317 |
+
int64_t* body_length);
|
| 318 |
+
|
| 319 |
+
/// \brief EXPERIMENTAL: Convert arrow::SparseTensor to a Message with minimal memory
|
| 320 |
+
/// allocation
|
| 321 |
+
///
|
| 322 |
+
/// The message is written out as followed:
|
| 323 |
+
/// \code
|
| 324 |
+
/// <metadata size> <metadata> <sparse index> <sparse tensor body>
|
| 325 |
+
/// \endcode
|
| 326 |
+
///
|
| 327 |
+
/// \param[in] sparse_tensor the SparseTensor to write
|
| 328 |
+
/// \param[in] pool MemoryPool to allocate space for metadata
|
| 329 |
+
/// \return the resulting Message
|
| 330 |
+
ARROW_EXPORT
|
| 331 |
+
Result<std::unique_ptr<Message>> GetSparseTensorMessage(const SparseTensor& sparse_tensor,
|
| 332 |
+
MemoryPool* pool);
|
| 333 |
+
|
| 334 |
+
/// \brief EXPERIMENTAL: Write arrow::SparseTensor as a contiguous message. The metadata,
|
| 335 |
+
/// sparse index, and body are written assuming 64-byte alignment. It is the
|
| 336 |
+
/// user's responsibility to ensure that the OutputStream has been aligned
|
| 337 |
+
/// to a 64-byte multiple before writing the message.
|
| 338 |
+
///
|
| 339 |
+
/// \param[in] sparse_tensor the SparseTensor to write
|
| 340 |
+
/// \param[in] dst the OutputStream to write to
|
| 341 |
+
/// \param[out] metadata_length the actual metadata length, including padding
|
| 342 |
+
/// \param[out] body_length the actual message body length
|
| 343 |
+
/// \return Status
|
| 344 |
+
ARROW_EXPORT
|
| 345 |
+
Status WriteSparseTensor(const SparseTensor& sparse_tensor, io::OutputStream* dst,
|
| 346 |
+
int32_t* metadata_length, int64_t* body_length);
|
| 347 |
+
|
| 348 |
+
/// \brief Compute IpcPayload for the given schema
|
| 349 |
+
/// \param[in] schema the Schema that is being serialized
|
| 350 |
+
/// \param[in] options options for serialization
|
| 351 |
+
/// \param[in] mapper object mapping dictionary fields to dictionary ids
|
| 352 |
+
/// \param[out] out the returned vector of IpcPayloads
|
| 353 |
+
/// \return Status
|
| 354 |
+
ARROW_EXPORT
|
| 355 |
+
Status GetSchemaPayload(const Schema& schema, const IpcWriteOptions& options,
|
| 356 |
+
const DictionaryFieldMapper& mapper, IpcPayload* out);
|
| 357 |
+
|
| 358 |
+
/// \brief Compute IpcPayload for a dictionary
|
| 359 |
+
/// \param[in] id the dictionary id
|
| 360 |
+
/// \param[in] dictionary the dictionary values
|
| 361 |
+
/// \param[in] options options for serialization
|
| 362 |
+
/// \param[out] payload the output IpcPayload
|
| 363 |
+
/// \return Status
|
| 364 |
+
ARROW_EXPORT
|
| 365 |
+
Status GetDictionaryPayload(int64_t id, const std::shared_ptr<Array>& dictionary,
|
| 366 |
+
const IpcWriteOptions& options, IpcPayload* payload);
|
| 367 |
+
|
| 368 |
+
/// \brief Compute IpcPayload for a dictionary
|
| 369 |
+
/// \param[in] id the dictionary id
|
| 370 |
+
/// \param[in] is_delta whether the dictionary is a delta dictionary
|
| 371 |
+
/// \param[in] dictionary the dictionary values
|
| 372 |
+
/// \param[in] options options for serialization
|
| 373 |
+
/// \param[out] payload the output IpcPayload
|
| 374 |
+
/// \return Status
|
| 375 |
+
ARROW_EXPORT
|
| 376 |
+
Status GetDictionaryPayload(int64_t id, bool is_delta,
|
| 377 |
+
const std::shared_ptr<Array>& dictionary,
|
| 378 |
+
const IpcWriteOptions& options, IpcPayload* payload);
|
| 379 |
+
|
| 380 |
+
/// \brief Compute IpcPayload for the given record batch
|
| 381 |
+
/// \param[in] batch the RecordBatch that is being serialized
|
| 382 |
+
/// \param[in] options options for serialization
|
| 383 |
+
/// \param[out] out the returned IpcPayload
|
| 384 |
+
/// \return Status
|
| 385 |
+
ARROW_EXPORT
|
| 386 |
+
Status GetRecordBatchPayload(const RecordBatch& batch, const IpcWriteOptions& options,
|
| 387 |
+
IpcPayload* out);
|
| 388 |
+
|
| 389 |
+
/// \brief Compute IpcPayload for the given record batch and custom metadata
|
| 390 |
+
/// \param[in] batch the RecordBatch that is being serialized
|
| 391 |
+
/// \param[in] custom_metadata the custom metadata to be serialized with the record batch
|
| 392 |
+
/// \param[in] options options for serialization
|
| 393 |
+
/// \param[out] out the returned IpcPayload
|
| 394 |
+
/// \return Status
|
| 395 |
+
ARROW_EXPORT
|
| 396 |
+
Status GetRecordBatchPayload(
|
| 397 |
+
const RecordBatch& batch,
|
| 398 |
+
const std::shared_ptr<const KeyValueMetadata>& custom_metadata,
|
| 399 |
+
const IpcWriteOptions& options, IpcPayload* out);
|
| 400 |
+
|
| 401 |
+
/// \brief Write an IPC payload to the given stream.
|
| 402 |
+
/// \param[in] payload the payload to write
|
| 403 |
+
/// \param[in] options options for serialization
|
| 404 |
+
/// \param[in] dst The stream to write the payload to.
|
| 405 |
+
/// \param[out] metadata_length the length of the serialized metadata
|
| 406 |
+
/// \return Status
|
| 407 |
+
ARROW_EXPORT
|
| 408 |
+
Status WriteIpcPayload(const IpcPayload& payload, const IpcWriteOptions& options,
|
| 409 |
+
io::OutputStream* dst, int32_t* metadata_length);
|
| 410 |
+
|
| 411 |
+
/// \brief Compute IpcPayload for the given sparse tensor
|
| 412 |
+
/// \param[in] sparse_tensor the SparseTensor that is being serialized
|
| 413 |
+
/// \param[in,out] pool for any required temporary memory allocations
|
| 414 |
+
/// \param[out] out the returned IpcPayload
|
| 415 |
+
/// \return Status
|
| 416 |
+
ARROW_EXPORT
|
| 417 |
+
Status GetSparseTensorPayload(const SparseTensor& sparse_tensor, MemoryPool* pool,
|
| 418 |
+
IpcPayload* out);
|
| 419 |
+
|
| 420 |
+
namespace internal {
|
| 421 |
+
|
| 422 |
+
// These internal APIs may change without warning or deprecation
|
| 423 |
+
|
| 424 |
+
class ARROW_EXPORT IpcPayloadWriter {
|
| 425 |
+
public:
|
| 426 |
+
virtual ~IpcPayloadWriter();
|
| 427 |
+
|
| 428 |
+
// Default implementation is a no-op
|
| 429 |
+
virtual Status Start();
|
| 430 |
+
|
| 431 |
+
virtual Status WritePayload(const IpcPayload& payload) = 0;
|
| 432 |
+
|
| 433 |
+
virtual Status Close() = 0;
|
| 434 |
+
};
|
| 435 |
+
|
| 436 |
+
/// Create a new IPC payload stream writer from stream sink. User is
|
| 437 |
+
/// responsible for closing the actual OutputStream.
|
| 438 |
+
///
|
| 439 |
+
/// \param[in] sink output stream to write to
|
| 440 |
+
/// \param[in] options options for serialization
|
| 441 |
+
/// \return Result<std::shared_ptr<IpcPayloadWriter>>
|
| 442 |
+
ARROW_EXPORT
|
| 443 |
+
Result<std::unique_ptr<IpcPayloadWriter>> MakePayloadStreamWriter(
|
| 444 |
+
io::OutputStream* sink, const IpcWriteOptions& options = IpcWriteOptions::Defaults());
|
| 445 |
+
|
| 446 |
+
/// Create a new IPC payload file writer from stream sink.
|
| 447 |
+
///
|
| 448 |
+
/// \param[in] sink output stream to write to
|
| 449 |
+
/// \param[in] schema the schema of the record batches to be written
|
| 450 |
+
/// \param[in] options options for serialization, optional
|
| 451 |
+
/// \param[in] metadata custom metadata for File Footer, optional
|
| 452 |
+
/// \return Status
|
| 453 |
+
ARROW_EXPORT
|
| 454 |
+
Result<std::unique_ptr<IpcPayloadWriter>> MakePayloadFileWriter(
|
| 455 |
+
io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
|
| 456 |
+
const IpcWriteOptions& options = IpcWriteOptions::Defaults(),
|
| 457 |
+
const std::shared_ptr<const KeyValueMetadata>& metadata = NULLPTR);
|
| 458 |
+
|
| 459 |
+
/// Create a new RecordBatchWriter from IpcPayloadWriter and schema.
|
| 460 |
+
///
|
| 461 |
+
/// The format is implicitly the IPC stream format (allowing dictionary
|
| 462 |
+
/// replacement and deltas).
|
| 463 |
+
///
|
| 464 |
+
/// \param[in] sink the IpcPayloadWriter to write to
|
| 465 |
+
/// \param[in] schema the schema of the record batches to be written
|
| 466 |
+
/// \param[in] options options for serialization
|
| 467 |
+
/// \return Result<std::unique_ptr<RecordBatchWriter>>
|
| 468 |
+
ARROW_EXPORT
|
| 469 |
+
Result<std::unique_ptr<RecordBatchWriter>> OpenRecordBatchWriter(
|
| 470 |
+
std::unique_ptr<IpcPayloadWriter> sink, const std::shared_ptr<Schema>& schema,
|
| 471 |
+
const IpcWriteOptions& options = IpcWriteOptions::Defaults());
|
| 472 |
+
|
| 473 |
+
} // namespace internal
|
| 474 |
+
} // namespace ipc
|
| 475 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/api.h
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include "arrow/json/options.h"
|
| 21 |
+
#include "arrow/json/reader.h"
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/chunked_builder.h
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <memory>
|
| 21 |
+
#include <vector>
|
| 22 |
+
|
| 23 |
+
#include "arrow/status.h"
|
| 24 |
+
#include "arrow/type_fwd.h"
|
| 25 |
+
#include "arrow/util/type_fwd.h"
|
| 26 |
+
#include "arrow/util/visibility.h"
|
| 27 |
+
|
| 28 |
+
namespace arrow {
|
| 29 |
+
namespace json {
|
| 30 |
+
|
| 31 |
+
class PromotionGraph;
|
| 32 |
+
|
| 33 |
+
class ARROW_EXPORT ChunkedArrayBuilder {
|
| 34 |
+
public:
|
| 35 |
+
virtual ~ChunkedArrayBuilder() = default;
|
| 36 |
+
|
| 37 |
+
/// Spawn a task that will try to convert and insert the given JSON block
|
| 38 |
+
virtual void Insert(int64_t block_index,
|
| 39 |
+
const std::shared_ptr<Field>& unconverted_field,
|
| 40 |
+
const std::shared_ptr<Array>& unconverted) = 0;
|
| 41 |
+
|
| 42 |
+
/// Return the final chunked array.
|
| 43 |
+
/// Every chunk must be inserted before this is called!
|
| 44 |
+
virtual Status Finish(std::shared_ptr<ChunkedArray>* out) = 0;
|
| 45 |
+
|
| 46 |
+
/// Finish current task group and substitute a new one
|
| 47 |
+
virtual Status ReplaceTaskGroup(
|
| 48 |
+
const std::shared_ptr<arrow::internal::TaskGroup>& task_group) = 0;
|
| 49 |
+
|
| 50 |
+
protected:
|
| 51 |
+
explicit ChunkedArrayBuilder(
|
| 52 |
+
const std::shared_ptr<arrow::internal::TaskGroup>& task_group)
|
| 53 |
+
: task_group_(task_group) {}
|
| 54 |
+
|
| 55 |
+
std::shared_ptr<arrow::internal::TaskGroup> task_group_;
|
| 56 |
+
};
|
| 57 |
+
|
| 58 |
+
/// create a chunked builder
|
| 59 |
+
///
|
| 60 |
+
/// if unexpected fields and promotion need to be handled, promotion_graph must be
|
| 61 |
+
/// non-null
|
| 62 |
+
ARROW_EXPORT Status MakeChunkedArrayBuilder(
|
| 63 |
+
const std::shared_ptr<arrow::internal::TaskGroup>& task_group, MemoryPool* pool,
|
| 64 |
+
const PromotionGraph* promotion_graph, const std::shared_ptr<DataType>& type,
|
| 65 |
+
std::shared_ptr<ChunkedArrayBuilder>* out);
|
| 66 |
+
|
| 67 |
+
} // namespace json
|
| 68 |
+
} // namespace arrow
|
loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/chunker.h
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Licensed to the Apache Software Foundation (ASF) under one
|
| 2 |
+
// or more contributor license agreements. See the NOTICE file
|
| 3 |
+
// distributed with this work for additional information
|
| 4 |
+
// regarding copyright ownership. The ASF licenses this file
|
| 5 |
+
// to you under the Apache License, Version 2.0 (the
|
| 6 |
+
// "License"); you may not use this file except in compliance
|
| 7 |
+
// with the License. You may obtain a copy of the License at
|
| 8 |
+
//
|
| 9 |
+
// http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
//
|
| 11 |
+
// Unless required by applicable law or agreed to in writing,
|
| 12 |
+
// software distributed under the License is distributed on an
|
| 13 |
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 14 |
+
// KIND, either express or implied. See the License for the
|
| 15 |
+
// specific language governing permissions and limitations
|
| 16 |
+
// under the License.
|
| 17 |
+
|
| 18 |
+
#pragma once
|
| 19 |
+
|
| 20 |
+
#include <memory>
|
| 21 |
+
|
| 22 |
+
#include "arrow/util/delimiting.h"
|
| 23 |
+
#include "arrow/util/macros.h"
|
| 24 |
+
#include "arrow/util/visibility.h"
|
| 25 |
+
|
| 26 |
+
namespace arrow {
|
| 27 |
+
namespace json {
|
| 28 |
+
|
| 29 |
+
struct ParseOptions;
|
| 30 |
+
|
| 31 |
+
ARROW_EXPORT
|
| 32 |
+
std::unique_ptr<Chunker> MakeChunker(const ParseOptions& options);
|
| 33 |
+
|
| 34 |
+
} // namespace json
|
| 35 |
+
} // namespace arrow
|