diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client.h new file mode 100644 index 0000000000000000000000000000000000000000..5e984edf7576e4e476ba6745db63b0b676e252b5 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client.h @@ -0,0 +1,434 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/// \brief Implementation of Flight RPC client. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "arrow/ipc/options.h" +#include "arrow/ipc/reader.h" +#include "arrow/ipc/writer.h" +#include "arrow/result.h" +#include "arrow/status.h" +#include "arrow/util/cancel.h" + +#include "arrow/flight/type_fwd.h" +#include "arrow/flight/types.h" // IWYU pragma: keep +#include "arrow/flight/visibility.h" + +namespace arrow { + +class RecordBatch; +class Schema; + +namespace flight { + +/// \brief A duration type for Flight call timeouts. +typedef std::chrono::duration TimeoutDuration; + +/// \brief Hints to the underlying RPC layer for Arrow Flight calls. +class ARROW_FLIGHT_EXPORT FlightCallOptions { + public: + /// Create a default set of call options. + FlightCallOptions(); + + /// \brief An optional timeout for this call. Negative durations + /// mean an implementation-defined default behavior will be used + /// instead. This is the default value. + TimeoutDuration timeout; + + /// \brief IPC reader options, if applicable for the call. + ipc::IpcReadOptions read_options; + + /// \brief IPC writer options, if applicable for the call. + ipc::IpcWriteOptions write_options; + + /// \brief Headers for client to add to context. + std::vector> headers; + + /// \brief A token to enable interactive user cancellation of long-running requests. + StopToken stop_token; + + /// \brief An optional memory manager to control where to allocate incoming data. + std::shared_ptr memory_manager; +}; + +/// \brief Indicate that the client attempted to write a message +/// larger than the soft limit set via write_size_limit_bytes. +class ARROW_FLIGHT_EXPORT FlightWriteSizeStatusDetail : public arrow::StatusDetail { + public: + explicit FlightWriteSizeStatusDetail(int64_t limit, int64_t actual) + : limit_(limit), actual_(actual) {} + const char* type_id() const override; + std::string ToString() const override; + int64_t limit() const { return limit_; } + int64_t actual() const { return actual_; } + + /// \brief Extract this status detail from a status, or return + /// nullptr if the status doesn't contain this status detail. + static std::shared_ptr UnwrapStatus( + const arrow::Status& status); + + private: + int64_t limit_; + int64_t actual_; +}; + +struct ARROW_FLIGHT_EXPORT FlightClientOptions { + /// \brief Root certificates to use for validating server + /// certificates. + std::string tls_root_certs; + /// \brief Override the hostname checked by TLS. Use with caution. + std::string override_hostname; + /// \brief The client certificate to use if using Mutual TLS + std::string cert_chain; + /// \brief The private key associated with the client certificate for Mutual TLS + std::string private_key; + /// \brief A list of client middleware to apply. + std::vector> middleware; + /// \brief A soft limit on the number of bytes to write in a single + /// batch when sending Arrow data to a server. + /// + /// Used to help limit server memory consumption. Only enabled if + /// positive. When enabled, FlightStreamWriter.Write* may yield a + /// IOError with error detail FlightWriteSizeStatusDetail. + int64_t write_size_limit_bytes = 0; + + /// \brief Generic connection options, passed to the underlying + /// transport; interpretation is implementation-dependent. + std::vector>> generic_options; + + /// \brief Use TLS without validating the server certificate. Use with caution. + bool disable_server_verification = false; + + /// \brief Get default options. + static FlightClientOptions Defaults(); +}; + +/// \brief A RecordBatchReader exposing Flight metadata and cancel +/// operations. +class ARROW_FLIGHT_EXPORT FlightStreamReader : public MetadataRecordBatchReader { + public: + /// \brief Try to cancel the call. + virtual void Cancel() = 0; + + using MetadataRecordBatchReader::ToRecordBatches; + /// \brief Consume entire stream as a vector of record batches + virtual arrow::Result>> ToRecordBatches( + const StopToken& stop_token) = 0; + + using MetadataRecordBatchReader::ToTable; + /// \brief Consume entire stream as a Table + arrow::Result> ToTable(const StopToken& stop_token); + + using MetadataRecordBatchReader::stats; + /// \brief Return current read statistics + virtual arrow::ipc::ReadStats stats() const = 0; +}; + +// Silence warning +// "non dll-interface class RecordBatchReader used as base for dll-interface class" +#ifdef _MSC_VER +# pragma warning(push) +# pragma warning(disable : 4275) +#endif + +/// \brief A RecordBatchWriter that also allows sending +/// application-defined metadata via the Flight protocol. +class ARROW_FLIGHT_EXPORT FlightStreamWriter : public MetadataRecordBatchWriter { + public: + /// \brief Indicate that the application is done writing to this stream. + /// + /// The application may not write to this stream after calling + /// this. This differs from closing the stream because this writer + /// may represent only one half of a readable and writable stream. + virtual Status DoneWriting() = 0; +}; + +#ifdef _MSC_VER +# pragma warning(pop) +#endif + +/// \brief A reader for application-specific metadata sent back to the +/// client during an upload. +class ARROW_FLIGHT_EXPORT FlightMetadataReader { + public: + virtual ~FlightMetadataReader(); + /// \brief Read a message from the server. + virtual Status ReadMetadata(std::shared_ptr* out) = 0; +}; + +/// \brief Client class for Arrow Flight RPC services. +class ARROW_FLIGHT_EXPORT FlightClient { + public: + ~FlightClient(); + + /// \brief Connect to an unauthenticated flight service + /// \param[in] location the URI + /// \return Arrow result with the created FlightClient, OK status may not indicate that + /// the connection was successful + static arrow::Result> Connect(const Location& location); + + /// \brief Connect to an unauthenticated flight service + /// \param[in] location the URI + /// \param[in] options Other options for setting up the client + /// \return Arrow result with the created FlightClient, OK status may not indicate that + /// the connection was successful + static arrow::Result> Connect( + const Location& location, const FlightClientOptions& options); + + /// \brief Authenticate to the server using the given handler. + /// \param[in] options Per-RPC options + /// \param[in] auth_handler The authentication mechanism to use + /// \return Status OK if the client authenticated successfully + Status Authenticate(const FlightCallOptions& options, + std::unique_ptr auth_handler); + + /// \brief Authenticate to the server using basic HTTP style authentication. + /// \param[in] options Per-RPC options + /// \param[in] username Username to use + /// \param[in] password Password to use + /// \return Arrow result with bearer token and status OK if client authenticated + /// successfully + arrow::Result> AuthenticateBasicToken( + const FlightCallOptions& options, const std::string& username, + const std::string& password); + + /// \brief Perform the indicated action, returning an iterator to the stream + /// of results, if any + /// \param[in] options Per-RPC options + /// \param[in] action the action to be performed + /// \return Arrow result with an iterator object for reading the returned results + arrow::Result> DoAction(const FlightCallOptions& options, + const Action& action); + arrow::Result> DoAction(const Action& action) { + return DoAction({}, action); + } + + /// \brief Perform the CancelFlightInfo action, returning a + /// CancelFlightInfoResult + /// + /// \param[in] options Per-RPC options + /// \param[in] request The CancelFlightInfoRequest + /// \return Arrow result with a CancelFlightInfoResult + arrow::Result CancelFlightInfo( + const FlightCallOptions& options, const CancelFlightInfoRequest& request); + arrow::Result CancelFlightInfo( + const CancelFlightInfoRequest& request) { + return CancelFlightInfo({}, request); + } + + /// \brief Perform the RenewFlightEndpoint action, returning a renewed + /// FlightEndpoint + /// + /// \param[in] options Per-RPC options + /// \param[in] request The RenewFlightEndpointRequest + /// \return Arrow result with a renewed FlightEndpoint + arrow::Result RenewFlightEndpoint( + const FlightCallOptions& options, const RenewFlightEndpointRequest& request); + arrow::Result RenewFlightEndpoint( + const RenewFlightEndpointRequest& request) { + return RenewFlightEndpoint({}, request); + } + + /// \brief Retrieve a list of available Action types + /// \param[in] options Per-RPC options + /// \return Arrow result with the available actions + arrow::Result> ListActions(const FlightCallOptions& options); + arrow::Result> ListActions() { + return ListActions(FlightCallOptions()); + } + + /// \brief Request access plan for a single flight, which may be an existing + /// dataset or a command to be executed + /// \param[in] options Per-RPC options + /// \param[in] descriptor the dataset request, whether a named dataset or + /// command + /// \return Arrow result with the FlightInfo describing where to access the dataset + arrow::Result> GetFlightInfo( + const FlightCallOptions& options, const FlightDescriptor& descriptor); + arrow::Result> GetFlightInfo( + const FlightDescriptor& descriptor) { + return GetFlightInfo({}, descriptor); + } + + /// \brief Asynchronous GetFlightInfo. + /// \param[in] options Per-RPC options + /// \param[in] descriptor the dataset request + /// \param[in] listener Callbacks for response and RPC completion + void GetFlightInfoAsync(const FlightCallOptions& options, + const FlightDescriptor& descriptor, + std::shared_ptr> listener); + void GetFlightInfoAsync(const FlightDescriptor& descriptor, + std::shared_ptr> listener) { + return GetFlightInfoAsync({}, descriptor, std::move(listener)); + } + + /// \brief Asynchronous GetFlightInfo returning a Future. + /// \param[in] options Per-RPC options + /// \param[in] descriptor the dataset request + arrow::Future GetFlightInfoAsync(const FlightCallOptions& options, + const FlightDescriptor& descriptor); + arrow::Future GetFlightInfoAsync(const FlightDescriptor& descriptor) { + return GetFlightInfoAsync({}, descriptor); + } + + /// \brief Request and poll a long running query + /// \param[in] options Per-RPC options + /// \param[in] descriptor the dataset request or a descriptor returned by a + /// prior PollFlightInfo call + /// \return Arrow result with the PollInfo describing the status of + /// the requested query + arrow::Result> PollFlightInfo( + const FlightCallOptions& options, const FlightDescriptor& descriptor); + arrow::Result> PollFlightInfo( + const FlightDescriptor& descriptor) { + return PollFlightInfo({}, descriptor); + } + + /// \brief Request schema for a single flight, which may be an existing + /// dataset or a command to be executed + /// \param[in] options Per-RPC options + /// \param[in] descriptor the dataset request, whether a named dataset or + /// command + /// \return Arrow result with the SchemaResult describing the dataset schema + arrow::Result> GetSchema( + const FlightCallOptions& options, const FlightDescriptor& descriptor); + + arrow::Result> GetSchema( + const FlightDescriptor& descriptor) { + return GetSchema({}, descriptor); + } + + /// \brief List all available flights known to the server + /// \return Arrow result with an iterator that returns a FlightInfo for each flight + arrow::Result> ListFlights(); + + /// \brief List available flights given indicated filter criteria + /// \param[in] options Per-RPC options + /// \param[in] criteria the filter criteria (opaque) + /// \return Arrow result with an iterator that returns a FlightInfo for each flight + arrow::Result> ListFlights( + const FlightCallOptions& options, const Criteria& criteria); + + /// \brief Given a flight ticket and schema, request to be sent the + /// stream. Returns record batch stream reader + /// \param[in] options Per-RPC options + /// \param[in] ticket The flight ticket to use + /// \return Arrow result with the returned RecordBatchReader + arrow::Result> DoGet( + const FlightCallOptions& options, const Ticket& ticket); + arrow::Result> DoGet(const Ticket& ticket) { + return DoGet({}, ticket); + } + + /// \brief DoPut return value + struct DoPutResult { + /// \brief a writer to write record batches to + std::unique_ptr writer; + /// \brief a reader for application metadata from the server + std::unique_ptr reader; + }; + /// \brief Upload data to a Flight described by the given + /// descriptor. The caller must call Close() on the returned stream + /// once they are done writing. + /// + /// The reader and writer are linked; closing the writer will also + /// close the reader. Use \a DoneWriting to only close the write + /// side of the channel. + /// + /// \param[in] options Per-RPC options + /// \param[in] descriptor the descriptor of the stream + /// \param[in] schema the schema for the data to upload + /// \return Arrow result with a DoPutResult struct holding a reader and a writer + arrow::Result DoPut(const FlightCallOptions& options, + const FlightDescriptor& descriptor, + const std::shared_ptr& schema); + + arrow::Result DoPut(const FlightDescriptor& descriptor, + const std::shared_ptr& schema) { + return DoPut({}, descriptor, schema); + } + + struct DoExchangeResult { + std::unique_ptr writer; + std::unique_ptr reader; + }; + arrow::Result DoExchange(const FlightCallOptions& options, + const FlightDescriptor& descriptor); + arrow::Result DoExchange(const FlightDescriptor& descriptor) { + return DoExchange({}, descriptor); + } + + /// \brief Set server session option(s) by name/value. Sessions are generally + /// persisted via HTTP cookies. + /// \param[in] options Per-RPC options + /// \param[in] request The server session options to set + ::arrow::Result SetSessionOptions( + const FlightCallOptions& options, const SetSessionOptionsRequest& request); + + /// \brief Get the current server session options. The session is generally + /// accessed via an HTTP cookie. + /// \param[in] options Per-RPC options + /// \param[in] request The (empty) GetSessionOptions request object. + ::arrow::Result GetSessionOptions( + const FlightCallOptions& options, const GetSessionOptionsRequest& request); + + /// \brief Close/invalidate the current server session. The session is generally + /// accessed via an HTTP cookie. + /// \param[in] options Per-RPC options + /// \param[in] request The (empty) CloseSession request object. + ::arrow::Result CloseSession(const FlightCallOptions& options, + const CloseSessionRequest& request); + + /// \brief Explicitly shut down and clean up the client. + /// + /// For backwards compatibility, this will be implicitly called by + /// the destructor if not already called, but this gives the + /// application no chance to handle errors, so it is recommended to + /// explicitly close the client. + /// + /// \since 8.0.0 + Status Close(); + + /// \brief Whether this client supports asynchronous methods. + bool supports_async() const; + + /// \brief Check whether this client supports asynchronous methods. + /// + /// This is like supports_async(), except that a detailed error message + /// is returned if async support is not available. If async support is + /// available, this function returns successfully. + Status CheckAsyncSupport() const; + + private: + FlightClient(); + Status CheckOpen() const; + std::unique_ptr transport_; + bool closed_; + int64_t write_size_limit_bytes_; +}; + +} // namespace flight +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_auth.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_auth.h new file mode 100644 index 0000000000000000000000000000000000000000..476128f595c243f0a2de6a381084db2bc12d9048 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_auth.h @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "arrow/flight/visibility.h" +#include "arrow/status.h" + +namespace arrow { + +namespace flight { + +/// \brief A reader for messages from the server during an +/// authentication handshake. +class ARROW_FLIGHT_EXPORT ClientAuthReader { + public: + virtual ~ClientAuthReader() = default; + virtual Status Read(std::string* response) = 0; +}; + +/// \brief A writer for messages to the server during an +/// authentication handshake. +class ARROW_FLIGHT_EXPORT ClientAuthSender { + public: + virtual ~ClientAuthSender() = default; + virtual Status Write(const std::string& token) = 0; +}; + +/// \brief An authentication implementation for a Flight service. +/// Authentication includes both an initial negotiation and a per-call +/// token validation. Implementations may choose to use either or both +/// mechanisms. +class ARROW_FLIGHT_EXPORT ClientAuthHandler { + public: + virtual ~ClientAuthHandler() = default; + /// \brief Authenticate the client on initial connection. The client + /// can send messages to/read responses from the server at any time. + /// \return Status OK if authenticated successfully + virtual Status Authenticate(ClientAuthSender* outgoing, ClientAuthReader* incoming) = 0; + /// \brief Get a per-call token. + /// \param[out] token The token to send to the server. + virtual Status GetToken(std::string* token) = 0; +}; + +} // namespace flight +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_cookie_middleware.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_cookie_middleware.h new file mode 100644 index 0000000000000000000000000000000000000000..622efd04412c80047090775aec2b96ad24637ed0 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_cookie_middleware.h @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Middleware implementation for sending and receiving HTTP cookies. + +#pragma once + +#include + +#include "arrow/flight/client_middleware.h" + +namespace arrow { +namespace flight { + +/// \brief Returns a ClientMiddlewareFactory that handles sending and receiving cookies. +ARROW_FLIGHT_EXPORT std::shared_ptr GetCookieFactory(); + +} // namespace flight +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_middleware.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_middleware.h new file mode 100644 index 0000000000000000000000000000000000000000..6a938e6c3a61b432a1be2e6185f7163690d6ac42 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_middleware.h @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Interfaces for defining middleware for Flight clients. Currently +// experimental. + +#pragma once + +#include + +#include "arrow/flight/middleware.h" +#include "arrow/flight/visibility.h" // IWYU pragma: keep +#include "arrow/status.h" + +namespace arrow { +namespace flight { + +/// \brief Client-side middleware for a call, instantiated per RPC. +/// +/// Middleware should be fast and must be infallible: there is no way +/// to reject the call or report errors from the middleware instance. +class ARROW_FLIGHT_EXPORT ClientMiddleware { + public: + virtual ~ClientMiddleware() = default; + + /// \brief A callback before headers are sent. Extra headers can be + /// added, but existing ones cannot be read. + virtual void SendingHeaders(AddCallHeaders* outgoing_headers) = 0; + + /// \brief A callback when headers are received from the server. + /// + /// This may be called more than once, since servers send both + /// headers and trailers. Some implementations (e.g. gRPC-Java, and + /// hence Arrow Flight in Java) may consolidate headers into + /// trailers if the RPC errored. + virtual void ReceivedHeaders(const CallHeaders& incoming_headers) = 0; + + /// \brief A callback after the call has completed. + virtual void CallCompleted(const Status& status) = 0; +}; + +/// \brief A factory for new middleware instances. +/// +/// If added to a client, this will be called for each RPC (including +/// Handshake) to give the opportunity to intercept the call. +/// +/// It is guaranteed that all client middleware methods are called +/// from the same thread that calls the RPC method implementation. +class ARROW_FLIGHT_EXPORT ClientMiddlewareFactory { + public: + virtual ~ClientMiddlewareFactory() = default; + + /// \brief A callback for the start of a new call. + /// + /// \param info Information about the call. + /// \param[out] middleware The middleware instance for this call. If + /// unset, will not add middleware to this call instance from + /// this factory. + virtual void StartCall(const CallInfo& info, + std::unique_ptr* middleware) = 0; +}; + +} // namespace flight +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_tracing_middleware.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_tracing_middleware.h new file mode 100644 index 0000000000000000000000000000000000000000..f7bd232a8d2b4beab5a57a94d5a2d1867031a3ea --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/client_tracing_middleware.h @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Middleware implementation for propagating OpenTelemetry spans. + +#pragma once + +#include + +#include "arrow/flight/client_middleware.h" + +namespace arrow { +namespace flight { + +/// \brief Returns a ClientMiddlewareFactory that handles sending OpenTelemetry spans. +ARROW_FLIGHT_EXPORT std::shared_ptr +MakeTracingClientMiddlewareFactory(); + +} // namespace flight +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/middleware.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/middleware.h new file mode 100644 index 0000000000000000000000000000000000000000..753c4522848e5551d18cde74038b21f0ea6d5cd7 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/middleware.h @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Interfaces for defining middleware for Flight clients and +// servers. + +#pragma once + +#include +#include +#include +#include + +#include "arrow/flight/types.h" +#include "arrow/status.h" + +namespace arrow { +namespace flight { + +/// \brief A write-only wrapper around headers for an RPC call. +class ARROW_FLIGHT_EXPORT AddCallHeaders { + public: + virtual ~AddCallHeaders() = default; + + /// \brief Add a header to be sent to the client. + /// + /// \param[in] key The header name. Must be lowercase ASCII; some + /// transports may reject invalid header names. + /// \param[in] value The header value. Some transports may only + /// accept binary header values if the header name ends in "-bin". + virtual void AddHeader(const std::string& key, const std::string& value) = 0; +}; + +/// \brief An enumeration of the RPC methods Flight implements. +enum class FlightMethod : char { + Invalid = 0, + Handshake = 1, + ListFlights = 2, + GetFlightInfo = 3, + GetSchema = 4, + DoGet = 5, + DoPut = 6, + DoAction = 7, + ListActions = 8, + DoExchange = 9, + PollFlightInfo = 10, +}; + +/// \brief Get a human-readable name for a Flight method. +ARROW_FLIGHT_EXPORT +std::string ToString(FlightMethod method); + +/// \brief Information about an instance of a Flight RPC. +struct ARROW_FLIGHT_EXPORT CallInfo { + public: + /// \brief The RPC method of this call. + FlightMethod method; +}; + +} // namespace flight +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/otel_logging.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/otel_logging.h new file mode 100644 index 0000000000000000000000000000000000000000..953a5c67cfea96c0f0e7920413f3997489337ecb --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/otel_logging.h @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "arrow/util/config.h" + +#ifdef ARROW_WITH_OPENTELEMETRY +# include "arrow/status.h" +# include "arrow/telemetry/logging.h" +# include "arrow/util/macros.h" + +namespace arrow::flight { + +ARROW_EXPORT Status +RegisterFlightOtelLoggers(const telemetry::OtelLoggingOptions& options); + +} // namespace arrow::flight +#endif diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/platform.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/platform.h new file mode 100644 index 0000000000000000000000000000000000000000..9ed3eb2a13c137b6315f0b160a226136a1d8eaec --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/platform.h @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Internal header. Platform-specific definitions for Flight. + +#pragma once + +#ifdef _MSC_VER + +// The protobuf documentation says that C4251 warnings when using the +// library are spurious and suppressed when the build the library and +// compiler, but must be also suppressed in downstream projects +# pragma warning(disable : 4251) + +#endif // _MSC_VER + +#include "arrow/util/config.h" // IWYU pragma: keep diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server.h new file mode 100644 index 0000000000000000000000000000000000000000..7ba0c7a22352eeba5f498425c78570fcc829fa95 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server.h @@ -0,0 +1,327 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Interfaces to use for defining Flight RPC servers. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "arrow/flight/server_auth.h" +#include "arrow/flight/type_fwd.h" +#include "arrow/flight/types.h" // IWYU pragma: keep +#include "arrow/flight/visibility.h" // IWYU pragma: keep +#include "arrow/ipc/dictionary.h" +#include "arrow/ipc/options.h" +#include "arrow/record_batch.h" + +namespace arrow { + +class Schema; +class Status; + +namespace flight { + +/// \brief Interface that produces a sequence of IPC payloads to be sent in +/// FlightData protobuf messages +class ARROW_FLIGHT_EXPORT FlightDataStream { + public: + virtual ~FlightDataStream(); + + virtual std::shared_ptr schema() = 0; + + /// \brief Compute FlightPayload containing serialized RecordBatch schema + virtual arrow::Result GetSchemaPayload() = 0; + + // When the stream is completed, the last payload written will have null + // metadata + virtual arrow::Result Next() = 0; + + virtual Status Close(); +}; + +/// \brief A basic implementation of FlightDataStream that will provide +/// a sequence of FlightData messages to be written to a stream +class ARROW_FLIGHT_EXPORT RecordBatchStream : public FlightDataStream { + public: + /// \param[in] reader produces a sequence of record batches + /// \param[in] options IPC options for writing + explicit RecordBatchStream( + const std::shared_ptr& reader, + const ipc::IpcWriteOptions& options = ipc::IpcWriteOptions::Defaults()); + ~RecordBatchStream() override; + + // inherit deprecated API + using FlightDataStream::GetSchemaPayload; + using FlightDataStream::Next; + + std::shared_ptr schema() override; + arrow::Result GetSchemaPayload() override; + + arrow::Result Next() override; + Status Close() override; + + private: + class RecordBatchStreamImpl; + std::unique_ptr impl_; +}; + +/// \brief A reader for IPC payloads uploaded by a client. Also allows +/// reading application-defined metadata via the Flight protocol. +class ARROW_FLIGHT_EXPORT FlightMessageReader : public MetadataRecordBatchReader { + public: + /// \brief Get the descriptor for this upload. + virtual const FlightDescriptor& descriptor() const = 0; +}; + +/// \brief A writer for application-specific metadata sent back to the +/// client during an upload. +class ARROW_FLIGHT_EXPORT FlightMetadataWriter { + public: + virtual ~FlightMetadataWriter(); + /// \brief Send a message to the client. + virtual Status WriteMetadata(const Buffer& app_metadata) = 0; +}; + +/// \brief A writer for IPC payloads to a client. Also allows sending +/// application-defined metadata via the Flight protocol. +/// +/// This class offers more control compared to FlightDataStream, +/// including the option to write metadata without data and the +/// ability to interleave reading and writing. +class ARROW_FLIGHT_EXPORT FlightMessageWriter : public MetadataRecordBatchWriter { + public: + virtual ~FlightMessageWriter() = default; +}; + +/// \brief Call state/contextual data. +class ARROW_FLIGHT_EXPORT ServerCallContext { + public: + virtual ~ServerCallContext() = default; + /// \brief The name of the authenticated peer (may be the empty string) + virtual const std::string& peer_identity() const = 0; + /// \brief The peer address (not validated) + virtual const std::string& peer() const = 0; + /// \brief Add a response header. This is only valid before the server + /// starts sending the response; generally this isn't an issue unless you + /// are implementing FlightDataStream, ResultStream, or similar interfaces + /// yourself, or during a DoExchange or DoPut. + virtual void AddHeader(const std::string& key, const std::string& value) const = 0; + /// \brief Add a response trailer. This is only valid before the server + /// sends the final status; generally this isn't an issue unless your RPC + /// handler launches a thread or similar. + virtual void AddTrailer(const std::string& key, const std::string& value) const = 0; + /// \brief Look up a middleware by key. Do not maintain a reference + /// to the object beyond the request body. + /// \return The middleware, or nullptr if not found. + virtual ServerMiddleware* GetMiddleware(const std::string& key) const = 0; + /// \brief Check if the current RPC has been cancelled (by the client, by + /// a network error, etc.). + virtual bool is_cancelled() const = 0; + /// \brief The headers sent by the client for this call. + virtual const CallHeaders& incoming_headers() const = 0; +}; + +class ARROW_FLIGHT_EXPORT FlightServerOptions { + public: + explicit FlightServerOptions(const Location& location_); + + ~FlightServerOptions(); + + /// \brief The host & port (or domain socket path) to listen on. + /// Use port 0 to bind to an available port. + Location location; + /// \brief The authentication handler to use. + std::shared_ptr auth_handler; + /// \brief A list of TLS certificate+key pairs to use. + std::vector tls_certificates; + /// \brief Enable mTLS and require that the client present a certificate. + bool verify_client; + /// \brief If using mTLS, the PEM-encoded root certificate to use. + std::string root_certificates; + /// \brief A list of server middleware to apply, along with a key to + /// identify them by. + /// + /// Middleware are always applied in the order provided. Duplicate + /// keys are an error. + std::vector>> + middleware; + + /// \brief An optional memory manager to control where to allocate incoming data. + std::shared_ptr memory_manager; + + /// \brief A Flight implementation-specific callback to customize + /// transport-specific options. + /// + /// Not guaranteed to be called. The type of the parameter is + /// specific to the Flight implementation. Users should take care to + /// link to the same transport implementation as Flight to avoid + /// runtime problems. See "Using Arrow C++ in your own project" in + /// the documentation for more details. + std::function builder_hook; +}; + +/// \brief Skeleton RPC server implementation which can be used to create +/// custom servers by implementing its abstract methods +class ARROW_FLIGHT_EXPORT FlightServerBase { + public: + FlightServerBase(); + virtual ~FlightServerBase(); + + // Lifecycle methods. + + /// \brief Initialize a Flight server listening at the given location. + /// This method must be called before any other method. + /// \param[in] options The configuration for this server. + Status Init(const FlightServerOptions& options); + + /// \brief Get the port that the Flight server is listening on. + /// This method must only be called after Init(). Will return a + /// non-positive value if no port exists (e.g. when listening on a + /// domain socket). + int port() const; + + /// \brief Get the address that the Flight server is listening on. + /// This method must only be called after Init(). + Location location() const; + + /// \brief Set the server to stop when receiving any of the given signal + /// numbers. + /// This method must be called before Serve(). + Status SetShutdownOnSignals(const std::vector sigs); + + /// \brief Start serving. + /// This method blocks until the server shuts down. + /// + /// The server will start to shut down when either Shutdown() is called + /// or one of the signals registered in SetShutdownOnSignals() is received. + Status Serve(); + + /// \brief Query whether Serve() was interrupted by a signal. + /// This method must be called after Serve() has returned. + /// + /// \return int the signal number that interrupted Serve(), if any, otherwise 0 + int GotSignal() const; + + /// \brief Shut down the server, blocking until current requests finish. + /// + /// Can be called from a signal handler or another thread while Serve() + /// blocks. Optionally a deadline can be set. Once the deadline expires + /// server will wait until remaining running calls complete. + /// + /// Should only be called once. + Status Shutdown(const std::chrono::system_clock::time_point* deadline = NULLPTR); + + /// \brief Block until server shuts down with Shutdown. + /// + /// Does not respond to signals like Serve(). + Status Wait(); + + // Implement these methods to create your own server. The default + // implementations will return a not-implemented result to the client + + /// \brief Retrieve a list of available fields given an optional opaque + /// criteria + /// \param[in] context The call context. + /// \param[in] criteria may be null + /// \param[out] listings the returned listings iterator + /// \return Status + virtual Status ListFlights(const ServerCallContext& context, const Criteria* criteria, + std::unique_ptr* listings); + + /// \brief Retrieve the schema and an access plan for the indicated + /// descriptor + /// \param[in] context The call context. + /// \param[in] request the dataset request, whether a named dataset or command + /// \param[out] info the returned flight info provider + /// \return Status + virtual Status GetFlightInfo(const ServerCallContext& context, + const FlightDescriptor& request, + std::unique_ptr* info); + + /// \brief Retrieve the current status of the target query + /// \param[in] context The call context. + /// \param[in] request the dataset request or a descriptor returned by a + /// prior PollFlightInfo call + /// \param[out] info the returned retry info provider + /// \return Status + virtual Status PollFlightInfo(const ServerCallContext& context, + const FlightDescriptor& request, + std::unique_ptr* info); + + /// \brief Retrieve the schema for the indicated descriptor + /// \param[in] context The call context. + /// \param[in] request the dataset request, whether a named dataset or command + /// \param[out] schema the returned flight schema provider + /// \return Status + virtual Status GetSchema(const ServerCallContext& context, + const FlightDescriptor& request, + std::unique_ptr* schema); + + /// \brief Get a stream of IPC payloads to put on the wire + /// \param[in] context The call context. + /// \param[in] request an opaque ticket + /// \param[out] stream the returned stream provider + /// \return Status + virtual Status DoGet(const ServerCallContext& context, const Ticket& request, + std::unique_ptr* stream); + + /// \brief Process a stream of IPC payloads sent from a client + /// \param[in] context The call context. + /// \param[in] reader a sequence of uploaded record batches + /// \param[in] writer send metadata back to the client + /// \return Status + virtual Status DoPut(const ServerCallContext& context, + std::unique_ptr reader, + std::unique_ptr writer); + + /// \brief Process a bidirectional stream of IPC payloads + /// \param[in] context The call context. + /// \param[in] reader a sequence of uploaded record batches + /// \param[in] writer send data back to the client + /// \return Status + virtual Status DoExchange(const ServerCallContext& context, + std::unique_ptr reader, + std::unique_ptr writer); + + /// \brief Execute an action, return stream of zero or more results + /// \param[in] context The call context. + /// \param[in] action the action to execute, with type and body + /// \param[out] result the result iterator + /// \return Status + virtual Status DoAction(const ServerCallContext& context, const Action& action, + std::unique_ptr* result); + + /// \brief Retrieve the list of available actions + /// \param[in] context The call context. + /// \param[out] actions a vector of available action types + /// \return Status + virtual Status ListActions(const ServerCallContext& context, + std::vector* actions); + + private: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace flight +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server_auth.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server_auth.h new file mode 100644 index 0000000000000000000000000000000000000000..406d6a9060e6404194222090e973a14ba9c84528 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server_auth.h @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/// \brief Server-side APIs to implement authentication for Flight. + +#pragma once + +#include + +#include "arrow/flight/type_fwd.h" +#include "arrow/flight/visibility.h" +#include "arrow/status.h" + +namespace arrow { + +namespace flight { + +/// \brief A reader for messages from the client during an +/// authentication handshake. +class ARROW_FLIGHT_EXPORT ServerAuthReader { + public: + virtual ~ServerAuthReader() = default; + virtual Status Read(std::string* token) = 0; +}; + +/// \brief A writer for messages to the client during an +/// authentication handshake. +class ARROW_FLIGHT_EXPORT ServerAuthSender { + public: + virtual ~ServerAuthSender() = default; + virtual Status Write(const std::string& message) = 0; +}; + +/// \brief An authentication implementation for a Flight service. +/// Authentication includes both an initial negotiation and a per-call +/// token validation. Implementations may choose to use either or both +/// mechanisms. +/// An implementation may need to track some state, e.g. a mapping of +/// client tokens to authenticated identities. +class ARROW_FLIGHT_EXPORT ServerAuthHandler { + public: + virtual ~ServerAuthHandler(); + /// \brief Authenticate the client on initial connection. The server + /// can send and read responses from the client at any time. + /// \param[in] context The call context. + /// \param[in] outgoing The writer for messages to the client. + /// \param[in] incoming The reader for messages from the client. + /// \return Status OK if this authentication is succeeded. + virtual Status Authenticate(const ServerCallContext& context, + ServerAuthSender* outgoing, ServerAuthReader* incoming) = 0; + /// \brief Validate a per-call client token. + /// \param[in] context The call context. + /// \param[in] token The client token. May be the empty string if + /// the client does not provide a token. + /// \param[out] peer_identity The identity of the peer, if this + /// authentication method supports it. + /// \return Status OK if the token is valid, any other status if + /// validation failed + virtual Status IsValid(const ServerCallContext& context, const std::string& token, + std::string* peer_identity) { + // TODO: We can make this pure virtual function when we remove + // the deprecated version. + ARROW_SUPPRESS_DEPRECATION_WARNING + return IsValid(token, peer_identity); + ARROW_UNSUPPRESS_DEPRECATION_WARNING + } + /// \brief Validate a per-call client token. + /// \param[in] token The client token. May be the empty string if + /// the client does not provide a token. + /// \param[out] peer_identity The identity of the peer, if this + /// authentication method supports it. + /// \return Status OK if the token is valid, any other status if + /// validation failed + /// \deprecated Deprecated in 13.0.0. Implement the IsValid() + /// with ServerCallContext version instead. + ARROW_DEPRECATED("Deprecated in 13.0.0. Use ServerCallContext overload instead.") + virtual Status IsValid(const std::string& token, std::string* peer_identity) { + return Status::NotImplemented(typeid(this).name(), "::IsValid() isn't implemented"); + } +}; + +/// \brief An authentication mechanism that does nothing. +class ARROW_FLIGHT_EXPORT NoOpAuthHandler : public ServerAuthHandler { + public: + ~NoOpAuthHandler() override; + Status Authenticate(const ServerCallContext& context, ServerAuthSender* outgoing, + ServerAuthReader* incoming) override; + Status IsValid(const ServerCallContext& context, const std::string& token, + std::string* peer_identity) override; +}; + +} // namespace flight +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server_middleware.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server_middleware.h new file mode 100644 index 0000000000000000000000000000000000000000..755727d1fccbe01434a54e54941585cb8390cb8e --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server_middleware.h @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Interfaces for defining middleware for Flight servers. + +#pragma once + +#include +#include + +#include "arrow/flight/middleware.h" +#include "arrow/flight/type_fwd.h" +#include "arrow/flight/visibility.h" // IWYU pragma: keep +#include "arrow/status.h" + +namespace arrow { +namespace flight { + +/// \brief Server-side middleware for a call, instantiated per RPC. +/// +/// Middleware should be fast and must be infallible: there is no way +/// to reject the call or report errors from the middleware instance. +class ARROW_FLIGHT_EXPORT ServerMiddleware { + public: + virtual ~ServerMiddleware() = default; + + /// \brief Unique name of middleware, used as alternative to RTTI + /// \return the string name of the middleware + virtual std::string name() const = 0; + + /// \brief A callback before headers are sent. Extra headers can be + /// added, but existing ones cannot be read. + virtual void SendingHeaders(AddCallHeaders* outgoing_headers) = 0; + + /// \brief A callback after the call has completed. + virtual void CallCompleted(const Status& status) = 0; +}; + +/// \brief A factory for new middleware instances. +/// +/// If added to a server, this will be called for each RPC (including +/// Handshake) to give the opportunity to intercept the call. +/// +/// It is guaranteed that all server middleware methods are called +/// from the same thread that calls the RPC method implementation. +class ARROW_FLIGHT_EXPORT ServerMiddlewareFactory { + public: + virtual ~ServerMiddlewareFactory() = default; + + /// \brief A callback for the start of a new call. + /// + /// Return a non-OK status to reject the call with the given status. + /// + /// \param[in] info Information about the call. + /// \param[in] context The call context. + /// \param[out] middleware The middleware instance for this call. If + /// null, no middleware will be added to this call instance from + /// this factory. + /// \return Status A non-OK status will reject the call with the + /// given status. Middleware previously in the chain will have + /// their CallCompleted callback called. Other middleware + /// factories will not be called. + virtual Status StartCall(const CallInfo& info, const ServerCallContext& context, + std::shared_ptr* middleware) = 0; +}; + +} // namespace flight +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server_tracing_middleware.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server_tracing_middleware.h new file mode 100644 index 0000000000000000000000000000000000000000..2113f6e940dee82894e2ddac88ffa0142dc8b67d --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/server_tracing_middleware.h @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Middleware implementation for propagating OpenTelemetry spans. + +#pragma once + +#include +#include +#include + +#include "arrow/flight/server_middleware.h" +#include "arrow/flight/visibility.h" +#include "arrow/status.h" + +namespace arrow { +namespace flight { + +/// \brief Returns a ServerMiddlewareFactory that handles receiving OpenTelemetry spans. +ARROW_FLIGHT_EXPORT std::shared_ptr +MakeTracingServerMiddlewareFactory(); + +/// \brief A server middleware that provides access to the +/// OpenTelemetry context, if present. +/// +/// Used to make the OpenTelemetry span available in Python. +class ARROW_FLIGHT_EXPORT TracingServerMiddleware : public ServerMiddleware { + public: + ~TracingServerMiddleware(); + + static constexpr const char kMiddlewareName[] = + "arrow::flight::TracingServerMiddleware"; + + std::string name() const override { return kMiddlewareName; } + void SendingHeaders(AddCallHeaders*) override; + void CallCompleted(const Status&) override; + + struct TraceKey { + std::string key; + std::string value; + }; + /// \brief Get the trace context. + std::vector GetTraceContext() const; + + private: + class Impl; + friend class TracingServerMiddlewareFactory; + + explicit TracingServerMiddleware(std::unique_ptr impl); + std::unique_ptr impl_; +}; + +} // namespace flight +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_auth_handlers.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_auth_handlers.h new file mode 100644 index 0000000000000000000000000000000000000000..47ebf28d4a1689729f7c4843385142d99966deff --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_auth_handlers.h @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "arrow/flight/client_auth.h" +#include "arrow/flight/server.h" +#include "arrow/flight/server_auth.h" +#include "arrow/flight/types.h" +#include "arrow/flight/visibility.h" +#include "arrow/status.h" + +// A pair of authentication handlers that check for a predefined password +// and set the peer identity to a predefined username. + +namespace arrow::flight { + +class ARROW_FLIGHT_EXPORT TestServerAuthHandler : public ServerAuthHandler { + public: + explicit TestServerAuthHandler(const std::string& username, + const std::string& password); + ~TestServerAuthHandler() override; + Status Authenticate(const ServerCallContext& context, ServerAuthSender* outgoing, + ServerAuthReader* incoming) override; + Status IsValid(const ServerCallContext& context, const std::string& token, + std::string* peer_identity) override; + + private: + std::string username_; + std::string password_; +}; + +class ARROW_FLIGHT_EXPORT TestServerBasicAuthHandler : public ServerAuthHandler { + public: + explicit TestServerBasicAuthHandler(const std::string& username, + const std::string& password); + ~TestServerBasicAuthHandler() override; + Status Authenticate(const ServerCallContext& context, ServerAuthSender* outgoing, + ServerAuthReader* incoming) override; + Status IsValid(const ServerCallContext& context, const std::string& token, + std::string* peer_identity) override; + + private: + BasicAuth basic_auth_; +}; + +class ARROW_FLIGHT_EXPORT TestClientAuthHandler : public ClientAuthHandler { + public: + explicit TestClientAuthHandler(const std::string& username, + const std::string& password); + ~TestClientAuthHandler() override; + Status Authenticate(ClientAuthSender* outgoing, ClientAuthReader* incoming) override; + Status GetToken(std::string* token) override; + + private: + std::string username_; + std::string password_; +}; + +class ARROW_FLIGHT_EXPORT TestClientBasicAuthHandler : public ClientAuthHandler { + public: + explicit TestClientBasicAuthHandler(const std::string& username, + const std::string& password); + ~TestClientBasicAuthHandler() override; + Status Authenticate(ClientAuthSender* outgoing, ClientAuthReader* incoming) override; + Status GetToken(std::string* token) override; + + private: + BasicAuth basic_auth_; + std::string token_; +}; + +} // namespace arrow::flight diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_definitions.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_definitions.h new file mode 100644 index 0000000000000000000000000000000000000000..c294ea02668e869809c3377acb43ee77d341b2e3 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_definitions.h @@ -0,0 +1,318 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Common test definitions for Flight. Individual transport +// implementations can instantiate these tests. +// +// While Googletest's value-parameterized tests would be a more +// natural way to do this, they cause runtime issues on MinGW/MSVC +// (Googletest thinks the test suite has been defined twice). + +#pragma once + +#include +#include +#include +#include +#include + +#include "arrow/flight/server.h" +#include "arrow/flight/types.h" +#include "arrow/util/macros.h" + +namespace arrow { +namespace flight { + +class ARROW_FLIGHT_EXPORT FlightTest { + protected: + virtual std::string transport() const = 0; + virtual bool supports_async() const { return false; } + virtual void SetUpTest() {} + virtual void TearDownTest() {} +}; + +/// Common tests of startup/shutdown +class ARROW_FLIGHT_EXPORT ConnectivityTest : public FlightTest { + public: + // Test methods + void TestGetPort(); + void TestBuilderHook(); + void TestShutdown(); + void TestShutdownWithDeadline(); + void TestBrokenConnection(); +}; + +#define ARROW_FLIGHT_TEST_CONNECTIVITY(FIXTURE) \ + static_assert(std::is_base_of::value, \ + ARROW_STRINGIFY(FIXTURE) " must inherit from ConnectivityTest"); \ + TEST_F(FIXTURE, GetPort) { TestGetPort(); } \ + TEST_F(FIXTURE, BuilderHook) { TestBuilderHook(); } \ + TEST_F(FIXTURE, Shutdown) { TestShutdown(); } \ + TEST_F(FIXTURE, ShutdownWithDeadline) { TestShutdownWithDeadline(); } \ + TEST_F(FIXTURE, BrokenConnection) { TestBrokenConnection(); } + +/// Common tests of data plane methods +class ARROW_FLIGHT_EXPORT DataTest : public FlightTest { + public: + void SetUpTest() override; + void TearDownTest() override; + Status ConnectClient(); + + // Test methods + void TestDoGetInts(); + void TestDoGetFloats(); + void TestDoGetDicts(); + void TestDoGetLargeBatch(); + void TestFlightDataStreamError(); + void TestOverflowServerBatch(); + void TestOverflowClientBatch(); + void TestDoExchange(); + void TestDoExchangeNoData(); + void TestDoExchangeWriteOnlySchema(); + void TestDoExchangeGet(); + void TestDoExchangePut(); + void TestDoExchangeEcho(); + void TestDoExchangeTotal(); + void TestDoExchangeError(); + void TestDoExchangeConcurrency(); + void TestDoExchangeUndrained(); + void TestIssue5095(); + + private: + void CheckDoGet( + const FlightDescriptor& descr, const RecordBatchVector& expected_batches, + std::function&)> check_endpoints); + void CheckDoGet(const Ticket& ticket, const RecordBatchVector& expected_batches); + + std::unique_ptr client_; + std::unique_ptr server_; +}; + +#define ARROW_FLIGHT_TEST_DATA(FIXTURE) \ + static_assert(std::is_base_of::value, \ + ARROW_STRINGIFY(FIXTURE) " must inherit from DataTest"); \ + TEST_F(FIXTURE, TestDoGetInts) { TestDoGetInts(); } \ + TEST_F(FIXTURE, TestDoGetFloats) { TestDoGetFloats(); } \ + TEST_F(FIXTURE, TestDoGetDicts) { TestDoGetDicts(); } \ + TEST_F(FIXTURE, TestDoGetLargeBatch) { TestDoGetLargeBatch(); } \ + TEST_F(FIXTURE, TestFlightDataStreamError) { TestFlightDataStreamError(); } \ + TEST_F(FIXTURE, TestOverflowServerBatch) { TestOverflowServerBatch(); } \ + TEST_F(FIXTURE, TestOverflowClientBatch) { TestOverflowClientBatch(); } \ + TEST_F(FIXTURE, TestDoExchange) { TestDoExchange(); } \ + TEST_F(FIXTURE, TestDoExchangeNoData) { TestDoExchangeNoData(); } \ + TEST_F(FIXTURE, TestDoExchangeWriteOnlySchema) { TestDoExchangeWriteOnlySchema(); } \ + TEST_F(FIXTURE, TestDoExchangeGet) { TestDoExchangeGet(); } \ + TEST_F(FIXTURE, TestDoExchangePut) { TestDoExchangePut(); } \ + TEST_F(FIXTURE, TestDoExchangeEcho) { TestDoExchangeEcho(); } \ + TEST_F(FIXTURE, TestDoExchangeTotal) { TestDoExchangeTotal(); } \ + TEST_F(FIXTURE, TestDoExchangeError) { TestDoExchangeError(); } \ + TEST_F(FIXTURE, TestDoExchangeConcurrency) { TestDoExchangeConcurrency(); } \ + TEST_F(FIXTURE, TestDoExchangeUndrained) { TestDoExchangeUndrained(); } \ + TEST_F(FIXTURE, TestIssue5095) { TestIssue5095(); } + +/// \brief Specific tests of DoPut. +class ARROW_FLIGHT_EXPORT DoPutTest : public FlightTest { + public: + void SetUpTest() override; + void TearDownTest() override; + void CheckBatches(const FlightDescriptor& expected_descriptor, + const RecordBatchVector& expected_batches); + void CheckDoPut(const FlightDescriptor& descr, const std::shared_ptr& schema, + const RecordBatchVector& batches); + + // Test methods + void TestInts(); + void TestFloats(); + void TestEmptyBatch(); + void TestDicts(); + void TestLargeBatch(); + void TestSizeLimit(); + void TestUndrained(); + + private: + std::unique_ptr client_; + std::unique_ptr server_; +}; + +#define ARROW_FLIGHT_TEST_DO_PUT(FIXTURE) \ + static_assert(std::is_base_of::value, \ + ARROW_STRINGIFY(FIXTURE) " must inherit from DoPutTest"); \ + TEST_F(FIXTURE, TestInts) { TestInts(); } \ + TEST_F(FIXTURE, TestFloats) { TestFloats(); } \ + TEST_F(FIXTURE, TestEmptyBatch) { TestEmptyBatch(); } \ + TEST_F(FIXTURE, TestDicts) { TestDicts(); } \ + TEST_F(FIXTURE, TestLargeBatch) { TestLargeBatch(); } \ + TEST_F(FIXTURE, TestSizeLimit) { TestSizeLimit(); } \ + TEST_F(FIXTURE, TestUndrained) { TestUndrained(); } + +class ARROW_FLIGHT_EXPORT AppMetadataTestServer : public FlightServerBase { + public: + virtual ~AppMetadataTestServer() = default; + + Status DoGet(const ServerCallContext& context, const Ticket& request, + std::unique_ptr* data_stream) override; + + Status DoPut(const ServerCallContext& context, + std::unique_ptr reader, + std::unique_ptr writer) override; +}; + +/// \brief Tests of app_metadata in data plane methods. +class ARROW_FLIGHT_EXPORT AppMetadataTest : public FlightTest { + public: + void SetUpTest() override; + void TearDownTest() override; + + // Test methods + void TestDoGet(); + void TestDoGetDictionaries(); + void TestDoPut(); + void TestDoPutDictionaries(); + void TestDoPutReadMetadata(); + + private: + std::unique_ptr client_; + std::unique_ptr server_; +}; + +#define ARROW_FLIGHT_TEST_APP_METADATA(FIXTURE) \ + static_assert(std::is_base_of::value, \ + ARROW_STRINGIFY(FIXTURE) " must inherit from AppMetadataTest"); \ + TEST_F(FIXTURE, TestDoGet) { TestDoGet(); } \ + TEST_F(FIXTURE, TestDoGetDictionaries) { TestDoGetDictionaries(); } \ + TEST_F(FIXTURE, TestDoPut) { TestDoPut(); } \ + TEST_F(FIXTURE, TestDoPutDictionaries) { TestDoPutDictionaries(); } \ + TEST_F(FIXTURE, TestDoPutReadMetadata) { TestDoPutReadMetadata(); } + +/// \brief Tests of IPC options in data plane methods. +class ARROW_FLIGHT_EXPORT IpcOptionsTest : public FlightTest { + public: + void SetUpTest() override; + void TearDownTest() override; + + // Test methods + void TestDoGetReadOptions(); + void TestDoPutWriteOptions(); + void TestDoExchangeClientWriteOptions(); + void TestDoExchangeClientWriteOptionsBegin(); + void TestDoExchangeServerWriteOptions(); + + private: + std::unique_ptr client_; + std::unique_ptr server_; +}; + +#define ARROW_FLIGHT_TEST_IPC_OPTIONS(FIXTURE) \ + static_assert(std::is_base_of::value, \ + ARROW_STRINGIFY(FIXTURE) " must inherit from IpcOptionsTest"); \ + TEST_F(FIXTURE, TestDoGetReadOptions) { TestDoGetReadOptions(); } \ + TEST_F(FIXTURE, TestDoPutWriteOptions) { TestDoPutWriteOptions(); } \ + TEST_F(FIXTURE, TestDoExchangeClientWriteOptions) { \ + TestDoExchangeClientWriteOptions(); \ + } \ + TEST_F(FIXTURE, TestDoExchangeClientWriteOptionsBegin) { \ + TestDoExchangeClientWriteOptionsBegin(); \ + } \ + TEST_F(FIXTURE, TestDoExchangeServerWriteOptions) { \ + TestDoExchangeServerWriteOptions(); \ + } + +/// \brief Tests of data plane methods with CUDA memory. +/// +/// If not built with ARROW_CUDA, tests are no-ops. +class ARROW_FLIGHT_EXPORT CudaDataTest : public FlightTest { + public: + void SetUpTest() override; + void TearDownTest() override; + + // Test methods + void TestDoGet(); + void TestDoPut(); + void TestDoExchange(); + + private: + class Impl; + std::unique_ptr client_; + std::unique_ptr server_; + std::shared_ptr impl_; +}; + +#define ARROW_FLIGHT_TEST_CUDA_DATA(FIXTURE) \ + static_assert(std::is_base_of::value, \ + ARROW_STRINGIFY(FIXTURE) " must inherit from CudaDataTest"); \ + TEST_F(FIXTURE, TestDoGet) { TestDoGet(); } \ + TEST_F(FIXTURE, TestDoPut) { TestDoPut(); } \ + TEST_F(FIXTURE, TestDoExchange) { TestDoExchange(); } + +/// \brief Tests of error handling. +class ARROW_FLIGHT_EXPORT ErrorHandlingTest : public FlightTest { + public: + void SetUpTest() override; + void TearDownTest() override; + + // Test methods + void TestGetFlightInfo(); + void TestGetFlightInfoMetadata(); + void TestAsyncGetFlightInfo(); + void TestDoPut(); + void TestDoExchange(); + + protected: + struct Impl; + + std::vector> GetHeaders(); + + std::shared_ptr impl_; + std::unique_ptr client_; + std::unique_ptr server_; +}; + +#define ARROW_FLIGHT_TEST_ERROR_HANDLING(FIXTURE) \ + static_assert(std::is_base_of::value, \ + ARROW_STRINGIFY(FIXTURE) " must inherit from ErrorHandlingTest"); \ + TEST_F(FIXTURE, TestAsyncGetFlightInfo) { TestAsyncGetFlightInfo(); } \ + TEST_F(FIXTURE, TestGetFlightInfo) { TestGetFlightInfo(); } \ + TEST_F(FIXTURE, TestGetFlightInfoMetadata) { TestGetFlightInfoMetadata(); } \ + TEST_F(FIXTURE, TestDoPut) { TestDoPut(); } \ + TEST_F(FIXTURE, TestDoExchange) { TestDoExchange(); } + +/// \brief Tests of the async client. +class ARROW_FLIGHT_EXPORT AsyncClientTest : public FlightTest { + public: + void SetUpTest() override; + void TearDownTest() override; + + // Test methods + void TestGetFlightInfo(); + void TestGetFlightInfoFuture(); + void TestListenerLifetime(); + + private: + std::unique_ptr client_; + std::unique_ptr server_; +}; + +// DISABLED TestListenerLifetime: https://github.com/apache/arrow/issues/45120 +#define ARROW_FLIGHT_TEST_ASYNC_CLIENT(FIXTURE) \ + static_assert(std::is_base_of::value, \ + ARROW_STRINGIFY(FIXTURE) " must inherit from AsyncClientTest"); \ + TEST_F(FIXTURE, TestGetFlightInfo) { TestGetFlightInfo(); } \ + TEST_F(FIXTURE, TestGetFlightInfoFuture) { TestGetFlightInfoFuture(); } \ + TEST_F(FIXTURE, DISABLED_TestListenerLifetime) { TestListenerLifetime(); } + +} // namespace flight +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_flight_server.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_flight_server.h new file mode 100644 index 0000000000000000000000000000000000000000..c286d329e8cfd48cd0a5b8e59a91fac0f0fa9c7b --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_flight_server.h @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "arrow/flight/server.h" +#include "arrow/flight/type_fwd.h" +#include "arrow/flight/visibility.h" +#include "arrow/status.h" + +namespace arrow::flight { + +class ARROW_FLIGHT_EXPORT TestFlightServer : public FlightServerBase { + public: + static std::unique_ptr Make(); + + Status ListFlights(const ServerCallContext& context, const Criteria* criteria, + std::unique_ptr* listings) override; + + Status GetFlightInfo(const ServerCallContext& context, const FlightDescriptor& request, + std::unique_ptr* out) override; + + Status DoGet(const ServerCallContext& context, const Ticket& request, + std::unique_ptr* data_stream) override; + + Status DoPut(const ServerCallContext&, std::unique_ptr reader, + std::unique_ptr writer) override; + + Status DoExchange(const ServerCallContext& context, + std::unique_ptr reader, + std::unique_ptr writer) override; + + // A simple example - act like DoGet. + Status RunExchangeGet(std::unique_ptr reader, + std::unique_ptr writer); + + // A simple example - act like DoPut + Status RunExchangePut(std::unique_ptr reader, + std::unique_ptr writer); + + // Read some number of record batches from the client, send a + // metadata message back with the count, then echo the batches back. + Status RunExchangeCounter(std::unique_ptr reader, + std::unique_ptr writer); + + // Read int64 batches from the client, each time sending back a + // batch with a running sum of columns. + Status RunExchangeTotal(std::unique_ptr reader, + std::unique_ptr writer); + + // Echo the client's messages back. + Status RunExchangeEcho(std::unique_ptr reader, + std::unique_ptr writer); + + // Regression test for ARROW-13253 + Status RunExchangeLargeBatch(std::unique_ptr, + std::unique_ptr writer); + + Status RunAction1(const Action& action, std::unique_ptr* out); + + Status RunAction2(std::unique_ptr* out); + + Status ListIncomingHeaders(const ServerCallContext& context, const Action& action, + std::unique_ptr* out); + + Status DoAction(const ServerCallContext& context, const Action& action, + std::unique_ptr* out) override; + + Status ListActions(const ServerCallContext& context, + std::vector* out) override; + + Status GetSchema(const ServerCallContext& context, const FlightDescriptor& request, + std::unique_ptr* schema) override; +}; + +} // namespace arrow::flight diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_util.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_util.h new file mode 100644 index 0000000000000000000000000000000000000000..3a14758fc9e7fd8c5ecf190129790b172f452f2a --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/test_util.h @@ -0,0 +1,188 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +#include "arrow/status.h" +#include "arrow/testing/gtest_util.h" +#include "arrow/testing/process.h" +#include "arrow/testing/util.h" + +#include "arrow/flight/client.h" +#include "arrow/flight/server.h" +#include "arrow/flight/types.h" +#include "arrow/flight/visibility.h" + +namespace arrow { +namespace flight { + +// ---------------------------------------------------------------------- +// Helpers to compare values for equality +ARROW_FLIGHT_EXPORT +void AssertEqual(const FlightInfo& expected, const FlightInfo& actual); + +// ---------------------------------------------------------------------- +// Fixture to use for running test servers + +class ARROW_FLIGHT_EXPORT TestServer { + public: + explicit TestServer(const std::string& executable_name) + : executable_name_(executable_name), port_(::arrow::GetListenPort()) {} + TestServer(const std::string& executable_name, int port) + : executable_name_(executable_name), port_(port) {} + TestServer(const std::string& executable_name, const std::string& unix_sock) + : executable_name_(executable_name), unix_sock_(unix_sock) {} + + Status Start(const std::vector& extra_args); + Status Start() { return Start({}); } + + void Stop(); + + bool IsRunning(); + + int port() const; + const std::string& unix_sock() const; + + private: + std::string executable_name_; + int port_; + std::string unix_sock_; + std::unique_ptr server_process_; +}; + +// Helper to initialize a server and matching client with callbacks to +// populate options. +template +Status MakeServer(const Location& location, std::unique_ptr* server, + std::unique_ptr* client, + std::function make_server_options, + std::function make_client_options, + Args&&... server_args) { + *server = std::make_unique(std::forward(server_args)...); + FlightServerOptions server_options(location); + RETURN_NOT_OK(make_server_options(&server_options)); + RETURN_NOT_OK((*server)->Init(server_options)); + std::string uri = + location.scheme() + "://127.0.0.1:" + std::to_string((*server)->port()); + ARROW_ASSIGN_OR_RAISE(auto real_location, Location::Parse(uri)); + FlightClientOptions client_options = FlightClientOptions::Defaults(); + RETURN_NOT_OK(make_client_options(&client_options)); + return FlightClient::Connect(real_location, client_options).Value(client); +} + +// Helper to initialize a server and matching client with callbacks to +// populate options. +template +Status MakeServer(std::unique_ptr* server, + std::unique_ptr* client, + std::function make_server_options, + std::function make_client_options, + Args&&... server_args) { + ARROW_ASSIGN_OR_RAISE(auto location, Location::ForGrpcTcp("localhost", 0)); + return MakeServer(location, server, client, std::move(make_server_options), + std::move(make_client_options), + std::forward(server_args)...); +} + +// ---------------------------------------------------------------------- +// A FlightDataStream that numbers the record batches +/// \brief A basic implementation of FlightDataStream that will provide +/// a sequence of FlightData messages to be written to a stream +class ARROW_FLIGHT_EXPORT NumberingStream : public FlightDataStream { + public: + explicit NumberingStream(std::unique_ptr stream); + + std::shared_ptr schema() override; + arrow::Result GetSchemaPayload() override; + arrow::Result Next() override; + + private: + int counter_; + std::shared_ptr stream_; +}; + +// ---------------------------------------------------------------------- +// Example data for test-server and unit tests + +ARROW_FLIGHT_EXPORT +std::shared_ptr ExampleIntSchema(); + +ARROW_FLIGHT_EXPORT +std::shared_ptr ExampleFloatSchema(); + +ARROW_FLIGHT_EXPORT +std::shared_ptr ExampleStringSchema(); + +ARROW_FLIGHT_EXPORT +std::shared_ptr ExampleDictSchema(); + +ARROW_FLIGHT_EXPORT +std::shared_ptr ExampleLargeSchema(); + +ARROW_FLIGHT_EXPORT +Status ExampleIntBatches(RecordBatchVector* out); + +ARROW_FLIGHT_EXPORT +Status ExampleFloatBatches(RecordBatchVector* out); + +ARROW_FLIGHT_EXPORT +Status ExampleDictBatches(RecordBatchVector* out); + +ARROW_FLIGHT_EXPORT +Status ExampleNestedBatches(RecordBatchVector* out); + +ARROW_FLIGHT_EXPORT +Status ExampleLargeBatches(RecordBatchVector* out); + +ARROW_FLIGHT_EXPORT +arrow::Result> VeryLargeBatch(); + +ARROW_FLIGHT_EXPORT +std::vector ExampleFlightInfo(); + +ARROW_FLIGHT_EXPORT +std::vector ExampleActionTypes(); + +ARROW_FLIGHT_EXPORT +FlightInfo MakeFlightInfo(const Schema& schema, const FlightDescriptor& descriptor, + const std::vector& endpoints, + int64_t total_records, int64_t total_bytes, bool ordered, + std::string app_metadata); + +ARROW_FLIGHT_EXPORT +FlightInfo MakeFlightInfo(const FlightDescriptor& descriptor, + const std::vector& endpoints, + int64_t total_records, int64_t total_bytes, bool ordered, + std::string app_metadata); + +ARROW_FLIGHT_EXPORT +Status ExampleTlsCertificates(std::vector* out); + +ARROW_FLIGHT_EXPORT +Status ExampleTlsCertificateRoot(CertKeyPair* out); + +} // namespace flight +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/transport.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/transport.h new file mode 100644 index 0000000000000000000000000000000000000000..2d894c6f8ffbd3e59ef0ed51179e727d0fe51c47 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/transport.h @@ -0,0 +1,298 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/// \file +/// Internal (but not private) interface for implementing +/// alternate network transports in Flight. +/// +/// To implement a transport, implement ServerTransport and +/// ClientTransport, and register the desired URI schemes with +/// TransportRegistry. Flight takes care of most of the per-RPC +/// details; transports only handle connections and providing a I/O +/// stream implementation (TransportDataStream). +/// +/// On the server side: +/// +/// 1. Applications subclass FlightServerBase and override RPC handlers. +/// 2. FlightServerBase::Init will look up and create a ServerTransport +/// based on the scheme of the Location given to it. +/// 3. The ServerTransport will start the actual server. (For instance, +/// for gRPC, it creates a gRPC server and registers a gRPC service.) +/// That server will handle connections. +/// 4. The transport should forward incoming calls to the server to the RPC +/// handlers defined on ServerTransport, which implements the actual +/// RPC handler using the interfaces here. Any I/O the RPC handler needs +/// to do is managed by transport-specific implementations of +/// TransportDataStream. +/// 5. ServerTransport calls FlightServerBase for the actual application +/// logic. +/// +/// On the client side: +/// +/// 1. Applications create a FlightClient with a Location. +/// 2. FlightClient will look up and create a ClientTransport based on +/// the scheme of the Location given to it. +/// 3. When calling a method on FlightClient, FlightClient will delegate to +/// the ClientTransport. There is some indirection, e.g. for DoGet, +/// FlightClient only requests that the ClientTransport start the +/// call and provide it with an I/O stream. The "Flight implementation" +/// itself still lives in FlightClient. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "arrow/flight/type_fwd.h" +#include "arrow/flight/types.h" +#include "arrow/flight/visibility.h" +#include "arrow/ipc/options.h" +#include "arrow/type_fwd.h" + +namespace arrow { +namespace ipc { +class Message; +} +namespace flight { +class FlightStatusDetail; +namespace internal { + +/// Internal, not user-visible type used for memory-efficient reads +struct ARROW_FLIGHT_EXPORT FlightData { + /// Used only for puts, may be null + std::unique_ptr descriptor; + + /// Non-length-prefixed Message header as described in format/Message.fbs + std::shared_ptr metadata; + + /// Application-defined metadata + std::shared_ptr app_metadata; + + /// Message body + std::shared_ptr body; + + /// Open IPC message from the metadata and body + ::arrow::Result> OpenMessage(); +}; + +/// \brief A transport-specific interface for reading/writing Arrow data. +/// +/// New transports will implement this to read/write IPC payloads to +/// the underlying stream. +class ARROW_FLIGHT_EXPORT TransportDataStream { + public: + virtual ~TransportDataStream() = default; + /// \brief Attempt to read the next FlightData message. + /// + /// \return success true if data was populated, false if there was + /// an error. For clients, the error can be retrieved from + /// Finish(Status). + virtual bool ReadData(FlightData* data); + /// \brief Attempt to write a FlightPayload. + /// + /// \param[in] payload The data to write. + /// \return true if the message was accepted by the transport, false + /// if not (e.g. due to client/server disconnect), Status if there + /// was an error (e.g. with the payload itself). + virtual arrow::Result WriteData(const FlightPayload& payload); + /// \brief Indicate that there are no more writes on this stream. + /// + /// This is only a hint for the underlying transport and may not + /// actually do anything. + virtual Status WritesDone(); +}; + +/// \brief A transport-specific interface for reading/writing Arrow +/// data for a client. +class ARROW_FLIGHT_EXPORT ClientDataStream : public TransportDataStream { + public: + /// \brief Attempt to read a non-data message. + /// + /// Only implemented for DoPut; mutually exclusive with + /// ReadData(FlightData*). + virtual bool ReadPutMetadata(std::shared_ptr* out); + /// \brief Attempt to cancel the call. + /// + /// This is only a hint and may not take effect immediately. The + /// client should still finish the call with Finish(Status) as usual. + virtual void TryCancel() {} + /// \brief Finish the call, reporting the server-sent status and/or + /// any client-side errors as appropriate. + /// + /// Implies WritesDone() and DoFinish(). + /// + /// \param[in] st A client-side status to combine with the + /// server-side error. That is, if an error occurs on the + /// client-side, call Finish(Status) to finish the server-side + /// call, get the server-side status, and merge the statuses + /// together so context is not lost. + Status Finish(Status st); + + protected: + /// \brief End the call, returning the final server status. + /// + /// For implementors: should imply WritesDone() (even if it does not + /// directly call it). + /// + /// Implies WritesDone(). + virtual Status DoFinish() = 0; +}; + +/// An implementation of a Flight client for a particular transport. +/// +/// Transports should override the methods they are capable of +/// supporting. The default method implementations return an error. +class ARROW_FLIGHT_EXPORT ClientTransport { + public: + virtual ~ClientTransport() = default; + + /// Initialize the client. + virtual Status Init(const FlightClientOptions& options, const Location& location, + const arrow::util::Uri& uri) = 0; + /// Close the client. Once this returns, the client is no longer usable. + virtual Status Close() = 0; + + virtual Status Authenticate(const FlightCallOptions& options, + std::unique_ptr auth_handler); + virtual arrow::Result> AuthenticateBasicToken( + const FlightCallOptions& options, const std::string& username, + const std::string& password); + virtual Status DoAction(const FlightCallOptions& options, const Action& action, + std::unique_ptr* results); + virtual Status ListActions(const FlightCallOptions& options, + std::vector* actions); + virtual Status GetFlightInfo(const FlightCallOptions& options, + const FlightDescriptor& descriptor, + std::unique_ptr* info); + virtual void GetFlightInfoAsync(const FlightCallOptions& options, + const FlightDescriptor& descriptor, + std::shared_ptr> listener); + virtual Status PollFlightInfo(const FlightCallOptions& options, + const FlightDescriptor& descriptor, + std::unique_ptr* info); + virtual arrow::Result> GetSchema( + const FlightCallOptions& options, const FlightDescriptor& descriptor); + virtual Status ListFlights(const FlightCallOptions& options, const Criteria& criteria, + std::unique_ptr* listing); + virtual Status DoGet(const FlightCallOptions& options, const Ticket& ticket, + std::unique_ptr* stream); + virtual Status DoPut(const FlightCallOptions& options, + std::unique_ptr* stream); + virtual Status DoExchange(const FlightCallOptions& options, + std::unique_ptr* stream); + + bool supports_async() const { return CheckAsyncSupport().ok(); } + virtual Status CheckAsyncSupport() const { + return Status::NotImplemented( + "this Flight transport does not support async operations"); + } + + static void SetAsyncRpc(AsyncListenerBase* listener, std::unique_ptr&& rpc); + static AsyncRpc* GetAsyncRpc(AsyncListenerBase* listener); + static std::unique_ptr ReleaseAsyncRpc(AsyncListenerBase* listener); +}; + +/// A registry of transport implementations. +class ARROW_FLIGHT_EXPORT TransportRegistry { + public: + using ClientFactory = std::function>()>; + using ServerFactory = std::function>( + FlightServerBase*, std::shared_ptr memory_manager)>; + + TransportRegistry(); + ~TransportRegistry(); + + arrow::Result> MakeClient( + const std::string& scheme) const; + arrow::Result> MakeServer( + const std::string& scheme, FlightServerBase* base, + std::shared_ptr memory_manager) const; + + Status RegisterClient(const std::string& scheme, ClientFactory factory); + Status RegisterServer(const std::string& scheme, ServerFactory factory); + + private: + class Impl; + std::unique_ptr impl_; +}; + +/// \brief Get the registry of transport implementations. +ARROW_FLIGHT_EXPORT +TransportRegistry* GetDefaultTransportRegistry(); + +//------------------------------------------------------------ +// Async APIs + +/// \brief Transport-specific state for an async RPC. +/// +/// Transport implementations may subclass this to store their own +/// state, and stash an instance in a user-supplied AsyncListener via +/// ClientTransport::GetAsyncRpc and ClientTransport::SetAsyncRpc. +class ARROW_FLIGHT_EXPORT AsyncRpc { + public: + virtual ~AsyncRpc() = default; + /// \brief Request cancellation of the RPC. + virtual void TryCancel() {} + + /// Only needed for DoPut/DoExchange + virtual void Begin(const FlightDescriptor& descriptor, std::shared_ptr schema) { + } + /// Only needed for DoPut/DoExchange + virtual void Write(arrow::flight::FlightStreamChunk chunk) {} + /// Only needed for DoPut/DoExchange + virtual void DoneWriting() {} +}; + +//------------------------------------------------------------ +// Error propagation helpers + +/// \brief Abstract error status. +/// +/// Transport implementations may use side channels (e.g. HTTP +/// trailers) to convey additional information to reconstruct the +/// original C++ status for implementations that can use it. +struct ARROW_FLIGHT_EXPORT TransportStatus { + TransportStatusCode code; + std::string message; + + /// \brief Convert a C++ status to an abstract transport status. + static TransportStatus FromStatus(const Status& arrow_status); + + /// \brief Reconstruct a string-encoded TransportStatus. + static TransportStatus FromCodeStringAndMessage(const std::string& code_str, + std::string message); + + /// \brief Convert an abstract transport status to a C++ status. + Status ToStatus() const; +}; + +/// \brief Convert the string representation of an Arrow status code +/// back to an Arrow status. +ARROW_FLIGHT_EXPORT +Status ReconstructStatus(const std::string& code_str, const Status& current_status, + std::optional message, + std::optional detail_message, + std::optional detail_bin, + std::shared_ptr detail); + +} // namespace internal +} // namespace flight +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/transport_server.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/transport_server.h new file mode 100644 index 0000000000000000000000000000000000000000..e0c483fc74f846893985666546a8d7c7b72ea6e0 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/transport_server.h @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "arrow/flight/transport.h" +#include "arrow/flight/type_fwd.h" +#include "arrow/flight/visibility.h" +#include "arrow/type_fwd.h" + +namespace arrow { +namespace ipc { +class Message; +} +namespace flight { +namespace internal { + +/// \brief A transport-specific interface for reading/writing Arrow +/// data for a server. +class ARROW_FLIGHT_EXPORT ServerDataStream : public TransportDataStream { + public: + /// \brief Attempt to write a non-data message. + /// + /// Only implemented for DoPut; mutually exclusive with + /// WriteData(const FlightPayload&). + virtual Status WritePutMetadata(const Buffer& payload); +}; + +/// \brief An implementation of a Flight server for a particular +/// transport. +/// +/// This class (the transport implementation) implements the underlying +/// server and handles connections/incoming RPC calls. It should forward RPC +/// calls to the RPC handlers defined on this class, which work in terms of +/// the generic interfaces above. The RPC handlers here then forward calls +/// to the underlying FlightServerBase instance that contains the actual +/// application RPC method handlers. +/// +/// Used by FlightServerBase to manage the server lifecycle. +class ARROW_FLIGHT_EXPORT ServerTransport { + public: + ServerTransport(FlightServerBase* base, std::shared_ptr memory_manager) + : base_(base), memory_manager_(std::move(memory_manager)) {} + virtual ~ServerTransport() = default; + + /// \name Server Lifecycle Methods + /// Transports implement these methods to start/shutdown the underlying + /// server. + /// @{ + /// \brief Initialize the server. + /// + /// This method should launch the server in a background thread, i.e. it + /// should not block. Once this returns, the server should be active. + virtual Status Init(const FlightServerOptions& options, + const arrow::util::Uri& uri) = 0; + /// \brief Shutdown the server. + /// + /// This should wait for active RPCs to finish. Once this returns, the + /// server is no longer listening. + virtual Status Shutdown() = 0; + /// \brief Shutdown the server with a deadline. + /// + /// This should wait for active RPCs to finish, or for the deadline to + /// expire. Once this returns, the server is no longer listening. + virtual Status Shutdown(const std::chrono::system_clock::time_point& deadline) = 0; + /// \brief Wait for the server to shutdown (but do not shut down the server). + /// + /// Once this returns, the server is no longer listening. + virtual Status Wait() = 0; + /// \brief Get the address the server is listening on, else an empty Location. + virtual Location location() const = 0; + ///@} + + /// \name RPC Handlers + /// Implementations of RPC handlers for Flight methods using the common + /// interfaces here. Transports should call these methods from their + /// server implementation to handle the actual RPC calls. + ///@{ + /// \brief Get the FlightServerBase. + /// + /// Intended as an escape hatch for now since not all methods have been + /// factored into a transport-agnostic interface. + FlightServerBase* base() const { return base_; } + /// \brief Implement DoGet in terms of a transport-level stream. + /// + /// \param[in] context The server context. + /// \param[in] request The request payload. + /// \param[in] stream The transport-specific data stream + /// implementation. Must implement WriteData(const + /// FlightPayload&). + Status DoGet(const ServerCallContext& context, const Ticket& request, + ServerDataStream* stream); + /// \brief Implement DoPut in terms of a transport-level stream. + /// + /// \param[in] context The server context. + /// \param[in] stream The transport-specific data stream + /// implementation. Must implement ReadData(FlightData*) + /// and WritePutMetadata(const Buffer&). + Status DoPut(const ServerCallContext& context, ServerDataStream* stream); + /// \brief Implement DoExchange in terms of a transport-level stream. + /// + /// \param[in] context The server context. + /// \param[in] stream The transport-specific data stream + /// implementation. Must implement ReadData(FlightData*) + /// and WriteData(const FlightPayload&). + Status DoExchange(const ServerCallContext& context, ServerDataStream* stream); + ///@} + + protected: + FlightServerBase* base_; + std::shared_ptr memory_manager_; +}; + +} // namespace internal +} // namespace flight +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/type_fwd.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/type_fwd.h new file mode 100644 index 0000000000000000000000000000000000000000..e6f6a4b9317fe009c6a2f47317d7f42687bd84f4 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/type_fwd.h @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +namespace arrow { +namespace internal { +class Uri; +} +namespace flight { +struct Action; +struct ActionType; +template +class AsyncListener; +class AsyncListenerBase; +class AsyncRpc; +struct BasicAuth; +class ClientAuthHandler; +class ClientMiddleware; +class ClientMiddlewareFactory; +struct Criteria; +class FlightCallOptions; +struct FlightClientOptions; +struct FlightDescriptor; +struct FlightEndpoint; +class FlightInfo; +class PollInfo; +class FlightListing; +class FlightMetadataReader; +class FlightMetadataWriter; +struct FlightPayload; +class FlightServerBase; +class FlightServerOptions; +class FlightStreamReader; +class FlightStreamWriter; +struct Location; +struct Result; +class ResultStream; +struct SchemaResult; +class ServerCallContext; +class ServerMiddleware; +class ServerMiddlewareFactory; +struct Ticket; +namespace internal { +class AsyncRpc; +class ClientTransport; +struct FlightData; +class ServerTransport; +} // namespace internal +} // namespace flight +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/types.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/types.h new file mode 100644 index 0000000000000000000000000000000000000000..99bbda68202ddc91bc5fede656ad76870b9d4ccc --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/types.h @@ -0,0 +1,1314 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Data structure for Flight RPC. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/buffer.h" +#include "arrow/flight/type_fwd.h" +#include "arrow/flight/visibility.h" +#include "arrow/ipc/options.h" +#include "arrow/ipc/writer.h" +#include "arrow/result.h" +#include "arrow/status.h" + +namespace arrow { + +class RecordBatch; +class Schema; +class Table; + +namespace ipc { +class DictionaryMemo; +struct ReadStats; +} // namespace ipc + +namespace util { +class Uri; +} // namespace util + +namespace flight { + +ARROW_FLIGHT_EXPORT +extern const char* kSchemeGrpc; +ARROW_FLIGHT_EXPORT +extern const char* kSchemeGrpcTcp; +ARROW_FLIGHT_EXPORT +extern const char* kSchemeGrpcUnix; +ARROW_FLIGHT_EXPORT +extern const char* kSchemeGrpcTls; + +class FlightClient; +class FlightServerBase; + +/// \brief A timestamp compatible with Protocol Buffer's +/// google.protobuf.Timestamp: +/// +/// https://protobuf.dev/reference/protobuf/google.protobuf/#timestamp +/// +/// > A Timestamp represents a point in time independent of any time +/// > zone or calendar, represented as seconds and fractions of +/// > seconds at nanosecond resolution in UTC Epoch time. It is +/// > encoded using the Proleptic Gregorian Calendar which extends the +/// > Gregorian calendar backwards to year one. It is encoded assuming +/// > all minutes are 60 seconds long, i.e. leap seconds are "smeared" +/// > so that no leap second table is needed for interpretation. Range +/// > is from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. +using Timestamp = + std::chrono::time_point; + +/// \brief A Flight-specific status code. Used to encode some +/// additional status codes into an Arrow Status. +enum class FlightStatusCode : int8_t { + /// An implementation error has occurred. + Internal, + /// A request timed out. + TimedOut, + /// A request was cancelled. + Cancelled, + /// We are not authenticated to the remote service. + Unauthenticated, + /// We do not have permission to make this request. + Unauthorized, + /// The remote service cannot handle this request at the moment. + Unavailable, + /// A request failed for some other reason + Failed +}; + +// Silence warning +// "non dll-interface class RecordBatchReader used as base for dll-interface class" +#ifdef _MSC_VER +# pragma warning(push) +# pragma warning(disable : 4275) +#endif + +/// \brief Flight-specific error information in a Status. +class ARROW_FLIGHT_EXPORT FlightStatusDetail : public arrow::StatusDetail { + public: + explicit FlightStatusDetail(FlightStatusCode code) : code_{code} {} + explicit FlightStatusDetail(FlightStatusCode code, std::string extra_info) + : code_{code}, extra_info_(std::move(extra_info)) {} + const char* type_id() const override; + std::string ToString() const override; + + /// \brief Get the Flight status code. + FlightStatusCode code() const; + /// \brief Get the extra error info + std::string extra_info() const; + /// \brief Get the human-readable name of the status code. + std::string CodeAsString() const; + /// \brief Set the extra error info + void set_extra_info(std::string extra_info); + + /// \brief Try to extract a \a FlightStatusDetail from any Arrow + /// status. + /// + /// \return a \a FlightStatusDetail if it could be unwrapped, \a + /// nullptr otherwise + static std::shared_ptr UnwrapStatus(const arrow::Status& status); + + private: + FlightStatusCode code_; + std::string extra_info_; +}; + +#ifdef _MSC_VER +# pragma warning(pop) +#endif + +/// \brief Make an appropriate Arrow status for the given +/// Flight-specific status. +/// +/// \param code The Flight status code. +/// \param message The message for the error. +/// \param extra_info Optional extra binary info for the error (eg protobuf) +ARROW_FLIGHT_EXPORT +Status MakeFlightError(FlightStatusCode code, std::string message, + std::string extra_info = {}); + +/// \brief Headers sent from the client or server. +/// +/// Header values are ordered. +using CallHeaders = std::multimap; + +/// \brief A TLS certificate plus key. +struct ARROW_FLIGHT_EXPORT CertKeyPair { + /// \brief The certificate in PEM format. + std::string pem_cert; + + /// \brief The key in PEM format. + std::string pem_key; +}; + +namespace internal { + +template +struct remove_unique_ptr { + using type = T; +}; + +template +struct remove_unique_ptr> { + using type = T; +}; + +// Base CRTP type +template +struct BaseType { + protected: + using SuperT = BaseType; + using SelfT = typename remove_unique_ptr::type; + + const SelfT& self() const { return static_cast(*this); } + SelfT& self() { return static_cast(*this); } + + public: + BaseType() = default; + + friend bool operator==(const SelfT& left, const SelfT& right) { + return left.Equals(right); + } + friend bool operator!=(const SelfT& left, const SelfT& right) { + return !left.Equals(right); + } + + /// \brief Serialize this message to its wire-format representation. + inline arrow::Result SerializeToString() const { + std::string out; + ARROW_RETURN_NOT_OK(self().SelfT::SerializeToString(&out)); + return out; + } + + inline static arrow::Result Deserialize(std::string_view serialized) { + T out; + ARROW_RETURN_NOT_OK(SelfT::Deserialize(serialized, &out)); + return out; + } + + inline arrow::Result> SerializeToBuffer() const { + std::string out; + ARROW_RETURN_NOT_OK(self().SelfT::SerializeToString(&out)); + return Buffer::FromString(std::move(out)); + } +}; + +} // namespace internal + +//------------------------------------------------------------ +// Wrapper types for Flight RPC protobuf messages + +// A wrapper around arrow.flight.protocol.HandshakeRequest is not defined +// A wrapper around arrow.flight.protocol.HandshakeResponse is not defined + +/// \brief message for simple auth +struct ARROW_FLIGHT_EXPORT BasicAuth : public internal::BaseType { + std::string username; + std::string password; + + BasicAuth() = default; + BasicAuth(std::string username, std::string password) + : username(std::move(username)), password(std::move(password)) {} + + std::string ToString() const; + bool Equals(const BasicAuth& other) const; + + using SuperT::Deserialize; + using SuperT::SerializeToString; + + /// \brief Serialize this message to its wire-format representation. + /// + /// Use `SerializeToString()` if you want a Result-returning version. + arrow::Status SerializeToString(std::string* out) const; + + /// \brief Deserialize this message from its wire-format representation. + /// + /// Use `Deserialize(serialized)` if you want a Result-returning version. + static arrow::Status Deserialize(std::string_view serialized, BasicAuth* out); +}; + +// A wrapper around arrow.flight.protocol.Empty is not defined + +/// \brief A type of action that can be performed with the DoAction RPC. +struct ARROW_FLIGHT_EXPORT ActionType : public internal::BaseType { + /// \brief The name of the action. + std::string type; + + /// \brief A human-readable description of the action. + std::string description; + + ActionType() = default; + + ActionType(std::string type, std::string description) + : type(std::move(type)), description(std::move(description)) {} + + std::string ToString() const; + bool Equals(const ActionType& other) const; + + using SuperT::Deserialize; + using SuperT::SerializeToString; + + /// \brief Serialize this message to its wire-format representation. + /// + /// Use `SerializeToString()` if you want a Result-returning version. + arrow::Status SerializeToString(std::string* out) const; + + /// \brief Deserialize this message from its wire-format representation. + /// + /// Use `Deserialize(serialized)` if you want a Result-returning version. + static arrow::Status Deserialize(std::string_view serialized, ActionType* out); + + static const ActionType kCancelFlightInfo; + static const ActionType kRenewFlightEndpoint; + static const ActionType kSetSessionOptions; + static const ActionType kGetSessionOptions; + static const ActionType kCloseSession; +}; + +/// \brief Opaque selection criteria for ListFlights RPC +struct ARROW_FLIGHT_EXPORT Criteria : public internal::BaseType { + /// Opaque criteria expression, dependent on server implementation + std::string expression; + + Criteria() = default; + Criteria(std::string expression) // NOLINT runtime/explicit + : expression(std::move(expression)) {} + + std::string ToString() const; + bool Equals(const Criteria& other) const; + + using SuperT::Deserialize; + using SuperT::SerializeToString; + + /// \brief Serialize this message to its wire-format representation. + /// + /// Use `SerializeToString()` if you want a Result-returning version. + arrow::Status SerializeToString(std::string* out) const; + + /// \brief Deserialize this message from its wire-format representation. + /// + /// Use `Deserialize(serialized)` if you want a Result-returning version. + static arrow::Status Deserialize(std::string_view serialized, Criteria* out); +}; + +/// \brief An action to perform with the DoAction RPC +struct ARROW_FLIGHT_EXPORT Action : public internal::BaseType { + /// The action type + std::string type; + + /// The action content as a Buffer + std::shared_ptr body; + + Action() = default; + Action(std::string type, std::shared_ptr body) + : type(std::move(type)), body(std::move(body)) {} + + std::string ToString() const; + bool Equals(const Action& other) const; + + using SuperT::Deserialize; + using SuperT::SerializeToString; + + /// \brief Serialize this message to its wire-format representation. + /// + /// Use `SerializeToString()` if you want a Result-returning version. + arrow::Status SerializeToString(std::string* out) const; + + /// \brief Deserialize this message from its wire-format representation. + /// + /// Use `Deserialize(serialized)` if you want a Result-returning version. + static arrow::Status Deserialize(std::string_view serialized, Action* out); +}; + +/// \brief Opaque result returned after executing an action +struct ARROW_FLIGHT_EXPORT Result : public internal::BaseType { + std::shared_ptr body; + + Result() = default; + Result(std::shared_ptr body) // NOLINT runtime/explicit + : body(std::move(body)) {} + + std::string ToString() const; + bool Equals(const Result& other) const; + + using SuperT::Deserialize; + using SuperT::SerializeToString; + + /// \brief Serialize this message to its wire-format representation. + /// + /// Use `SerializeToString()` if you want a Result-returning version. + arrow::Status SerializeToString(std::string* out) const; + + /// \brief Deserialize this message from its wire-format representation. + /// + /// Use `Deserialize(serialized)` if you want a Result-returning version. + static arrow::Status Deserialize(std::string_view serialized, Result* out); +}; + +/// \brief Schema result returned after a schema request RPC +struct ARROW_FLIGHT_EXPORT SchemaResult : public internal::BaseType { + public: + SchemaResult() = default; + explicit SchemaResult(std::string schema) : raw_schema_(std::move(schema)) {} + + /// \brief Factory method to construct a SchemaResult. + static arrow::Result> Make(const Schema& schema); + + /// \brief return schema + /// \param[in,out] dictionary_memo for dictionary bookkeeping, will + /// be modified + /// \return Arrow result with the reconstructed Schema + arrow::Result> GetSchema( + ipc::DictionaryMemo* dictionary_memo) const; + + const std::string& serialized_schema() const { return raw_schema_; } + + std::string ToString() const; + bool Equals(const SchemaResult& other) const; + + using SuperT::Deserialize; + using SuperT::SerializeToString; + + /// \brief Serialize this message to its wire-format representation. + /// + /// Use `SerializeToString()` if you want a Result-returning version. + arrow::Status SerializeToString(std::string* out) const; + + /// \brief Deserialize this message from its wire-format representation. + /// + /// Use `Deserialize(serialized)` if you want a Result-returning version. + static arrow::Status Deserialize(std::string_view serialized, SchemaResult* out); + + private: + std::string raw_schema_; +}; + +/// \brief A request to retrieve or generate a dataset +struct ARROW_FLIGHT_EXPORT FlightDescriptor + : public internal::BaseType { + enum DescriptorType { + UNKNOWN = 0, /// Unused + PATH = 1, /// Named path identifying a dataset + CMD = 2 /// Opaque command to generate a dataset + }; + + /// The descriptor type + DescriptorType type = UNKNOWN; + + /// Opaque value used to express a command. Should only be defined when type + /// is CMD + std::string cmd; + + /// List of strings identifying a particular dataset. Should only be defined + /// when type is PATH + std::vector path; + + FlightDescriptor(); + FlightDescriptor(DescriptorType type, std::string cmd, + std::vector path) noexcept; + ~FlightDescriptor(); + + /// \brief Get a human-readable form of this descriptor. + std::string ToString() const; + bool Equals(const FlightDescriptor& other) const; + + using SuperT::Deserialize; + using SuperT::SerializeToString; + + /// \brief Get the wire-format representation of this type. + /// + /// Useful when interoperating with non-Flight systems (e.g. REST + /// services) that may want to return Flight types. + /// + /// Use `SerializeToString()` if you want a Result-returning version. + arrow::Status SerializeToString(std::string* out) const; + + /// \brief Parse the wire-format representation of this type. + /// + /// Useful when interoperating with non-Flight systems (e.g. REST + /// services) that may want to return Flight types. + /// + /// Use `Deserialize(serialized)` if you want a Result-returning version. + static arrow::Status Deserialize(std::string_view serialized, FlightDescriptor* out); + + // Convenience factory functions + + static FlightDescriptor Command(std::string cmd) { + return FlightDescriptor{CMD, std::move(cmd), {}}; + } + + static FlightDescriptor Path(std::vector path) { + return FlightDescriptor{PATH, "", std::move(path)}; + } +}; + +/// \brief Data structure providing an opaque identifier or credential to use +/// when requesting a data stream with the DoGet RPC +struct ARROW_FLIGHT_EXPORT Ticket : public internal::BaseType { + std::string ticket; + + Ticket() = default; + Ticket(std::string ticket) // NOLINT runtime/explicit + : ticket(std::move(ticket)) {} + + std::string ToString() const; + bool Equals(const Ticket& other) const; + + using SuperT::Deserialize; + using SuperT::SerializeToString; + + /// \brief Get the wire-format representation of this type. + /// + /// Useful when interoperating with non-Flight systems (e.g. REST + /// services) that may want to return Flight types. + /// + /// Use `SerializeToString()` if you want a Result-returning version. + arrow::Status SerializeToString(std::string* out) const; + + /// \brief Parse the wire-format representation of this type. + /// + /// Useful when interoperating with non-Flight systems (e.g. REST + /// services) that may want to return Flight types. + /// + /// Use `Deserialize(serialized)` if you want a Result-returning version. + static arrow::Status Deserialize(std::string_view serialized, Ticket* out); +}; + +/// \brief A host location (a URI) +struct ARROW_FLIGHT_EXPORT Location : public internal::BaseType { + public: + /// \brief Initialize a blank location. + Location(); + + ~Location(); + + /// \brief Initialize a location by parsing a URI string + static arrow::Result Parse(const std::string& uri_string); + + /// \brief Get the fallback URI. + /// + /// arrow-flight-reuse-connection://? means that a client may attempt to + /// reuse an existing connection to a Flight service to fetch data instead + /// of creating a new connection to one of the other locations listed in a + /// FlightEndpoint response. + static const Location& ReuseConnection(); + + /// \brief Initialize a location for a non-TLS, gRPC-based Flight + /// service from a host and port + /// \param[in] host The hostname to connect to + /// \param[in] port The port + /// \return Arrow result with the resulting location + static arrow::Result ForGrpcTcp(const std::string& host, const int port); + + /// \brief Initialize a location for a TLS-enabled, gRPC-based Flight + /// service from a host and port + /// \param[in] host The hostname to connect to + /// \param[in] port The port + /// \return Arrow result with the resulting location + static arrow::Result ForGrpcTls(const std::string& host, const int port); + + /// \brief Initialize a location for a domain socket-based Flight + /// service + /// \param[in] path The path to the domain socket + /// \return Arrow result with the resulting location + static arrow::Result ForGrpcUnix(const std::string& path); + + /// \brief Initialize a location based on a URI scheme + static arrow::Result ForScheme(const std::string& scheme, + const std::string& host, const int port); + + /// \brief Get the scheme of this URI. + std::string scheme() const; + + /// \brief Get a representation of this URI as a string. + std::string ToString() const; + bool Equals(const Location& other) const; + + using SuperT::Deserialize; + using SuperT::SerializeToString; + + /// \brief Serialize this message to its wire-format representation. + /// + /// Use `SerializeToString()` if you want a Result-returning version. + arrow::Status SerializeToString(std::string* out) const; + + /// \brief Deserialize this message from its wire-format representation. + /// + /// Use `Deserialize(serialized)` if you want a Result-returning version. + static arrow::Status Deserialize(std::string_view serialized, Location* out); + + private: + friend class FlightClient; + friend class FlightServerBase; + std::shared_ptr uri_; +}; + +/// \brief A flight ticket and list of locations where the ticket can be +/// redeemed +struct ARROW_FLIGHT_EXPORT FlightEndpoint : public internal::BaseType { + /// Opaque ticket identify; use with DoGet RPC + Ticket ticket; + + /// List of locations where ticket can be redeemed. If the list is empty, the + /// ticket can only be redeemed on the current service where the ticket was + /// generated + std::vector locations; + + /// Expiration time of this stream. If present, clients may assume + /// they can retry DoGet requests. Otherwise, clients should avoid + /// retrying DoGet requests. + std::optional expiration_time; + + /// Opaque Application-defined metadata + std::string app_metadata; + + FlightEndpoint() = default; + FlightEndpoint(Ticket ticket, std::vector locations, + std::optional expiration_time, std::string app_metadata) + : ticket(std::move(ticket)), + locations(std::move(locations)), + expiration_time(expiration_time), + app_metadata(std::move(app_metadata)) {} + + std::string ToString() const; + bool Equals(const FlightEndpoint& other) const; + + using SuperT::Deserialize; + using SuperT::SerializeToString; + + /// \brief Serialize this message to its wire-format representation. + /// + /// Use `SerializeToString()` if you want a Result-returning version. + arrow::Status SerializeToString(std::string* out) const; + + /// \brief Deserialize this message from its wire-format representation. + /// + /// Use `Deserialize(serialized)` if you want a Result-returning version. + static arrow::Status Deserialize(std::string_view serialized, FlightEndpoint* out); +}; + +/// \brief The access coordinates for retrieval of a dataset, returned by +/// GetFlightInfo +class ARROW_FLIGHT_EXPORT FlightInfo + : public internal::BaseType> { + public: + struct Data { + std::string schema; + FlightDescriptor descriptor; + std::vector endpoints; + int64_t total_records = -1; + int64_t total_bytes = -1; + bool ordered = false; + std::string app_metadata; + }; + + explicit FlightInfo(Data data) : data_(std::move(data)), reconstructed_schema_(false) {} + + /// \brief Factory method to construct a FlightInfo. + static arrow::Result Make(const Schema& schema, + const FlightDescriptor& descriptor, + const std::vector& endpoints, + int64_t total_records, int64_t total_bytes, + bool ordered = false, + std::string app_metadata = ""); + + /// \brief Factory method to construct a FlightInfo. + static arrow::Result Make(const std::shared_ptr& schema, + const FlightDescriptor& descriptor, + const std::vector& endpoints, + int64_t total_records, int64_t total_bytes, + bool ordered = false, + std::string app_metadata = ""); + + /// \brief Deserialize the Arrow schema of the dataset. Populate any + /// dictionary encoded fields into a DictionaryMemo for + /// bookkeeping + /// \param[in,out] dictionary_memo for dictionary bookkeeping, will + /// be modified + /// \return Arrow result with the reconstructed Schema. Note that the schema + /// may be nullptr, as the schema is optional. + arrow::Result> GetSchema( + ipc::DictionaryMemo* dictionary_memo) const; + + const std::string& serialized_schema() const { return data_.schema; } + + /// The descriptor associated with this flight, may not be set + const FlightDescriptor& descriptor() const { return data_.descriptor; } + + /// A list of endpoints associated with the flight (dataset). To consume the + /// whole flight, all endpoints must be consumed + const std::vector& endpoints() const { return data_.endpoints; } + + /// The total number of records (rows) in the dataset. If unknown, set to -1 + int64_t total_records() const { return data_.total_records; } + + /// The total number of bytes in the dataset. If unknown, set to -1 + int64_t total_bytes() const { return data_.total_bytes; } + + /// Whether endpoints are in the same order as the data. + bool ordered() const { return data_.ordered; } + + /// Application-defined opaque metadata + const std::string& app_metadata() const { return data_.app_metadata; } + + using SuperT::Deserialize; + using SuperT::SerializeToString; + + /// \brief Get the wire-format representation of this type. + /// + /// Useful when interoperating with non-Flight systems (e.g. REST + /// services) that may want to return Flight types. + /// + /// Use `SerializeToString()` if you want a Result-returning version. + arrow::Status SerializeToString(std::string* out) const; + + /// \brief Parse the wire-format representation of this type. + /// + /// Useful when interoperating with non-Flight systems (e.g. REST + /// services) that may want to return Flight types. + /// + /// Use `Deserialize(serialized)` if you want a Result-returning version. + static arrow::Status Deserialize(std::string_view serialized, + std::unique_ptr* out); + + std::string ToString() const; + + /// Compare two FlightInfo for equality. This will compare the + /// serialized schema representations, NOT the logical equality of + /// the schemas. + bool Equals(const FlightInfo& other) const; + + private: + Data data_; + mutable std::shared_ptr schema_; + mutable bool reconstructed_schema_; +}; + +/// \brief The information to process a long-running query. +class ARROW_FLIGHT_EXPORT PollInfo + : public internal::BaseType> { + public: + /// The currently available results so far. + std::unique_ptr info = NULLPTR; + /// The descriptor the client should use on the next try. If unset, + /// the query is complete. + std::optional descriptor = std::nullopt; + /// Query progress. Must be in [0.0, 1.0] but need not be + /// monotonic or nondecreasing. If unknown, do not set. + std::optional progress = std::nullopt; + /// Expiration time for this request. After this passes, the server + /// might not accept the poll descriptor anymore (and the query may + /// be cancelled). This may be updated on a call to PollFlightInfo. + std::optional expiration_time = std::nullopt; + + PollInfo() + : info(NULLPTR), + descriptor(std::nullopt), + progress(std::nullopt), + expiration_time(std::nullopt) {} + + PollInfo(std::unique_ptr info, std::optional descriptor, + std::optional progress, std::optional expiration_time) + : info(std::move(info)), + descriptor(std::move(descriptor)), + progress(progress), + expiration_time(expiration_time) {} + + PollInfo(const PollInfo& other) + : info(other.info ? std::make_unique(*other.info) : NULLPTR), + descriptor(other.descriptor), + progress(other.progress), + expiration_time(other.expiration_time) {} + PollInfo(PollInfo&& other) noexcept = default; + ~PollInfo() = default; + PollInfo& operator=(const PollInfo& other) { + info = other.info ? std::make_unique(*other.info) : NULLPTR; + descriptor = other.descriptor; + progress = other.progress; + expiration_time = other.expiration_time; + return *this; + } + PollInfo& operator=(PollInfo&& other) = default; + + using SuperT::Deserialize; + using SuperT::SerializeToString; + + /// \brief Get the wire-format representation of this type. + /// + /// Useful when interoperating with non-Flight systems (e.g. REST + /// services) that may want to return Flight types. + /// + /// Use `SerializeToString()` if you want a Result-returning version. + arrow::Status SerializeToString(std::string* out) const; + + /// \brief Parse the wire-format representation of this type. + /// + /// Useful when interoperating with non-Flight systems (e.g. REST + /// services) that may want to return Flight types. + /// + /// Use `Deserialize(serialized)` if you want a Result-returning version. + static arrow::Status Deserialize(std::string_view serialized, + std::unique_ptr* out); + + std::string ToString() const; + + /// Compare two PollInfo for equality. This will compare the + /// serialized schema representations, NOT the logical equality of + /// the schemas. + bool Equals(const PollInfo& other) const; +}; + +/// \brief The request of the CancelFlightInfoRequest action. +struct ARROW_FLIGHT_EXPORT CancelFlightInfoRequest + : public internal::BaseType { + std::unique_ptr info; + + CancelFlightInfoRequest() = default; + CancelFlightInfoRequest(std::unique_ptr info) // NOLINT runtime/explicit + : info(std::move(info)) {} + + std::string ToString() const; + bool Equals(const CancelFlightInfoRequest& other) const; + + using SuperT::Deserialize; + using SuperT::SerializeToString; + + /// \brief Serialize this message to its wire-format representation. + /// + /// Use `SerializeToString()` if you want a Result-returning version. + arrow::Status SerializeToString(std::string* out) const; + + /// \brief Deserialize this message from its wire-format representation. + /// + /// Use `Deserialize(serialized)` if you want a Result-returning version. + static arrow::Status Deserialize(std::string_view serialized, + CancelFlightInfoRequest* out); +}; + +enum class CancelStatus { + /// The cancellation status is unknown. Servers should avoid using + /// this value (send a kNotCancellable if the requested FlightInfo + /// is not known). Clients can retry the request. + kUnspecified = 0, + /// The cancellation request is complete. Subsequent requests with + /// the same payload may return kCancelled or a kNotCancellable error. + kCancelled = 1, + /// The cancellation request is in progress. The client may retry + /// the cancellation request. + kCancelling = 2, + // The FlightInfo is not cancellable. The client should not retry the + // cancellation request. + kNotCancellable = 3, +}; + +/// \brief The result of the CancelFlightInfo action. +struct ARROW_FLIGHT_EXPORT CancelFlightInfoResult + : public internal::BaseType { + CancelStatus status = CancelStatus::kUnspecified; + + CancelFlightInfoResult() = default; + CancelFlightInfoResult(CancelStatus status) // NOLINT runtime/explicit + : status(status) {} + + std::string ToString() const; + bool Equals(const CancelFlightInfoResult& other) const; + + using SuperT::Deserialize; + using SuperT::SerializeToString; + + /// \brief Serialize this message to its wire-format representation. + /// + /// Use `SerializeToString()` if you want a Result-returning version. + arrow::Status SerializeToString(std::string* out) const; + + /// \brief Deserialize this message from its wire-format representation. + /// + /// Use `Deserialize(serialized)` if you want a Result-returning version. + static arrow::Status Deserialize(std::string_view serialized, + CancelFlightInfoResult* out); +}; + +ARROW_FLIGHT_EXPORT +std::ostream& operator<<(std::ostream& os, CancelStatus status); + +/// \brief The request of the RenewFlightEndpoint action. +struct ARROW_FLIGHT_EXPORT RenewFlightEndpointRequest + : public internal::BaseType { + FlightEndpoint endpoint; + + RenewFlightEndpointRequest() = default; + explicit RenewFlightEndpointRequest(FlightEndpoint endpoint) + : endpoint(std::move(endpoint)) {} + + std::string ToString() const; + bool Equals(const RenewFlightEndpointRequest& other) const; + + using SuperT::Deserialize; + using SuperT::SerializeToString; + + /// \brief Serialize this message to its wire-format representation. + /// + /// Use `SerializeToString()` if you want a Result-returning version. + arrow::Status SerializeToString(std::string* out) const; + + /// \brief Deserialize this message from its wire-format representation. + /// + /// Use `Deserialize(serialized)` if you want a Result-returning version. + static arrow::Status Deserialize(std::string_view serialized, + RenewFlightEndpointRequest* out); +}; + +// FlightData in Flight.proto maps to FlightPayload here. + +/// \brief Staging data structure for messages about to be put on the wire +/// +/// This structure corresponds to FlightData in the protocol. +struct ARROW_FLIGHT_EXPORT FlightPayload { + std::shared_ptr descriptor; + std::shared_ptr app_metadata; + ipc::IpcPayload ipc_message; + + FlightPayload() = default; + FlightPayload(std::shared_ptr descriptor, std::shared_ptr app_metadata, + ipc::IpcPayload ipc_message) + : descriptor(std::move(descriptor)), + app_metadata(std::move(app_metadata)), + ipc_message(std::move(ipc_message)) {} + + /// \brief Check that the payload can be written to the wire. + Status Validate() const; + + /// \brief Serialize this payload to a vector of buffers. + arrow::Result SerializeToBuffers() const; +}; + +// A wrapper around arrow.flight.protocol.PutResult is not defined + +// Session management messages + +/// \brief Variant supporting all possible value types for {Set,Get}SessionOptions +/// +/// By convention, an attempt to set a valueless (std::monostate) SessionOptionValue +/// should attempt to unset or clear the named option value on the server. +using SessionOptionValue = std::variant>; +std::ostream& operator<<(std::ostream& os, const SessionOptionValue& v); + +/// \brief A request to set a set of session options by name/value. +struct ARROW_FLIGHT_EXPORT SetSessionOptionsRequest + : public internal::BaseType { + std::map session_options; + + SetSessionOptionsRequest() = default; + explicit SetSessionOptionsRequest( + std::map session_options) + : session_options(std::move(session_options)) {} + + std::string ToString() const; + bool Equals(const SetSessionOptionsRequest& other) const; + + using SuperT::Deserialize; + using SuperT::SerializeToString; + + /// \brief Serialize this message to its wire-format representation. + /// + /// Use `SerializeToString()` if you want a Result-returning version. + arrow::Status SerializeToString(std::string* out) const; + + /// \brief Deserialize this message from its wire-format representation. + /// + /// Use `Deserialize(serialized)` if you want a Result-returning version. + static arrow::Status Deserialize(std::string_view serialized, + SetSessionOptionsRequest* out); +}; + +/// \brief The result of setting a session option. +enum class SetSessionOptionErrorValue : int8_t { + /// \brief The status of setting the option is unknown. + /// + /// Servers should avoid using this value (send a NOT_FOUND error if the requested + /// session is not known). Clients can retry the request. + kUnspecified, + /// \brief The given session option name is invalid. + kInvalidName, + /// \brief The session option value or type is invalid. + kInvalidValue, + /// \brief The session option cannot be set. + kError +}; +std::string ToString(const SetSessionOptionErrorValue& error_value); +std::ostream& operator<<(std::ostream& os, const SetSessionOptionErrorValue& error_value); + +/// \brief The result(s) of setting session option(s). +struct ARROW_FLIGHT_EXPORT SetSessionOptionsResult + : public internal::BaseType { + struct Error { + SetSessionOptionErrorValue value; + + bool Equals(const Error& other) const { return value == other.value; } + friend bool operator==(const Error& left, const Error& right) { + return left.Equals(right); + } + friend bool operator!=(const Error& left, const Error& right) { + return !(left == right); + } + }; + + std::map errors; + + SetSessionOptionsResult() = default; + SetSessionOptionsResult(std::map errors) // NOLINT runtime/explicit + : errors(std::move(errors)) {} + + std::string ToString() const; + bool Equals(const SetSessionOptionsResult& other) const; + + using SuperT::Deserialize; + using SuperT::SerializeToString; + + /// \brief Serialize this message to its wire-format representation. + /// + /// Use `SerializeToString()` if you want a Result-returning version. + arrow::Status SerializeToString(std::string* out) const; + + /// \brief Deserialize this message from its wire-format representation. + /// + /// Use `Deserialize(serialized)` if you want a Result-returning version. + static arrow::Status Deserialize(std::string_view serialized, + SetSessionOptionsResult* out); +}; + +/// \brief A request to get current session options. +struct ARROW_FLIGHT_EXPORT GetSessionOptionsRequest + : public internal::BaseType { + GetSessionOptionsRequest() = default; + + std::string ToString() const; + bool Equals(const GetSessionOptionsRequest& other) const; + + using SuperT::Deserialize; + using SuperT::SerializeToString; + + /// \brief Serialize this message to its wire-format representation. + /// + /// Use `SerializeToString()` if you want a Result-returning version. + arrow::Status SerializeToString(std::string* out) const; + + /// \brief Deserialize this message from its wire-format representation. + /// + /// Use `Deserialize(serialized)` if you want a Result-returning version. + static arrow::Status Deserialize(std::string_view serialized, + GetSessionOptionsRequest* out); +}; + +/// \brief The current session options. +struct ARROW_FLIGHT_EXPORT GetSessionOptionsResult + : public internal::BaseType { + std::map session_options; + + GetSessionOptionsResult() = default; + GetSessionOptionsResult( // NOLINT runtime/explicit + std::map session_options) + : session_options(std::move(session_options)) {} + + std::string ToString() const; + bool Equals(const GetSessionOptionsResult& other) const; + + using SuperT::Deserialize; + using SuperT::SerializeToString; + + /// \brief Serialize this message to its wire-format representation. + /// + /// Use `SerializeToString()` if you want a Result-returning version. + arrow::Status SerializeToString(std::string* out) const; + + /// \brief Deserialize this message from its wire-format representation. + /// + /// Use `Deserialize(serialized)` if you want a Result-returning version. + static arrow::Status Deserialize(std::string_view serialized, + GetSessionOptionsResult* out); +}; + +/// \brief A request to close the open client session. +struct ARROW_FLIGHT_EXPORT CloseSessionRequest + : public internal::BaseType { + CloseSessionRequest() = default; + + std::string ToString() const; + bool Equals(const CloseSessionRequest& other) const; + + using SuperT::Deserialize; + using SuperT::SerializeToString; + + /// \brief Serialize this message to its wire-format representation. + /// + /// Use `SerializeToString()` if you want a Result-returning version. + arrow::Status SerializeToString(std::string* out) const; + + /// \brief Deserialize this message from its wire-format representation. + /// + /// Use `Deserialize(serialized)` if you want a Result-returning version. + static arrow::Status Deserialize(std::string_view serialized, CloseSessionRequest* out); +}; + +/// \brief The result of closing a session. +enum class CloseSessionStatus : int8_t { + // \brief The session close status is unknown. + // + // Servers should avoid using this value (send a NOT_FOUND error if the requested + // session is not known). Clients can retry the request. + kUnspecified, + // \brief The session close request is complete. + // + // Subsequent requests with the same session produce a NOT_FOUND error. + kClosed, + // \brief The session close request is in progress. + // + // The client may retry the request. + kClosing, + // \brief The session is not closeable. + // + // The client should not retry the request. + kNotClosable +}; +std::string ToString(const CloseSessionStatus& status); +std::ostream& operator<<(std::ostream& os, const CloseSessionStatus& status); + +/// \brief The result of attempting to close the client session. +struct ARROW_FLIGHT_EXPORT CloseSessionResult + : public internal::BaseType { + CloseSessionStatus status; + + CloseSessionResult() = default; + CloseSessionResult(CloseSessionStatus status) // NOLINT runtime/explicit + : status(status) {} + + std::string ToString() const; + bool Equals(const CloseSessionResult& other) const; + + using SuperT::Deserialize; + using SuperT::SerializeToString; + + /// \brief Serialize this message to its wire-format representation. + /// + /// Use `SerializeToString()` if you want a Result-returning version. + arrow::Status SerializeToString(std::string* out) const; + + /// \brief Deserialize this message from its wire-format representation. + /// + /// Use `Deserialize(serialized)` if you want a Result-returning version. + static arrow::Status Deserialize(std::string_view serialized, CloseSessionResult* out); +}; + +//------------------------------------------------------------ + +/// \brief An iterator to FlightInfo instances returned by ListFlights. +class ARROW_FLIGHT_EXPORT FlightListing { + public: + virtual ~FlightListing() = default; + + /// \brief Retrieve the next FlightInfo from the iterator. + /// \return Arrow result with a single FlightInfo. Set to \a nullptr if there + /// are none left. + virtual arrow::Result> Next() = 0; +}; + +/// \brief An iterator to Result instances returned by DoAction. +class ARROW_FLIGHT_EXPORT ResultStream { + public: + virtual ~ResultStream() = default; + + /// \brief Retrieve the next Result from the iterator. + /// \return Arrow result with a single Result. Set to \a nullptr if there are none left. + virtual arrow::Result> Next() = 0; + + /// \brief Read and drop the remaining messages to get the error (if any) from a server. + /// \return Status OK if this is no error from a server, any other status if a + /// server returns an error. + Status Drain(); +}; + +/// \brief A holder for a RecordBatch with associated Flight metadata. +struct ARROW_FLIGHT_EXPORT FlightStreamChunk { + public: + FlightStreamChunk() noexcept; + ~FlightStreamChunk(); + + std::shared_ptr data; + std::shared_ptr app_metadata; +}; + +/// \brief An interface to read Flight data with metadata. +class ARROW_FLIGHT_EXPORT MetadataRecordBatchReader { + public: + virtual ~MetadataRecordBatchReader() = default; + + /// \brief Get the schema for this stream. + virtual arrow::Result> GetSchema() = 0; + + /// \brief Get the next message from Flight. If the stream is + /// finished, then the members of \a FlightStreamChunk will be + /// nullptr. + virtual arrow::Result Next() = 0; + + /// \brief Consume entire stream as a vector of record batches + virtual arrow::Result>> ToRecordBatches(); + + /// \brief Consume entire stream as a Table + virtual arrow::Result> ToTable(); + + /// \brief Return current read statistics + virtual arrow::ipc::ReadStats stats() const = 0; +}; + +/// \brief Convert a MetadataRecordBatchReader to a regular RecordBatchReader. +ARROW_FLIGHT_EXPORT +arrow::Result> MakeRecordBatchReader( + std::shared_ptr reader); + +/// \brief An interface to write IPC payloads with metadata. +class ARROW_FLIGHT_EXPORT MetadataRecordBatchWriter : public ipc::RecordBatchWriter { + public: + virtual ~MetadataRecordBatchWriter() = default; + /// \brief Begin writing data with the given schema. Only used with \a DoExchange. + virtual Status Begin(const std::shared_ptr& schema, + const ipc::IpcWriteOptions& options) = 0; + virtual Status Begin(const std::shared_ptr& schema); + virtual Status WriteMetadata(std::shared_ptr app_metadata) = 0; + virtual Status WriteWithMetadata(const RecordBatch& batch, + std::shared_ptr app_metadata) = 0; +}; + +/// \brief A FlightListing implementation based on a vector of +/// FlightInfo objects. +/// +/// This can be iterated once, then it is consumed. +class ARROW_FLIGHT_EXPORT SimpleFlightListing : public FlightListing { + public: + explicit SimpleFlightListing(const std::vector& flights); + explicit SimpleFlightListing(std::vector&& flights); + + arrow::Result> Next() override; + + private: + int position_; + std::vector flights_; +}; + +/// \brief A ResultStream implementation based on a vector of +/// Result objects. +/// +/// This can be iterated once, then it is consumed. +class ARROW_FLIGHT_EXPORT SimpleResultStream : public ResultStream { + public: + explicit SimpleResultStream(std::vector&& results); + arrow::Result> Next() override; + + private: + std::vector results_; + size_t position_; +}; + +/// \defgroup flight-error Error Handling +/// Types for handling errors from RPCs. Flight uses a set of status +/// codes standardized across Flight implementations, so these types +/// let applications work directly with those codes instead of having +/// to translate to and from Arrow Status. +/// @{ + +/// \brief Abstract status code for an RPC as per the Flight +/// specification. +enum class TransportStatusCode { + /// \brief No error. + kOk = 0, + /// \brief An unknown error occurred. + kUnknown = 1, + /// \brief An error occurred in the transport implementation, or an + /// error internal to the service implementation occurred. + kInternal = 2, + /// \brief An argument is invalid. + kInvalidArgument = 3, + /// \brief The request timed out. + kTimedOut = 4, + /// \brief An argument is not necessarily invalid, but references + /// some resource that does not exist. Prefer over + /// kInvalidArgument where applicable. + kNotFound = 5, + /// \brief The request attempted to create some resource that does + /// not exist. + kAlreadyExists = 6, + /// \brief The request was explicitly cancelled. + kCancelled = 7, + /// \brief The client is not authenticated. + kUnauthenticated = 8, + /// \brief The client is not authorized to perform this request. + kUnauthorized = 9, + /// \brief The request is not implemented + kUnimplemented = 10, + /// \brief There is a network connectivity error, or some resource + /// is otherwise unavailable. Most likely a temporary condition. + kUnavailable = 11, +}; + +/// \brief Convert a code to a string. +std::string ToString(TransportStatusCode code); + +/// \brief An error from an RPC call, using Flight error codes directly +/// instead of trying to translate to Arrow Status. +/// +/// Currently, only attached to the Status passed to AsyncListener::OnFinish. +class ARROW_FLIGHT_EXPORT TransportStatusDetail : public StatusDetail { + public: + constexpr static const char* kTypeId = "flight::TransportStatusDetail"; + explicit TransportStatusDetail(TransportStatusCode code, std::string message, + std::vector> details) + : code_(code), message_(std::move(message)), details_(std::move(details)) {} + const char* type_id() const override { return kTypeId; } + std::string ToString() const override; + + static std::optional> Unwrap( + const Status& status); + + TransportStatusCode code() const { return code_; } + std::string_view message() const { return message_; } + const std::vector>& details() const { + return details_; + } + + private: + TransportStatusCode code_; + std::string message_; + std::vector> details_; +}; + +/// @} + +} // namespace flight +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/types_async.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/types_async.h new file mode 100644 index 0000000000000000000000000000000000000000..3cdc999e3556fafc03f359520e0f353e9561751f --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/types_async.h @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "arrow/flight/type_fwd.h" +#include "arrow/flight/types.h" +#include "arrow/ipc/options.h" +#include "arrow/type_fwd.h" + +namespace arrow::flight { + +/// \defgroup flight-async Async Flight Types +/// Common types used for asynchronous Flight APIs. +/// @{ + +/// \brief Non-templated state for an async RPC. +class ARROW_FLIGHT_EXPORT AsyncListenerBase { + public: + AsyncListenerBase(); + virtual ~AsyncListenerBase(); + + /// \brief Request cancellation of the RPC. + /// + /// The RPC is not cancelled until AsyncListener::OnFinish is called. + void TryCancel(); + + private: + friend class arrow::flight::internal::ClientTransport; + + /// Transport-specific state for this RPC. Transport + /// implementations may store and retrieve state here via + /// ClientTransport::SetAsyncRpc and ClientTransport::GetAsyncRpc. + std::unique_ptr rpc_state_; +}; + +/// \brief Callbacks for results from async RPCs. +/// +/// A single listener may not be used for multiple concurrent RPC +/// calls. The application MUST hold the listener alive until +/// OnFinish() is called and has finished. +template +class ARROW_FLIGHT_EXPORT AsyncListener : public AsyncListenerBase { + public: + /// \brief Get the next server result. + /// + /// This will never be called concurrently with itself or OnFinish. + virtual void OnNext(T message) = 0; + /// \brief Get the final status. + /// + /// This will never be called concurrently with itself or OnNext. If the + /// error comes from the remote server, then a TransportStatusDetail will be + /// attached. Otherwise, the error is generated by the client-side + /// transport and will not have a TransportStatusDetail. + virtual void OnFinish(Status status) = 0; +}; + +/// @} + +} // namespace arrow::flight diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/visibility.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/visibility.h new file mode 100644 index 0000000000000000000000000000000000000000..c85a5f41b14fd266e3099e44ba2234e595ab8c96 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/flight/visibility.h @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#if defined(_WIN32) || defined(__CYGWIN__) +# if defined(_MSC_VER) +# pragma warning(push) +# pragma warning(disable : 4251) +# else +# pragma GCC diagnostic ignored "-Wattributes" +# endif + +# ifdef ARROW_FLIGHT_STATIC +# define ARROW_FLIGHT_EXPORT +# elif defined(ARROW_FLIGHT_EXPORTING) +# define ARROW_FLIGHT_EXPORT __declspec(dllexport) +# else +# define ARROW_FLIGHT_EXPORT __declspec(dllimport) +# endif + +# define ARROW_FLIGHT_NO_EXPORT +#else // Not Windows +# ifndef ARROW_FLIGHT_EXPORT +# define ARROW_FLIGHT_EXPORT __attribute__((visibility("default"))) +# endif +# ifndef ARROW_FLIGHT_NO_EXPORT +# define ARROW_FLIGHT_NO_EXPORT __attribute__((visibility("hidden"))) +# endif +#endif // Non-Windows + +#if defined(_MSC_VER) +# pragma warning(pop) +#endif diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/api.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/api.h new file mode 100644 index 0000000000000000000000000000000000000000..b178e04f4cc95b0c35700f67b98c1b4d5ce146d8 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/api.h @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "arrow/io/buffered.h" +#include "arrow/io/compressed.h" +#include "arrow/io/file.h" +#include "arrow/io/hdfs.h" +#include "arrow/io/interfaces.h" +#include "arrow/io/memory.h" diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/buffered.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/buffered.h new file mode 100644 index 0000000000000000000000000000000000000000..2fb4bb629624062dd4ba902f9e49755d219e928e --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/buffered.h @@ -0,0 +1,168 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Buffered stream implementations + +#pragma once + +#include +#include +#include + +#include "arrow/io/concurrency.h" +#include "arrow/io/interfaces.h" +#include "arrow/util/visibility.h" + +namespace arrow { + +class Buffer; +class MemoryPool; +class Status; + +namespace io { + +class ARROW_EXPORT BufferedOutputStream : public OutputStream { + public: + ~BufferedOutputStream() override; + + /// \brief Create a buffered output stream wrapping the given output stream. + /// \param[in] buffer_size the size of the temporary write buffer + /// \param[in] pool a MemoryPool to use for allocations + /// \param[in] raw another OutputStream + /// \return the created BufferedOutputStream + static Result> Create( + int64_t buffer_size, MemoryPool* pool, std::shared_ptr raw); + + /// \brief Resize internal buffer + /// \param[in] new_buffer_size the new buffer size + /// \return Status + Status SetBufferSize(int64_t new_buffer_size); + + /// \brief Return the current size of the internal buffer + int64_t buffer_size() const; + + /// \brief Return the number of remaining bytes that have not been flushed to + /// the raw OutputStream + int64_t bytes_buffered() const; + + /// \brief Flush any buffered writes and release the raw + /// OutputStream. Further operations on this object are invalid + /// \return the underlying OutputStream + Result> Detach(); + + // OutputStream interface + + /// \brief Close the buffered output stream. This implicitly closes the + /// underlying raw output stream. + Status Close() override; + Status Abort() override; + bool closed() const override; + + Result Tell() const override; + // Write bytes to the stream. Thread-safe + Status Write(const void* data, int64_t nbytes) override; + Status Write(const std::shared_ptr& data) override; + + Status Flush() override; + + /// \brief Return the underlying raw output stream. + std::shared_ptr raw() const; + + private: + explicit BufferedOutputStream(std::shared_ptr raw, MemoryPool* pool); + + class ARROW_NO_EXPORT Impl; + std::unique_ptr impl_; +}; + +/// \class BufferedInputStream +/// \brief An InputStream that performs buffered reads from an unbuffered +/// InputStream, which can mitigate the overhead of many small reads in some +/// cases +class ARROW_EXPORT BufferedInputStream + : public internal::InputStreamConcurrencyWrapper { + public: + ~BufferedInputStream() override; + + /// \brief Create a BufferedInputStream from a raw InputStream + /// \param[in] buffer_size the size of the temporary read buffer + /// \param[in] pool a MemoryPool to use for allocations + /// \param[in] raw a raw InputStream + /// \param[in] raw_read_bound a bound on the maximum number of bytes + /// to read from the raw input stream. The default -1 indicates that + /// it is unbounded + /// \return the created BufferedInputStream + static Result> Create( + int64_t buffer_size, MemoryPool* pool, std::shared_ptr raw, + int64_t raw_read_bound = -1); + + /// \brief Resize internal read buffer; calls to Read(...) will read at least + /// this many bytes from the raw InputStream if possible. + /// \param[in] new_buffer_size the new read buffer size + /// \return Status + Status SetBufferSize(int64_t new_buffer_size); + + /// \brief Return the number of remaining bytes in the read buffer + int64_t bytes_buffered() const; + + /// \brief Return the current size of the internal buffer + int64_t buffer_size() const; + + /// \brief Release the raw InputStream. Any data buffered will be + /// discarded. Further operations on this object are invalid + /// \return raw the underlying InputStream + std::shared_ptr Detach(); + + /// \brief Return the unbuffered InputStream + std::shared_ptr raw() const; + + // InputStream APIs + + bool closed() const override; + Result> ReadMetadata() override; + Future> ReadMetadataAsync( + const IOContext& io_context) override; + + private: + friend InputStreamConcurrencyWrapper; + + explicit BufferedInputStream(std::shared_ptr raw, MemoryPool* pool, + int64_t raw_total_bytes_bound); + + Status DoClose(); + Status DoAbort() override; + + /// \brief Returns the position of the buffered stream, though the position + /// of the unbuffered stream may be further advanced. + Result DoTell() const; + + Result DoRead(int64_t nbytes, void* out); + + /// \brief Read into buffer. + Result> DoRead(int64_t nbytes); + + /// \brief Return a zero-copy string view referencing buffered data, + /// but do not advance the position of the stream. Buffers data and + /// expands the buffer size if necessary + Result DoPeek(int64_t nbytes) override; + + class ARROW_NO_EXPORT Impl; + std::unique_ptr impl_; +}; + +} // namespace io +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/caching.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/caching.h new file mode 100644 index 0000000000000000000000000000000000000000..2a8ecb5fecfb8f113eb2cef94d775a8ee5a01fe9 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/caching.h @@ -0,0 +1,157 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "arrow/io/interfaces.h" +#include "arrow/util/type_fwd.h" +#include "arrow/util/visibility.h" + +namespace arrow { +namespace io { + +struct ARROW_EXPORT CacheOptions { + static constexpr double kDefaultIdealBandwidthUtilizationFrac = 0.9; + static constexpr int64_t kDefaultMaxIdealRequestSizeMib = 64; + + /// \brief The maximum distance in bytes between two consecutive + /// ranges; beyond this value, ranges are not combined + int64_t hole_size_limit; + /// \brief The maximum size in bytes of a combined range; if + /// combining two consecutive ranges would produce a range of a + /// size greater than this, they are not combined + int64_t range_size_limit; + /// \brief A lazy cache does not perform any I/O until requested. + /// lazy = false: request all byte ranges when PreBuffer or WillNeed is called. + /// lazy = True, prefetch_limit = 0: request merged byte ranges only after the reader + /// needs them. + /// lazy = True, prefetch_limit = k: prefetch up to k merged byte ranges ahead of the + /// range that is currently being read. + bool lazy; + /// \brief The maximum number of ranges to be prefetched. This is only used + /// for lazy cache to asynchronously read some ranges after reading the target range. + int64_t prefetch_limit = 0; + + bool operator==(const CacheOptions& other) const { + return hole_size_limit == other.hole_size_limit && + range_size_limit == other.range_size_limit && lazy == other.lazy && + prefetch_limit == other.prefetch_limit; + } + + /// \brief Construct CacheOptions from network storage metrics (e.g. S3). + /// + /// \param[in] time_to_first_byte_millis Seek-time or Time-To-First-Byte (TTFB) in + /// milliseconds, also called call setup latency of a new read request. + /// The value is a positive integer. + /// \param[in] transfer_bandwidth_mib_per_sec Data transfer Bandwidth (BW) in MiB/sec + /// (per connection). + /// The value is a positive integer. + /// \param[in] ideal_bandwidth_utilization_frac Transfer bandwidth utilization fraction + /// (per connection) to maximize the net data load. + /// The value is a positive double precision number less than 1. + /// \param[in] max_ideal_request_size_mib The maximum single data request size (in MiB) + /// to maximize the net data load. + /// The value is a positive integer. + /// \return A new instance of CacheOptions. + static CacheOptions MakeFromNetworkMetrics( + int64_t time_to_first_byte_millis, int64_t transfer_bandwidth_mib_per_sec, + double ideal_bandwidth_utilization_frac = kDefaultIdealBandwidthUtilizationFrac, + int64_t max_ideal_request_size_mib = kDefaultMaxIdealRequestSizeMib); + + static CacheOptions Defaults(); + static CacheOptions LazyDefaults(); +}; + +namespace internal { + +/// \brief A read cache designed to hide IO latencies when reading. +/// +/// This class takes multiple byte ranges that an application expects to read, and +/// coalesces them into fewer, larger read requests, which benefits performance on some +/// filesystems, particularly remote ones like Amazon S3. By default, it also issues +/// these read requests in parallel up front. +/// +/// To use: +/// 1. Cache() the ranges you expect to read in the future. Ideally, these ranges have +/// the exact offset and length that will later be read. The cache will combine those +/// ranges according to parameters (see constructor). +/// +/// By default, the cache will also start fetching the combined ranges in parallel in +/// the background, unless CacheOptions.lazy is set. +/// +/// 2. Call WaitFor() to be notified when the given ranges have been read. If +/// CacheOptions.lazy is set, I/O will be triggered in the background here instead. +/// This can be done in parallel (e.g. if parsing a file, call WaitFor() for each +/// chunk of the file that can be parsed in parallel). +/// +/// 3. Call Read() to retrieve the actual data for the given ranges. +/// A synchronous application may skip WaitFor() and just call Read() - it will still +/// benefit from coalescing and parallel fetching. +class ARROW_EXPORT ReadRangeCache { + public: + static constexpr int64_t kDefaultHoleSizeLimit = 8192; + static constexpr int64_t kDefaultRangeSizeLimit = 32 * 1024 * 1024; + + /// Construct a read cache with default + explicit ReadRangeCache(std::shared_ptr file, IOContext ctx) + : ReadRangeCache(file, file.get(), std::move(ctx), CacheOptions::Defaults()) {} + + /// Construct a read cache with given options + explicit ReadRangeCache(std::shared_ptr file, IOContext ctx, + CacheOptions options) + : ReadRangeCache(file, file.get(), std::move(ctx), options) {} + + /// Construct a read cache with an unowned file + ReadRangeCache(RandomAccessFile* file, IOContext ctx, CacheOptions options) + : ReadRangeCache(NULLPTR, file, std::move(ctx), options) {} + + ~ReadRangeCache(); + + /// \brief Cache the given ranges in the background. + /// + /// The caller must ensure that the ranges do not overlap with each other, + /// nor with previously cached ranges. Otherwise, behaviour will be undefined. + Status Cache(std::vector ranges); + + /// \brief Read a range previously given to Cache(). + Result> Read(ReadRange range); + + /// \brief Wait until all ranges added so far have been cached. + Future<> Wait(); + + /// \brief Wait until all given ranges have been cached. + Future<> WaitFor(std::vector ranges); + + protected: + struct Impl; + struct LazyImpl; + + ReadRangeCache(std::shared_ptr owned_file, RandomAccessFile* file, + IOContext ctx, CacheOptions options); + + std::unique_ptr impl_; +}; + +} // namespace internal +} // namespace io +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/compressed.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/compressed.h new file mode 100644 index 0000000000000000000000000000000000000000..da997ab44a6e25c91a9aed99c1ad971427c7bd19 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/compressed.h @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Compressed stream implementations + +#pragma once + +#include +#include + +#include "arrow/io/concurrency.h" +#include "arrow/io/interfaces.h" +#include "arrow/util/visibility.h" + +namespace arrow { + +class MemoryPool; +class Status; + +namespace util { + +class Codec; + +} // namespace util + +namespace io { + +class ARROW_EXPORT CompressedOutputStream : public OutputStream { + public: + ~CompressedOutputStream() override; + + /// \brief Create a compressed output stream wrapping the given output stream. + /// + /// The codec must be capable of streaming compression. Some codecs, + /// like Snappy, are not able to do so. + static Result> Make( + util::Codec* codec, const std::shared_ptr& raw, + MemoryPool* pool = default_memory_pool()); + + // OutputStream interface + + /// \brief Close the compressed output stream. This implicitly closes the + /// underlying raw output stream. + Status Close() override; + Status Abort() override; + bool closed() const override; + + Result Tell() const override; + + Status Write(const void* data, int64_t nbytes) override; + /// \cond FALSE + using Writable::Write; + /// \endcond + Status Flush() override; + + /// \brief Return the underlying raw output stream. + std::shared_ptr raw() const; + + private: + ARROW_DISALLOW_COPY_AND_ASSIGN(CompressedOutputStream); + + CompressedOutputStream() = default; + + class ARROW_NO_EXPORT Impl; + std::unique_ptr impl_; +}; + +class ARROW_EXPORT CompressedInputStream + : public internal::InputStreamConcurrencyWrapper { + public: + ~CompressedInputStream() override; + + /// \brief Create a compressed input stream wrapping the given input stream. + /// + /// The codec must be capable of streaming decompression. Some codecs, + /// like Snappy, are not able to do so. + static Result> Make( + util::Codec* codec, const std::shared_ptr& raw, + MemoryPool* pool = default_memory_pool()); + + // InputStream interface + + bool closed() const override; + Result> ReadMetadata() override; + Future> ReadMetadataAsync( + const IOContext& io_context) override; + + /// \brief Return the underlying raw input stream. + std::shared_ptr raw() const; + + private: + friend InputStreamConcurrencyWrapper; + ARROW_DISALLOW_COPY_AND_ASSIGN(CompressedInputStream); + + CompressedInputStream() = default; + + /// \brief Close the compressed input stream. This implicitly closes the + /// underlying raw input stream. + Status DoClose(); + Status DoAbort() override; + Result DoTell() const; + Result DoRead(int64_t nbytes, void* out); + Result> DoRead(int64_t nbytes); + + class ARROW_NO_EXPORT Impl; + std::unique_ptr impl_; +}; + +} // namespace io +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/concurrency.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/concurrency.h new file mode 100644 index 0000000000000000000000000000000000000000..1ab6a22d474837faf309a654e9bec367dbe7cfef --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/concurrency.h @@ -0,0 +1,263 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "arrow/io/interfaces.h" +#include "arrow/result.h" +#include "arrow/status.h" +#include "arrow/util/checked_cast.h" +#include "arrow/util/macros.h" +#include "arrow/util/visibility.h" + +namespace arrow { +namespace io { +namespace internal { + +template +class SharedLockGuard { + public: + explicit SharedLockGuard(LockType* lock) : lock_(lock) { lock_->LockShared(); } + + ~SharedLockGuard() { lock_->UnlockShared(); } + + protected: + LockType* lock_; +}; + +template +class ExclusiveLockGuard { + public: + explicit ExclusiveLockGuard(LockType* lock) : lock_(lock) { lock_->LockExclusive(); } + + ~ExclusiveLockGuard() { lock_->UnlockExclusive(); } + + protected: + LockType* lock_; +}; + +// Debug concurrency checker that marks "shared" and "exclusive" code sections, +// aborting if the concurrency rules get violated. Does nothing in release mode. +// Note that we intentionally use the same class declaration in debug and +// release builds in order to avoid runtime failures when e.g. loading a +// release-built DLL with a debug-built application, or the reverse. + +class ARROW_EXPORT SharedExclusiveChecker { + public: + SharedExclusiveChecker(); + void LockShared(); + void UnlockShared(); + void LockExclusive(); + void UnlockExclusive(); + + SharedLockGuard shared_guard() { + return SharedLockGuard(this); + } + + ExclusiveLockGuard exclusive_guard() { + return ExclusiveLockGuard(this); + } + + protected: + struct Impl; + std::shared_ptr impl_; +}; + +// Concurrency wrappers for IO classes that check the correctness of +// concurrent calls to various methods. It is not necessary to wrap all +// IO classes with these, only a few core classes that get used in tests. +// +// We're not using virtual inheritance here as virtual bases have poorly +// understood semantic overhead which we'd be passing on to implementers +// and users of these interfaces. Instead, we just duplicate the method +// wrappers between those two classes. + +template +class InputStreamConcurrencyWrapper : public InputStream { + public: + Status Close() final { + auto guard = lock_.exclusive_guard(); + return derived()->DoClose(); + } + + Status Abort() final { + auto guard = lock_.exclusive_guard(); + return derived()->DoAbort(); + } + + Result Tell() const final { + auto guard = lock_.exclusive_guard(); + return derived()->DoTell(); + } + + Result Read(int64_t nbytes, void* out) final { + auto guard = lock_.exclusive_guard(); + return derived()->DoRead(nbytes, out); + } + + Result> Read(int64_t nbytes) final { + auto guard = lock_.exclusive_guard(); + return derived()->DoRead(nbytes); + } + + Result Peek(int64_t nbytes) final { + auto guard = lock_.exclusive_guard(); + return derived()->DoPeek(nbytes); + } + + /* + Methods to implement in derived class: + + Status DoClose(); + Result DoTell() const; + Result DoRead(int64_t nbytes, void* out); + Result> DoRead(int64_t nbytes); + + And optionally: + + Status DoAbort() override; + Result DoPeek(int64_t nbytes) override; + + These methods should be protected in the derived class and + InputStreamConcurrencyWrapper declared as a friend with + + friend InputStreamConcurrencyWrapper; + */ + + protected: + // Default implementations. They are virtual because the derived class may + // have derived classes itself. + virtual Status DoAbort() { return derived()->DoClose(); } + + virtual Result DoPeek(int64_t ARROW_ARG_UNUSED(nbytes)) { + return Status::NotImplemented("Peek not implemented"); + } + + Derived* derived() { return ::arrow::internal::checked_cast(this); } + + const Derived* derived() const { + return ::arrow::internal::checked_cast(this); + } + + mutable SharedExclusiveChecker lock_; +}; + +template +class RandomAccessFileConcurrencyWrapper : public RandomAccessFile { + public: + Status Close() final { + auto guard = lock_.exclusive_guard(); + return derived()->DoClose(); + } + + Status Abort() final { + auto guard = lock_.exclusive_guard(); + return derived()->DoAbort(); + } + + Result Tell() const final { + auto guard = lock_.exclusive_guard(); + return derived()->DoTell(); + } + + Result Read(int64_t nbytes, void* out) final { + auto guard = lock_.exclusive_guard(); + return derived()->DoRead(nbytes, out); + } + + Result> Read(int64_t nbytes) final { + auto guard = lock_.exclusive_guard(); + return derived()->DoRead(nbytes); + } + + Result Peek(int64_t nbytes) final { + auto guard = lock_.exclusive_guard(); + return derived()->DoPeek(nbytes); + } + + Status Seek(int64_t position) final { + auto guard = lock_.exclusive_guard(); + return derived()->DoSeek(position); + } + + Result GetSize() final { + auto guard = lock_.shared_guard(); + return derived()->DoGetSize(); + } + + // NOTE: ReadAt doesn't use stream pointer, but it is allowed to update it + // (it's the case on Windows when using ReadFileEx). + // So any method that relies on the current position (even if it doesn't + // update it, such as Peek) cannot run in parallel with ReadAt and has + // to use the exclusive_guard. + + Result ReadAt(int64_t position, int64_t nbytes, void* out) final { + auto guard = lock_.shared_guard(); + return derived()->DoReadAt(position, nbytes, out); + } + + Result> ReadAt(int64_t position, int64_t nbytes) final { + auto guard = lock_.shared_guard(); + return derived()->DoReadAt(position, nbytes); + } + + /* + Methods to implement in derived class: + + Status DoClose(); + Result DoTell() const; + Result DoRead(int64_t nbytes, void* out); + Result> DoRead(int64_t nbytes); + Status DoSeek(int64_t position); + Result DoGetSize() + Result DoReadAt(int64_t position, int64_t nbytes, void* out); + Result> DoReadAt(int64_t position, int64_t nbytes); + + And optionally: + + Status DoAbort() override; + Result DoPeek(int64_t nbytes) override; + + These methods should be protected in the derived class and + RandomAccessFileConcurrencyWrapper declared as a friend with + + friend RandomAccessFileConcurrencyWrapper; + */ + + protected: + // Default implementations. They are virtual because the derived class may + // have derived classes itself. + virtual Status DoAbort() { return derived()->DoClose(); } + + virtual Result DoPeek(int64_t ARROW_ARG_UNUSED(nbytes)) { + return Status::NotImplemented("Peek not implemented"); + } + + Derived* derived() { return ::arrow::internal::checked_cast(this); } + + const Derived* derived() const { + return ::arrow::internal::checked_cast(this); + } + + mutable SharedExclusiveChecker lock_; +}; + +} // namespace internal +} // namespace io +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/file.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/file.h new file mode 100644 index 0000000000000000000000000000000000000000..8f7235a138ae7a84d05dbb743edc0437895e44ca --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/file.h @@ -0,0 +1,221 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// IO interface implementations for OS files + +#pragma once + +#include +#include +#include +#include + +#include "arrow/io/concurrency.h" +#include "arrow/io/interfaces.h" +#include "arrow/util/visibility.h" + +namespace arrow { + +class Buffer; +class MemoryPool; +class Status; + +namespace io { + +/// \brief An operating system file open in write-only mode. +class ARROW_EXPORT FileOutputStream : public OutputStream { + public: + ~FileOutputStream() override; + + /// \brief Open a local file for writing, truncating any existing file + /// \param[in] path with UTF8 encoding + /// \param[in] append append to existing file, otherwise truncate to 0 bytes + /// \return an open FileOutputStream + /// + /// When opening a new file, any existing file with the indicated path is + /// truncated to 0 bytes, deleting any existing data + static Result> Open(const std::string& path, + bool append = false); + + /// \brief Open a file descriptor for writing. The underlying file isn't + /// truncated. + /// \param[in] fd file descriptor + /// \return an open FileOutputStream + /// + /// The file descriptor becomes owned by the OutputStream, and will be closed + /// on Close() or destruction. + static Result> Open(int fd); + + // OutputStream interface + Status Close() override; + bool closed() const override; + Result Tell() const override; + + // Write bytes to the stream. Thread-safe + Status Write(const void* data, int64_t nbytes) override; + /// \cond FALSE + using Writable::Write; + /// \endcond + + int file_descriptor() const; + + private: + FileOutputStream(); + + class ARROW_NO_EXPORT FileOutputStreamImpl; + std::unique_ptr impl_; +}; + +/// \brief An operating system file open in read-only mode. +/// +/// Reads through this implementation are unbuffered. If many small reads +/// need to be issued, it is recommended to use a buffering layer for good +/// performance. +class ARROW_EXPORT ReadableFile + : public internal::RandomAccessFileConcurrencyWrapper { + public: + ~ReadableFile() override; + + /// \brief Open a local file for reading + /// \param[in] path with UTF8 encoding + /// \param[in] pool a MemoryPool for memory allocations + /// \return ReadableFile instance + static Result> Open( + const std::string& path, MemoryPool* pool = default_memory_pool()); + + /// \brief Open a local file for reading + /// \param[in] fd file descriptor + /// \param[in] pool a MemoryPool for memory allocations + /// \return ReadableFile instance + /// + /// The file descriptor becomes owned by the ReadableFile, and will be closed + /// on Close() or destruction. + static Result> Open( + int fd, MemoryPool* pool = default_memory_pool()); + + bool closed() const override; + + int file_descriptor() const; + + Status WillNeed(const std::vector& ranges) override; + + private: + friend RandomAccessFileConcurrencyWrapper; + + explicit ReadableFile(MemoryPool* pool); + + Status DoClose(); + Result DoTell() const; + Result DoRead(int64_t nbytes, void* buffer); + Result> DoRead(int64_t nbytes); + + /// \brief Thread-safe implementation of ReadAt + Result DoReadAt(int64_t position, int64_t nbytes, void* out); + + /// \brief Thread-safe implementation of ReadAt + Result> DoReadAt(int64_t position, int64_t nbytes); + + Result DoGetSize(); + Status DoSeek(int64_t position); + + class ARROW_NO_EXPORT ReadableFileImpl; + std::unique_ptr impl_; +}; + +/// \brief A file interface that uses memory-mapped files for memory interactions +/// +/// This implementation supports zero-copy reads. The same class is used +/// for both reading and writing. +/// +/// If opening a file in a writable mode, it is not truncated first as with +/// FileOutputStream. +class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface { + public: + ~MemoryMappedFile() override; + + /// Create new file with indicated size, return in read/write mode + static Result> Create(const std::string& path, + int64_t size); + + // mmap() with whole file + static Result> Open(const std::string& path, + FileMode::type mode); + + // mmap() with a region of file, the offset must be a multiple of the page size + static Result> Open(const std::string& path, + FileMode::type mode, + const int64_t offset, + const int64_t length); + + Status Close() override; + + bool closed() const override; + + Result Tell() const override; + + Status Seek(int64_t position) override; + + // Required by RandomAccessFile, copies memory into out. Not thread-safe + Result Read(int64_t nbytes, void* out) override; + + // Zero copy read, moves position pointer. Not thread-safe + Result> Read(int64_t nbytes) override; + + // Zero-copy read, leaves position unchanged. Acquires a reader lock + // for the duration of slice creation (typically very short). Is thread-safe. + Result> ReadAt(int64_t position, int64_t nbytes) override; + + // Raw copy of the memory at specified position. Thread-safe, but + // locks out other readers for the duration of memcpy. Prefer the + // zero copy method + Result ReadAt(int64_t position, int64_t nbytes, void* out) override; + + // Synchronous ReadAsync override + Future> ReadAsync(const IOContext&, int64_t position, + int64_t nbytes) override; + + Status WillNeed(const std::vector& ranges) override; + + bool supports_zero_copy() const override; + + /// Write data at the current position in the file. Thread-safe + Status Write(const void* data, int64_t nbytes) override; + /// \cond FALSE + using Writable::Write; + /// \endcond + + /// Set the size of the map to new_size. + Status Resize(int64_t new_size); + + /// Write data at a particular position in the file. Thread-safe + Status WriteAt(int64_t position, const void* data, int64_t nbytes) override; + + Result GetSize() override; + + int file_descriptor() const; + + private: + MemoryMappedFile(); + + Status WriteInternal(const void* data, int64_t nbytes); + + class ARROW_NO_EXPORT MemoryMap; + std::shared_ptr memory_map_; +}; + +} // namespace io +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/hdfs.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/hdfs.h new file mode 100644 index 0000000000000000000000000000000000000000..c6a67b67feabbf6ba97f9422bff72a2a293727aa --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/hdfs.h @@ -0,0 +1,284 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "arrow/io/interfaces.h" +#include "arrow/util/macros.h" +#include "arrow/util/visibility.h" + +namespace arrow { + +class Buffer; +class MemoryPool; +class Status; + +namespace io { + +class HdfsReadableFile; +class HdfsOutputStream; + +/// DEPRECATED. Use the FileSystem API in arrow::fs instead. +struct ObjectType { + enum type { FILE, DIRECTORY }; +}; + +/// DEPRECATED. Use the FileSystem API in arrow::fs instead. +struct ARROW_EXPORT FileStatistics { + /// Size of file, -1 if finding length is unsupported + int64_t size; + ObjectType::type kind; +}; + +class ARROW_EXPORT FileSystem { + public: + virtual ~FileSystem() = default; + + virtual Status MakeDirectory(const std::string& path) = 0; + + virtual Status DeleteDirectory(const std::string& path) = 0; + + virtual Status GetChildren(const std::string& path, + std::vector* listing) = 0; + + virtual Status Rename(const std::string& src, const std::string& dst) = 0; + + virtual Status Stat(const std::string& path, FileStatistics* stat) = 0; +}; + +struct HdfsPathInfo { + ObjectType::type kind; + + std::string name; + std::string owner; + std::string group; + + // Access times in UNIX timestamps (seconds) + int64_t size; + int64_t block_size; + + int32_t last_modified_time; + int32_t last_access_time; + + int16_t replication; + int16_t permissions; +}; + +struct HdfsConnectionConfig { + std::string host; + int port; + std::string user; + std::string kerb_ticket; + std::unordered_map extra_conf; +}; + +class ARROW_EXPORT HadoopFileSystem : public FileSystem { + public: + ~HadoopFileSystem() override; + + // Connect to an HDFS cluster given a configuration + // + // @param config (in): configuration for connecting + // @param fs (out): the created client + // @returns Status + static Status Connect(const HdfsConnectionConfig* config, + std::shared_ptr* fs); + + // Create directory and all parents + // + // @param path (in): absolute HDFS path + // @returns Status + Status MakeDirectory(const std::string& path) override; + + // Delete file or directory + // @param path absolute path to data + // @param recursive if path is a directory, delete contents as well + // @returns error status on failure + Status Delete(const std::string& path, bool recursive = false); + + Status DeleteDirectory(const std::string& path) override; + + // Disconnect from cluster + // + // @returns Status + Status Disconnect(); + + // @param path (in): absolute HDFS path + // @returns bool, true if the path exists, false if not (or on error) + bool Exists(const std::string& path); + + // @param path (in): absolute HDFS path + // @param info (out) + // @returns Status + Status GetPathInfo(const std::string& path, HdfsPathInfo* info); + + // @param nbytes (out): total capacity of the filesystem + // @returns Status + Status GetCapacity(int64_t* nbytes); + + // @param nbytes (out): total bytes used of the filesystem + // @returns Status + Status GetUsed(int64_t* nbytes); + + Status GetChildren(const std::string& path, std::vector* listing) override; + + /// List directory contents + /// + /// If path is a relative path, returned values will be absolute paths or URIs + /// starting from the current working directory. + Status ListDirectory(const std::string& path, std::vector* listing); + + /// Return the filesystem's current working directory. + /// + /// The working directory is the base path for all relative paths given to + /// other APIs. + /// NOTE: this actually returns a URI. + Status GetWorkingDirectory(std::string* out); + + /// Change + /// + /// @param path file path to change + /// @param owner pass null for no change + /// @param group pass null for no change + Status Chown(const std::string& path, const char* owner, const char* group); + + /// Change path permissions + /// + /// \param path Absolute path in file system + /// \param mode Mode bitset + /// \return Status + Status Chmod(const std::string& path, int mode); + + // Move file or directory from source path to destination path within the + // current filesystem + Status Rename(const std::string& src, const std::string& dst) override; + + Status Copy(const std::string& src, const std::string& dst); + + Status Move(const std::string& src, const std::string& dst); + + Status Stat(const std::string& path, FileStatistics* stat) override; + + // TODO(wesm): GetWorkingDirectory, SetWorkingDirectory + + // Open an HDFS file in READ mode. Returns error + // status if the file is not found. + // + // @param path complete file path + Status OpenReadable(const std::string& path, int32_t buffer_size, + std::shared_ptr* file); + + Status OpenReadable(const std::string& path, int32_t buffer_size, + const io::IOContext& io_context, + std::shared_ptr* file); + + Status OpenReadable(const std::string& path, std::shared_ptr* file); + + Status OpenReadable(const std::string& path, const io::IOContext& io_context, + std::shared_ptr* file); + + // FileMode::WRITE options + // @param path complete file path + // @param buffer_size 0 by default + // @param replication 0 by default + // @param default_block_size 0 by default + Status OpenWritable(const std::string& path, bool append, int32_t buffer_size, + int16_t replication, int64_t default_block_size, + std::shared_ptr* file); + + Status OpenWritable(const std::string& path, bool append, + std::shared_ptr* file); + + private: + friend class HdfsReadableFile; + friend class HdfsOutputStream; + + class ARROW_NO_EXPORT HadoopFileSystemImpl; + std::unique_ptr impl_; + + HadoopFileSystem(); + ARROW_DISALLOW_COPY_AND_ASSIGN(HadoopFileSystem); +}; + +class ARROW_EXPORT HdfsReadableFile : public RandomAccessFile { + public: + ~HdfsReadableFile() override; + + Status Close() override; + + bool closed() const override; + + // NOTE: If you wish to read a particular range of a file in a multithreaded + // context, you may prefer to use ReadAt to avoid locking issues + Result Read(int64_t nbytes, void* out) override; + Result> Read(int64_t nbytes) override; + Result ReadAt(int64_t position, int64_t nbytes, void* out) override; + Result> ReadAt(int64_t position, int64_t nbytes) override; + + Status Seek(int64_t position) override; + Result Tell() const override; + Result GetSize() override; + + private: + explicit HdfsReadableFile(const io::IOContext&); + + class ARROW_NO_EXPORT HdfsReadableFileImpl; + std::unique_ptr impl_; + + friend class HadoopFileSystem::HadoopFileSystemImpl; + + ARROW_DISALLOW_COPY_AND_ASSIGN(HdfsReadableFile); +}; + +// Naming this file OutputStream because it does not support seeking (like the +// WritableFile interface) +class ARROW_EXPORT HdfsOutputStream : public OutputStream { + public: + ~HdfsOutputStream() override; + + Status Close() override; + + bool closed() const override; + + using OutputStream::Write; + Status Write(const void* buffer, int64_t nbytes) override; + + Status Flush() override; + + Result Tell() const override; + + private: + class ARROW_NO_EXPORT HdfsOutputStreamImpl; + std::unique_ptr impl_; + + friend class HadoopFileSystem::HadoopFileSystemImpl; + + HdfsOutputStream(); + + ARROW_DISALLOW_COPY_AND_ASSIGN(HdfsOutputStream); +}; + +ARROW_EXPORT Status HaveLibHdfs(); + +} // namespace io +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/interfaces.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/interfaces.h new file mode 100644 index 0000000000000000000000000000000000000000..b21794d9566f18d185083cb18aac2c4012970b22 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/interfaces.h @@ -0,0 +1,362 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "arrow/io/type_fwd.h" +#include "arrow/type_fwd.h" +#include "arrow/util/cancel.h" +#include "arrow/util/macros.h" +#include "arrow/util/type_fwd.h" +#include "arrow/util/visibility.h" + +namespace arrow { +namespace io { + +struct ReadRange { + int64_t offset; + int64_t length; + + friend bool operator==(const ReadRange& left, const ReadRange& right) { + return (left.offset == right.offset && left.length == right.length); + } + friend bool operator!=(const ReadRange& left, const ReadRange& right) { + return !(left == right); + } + + bool Contains(const ReadRange& other) const { + return (offset <= other.offset && offset + length >= other.offset + other.length); + } +}; + +/// EXPERIMENTAL: options provider for IO tasks +/// +/// Includes an Executor (which will be used to execute asynchronous reads), +/// a MemoryPool (which will be used to allocate buffers when zero copy reads +/// are not possible), and an external id (in case the executor receives tasks from +/// multiple sources and must distinguish tasks associated with this IOContext). +struct ARROW_EXPORT IOContext { + // No specified executor: will use a global IO thread pool + IOContext() : IOContext(default_memory_pool(), StopToken::Unstoppable()) {} + + explicit IOContext(StopToken stop_token) + : IOContext(default_memory_pool(), std::move(stop_token)) {} + + explicit IOContext(MemoryPool* pool, StopToken stop_token = StopToken::Unstoppable()); + + explicit IOContext(MemoryPool* pool, ::arrow::internal::Executor* executor, + StopToken stop_token = StopToken::Unstoppable(), + int64_t external_id = -1) + : pool_(pool), + executor_(executor), + external_id_(external_id), + stop_token_(std::move(stop_token)) {} + + explicit IOContext(::arrow::internal::Executor* executor, + StopToken stop_token = StopToken::Unstoppable(), + int64_t external_id = -1) + : pool_(default_memory_pool()), + executor_(executor), + external_id_(external_id), + stop_token_(std::move(stop_token)) {} + + MemoryPool* pool() const { return pool_; } + + ::arrow::internal::Executor* executor() const { return executor_; } + + // An application-specific ID, forwarded to executor task submissions + int64_t external_id() const { return external_id_; } + + StopToken stop_token() const { return stop_token_; } + + private: + MemoryPool* pool_; + ::arrow::internal::Executor* executor_; + int64_t external_id_; + StopToken stop_token_; +}; + +class ARROW_EXPORT FileInterface : public std::enable_shared_from_this { + public: + virtual ~FileInterface() = 0; + + /// \brief Close the stream cleanly + /// + /// For writable streams, this will attempt to flush any pending data + /// before releasing the underlying resource. + /// + /// After Close() is called, closed() returns true and the stream is not + /// available for further operations. + virtual Status Close() = 0; + + /// \brief Close the stream asynchronously + /// + /// By default, this will just submit the synchronous Close() to the + /// default I/O thread pool. Subclasses may implement this in a more + /// efficient manner. + virtual Future<> CloseAsync(); + + /// \brief Close the stream abruptly + /// + /// This method does not guarantee that any pending data is flushed. + /// It merely releases any underlying resource used by the stream for + /// its operation. + /// + /// After Abort() is called, closed() returns true and the stream is not + /// available for further operations. + virtual Status Abort(); + + /// \brief Return the position in this stream + virtual Result Tell() const = 0; + + /// \brief Return whether the stream is closed + virtual bool closed() const = 0; + + FileMode::type mode() const { return mode_; } + + protected: + FileInterface() : mode_(FileMode::READ) {} + FileMode::type mode_; + void set_mode(FileMode::type mode) { mode_ = mode; } + + private: + ARROW_DISALLOW_COPY_AND_ASSIGN(FileInterface); +}; + +class ARROW_EXPORT Seekable { + public: + virtual ~Seekable() = default; + virtual Status Seek(int64_t position) = 0; +}; + +class ARROW_EXPORT Writable { + public: + virtual ~Writable() = default; + + /// \brief Write the given data to the stream + /// + /// This method always processes the bytes in full. Depending on the + /// semantics of the stream, the data may be written out immediately, + /// held in a buffer, or written asynchronously. In the case where + /// the stream buffers the data, it will be copied. To avoid potentially + /// large copies, use the Write variant that takes an owned Buffer. + virtual Status Write(const void* data, int64_t nbytes) = 0; + + /// \brief Write the given data to the stream + /// + /// Since the Buffer owns its memory, this method can avoid a copy if + /// buffering is required. See Write(const void*, int64_t) for details. + virtual Status Write(const std::shared_ptr& data); + + /// \brief Flush buffered bytes, if any + virtual Status Flush(); + + Status Write(std::string_view data); +}; + +class ARROW_EXPORT Readable { + public: + virtual ~Readable() = default; + + /// \brief Read data from current file position. + /// + /// Read at most `nbytes` from the current file position into `out`. + /// The number of bytes read is returned. + virtual Result Read(int64_t nbytes, void* out) = 0; + + /// \brief Read data from current file position. + /// + /// Read at most `nbytes` from the current file position. Less bytes may + /// be read if EOF is reached. This method updates the current file position. + /// + /// In some cases (e.g. a memory-mapped file), this method may avoid a + /// memory copy. + virtual Result> Read(int64_t nbytes) = 0; + + /// EXPERIMENTAL: The IOContext associated with this file. + /// + /// By default, this is the same as default_io_context(), but it may be + /// overridden by subclasses. + virtual const IOContext& io_context() const; +}; + +class ARROW_EXPORT OutputStream : virtual public FileInterface, public Writable { + protected: + OutputStream() = default; +}; + +class ARROW_EXPORT InputStream : virtual public FileInterface, virtual public Readable { + public: + /// \brief Advance or skip stream indicated number of bytes + /// \param[in] nbytes the number to move forward + /// \return Status + Status Advance(int64_t nbytes); + + /// \brief Return zero-copy string_view to upcoming bytes. + /// + /// Do not modify the stream position. The view becomes invalid after + /// any operation on the stream. May trigger buffering if the requested + /// size is larger than the number of buffered bytes. + /// + /// May return NotImplemented on streams that don't support it. + /// + /// \param[in] nbytes the maximum number of bytes to see + virtual Result Peek(int64_t nbytes); + + /// \brief Return true if InputStream is capable of zero copy Buffer reads + /// + /// Zero copy reads imply the use of Buffer-returning Read() overloads. + virtual bool supports_zero_copy() const; + + /// \brief Read and return stream metadata + /// + /// If the stream implementation doesn't support metadata, empty metadata + /// is returned. Note that it is allowed to return a null pointer rather + /// than an allocated empty metadata. + virtual Result> ReadMetadata(); + + /// \brief Read stream metadata asynchronously + virtual Future> ReadMetadataAsync( + const IOContext& io_context); + Future> ReadMetadataAsync(); + + protected: + InputStream() = default; +}; + +class ARROW_EXPORT RandomAccessFile : public InputStream, public Seekable { + public: + /// Necessary because we hold a std::unique_ptr + ~RandomAccessFile() override; + + /// \brief Create an isolated InputStream that reads a segment of a + /// RandomAccessFile. Multiple such stream can be created and used + /// independently without interference + /// \param[in] file a file instance + /// \param[in] file_offset the starting position in the file + /// \param[in] nbytes the extent of bytes to read. The file should have + /// sufficient bytes available + static Result> GetStream( + std::shared_ptr file, int64_t file_offset, int64_t nbytes); + + /// \brief Return the total file size in bytes. + /// + /// This method does not read or move the current file position, so is safe + /// to call concurrently with e.g. ReadAt(). + virtual Result GetSize() = 0; + + /// \brief Read data from given file position. + /// + /// At most `nbytes` bytes are read. The number of bytes read is returned + /// (it can be less than `nbytes` if EOF is reached). + /// + /// This method can be safely called from multiple threads concurrently. + /// It is unspecified whether this method updates the file position or not. + /// + /// The default RandomAccessFile-provided implementation uses Seek() and Read(), + /// but subclasses may override it with a more efficient implementation + /// that doesn't depend on implicit file positioning. + /// + /// \param[in] position Where to read bytes from + /// \param[in] nbytes The number of bytes to read + /// \param[out] out The buffer to read bytes into + /// \return The number of bytes read, or an error + virtual Result ReadAt(int64_t position, int64_t nbytes, void* out); + + /// \brief Read data from given file position. + /// + /// At most `nbytes` bytes are read, but it can be less if EOF is reached. + /// + /// \param[in] position Where to read bytes from + /// \param[in] nbytes The number of bytes to read + /// \return A buffer containing the bytes read, or an error + virtual Result> ReadAt(int64_t position, int64_t nbytes); + + /// EXPERIMENTAL: Read data asynchronously. + virtual Future> ReadAsync(const IOContext&, int64_t position, + int64_t nbytes); + + /// EXPERIMENTAL: Read data asynchronously, using the file's IOContext. + Future> ReadAsync(int64_t position, int64_t nbytes); + + /// EXPERIMENTAL: Explicit multi-read. + /// \brief Request multiple reads at once + /// + /// The underlying filesystem may optimize these reads by coalescing small reads into + /// large reads or by breaking up large reads into multiple parallel smaller reads. The + /// reads should be issued in parallel if it makes sense for the filesystem. + /// + /// One future will be returned for each input read range. Multiple returned futures + /// may correspond to a single read. Or, a single returned future may be a combined + /// result of several individual reads. + /// + /// \param[in] ranges The ranges to read + /// \return A future that will complete with the data from the requested range is + /// available + virtual std::vector>> ReadManyAsync( + const IOContext&, const std::vector& ranges); + + /// EXPERIMENTAL: Explicit multi-read, using the file's IOContext. + std::vector>> ReadManyAsync( + const std::vector& ranges); + + /// EXPERIMENTAL: Inform that the given ranges may be read soon. + /// + /// Some implementations might arrange to prefetch some of the data. + /// However, no guarantee is made and the default implementation does nothing. + /// For robust prefetching, use ReadAt() or ReadAsync(). + virtual Status WillNeed(const std::vector& ranges); + + protected: + RandomAccessFile(); + + private: + struct ARROW_NO_EXPORT Impl; + std::unique_ptr interface_impl_; +}; + +class ARROW_EXPORT WritableFile : public OutputStream, public Seekable { + public: + virtual Status WriteAt(int64_t position, const void* data, int64_t nbytes) = 0; + + protected: + WritableFile() = default; +}; + +class ARROW_EXPORT ReadWriteFileInterface : public RandomAccessFile, public WritableFile { + protected: + ReadWriteFileInterface() { RandomAccessFile::set_mode(FileMode::READWRITE); } +}; + +/// \brief Return an iterator on an input stream +/// +/// The iterator yields a fixed-size block on each Next() call, except the +/// last block in the stream which may be smaller. +/// Once the end of stream is reached, Next() returns nullptr +/// (unlike InputStream::Read() which returns an empty buffer). +ARROW_EXPORT +Result>> MakeInputStreamIterator( + std::shared_ptr stream, int64_t block_size); + +} // namespace io +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/memory.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/memory.h new file mode 100644 index 0000000000000000000000000000000000000000..ab97c9fc1513a7758c5aa8576763c557ac69fa80 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/memory.h @@ -0,0 +1,197 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Public API for different memory sharing / IO mechanisms + +#pragma once + +#include +#include +#include +#include + +#include "arrow/io/concurrency.h" +#include "arrow/io/interfaces.h" +#include "arrow/type_fwd.h" +#include "arrow/util/visibility.h" + +namespace arrow { + +class Status; + +namespace io { + +/// \brief An output stream that writes to a resizable buffer +class ARROW_EXPORT BufferOutputStream : public OutputStream { + public: + explicit BufferOutputStream(const std::shared_ptr& buffer); + + /// \brief Create in-memory output stream with indicated capacity using a + /// memory pool + /// \param[in] initial_capacity the initial allocated internal capacity of + /// the OutputStream + /// \param[in,out] pool a MemoryPool to use for allocations + /// \return the created stream + static Result> Create( + int64_t initial_capacity = 4096, MemoryPool* pool = default_memory_pool()); + + ~BufferOutputStream() override; + + // Implement the OutputStream interface + + /// Close the stream, preserving the buffer (retrieve it with Finish()). + Status Close() override; + bool closed() const override; + Result Tell() const override; + Status Write(const void* data, int64_t nbytes) override; + + /// \cond FALSE + using OutputStream::Write; + /// \endcond + + /// Close the stream and return the buffer + Result> Finish(); + + /// \brief Initialize state of OutputStream with newly allocated memory and + /// set position to 0 + /// \param[in] initial_capacity the starting allocated capacity + /// \param[in,out] pool the memory pool to use for allocations + /// \return Status + Status Reset(int64_t initial_capacity = 1024, MemoryPool* pool = default_memory_pool()); + + int64_t capacity() const { return capacity_; } + + private: + BufferOutputStream(); + + // Ensures there is sufficient space available to write nbytes + Status Reserve(int64_t nbytes); + + std::shared_ptr buffer_; + bool is_open_; + int64_t capacity_; + int64_t position_; + uint8_t* mutable_data_; +}; + +/// \brief A helper class to track the size of allocations +/// +/// Writes to this stream do not copy or retain any data, they just bump +/// a size counter that can be later used to know exactly which data size +/// needs to be allocated for actual writing. +class ARROW_EXPORT MockOutputStream : public OutputStream { + public: + MockOutputStream() : extent_bytes_written_(0), is_open_(true) {} + + // Implement the OutputStream interface + Status Close() override; + bool closed() const override; + Result Tell() const override; + Status Write(const void* data, int64_t nbytes) override; + /// \cond FALSE + using Writable::Write; + /// \endcond + + int64_t GetExtentBytesWritten() const { return extent_bytes_written_; } + + private: + int64_t extent_bytes_written_; + bool is_open_; +}; + +/// \brief An output stream that writes into a fixed-size mutable buffer +class ARROW_EXPORT FixedSizeBufferWriter : public WritableFile { + public: + /// Input buffer must be mutable, will abort if not + explicit FixedSizeBufferWriter(const std::shared_ptr& buffer); + ~FixedSizeBufferWriter() override; + + Status Close() override; + bool closed() const override; + Status Seek(int64_t position) override; + Result Tell() const override; + Status Write(const void* data, int64_t nbytes) override; + /// \cond FALSE + using Writable::Write; + /// \endcond + + Status WriteAt(int64_t position, const void* data, int64_t nbytes) override; + + void set_memcopy_threads(int num_threads); + void set_memcopy_blocksize(int64_t blocksize); + void set_memcopy_threshold(int64_t threshold); + + protected: + class FixedSizeBufferWriterImpl; + std::unique_ptr impl_; +}; + +/// \class BufferReader +/// \brief Random access zero-copy reads on an arrow::Buffer +class ARROW_EXPORT BufferReader + : public internal::RandomAccessFileConcurrencyWrapper { + public: + /// \brief Instantiate from std::shared_ptr. + /// + /// This is a zero-copy constructor. + explicit BufferReader(std::shared_ptr buffer); + + /// \brief Instantiate from std::string. Owns data. + static std::unique_ptr FromString(std::string data); + + bool closed() const override; + + bool supports_zero_copy() const override; + + std::shared_ptr buffer() const { return buffer_; } + + // Synchronous ReadAsync override + Future> ReadAsync(const IOContext&, int64_t position, + int64_t nbytes) override; + Status WillNeed(const std::vector& ranges) override; + + protected: + friend RandomAccessFileConcurrencyWrapper; + + Status DoClose(); + + Result DoRead(int64_t nbytes, void* buffer); + Result> DoRead(int64_t nbytes); + Result DoReadAt(int64_t position, int64_t nbytes, void* out); + Result> DoReadAt(int64_t position, int64_t nbytes); + Result DoPeek(int64_t nbytes) override; + + Result DoTell() const; + Status DoSeek(int64_t position); + Result DoGetSize(); + + Status CheckClosed() const { + if (!is_open_) { + return Status::Invalid("Operation forbidden on closed BufferReader"); + } + return Status::OK(); + } + + std::shared_ptr buffer_; + const uint8_t* data_; + int64_t size_; + int64_t position_; + bool is_open_; +}; + +} // namespace io +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/mman.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/mman.h new file mode 100644 index 0000000000000000000000000000000000000000..42fb240ac9620cf84b854993edd396673ac3f5ef --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/mman.h @@ -0,0 +1,169 @@ +// Copyright https://code.google.com/p/mman-win32/ +// +// Licensed under the MIT License; +// You may obtain a copy of the License at +// +// https://opensource.org/licenses/MIT + +#pragma once + +#include "arrow/util/windows_compatibility.h" + +#include +#include +#include + +#include + +#define PROT_NONE 0 +#define PROT_READ 1 +#define PROT_WRITE 2 +#define PROT_EXEC 4 + +#define MAP_FILE 0 +#define MAP_SHARED 1 +#define MAP_PRIVATE 2 +#define MAP_TYPE 0xf +#define MAP_FIXED 0x10 +#define MAP_ANONYMOUS 0x20 +#define MAP_ANON MAP_ANONYMOUS + +#define MAP_FAILED ((void*)-1) + +/* Flags for msync. */ +#define MS_ASYNC 1 +#define MS_SYNC 2 +#define MS_INVALIDATE 4 + +#ifndef FILE_MAP_EXECUTE +# define FILE_MAP_EXECUTE 0x0020 +#endif + +static inline int __map_mman_error(const DWORD err, const int deferr) { + if (err == 0) return 0; + // TODO: implement + return err; +} + +static inline DWORD __map_mmap_prot_page(const int prot) { + DWORD protect = 0; + + if (prot == PROT_NONE) return protect; + + if ((prot & PROT_EXEC) != 0) { + protect = ((prot & PROT_WRITE) != 0) ? PAGE_EXECUTE_READWRITE : PAGE_EXECUTE_READ; + } else { + protect = ((prot & PROT_WRITE) != 0) ? PAGE_READWRITE : PAGE_READONLY; + } + + return protect; +} + +static inline DWORD __map_mmap_prot_file(const int prot) { + DWORD desiredAccess = 0; + + if (prot == PROT_NONE) return desiredAccess; + + if ((prot & PROT_READ) != 0) desiredAccess |= FILE_MAP_READ; + if ((prot & PROT_WRITE) != 0) desiredAccess |= FILE_MAP_WRITE; + if ((prot & PROT_EXEC) != 0) desiredAccess |= FILE_MAP_EXECUTE; + + return desiredAccess; +} + +static inline void* mmap(void* addr, size_t len, int prot, int flags, int fildes, + off_t off) { + HANDLE fm, h; + + void* map = MAP_FAILED; + const uint64_t off64 = static_cast(off); + const uint64_t maxSize = off64 + len; + + const DWORD dwFileOffsetLow = static_cast(off64 & 0xFFFFFFFFUL); + const DWORD dwFileOffsetHigh = static_cast((off64 >> 32) & 0xFFFFFFFFUL); + const DWORD dwMaxSizeLow = static_cast(maxSize & 0xFFFFFFFFUL); + const DWORD dwMaxSizeHigh = static_cast((maxSize >> 32) & 0xFFFFFFFFUL); + + const DWORD protect = __map_mmap_prot_page(prot); + const DWORD desiredAccess = __map_mmap_prot_file(prot); + + errno = 0; + + if (len == 0 + /* Unsupported flag combinations */ + || (flags & MAP_FIXED) != 0 + /* Unsupported protection combinations */ + || prot == PROT_EXEC) { + errno = EINVAL; + return MAP_FAILED; + } + + h = ((flags & MAP_ANONYMOUS) == 0) ? (HANDLE)_get_osfhandle(fildes) + : INVALID_HANDLE_VALUE; + + if ((flags & MAP_ANONYMOUS) == 0 && h == INVALID_HANDLE_VALUE) { + errno = EBADF; + return MAP_FAILED; + } + + fm = CreateFileMapping(h, NULL, protect, dwMaxSizeHigh, dwMaxSizeLow, NULL); + + if (fm == NULL) { + errno = __map_mman_error(GetLastError(), EPERM); + return MAP_FAILED; + } + + map = MapViewOfFile(fm, desiredAccess, dwFileOffsetHigh, dwFileOffsetLow, len); + + CloseHandle(fm); + + if (map == NULL) { + errno = __map_mman_error(GetLastError(), EPERM); + return MAP_FAILED; + } + + return map; +} + +static inline int munmap(void* addr, size_t len) { + if (UnmapViewOfFile(addr)) return 0; + + errno = __map_mman_error(GetLastError(), EPERM); + + return -1; +} + +static inline int mprotect(void* addr, size_t len, int prot) { + DWORD newProtect = __map_mmap_prot_page(prot); + DWORD oldProtect = 0; + + if (VirtualProtect(addr, len, newProtect, &oldProtect)) return 0; + + errno = __map_mman_error(GetLastError(), EPERM); + + return -1; +} + +static inline int msync(void* addr, size_t len, int flags) { + if (FlushViewOfFile(addr, len)) return 0; + + errno = __map_mman_error(GetLastError(), EPERM); + + return -1; +} + +static inline int mlock(const void* addr, size_t len) { + if (VirtualLock((LPVOID)addr, len)) return 0; + + errno = __map_mman_error(GetLastError(), EPERM); + + return -1; +} + +static inline int munlock(const void* addr, size_t len) { + if (VirtualUnlock((LPVOID)addr, len)) return 0; + + errno = __map_mman_error(GetLastError(), EPERM); + + return -1; +} diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/slow.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/slow.h new file mode 100644 index 0000000000000000000000000000000000000000..f87f9acbcd21c5ed8cac4f614738e9607347aacf --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/slow.h @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Slow stream implementations, mainly for testing and benchmarking + +#pragma once + +#include +#include +#include + +#include "arrow/io/interfaces.h" +#include "arrow/util/visibility.h" + +namespace arrow { + +class Buffer; +class Status; + +namespace io { + +class ARROW_EXPORT LatencyGenerator { + public: + virtual ~LatencyGenerator(); + + void Sleep(); + + virtual double NextLatency() = 0; + + static std::shared_ptr Make(double average_latency); + static std::shared_ptr Make(double average_latency, int32_t seed); +}; + +// XXX use ConcurrencyWrapper? It could increase chances of finding a race. + +template +class SlowInputStreamBase : public StreamType { + public: + SlowInputStreamBase(std::shared_ptr stream, + std::shared_ptr latencies) + : stream_(std::move(stream)), latencies_(std::move(latencies)) {} + + SlowInputStreamBase(std::shared_ptr stream, double average_latency) + : stream_(std::move(stream)), latencies_(LatencyGenerator::Make(average_latency)) {} + + SlowInputStreamBase(std::shared_ptr stream, double average_latency, + int32_t seed) + : stream_(std::move(stream)), + latencies_(LatencyGenerator::Make(average_latency, seed)) {} + + protected: + std::shared_ptr stream_; + std::shared_ptr latencies_; +}; + +/// \brief An InputStream wrapper that makes reads slower. +/// +/// Read() calls are made slower by an average latency (in seconds). +/// Actual latencies form a normal distribution closely centered +/// on the average latency. +/// Other calls are forwarded directly. +class ARROW_EXPORT SlowInputStream : public SlowInputStreamBase { + public: + ~SlowInputStream() override; + + using SlowInputStreamBase::SlowInputStreamBase; + + Status Close() override; + Status Abort() override; + bool closed() const override; + + Result Read(int64_t nbytes, void* out) override; + Result> Read(int64_t nbytes) override; + Result Peek(int64_t nbytes) override; + + Result Tell() const override; +}; + +/// \brief A RandomAccessFile wrapper that makes reads slower. +/// +/// Similar to SlowInputStream, but allows random access and seeking. +class ARROW_EXPORT SlowRandomAccessFile : public SlowInputStreamBase { + public: + ~SlowRandomAccessFile() override; + + using SlowInputStreamBase::SlowInputStreamBase; + + Status Close() override; + Status Abort() override; + bool closed() const override; + + Result Read(int64_t nbytes, void* out) override; + Result> Read(int64_t nbytes) override; + Result ReadAt(int64_t position, int64_t nbytes, void* out) override; + Result> ReadAt(int64_t position, int64_t nbytes) override; + Result Peek(int64_t nbytes) override; + + Result GetSize() override; + Status Seek(int64_t position) override; + Result Tell() const override; +}; + +} // namespace io +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/stdio.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/stdio.h new file mode 100644 index 0000000000000000000000000000000000000000..ca5469a0b153f7aabd4f7eda86aa87680425233e --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/stdio.h @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "arrow/io/interfaces.h" +#include "arrow/util/visibility.h" + +namespace arrow { +namespace io { + +// Output stream that just writes to stdout. +class ARROW_EXPORT StdoutStream : public OutputStream { + public: + StdoutStream(); + ~StdoutStream() override {} + + Status Close() override; + bool closed() const override; + + Result Tell() const override; + + Status Write(const void* data, int64_t nbytes) override; + + private: + int64_t pos_; +}; + +// Output stream that just writes to stderr. +class ARROW_EXPORT StderrStream : public OutputStream { + public: + StderrStream(); + ~StderrStream() override {} + + Status Close() override; + bool closed() const override; + + Result Tell() const override; + + Status Write(const void* data, int64_t nbytes) override; + + private: + int64_t pos_; +}; + +// Input stream that just reads from stdin. +class ARROW_EXPORT StdinStream : public InputStream { + public: + StdinStream(); + ~StdinStream() override {} + + Status Close() override; + bool closed() const override; + + Result Tell() const override; + + Result Read(int64_t nbytes, void* out) override; + + Result> Read(int64_t nbytes) override; + + private: + int64_t pos_; +}; + +} // namespace io +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/test_common.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/test_common.h new file mode 100644 index 0000000000000000000000000000000000000000..a496566eb28ce570cb1470f6008b3350f0aab70a --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/test_common.h @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "arrow/io/interfaces.h" +#include "arrow/testing/visibility.h" +#include "arrow/type_fwd.h" + +namespace arrow { +namespace io { + +class MemoryMappedFile; + +ARROW_TESTING_EXPORT +void AssertFileContents(const std::string& path, const std::string& contents); + +ARROW_TESTING_EXPORT bool FileExists(const std::string& path); + +ARROW_TESTING_EXPORT Status PurgeLocalFileFromOsCache(const std::string& path); + +ARROW_TESTING_EXPORT +Status ZeroMemoryMap(MemoryMappedFile* file); + +class ARROW_TESTING_EXPORT MemoryMapFixture { + public: + void TearDown(); + + void CreateFile(const std::string& path, int64_t size); + + Result> InitMemoryMap(int64_t size, + const std::string& path); + + void AppendFile(const std::string& path); + + private: + std::vector tmp_files_; +}; + +class ARROW_TESTING_EXPORT TrackedRandomAccessFile : public io::RandomAccessFile { + public: + virtual int64_t num_reads() const = 0; + virtual int64_t bytes_read() const = 0; + virtual const std::vector& get_read_ranges() const = 0; + virtual void ResetStats() = 0; + + static std::unique_ptr Make(io::RandomAccessFile* target); +}; + +} // namespace io +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/transform.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/transform.h new file mode 100644 index 0000000000000000000000000000000000000000..90f9efa8a55747b8435e1ece65d1879656455fed --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/transform.h @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Transform stream implementations + +#pragma once + +#include +#include +#include +#include + +#include "arrow/io/interfaces.h" +#include "arrow/util/visibility.h" + +namespace arrow { +namespace io { + +class ARROW_EXPORT TransformInputStream : public InputStream { + public: + using TransformFunc = + std::function>(const std::shared_ptr&)>; + + TransformInputStream(std::shared_ptr wrapped, TransformFunc transform); + ~TransformInputStream() override; + + Status Close() override; + Status Abort() override; + bool closed() const override; + + Result Read(int64_t nbytes, void* out) override; + Result> Read(int64_t nbytes) override; + + Result> ReadMetadata() override; + Future> ReadMetadataAsync( + const IOContext& io_context) override; + + Result Tell() const override; + + protected: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace io +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/type_fwd.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/type_fwd.h new file mode 100644 index 0000000000000000000000000000000000000000..89554a256e075069e64601fec569cf75ac2c1f84 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/io/type_fwd.h @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "arrow/type_fwd.h" +#include "arrow/util/visibility.h" + +namespace arrow { +namespace io { + +struct FileMode { + enum type { READ, WRITE, READWRITE }; +}; + +struct IOContext; +struct CacheOptions; + +/// EXPERIMENTAL: convenience global singleton for default IOContext settings +ARROW_EXPORT +const IOContext& default_io_context(); + +/// \brief Get the capacity of the global I/O thread pool +/// +/// Return the number of worker threads in the thread pool to which +/// Arrow dispatches various I/O-bound tasks. This is an ideal number, +/// not necessarily the exact number of threads at a given point in time. +/// +/// You can change this number using SetIOThreadPoolCapacity(). +ARROW_EXPORT int GetIOThreadPoolCapacity(); + +/// \brief Set the capacity of the global I/O thread pool +/// +/// Set the number of worker threads in the thread pool to which +/// Arrow dispatches various I/O-bound tasks. +/// +/// The current number is returned by GetIOThreadPoolCapacity(). +ARROW_EXPORT Status SetIOThreadPoolCapacity(int threads); + +class FileInterface; +class Seekable; +class Writable; +class Readable; +class OutputStream; +class FileOutputStream; +class InputStream; +class ReadableFile; +class RandomAccessFile; +class MemoryMappedFile; +class WritableFile; +class ReadWriteFileInterface; + +class LatencyGenerator; + +class BufferOutputStream; +class BufferReader; +class CompressedInputStream; +class CompressedOutputStream; +class BufferedInputStream; +class BufferedOutputStream; + +} // namespace io +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/api.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/api.h new file mode 100644 index 0000000000000000000000000000000000000000..a9b1e78846b6a655cf4ff651f833cd3d7d18d3e2 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/api.h @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "arrow/ipc/dictionary.h" +#include "arrow/ipc/feather.h" +#include "arrow/ipc/message.h" +#include "arrow/ipc/reader.h" +#include "arrow/ipc/writer.h" diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/dictionary.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/dictionary.h new file mode 100644 index 0000000000000000000000000000000000000000..7f1629c427fd17e981a8a5b17a0893226c167b21 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/dictionary.h @@ -0,0 +1,177 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Tools for dictionaries in IPC context + +#pragma once + +#include +#include +#include +#include + +#include "arrow/result.h" +#include "arrow/status.h" +#include "arrow/type_fwd.h" +#include "arrow/util/macros.h" +#include "arrow/util/visibility.h" + +namespace arrow { +namespace ipc { + +namespace internal { + +class FieldPosition { + public: + FieldPosition() : parent_(NULLPTR), index_(-1), depth_(0) {} + + FieldPosition child(int index) const { return {this, index}; } + + std::vector path() const { + std::vector path(depth_); + const FieldPosition* cur = this; + for (int i = depth_ - 1; i >= 0; --i) { + path[i] = cur->index_; + cur = cur->parent_; + } + return path; + } + + protected: + FieldPosition(const FieldPosition* parent, int index) + : parent_(parent), index_(index), depth_(parent->depth_ + 1) {} + + const FieldPosition* parent_; + int index_; + int depth_; +}; + +} // namespace internal + +/// \brief Map fields in a schema to dictionary ids +/// +/// The mapping is structural, i.e. the field path (as a vector of indices) +/// is associated to the dictionary id. A dictionary id may be associated +/// to multiple fields. +class ARROW_EXPORT DictionaryFieldMapper { + public: + DictionaryFieldMapper(); + explicit DictionaryFieldMapper(const Schema& schema); + ~DictionaryFieldMapper(); + + Status AddSchemaFields(const Schema& schema); + Status AddField(int64_t id, std::vector field_path); + + Result GetFieldId(std::vector field_path) const; + + int num_fields() const; + + /// \brief Returns number of unique dictionaries, taking into + /// account that different fields can share the same dictionary. + int num_dicts() const; + + private: + struct Impl; + std::unique_ptr impl_; +}; + +using DictionaryVector = std::vector>>; + +/// \brief Memoization data structure for reading dictionaries from IPC streams +/// +/// This structure tracks the following associations: +/// - field position (structural) -> dictionary id +/// - dictionary id -> value type +/// - dictionary id -> dictionary (value) data +/// +/// Together, they allow resolving dictionary data when reading an IPC stream, +/// using metadata recorded in the schema message and data recorded in the +/// dictionary batch messages (see ResolveDictionaries). +/// +/// This structure isn't useful for writing an IPC stream, where only +/// DictionaryFieldMapper is necessary. +class ARROW_EXPORT DictionaryMemo { + public: + DictionaryMemo(); + ~DictionaryMemo(); + + DictionaryFieldMapper& fields(); + const DictionaryFieldMapper& fields() const; + + /// \brief Return current dictionary corresponding to a particular + /// id. Returns KeyError if id not found + Result> GetDictionary(int64_t id, MemoryPool* pool) const; + + /// \brief Return dictionary value type corresponding to a + /// particular dictionary id. + Result> GetDictionaryType(int64_t id) const; + + /// \brief Return true if we have a dictionary for the input id + bool HasDictionary(int64_t id) const; + + /// \brief Add a dictionary value type to the memo with a particular id. + /// Returns KeyError if a different type is already registered with the same id. + Status AddDictionaryType(int64_t id, const std::shared_ptr& type); + + /// \brief Add a dictionary to the memo with a particular id. Returns + /// KeyError if that dictionary already exists + Status AddDictionary(int64_t id, const std::shared_ptr& dictionary); + + /// \brief Append a dictionary delta to the memo with a particular id. Returns + /// KeyError if that dictionary does not exists + Status AddDictionaryDelta(int64_t id, const std::shared_ptr& dictionary); + + /// \brief Add a dictionary to the memo if it does not have one with the id, + /// otherwise, replace the dictionary with the new one. + /// + /// Return true if the dictionary was added, false if replaced. + Result AddOrReplaceDictionary(int64_t id, + const std::shared_ptr& dictionary); + + private: + struct Impl; + std::unique_ptr impl_; +}; + +// For writing: collect dictionary entries to write to the IPC stream, in order +// (i.e. inner dictionaries before dependent outer dictionaries). +ARROW_EXPORT +Result CollectDictionaries(const RecordBatch& batch, + const DictionaryFieldMapper& mapper); + +// For reading: resolve all dictionaries in columns, according to the field +// mapping and dictionary arrays stored in memo. +// Columns may be sparse, i.e. some entries may be left null +// (e.g. if an inclusion mask was used). +ARROW_EXPORT +Status ResolveDictionaries(const ArrayDataVector& columns, const DictionaryMemo& memo, + MemoryPool* pool); + +namespace internal { + +// Like CollectDictionaries above, but uses the memo's DictionaryFieldMapper +// and all collected dictionaries are added to the memo using AddDictionary. +// +// This is used as a shortcut in some roundtripping tests (to avoid emitting +// any actual dictionary batches). +ARROW_EXPORT +Status CollectDictionaries(const RecordBatch& batch, DictionaryMemo* memo); + +} // namespace internal + +} // namespace ipc +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/feather.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/feather.h new file mode 100644 index 0000000000000000000000000000000000000000..538cbb705118b76bcd122a5f364143c633adf451 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/feather.h @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Public API for the "Feather" file format, originally created at +// http://github.com/wesm/feather + +#pragma once + +#include +#include +#include +#include + +#include "arrow/ipc/options.h" +#include "arrow/type_fwd.h" +#include "arrow/util/compression.h" +#include "arrow/util/visibility.h" + +namespace arrow { + +class Schema; +class Status; +class Table; + +namespace io { + +class OutputStream; +class RandomAccessFile; + +} // namespace io + +namespace ipc { +namespace feather { + +static constexpr const int kFeatherV1Version = 2; +static constexpr const int kFeatherV2Version = 3; + +// ---------------------------------------------------------------------- +// Metadata accessor classes + +/// \class Reader +/// \brief An interface for reading columns from Feather files +class ARROW_EXPORT Reader { + public: + virtual ~Reader() = default; + + /// \brief Open a Feather file from a RandomAccessFile interface + /// + /// \param[in] source a RandomAccessFile instance + /// \return the table reader + static Result> Open( + const std::shared_ptr& source); + + /// \brief Open a Feather file from a RandomAccessFile interface + /// with IPC Read options + /// + /// \param[in] source a RandomAccessFile instance + /// \param[in] options IPC Read options + /// \return the table reader + static Result> Open( + const std::shared_ptr& source, const IpcReadOptions& options); + + /// \brief Return the version number of the Feather file + virtual int version() const = 0; + + virtual std::shared_ptr schema() const = 0; + + /// \brief Read all columns from the file as an arrow::Table. + /// + /// \param[out] out the returned table + /// \return Status + /// + /// This function is zero-copy if the file source supports zero-copy reads + virtual Status Read(std::shared_ptr* out) = 0; + + /// \brief Read only the specified columns from the file as an arrow::Table. + /// + /// \param[in] indices the column indices to read + /// \param[out] out the returned table + /// \return Status + /// + /// This function is zero-copy if the file source supports zero-copy reads + virtual Status Read(const std::vector& indices, std::shared_ptr
* out) = 0; + + /// \brief Read only the specified columns from the file as an arrow::Table. + /// + /// \param[in] names the column names to read + /// \param[out] out the returned table + /// \return Status + /// + /// This function is zero-copy if the file source supports zero-copy reads + virtual Status Read(const std::vector& names, + std::shared_ptr
* out) = 0; +}; + +struct ARROW_EXPORT WriteProperties { + static WriteProperties Defaults(); + + static WriteProperties DefaultsV1() { + WriteProperties props = Defaults(); + props.version = kFeatherV1Version; + return props; + } + + /// Feather file version number + /// + /// version 2: "Feather V1" Apache Arrow <= 0.16.0 + /// version 3: "Feather V2" Apache Arrow > 0.16.0 + int version = kFeatherV2Version; + + // Parameters for Feather V2 only + + /// Number of rows per intra-file chunk. Use smaller chunksize when you need + /// faster random row access + int64_t chunksize = 1LL << 16; + + /// Compression type to use. Only UNCOMPRESSED, LZ4_FRAME, and ZSTD are + /// supported. The default compression returned by Defaults() is LZ4 if the + /// project is built with support for it, otherwise + /// UNCOMPRESSED. UNCOMPRESSED is set as the object default here so that if + /// WriteProperties::Defaults() is not used, the default constructor for + /// WriteProperties will work regardless of the options used to build the C++ + /// project. + Compression::type compression = Compression::UNCOMPRESSED; + + /// Compressor-specific compression level + int compression_level = ::arrow::util::kUseDefaultCompressionLevel; +}; + +ARROW_EXPORT +Status WriteTable(const Table& table, io::OutputStream* dst, + const WriteProperties& properties = WriteProperties::Defaults()); + +} // namespace feather +} // namespace ipc +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/message.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/message.h new file mode 100644 index 0000000000000000000000000000000000000000..292b89d505b8feffb53f9b7238fca869ee943ec3 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/message.h @@ -0,0 +1,585 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// C++ object model and user API for interprocess schema messaging + +#pragma once + +#include +#include +#include +#include +#include + +#include "arrow/io/type_fwd.h" +#include "arrow/ipc/type_fwd.h" +#include "arrow/result.h" +#include "arrow/status.h" +#include "arrow/type_fwd.h" +#include "arrow/util/macros.h" +#include "arrow/util/visibility.h" + +namespace arrow { +namespace ipc { + +struct IpcWriteOptions; + +// Read interface classes. We do not fully deserialize the flatbuffers so that +// individual fields metadata can be retrieved from very large schema without +// + +/// \class Message +/// \brief An IPC message including metadata and body +class ARROW_EXPORT Message { + public: + /// \brief Construct message, but do not validate + /// + /// Use at your own risk; Message::Open has more metadata validation + Message(std::shared_ptr metadata, std::shared_ptr body); + + ~Message(); + + /// \brief Create and validate a Message instance from two buffers + /// + /// \param[in] metadata a buffer containing the Flatbuffer metadata + /// \param[in] body a buffer containing the message body, which may be null + /// \return the created message + static Result> Open(std::shared_ptr metadata, + std::shared_ptr body); + + /// \brief Read message body and create Message given Flatbuffer metadata + /// \param[in] metadata containing a serialized Message flatbuffer + /// \param[in] stream an InputStream + /// \return the created Message + /// + /// \note If stream supports zero-copy, this is zero-copy + static Result> ReadFrom(std::shared_ptr metadata, + io::InputStream* stream); + + /// \brief Read message body from position in file, and create Message given + /// the Flatbuffer metadata + /// \param[in] offset the position in the file where the message body starts. + /// \param[in] metadata containing a serialized Message flatbuffer + /// \param[in] file the seekable file interface to read from + /// \return the created Message + /// + /// \note If file supports zero-copy, this is zero-copy + static Result> ReadFrom(const int64_t offset, + std::shared_ptr metadata, + io::RandomAccessFile* file); + + /// \brief Return true if message type and contents are equal + /// + /// \param other another message + /// \return true if contents equal + bool Equals(const Message& other) const; + + /// \brief the Message metadata + /// + /// \return buffer + std::shared_ptr metadata() const; + + /// \brief Custom metadata serialized in metadata Flatbuffer. Returns nullptr + /// when none set + const std::shared_ptr& custom_metadata() const; + + /// \brief the Message body, if any + /// + /// \return buffer is null if no body + std::shared_ptr body() const; + + /// \brief The expected body length according to the metadata, for + /// verification purposes + int64_t body_length() const; + + /// \brief The Message type + MessageType type() const; + + /// \brief The Message metadata version + MetadataVersion metadata_version() const; + + const void* header() const; + + /// \brief Write length-prefixed metadata and body to output stream + /// + /// \param[in] file output stream to write to + /// \param[in] options IPC writing options including alignment + /// \param[out] output_length the number of bytes written + /// \return Status + Status SerializeTo(io::OutputStream* file, const IpcWriteOptions& options, + int64_t* output_length) const; + + /// \brief Return true if the Message metadata passes Flatbuffer validation + bool Verify() const; + + /// \brief Whether a given message type needs a body. + static bool HasBody(MessageType type) { + return type != MessageType::NONE && type != MessageType::SCHEMA; + } + + private: + // Hide serialization details from user API + class MessageImpl; + std::unique_ptr impl_; + + ARROW_DISALLOW_COPY_AND_ASSIGN(Message); +}; + +ARROW_EXPORT std::string FormatMessageType(MessageType type); + +/// \class MessageDecoderListener +/// \brief An abstract class to listen events from MessageDecoder. +/// +/// This API is EXPERIMENTAL. +/// +/// \since 0.17.0 +class ARROW_EXPORT MessageDecoderListener { + public: + virtual ~MessageDecoderListener() = default; + + /// \brief Called when a message is decoded. + /// + /// MessageDecoder calls this method when it decodes a message. This + /// method is called multiple times when the target stream has + /// multiple messages. + /// + /// \param[in] message a decoded message + /// \return Status + virtual Status OnMessageDecoded(std::unique_ptr message) = 0; + + /// \brief Called when the decoder state is changed to + /// MessageDecoder::State::INITIAL. + /// + /// The default implementation just returns arrow::Status::OK(). + /// + /// \return Status + virtual Status OnInitial(); + + /// \brief Called when the decoder state is changed to + /// MessageDecoder::State::METADATA_LENGTH. + /// + /// The default implementation just returns arrow::Status::OK(). + /// + /// \return Status + virtual Status OnMetadataLength(); + + /// \brief Called when the decoder state is changed to + /// MessageDecoder::State::METADATA. + /// + /// The default implementation just returns arrow::Status::OK(). + /// + /// \return Status + virtual Status OnMetadata(); + + /// \brief Called when the decoder state is changed to + /// MessageDecoder::State::BODY. + /// + /// The default implementation just returns arrow::Status::OK(). + /// + /// \return Status + virtual Status OnBody(); + + /// \brief Called when the decoder state is changed to + /// MessageDecoder::State::EOS. + /// + /// The default implementation just returns arrow::Status::OK(). + /// + /// \return Status + virtual Status OnEOS(); +}; + +/// \class AssignMessageDecoderListener +/// \brief Assign a message decoded by MessageDecoder. +/// +/// This API is EXPERIMENTAL. +/// +/// \since 0.17.0 +class ARROW_EXPORT AssignMessageDecoderListener : public MessageDecoderListener { + public: + /// \brief Construct a listener that assigns a decoded message to the + /// specified location. + /// + /// \param[in] message a location to store the received message + explicit AssignMessageDecoderListener(std::unique_ptr* message) + : message_(message) {} + + virtual ~AssignMessageDecoderListener() = default; + + Status OnMessageDecoded(std::unique_ptr message) override { + *message_ = std::move(message); + return Status::OK(); + } + + private: + std::unique_ptr* message_; + + ARROW_DISALLOW_COPY_AND_ASSIGN(AssignMessageDecoderListener); +}; + +/// \class MessageDecoder +/// \brief Push style message decoder that receives data from user. +/// +/// This API is EXPERIMENTAL. +/// +/// \since 0.17.0 +class ARROW_EXPORT MessageDecoder { + public: + /// \brief State for reading a message + enum State { + /// The initial state. It requires one of the followings as the next data: + /// + /// * int32_t continuation token + /// * int32_t end-of-stream mark (== 0) + /// * int32_t metadata length (backward compatibility for + /// reading old IPC messages produced prior to version 0.15.0 + INITIAL, + + /// It requires int32_t metadata length. + METADATA_LENGTH, + + /// It requires metadata. + METADATA, + + /// It requires message body. + BODY, + + /// The end-of-stream state. No more data is processed. + EOS, + }; + + /// \brief Construct a message decoder. + /// + /// \param[in] listener a MessageDecoderListener that responds events from + /// the decoder + /// \param[in] pool an optional MemoryPool to copy metadata on the + /// \param[in] skip_body if true the body will be skipped even if the message has a body + /// CPU, if required + explicit MessageDecoder(std::shared_ptr listener, + MemoryPool* pool = default_memory_pool(), + bool skip_body = false); + + /// \brief Construct a message decoder with the specified state. + /// + /// This is a construct for advanced users that know how to decode + /// Message. + /// + /// \param[in] listener a MessageDecoderListener that responds events from + /// the decoder + /// \param[in] initial_state an initial state of the decode + /// \param[in] initial_next_required_size the number of bytes needed + /// to run the next action + /// \param[in] pool an optional MemoryPool to copy metadata on the + /// CPU, if required + /// \param[in] skip_body if true the body will be skipped even if the message has a body + MessageDecoder(std::shared_ptr listener, State initial_state, + int64_t initial_next_required_size, + MemoryPool* pool = default_memory_pool(), bool skip_body = false); + + virtual ~MessageDecoder(); + + /// \brief Feed data to the decoder as a raw data. + /// + /// If the decoder can decode one or more messages by the data, the + /// decoder calls listener->OnMessageDecoded() with a decoded + /// message multiple times. + /// + /// If the state of the decoder is changed, corresponding callbacks + /// on listener is called: + /// + /// * MessageDecoder::State::INITIAL: listener->OnInitial() + /// * MessageDecoder::State::METADATA_LENGTH: listener->OnMetadataLength() + /// * MessageDecoder::State::METADATA: listener->OnMetadata() + /// * MessageDecoder::State::BODY: listener->OnBody() + /// * MessageDecoder::State::EOS: listener->OnEOS() + /// + /// \param[in] data a raw data to be processed. This data isn't + /// copied. The passed memory must be kept alive through message + /// processing. + /// \param[in] size raw data size. + /// \return Status + Status Consume(const uint8_t* data, int64_t size); + + /// \brief Feed data to the decoder as a Buffer. + /// + /// If the decoder can decode one or more messages by the Buffer, + /// the decoder calls listener->OnMessageDecoded() with a decoded + /// message multiple times. + /// + /// \param[in] buffer a Buffer to be processed. + /// \return Status + Status Consume(std::shared_ptr buffer); + + /// \brief Return the number of bytes needed to advance the state of + /// the decoder. + /// + /// This method is provided for users who want to optimize performance. + /// Normal users don't need to use this method. + /// + /// Here is an example usage for normal users: + /// + /// ~~~{.cpp} + /// decoder.Consume(buffer1); + /// decoder.Consume(buffer2); + /// decoder.Consume(buffer3); + /// ~~~ + /// + /// Decoder has internal buffer. If consumed data isn't enough to + /// advance the state of the decoder, consumed data is buffered to + /// the internal buffer. It causes performance overhead. + /// + /// If you pass next_required_size() size data to each Consume() + /// call, the decoder doesn't use its internal buffer. It improves + /// performance. + /// + /// Here is an example usage to avoid using internal buffer: + /// + /// ~~~{.cpp} + /// buffer1 = get_data(decoder.next_required_size()); + /// decoder.Consume(buffer1); + /// buffer2 = get_data(decoder.next_required_size()); + /// decoder.Consume(buffer2); + /// ~~~ + /// + /// Users can use this method to avoid creating small + /// chunks. Message body must be contiguous data. If users pass + /// small chunks to the decoder, the decoder needs concatenate small + /// chunks internally. It causes performance overhead. + /// + /// Here is an example usage to reduce small chunks: + /// + /// ~~~{.cpp} + /// buffer = AllocateResizableBuffer(); + /// while ((small_chunk = get_data(&small_chunk_size))) { + /// auto current_buffer_size = buffer->size(); + /// buffer->Resize(current_buffer_size + small_chunk_size); + /// memcpy(buffer->mutable_data() + current_buffer_size, + /// small_chunk, + /// small_chunk_size); + /// if (buffer->size() < decoder.next_required_size()) { + /// continue; + /// } + /// std::shared_ptr chunk(buffer.release()); + /// decoder.Consume(chunk); + /// buffer = AllocateResizableBuffer(); + /// } + /// if (buffer->size() > 0) { + /// std::shared_ptr chunk(buffer.release()); + /// decoder.Consume(chunk); + /// } + /// ~~~ + /// + /// \return the number of bytes needed to advance the state of the + /// decoder + int64_t next_required_size() const; + + /// \brief Return the current state of the decoder. + /// + /// This method is provided for users who want to optimize performance. + /// Normal users don't need to use this method. + /// + /// Decoder doesn't need Buffer to process data on the + /// MessageDecoder::State::INITIAL state and the + /// MessageDecoder::State::METADATA_LENGTH. Creating Buffer has + /// performance overhead. Advanced users can avoid creating Buffer + /// by checking the current state of the decoder: + /// + /// ~~~{.cpp} + /// switch (decoder.state()) { + /// MessageDecoder::State::INITIAL: + /// MessageDecoder::State::METADATA_LENGTH: + /// { + /// uint8_t data[sizeof(int32_t)]; + /// auto data_size = input->Read(decoder.next_required_size(), data); + /// decoder.Consume(data, data_size); + /// } + /// break; + /// default: + /// { + /// auto buffer = input->Read(decoder.next_required_size()); + /// decoder.Consume(buffer); + /// } + /// break; + /// } + /// ~~~ + /// + /// \return the current state + State state() const; + + private: + class MessageDecoderImpl; + std::unique_ptr impl_; + + ARROW_DISALLOW_COPY_AND_ASSIGN(MessageDecoder); +}; + +/// \brief Abstract interface for a sequence of messages +/// \since 0.5.0 +class ARROW_EXPORT MessageReader { + public: + virtual ~MessageReader() = default; + + /// \brief Create MessageReader that reads from InputStream + static std::unique_ptr Open(io::InputStream* stream); + + /// \brief Create MessageReader that reads from owned InputStream + static std::unique_ptr Open( + const std::shared_ptr& owned_stream); + + /// \brief Read next Message from the interface + /// + /// \return an arrow::ipc::Message instance + virtual Result> ReadNextMessage() = 0; +}; + +// the first parameter of the function should be a pointer to metadata (aka. +// org::apache::arrow::flatbuf::RecordBatch*) +using FieldsLoaderFunction = std::function; + +/// \brief Read encapsulated IPC message from position in file +/// +/// Read a length-prefixed message flatbuffer starting at the indicated file +/// offset. If the message has a body with non-zero length, it will also be +/// read +/// +/// The metadata_length includes at least the length prefix and the flatbuffer +/// +/// \param[in] offset the position in the file where the message starts. The +/// first 4 bytes after the offset are the message length +/// \param[in] metadata_length the total number of bytes to read from file +/// \param[in] file the seekable file interface to read from +/// \param[in] fields_loader the function for loading subset of fields from the given file +/// \return the message read + +ARROW_EXPORT +Result> ReadMessage( + const int64_t offset, const int32_t metadata_length, io::RandomAccessFile* file, + const FieldsLoaderFunction& fields_loader = {}); + +/// \brief Read encapsulated IPC message from position in file +/// +/// Read a length-prefixed message flatbuffer starting at the indicated file +/// offset. +/// +/// The metadata_length includes at least the length prefix and the flatbuffer +/// +/// \param[in] offset the position in the file where the message starts. The +/// first 4 bytes after the offset are the message length +/// \param[in] metadata_length the total number of bytes to read from file +/// \param[in] body_length the number of bytes for the message body +/// \param[in] file the seekable file interface to read from +/// \return the message read + +ARROW_EXPORT +Result> ReadMessage(const int64_t offset, + const int32_t metadata_length, + const int64_t body_length, + io::RandomAccessFile* file); + +/// \brief Read encapsulated IPC message from cached buffers +/// +/// The buffers should contain an entire message. Partial reads are not handled. +/// +/// This method can be used to read just the metadata by passing in a nullptr for the +/// body. The body will then be skipped and the body size will not be validated. +/// +/// If the body buffer is provided then it must be the complete body buffer +/// +/// This is similar to Message::Open but performs slightly more validation (e.g. checks +/// to see that the metadata length is correct and that the body is the size the metadata +/// expected) +/// +/// \param metadata The bytes for the metadata +/// \param body The bytes for the body +/// \return The message represented by the buffers +ARROW_EXPORT Result> ReadMessage( + std::shared_ptr metadata, std::shared_ptr body); + +ARROW_EXPORT +Future> ReadMessageAsync( + const int64_t offset, const int32_t metadata_length, const int64_t body_length, + io::RandomAccessFile* file, const io::IOContext& context = io::default_io_context()); + +/// \brief Advance stream to an 8-byte offset if its position is not a multiple +/// of 8 already +/// \param[in] stream an input stream +/// \param[in] alignment the byte multiple for the metadata prefix, usually 8 +/// or 64, to ensure the body starts on a multiple of that alignment +/// \return Status +ARROW_EXPORT +Status AlignStream(io::InputStream* stream, int32_t alignment = 8); + +/// \brief Advance stream to an 8-byte offset if its position is not a multiple +/// of 8 already +/// \param[in] stream an output stream +/// \param[in] alignment the byte multiple for the metadata prefix, usually 8 +/// or 64, to ensure the body starts on a multiple of that alignment +/// \return Status +ARROW_EXPORT +Status AlignStream(io::OutputStream* stream, int32_t alignment = 8); + +/// \brief Return error Status if file position is not a multiple of the +/// indicated alignment +ARROW_EXPORT +Status CheckAligned(io::FileInterface* stream, int32_t alignment = 8); + +/// \brief Read encapsulated IPC message (metadata and body) from InputStream +/// +/// Returns null if there are not enough bytes available or the +/// message length is 0 (e.g. EOS in a stream) +/// +/// \param[in] stream an input stream +/// \param[in] pool an optional MemoryPool to copy metadata on the CPU, if required +/// \return Message +ARROW_EXPORT +Result> ReadMessage(io::InputStream* stream, + MemoryPool* pool = default_memory_pool()); + +/// \brief Feed data from InputStream to MessageDecoder to decode an +/// encapsulated IPC message (metadata and body) +/// +/// This API is EXPERIMENTAL. +/// +/// \param[in] decoder a decoder +/// \param[in] stream an input stream +/// \return Status +/// +/// \since 0.17.0 +ARROW_EXPORT +Status DecodeMessage(MessageDecoder* decoder, io::InputStream* stream); + +/// Write encapsulated IPC message Does not make assumptions about +/// whether the stream is aligned already. Can write legacy (pre +/// version 0.15.0) IPC message if option set +/// +/// continuation: 0xFFFFFFFF +/// message_size: int32 +/// message: const void* +/// padding +/// +/// +/// \param[in] message a buffer containing the metadata to write +/// \param[in] options IPC writing options, including alignment and +/// legacy message support +/// \param[in,out] file the OutputStream to write to +/// \param[out] message_length the total size of the payload written including +/// padding +/// \return Status +Status WriteMessage(const Buffer& message, const IpcWriteOptions& options, + io::OutputStream* file, int32_t* message_length); + +} // namespace ipc +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/options.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/options.h new file mode 100644 index 0000000000000000000000000000000000000000..9cd0999d2c243e89c1bac43df44c0aecf1a13656 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/options.h @@ -0,0 +1,201 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "arrow/io/caching.h" +#include "arrow/ipc/type_fwd.h" +#include "arrow/status.h" +#include "arrow/type_fwd.h" +#include "arrow/util/align_util.h" +#include "arrow/util/compression.h" +#include "arrow/util/visibility.h" + +namespace arrow { + +class MemoryPool; + +namespace ipc { + +// ARROW-109: We set this number arbitrarily to help catch user mistakes. For +// deeply nested schemas, it is expected the user will indicate explicitly the +// maximum allowed recursion depth +constexpr int kMaxNestingDepth = 64; + +/// \brief Options for writing Arrow IPC messages +struct ARROW_EXPORT IpcWriteOptions { + /// \brief If true, allow field lengths that don't fit in a signed 32-bit int. + /// + /// Some implementations may not be able to parse streams created with this option. + bool allow_64bit = false; + + /// \brief The maximum permitted schema nesting depth. + int max_recursion_depth = kMaxNestingDepth; + + /// \brief Write padding after memory buffers up to this multiple of bytes. + int32_t alignment = 8; + + /// \brief Write the pre-0.15.0 IPC message format + /// + /// This legacy format consists of a 4-byte prefix instead of 8-byte. + bool write_legacy_ipc_format = false; + + /// \brief The memory pool to use for allocations made during IPC writing + /// + /// While Arrow IPC is predominantly zero-copy, it may have to allocate + /// memory in some cases (for example if compression is enabled). + MemoryPool* memory_pool = default_memory_pool(); + + /// \brief Compression codec to use for record batch body buffers + /// + /// May only be UNCOMPRESSED, LZ4_FRAME and ZSTD. + std::shared_ptr codec; + + /// \brief Minimum space savings percentage required for compression to be applied + /// + /// Space savings is calculated as (1.0 - compressed_size / uncompressed_size). + /// + /// For example, if min_space_savings = 0.1, a 100-byte body buffer won't undergo + /// compression if its expected compressed size exceeds 90 bytes. If this option is + /// unset, compression will be used indiscriminately. If no codec was supplied, this + /// option is ignored. + /// + /// Values outside of the range [0,1] are handled as errors. + /// + /// Note that enabling this option may result in unreadable data for Arrow C++ versions + /// prior to 12.0.0. + std::optional min_space_savings; + + /// \brief Use global CPU thread pool to parallelize any computational tasks + /// like compression + bool use_threads = true; + + /// \brief Whether to emit dictionary deltas + /// + /// If false, a changed dictionary for a given field will emit a full + /// dictionary replacement. + /// If true, a changed dictionary will be compared against the previous + /// version. If possible, a dictionary delta will be emitted, otherwise + /// a full dictionary replacement. + /// + /// Default is false to maximize stream compatibility. + /// + /// Also, note that if a changed dictionary is a nested dictionary, + /// then a delta is never emitted, for compatibility with the read path. + bool emit_dictionary_deltas = false; + + /// \brief Whether to unify dictionaries for the IPC file format + /// + /// The IPC file format doesn't support dictionary replacements. + /// Therefore, chunks of a column with a dictionary type must have the same + /// dictionary in each record batch (or an extended dictionary + delta). + /// + /// If this option is true, RecordBatchWriter::WriteTable will attempt + /// to unify dictionaries across each table column. If this option is + /// false, incompatible dictionaries across a table column will simply + /// raise an error. + /// + /// Note that enabling this option has a runtime cost. Also, not all types + /// currently support dictionary unification. + /// + /// This option is ignored for IPC streams, which support dictionary replacement + /// and deltas. + bool unify_dictionaries = false; + + /// \brief Format version to use for IPC messages and their metadata. + /// + /// Presently using V5 version (readable by 1.0.0 and later). + /// V4 is also available (readable by 0.8.0 and later). + MetadataVersion metadata_version = MetadataVersion::V5; + + static IpcWriteOptions Defaults(); +}; + +/// \brief Alignment of data in memory +/// Alignment values larger than 0 are taken directly as byte alignment value +/// See util::EnsureAlignment(..., int64_t alignment, ...) +enum class Alignment : int64_t { + /// \brief data is aligned depending on the actual data type + kDataTypeSpecificAlignment = util::kValueAlignment, + /// \brief no particular alignment enforced + kAnyAlignment = 0, + /// \brief data is aligned to 64-byte boundary + k64ByteAlignment = 64 +}; + +/// \brief Options for reading Arrow IPC messages +struct ARROW_EXPORT IpcReadOptions { + /// \brief The maximum permitted schema nesting depth. + int max_recursion_depth = kMaxNestingDepth; + + /// \brief The memory pool to use for allocations made during IPC reading + /// + /// While Arrow IPC is predominantly zero-copy, it may have to allocate + /// memory in some cases (for example if compression is enabled). + MemoryPool* memory_pool = default_memory_pool(); + + /// \brief Top-level schema fields to include when deserializing RecordBatch. + /// + /// If empty (the default), return all deserialized fields. + /// If non-empty, the values are the indices of fields in the top-level schema. + std::vector included_fields; + + /// \brief Use global CPU thread pool to parallelize any computational tasks + /// like decompression + bool use_threads = true; + + /// \brief Whether to convert incoming data to platform-native endianness + /// + /// If the endianness of the received schema is not equal to platform-native + /// endianness, then all buffers with endian-sensitive data will be byte-swapped. + /// This includes the value buffers of numeric types, temporal types, decimal + /// types, as well as the offset buffers of variable-sized binary and list-like + /// types. + /// + /// Endianness conversion is achieved by the RecordBatchFileReader, + /// RecordBatchStreamReader and StreamDecoder classes. + bool ensure_native_endian = true; + + /// \brief How to align data if mis-aligned + /// + /// Data is copied to aligned memory locations allocated via the + /// MemoryPool configured as \ref arrow::ipc::IpcReadOptions::memory_pool. + /// Some use cases might require data to have a specific alignment, for example, + /// for the data buffer of an Int32 array to be aligned on a 4-byte boundary. + /// + /// Default (kAnyAlignment) keeps the alignment as is, so no copy of data occurs. + Alignment ensure_alignment = Alignment::kAnyAlignment; + + /// \brief Options to control caching behavior when pre-buffering is requested + /// + /// The lazy property will always be reset to true to deliver the expected behavior + io::CacheOptions pre_buffer_cache_options = io::CacheOptions::LazyDefaults(); + + static IpcReadOptions Defaults(); +}; + +namespace internal { + +Status CheckCompressionSupported(Compression::type codec); + +} // namespace internal +} // namespace ipc +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/reader.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/reader.h new file mode 100644 index 0000000000000000000000000000000000000000..d3526bf455d3bb9d9b49f8acff763072f86aa245 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/reader.h @@ -0,0 +1,638 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Read Arrow files and streams + +#pragma once + +#include +#include +#include +#include +#include + +#include "arrow/io/caching.h" +#include "arrow/io/type_fwd.h" +#include "arrow/ipc/message.h" +#include "arrow/ipc/options.h" +#include "arrow/record_batch.h" +#include "arrow/result.h" +#include "arrow/type_fwd.h" +#include "arrow/util/async_generator.h" +#include "arrow/util/macros.h" +#include "arrow/util/visibility.h" + +namespace arrow { +namespace ipc { + +class DictionaryMemo; +struct IpcPayload; + +using RecordBatchReader = ::arrow::RecordBatchReader; + +struct ReadStats { + /// Number of IPC messages read. + int64_t num_messages = 0; + /// Number of record batches read. + int64_t num_record_batches = 0; + /// Number of dictionary batches read. + /// + /// Note: num_dictionary_batches >= num_dictionary_deltas + num_replaced_dictionaries + int64_t num_dictionary_batches = 0; + + /// Number of dictionary deltas read. + int64_t num_dictionary_deltas = 0; + /// Number of replaced dictionaries (i.e. where a dictionary batch replaces + /// an existing dictionary with an unrelated new dictionary). + int64_t num_replaced_dictionaries = 0; +}; + +/// \brief Synchronous batch stream reader that reads from io::InputStream +/// +/// This class reads the schema (plus any dictionaries) as the first messages +/// in the stream, followed by record batches. For more granular zero-copy +/// reads see the ReadRecordBatch functions +class ARROW_EXPORT RecordBatchStreamReader : public RecordBatchReader { + public: + /// Create batch reader from generic MessageReader. + /// This will take ownership of the given MessageReader. + /// + /// \param[in] message_reader a MessageReader implementation + /// \param[in] options any IPC reading options (optional) + /// \return the created batch reader + static Result> Open( + std::unique_ptr message_reader, + const IpcReadOptions& options = IpcReadOptions::Defaults()); + + /// \brief Record batch stream reader from InputStream + /// + /// \param[in] stream an input stream instance. Must stay alive throughout + /// lifetime of stream reader + /// \param[in] options any IPC reading options (optional) + /// \return the created batch reader + static Result> Open( + io::InputStream* stream, + const IpcReadOptions& options = IpcReadOptions::Defaults()); + + /// \brief Open stream and retain ownership of stream object + /// \param[in] stream the input stream + /// \param[in] options any IPC reading options (optional) + /// \return the created batch reader + static Result> Open( + const std::shared_ptr& stream, + const IpcReadOptions& options = IpcReadOptions::Defaults()); + + /// \brief Return current read statistics + virtual ReadStats stats() const = 0; +}; + +/// \brief Reads the record batch file format +class ARROW_EXPORT RecordBatchFileReader + : public std::enable_shared_from_this { + public: + virtual ~RecordBatchFileReader() = default; + + /// \brief Open a RecordBatchFileReader + /// + /// Open a file-like object that is assumed to be self-contained; i.e., the + /// end of the file interface is the end of the Arrow file. Note that there + /// can be any amount of data preceding the Arrow-formatted data, because we + /// need only locate the end of the Arrow file stream to discover the metadata + /// and then proceed to read the data into memory. + static Result> Open( + io::RandomAccessFile* file, + const IpcReadOptions& options = IpcReadOptions::Defaults()); + + /// \brief Open a RecordBatchFileReader + /// If the file is embedded within some larger file or memory region, you can + /// pass the absolute memory offset to the end of the file (which contains the + /// metadata footer). The metadata must have been written with memory offsets + /// relative to the start of the containing file + /// + /// \param[in] file the data source + /// \param[in] footer_offset the position of the end of the Arrow file + /// \param[in] options options for IPC reading + /// \return the returned reader + static Result> Open( + io::RandomAccessFile* file, int64_t footer_offset, + const IpcReadOptions& options = IpcReadOptions::Defaults()); + + /// \brief Version of Open that retains ownership of file + /// + /// \param[in] file the data source + /// \param[in] options options for IPC reading + /// \return the returned reader + static Result> Open( + const std::shared_ptr& file, + const IpcReadOptions& options = IpcReadOptions::Defaults()); + + /// \brief Version of Open that retains ownership of file + /// + /// \param[in] file the data source + /// \param[in] footer_offset the position of the end of the Arrow file + /// \param[in] options options for IPC reading + /// \return the returned reader + static Result> Open( + const std::shared_ptr& file, int64_t footer_offset, + const IpcReadOptions& options = IpcReadOptions::Defaults()); + + /// \brief Open a file asynchronously (owns the file). + static Future> OpenAsync( + const std::shared_ptr& file, + const IpcReadOptions& options = IpcReadOptions::Defaults()); + + /// \brief Open a file asynchronously (borrows the file). + static Future> OpenAsync( + io::RandomAccessFile* file, + const IpcReadOptions& options = IpcReadOptions::Defaults()); + + /// \brief Open a file asynchronously (owns the file). + static Future> OpenAsync( + const std::shared_ptr& file, int64_t footer_offset, + const IpcReadOptions& options = IpcReadOptions::Defaults()); + + /// \brief Open a file asynchronously (borrows the file). + static Future> OpenAsync( + io::RandomAccessFile* file, int64_t footer_offset, + const IpcReadOptions& options = IpcReadOptions::Defaults()); + + /// \brief The schema read from the file + virtual std::shared_ptr schema() const = 0; + + /// \brief Returns the number of record batches in the file + virtual int num_record_batches() const = 0; + + /// \brief Return the metadata version from the file metadata + virtual MetadataVersion version() const = 0; + + /// \brief Return the contents of the custom_metadata field from the file's + /// Footer + virtual std::shared_ptr metadata() const = 0; + + /// \brief Read a particular record batch from the file. Does not copy memory + /// if the input source supports zero-copy. + /// + /// \param[in] i the index of the record batch to return + /// \return the read batch + virtual Result> ReadRecordBatch(int i) = 0; + + /// \brief Read a particular record batch along with its custom metadata from the file. + /// Does not copy memory if the input source supports zero-copy. + /// + /// \param[in] i the index of the record batch to return + /// \return a struct containing the read batch and its custom metadata + virtual Result ReadRecordBatchWithCustomMetadata(int i) = 0; + + /// \brief Return current read statistics + virtual ReadStats stats() const = 0; + + /// \brief Computes the total number of rows in the file. + virtual Result CountRows() = 0; + + /// \brief Begin loading metadata for the desired batches into memory. + /// + /// This method will also begin loading all dictionaries messages into memory. + /// + /// For a regular file this will immediately begin disk I/O in the background on a + /// thread on the IOContext's thread pool. If the file is memory mapped this will + /// ensure the memory needed for the metadata is paged from disk into memory + /// + /// \param indices Indices of the batches to prefetch + /// If empty then all batches will be prefetched. + virtual Status PreBufferMetadata(const std::vector& indices) = 0; + + /// \brief Get a reentrant generator of record batches. + /// + /// \param[in] coalesce If true, enable I/O coalescing. + /// \param[in] io_context The IOContext to use (controls which thread pool + /// is used for I/O). + /// \param[in] cache_options Options for coalescing (if enabled). + /// \param[in] executor Optionally, an executor to use for decoding record + /// batches. This is generally only a benefit for very wide and/or + /// compressed batches. + virtual Result>> GetRecordBatchGenerator( + const bool coalesce = false, + const io::IOContext& io_context = io::default_io_context(), + const io::CacheOptions cache_options = io::CacheOptions::LazyDefaults(), + arrow::internal::Executor* executor = NULLPTR) = 0; + + /// \brief Collect all batches as a vector of record batches + Result ToRecordBatches(); + + /// \brief Collect all batches and concatenate as arrow::Table + Result> ToTable(); +}; + +/// \brief A general listener class to receive events. +/// +/// You must implement callback methods for interested events. +/// +/// This API is EXPERIMENTAL. +/// +/// \since 0.17.0 +class ARROW_EXPORT Listener { + public: + virtual ~Listener() = default; + + /// \brief Called when end-of-stream is received. + /// + /// The default implementation just returns arrow::Status::OK(). + /// + /// \return Status + /// + /// \see StreamDecoder + virtual Status OnEOS(); + + /// \brief Called when a record batch is decoded and + /// OnRecordBatchWithMetadataDecoded() isn't overridden. + /// + /// The default implementation just returns + /// arrow::Status::NotImplemented(). + /// + /// \param[in] record_batch a record batch decoded + /// \return Status + /// + /// \see StreamDecoder + virtual Status OnRecordBatchDecoded(std::shared_ptr record_batch); + + /// \brief Called when a record batch with custom metadata is decoded. + /// + /// The default implementation just calls OnRecordBatchDecoded() + /// without custom metadata. + /// + /// \param[in] record_batch_with_metadata a record batch with custom + /// metadata decoded + /// \return Status + /// + /// \see StreamDecoder + /// + /// \since 13.0.0 + virtual Status OnRecordBatchWithMetadataDecoded( + RecordBatchWithMetadata record_batch_with_metadata); + + /// \brief Called when a schema is decoded. + /// + /// The default implementation just returns arrow::Status::OK(). + /// + /// \param[in] schema a schema decoded + /// \return Status + /// + /// \see StreamDecoder + virtual Status OnSchemaDecoded(std::shared_ptr schema); + + /// \brief Called when a schema is decoded. + /// + /// The default implementation just calls OnSchemaDecoded(schema) + /// (without filtered_schema) to keep backward compatibility. + /// + /// \param[in] schema a schema decoded + /// \param[in] filtered_schema a filtered schema that only has read fields + /// \return Status + /// + /// \see StreamDecoder + /// + /// \since 13.0.0 + virtual Status OnSchemaDecoded(std::shared_ptr schema, + std::shared_ptr filtered_schema); +}; + +/// \brief Collect schema and record batches decoded by StreamDecoder. +/// +/// This API is EXPERIMENTAL. +/// +/// \since 0.17.0 +class ARROW_EXPORT CollectListener : public Listener { + public: + CollectListener() : schema_(), filtered_schema_(), record_batches_(), metadatas_() {} + virtual ~CollectListener() = default; + + Status OnSchemaDecoded(std::shared_ptr schema, + std::shared_ptr filtered_schema) override { + schema_ = std::move(schema); + filtered_schema_ = std::move(filtered_schema); + return Status::OK(); + } + + Status OnRecordBatchWithMetadataDecoded( + RecordBatchWithMetadata record_batch_with_metadata) override { + record_batches_.push_back(std::move(record_batch_with_metadata.batch)); + metadatas_.push_back(std::move(record_batch_with_metadata.custom_metadata)); + return Status::OK(); + } + + /// \return the decoded schema + std::shared_ptr schema() const { return schema_; } + + /// \return the filtered schema + std::shared_ptr filtered_schema() const { return filtered_schema_; } + + /// \return the all decoded record batches + const std::vector>& record_batches() const { + return record_batches_; + } + + /// \return the all decoded metadatas + const std::vector>& metadatas() const { + return metadatas_; + } + + /// \return the number of collected record batches + int64_t num_record_batches() const { return record_batches_.size(); } + + /// \return the last decoded record batch and remove it from + /// record_batches + std::shared_ptr PopRecordBatch() { + auto record_batch_with_metadata = PopRecordBatchWithMetadata(); + return std::move(record_batch_with_metadata.batch); + } + + /// \return the last decoded record batch with custom metadata and + /// remove it from record_batches + RecordBatchWithMetadata PopRecordBatchWithMetadata() { + RecordBatchWithMetadata record_batch_with_metadata; + if (record_batches_.empty()) { + return record_batch_with_metadata; + } + record_batch_with_metadata.batch = std::move(record_batches_.back()); + record_batch_with_metadata.custom_metadata = std::move(metadatas_.back()); + record_batches_.pop_back(); + metadatas_.pop_back(); + return record_batch_with_metadata; + } + + private: + std::shared_ptr schema_; + std::shared_ptr filtered_schema_; + std::vector> record_batches_; + std::vector> metadatas_; +}; + +/// \brief Push style stream decoder that receives data from user. +/// +/// This class decodes the Apache Arrow IPC streaming format data. +/// +/// This API is EXPERIMENTAL. +/// +/// \see https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format +/// +/// \since 0.17.0 +class ARROW_EXPORT StreamDecoder { + public: + /// \brief Construct a stream decoder. + /// + /// \param[in] listener a Listener that must implement + /// Listener::OnRecordBatchDecoded() to receive decoded record batches + /// \param[in] options any IPC reading options (optional) + StreamDecoder(std::shared_ptr listener, + IpcReadOptions options = IpcReadOptions::Defaults()); + + virtual ~StreamDecoder(); + + /// \brief Feed data to the decoder as a raw data. + /// + /// If the decoder can read one or more record batches by the data, + /// the decoder calls listener->OnRecordBatchDecoded() with a + /// decoded record batch multiple times. + /// + /// \param[in] data a raw data to be processed. This data isn't + /// copied. The passed memory must be kept alive through record + /// batch processing. + /// \param[in] size raw data size. + /// \return Status + Status Consume(const uint8_t* data, int64_t size); + + /// \brief Feed data to the decoder as a Buffer. + /// + /// If the decoder can read one or more record batches by the + /// Buffer, the decoder calls listener->RecordBatchReceived() with a + /// decoded record batch multiple times. + /// + /// \param[in] buffer a Buffer to be processed. + /// \return Status + Status Consume(std::shared_ptr buffer); + + /// \brief Reset the internal status. + /// + /// You can reuse this decoder for new stream after calling + /// this. + /// + /// \return Status + Status Reset(); + + /// \return the shared schema of the record batches in the stream + std::shared_ptr schema() const; + + /// \brief Return the number of bytes needed to advance the state of + /// the decoder. + /// + /// This method is provided for users who want to optimize performance. + /// Normal users don't need to use this method. + /// + /// Here is an example usage for normal users: + /// + /// ~~~{.cpp} + /// decoder.Consume(buffer1); + /// decoder.Consume(buffer2); + /// decoder.Consume(buffer3); + /// ~~~ + /// + /// Decoder has internal buffer. If consumed data isn't enough to + /// advance the state of the decoder, consumed data is buffered to + /// the internal buffer. It causes performance overhead. + /// + /// If you pass next_required_size() size data to each Consume() + /// call, the decoder doesn't use its internal buffer. It improves + /// performance. + /// + /// Here is an example usage to avoid using internal buffer: + /// + /// ~~~{.cpp} + /// buffer1 = get_data(decoder.next_required_size()); + /// decoder.Consume(buffer1); + /// buffer2 = get_data(decoder.next_required_size()); + /// decoder.Consume(buffer2); + /// ~~~ + /// + /// Users can use this method to avoid creating small chunks. Record + /// batch data must be contiguous data. If users pass small chunks + /// to the decoder, the decoder needs concatenate small chunks + /// internally. It causes performance overhead. + /// + /// Here is an example usage to reduce small chunks: + /// + /// ~~~{.cpp} + /// buffer = AllocateResizableBuffer(); + /// while ((small_chunk = get_data(&small_chunk_size))) { + /// auto current_buffer_size = buffer->size(); + /// buffer->Resize(current_buffer_size + small_chunk_size); + /// memcpy(buffer->mutable_data() + current_buffer_size, + /// small_chunk, + /// small_chunk_size); + /// if (buffer->size() < decoder.next_required_size()) { + /// continue; + /// } + /// std::shared_ptr chunk(buffer.release()); + /// decoder.Consume(chunk); + /// buffer = AllocateResizableBuffer(); + /// } + /// if (buffer->size() > 0) { + /// std::shared_ptr chunk(buffer.release()); + /// decoder.Consume(chunk); + /// } + /// ~~~ + /// + /// \return the number of bytes needed to advance the state of the + /// decoder + int64_t next_required_size() const; + + /// \brief Return current read statistics + ReadStats stats() const; + + private: + class StreamDecoderImpl; + std::unique_ptr impl_; + + ARROW_DISALLOW_COPY_AND_ASSIGN(StreamDecoder); +}; + +// Generic read functions; does not copy data if the input supports zero copy reads + +/// \brief Read Schema from stream serialized as a single IPC message +/// and populate any dictionary-encoded fields into a DictionaryMemo +/// +/// \param[in] stream an InputStream +/// \param[in] dictionary_memo for recording dictionary-encoded fields +/// \return the output Schema +/// +/// If record batches follow the schema, it is better to use +/// RecordBatchStreamReader +ARROW_EXPORT +Result> ReadSchema(io::InputStream* stream, + DictionaryMemo* dictionary_memo); + +/// \brief Read Schema from encapsulated Message +/// +/// \param[in] message the message containing the Schema IPC metadata +/// \param[in] dictionary_memo DictionaryMemo for recording dictionary-encoded +/// fields. Can be nullptr if you are sure there are no +/// dictionary-encoded fields +/// \return the resulting Schema +ARROW_EXPORT +Result> ReadSchema(const Message& message, + DictionaryMemo* dictionary_memo); + +/// Read record batch as encapsulated IPC message with metadata size prefix and +/// header +/// +/// \param[in] schema the record batch schema +/// \param[in] dictionary_memo DictionaryMemo which has any +/// dictionaries. Can be nullptr if you are sure there are no +/// dictionary-encoded fields +/// \param[in] options IPC options for reading +/// \param[in] stream the file where the batch is located +/// \return the read record batch +ARROW_EXPORT +Result> ReadRecordBatch( + const std::shared_ptr& schema, const DictionaryMemo* dictionary_memo, + const IpcReadOptions& options, io::InputStream* stream); + +/// \brief Read record batch from message +/// +/// \param[in] message a Message containing the record batch metadata +/// \param[in] schema the record batch schema +/// \param[in] dictionary_memo DictionaryMemo which has any +/// dictionaries. Can be nullptr if you are sure there are no +/// dictionary-encoded fields +/// \param[in] options IPC options for reading +/// \return the read record batch +ARROW_EXPORT +Result> ReadRecordBatch( + const Message& message, const std::shared_ptr& schema, + const DictionaryMemo* dictionary_memo, const IpcReadOptions& options); + +/// Read record batch from file given metadata and schema +/// +/// \param[in] metadata a Message containing the record batch metadata +/// \param[in] schema the record batch schema +/// \param[in] dictionary_memo DictionaryMemo which has any +/// dictionaries. Can be nullptr if you are sure there are no +/// dictionary-encoded fields +/// \param[in] file a random access file +/// \param[in] options options for deserialization +/// \return the read record batch +ARROW_EXPORT +Result> ReadRecordBatch( + const Buffer& metadata, const std::shared_ptr& schema, + const DictionaryMemo* dictionary_memo, const IpcReadOptions& options, + io::RandomAccessFile* file); + +/// \brief Read arrow::Tensor as encapsulated IPC message in file +/// +/// \param[in] file an InputStream pointed at the start of the message +/// \return the read tensor +ARROW_EXPORT +Result> ReadTensor(io::InputStream* file); + +/// \brief EXPERIMENTAL: Read arrow::Tensor from IPC message +/// +/// \param[in] message a Message containing the tensor metadata and body +/// \return the read tensor +ARROW_EXPORT +Result> ReadTensor(const Message& message); + +/// \brief EXPERIMENTAL: Read arrow::SparseTensor as encapsulated IPC message in file +/// +/// \param[in] file an InputStream pointed at the start of the message +/// \return the read sparse tensor +ARROW_EXPORT +Result> ReadSparseTensor(io::InputStream* file); + +/// \brief EXPERIMENTAL: Read arrow::SparseTensor from IPC message +/// +/// \param[in] message a Message containing the tensor metadata and body +/// \return the read sparse tensor +ARROW_EXPORT +Result> ReadSparseTensor(const Message& message); + +namespace internal { + +// These internal APIs may change without warning or deprecation + +/// \brief EXPERIMENTAL: Read arrow::SparseTensorFormat::type from a metadata +/// \param[in] metadata a Buffer containing the sparse tensor metadata +/// \return the count of the body buffers +ARROW_EXPORT +Result ReadSparseTensorBodyBufferCount(const Buffer& metadata); + +/// \brief EXPERIMENTAL: Read arrow::SparseTensor from an IpcPayload +/// \param[in] payload a IpcPayload contains a serialized SparseTensor +/// \return the read sparse tensor +ARROW_EXPORT +Result> ReadSparseTensorPayload(const IpcPayload& payload); + +// For fuzzing targets +ARROW_EXPORT +Status FuzzIpcStream(const uint8_t* data, int64_t size); +ARROW_EXPORT +Status FuzzIpcTensorStream(const uint8_t* data, int64_t size); +ARROW_EXPORT +Status FuzzIpcFile(const uint8_t* data, int64_t size); + +} // namespace internal + +} // namespace ipc +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/test_common.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/test_common.h new file mode 100644 index 0000000000000000000000000000000000000000..f6982b3e11a7c0eb6e6e06b8a0d996d11c9882ad --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/test_common.h @@ -0,0 +1,198 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "arrow/array.h" +#include "arrow/record_batch.h" +#include "arrow/status.h" +#include "arrow/testing/visibility.h" +#include "arrow/type.h" + +namespace arrow { +namespace ipc { +namespace test { + +// A typedef used for test parameterization +typedef Status MakeRecordBatch(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +void CompareArraysDetailed(int index, const Array& result, const Array& expected); + +ARROW_TESTING_EXPORT +void CompareBatchColumnsDetailed(const RecordBatch& result, const RecordBatch& expected); + +ARROW_TESTING_EXPORT +Status MakeRandomInt32Array(int64_t length, bool include_nulls, MemoryPool* pool, + std::shared_ptr* out, uint32_t seed = 0, + int32_t min = 0, int32_t max = 1000); + +ARROW_TESTING_EXPORT +Status MakeRandomInt64Array(int64_t length, bool include_nulls, MemoryPool* pool, + std::shared_ptr* out, uint32_t seed = 0); + +ARROW_TESTING_EXPORT +Status MakeRandomListArray(const std::shared_ptr& child_array, int num_lists, + bool include_nulls, MemoryPool* pool, + std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeRandomLargeListArray(const std::shared_ptr& child_array, int num_lists, + bool include_nulls, MemoryPool* pool, + std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeRandomBooleanArray(const int length, bool include_nulls, + std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeBooleanBatchSized(const int length, std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeBooleanBatch(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeIntBatchSized(int length, std::shared_ptr* out, + uint32_t seed = 0); + +ARROW_TESTING_EXPORT +Status MakeIntRecordBatch(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeFloat3264BatchSized(int length, std::shared_ptr* out, + uint32_t seed = 0); + +ARROW_TESTING_EXPORT +Status MakeFloat3264Batch(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeFloatBatchSized(int length, std::shared_ptr* out, + uint32_t seed = 0); + +ARROW_TESTING_EXPORT +Status MakeFloatBatch(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeRandomStringArray(int64_t length, bool include_nulls, MemoryPool* pool, + std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeStringTypesRecordBatch(std::shared_ptr* out, + bool with_nulls = true, bool with_view_types = true); + +ARROW_TESTING_EXPORT +Status MakeStringTypesRecordBatchWithNulls(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeNullRecordBatch(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeListRecordBatchSized(int length, std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeListRecordBatch(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeListViewRecordBatchSized(int length, std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeListViewRecordBatch(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeFixedSizeListRecordBatch(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeZeroLengthRecordBatch(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeNonNullRecordBatch(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeDeeplyNestedList(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeDeeplyNestedListView(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeStruct(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeRunEndEncoded(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeUnion(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeDictionary(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeDictionaryFlat(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeNestedDictionary(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeMap(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeMapOfDictionary(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeDates(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeTimestamps(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeIntervals(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeTimes(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeFWBinary(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeDecimal(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeNull(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeUuid(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeComplex128(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeDictExtension(std::shared_ptr* out); + +ARROW_TESTING_EXPORT +Status MakeRandomTensor(const std::shared_ptr& type, + const std::vector& shape, bool row_major_p, + std::shared_ptr* out, uint32_t seed = 0); + +ARROW_TESTING_EXPORT Status RoundtripBatch(const std::shared_ptr& batch, + std::shared_ptr* out); + +} // namespace test +} // namespace ipc +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/type_fwd.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/type_fwd.h new file mode 100644 index 0000000000000000000000000000000000000000..7d32096ea7ebf13a7e3fa1eeab97129217f54c03 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/type_fwd.h @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +namespace arrow { +namespace ipc { + +enum class MetadataVersion : char { + /// 0.1.0 + V1, + + /// 0.2.0 + V2, + + /// 0.3.0 to 0.7.1 + V3, + + /// 0.8.0 to 0.17.0 + V4, + + /// >= 1.0.0 + V5 +}; + +class Message; +enum class MessageType { + NONE, + SCHEMA, + DICTIONARY_BATCH, + RECORD_BATCH, + TENSOR, + SPARSE_TENSOR +}; + +struct IpcReadOptions; +struct IpcWriteOptions; + +class MessageReader; + +class RecordBatchStreamReader; +class RecordBatchFileReader; +class RecordBatchWriter; + +class DictionaryFieldMapper; +class DictionaryMemo; + +namespace feather { + +class Reader; + +} // namespace feather +} // namespace ipc +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/util.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/util.h new file mode 100644 index 0000000000000000000000000000000000000000..2c913f232953fcb2947584b28573a4fa7b941296 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/util.h @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +namespace arrow { +namespace ipc { + +// Buffers are padded to 64-byte boundaries (for SIMD) +static constexpr int32_t kArrowAlignment = 64; + +// Tensors are padded to 64-byte boundaries +static constexpr int32_t kTensorAlignment = 64; + +// Align on 8-byte boundaries in IPC +static constexpr int32_t kArrowIpcAlignment = 8; + +static constexpr uint8_t kPaddingBytes[kArrowAlignment] = {0}; + +static inline int64_t PaddedLength(int64_t nbytes, int32_t alignment = kArrowAlignment) { + return ((nbytes + alignment - 1) / alignment) * alignment; +} + +} // namespace ipc +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/writer.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/writer.h new file mode 100644 index 0000000000000000000000000000000000000000..f44fd4fbb10e06cc45e4b51820b6e68e71203741 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/ipc/writer.h @@ -0,0 +1,475 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Implement Arrow streaming binary format + +#pragma once + +#include +#include +#include + +#include "arrow/ipc/dictionary.h" // IWYU pragma: export +#include "arrow/ipc/message.h" +#include "arrow/ipc/options.h" +#include "arrow/result.h" +#include "arrow/util/macros.h" +#include "arrow/util/visibility.h" + +namespace arrow { + +class Array; +class Buffer; +class MemoryManager; +class MemoryPool; +class RecordBatch; +class Schema; +class Status; +class Table; +class Tensor; +class SparseTensor; + +namespace io { + +class OutputStream; + +} // namespace io + +namespace ipc { + +/// \brief Intermediate data structure with metadata header, and zero +/// or more buffers for the message body. +struct IpcPayload { + MessageType type = MessageType::NONE; + std::shared_ptr metadata; + std::vector> body_buffers; + std::vector variadic_buffer_counts; + int64_t body_length = 0; // serialized body length (padded, maybe compressed) + int64_t raw_body_length = 0; // initial uncompressed body length +}; + +struct WriteStats { + /// Number of IPC messages written. + int64_t num_messages = 0; + /// Number of record batches written. + int64_t num_record_batches = 0; + /// Number of dictionary batches written. + /// + /// Note: num_dictionary_batches >= num_dictionary_deltas + num_replaced_dictionaries + int64_t num_dictionary_batches = 0; + + /// Number of dictionary deltas written. + int64_t num_dictionary_deltas = 0; + /// Number of replaced dictionaries (i.e. where a dictionary batch replaces + /// an existing dictionary with an unrelated new dictionary). + int64_t num_replaced_dictionaries = 0; + + /// Total size in bytes of record batches emitted. + /// The "raw" size counts the original buffer sizes, while the "serialized" size + /// includes padding and (optionally) compression. + int64_t total_raw_body_size = 0; + int64_t total_serialized_body_size = 0; +}; + +/// \class RecordBatchWriter +/// \brief Abstract interface for writing a stream of record batches +class ARROW_EXPORT RecordBatchWriter { + public: + virtual ~RecordBatchWriter(); + + /// \brief Write a record batch to the stream + /// + /// \param[in] batch the record batch to write to the stream + /// \return Status + virtual Status WriteRecordBatch(const RecordBatch& batch) = 0; + + /// \brief Write a record batch with custom metadata to the stream + /// + /// \param[in] batch the record batch to write to the stream + /// \param[in] custom_metadata the record batch's custom metadata to write to the stream + /// \return Status + virtual Status WriteRecordBatch( + const RecordBatch& batch, + const std::shared_ptr& custom_metadata); + + /// \brief Write possibly-chunked table by creating sequence of record batches + /// \param[in] table table to write + /// \return Status + Status WriteTable(const Table& table); + + /// \brief Write Table with a particular chunksize + /// \param[in] table table to write + /// \param[in] max_chunksize maximum number of rows for table chunks. To + /// indicate that no maximum should be enforced, pass -1. + /// \return Status + virtual Status WriteTable(const Table& table, int64_t max_chunksize); + + /// \brief Perform any logic necessary to finish the stream + /// + /// \return Status + virtual Status Close() = 0; + + /// \brief Return current write statistics + virtual WriteStats stats() const = 0; +}; + +/// \defgroup record-batch-writer-factories Functions for creating RecordBatchWriter +/// instances +/// +/// @{ + +/// Create a new IPC stream writer from stream sink and schema. User is +/// responsible for closing the actual OutputStream. +/// +/// \param[in] sink output stream to write to +/// \param[in] schema the schema of the record batches to be written +/// \param[in] options options for serialization +/// \return Result> +ARROW_EXPORT +Result> MakeStreamWriter( + io::OutputStream* sink, const std::shared_ptr& schema, + const IpcWriteOptions& options = IpcWriteOptions::Defaults()); + +/// Create a new IPC stream writer from stream sink and schema. User is +/// responsible for closing the actual OutputStream. +/// +/// \param[in] sink output stream to write to +/// \param[in] schema the schema of the record batches to be written +/// \param[in] options options for serialization +/// \return Result> +ARROW_EXPORT +Result> MakeStreamWriter( + std::shared_ptr sink, const std::shared_ptr& schema, + const IpcWriteOptions& options = IpcWriteOptions::Defaults()); + +/// Create a new IPC file writer from stream sink and schema +/// +/// \param[in] sink output stream to write to +/// \param[in] schema the schema of the record batches to be written +/// \param[in] options options for serialization, optional +/// \param[in] metadata custom metadata for File Footer, optional +/// \return Result> +ARROW_EXPORT +Result> MakeFileWriter( + io::OutputStream* sink, const std::shared_ptr& schema, + const IpcWriteOptions& options = IpcWriteOptions::Defaults(), + const std::shared_ptr& metadata = NULLPTR); + +/// Create a new IPC file writer from stream sink and schema +/// +/// \param[in] sink output stream to write to +/// \param[in] schema the schema of the record batches to be written +/// \param[in] options options for serialization, optional +/// \param[in] metadata custom metadata for File Footer, optional +/// \return Result> +ARROW_EXPORT +Result> MakeFileWriter( + std::shared_ptr sink, const std::shared_ptr& schema, + const IpcWriteOptions& options = IpcWriteOptions::Defaults(), + const std::shared_ptr& metadata = NULLPTR); + +/// @} + +/// \brief Low-level API for writing a record batch (without schema) +/// to an OutputStream as encapsulated IPC message. See Arrow format +/// documentation for more detail. +/// +/// \param[in] batch the record batch to write +/// \param[in] buffer_start_offset the start offset to use in the buffer metadata, +/// generally should be 0 +/// \param[in] dst an OutputStream +/// \param[out] metadata_length the size of the length-prefixed flatbuffer +/// including padding to a 64-byte boundary +/// \param[out] body_length the size of the contiguous buffer block plus +/// \param[in] options options for serialization +/// \return Status +ARROW_EXPORT +Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset, + io::OutputStream* dst, int32_t* metadata_length, + int64_t* body_length, const IpcWriteOptions& options); + +/// \brief Serialize record batch as encapsulated IPC message in a new buffer +/// +/// \param[in] batch the record batch +/// \param[in] options the IpcWriteOptions to use for serialization +/// \return the serialized message +ARROW_EXPORT +Result> SerializeRecordBatch(const RecordBatch& batch, + const IpcWriteOptions& options); + +/// \brief Serialize record batch as encapsulated IPC message in a new buffer +/// +/// \param[in] batch the record batch +/// \param[in] mm a MemoryManager to allocate memory from +/// \return the serialized message +ARROW_EXPORT +Result> SerializeRecordBatch(const RecordBatch& batch, + std::shared_ptr mm); + +/// \brief Write record batch to OutputStream +/// +/// \param[in] batch the record batch to write +/// \param[in] options the IpcWriteOptions to use for serialization +/// \param[in] out the OutputStream to write the output to +/// \return Status +/// +/// If writing to pre-allocated memory, you can use +/// arrow::ipc::GetRecordBatchSize to compute how much space is required +ARROW_EXPORT +Status SerializeRecordBatch(const RecordBatch& batch, const IpcWriteOptions& options, + io::OutputStream* out); + +/// \brief Serialize schema as encapsulated IPC message +/// +/// \param[in] schema the schema to write +/// \param[in] pool a MemoryPool to allocate memory from +/// \return the serialized schema +ARROW_EXPORT +Result> SerializeSchema(const Schema& schema, + MemoryPool* pool = default_memory_pool()); + +/// \brief Write multiple record batches to OutputStream, including schema +/// \param[in] batches a vector of batches. Must all have same schema +/// \param[in] options options for serialization +/// \param[out] dst an OutputStream +/// \return Status +ARROW_EXPORT +Status WriteRecordBatchStream(const std::vector>& batches, + const IpcWriteOptions& options, io::OutputStream* dst); + +/// \brief Compute the number of bytes needed to write an IPC payload +/// including metadata +/// +/// \param[in] payload the IPC payload to write +/// \param[in] options write options +/// \return the size of the complete encapsulated message +ARROW_EXPORT +int64_t GetPayloadSize(const IpcPayload& payload, + const IpcWriteOptions& options = IpcWriteOptions::Defaults()); + +/// \brief Compute the number of bytes needed to write a record batch including metadata +/// +/// \param[in] batch the record batch to write +/// \param[out] size the size of the complete encapsulated message +/// \return Status +ARROW_EXPORT +Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size); + +/// \brief Compute the number of bytes needed to write a record batch including metadata +/// +/// \param[in] batch the record batch to write +/// \param[in] options options for serialization +/// \param[out] size the size of the complete encapsulated message +/// \return Status +ARROW_EXPORT +Status GetRecordBatchSize(const RecordBatch& batch, const IpcWriteOptions& options, + int64_t* size); + +/// \brief Compute the number of bytes needed to write a tensor including metadata +/// +/// \param[in] tensor the tensor to write +/// \param[out] size the size of the complete encapsulated message +/// \return Status +ARROW_EXPORT +Status GetTensorSize(const Tensor& tensor, int64_t* size); + +/// \brief EXPERIMENTAL: Convert arrow::Tensor to a Message with minimal memory +/// allocation +/// +/// \param[in] tensor the Tensor to write +/// \param[in] pool MemoryPool to allocate space for metadata +/// \return the resulting Message +ARROW_EXPORT +Result> GetTensorMessage(const Tensor& tensor, MemoryPool* pool); + +/// \brief Write arrow::Tensor as a contiguous message. +/// +/// The metadata and body are written assuming 64-byte alignment. It is the +/// user's responsibility to ensure that the OutputStream has been aligned +/// to a 64-byte multiple before writing the message. +/// +/// The message is written out as followed: +/// \code +/// +/// \endcode +/// +/// \param[in] tensor the Tensor to write +/// \param[in] dst the OutputStream to write to +/// \param[out] metadata_length the actual metadata length, including padding +/// \param[out] body_length the actual message body length +/// \return Status +ARROW_EXPORT +Status WriteTensor(const Tensor& tensor, io::OutputStream* dst, int32_t* metadata_length, + int64_t* body_length); + +/// \brief EXPERIMENTAL: Convert arrow::SparseTensor to a Message with minimal memory +/// allocation +/// +/// The message is written out as followed: +/// \code +/// +/// \endcode +/// +/// \param[in] sparse_tensor the SparseTensor to write +/// \param[in] pool MemoryPool to allocate space for metadata +/// \return the resulting Message +ARROW_EXPORT +Result> GetSparseTensorMessage(const SparseTensor& sparse_tensor, + MemoryPool* pool); + +/// \brief EXPERIMENTAL: Write arrow::SparseTensor as a contiguous message. The metadata, +/// sparse index, and body are written assuming 64-byte alignment. It is the +/// user's responsibility to ensure that the OutputStream has been aligned +/// to a 64-byte multiple before writing the message. +/// +/// \param[in] sparse_tensor the SparseTensor to write +/// \param[in] dst the OutputStream to write to +/// \param[out] metadata_length the actual metadata length, including padding +/// \param[out] body_length the actual message body length +/// \return Status +ARROW_EXPORT +Status WriteSparseTensor(const SparseTensor& sparse_tensor, io::OutputStream* dst, + int32_t* metadata_length, int64_t* body_length); + +/// \brief Compute IpcPayload for the given schema +/// \param[in] schema the Schema that is being serialized +/// \param[in] options options for serialization +/// \param[in] mapper object mapping dictionary fields to dictionary ids +/// \param[out] out the returned vector of IpcPayloads +/// \return Status +ARROW_EXPORT +Status GetSchemaPayload(const Schema& schema, const IpcWriteOptions& options, + const DictionaryFieldMapper& mapper, IpcPayload* out); + +/// \brief Compute IpcPayload for a dictionary +/// \param[in] id the dictionary id +/// \param[in] dictionary the dictionary values +/// \param[in] options options for serialization +/// \param[out] payload the output IpcPayload +/// \return Status +ARROW_EXPORT +Status GetDictionaryPayload(int64_t id, const std::shared_ptr& dictionary, + const IpcWriteOptions& options, IpcPayload* payload); + +/// \brief Compute IpcPayload for a dictionary +/// \param[in] id the dictionary id +/// \param[in] is_delta whether the dictionary is a delta dictionary +/// \param[in] dictionary the dictionary values +/// \param[in] options options for serialization +/// \param[out] payload the output IpcPayload +/// \return Status +ARROW_EXPORT +Status GetDictionaryPayload(int64_t id, bool is_delta, + const std::shared_ptr& dictionary, + const IpcWriteOptions& options, IpcPayload* payload); + +/// \brief Compute IpcPayload for the given record batch +/// \param[in] batch the RecordBatch that is being serialized +/// \param[in] options options for serialization +/// \param[out] out the returned IpcPayload +/// \return Status +ARROW_EXPORT +Status GetRecordBatchPayload(const RecordBatch& batch, const IpcWriteOptions& options, + IpcPayload* out); + +/// \brief Compute IpcPayload for the given record batch and custom metadata +/// \param[in] batch the RecordBatch that is being serialized +/// \param[in] custom_metadata the custom metadata to be serialized with the record batch +/// \param[in] options options for serialization +/// \param[out] out the returned IpcPayload +/// \return Status +ARROW_EXPORT +Status GetRecordBatchPayload( + const RecordBatch& batch, + const std::shared_ptr& custom_metadata, + const IpcWriteOptions& options, IpcPayload* out); + +/// \brief Write an IPC payload to the given stream. +/// \param[in] payload the payload to write +/// \param[in] options options for serialization +/// \param[in] dst The stream to write the payload to. +/// \param[out] metadata_length the length of the serialized metadata +/// \return Status +ARROW_EXPORT +Status WriteIpcPayload(const IpcPayload& payload, const IpcWriteOptions& options, + io::OutputStream* dst, int32_t* metadata_length); + +/// \brief Compute IpcPayload for the given sparse tensor +/// \param[in] sparse_tensor the SparseTensor that is being serialized +/// \param[in,out] pool for any required temporary memory allocations +/// \param[out] out the returned IpcPayload +/// \return Status +ARROW_EXPORT +Status GetSparseTensorPayload(const SparseTensor& sparse_tensor, MemoryPool* pool, + IpcPayload* out); + +namespace internal { + +// These internal APIs may change without warning or deprecation + +class ARROW_EXPORT IpcPayloadWriter { + public: + virtual ~IpcPayloadWriter(); + + // Default implementation is a no-op + virtual Status Start(); + + virtual Status WritePayload(const IpcPayload& payload) = 0; + + virtual Status Close() = 0; +}; + +/// Create a new IPC payload stream writer from stream sink. User is +/// responsible for closing the actual OutputStream. +/// +/// \param[in] sink output stream to write to +/// \param[in] options options for serialization +/// \return Result> +ARROW_EXPORT +Result> MakePayloadStreamWriter( + io::OutputStream* sink, const IpcWriteOptions& options = IpcWriteOptions::Defaults()); + +/// Create a new IPC payload file writer from stream sink. +/// +/// \param[in] sink output stream to write to +/// \param[in] schema the schema of the record batches to be written +/// \param[in] options options for serialization, optional +/// \param[in] metadata custom metadata for File Footer, optional +/// \return Status +ARROW_EXPORT +Result> MakePayloadFileWriter( + io::OutputStream* sink, const std::shared_ptr& schema, + const IpcWriteOptions& options = IpcWriteOptions::Defaults(), + const std::shared_ptr& metadata = NULLPTR); + +/// Create a new RecordBatchWriter from IpcPayloadWriter and schema. +/// +/// The format is implicitly the IPC stream format (allowing dictionary +/// replacement and deltas). +/// +/// \param[in] sink the IpcPayloadWriter to write to +/// \param[in] schema the schema of the record batches to be written +/// \param[in] options options for serialization +/// \return Result> +ARROW_EXPORT +Result> OpenRecordBatchWriter( + std::unique_ptr sink, const std::shared_ptr& schema, + const IpcWriteOptions& options = IpcWriteOptions::Defaults()); + +} // namespace internal +} // namespace ipc +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/api.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/api.h new file mode 100644 index 0000000000000000000000000000000000000000..0bc35a66fd4a147f3fc82e9d477a98144663802d --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/api.h @@ -0,0 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "arrow/json/options.h" +#include "arrow/json/reader.h" diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/chunked_builder.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/chunked_builder.h new file mode 100644 index 0000000000000000000000000000000000000000..521f895b6300150bcb105704e919df15c1ed9878 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/chunked_builder.h @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "arrow/status.h" +#include "arrow/type_fwd.h" +#include "arrow/util/type_fwd.h" +#include "arrow/util/visibility.h" + +namespace arrow { +namespace json { + +class PromotionGraph; + +class ARROW_EXPORT ChunkedArrayBuilder { + public: + virtual ~ChunkedArrayBuilder() = default; + + /// Spawn a task that will try to convert and insert the given JSON block + virtual void Insert(int64_t block_index, + const std::shared_ptr& unconverted_field, + const std::shared_ptr& unconverted) = 0; + + /// Return the final chunked array. + /// Every chunk must be inserted before this is called! + virtual Status Finish(std::shared_ptr* out) = 0; + + /// Finish current task group and substitute a new one + virtual Status ReplaceTaskGroup( + const std::shared_ptr& task_group) = 0; + + protected: + explicit ChunkedArrayBuilder( + const std::shared_ptr& task_group) + : task_group_(task_group) {} + + std::shared_ptr task_group_; +}; + +/// create a chunked builder +/// +/// if unexpected fields and promotion need to be handled, promotion_graph must be +/// non-null +ARROW_EXPORT Status MakeChunkedArrayBuilder( + const std::shared_ptr& task_group, MemoryPool* pool, + const PromotionGraph* promotion_graph, const std::shared_ptr& type, + std::shared_ptr* out); + +} // namespace json +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/chunker.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/chunker.h new file mode 100644 index 0000000000000000000000000000000000000000..f43d01c14b78ea415184779f91fe4ffc33b6e047 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/chunker.h @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "arrow/util/delimiting.h" +#include "arrow/util/macros.h" +#include "arrow/util/visibility.h" + +namespace arrow { +namespace json { + +struct ParseOptions; + +ARROW_EXPORT +std::unique_ptr MakeChunker(const ParseOptions& options); + +} // namespace json +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/converter.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/converter.h new file mode 100644 index 0000000000000000000000000000000000000000..11f7480bd73b371e3119f3d509dfd4192910786e --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/converter.h @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "arrow/status.h" +#include "arrow/util/macros.h" +#include "arrow/util/visibility.h" + +namespace arrow { + +class Array; +class DataType; +class Field; +class MemoryPool; + +namespace json { + +/// \brief interface for conversion of Arrays +/// +/// Converters are not required to be correct for arbitrary input- only +/// for unconverted arrays emitted by a corresponding parser. +class ARROW_EXPORT Converter { + public: + virtual ~Converter() = default; + + /// convert an array + /// on failure, this converter may be promoted to another converter which + /// *can* convert the given input. + virtual Status Convert(const std::shared_ptr& in, + std::shared_ptr* out) = 0; + + std::shared_ptr out_type() const { return out_type_; } + + MemoryPool* pool() { return pool_; } + + protected: + ARROW_DISALLOW_COPY_AND_ASSIGN(Converter); + + Converter(MemoryPool* pool, const std::shared_ptr& out_type) + : pool_(pool), out_type_(out_type) {} + + MemoryPool* pool_; + std::shared_ptr out_type_; +}; + +/// \brief produce a single converter to the specified out_type +ARROW_EXPORT Status MakeConverter(const std::shared_ptr& out_type, + MemoryPool* pool, std::shared_ptr* out); + +class ARROW_EXPORT PromotionGraph { + public: + virtual ~PromotionGraph() = default; + + /// \brief produce a valid field which will be inferred as null + virtual std::shared_ptr Null(const std::string& name) const = 0; + + /// \brief given an unexpected field encountered during parsing, return a type to which + /// it may be convertible (may return null if none is available) + virtual std::shared_ptr Infer( + const std::shared_ptr& unexpected_field) const = 0; + + /// \brief given a type to which conversion failed, return a promoted type to which + /// conversion may succeed (may return null if none is available) + virtual std::shared_ptr Promote( + const std::shared_ptr& failed, + const std::shared_ptr& unexpected_field) const = 0; + + protected: + ARROW_DISALLOW_COPY_AND_ASSIGN(PromotionGraph); + PromotionGraph() = default; +}; + +ARROW_EXPORT const PromotionGraph* GetPromotionGraph(); + +} // namespace json +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/from_string.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/from_string.h new file mode 100644 index 0000000000000000000000000000000000000000..667b628b16b1beafabc44d52a17726464b8153c2 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/from_string.h @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Implement a simple JSON representation format for arrays + +#pragma once + +#include +#include +#include + +#include "arrow/status.h" +#include "arrow/type_fwd.h" +#include "arrow/util/visibility.h" + +namespace arrow { + +class Array; +class DataType; + +namespace json { + +/// \defgroup array-from-json-string FromJSONString Helpers +/// +/// These helpers are intended to be used in examples, tests, or for quick +/// prototyping and are not intended to be used where performance matters. +/// +/// See the User Guide for +/// more information. +/// +/// @{ + +/// \brief Create an Array from a JSON string +/// +/// \code {.cpp} +/// Result> maybe_array = +/// ArrayFromJSONString(int64(), "[2, 3, null, 7, 11]"); +/// \endcode +ARROW_EXPORT +Result> ArrayFromJSONString(const std::shared_ptr&, + const std::string& json); + +/// \copydoc ArrayFromJSONString(const std::shared_ptr&, const std::string&) +ARROW_EXPORT +Result> ArrayFromJSONString(const std::shared_ptr&, + std::string_view json); + +/// \copydoc ArrayFromJSONString(const std::shared_ptr&, const std::string&) +ARROW_EXPORT +Result> ArrayFromJSONString(const std::shared_ptr&, + const char* json); + +/// \brief Create a ChunkedArray from a JSON string +/// +/// \code {.cpp} +/// Result> maybe_chunked_array = +/// ChunkedArrayFromJSONString(int64(), {R"([5, 10])", R"([null])", R"([16])"}); +/// \endcode +ARROW_EXPORT +Result> ChunkedArrayFromJSONString( + const std::shared_ptr& type, const std::vector& json_strings); + +/// \brief Create a DictionaryArray from a JSON string +/// +/// \code {.cpp} +/// Result> maybe_dict_array = +/// DictArrayFromJSONString(dictionary(int32(), utf8()), "[0, 1, 0, 2, 0, 3]", +/// R"(["k1", "k2", "k3", "k4"])"); +/// \endcode +ARROW_EXPORT +Result> DictArrayFromJSONString(const std::shared_ptr&, + std::string_view indices_json, + std::string_view dictionary_json); + +/// \brief Create a Scalar from a JSON string +/// \code {.cpp} +/// Result> maybe_scalar = +/// ScalarFromJSONString(float64(), "42", &scalar); +/// \endcode +ARROW_EXPORT +Result> ScalarFromJSONString(const std::shared_ptr&, + std::string_view json); + +/// \brief Create a DictionaryScalar from a JSON string +/// \code {.cpp} +/// Result> maybe_dict_scalar = +/// DictScalarFromJSONString(dictionary(int32(), utf8()), "3", R"(["k1", "k2", "k3", +/// "k4"])", &scalar); +/// \endcode +ARROW_EXPORT +Result> DictScalarFromJSONString( + const std::shared_ptr&, std::string_view index_json, + std::string_view dictionary_json); + +/// @} + +} // namespace json +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/object_parser.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/object_parser.h new file mode 100644 index 0000000000000000000000000000000000000000..c3c23ad31d9d316b90f58e8605f2e532d7b46b2b --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/object_parser.h @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "arrow/result.h" +#include "arrow/util/visibility.h" + +namespace arrow { +namespace json { +namespace internal { + +/// This class is a helper to parse a json object from a string. +/// It uses rapidjson::Document in implementation. +class ARROW_EXPORT ObjectParser { + public: + ObjectParser(); + ~ObjectParser(); + + Status Parse(std::string_view json); + + Result GetString(const char* key) const; + + Result GetBool(const char* key) const; + + // Get all members of the object as a map from string keys to string values + Result> GetStringMap() const; + + private: + class Impl; + std::unique_ptr impl_; +}; + +} // namespace internal +} // namespace json +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/object_writer.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/object_writer.h new file mode 100644 index 0000000000000000000000000000000000000000..e68181deec58370fb18239f5296fb00e9ced7d59 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/object_writer.h @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "arrow/util/visibility.h" + +namespace arrow { +namespace json { +namespace internal { + +/// This class is a helper to serialize a json object to a string. +/// It uses rapidjson in implementation. +class ARROW_EXPORT ObjectWriter { + public: + ObjectWriter(); + ~ObjectWriter(); + + void SetString(std::string_view key, std::string_view value); + void SetBool(std::string_view key, bool value); + + std::string Serialize(); + + private: + class Impl; + std::unique_ptr impl_; +}; + +} // namespace internal +} // namespace json +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/options.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/options.h new file mode 100644 index 0000000000000000000000000000000000000000..c8b63e698c6e36ae3bb4274441fd2ddbd4097140 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/options.h @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "arrow/json/type_fwd.h" +#include "arrow/util/visibility.h" + +namespace arrow { + +class DataType; +class Schema; + +namespace json { + +enum class UnexpectedFieldBehavior : char { + /// Unexpected JSON fields are ignored + Ignore, + /// Unexpected JSON fields error out + Error, + /// Unexpected JSON fields are type-inferred and included in the output + InferType +}; + +struct ARROW_EXPORT ParseOptions { + // Parsing options + + /// Optional explicit schema (disables type inference on those fields) + std::shared_ptr explicit_schema; + + /// Whether objects may be printed across multiple lines (for example pretty-printed) + /// + /// If true, parsing may be slower. + bool newlines_in_values = false; + + /// How JSON fields outside of explicit_schema (if given) are treated + UnexpectedFieldBehavior unexpected_field_behavior = UnexpectedFieldBehavior::InferType; + + /// Create parsing options with default values + static ParseOptions Defaults(); +}; + +struct ARROW_EXPORT ReadOptions { + // Reader options + + /// Whether to use the global CPU thread pool + bool use_threads = true; + /// Block size we request from the IO layer; also determines the size of + /// chunks when use_threads is true + int32_t block_size = 1 << 20; // 1 MB + + /// Create read options with default values + static ReadOptions Defaults(); +}; + +} // namespace json +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/parser.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/parser.h new file mode 100644 index 0000000000000000000000000000000000000000..d7b42f2dbacdeb09ee0471d8e2a8a34e03b21143 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/parser.h @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "arrow/json/options.h" +#include "arrow/status.h" +#include "arrow/util/key_value_metadata.h" +#include "arrow/util/macros.h" +#include "arrow/util/visibility.h" + +namespace arrow { + +class Array; +class Buffer; +class MemoryPool; +class KeyValueMetadata; +class ResizableBuffer; + +namespace json { + +struct Kind { + enum type : uint8_t { + kNull, + kBoolean, + kNumber, + kString, + kArray, + kObject, + kNumberOrString + }; + + static const std::string& Name(Kind::type); + + static const std::shared_ptr& Tag(Kind::type); + + static Kind::type FromTag(const std::shared_ptr& tag); + + static Status ForType(const DataType& type, Kind::type* kind); +}; + +/// \class BlockParser +/// \brief A reusable block-based parser for JSON data +/// +/// The parser takes a block of newline delimited JSON data and extracts Arrays +/// of unconverted strings which can be fed to a Converter to obtain a usable Array. +/// +/// Note that in addition to parse errors (such as malformed JSON) some conversion +/// errors are caught at parse time: +/// - A null value in non-nullable column +/// - Change in the JSON kind of a column. For example, if an explicit schema is provided +/// which stipulates that field "a" is integral, a row of {"a": "not a number"} will +/// result in an error. This also applies to fields outside an explicit schema. +class ARROW_EXPORT BlockParser { + public: + virtual ~BlockParser() = default; + + /// \brief Reserve storage for scalars parsed from a block of json + virtual Status ReserveScalarStorage(int64_t nbytes) = 0; + + /// \brief Parse a block of data + virtual Status Parse(const std::shared_ptr& json) = 0; + + /// \brief Extract parsed data + virtual Status Finish(std::shared_ptr* parsed) = 0; + + /// \brief Return the number of parsed rows + int32_t num_rows() const { return num_rows_; } + + /// \brief Construct a BlockParser + /// + /// \param[in] pool MemoryPool to use when constructing parsed array + /// \param[in] options ParseOptions to use when parsing JSON + /// \param[out] out constructed BlockParser + static Status Make(MemoryPool* pool, const ParseOptions& options, + std::unique_ptr* out); + + static Status Make(const ParseOptions& options, std::unique_ptr* out); + + protected: + ARROW_DISALLOW_COPY_AND_ASSIGN(BlockParser); + + explicit BlockParser(MemoryPool* pool) : pool_(pool) {} + + MemoryPool* pool_; + int32_t num_rows_ = 0; +}; + +} // namespace json +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/rapidjson_defs.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/rapidjson_defs.h new file mode 100644 index 0000000000000000000000000000000000000000..e431f4c90bc8f95912468b9c271bde41bd966a4c --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/rapidjson_defs.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Include this file before including any RapidJSON headers. + +#pragma once + +#define RAPIDJSON_HAS_STDSTRING 1 +#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 +#define RAPIDJSON_HAS_CXX11_RANGE_FOR 1 + +// rapidjson will be defined in namespace arrow::rapidjson +#define RAPIDJSON_NAMESPACE arrow::rapidjson +#define RAPIDJSON_NAMESPACE_BEGIN \ + namespace arrow { \ + namespace rapidjson { +#define RAPIDJSON_NAMESPACE_END \ + } \ + } + +// enable SIMD whitespace skipping, if available +#if defined(ARROW_HAVE_SSE4_2) +# define RAPIDJSON_SSE2 1 +# define RAPIDJSON_SSE42 1 +#endif + +#if defined(ARROW_HAVE_NEON) +# define RAPIDJSON_NEON 1 +#endif diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/reader.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/reader.h new file mode 100644 index 0000000000000000000000000000000000000000..9644a8d8dd31123bfa8f2468296aa646f96e5875 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/reader.h @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "arrow/io/type_fwd.h" +#include "arrow/json/options.h" +#include "arrow/record_batch.h" +#include "arrow/result.h" +#include "arrow/status.h" +#include "arrow/util/macros.h" +#include "arrow/util/type_fwd.h" +#include "arrow/util/visibility.h" + +namespace arrow { +namespace json { + +/// A class that reads an entire JSON file into a Arrow Table +/// +/// The file is expected to consist of individual line-separated JSON objects +class ARROW_EXPORT TableReader { + public: + virtual ~TableReader() = default; + + /// Read the entire JSON file and convert it to a Arrow Table + virtual Result> Read() = 0; + + /// Create a TableReader instance + static Result> Make(MemoryPool* pool, + std::shared_ptr input, + const ReadOptions&, + const ParseOptions&); +}; + +ARROW_EXPORT Result> ParseOne(ParseOptions options, + std::shared_ptr json); + +/// \brief A class that reads a JSON file incrementally +/// +/// JSON data is read from a stream in fixed-size blocks (configurable with +/// `ReadOptions::block_size`). Each block is converted to a `RecordBatch`. Yielded +/// batches have a consistent schema but may differ in row count. +/// +/// The supplied `ParseOptions` are used to determine a schema, based either on a +/// provided explicit schema or inferred from the first non-empty block. +/// Afterwards, the target schema is frozen. If `UnexpectedFieldBehavior::InferType` is +/// specified, unexpected fields will only be inferred for the first block. Afterwards +/// they'll be treated as errors. +/// +/// If `ReadOptions::use_threads` is `true`, each block's parsing/decoding task will be +/// parallelized on the given `cpu_executor` (with readahead corresponding to the +/// executor's capacity). If an executor isn't provided, the global thread pool will be +/// used. +/// +/// If `ReadOptions::use_threads` is `false`, computations will be run on the calling +/// thread and `cpu_executor` will be ignored. +class ARROW_EXPORT StreamingReader : public RecordBatchReader { + public: + virtual ~StreamingReader() = default; + + /// \brief Read the next `RecordBatch` asynchronously + /// This function is async-reentrant (but not synchronously reentrant). However, if + /// threading is disabled, this will block until completion. + virtual Future> ReadNextAsync() = 0; + + /// Get the number of bytes which have been successfully converted to record batches + /// and consumed + [[nodiscard]] virtual int64_t bytes_processed() const = 0; + + /// \brief Create a `StreamingReader` from an `InputStream` + /// Blocks until the initial batch is loaded + /// + /// \param[in] stream JSON source stream + /// \param[in] read_options Options for reading + /// \param[in] parse_options Options for chunking, parsing, and conversion + /// \param[in] io_context Context for IO operations (optional) + /// \param[in] cpu_executor Executor for computation tasks (optional) + /// \return The initialized reader + static Result> Make( + std::shared_ptr stream, const ReadOptions& read_options, + const ParseOptions& parse_options, + const io::IOContext& io_context = io::default_io_context(), + ::arrow::internal::Executor* cpu_executor = NULLPTR); + + /// \brief Create a `StreamingReader` from an `InputStream` asynchronously + /// Returned future completes after loading the first batch + /// + /// \param[in] stream JSON source stream + /// \param[in] read_options Options for reading + /// \param[in] parse_options Options for chunking, parsing, and conversion + /// \param[in] io_context Context for IO operations (optional) + /// \param[in] cpu_executor Executor for computation tasks (optional) + /// \return Future for the initialized reader + static Future> MakeAsync( + std::shared_ptr stream, const ReadOptions& read_options, + const ParseOptions& parse_options, + const io::IOContext& io_context = io::default_io_context(), + ::arrow::internal::Executor* cpu_executor = NULLPTR); +}; + +} // namespace json +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/test_common.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/test_common.h new file mode 100644 index 0000000000000000000000000000000000000000..f36cc4ea3ac4f33d32215ce51faa959aa889bc09 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/test_common.h @@ -0,0 +1,330 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/array.h" +#include "arrow/array/builder_binary.h" +#include "arrow/io/memory.h" +#include "arrow/json/converter.h" +#include "arrow/json/options.h" +#include "arrow/json/parser.h" +#include "arrow/json/rapidjson_defs.h" +#include "arrow/testing/gtest_util.h" +#include "arrow/testing/random.h" +#include "arrow/type.h" +#include "arrow/util/checked_cast.h" +#include "arrow/visit_type_inline.h" + +#include "rapidjson/document.h" +#include "rapidjson/prettywriter.h" +#include "rapidjson/reader.h" +#include "rapidjson/writer.h" + +namespace arrow { + +using internal::checked_cast; + +namespace json { + +namespace rj = arrow::rapidjson; + +using rj::StringBuffer; +using std::string_view; +using Writer = rj::Writer; + +struct GenerateOptions { + // Probability of a field being written + double field_probability = 1.0; + // Probability of a value being null + double null_probability = 0.2; + // Whether to randomize the order of written fields + bool randomize_field_order = false; + + static constexpr GenerateOptions Defaults() { return GenerateOptions{}; } +}; + +inline static Status OK(bool ok) { return ok ? Status::OK() : Status::Invalid(""); } + +template +inline static Status Generate( + const std::shared_ptr& type, Engine& e, Writer* writer, + const GenerateOptions& options = GenerateOptions::Defaults()); + +template +inline static Status Generate( + const std::vector>& fields, Engine& e, Writer* writer, + const GenerateOptions& options = GenerateOptions::Defaults()); + +template +inline static Status Generate( + const std::shared_ptr& schm, Engine& e, Writer* writer, + const GenerateOptions& options = GenerateOptions::Defaults()) { + return Generate(schm->fields(), e, writer, options); +} + +template +struct GenerateImpl { + Status Visit(const NullType&) { return OK(writer.Null()); } + + Status Visit(const BooleanType&) { + return OK(writer.Bool(std::uniform_int_distribution{}(e) & 1)); + } + + template + enable_if_physical_unsigned_integer Visit(const T&) { + auto val = std::uniform_int_distribution<>{}(e); + return OK(writer.Uint64(static_cast(val))); + } + + template + enable_if_physical_signed_integer Visit(const T&) { + auto val = std::uniform_int_distribution<>{}(e); + return OK(writer.Int64(static_cast(val))); + } + + template + enable_if_physical_floating_point Visit(const T&) { + auto val = std::normal_distribution{0, 1 << 10}(e); + return OK(writer.Double(val)); + } + + Status GenerateUtf8(const DataType&) { + auto num_codepoints = std::poisson_distribution<>{4}(e); + auto seed = std::uniform_int_distribution{}(e); + std::string s = RandomUtf8String(seed, num_codepoints); + return OK(writer.String(s)); + } + + template + enable_if_base_binary Visit(const T& t) { + return GenerateUtf8(t); + } + + Status Visit(const BinaryViewType& t) { return GenerateUtf8(t); } + + template + enable_if_list_like Visit(const T& t) { + auto size = std::poisson_distribution<>{4}(e); + writer.StartArray(); + for (int i = 0; i < size; ++i) { + RETURN_NOT_OK(Generate(t.value_type(), e, &writer, options)); + } + return OK(writer.EndArray(size)); + } + + Status Visit(const ListViewType& t) { return NotImplemented(t); } + + Status Visit(const LargeListViewType& t) { return NotImplemented(t); } + + Status Visit(const StructType& t) { return Generate(t.fields(), e, &writer, options); } + + Status Visit(const DayTimeIntervalType& t) { return NotImplemented(t); } + + Status Visit(const MonthDayNanoIntervalType& t) { return NotImplemented(t); } + + Status Visit(const DictionaryType& t) { return NotImplemented(t); } + + Status Visit(const ExtensionType& t) { return NotImplemented(t); } + + Status Visit(const Decimal128Type& t) { return NotImplemented(t); } + + Status Visit(const FixedSizeBinaryType& t) { return NotImplemented(t); } + + Status Visit(const UnionType& t) { return NotImplemented(t); } + + Status Visit(const RunEndEncodedType& t) { return NotImplemented(t); } + + Status NotImplemented(const DataType& t) { + return Status::NotImplemented("random generation of arrays of type ", t); + } + + Engine& e; + rj::Writer& writer; + const GenerateOptions& options; +}; + +template +inline static Status Generate(const std::shared_ptr& type, Engine& e, + Writer* writer, const GenerateOptions& options) { + if (std::bernoulli_distribution(options.null_probability)(e)) { + writer->Null(); + return Status::OK(); + } + GenerateImpl visitor = {e, *writer, options}; + return VisitTypeInline(*type, &visitor); +} + +template +inline static Status Generate(const std::vector>& fields, + Engine& e, Writer* writer, const GenerateOptions& options) { + RETURN_NOT_OK(OK(writer->StartObject())); + + int num_fields = 0; + auto write_field = [&](const Field& f) { + ++num_fields; + writer->Key(f.name().c_str()); + return Generate(f.type(), e, writer, options); + }; + + std::bernoulli_distribution bool_dist(options.field_probability); + if (options.randomize_field_order) { + std::vector indices; + indices.reserve(static_cast(fields.size() * options.field_probability)); + for (size_t i = 0; i < fields.size(); ++i) { + if (bool_dist(e)) { + indices.push_back(i); + } + } + std::shuffle(indices.begin(), indices.end(), e); + for (auto i : indices) { + RETURN_NOT_OK(write_field(*fields[i])); + } + } else { + for (const auto& f : fields) { + if (bool_dist(e)) { + RETURN_NOT_OK(write_field(*f)); + } + } + } + + return OK(writer->EndObject(num_fields)); +} + +inline static Status MakeStream(string_view src_str, + std::shared_ptr* out) { + auto src = std::make_shared(src_str); + *out = std::make_shared(src); + return Status::OK(); +} + +// scalar values (numbers and strings) are parsed into a +// dictionary. This can be decoded for ease of comparison +inline static Status DecodeStringDictionary(const DictionaryArray& dict_array, + std::shared_ptr* decoded) { + const StringArray& dict = checked_cast(*dict_array.dictionary()); + const Int32Array& indices = checked_cast(*dict_array.indices()); + StringBuilder builder; + RETURN_NOT_OK(builder.Resize(indices.length())); + for (int64_t i = 0; i < indices.length(); ++i) { + if (indices.IsNull(i)) { + builder.UnsafeAppendNull(); + continue; + } + auto value = dict.GetView(indices.GetView(i)); + RETURN_NOT_OK(builder.ReserveData(value.size())); + builder.UnsafeAppend(value); + } + return builder.Finish(decoded); +} + +inline static Status ParseFromString(ParseOptions options, string_view src_str, + std::shared_ptr* parsed) { + auto src = std::make_shared(src_str); + std::unique_ptr parser; + RETURN_NOT_OK(BlockParser::Make(options, &parser)); + RETURN_NOT_OK(parser->Parse(src)); + return parser->Finish(parsed); +} + +inline static Status ParseFromString(ParseOptions options, string_view src_str, + std::shared_ptr* parsed) { + std::shared_ptr parsed_non_struct; + RETURN_NOT_OK(ParseFromString(options, src_str, &parsed_non_struct)); + *parsed = internal::checked_pointer_cast(parsed_non_struct); + return Status::OK(); +} + +static inline std::string PrettyPrint(string_view one_line) { + rj::Document document; + + // Must pass size to avoid ASAN issues. + document.Parse(one_line.data(), one_line.size()); + rj::StringBuffer sb; + rj::PrettyWriter writer(sb); + document.Accept(writer); + return sb.GetString(); +} + +template +std::string RowsOfOneColumn(std::string_view name, std::initializer_list values, + decltype(std::to_string(*values.begin()))* = nullptr) { + std::stringstream ss; + for (auto value : values) { + ss << R"({")" << name << R"(":)" << std::to_string(value) << "}\n"; + } + return ss.str(); +} + +inline std::string RowsOfOneColumn(std::string_view name, + std::initializer_list values) { + std::stringstream ss; + for (auto value : values) { + ss << R"({")" << name << R"(":)" << value << "}\n"; + } + return ss.str(); +} + +inline static std::string scalars_only_src() { + return R"( + { "hello": 3.5, "world": false, "yo": "thing" } + { "hello": 3.25, "world": null } + { "hello": 3.125, "world": null, "yo": "\u5fcd" } + { "hello": 0.0, "world": true, "yo": null } + )"; +} + +inline static std::string nested_src() { + return R"( + { "hello": 3.5, "world": false, "yo": "thing", "arr": [1, 2, 3], "nuf": {} } + { "hello": 3.25, "world": null, "arr": [2], "nuf": null } + { "hello": 3.125, "world": null, "yo": "\u5fcd", "arr": [], "nuf": { "ps": 78 } } + { "hello": 0.0, "world": true, "yo": null, "arr": null, "nuf": { "ps": 90 } } + )"; +} + +inline static std::string null_src() { + return R"( + { "plain": null, "list1": [], "list2": [], "struct": { "plain": null } } + { "plain": null, "list1": [], "list2": [null], "struct": {} } + )"; +} + +inline static std::string unquoted_decimal_src() { + return R"( + { "price": 30.04, "cost":30.001 } + { "price": 1.23, "cost":1.229 } + )"; +} + +inline static std::string mixed_decimal_src() { + return R"( + { "price": 30.04, "cost": 30.001 } + { "price": "1.23", "cost": "1.229" } + )"; +} + +} // namespace json +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/type_fwd.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/type_fwd.h new file mode 100644 index 0000000000000000000000000000000000000000..457bd557badde19c0ffca7a3fbcd5a91613524f8 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/json/type_fwd.h @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace arrow { +namespace json { + +class TableReader; +struct ReadOptions; +struct ParseOptions; + +} // namespace json +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/memory_pool.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/memory_pool.h new file mode 100644 index 0000000000000000000000000000000000000000..c8df00bd6ec648a5cdc96ea2f8001ce54edb0463 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/memory_pool.h @@ -0,0 +1,348 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "arrow/result.h" +#include "arrow/status.h" +#include "arrow/type_fwd.h" +#include "arrow/util/macros.h" +#include "arrow/util/visibility.h" + +namespace arrow { + +namespace internal { + +/////////////////////////////////////////////////////////////////////// +// Helper tracking memory statistics + +/// \brief Memory pool statistics +/// +/// 64-byte aligned so that all atomic values are on the same cache line. +class alignas(64) MemoryPoolStats { + private: + // All atomics are updated according to Acquire-Release ordering. + // https://en.cppreference.com/w/cpp/atomic/memory_order#Release-Acquire_ordering + // + // max_memory_, total_allocated_bytes_, and num_allocs_ only go up (they are + // monotonically increasing) which can allow some optimizations. + std::atomic max_memory_{0}; + std::atomic bytes_allocated_{0}; + std::atomic total_allocated_bytes_{0}; + std::atomic num_allocs_{0}; + + public: + int64_t max_memory() const { return max_memory_.load(std::memory_order_acquire); } + + int64_t bytes_allocated() const { + return bytes_allocated_.load(std::memory_order_acquire); + } + + int64_t total_bytes_allocated() const { + return total_allocated_bytes_.load(std::memory_order_acquire); + } + + int64_t num_allocations() const { return num_allocs_.load(std::memory_order_acquire); } + + inline void DidAllocateBytes(int64_t size) { + // Issue the load before everything else. max_memory_ is monotonically increasing, + // so we can use a relaxed load before the read-modify-write. + auto max_memory = max_memory_.load(std::memory_order_relaxed); + const auto old_bytes_allocated = + bytes_allocated_.fetch_add(size, std::memory_order_acq_rel); + // Issue store operations on values that we don't depend on to proceed + // with execution. When done, max_memory and old_bytes_allocated have + // a higher chance of being available on CPU registers. This also has the + // nice side-effect of putting 3 atomic stores close to each other in the + // instruction stream. + total_allocated_bytes_.fetch_add(size, std::memory_order_acq_rel); + num_allocs_.fetch_add(1, std::memory_order_acq_rel); + + // If other threads are updating max_memory_ concurrently we leave the loop without + // updating knowing that it already reached a value even higher than ours. + const auto allocated = old_bytes_allocated + size; + while (max_memory < allocated && !max_memory_.compare_exchange_weak( + /*expected=*/max_memory, /*desired=*/allocated, + std::memory_order_acq_rel)) { + } + } + + inline void DidReallocateBytes(int64_t old_size, int64_t new_size) { + if (new_size > old_size) { + DidAllocateBytes(new_size - old_size); + } else { + DidFreeBytes(old_size - new_size); + } + } + + inline void DidFreeBytes(int64_t size) { + bytes_allocated_.fetch_sub(size, std::memory_order_acq_rel); + } +}; + +} // namespace internal + +/// Base class for memory allocation on the CPU. +/// +/// Besides tracking the number of allocated bytes, the allocator also should +/// take care of the required 64-byte alignment. +class ARROW_EXPORT MemoryPool { + public: + virtual ~MemoryPool() = default; + + /// \brief EXPERIMENTAL. Create a new instance of the default MemoryPool + static std::unique_ptr CreateDefault(); + + /// Allocate a new memory region of at least size bytes. + /// + /// The allocated region shall be 64-byte aligned. + Status Allocate(int64_t size, uint8_t** out) { + return Allocate(size, kDefaultBufferAlignment, out); + } + + /// Allocate a new memory region of at least size bytes aligned to alignment. + virtual Status Allocate(int64_t size, int64_t alignment, uint8_t** out) = 0; + + /// Resize an already allocated memory section. + /// + /// As by default most default allocators on a platform don't support aligned + /// reallocation, this function can involve a copy of the underlying data. + virtual Status Reallocate(int64_t old_size, int64_t new_size, int64_t alignment, + uint8_t** ptr) = 0; + Status Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) { + return Reallocate(old_size, new_size, kDefaultBufferAlignment, ptr); + } + + /// Free an allocated region. + /// + /// @param buffer Pointer to the start of the allocated memory region + /// @param size Allocated size located at buffer. An allocator implementation + /// may use this for tracking the amount of allocated bytes as well as for + /// faster deallocation if supported by its backend. + /// @param alignment The alignment of the allocation. Defaults to 64 bytes. + virtual void Free(uint8_t* buffer, int64_t size, int64_t alignment) = 0; + void Free(uint8_t* buffer, int64_t size) { + Free(buffer, size, kDefaultBufferAlignment); + } + + /// Return unused memory to the OS + /// + /// Only applies to allocators that hold onto unused memory. This will be + /// best effort, a memory pool may not implement this feature or may be + /// unable to fulfill the request due to fragmentation. + virtual void ReleaseUnused() {} + + /// Print statistics + /// + /// Print allocation statistics on stderr. The output format is + /// implementation-specific. Not all memory pools implement this method. + virtual void PrintStats() {} + + /// The number of bytes that were allocated and not yet free'd through + /// this allocator. + virtual int64_t bytes_allocated() const = 0; + + /// Return peak memory allocation in this memory pool + /// + /// \return Maximum bytes allocated. If not known (or not implemented), + /// returns -1 + virtual int64_t max_memory() const; + + /// The number of bytes that were allocated. + virtual int64_t total_bytes_allocated() const = 0; + + /// The number of allocations or reallocations that were requested. + virtual int64_t num_allocations() const = 0; + + /// The name of the backend used by this MemoryPool (e.g. "system" or "jemalloc"). + virtual std::string backend_name() const = 0; + + protected: + MemoryPool() = default; +}; + +class ARROW_EXPORT LoggingMemoryPool : public MemoryPool { + public: + explicit LoggingMemoryPool(MemoryPool* pool); + ~LoggingMemoryPool() override = default; + + using MemoryPool::Allocate; + using MemoryPool::Free; + using MemoryPool::Reallocate; + + Status Allocate(int64_t size, int64_t alignment, uint8_t** out) override; + Status Reallocate(int64_t old_size, int64_t new_size, int64_t alignment, + uint8_t** ptr) override; + void Free(uint8_t* buffer, int64_t size, int64_t alignment) override; + void ReleaseUnused() override; + void PrintStats() override; + + int64_t bytes_allocated() const override; + + int64_t max_memory() const override; + + int64_t total_bytes_allocated() const override; + + int64_t num_allocations() const override; + + std::string backend_name() const override; + + private: + MemoryPool* pool_; +}; + +/// Derived class for memory allocation. +/// +/// Tracks the number of bytes and maximum memory allocated through its direct +/// calls. Actual allocation is delegated to MemoryPool class. +class ARROW_EXPORT ProxyMemoryPool : public MemoryPool { + public: + explicit ProxyMemoryPool(MemoryPool* pool); + ~ProxyMemoryPool() override; + + using MemoryPool::Allocate; + using MemoryPool::Free; + using MemoryPool::Reallocate; + + Status Allocate(int64_t size, int64_t alignment, uint8_t** out) override; + Status Reallocate(int64_t old_size, int64_t new_size, int64_t alignment, + uint8_t** ptr) override; + void Free(uint8_t* buffer, int64_t size, int64_t alignment) override; + void ReleaseUnused() override; + void PrintStats() override; + + int64_t bytes_allocated() const override; + + int64_t max_memory() const override; + + int64_t total_bytes_allocated() const override; + + int64_t num_allocations() const override; + + std::string backend_name() const override; + + private: + class ProxyMemoryPoolImpl; + std::unique_ptr impl_; +}; + +/// EXPERIMENTAL MemoryPool wrapper with an upper limit +/// +/// Checking for limits is not done in a fully thread-safe way, therefore +/// multi-threaded allocations might be able to go successfully above the +/// configured limit. +class ARROW_EXPORT CappedMemoryPool : public MemoryPool { + public: + CappedMemoryPool(MemoryPool* wrapped_pool, int64_t bytes_allocated_limit) + : wrapped_(wrapped_pool), bytes_allocated_limit_(bytes_allocated_limit) {} + + using MemoryPool::Allocate; + using MemoryPool::Reallocate; + + Status Allocate(int64_t size, int64_t alignment, uint8_t** out) override; + Status Reallocate(int64_t old_size, int64_t new_size, int64_t alignment, + uint8_t** ptr) override; + void Free(uint8_t* buffer, int64_t size, int64_t alignment) override; + + void ReleaseUnused() override { wrapped_->ReleaseUnused(); } + + void PrintStats() override { wrapped_->PrintStats(); } + + int64_t bytes_allocated() const override { return wrapped_->bytes_allocated(); } + + int64_t max_memory() const override { return wrapped_->max_memory(); } + + int64_t total_bytes_allocated() const override { + return wrapped_->total_bytes_allocated(); + } + + int64_t num_allocations() const override { return wrapped_->num_allocations(); } + + std::string backend_name() const override { return wrapped_->backend_name(); } + + private: + Status OutOfMemory(int64_t current_allocated, int64_t requested) const; + + MemoryPool* wrapped_; + const int64_t bytes_allocated_limit_; +}; + +/// \brief Return a process-wide memory pool based on the system allocator. +ARROW_EXPORT MemoryPool* system_memory_pool(); + +/// \brief Return a process-wide memory pool based on jemalloc. +/// +/// May return NotImplemented if jemalloc is not available. +ARROW_EXPORT Status jemalloc_memory_pool(MemoryPool** out); + +/// \brief Set jemalloc memory page purging behavior for future-created arenas +/// to the indicated number of milliseconds. See dirty_decay_ms and +/// muzzy_decay_ms options in jemalloc for a description of what these do. The +/// default is configured to 1000 (1 second) which releases memory more +/// aggressively to the operating system than the jemalloc default of 10 +/// seconds. If you set the value to 0, dirty / muzzy pages will be released +/// immediately rather than with a time decay, but this may reduce application +/// performance. +ARROW_EXPORT +Status jemalloc_set_decay_ms(int ms); + +/// \brief Get basic statistics from jemalloc's mallctl. +/// See the MALLCTL NAMESPACE section in jemalloc project documentation for +/// available stats. +ARROW_EXPORT +Result jemalloc_get_stat(const char* name); + +/// \brief Reset the counter for peak bytes allocated in the calling thread to zero. +/// This affects subsequent calls to thread.peak.read, but not the values returned by +/// thread.allocated or thread.deallocated. +ARROW_EXPORT +Status jemalloc_peak_reset(); + +/// \brief Print summary statistics in human-readable form to stderr. +/// See malloc_stats_print documentation in jemalloc project documentation for +/// available opt flags. +ARROW_EXPORT +Status jemalloc_stats_print(const char* opts = ""); + +/// \brief Print summary statistics in human-readable form using a callback +/// See malloc_stats_print documentation in jemalloc project documentation for +/// available opt flags. +ARROW_EXPORT +Status jemalloc_stats_print(std::function write_cb, + const char* opts = ""); + +/// \brief Get summary statistics in human-readable form. +/// See malloc_stats_print documentation in jemalloc project documentation for +/// available opt flags. +ARROW_EXPORT +Result jemalloc_stats_string(const char* opts = ""); + +/// \brief Return a process-wide memory pool based on mimalloc. +/// +/// May return NotImplemented if mimalloc is not available. +ARROW_EXPORT Status mimalloc_memory_pool(MemoryPool** out); + +/// \brief Return the names of the backends supported by this Arrow build. +ARROW_EXPORT std::vector SupportedMemoryBackendNames(); + +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/memory_pool_test.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/memory_pool_test.h new file mode 100644 index 0000000000000000000000000000000000000000..d1dede49a077d384b119542cc5867b7b0dc7111b --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/memory_pool_test.h @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include + +#include "arrow/memory_pool.h" +#include "arrow/status.h" +#include "arrow/testing/gtest_util.h" + +namespace arrow { + +class TestMemoryPoolBase : public ::testing::Test { + public: + virtual ::arrow::MemoryPool* memory_pool() = 0; + + void TestMemoryTracking() { + auto pool = memory_pool(); + + uint8_t* data; + const auto old_bytes_allocated = pool->bytes_allocated(); + ASSERT_OK(pool->Allocate(100, &data)); + EXPECT_EQ(static_cast(0), reinterpret_cast(data) % 64); + ASSERT_EQ(old_bytes_allocated + 100, pool->bytes_allocated()); + + uint8_t* data2; + ASSERT_OK(pool->Allocate(27, &data2)); + EXPECT_EQ(static_cast(0), reinterpret_cast(data2) % 64); + ASSERT_EQ(old_bytes_allocated + 127, pool->bytes_allocated()); + + pool->Free(data, 100); + ASSERT_EQ(old_bytes_allocated + 27, pool->bytes_allocated()); + pool->Free(data2, 27); + ASSERT_EQ(old_bytes_allocated, pool->bytes_allocated()); + } + + void TestOOM() { + auto pool = memory_pool(); + + uint8_t* data; + int64_t max_alloc = std::min(std::numeric_limits::max(), + std::numeric_limits::max()); + // subtract 63 to prevent overflow after the size is aligned + for (int64_t to_alloc : {max_alloc, max_alloc - 63, max_alloc - 127}) { + ASSERT_RAISES(OutOfMemory, pool->Allocate(to_alloc, &data)); + } + } + + void TestReallocate() { + auto pool = memory_pool(); + + uint8_t* data; + ASSERT_OK(pool->Allocate(10, &data)); + ASSERT_EQ(10, pool->bytes_allocated()); + data[0] = 35; + data[9] = 12; + + // Expand + ASSERT_OK(pool->Reallocate(10, 20, &data)); + ASSERT_EQ(data[9], 12); + ASSERT_EQ(20, pool->bytes_allocated()); + + // Shrink + ASSERT_OK(pool->Reallocate(20, 5, &data)); + ASSERT_EQ(data[0], 35); + ASSERT_EQ(5, pool->bytes_allocated()); + + // Free + pool->Free(data, 5); + ASSERT_EQ(0, pool->bytes_allocated()); + } + + void TestAlignment() { + auto pool = memory_pool(); + { + uint8_t* data64; + ASSERT_OK(pool->Allocate(10, &data64)); + ASSERT_EQ(reinterpret_cast(data64) % kDefaultBufferAlignment, 0); + pool->Free(data64, 10); + } + + { + uint8_t* data512; + ASSERT_OK(pool->Allocate(10, 512, &data512)); + ASSERT_EQ(reinterpret_cast(data512) % 512, 0); + pool->Free(data512, 10, 512); + } + } +}; + +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/pretty_print.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/pretty_print.h new file mode 100644 index 0000000000000000000000000000000000000000..e2041feb56ddf72ccdbc9fd5f9667c3558e1191f --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/pretty_print.h @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "arrow/util/visibility.h" + +namespace arrow { + +class Array; +class ChunkedArray; +class RecordBatch; +class Schema; +class Status; +class Table; + +/// \class PrettyPrintDelimiters +/// \brief Options for controlling which delimiters to use when printing +/// an Array or ChunkedArray. +struct ARROW_EXPORT PrettyPrintDelimiters { + /// Delimiter to use when opening an Array or ChunkedArray (e.g. "[") + std::string open = "["; + + /// Delimiter to use when closing an Array or ChunkedArray (e.g. "]") + std::string close = "]"; + + /// Delimiter for separating individual elements of an Array (e.g. ","), + /// or individual chunks of a ChunkedArray + std::string element = ","; + + /// Create a PrettyPrintDelimiters instance with default values + static PrettyPrintDelimiters Defaults() { return PrettyPrintDelimiters(); } +}; + +/// \class PrettyPrintOptions +/// \brief Options for controlling how various Arrow types should be printed. +struct ARROW_EXPORT PrettyPrintOptions { + PrettyPrintOptions() = default; + + PrettyPrintOptions(int indent, // NOLINT runtime/explicit + int window = 10, int indent_size = 2, std::string null_rep = "null", + bool skip_new_lines = false, bool truncate_metadata = true, + int container_window = 2, int element_size_limit = 100) + : indent(indent), + indent_size(indent_size), + window(window), + container_window(container_window), + null_rep(std::move(null_rep)), + skip_new_lines(skip_new_lines), + truncate_metadata(truncate_metadata), + element_size_limit(element_size_limit) {} + + /// Create a PrettyPrintOptions instance with default values + static PrettyPrintOptions Defaults() { return PrettyPrintOptions(); } + + /// Number of spaces to shift entire formatted object to the right + int indent = 0; + + /// Size of internal indents + int indent_size = 2; + + /// Maximum number of elements to show at the beginning and at the end. + int window = 10; + + /// Maximum number of elements to show at the beginning and at the end, for elements + /// that are containers (that is, list in ListArray and chunks in ChunkedArray) + int container_window = 2; + + /// String to use for representing a null value, defaults to "null" + std::string null_rep = "null"; + + /// Skip new lines between elements, defaults to false + bool skip_new_lines = false; + + /// Limit display of each KeyValueMetadata key/value pair to a single line at + /// 80 character width + bool truncate_metadata = true; + + /// If true, display field metadata when pretty-printing a Schema + bool show_field_metadata = true; + + /// If true, display schema metadata when pretty-printing a Schema + bool show_schema_metadata = true; + + /// Limit each element to specified number of characters, defaults to 100 + int element_size_limit = 100; + + /// Delimiters to use when printing an Array + PrettyPrintDelimiters array_delimiters = PrettyPrintDelimiters::Defaults(); + + /// Delimiters to use when printing a ChunkedArray + PrettyPrintDelimiters chunked_array_delimiters = PrettyPrintDelimiters::Defaults(); +}; + +/// \brief Print human-readable representation of RecordBatch +ARROW_EXPORT +Status PrettyPrint(const RecordBatch& batch, int indent, std::ostream* sink); + +ARROW_EXPORT +Status PrettyPrint(const RecordBatch& batch, const PrettyPrintOptions& options, + std::ostream* sink); + +/// \brief Print human-readable representation of Table +ARROW_EXPORT +Status PrettyPrint(const Table& table, const PrettyPrintOptions& options, + std::ostream* sink); + +/// \brief Print human-readable representation of Array +ARROW_EXPORT +Status PrettyPrint(const Array& arr, int indent, std::ostream* sink); + +/// \brief Print human-readable representation of Array +ARROW_EXPORT +Status PrettyPrint(const Array& arr, const PrettyPrintOptions& options, + std::ostream* sink); + +/// \brief Print human-readable representation of Array +ARROW_EXPORT +Status PrettyPrint(const Array& arr, const PrettyPrintOptions& options, + std::string* result); + +/// \brief Print human-readable representation of ChunkedArray +ARROW_EXPORT +Status PrettyPrint(const ChunkedArray& chunked_arr, const PrettyPrintOptions& options, + std::ostream* sink); + +/// \brief Print human-readable representation of ChunkedArray +ARROW_EXPORT +Status PrettyPrint(const ChunkedArray& chunked_arr, const PrettyPrintOptions& options, + std::string* result); + +ARROW_EXPORT +Status PrettyPrint(const Schema& schema, const PrettyPrintOptions& options, + std::ostream* sink); + +ARROW_EXPORT +Status PrettyPrint(const Schema& schema, const PrettyPrintOptions& options, + std::string* result); + +ARROW_EXPORT +Status DebugPrint(const Array& arr, int indent); + +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/python/api.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/python/api.h new file mode 100644 index 0000000000000000000000000000000000000000..2b4263943a2d701202ad14b249118902096eeba6 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/python/api.h @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "arrow/python/arrow_to_pandas.h" +#include "arrow/python/common.h" +#include "arrow/python/datetime.h" +#include "arrow/python/helpers.h" +#include "arrow/python/inference.h" +#include "arrow/python/io.h" +#include "arrow/python/numpy_convert.h" +#include "arrow/python/numpy_to_arrow.h" +#include "arrow/python/python_to_arrow.h" +#include "arrow/python/util.h" diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/python/arrow_to_pandas.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/python/arrow_to_pandas.h new file mode 100644 index 0000000000000000000000000000000000000000..23e9d0c77ec3019c02c354e79edd539fe393dc60 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/python/arrow_to_pandas.h @@ -0,0 +1,165 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Functions for converting between pandas's NumPy-based data representation +// and Arrow data structures + +#pragma once + +#include "arrow/python/platform.h" + +#include +#include +#include + +#include "arrow/memory_pool.h" +#include "arrow/python/visibility.h" + +namespace arrow { + +class Array; +class ChunkedArray; +class Column; +class DataType; +class MemoryPool; +class Status; +class Table; + +namespace py { + +enum class MapConversionType { + DEFAULT, // convert arrow maps to assoc lists (list of kev-value tuples) in Pandas + LOSSY, // report warnings when lossiness is encountered due to duplicate keys + STRICT_, // raise a Python exception when lossiness is encountered due to duplicate + // keys +}; + +struct PandasOptions { + bool HasCategoricalColumns() const { + return categorical_columns && !categorical_columns->empty(); + } + + bool IsCategoricalColumn(const std::string& name) const { + return categorical_columns && categorical_columns->count(name); + } + + bool HasExtensionColumns() const { + return extension_columns && !extension_columns->empty(); + } + + bool IsExtensionColumn(const std::string& name) const { + return extension_columns && extension_columns->count(name); + } + + /// arrow::MemoryPool to use for memory allocations + MemoryPool* pool = default_memory_pool(); + + /// If true, we will convert all string columns to categoricals + bool strings_to_categorical = false; + bool zero_copy_only = false; + bool integer_object_nulls = false; + bool date_as_object = false; + bool timestamp_as_object = false; + bool use_threads = false; + + /// Coerce all date and timestamp to datetime64[ns] + bool coerce_temporal_nanoseconds = false; + + /// Used to maintain backwards compatibility for + /// timezone bugs (see ARROW-9528). Should be removed + /// after Arrow 2.0 release. + bool ignore_timezone = false; + + /// \brief If true, do not create duplicate PyObject versions of equal + /// objects. This only applies to immutable objects like strings or datetime + /// objects + bool deduplicate_objects = false; + + /// \brief For certain data types, a cast is needed in order to store the + /// data in a pandas DataFrame or Series (e.g. timestamps are always stored + /// as nanoseconds in pandas). This option controls whether it is a safe + /// cast or not. + bool safe_cast = true; + + /// \brief If true, create one block per column rather than consolidated + /// blocks (1 per data type). Do zero-copy wrapping when there are no + /// nulls. pandas currently will consolidate the blocks on its own, causing + /// increased memory use, so keep this in mind if you are working on a + /// memory-constrained situation. + bool split_blocks = false; + + /// \brief If true, allow non-writable zero-copy views to be created for + /// single column blocks. This option is also used to provide zero copy for + /// Series data + bool allow_zero_copy_blocks = false; + + /// \brief If true, attempt to deallocate buffers in passed Arrow object if + /// it is the only remaining shared_ptr copy of it. See ARROW-3789 for + /// original context for this feature. Only currently implemented for Table + /// conversions + bool self_destruct = false; + + /// \brief The default behavior (DEFAULT), is to convert Arrow Map arrays to + /// Python association lists (list-of-tuples) in the same order as the Arrow + /// Map, as in [(key1, value1), (key2, value2), ...] + /// If LOSSY or STRICT, convert Arrow Map arrays to native Python dicts. + /// This can change the ordering of (key, value) pairs, and will deduplicate + /// multiple keys, resulting in a possible loss of data. + /// If 'lossy', this key deduplication results in a warning printed + /// when detected. If 'strict', this instead results in an exception + /// being raised when detected. + MapConversionType maps_as_pydicts = MapConversionType::DEFAULT; + + // Used internally for nested arrays. + bool decode_dictionaries = false; + + // Columns that should be casted to categorical + // + // This is wrapped in a shared_ptr because this struct is copied internally for + // each column or nested field (see GH-47861). + std::shared_ptr> categorical_columns; + + // Columns that should be passed through to be converted to + // ExtensionArray/Block + std::shared_ptr> extension_columns; + + // Used internally to decipher between to_numpy() and to_pandas() when + // the expected output differs + bool to_numpy = false; +}; + +ARROW_PYTHON_EXPORT +Status ConvertArrayToPandas(const PandasOptions& options, std::shared_ptr arr, + PyObject* py_ref, PyObject** out); + +ARROW_PYTHON_EXPORT +Status ConvertChunkedArrayToPandas(const PandasOptions& options, + std::shared_ptr col, PyObject* py_ref, + PyObject** out); + +// Convert a whole table as efficiently as possible to a pandas.DataFrame. +// +// The returned Python object is a list of tuples consisting of the exact 2D +// BlockManager structure of the pandas.DataFrame used as of pandas 0.19.x. +// +// tuple item: (indices: ndarray[int32], block: ndarray[TYPE, ndim=2]) +ARROW_PYTHON_EXPORT +Status ConvertTableToPandas(const PandasOptions& options, std::shared_ptr
table, + PyObject** out); + +} // namespace py +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/python/async.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/python/async.h new file mode 100644 index 0000000000000000000000000000000000000000..601fcc8fc6873944832ae1675b59cb0dbb0316a6 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/python/async.h @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "arrow/python/common.h" +#include "arrow/status.h" +#include "arrow/util/future.h" + +namespace arrow::py { + +/// \brief Bind a Python callback to an arrow::Future. +/// +/// If the Future finishes successfully, py_wrapper is called with its +/// result value and should return a PyObject*. If py_wrapper is successful, +/// py_cb is called with its return value. +/// +/// If either the Future or py_wrapper fails, py_cb is called with the +/// associated Python exception. +/// +/// \param future The future to bind to. +/// \param py_cb The Python callback function. Will be passed the result of +/// py_wrapper, or a Python exception if the future failed or one was +/// raised by py_wrapper. +/// \param py_wrapper A function (likely defined in Cython) to convert the C++ +/// result of the future to a Python object. +template +void BindFuture(Future future, PyObject* py_cb, PyWrapper py_wrapper) { + Py_INCREF(py_cb); + OwnedRefNoGIL cb_ref(py_cb); + + auto future_cb = [cb_ref = std::move(cb_ref), + py_wrapper = std::move(py_wrapper)](Result result) { + SafeCallIntoPythonVoid([&]() { + OwnedRef py_value_or_exc{WrapResult(std::move(result), std::move(py_wrapper))}; + Py_XDECREF( + PyObject_CallFunctionObjArgs(cb_ref.obj(), py_value_or_exc.obj(), NULLPTR)); + ARROW_WARN_NOT_OK(CheckPyError(), "Internal error in async call"); + }); + }; + future.AddCallback(std::move(future_cb)); +} + +} // namespace arrow::py diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/python/benchmark.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/python/benchmark.h new file mode 100644 index 0000000000000000000000000000000000000000..62a1f2d807bd911a8fc86edb2b691191b70bdc11 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/python/benchmark.h @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "arrow/python/platform.h" + +#include "arrow/python/visibility.h" + +namespace arrow { +namespace py { +namespace benchmark { + +// Micro-benchmark routines for use from ASV + +// Run PandasObjectIsNull() once over every object in *list* +ARROW_PYTHON_EXPORT +void Benchmark_PandasObjectIsNull(PyObject* list); + +} // namespace benchmark +} // namespace py +} // namespace arrow diff --git a/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/python/common.h b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/python/common.h new file mode 100644 index 0000000000000000000000000000000000000000..13df38ec1f91967ada18e1e28886fb0371a732f9 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/pyarrow/include/arrow/python/common.h @@ -0,0 +1,471 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "arrow/buffer.h" +#include "arrow/python/pyarrow.h" +#include "arrow/python/visibility.h" +#include "arrow/result.h" +#include "arrow/util/macros.h" + +namespace arrow { + +class MemoryPool; +template +class Result; + +namespace py { + +// Convert current Python error to a Status. The Python error state is cleared +// and can be restored with RestorePyError(). +ARROW_PYTHON_EXPORT Status ConvertPyError(StatusCode code = StatusCode::UnknownError); +// Query whether the given Status is a Python error (as wrapped by ConvertPyError()). +ARROW_PYTHON_EXPORT bool IsPyError(const Status& status); +// Restore a Python error wrapped in a Status. +ARROW_PYTHON_EXPORT void RestorePyError(const Status& status); + +// Catch a pending Python exception and return the corresponding Status. +// If no exception is pending, Status::OK() is returned. +inline Status CheckPyError(StatusCode code = StatusCode::UnknownError) { + if (ARROW_PREDICT_TRUE(!PyErr_Occurred())) { + return Status::OK(); + } else { + return ConvertPyError(code); + } +} + +#define RETURN_IF_PYERROR() ARROW_RETURN_NOT_OK(CheckPyError()) + +#define PY_RETURN_IF_ERROR(CODE) ARROW_RETURN_NOT_OK(CheckPyError(CODE)) + +// For Cython, as you can't define template C++ functions in Cython, only use them. +// This function can set a Python exception. It assumes that T has a (cheap) +// default constructor. +template +T GetResultValue(Result result) { + if (ARROW_PREDICT_TRUE(result.ok())) { + return *std::move(result); + } else { + int r = internal::check_status(result.status()); // takes the GIL + assert(r == -1); // should have errored out + ARROW_UNUSED(r); + return {}; + } +} + +/// \brief Wrap a Result and return the corresponding Python object. +/// +/// If the Result is successful, py_wrapper is called with its result value +/// and should return a PyObject*. If py_wrapper is successful (returns +/// a non-NULL value), its return value is returned. +/// +/// If either the Result or py_wrapper fails, the associated Python exception +/// is raised and NULL is returned. +// +/// \param result The Result whose value to wrap in a Python object. +/// \param py_wrapper A function (likely defined in Cython) to convert the C++ +/// value of the Result to a Python object. +/// \return A new Python reference, or NULL if an exception occurred +template +PyObject* WrapResult(Result result, PyWrapper&& py_wrapper) { + static_assert(std::is_same_v()))>, + "PyWrapper argument to WrapResult should return a PyObject* " + "when called with a T*"); + Status st = result.status(); + if (st.ok()) { + PyObject* py_value = py_wrapper(result.MoveValueUnsafe()); + st = CheckPyError(); + if (st.ok()) { + return py_value; + } + Py_XDECREF(py_value); // should be null, but who knows + } + // Status is an error, convert it to an exception. + return internal::convert_status(st); +} + +// A RAII-style helper that ensures the GIL is acquired inside a lexical block. +class ARROW_PYTHON_EXPORT PyAcquireGIL { + public: + PyAcquireGIL() : acquired_gil_(false) { acquire(); } + + ~PyAcquireGIL() { release(); } + + void acquire() { + if (!acquired_gil_) { + state_ = PyGILState_Ensure(); + acquired_gil_ = true; + } + } + + // idempotent + void release() { + if (acquired_gil_) { + PyGILState_Release(state_); + acquired_gil_ = false; + } + } + + private: + bool acquired_gil_; + PyGILState_STATE state_; + ARROW_DISALLOW_COPY_AND_ASSIGN(PyAcquireGIL); +}; + +// A RAII-style helper that releases the GIL until the end of a lexical block +class ARROW_PYTHON_EXPORT PyReleaseGIL { + public: + PyReleaseGIL() : ptr_(PyEval_SaveThread(), &unique_ptr_deleter) {} + + private: + static void unique_ptr_deleter(PyThreadState* state) { + if (state) { + PyEval_RestoreThread(state); + } + } + std::unique_ptr ptr_; +}; + +// A helper to call safely into the Python interpreter from arbitrary C++ code. +// The GIL is acquired, and the current thread's error status is preserved. +template +auto SafeCallIntoPython(Function&& func) -> decltype(func()) { + PyAcquireGIL lock; + PyObject* exc_type; + PyObject* exc_value; + PyObject* exc_traceback; + PyErr_Fetch(&exc_type, &exc_value, &exc_traceback); + auto maybe_status = std::forward(func)(); + // If the return Status is a "Python error", the current Python error status + // describes the error and shouldn't be clobbered. + if (!IsPyError(::arrow::ToStatus(maybe_status)) && exc_type != NULLPTR) { + PyErr_Restore(exc_type, exc_value, exc_traceback); + } + return maybe_status; +} + +template +auto SafeCallIntoPythonVoid(Function&& func) -> decltype(func()) { + PyAcquireGIL lock; + PyObject* exc_type; + PyObject* exc_value; + PyObject* exc_traceback; + PyErr_Fetch(&exc_type, &exc_value, &exc_traceback); + func(); + if (exc_type != NULLPTR) { + PyErr_Restore(exc_type, exc_value, exc_traceback); + } +} + +// A RAII primitive that DECREFs the underlying PyObject* when it +// goes out of scope. +class ARROW_PYTHON_EXPORT OwnedRef { + public: + OwnedRef() : obj_(NULLPTR) {} + OwnedRef(OwnedRef&& other) : OwnedRef(other.detach()) {} + explicit OwnedRef(PyObject* obj) : obj_(obj) {} + + OwnedRef& operator=(OwnedRef&& other) { + obj_ = other.detach(); + return *this; + } + + ~OwnedRef() { + // GH-38626: destructor may be called after the Python interpreter is finalized. + if (Py_IsInitialized()) { + reset(); + } + } + + void reset(PyObject* obj) { + Py_XDECREF(obj_); + obj_ = obj; + } + + void reset() { reset(NULLPTR); } + + PyObject* detach() { + PyObject* result = obj_; + obj_ = NULLPTR; + return result; + } + + PyObject* obj() const { return obj_; } + + PyObject** ref() { return &obj_; } + + operator bool() const { return obj_ != NULLPTR; } + + private: + ARROW_DISALLOW_COPY_AND_ASSIGN(OwnedRef); + + PyObject* obj_; +}; + +// Same as OwnedRef, but ensures the GIL is taken when it goes out of scope. +// This is for situations where the GIL is not always known to be held +// (e.g. if it is released in the middle of a function for performance reasons) +class ARROW_PYTHON_EXPORT OwnedRefNoGIL : public OwnedRef { + public: + OwnedRefNoGIL() : OwnedRef() {} + OwnedRefNoGIL(OwnedRefNoGIL&& other) : OwnedRef(other.detach()) {} + explicit OwnedRefNoGIL(PyObject* obj) : OwnedRef(obj) {} + + ~OwnedRefNoGIL() { + // GH-38626: destructor may be called after the Python interpreter is finalized. + if (Py_IsInitialized() && obj() != NULLPTR) { + PyAcquireGIL lock; + reset(); + } + } +}; + +template