ENISE1/hf_env/lib/python3.14/site-packages/pyarrow/includes/libarrow_fs.pxd

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# distutils: language = c++

from pyarrow.includes.common cimport *
from pyarrow.includes.libarrow cimport *
from pyarrow.includes.libarrow_python cimport CTimePoint

cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil:

    ctypedef enum CFileType "arrow::fs::FileType":
        CFileType_NotFound "arrow::fs::FileType::NotFound"
        CFileType_Unknown "arrow::fs::FileType::Unknown"
        CFileType_File "arrow::fs::FileType::File"
        CFileType_Directory "arrow::fs::FileType::Directory"

    cdef cppclass CFileInfo "arrow::fs::FileInfo":
        CFileInfo()
        CFileInfo(CFileInfo)
        CFileInfo& operator=(CFileInfo)
        CFileInfo(const CFileInfo&)
        CFileInfo& operator=(const CFileInfo&)

        CFileType type()
        void set_type(CFileType type)
        c_string path()
        void set_path(const c_string& path)
        c_string base_name()
        int64_t size()
        void set_size(int64_t size)
        c_string extension()
        CTimePoint mtime()
        void set_mtime(CTimePoint mtime)

    cdef cppclass CFileSelector "arrow::fs::FileSelector":
        CFileSelector()
        c_string base_dir
        c_bool allow_not_found
        c_bool recursive

    cdef cppclass CFileLocator "arrow::fs::FileLocator":
        shared_ptr[CFileSystem] filesystem
        c_string path

    cdef cppclass CFileSystem "arrow::fs::FileSystem":
        shared_ptr[CFileSystem] shared_from_this()
        c_string type_name() const
        CResult[c_string] NormalizePath(c_string path)
        CResult[c_string] MakeUri(c_string path)
        CResult[CFileInfo] GetFileInfo(const c_string& path)
        CResult[vector[CFileInfo]] GetFileInfo(
            const vector[c_string]& paths)
        CResult[vector[CFileInfo]] GetFileInfo(const CFileSelector& select)
        CStatus CreateDir(const c_string& path, c_bool recursive)
        CStatus DeleteDir(const c_string& path)
        CStatus DeleteDirContents(const c_string& path, c_bool missing_dir_ok)
        CStatus DeleteRootDirContents()
        CStatus DeleteFile(const c_string& path)
        CStatus DeleteFiles(const vector[c_string]& paths)
        CStatus Move(const c_string& src, const c_string& dest)
        CStatus CopyFile(const c_string& src, const c_string& dest)
        CResult[shared_ptr[CInputStream]] OpenInputStream(
            const c_string& path)
        CResult[shared_ptr[CRandomAccessFile]] OpenInputFile(
            const c_string& path)
        CResult[shared_ptr[COutputStream]] OpenOutputStream(
            const c_string& path, const shared_ptr[const CKeyValueMetadata]&)
        CResult[shared_ptr[COutputStream]] OpenAppendStream(
            const c_string& path, const shared_ptr[const CKeyValueMetadata]&)
        c_bool Equals(const CFileSystem& other)
        c_bool Equals(shared_ptr[CFileSystem] other)

    CResult[shared_ptr[CFileSystem]] CFileSystemFromUri \
        "arrow::fs::FileSystemFromUri"(const c_string& uri)
    CResult[shared_ptr[CFileSystem]] CFileSystemFromUri \
        "arrow::fs::FileSystemFromUri"(const c_string& uri, c_string* out_path)
    CResult[shared_ptr[CFileSystem]] CFileSystemFromUriOrPath \
        "arrow::fs::FileSystemFromUriOrPath"(const c_string& uri,
                                             c_string* out_path)

    cdef cppclass CFileSystemGlobalOptions \
            "arrow::fs::FileSystemGlobalOptions":
        c_string tls_ca_file_path
        c_string tls_ca_dir_path

    CStatus CFileSystemsInitialize "arrow::fs::Initialize" \
        (const CFileSystemGlobalOptions& options)

    cdef cppclass CSubTreeFileSystem \
            "arrow::fs::SubTreeFileSystem"(CFileSystem):
        CSubTreeFileSystem(const c_string& base_path,
                           shared_ptr[CFileSystem] base_fs)
        c_string base_path()
        shared_ptr[CFileSystem] base_fs()

    ctypedef enum CS3LogLevel "arrow::fs::S3LogLevel":
        CS3LogLevel_Off "arrow::fs::S3LogLevel::Off"
        CS3LogLevel_Fatal "arrow::fs::S3LogLevel::Fatal"
        CS3LogLevel_Error "arrow::fs::S3LogLevel::Error"
        CS3LogLevel_Warn "arrow::fs::S3LogLevel::Warn"
        CS3LogLevel_Info "arrow::fs::S3LogLevel::Info"
        CS3LogLevel_Debug "arrow::fs::S3LogLevel::Debug"
        CS3LogLevel_Trace "arrow::fs::S3LogLevel::Trace"

    cdef struct CS3GlobalOptions "arrow::fs::S3GlobalOptions":
        CS3LogLevel log_level
        int num_event_loop_threads

    cdef cppclass CS3ProxyOptions "arrow::fs::S3ProxyOptions":
        c_string scheme
        c_string host
        int port
        c_string username
        c_string password
        c_bool Equals(const CS3ProxyOptions& other)

        @staticmethod
        CResult[CS3ProxyOptions] FromUriString "FromUri"(
            const c_string& uri_string)

    ctypedef enum CS3CredentialsKind "arrow::fs::S3CredentialsKind":
        CS3CredentialsKind_Anonymous "arrow::fs::S3CredentialsKind::Anonymous"
        CS3CredentialsKind_Default "arrow::fs::S3CredentialsKind::Default"
        CS3CredentialsKind_Explicit "arrow::fs::S3CredentialsKind::Explicit"
        CS3CredentialsKind_Role "arrow::fs::S3CredentialsKind::Role"
        CS3CredentialsKind_WebIdentity \
            "arrow::fs::S3CredentialsKind::WebIdentity"

    cdef cppclass CS3RetryStrategy "arrow::fs::S3RetryStrategy":
        @staticmethod
        shared_ptr[CS3RetryStrategy] GetAwsDefaultRetryStrategy(int64_t max_attempts)

        @staticmethod
        shared_ptr[CS3RetryStrategy] GetAwsStandardRetryStrategy(int64_t max_attempts)

    cdef cppclass CS3Options "arrow::fs::S3Options":
        c_string region
        double connect_timeout
        double request_timeout
        c_string endpoint_override
        c_string scheme
        c_bool background_writes
        c_bool allow_delayed_open
        c_bool allow_bucket_creation
        c_bool allow_bucket_deletion
        c_bool check_directory_existence_before_creation
        c_bool force_virtual_addressing
        c_string tls_ca_file_path
        shared_ptr[const CKeyValueMetadata] default_metadata
        c_string role_arn
        c_string session_name
        c_string external_id
        int load_frequency
        CS3ProxyOptions proxy_options
        CS3CredentialsKind credentials_kind
        shared_ptr[CS3RetryStrategy] retry_strategy
        void ConfigureDefaultCredentials()
        void ConfigureAccessKey(const c_string& access_key,
                                const c_string& secret_key,
                                const c_string& session_token)
        c_string GetAccessKey()
        c_string GetSecretKey()
        c_string GetSessionToken()
        c_bool Equals(const CS3Options& other)

        @staticmethod
        CS3Options Defaults()

        @staticmethod
        CS3Options Anonymous()

        @staticmethod
        CS3Options FromAccessKey(const c_string& access_key,
                                 const c_string& secret_key,
                                 const c_string& session_token)

        @staticmethod
        CS3Options FromAssumeRole(const c_string& role_arn,
                                  const c_string& session_name,
                                  const c_string& external_id,
                                  const int load_frequency)

    cdef cppclass CS3FileSystem "arrow::fs::S3FileSystem"(CFileSystem):
        @staticmethod
        CResult[shared_ptr[CS3FileSystem]] Make(const CS3Options& options)
        CS3Options options()
        c_string region()

    cdef CStatus CInitializeS3 "arrow::fs::InitializeS3"(
        const CS3GlobalOptions& options)
    cdef CStatus CEnsureS3Initialized "arrow::fs::EnsureS3Initialized"()
    cdef CStatus CFinalizeS3 "arrow::fs::FinalizeS3"()
    cdef CStatus CEnsureS3Finalized "arrow::fs::EnsureS3Finalized"()

    cdef CResult[c_string] ResolveS3BucketRegion(const c_string& bucket)

    cdef cppclass CGcsCredentials "arrow::fs::GcsCredentials":
        c_bool anonymous()
        CTimePoint expiration()
        c_string access_token()
        c_string target_service_account()

    cdef cppclass CGcsOptions "arrow::fs::GcsOptions":
        CGcsCredentials credentials
        c_string endpoint_override
        c_string scheme
        c_string default_bucket_location
        optional[c_string] project_id
        optional[double] retry_limit_seconds
        shared_ptr[const CKeyValueMetadata] default_metadata
        c_bool Equals(const CS3Options& other)

        @staticmethod
        CGcsOptions Defaults()

        @staticmethod
        CGcsOptions Anonymous()

        @staticmethod
        CGcsOptions FromAccessToken(const c_string& access_token,
                                    CTimePoint expiration)

        @staticmethod
        CGcsOptions FromImpersonatedServiceAccount(const CGcsCredentials& base_credentials,
                                                   c_string& target_service_account)

    cdef cppclass CGcsFileSystem "arrow::fs::GcsFileSystem":
        @staticmethod
        CResult[shared_ptr[CGcsFileSystem]] Make(const CGcsOptions& options)
        CGcsOptions options()

    cdef cppclass CAzureOptions "arrow::fs::AzureOptions":
        c_string account_name
        c_string blob_storage_authority
        c_string dfs_storage_authority
        c_string blob_storage_scheme
        c_string dfs_storage_scheme

        c_bool Equals(const CAzureOptions& other)
        CStatus ConfigureDefaultCredential()
        CStatus ConfigureAccountKeyCredential(c_string account_key)
        CStatus ConfigureSASCredential(c_string sas_token)
        CStatus ConfigureManagedIdentityCredential(c_string client_id)
        CStatus ConfigureClientSecretCredential(c_string tenant_id,
                                                c_string client_id,
                                                c_string client_secret)

    cdef cppclass CAzureFileSystem "arrow::fs::AzureFileSystem":
        @staticmethod
        CResult[shared_ptr[CAzureFileSystem]] Make(const CAzureOptions& options)
        CAzureOptions options()

    cdef cppclass CHdfsOptions "arrow::fs::HdfsOptions":
        HdfsConnectionConfig connection_config
        int32_t buffer_size
        int16_t replication
        int64_t default_block_size

        @staticmethod
        CResult[CHdfsOptions] FromUriString "FromUri"(
            const c_string& uri_string)
        void ConfigureEndPoint(c_string host, int port)
        void ConfigureDriver(c_bool use_hdfs3)
        void ConfigureReplication(int16_t replication)
        void ConfigureUser(c_string user_name)
        void ConfigureBufferSize(int32_t buffer_size)
        void ConfigureBlockSize(int64_t default_block_size)
        void ConfigureKerberosTicketCachePath(c_string path)
        void ConfigureExtraConf(c_string key, c_string value)

    cdef cppclass CHadoopFileSystem "arrow::fs::HadoopFileSystem"(CFileSystem):
        @staticmethod
        CResult[shared_ptr[CHadoopFileSystem]] Make(
            const CHdfsOptions& options)
        CHdfsOptions options()

    cdef cppclass CMockFileSystem "arrow::fs::internal::MockFileSystem"(
            CFileSystem):
        CMockFileSystem(CTimePoint current_time)

    CStatus CCopyFiles "arrow::fs::CopyFiles"(
        const vector[CFileLocator]& sources,
        const vector[CFileLocator]& destinations,
        const CIOContext& io_context,
        int64_t chunk_size, c_bool use_threads)
    CStatus CCopyFilesWithSelector "arrow::fs::CopyFiles"(
        const shared_ptr[CFileSystem]& source_fs,
        const CFileSelector& source_sel,
        const shared_ptr[CFileSystem]& destination_fs,
        const c_string& destination_base_dir,
        const CIOContext& io_context,
        int64_t chunk_size, c_bool use_threads)


# Callbacks for implementing Python filesystems
# Use typedef to emulate syntax for std::function<void(..)>
ctypedef void CallbackGetTypeName(object, c_string*)
ctypedef c_bool CallbackEquals(object, const CFileSystem&)

ctypedef void CallbackGetFileInfo(object, const c_string&, CFileInfo*)
ctypedef void CallbackGetFileInfoVector(object, const vector[c_string]&,
                                        vector[CFileInfo]*)
ctypedef void CallbackGetFileInfoSelector(object, const CFileSelector&,
                                          vector[CFileInfo]*)
ctypedef void CallbackCreateDir(object, const c_string&, c_bool)
ctypedef void CallbackDeleteDir(object, const c_string&)
ctypedef void CallbackDeleteDirContents(object, const c_string&, c_bool)
ctypedef void CallbackDeleteRootDirContents(object)
ctypedef void CallbackDeleteFile(object, const c_string&)
ctypedef void CallbackMove(object, const c_string&, const c_string&)
ctypedef void CallbackCopyFile(object, const c_string&, const c_string&)

ctypedef void CallbackOpenInputStream(object, const c_string&,
                                      shared_ptr[CInputStream]*)
ctypedef void CallbackOpenInputFile(object, const c_string&,
                                    shared_ptr[CRandomAccessFile]*)
ctypedef void CallbackOpenOutputStream(
    object, const c_string&, const shared_ptr[const CKeyValueMetadata]&,
    shared_ptr[COutputStream]*)
ctypedef void CallbackNormalizePath(object, const c_string&, c_string*)

cdef extern from "arrow/python/filesystem.h" namespace "arrow::py::fs" nogil:

    cdef cppclass CPyFileSystemVtable "arrow::py::fs::PyFileSystemVtable":
        PyFileSystemVtable()
        function[CallbackGetTypeName] get_type_name
        function[CallbackEquals] equals
        function[CallbackGetFileInfo] get_file_info
        function[CallbackGetFileInfoVector] get_file_info_vector
        function[CallbackGetFileInfoSelector] get_file_info_selector
        function[CallbackCreateDir] create_dir
        function[CallbackDeleteDir] delete_dir
        function[CallbackDeleteDirContents] delete_dir_contents
        function[CallbackDeleteRootDirContents] delete_root_dir_contents
        function[CallbackDeleteFile] delete_file
        function[CallbackMove] move
        function[CallbackCopyFile] copy_file
        function[CallbackOpenInputStream] open_input_stream
        function[CallbackOpenInputFile] open_input_file
        function[CallbackOpenOutputStream] open_output_stream
        function[CallbackOpenOutputStream] open_append_stream
        function[CallbackNormalizePath] normalize_path

    cdef cppclass CPyFileSystem "arrow::py::fs::PyFileSystem":
        @staticmethod
        shared_ptr[CPyFileSystem] Make(object handler,
                                       CPyFileSystemVtable vtable)

        PyObject* handler()