Spaces:
Paused
Paused
File size: 1,765 Bytes
aceb1b2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 | """
Data Sources Module
This module provides extensible data loading from various sources including
local files, URLs, cloud storage (Google Drive, Dropbox, S3), databases,
and other remote sources.
The module follows a singleton pattern for the DataSourceManager, similar
to other managers in Potato (ItemStateManager, UserStateManager).
Example usage:
from potato.data_sources import (
init_data_source_manager,
get_data_source_manager,
clear_data_source_manager
)
# Initialize with config
manager = init_data_source_manager(config)
# Load initial data
manager.load_initial_data()
# Get manager later
manager = get_data_source_manager()
# Load more data incrementally
manager.load_more(source_id, count=500)
"""
from potato.data_sources.manager import (
init_data_source_manager,
get_data_source_manager,
clear_data_source_manager,
DataSourceManager,
)
from potato.data_sources.base import (
DataSource,
SourceType,
SourceConfig,
)
from potato.data_sources.credentials import (
CredentialManager,
substitute_env_vars,
)
from potato.data_sources.cache_manager import (
CacheManager,
CacheEntry,
)
from potato.data_sources.partial_reader import (
PartialReader,
PartialReadState,
)
__all__ = [
# Manager functions
"init_data_source_manager",
"get_data_source_manager",
"clear_data_source_manager",
"DataSourceManager",
# Base classes
"DataSource",
"SourceType",
"SourceConfig",
# Credential management
"CredentialManager",
"substitute_env_vars",
# Cache management
"CacheManager",
"CacheEntry",
# Partial/incremental loading
"PartialReader",
"PartialReadState",
]
|