Spaces:
Sleeping
Sleeping
File size: 1,394 Bytes
d576da9 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
import os
import zipfile
import gdown
from cnnClassifier import logger
from cnnClassifier.utils.common import get_size
from cnnClassifier.entity.config_entity import DataIngestionConfig
class DataIngestion:
def __init__(self, config: DataIngestionConfig):
self.config = config
def download_file(self)-> str:
'''
Fetch data from the url
'''
try:
dataset_url = self.config.source_URL
zip_download_dir = self.config.local_data_file
os.makedirs("artifacts/data_ingestion", exist_ok=True)
logger.info(f"Downloading data from {dataset_url} into file {zip_download_dir}")
file_id = dataset_url.split("/")[-2]
prefix = 'https://drive.google.com/uc?/export=download&id='
gdown.download(prefix+file_id,zip_download_dir)
logger.info(f"Downloaded data from {dataset_url} into file {zip_download_dir}")
except Exception as e:
raise e
def extract_zip_file(self):
"""
zip_file_path: str
Extracts the zip file into the data directory
Function returns None
"""
unzip_path = self.config.unzip_dir
os.makedirs(unzip_path, exist_ok=True)
with zipfile.ZipFile(self.config.local_data_file, 'r') as zip_ref:
zip_ref.extractall(unzip_path)
|