Spaces:
Sleeping
Sleeping
added backend logic for upload and metadata module
Browse files- backend/app/api/__init__.py +1 -0
- backend/app/api/animation.py +14 -0
- backend/app/api/export.py +24 -0
- backend/app/api/health.py +16 -0
- backend/app/api/interpolation.py +28 -0
- backend/app/api/metadata.py +34 -0
- backend/app/api/metrics.py +14 -0
- backend/app/api/upload.py +10 -0
- backend/app/api/visualization.py +22 -0
- backend/app/core/__init__.py +1 -0
- backend/app/core/config.py +1 -0
- backend/app/core/exceptions.py +1 -0
- backend/app/jobs/__init__.py +1 -0
- backend/app/jobs/cleanup_job.py +1 -0
- backend/app/jobs/export_job.py +1 -0
- backend/app/jobs/interpolation_job.py +1 -0
- backend/app/main.py +32 -0
- backend/app/schemas/__init__.py +1 -0
- backend/app/schemas/animation.py +7 -0
- backend/app/schemas/common.py +18 -0
- backend/app/schemas/export.py +15 -0
- backend/app/schemas/interpolation.py +17 -0
- backend/app/schemas/metadata.py +44 -0
- backend/app/schemas/metrics.py +12 -0
- backend/app/schemas/upload.py +10 -0
- backend/app/schemas/visualization.py +7 -0
- backend/app/services/__init__.py +1 -0
- backend/app/services/inference/__init__.py +1 -0
- backend/app/services/inference/model_loader.py +1 -0
- backend/app/services/inference/onnx_runtime.py +1 -0
- backend/app/services/inference/rife.py +1 -0
- backend/app/services/scientific/__init__.py +1 -0
- backend/app/services/scientific/base_parser.py +11 -0
- backend/app/services/scientific/hdf_parser.py +156 -0
- backend/app/services/scientific/metadata_service.py +64 -0
- backend/app/services/scientific/metrics.py +1 -0
- backend/app/services/scientific/netcdf_parser.py +110 -0
- backend/app/services/scientific/visualization.py +1 -0
- backend/app/services/upload_service.py +56 -0
- backend/app/storage/__init__.py +1 -0
- backend/app/utils/__init__.py +1 -0
- backend/requirements.txt +10 -0
- backend/tests/test_metadata.py +104 -0
backend/app/api/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Placeholder
|
backend/app/api/animation.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter
|
| 2 |
+
from app.schemas.common import ApiResponse
|
| 3 |
+
from app.schemas.animation import AnimationSequenceResponse
|
| 4 |
+
|
| 5 |
+
router = APIRouter()
|
| 6 |
+
|
| 7 |
+
@router.get("/{file_id}/sequence", response_model=ApiResponse)
|
| 8 |
+
async def get_sequence(file_id: str, start_time: str = "", end_time: str = "", fps: int = 10):
|
| 9 |
+
# Placeholder
|
| 10 |
+
return ApiResponse(
|
| 11 |
+
success=True,
|
| 12 |
+
message="Animation sequence retrieved",
|
| 13 |
+
data=None
|
| 14 |
+
)
|
backend/app/api/export.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter
|
| 2 |
+
from app.schemas.common import ApiResponse
|
| 3 |
+
from app.schemas.export import ExportRequest, ExportJobStatusResponse
|
| 4 |
+
from app.schemas.interpolation import JobResponse
|
| 5 |
+
|
| 6 |
+
router = APIRouter()
|
| 7 |
+
|
| 8 |
+
@router.post("/job", response_model=ApiResponse)
|
| 9 |
+
async def create_export_job(request: ExportRequest):
|
| 10 |
+
# Placeholder
|
| 11 |
+
return ApiResponse(
|
| 12 |
+
success=True,
|
| 13 |
+
message="Export job queued",
|
| 14 |
+
data=JobResponse(job_id="dummy-export-job", status="preparing")
|
| 15 |
+
)
|
| 16 |
+
|
| 17 |
+
@router.get("/status/{job_id}", response_model=ApiResponse)
|
| 18 |
+
async def get_export_status(job_id: str):
|
| 19 |
+
# Placeholder
|
| 20 |
+
return ApiResponse(
|
| 21 |
+
success=True,
|
| 22 |
+
message="Export status retrieved",
|
| 23 |
+
data=ExportJobStatusResponse(status="processing", progress=10.0)
|
| 24 |
+
)
|
backend/app/api/health.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter
|
| 2 |
+
from app.schemas.common import ApiResponse
|
| 3 |
+
|
| 4 |
+
router = APIRouter()
|
| 5 |
+
|
| 6 |
+
@router.get("/")
|
| 7 |
+
async def health_check():
|
| 8 |
+
return ApiResponse(success=True, message="API is healthy", data=None)
|
| 9 |
+
|
| 10 |
+
@router.get("/gpu")
|
| 11 |
+
async def health_gpu():
|
| 12 |
+
return ApiResponse(success=True, message="GPU status", data={"cuda_available": False})
|
| 13 |
+
|
| 14 |
+
@router.get("/models")
|
| 15 |
+
async def health_models():
|
| 16 |
+
return ApiResponse(success=True, message="Models status", data={"rife": "not_loaded"})
|
backend/app/api/interpolation.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter
|
| 2 |
+
from app.schemas.common import ApiResponse
|
| 3 |
+
from app.schemas.interpolation import InterpolationRequest, JobResponse, JobStatusResponse
|
| 4 |
+
|
| 5 |
+
router = APIRouter()
|
| 6 |
+
|
| 7 |
+
@router.post("/generate", response_model=ApiResponse)
|
| 8 |
+
async def generate_interpolation(request: InterpolationRequest):
|
| 9 |
+
# Placeholder
|
| 10 |
+
return ApiResponse(
|
| 11 |
+
success=True,
|
| 12 |
+
message="Job queued",
|
| 13 |
+
data=JobResponse(job_id="dummy-job", status="queued")
|
| 14 |
+
)
|
| 15 |
+
|
| 16 |
+
@router.get("/status/{job_id}", response_model=ApiResponse)
|
| 17 |
+
async def get_status(job_id: str):
|
| 18 |
+
# Placeholder
|
| 19 |
+
return ApiResponse(
|
| 20 |
+
success=True,
|
| 21 |
+
message="Status retrieved",
|
| 22 |
+
data=JobStatusResponse(status="processing", progress=50.0)
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
@router.get("/events/{job_id}")
|
| 26 |
+
async def get_events(job_id: str):
|
| 27 |
+
# Placeholder for SSE
|
| 28 |
+
return {"message": "SSE endpoint placeholder"}
|
backend/app/api/metadata.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter, HTTPException
|
| 2 |
+
from app.schemas.common import ApiResponse
|
| 3 |
+
from app.services.scientific.metadata_service import MetadataService
|
| 4 |
+
import os
|
| 5 |
+
|
| 6 |
+
router = APIRouter()
|
| 7 |
+
|
| 8 |
+
# Note: Depending on where uploads are stored, we assume a storage path.
|
| 9 |
+
# Assuming standard backend/storage/uploads based on previous contexts.
|
| 10 |
+
STORAGE_DIR = os.path.join(os.getcwd(), "storage", "uploads")
|
| 11 |
+
|
| 12 |
+
@router.get("/{file_id}", response_model=ApiResponse)
|
| 13 |
+
async def get_metadata(file_id: str):
|
| 14 |
+
# Search for the file in the storage directory
|
| 15 |
+
file_path = None
|
| 16 |
+
for ext in [".nc", ".h5", ".hdf5"]:
|
| 17 |
+
possible_path = os.path.join(STORAGE_DIR, f"{file_id}{ext}")
|
| 18 |
+
if os.path.exists(possible_path):
|
| 19 |
+
file_path = possible_path
|
| 20 |
+
break
|
| 21 |
+
|
| 22 |
+
if not file_path:
|
| 23 |
+
raise HTTPException(status_code=404, detail="File not found")
|
| 24 |
+
|
| 25 |
+
try:
|
| 26 |
+
metadata_response = MetadataService.extract_metadata(file_id, file_path)
|
| 27 |
+
return ApiResponse(
|
| 28 |
+
success=True,
|
| 29 |
+
message="Metadata retrieved successfully",
|
| 30 |
+
data=metadata_response
|
| 31 |
+
)
|
| 32 |
+
except Exception as e:
|
| 33 |
+
raise HTTPException(status_code=500, detail=f"Failed to extract metadata: {str(e)}")
|
| 34 |
+
|
backend/app/api/metrics.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter
|
| 2 |
+
from app.schemas.common import ApiResponse
|
| 3 |
+
from app.schemas.metrics import MetricsRequest, MetricsResponse
|
| 4 |
+
|
| 5 |
+
router = APIRouter()
|
| 6 |
+
|
| 7 |
+
@router.post("/compare", response_model=ApiResponse)
|
| 8 |
+
async def compare_frames(request: MetricsRequest):
|
| 9 |
+
# Placeholder
|
| 10 |
+
return ApiResponse(
|
| 11 |
+
success=True,
|
| 12 |
+
message="Metrics computed",
|
| 13 |
+
data=None
|
| 14 |
+
)
|
backend/app/api/upload.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter, UploadFile, File
|
| 2 |
+
from app.schemas.upload import UploadResponse
|
| 3 |
+
from app.services.upload_service import UploadService
|
| 4 |
+
|
| 5 |
+
router = APIRouter()
|
| 6 |
+
|
| 7 |
+
@router.post("", response_model=UploadResponse)
|
| 8 |
+
async def upload_file(file: UploadFile = File(...)):
|
| 9 |
+
result = await UploadService.process_upload(file)
|
| 10 |
+
return result
|
backend/app/api/visualization.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter
|
| 2 |
+
from app.schemas.common import ApiResponse
|
| 3 |
+
|
| 4 |
+
router = APIRouter()
|
| 5 |
+
|
| 6 |
+
@router.get("/{file_id}/frame", response_model=ApiResponse)
|
| 7 |
+
async def get_frame(file_id: str, time_index: int = 0, variable: str = ""):
|
| 8 |
+
# Placeholder
|
| 9 |
+
return ApiResponse(
|
| 10 |
+
success=True,
|
| 11 |
+
message="Frame retrieved",
|
| 12 |
+
data=None
|
| 13 |
+
)
|
| 14 |
+
|
| 15 |
+
@router.get("/{file_id}/thumbnail", response_model=ApiResponse)
|
| 16 |
+
async def get_thumbnail(file_id: str):
|
| 17 |
+
# Placeholder
|
| 18 |
+
return ApiResponse(
|
| 19 |
+
success=True,
|
| 20 |
+
message="Thumbnail retrieved",
|
| 21 |
+
data={"url": "dummy_url"}
|
| 22 |
+
)
|
backend/app/core/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Placeholder
|
backend/app/core/config.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Placeholder
|
backend/app/core/exceptions.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Placeholder
|
backend/app/jobs/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Placeholder
|
backend/app/jobs/cleanup_job.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Placeholder
|
backend/app/jobs/export_job.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Placeholder
|
backend/app/jobs/interpolation_job.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Placeholder
|
backend/app/main.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import FastAPI
|
| 2 |
+
from fastapi.middleware.cors import CORSMiddleware
|
| 3 |
+
from .api import upload, metadata, visualization, interpolation, metrics, animation, export, health
|
| 4 |
+
|
| 5 |
+
app = FastAPI(
|
| 6 |
+
title="Fill the Frames API",
|
| 7 |
+
description="Scientific backend for satellite frame interpolation and visualization",
|
| 8 |
+
version="0.1.0"
|
| 9 |
+
)
|
| 10 |
+
|
| 11 |
+
# CORS configuration
|
| 12 |
+
app.add_middleware(
|
| 13 |
+
CORSMiddleware,
|
| 14 |
+
allow_origins=["*"], # Update for production
|
| 15 |
+
allow_credentials=True,
|
| 16 |
+
allow_methods=["*"],
|
| 17 |
+
allow_headers=["*"],
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
# Register routers
|
| 21 |
+
app.include_router(health.router, tags=["Health"])
|
| 22 |
+
app.include_router(upload.router, prefix="/api/v1/upload", tags=["Upload"])
|
| 23 |
+
app.include_router(metadata.router, prefix="/api/v1/metadata", tags=["Metadata"])
|
| 24 |
+
app.include_router(visualization.router, prefix="/api/v1/visualization", tags=["Visualization"])
|
| 25 |
+
app.include_router(interpolation.router, prefix="/api/v1/interpolation", tags=["Interpolation"])
|
| 26 |
+
app.include_router(metrics.router, prefix="/api/v1/metrics", tags=["Metrics"])
|
| 27 |
+
app.include_router(animation.router, prefix="/api/v1/animation", tags=["Animation"])
|
| 28 |
+
app.include_router(export.router, prefix="/api/v1/export", tags=["Export"])
|
| 29 |
+
|
| 30 |
+
@app.get("/")
|
| 31 |
+
def read_root():
|
| 32 |
+
return {"message": "Welcome to Fill the Frames API"}
|
backend/app/schemas/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Placeholder
|
backend/app/schemas/animation.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pydantic import BaseModel
|
| 2 |
+
from typing import List
|
| 3 |
+
from .visualization import FrameDataResponse
|
| 4 |
+
|
| 5 |
+
class AnimationSequenceResponse(BaseModel):
|
| 6 |
+
frames: List[FrameDataResponse]
|
| 7 |
+
total_frames: int
|
backend/app/schemas/common.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pydantic import BaseModel
|
| 2 |
+
from typing import Any
|
| 3 |
+
from datetime import datetime
|
| 4 |
+
|
| 5 |
+
class ApiResponse(BaseModel):
|
| 6 |
+
success: bool
|
| 7 |
+
message: str
|
| 8 |
+
data: Any
|
| 9 |
+
|
| 10 |
+
class FrameData(BaseModel):
|
| 11 |
+
frame_id: str
|
| 12 |
+
timestamp: datetime
|
| 13 |
+
variable: str
|
| 14 |
+
width: int
|
| 15 |
+
height: int
|
| 16 |
+
min_value: float
|
| 17 |
+
max_value: float
|
| 18 |
+
source: str
|
backend/app/schemas/export.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pydantic import BaseModel
|
| 2 |
+
from typing import Optional
|
| 3 |
+
|
| 4 |
+
class ExportRequest(BaseModel):
|
| 5 |
+
target_id: str
|
| 6 |
+
format: str
|
| 7 |
+
resolution: str
|
| 8 |
+
include_metadata: bool
|
| 9 |
+
include_metrics: bool
|
| 10 |
+
include_animation: bool = False
|
| 11 |
+
|
| 12 |
+
class ExportJobStatusResponse(BaseModel):
|
| 13 |
+
status: str
|
| 14 |
+
progress: float
|
| 15 |
+
download_url: Optional[str] = None
|
backend/app/schemas/interpolation.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pydantic import BaseModel
|
| 2 |
+
from typing import Optional
|
| 3 |
+
from .visualization import FrameDataResponse
|
| 4 |
+
|
| 5 |
+
class InterpolationRequest(BaseModel):
|
| 6 |
+
file_id: str
|
| 7 |
+
time_ratio: float
|
| 8 |
+
model: str
|
| 9 |
+
|
| 10 |
+
class JobResponse(BaseModel):
|
| 11 |
+
job_id: str
|
| 12 |
+
status: str
|
| 13 |
+
|
| 14 |
+
class JobStatusResponse(BaseModel):
|
| 15 |
+
status: str
|
| 16 |
+
progress: float
|
| 17 |
+
result_frame: Optional[FrameDataResponse] = None
|
backend/app/schemas/metadata.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pydantic import BaseModel
|
| 2 |
+
from typing import List, Dict, Any, Optional
|
| 3 |
+
|
| 4 |
+
class DimensionInfo(BaseModel):
|
| 5 |
+
name: str
|
| 6 |
+
size: int
|
| 7 |
+
|
| 8 |
+
class VariableInfo(BaseModel):
|
| 9 |
+
name: str
|
| 10 |
+
datatype: str
|
| 11 |
+
dimensions: List[str]
|
| 12 |
+
shape: List[int]
|
| 13 |
+
attributes: Dict[str, Any]
|
| 14 |
+
min_value: Optional[float] = None
|
| 15 |
+
max_value: Optional[float] = None
|
| 16 |
+
|
| 17 |
+
class CoordinateInfo(BaseModel):
|
| 18 |
+
latitude: Optional[str] = None
|
| 19 |
+
longitude: Optional[str] = None
|
| 20 |
+
projection: Optional[str] = None
|
| 21 |
+
|
| 22 |
+
class TemporalInfo(BaseModel):
|
| 23 |
+
start_time: Optional[str] = None
|
| 24 |
+
end_time: Optional[str] = None
|
| 25 |
+
time_steps: Optional[int] = None
|
| 26 |
+
|
| 27 |
+
class DatasetSummary(BaseModel):
|
| 28 |
+
file_format: str
|
| 29 |
+
variable_count: int
|
| 30 |
+
dimension_count: int
|
| 31 |
+
coordinate_count: int
|
| 32 |
+
dataset_size: int
|
| 33 |
+
|
| 34 |
+
class MetadataResponse(BaseModel):
|
| 35 |
+
file_id: str
|
| 36 |
+
filename: str
|
| 37 |
+
size: int
|
| 38 |
+
format: str
|
| 39 |
+
global_attributes: Dict[str, Any]
|
| 40 |
+
dimensions: List[DimensionInfo]
|
| 41 |
+
variables: List[VariableInfo]
|
| 42 |
+
coordinates: CoordinateInfo
|
| 43 |
+
temporal_info: TemporalInfo
|
| 44 |
+
summary: DatasetSummary
|
backend/app/schemas/metrics.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pydantic import BaseModel
|
| 2 |
+
from typing import List, Dict
|
| 3 |
+
from .visualization import FrameDataResponse
|
| 4 |
+
|
| 5 |
+
class MetricsRequest(BaseModel):
|
| 6 |
+
frame_a_id: str
|
| 7 |
+
frame_b_id: str
|
| 8 |
+
metrics: List[str]
|
| 9 |
+
|
| 10 |
+
class MetricsResponse(BaseModel):
|
| 11 |
+
scores: Dict[str, float]
|
| 12 |
+
difference_map: FrameDataResponse
|
backend/app/schemas/upload.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pydantic import BaseModel, Field
|
| 2 |
+
|
| 3 |
+
class UploadResponse(BaseModel):
|
| 4 |
+
success: bool = True
|
| 5 |
+
fileId: str = Field(..., alias="fileId")
|
| 6 |
+
filename: str
|
| 7 |
+
status: str
|
| 8 |
+
|
| 9 |
+
class Config:
|
| 10 |
+
populate_by_name = True
|
backend/app/schemas/visualization.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pydantic import BaseModel
|
| 2 |
+
from typing import List
|
| 3 |
+
from .common import FrameData
|
| 4 |
+
|
| 5 |
+
class FrameDataResponse(BaseModel):
|
| 6 |
+
frame_metadata: FrameData
|
| 7 |
+
z: List[List[float]]
|
backend/app/services/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Placeholder
|
backend/app/services/inference/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Placeholder
|
backend/app/services/inference/model_loader.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Placeholder
|
backend/app/services/inference/onnx_runtime.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Placeholder
|
backend/app/services/inference/rife.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Placeholder
|
backend/app/services/scientific/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Placeholder
|
backend/app/services/scientific/base_parser.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from abc import ABC, abstractmethod
|
| 2 |
+
from typing import Dict, Any
|
| 3 |
+
|
| 4 |
+
class BaseDatasetParser(ABC):
|
| 5 |
+
@abstractmethod
|
| 6 |
+
def load_dataset(self, file_path: str):
|
| 7 |
+
pass
|
| 8 |
+
|
| 9 |
+
@abstractmethod
|
| 10 |
+
def extract_metadata(self) -> Dict[str, Any]:
|
| 11 |
+
pass
|
backend/app/services/scientific/hdf_parser.py
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import h5py
|
| 2 |
+
import numpy as np
|
| 3 |
+
from typing import Dict, Any, List
|
| 4 |
+
from .base_parser import BaseDatasetParser
|
| 5 |
+
|
| 6 |
+
class HDFParser(BaseDatasetParser):
|
| 7 |
+
def __init__(self):
|
| 8 |
+
self.file = None
|
| 9 |
+
|
| 10 |
+
def load_dataset(self, file_path: str):
|
| 11 |
+
self.file = h5py.File(file_path, "r")
|
| 12 |
+
|
| 13 |
+
def _convert_type(self, val):
|
| 14 |
+
if isinstance(val, (np.integer, int)):
|
| 15 |
+
return int(val)
|
| 16 |
+
if isinstance(val, (np.floating, float)):
|
| 17 |
+
return float(val)
|
| 18 |
+
if isinstance(val, np.ndarray):
|
| 19 |
+
if val.size == 1:
|
| 20 |
+
return self._convert_type(val.item())
|
| 21 |
+
return [self._convert_type(v) for v in val.tolist()]
|
| 22 |
+
if isinstance(val, bytes):
|
| 23 |
+
try:
|
| 24 |
+
return val.decode('utf-8')
|
| 25 |
+
except Exception:
|
| 26 |
+
return str(val)
|
| 27 |
+
return str(val)
|
| 28 |
+
|
| 29 |
+
def extract_metadata(self) -> Dict[str, Any]:
|
| 30 |
+
if self.file is None:
|
| 31 |
+
raise ValueError("Dataset not loaded")
|
| 32 |
+
|
| 33 |
+
# Global Attributes
|
| 34 |
+
global_attributes = {k: self._convert_type(v) for k, v in self.file.attrs.items()}
|
| 35 |
+
|
| 36 |
+
variables = []
|
| 37 |
+
dimensions_map = {}
|
| 38 |
+
coordinate_names = []
|
| 39 |
+
|
| 40 |
+
# Helper to traverse HDF5 recursively
|
| 41 |
+
def visit_func(name, node):
|
| 42 |
+
if isinstance(node, h5py.Dataset):
|
| 43 |
+
var_attrs = {k: self._convert_type(v) for k, v in node.attrs.items()}
|
| 44 |
+
|
| 45 |
+
# Check if it's a dimension scale (often used as coordinates in HDF5/netCDF4-hdf5)
|
| 46 |
+
is_scale = False
|
| 47 |
+
if "CLASS" in var_attrs and var_attrs["CLASS"] == "DIMENSION_SCALE":
|
| 48 |
+
is_scale = True
|
| 49 |
+
coordinate_names.append(name.split('/')[-1])
|
| 50 |
+
|
| 51 |
+
# Extract dimension sizes
|
| 52 |
+
dims = []
|
| 53 |
+
for i, dim in enumerate(node.dims):
|
| 54 |
+
if len(dim) > 0 and dim[0].name:
|
| 55 |
+
dim_name = dim[0].name.split('/')[-1]
|
| 56 |
+
dims.append(dim_name)
|
| 57 |
+
dimensions_map[dim_name] = node.shape[i]
|
| 58 |
+
else:
|
| 59 |
+
dim_name = f"{name}_dim_{i}"
|
| 60 |
+
dims.append(dim_name)
|
| 61 |
+
dimensions_map[dim_name] = node.shape[i]
|
| 62 |
+
|
| 63 |
+
if not dims:
|
| 64 |
+
# If dims aren't formally defined, make placeholder names
|
| 65 |
+
for i, s in enumerate(node.shape):
|
| 66 |
+
dim_name = f"dim_{i}"
|
| 67 |
+
dims.append(dim_name)
|
| 68 |
+
dimensions_map[dim_name] = s
|
| 69 |
+
|
| 70 |
+
min_val = None
|
| 71 |
+
max_val = None
|
| 72 |
+
if node.size > 0 and node.size < 100000:
|
| 73 |
+
try:
|
| 74 |
+
# Only read small datasets to avoid memory issues
|
| 75 |
+
data = node[:]
|
| 76 |
+
min_val = float(np.min(data))
|
| 77 |
+
max_val = float(np.max(data))
|
| 78 |
+
except Exception:
|
| 79 |
+
pass
|
| 80 |
+
|
| 81 |
+
variables.append({
|
| 82 |
+
"name": name,
|
| 83 |
+
"datatype": str(node.dtype),
|
| 84 |
+
"dimensions": dims,
|
| 85 |
+
"shape": list(node.shape),
|
| 86 |
+
"attributes": var_attrs,
|
| 87 |
+
"min_value": min_val,
|
| 88 |
+
"max_value": max_val
|
| 89 |
+
})
|
| 90 |
+
|
| 91 |
+
self.file.visititems(visit_func)
|
| 92 |
+
|
| 93 |
+
dimensions = [{"name": str(k), "size": int(v)} for k, v in dimensions_map.items()]
|
| 94 |
+
|
| 95 |
+
# Coordinates
|
| 96 |
+
lat_names = ["lat", "latitude", "y", "geolocation/latitude"]
|
| 97 |
+
lon_names = ["lon", "longitude", "x", "geolocation/longitude"]
|
| 98 |
+
|
| 99 |
+
lat_coord = next((c for c in coordinate_names if c.lower() in lat_names), None)
|
| 100 |
+
lon_coord = next((c for c in coordinate_names if c.lower() in lon_names), None)
|
| 101 |
+
|
| 102 |
+
# Sometimes lat/lon are just variables not marked as scales
|
| 103 |
+
if not lat_coord:
|
| 104 |
+
lat_coord = next((v["name"] for v in variables if v["name"].split('/')[-1].lower() in lat_names), None)
|
| 105 |
+
if not lon_coord:
|
| 106 |
+
lon_coord = next((v["name"] for v in variables if v["name"].split('/')[-1].lower() in lon_names), None)
|
| 107 |
+
|
| 108 |
+
projection = global_attributes.get("projection") or global_attributes.get("grid_mapping_name")
|
| 109 |
+
for var in variables:
|
| 110 |
+
if "grid_mapping" in var["attributes"] or "grid_mapping_name" in var["attributes"]:
|
| 111 |
+
projection = var["attributes"].get("grid_mapping_name") or "mapped"
|
| 112 |
+
break
|
| 113 |
+
|
| 114 |
+
coordinates = {
|
| 115 |
+
"latitude": lat_coord,
|
| 116 |
+
"longitude": lon_coord,
|
| 117 |
+
"projection": str(projection) if projection else None
|
| 118 |
+
}
|
| 119 |
+
|
| 120 |
+
# Temporal Information
|
| 121 |
+
time_names = ["time", "valid_time", "timestamp", "forecast_time"]
|
| 122 |
+
time_coord_name = next((c for c in coordinate_names if c.lower() in time_names), None)
|
| 123 |
+
if not time_coord_name:
|
| 124 |
+
time_coord_name = next((v["name"] for v in variables if v["name"].split('/')[-1].lower() in time_names), None)
|
| 125 |
+
|
| 126 |
+
temporal_info = {
|
| 127 |
+
"start_time": None,
|
| 128 |
+
"end_time": None,
|
| 129 |
+
"time_steps": None
|
| 130 |
+
}
|
| 131 |
+
|
| 132 |
+
if time_coord_name and time_coord_name in self.file:
|
| 133 |
+
t_var = self.file[time_coord_name]
|
| 134 |
+
temporal_info["time_steps"] = int(t_var.size)
|
| 135 |
+
if t_var.size > 0:
|
| 136 |
+
try:
|
| 137 |
+
data = t_var[:]
|
| 138 |
+
temporal_info["start_time"] = str(data[0])
|
| 139 |
+
temporal_info["end_time"] = str(data[-1])
|
| 140 |
+
except Exception:
|
| 141 |
+
pass
|
| 142 |
+
|
| 143 |
+
return {
|
| 144 |
+
"global_attributes": global_attributes,
|
| 145 |
+
"dimensions": dimensions,
|
| 146 |
+
"variables": variables,
|
| 147 |
+
"coordinates": coordinates,
|
| 148 |
+
"temporal_info": temporal_info,
|
| 149 |
+
"variable_count": len(variables),
|
| 150 |
+
"dimension_count": len(dimensions),
|
| 151 |
+
"coordinate_count": len(coordinate_names)
|
| 152 |
+
}
|
| 153 |
+
|
| 154 |
+
def close(self):
|
| 155 |
+
if self.file is not None:
|
| 156 |
+
self.file.close()
|
backend/app/services/scientific/metadata_service.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import logging
|
| 3 |
+
from typing import Dict, Any
|
| 4 |
+
|
| 5 |
+
from app.schemas.metadata import MetadataResponse, DimensionInfo, VariableInfo, CoordinateInfo, TemporalInfo, DatasetSummary
|
| 6 |
+
from .netcdf_parser import NetCDFParser
|
| 7 |
+
from .hdf_parser import HDFParser
|
| 8 |
+
|
| 9 |
+
logger = logging.getLogger(__name__)
|
| 10 |
+
|
| 11 |
+
class MetadataService:
|
| 12 |
+
@staticmethod
|
| 13 |
+
def get_parser(file_path: str):
|
| 14 |
+
ext = os.path.splitext(file_path)[1].lower()
|
| 15 |
+
if ext == ".nc":
|
| 16 |
+
return NetCDFParser()
|
| 17 |
+
elif ext in [".h5", ".hdf5"]:
|
| 18 |
+
return HDFParser()
|
| 19 |
+
else:
|
| 20 |
+
raise ValueError(f"Unsupported file extension: {ext}")
|
| 21 |
+
|
| 22 |
+
@staticmethod
|
| 23 |
+
def extract_metadata(file_id: str, file_path: str) -> MetadataResponse:
|
| 24 |
+
logger.info(f"Opening dataset {file_id} at {file_path}")
|
| 25 |
+
parser = None
|
| 26 |
+
try:
|
| 27 |
+
parser = MetadataService.get_parser(file_path)
|
| 28 |
+
parser.load_dataset(file_path)
|
| 29 |
+
|
| 30 |
+
raw_metadata = parser.extract_metadata()
|
| 31 |
+
|
| 32 |
+
file_size = os.path.getsize(file_path)
|
| 33 |
+
file_format = os.path.splitext(file_path)[1].lower().strip(".")
|
| 34 |
+
|
| 35 |
+
summary = DatasetSummary(
|
| 36 |
+
file_format=file_format,
|
| 37 |
+
variable_count=raw_metadata["variable_count"],
|
| 38 |
+
dimension_count=raw_metadata["dimension_count"],
|
| 39 |
+
coordinate_count=raw_metadata["coordinate_count"],
|
| 40 |
+
dataset_size=file_size
|
| 41 |
+
)
|
| 42 |
+
|
| 43 |
+
response = MetadataResponse(
|
| 44 |
+
file_id=file_id,
|
| 45 |
+
filename=os.path.basename(file_path),
|
| 46 |
+
size=file_size,
|
| 47 |
+
format=file_format,
|
| 48 |
+
global_attributes=raw_metadata["global_attributes"],
|
| 49 |
+
dimensions=[DimensionInfo(**d) for d in raw_metadata["dimensions"]],
|
| 50 |
+
variables=[VariableInfo(**v) for v in raw_metadata["variables"]],
|
| 51 |
+
coordinates=CoordinateInfo(**raw_metadata["coordinates"]),
|
| 52 |
+
temporal_info=TemporalInfo(**raw_metadata["temporal_info"]),
|
| 53 |
+
summary=summary
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
logger.info(f"Metadata extracted successfully for {file_id}")
|
| 57 |
+
return response
|
| 58 |
+
|
| 59 |
+
except Exception as e:
|
| 60 |
+
logger.error(f"Metadata extraction failed for {file_id}: {str(e)}")
|
| 61 |
+
raise e
|
| 62 |
+
finally:
|
| 63 |
+
if parser is not None:
|
| 64 |
+
parser.close()
|
backend/app/services/scientific/metrics.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Placeholder
|
backend/app/services/scientific/netcdf_parser.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import xarray as xr
|
| 2 |
+
import numpy as np
|
| 3 |
+
from typing import Dict, Any
|
| 4 |
+
from .base_parser import BaseDatasetParser
|
| 5 |
+
|
| 6 |
+
class NetCDFParser(BaseDatasetParser):
|
| 7 |
+
def __init__(self):
|
| 8 |
+
self.ds = None
|
| 9 |
+
|
| 10 |
+
def load_dataset(self, file_path: str):
|
| 11 |
+
self.ds = xr.open_dataset(file_path, engine="netcdf4")
|
| 12 |
+
|
| 13 |
+
def _convert_type(self, val):
|
| 14 |
+
if isinstance(val, (np.integer, int)):
|
| 15 |
+
return int(val)
|
| 16 |
+
if isinstance(val, (np.floating, float)):
|
| 17 |
+
return float(val)
|
| 18 |
+
if isinstance(val, np.ndarray):
|
| 19 |
+
return val.tolist()
|
| 20 |
+
return str(val)
|
| 21 |
+
|
| 22 |
+
def extract_metadata(self) -> Dict[str, Any]:
|
| 23 |
+
if self.ds is None:
|
| 24 |
+
raise ValueError("Dataset not loaded")
|
| 25 |
+
|
| 26 |
+
# Global Attributes
|
| 27 |
+
global_attributes = {k: self._convert_type(v) for k, v in self.ds.attrs.items()}
|
| 28 |
+
|
| 29 |
+
# Dimensions
|
| 30 |
+
dimensions = [{"name": str(k), "size": int(v)} for k, v in self.ds.sizes.items()]
|
| 31 |
+
|
| 32 |
+
# Variables
|
| 33 |
+
variables = []
|
| 34 |
+
for name, var in self.ds.variables.items():
|
| 35 |
+
# Skip if it's purely a coordinate (optional, but requested all variables? Usually we list all)
|
| 36 |
+
min_val = None
|
| 37 |
+
max_val = None
|
| 38 |
+
# Only calculate min/max if it's not a huge dataset or inexpensive. For metadata, we can sample or ignore.
|
| 39 |
+
# To be safe and inexpensive, we might just skip or do min/max on small arrays.
|
| 40 |
+
if var.size < 100000: # arbitrary small limit
|
| 41 |
+
try:
|
| 42 |
+
min_val = float(var.min().values)
|
| 43 |
+
max_val = float(var.max().values)
|
| 44 |
+
except Exception:
|
| 45 |
+
pass
|
| 46 |
+
|
| 47 |
+
variables.append({
|
| 48 |
+
"name": str(name),
|
| 49 |
+
"datatype": str(var.dtype),
|
| 50 |
+
"dimensions": list(var.dims),
|
| 51 |
+
"shape": list(var.shape),
|
| 52 |
+
"attributes": {k: self._convert_type(v) for k, v in var.attrs.items()},
|
| 53 |
+
"min_value": min_val,
|
| 54 |
+
"max_value": max_val
|
| 55 |
+
})
|
| 56 |
+
|
| 57 |
+
# Coordinates
|
| 58 |
+
lat_names = ["lat", "latitude", "y"]
|
| 59 |
+
lon_names = ["lon", "longitude", "x"]
|
| 60 |
+
|
| 61 |
+
lat_coord = next((c for c in self.ds.coords if c.lower() in lat_names), None)
|
| 62 |
+
lon_coord = next((c for c in self.ds.coords if c.lower() in lon_names), None)
|
| 63 |
+
|
| 64 |
+
# Look for projection attributes or standard grid mappings
|
| 65 |
+
projection = global_attributes.get("projection") or global_attributes.get("grid_mapping_name")
|
| 66 |
+
for var in variables:
|
| 67 |
+
if "grid_mapping" in var["attributes"] or "grid_mapping_name" in var["attributes"]:
|
| 68 |
+
projection = var["attributes"].get("grid_mapping_name") or var["attributes"].get("grid_mapping") or "mapped"
|
| 69 |
+
break
|
| 70 |
+
|
| 71 |
+
coordinates = {
|
| 72 |
+
"latitude": lat_coord,
|
| 73 |
+
"longitude": lon_coord,
|
| 74 |
+
"projection": str(projection) if projection else None
|
| 75 |
+
}
|
| 76 |
+
|
| 77 |
+
# Temporal Information
|
| 78 |
+
time_names = ["time", "valid_time", "timestamp", "forecast_time"]
|
| 79 |
+
time_coord = next((c for c in self.ds.coords if c.lower() in time_names), None)
|
| 80 |
+
|
| 81 |
+
temporal_info = {
|
| 82 |
+
"start_time": None,
|
| 83 |
+
"end_time": None,
|
| 84 |
+
"time_steps": None
|
| 85 |
+
}
|
| 86 |
+
|
| 87 |
+
if time_coord is not None:
|
| 88 |
+
t_var = self.ds[time_coord]
|
| 89 |
+
temporal_info["time_steps"] = int(t_var.size)
|
| 90 |
+
if t_var.size > 0:
|
| 91 |
+
try:
|
| 92 |
+
temporal_info["start_time"] = str(t_var.values[0])
|
| 93 |
+
temporal_info["end_time"] = str(t_var.values[-1])
|
| 94 |
+
except Exception:
|
| 95 |
+
pass
|
| 96 |
+
|
| 97 |
+
return {
|
| 98 |
+
"global_attributes": global_attributes,
|
| 99 |
+
"dimensions": dimensions,
|
| 100 |
+
"variables": variables,
|
| 101 |
+
"coordinates": coordinates,
|
| 102 |
+
"temporal_info": temporal_info,
|
| 103 |
+
"variable_count": len(variables),
|
| 104 |
+
"dimension_count": len(dimensions),
|
| 105 |
+
"coordinate_count": len(self.ds.coords)
|
| 106 |
+
}
|
| 107 |
+
|
| 108 |
+
def close(self):
|
| 109 |
+
if self.ds is not None:
|
| 110 |
+
self.ds.close()
|
backend/app/services/scientific/visualization.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Placeholder
|
backend/app/services/upload_service.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import uuid
|
| 3 |
+
import shutil
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from fastapi import UploadFile, HTTPException
|
| 6 |
+
from typing import Dict, Any
|
| 7 |
+
|
| 8 |
+
UPLOAD_DIR = Path("storage/uploads")
|
| 9 |
+
|
| 10 |
+
ALLOWED_EXTENSIONS = {".nc", ".h5", ".hdf5"}
|
| 11 |
+
ALLOWED_MIME_TYPES = {
|
| 12 |
+
"application/x-netcdf",
|
| 13 |
+
"application/netcdf",
|
| 14 |
+
"application/x-hdf5",
|
| 15 |
+
"application/x-hdf",
|
| 16 |
+
"application/octet-stream",
|
| 17 |
+
}
|
| 18 |
+
|
| 19 |
+
MAX_FILE_SIZE = 100 * 1024 * 1024 # 100 MB
|
| 20 |
+
|
| 21 |
+
class UploadService:
|
| 22 |
+
@staticmethod
|
| 23 |
+
async def process_upload(file: UploadFile) -> Dict[str, Any]:
|
| 24 |
+
if not file.filename:
|
| 25 |
+
raise HTTPException(status_code=400, detail="Filename missing")
|
| 26 |
+
|
| 27 |
+
ext = Path(file.filename).suffix.lower()
|
| 28 |
+
if ext not in ALLOWED_EXTENSIONS:
|
| 29 |
+
raise HTTPException(status_code=400, detail=f"File extension {ext} not allowed. Supported: .nc, .h5, .hdf5")
|
| 30 |
+
|
| 31 |
+
if file.content_type not in ALLOWED_MIME_TYPES:
|
| 32 |
+
raise HTTPException(status_code=400, detail=f"Invalid MIME type: {file.content_type}")
|
| 33 |
+
|
| 34 |
+
# File size validation
|
| 35 |
+
file.file.seek(0, os.SEEK_END)
|
| 36 |
+
file_size = file.file.tell()
|
| 37 |
+
file.file.seek(0)
|
| 38 |
+
|
| 39 |
+
if file_size > MAX_FILE_SIZE:
|
| 40 |
+
raise HTTPException(status_code=400, detail="File size exceeds maximum allowed limit (100MB)")
|
| 41 |
+
|
| 42 |
+
UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
|
| 43 |
+
|
| 44 |
+
file_id = str(uuid.uuid4())
|
| 45 |
+
safe_filename = f"{file_id}{ext}"
|
| 46 |
+
file_path = UPLOAD_DIR / safe_filename
|
| 47 |
+
|
| 48 |
+
with open(file_path, "wb") as buffer:
|
| 49 |
+
shutil.copyfileobj(file.file, buffer)
|
| 50 |
+
|
| 51 |
+
return {
|
| 52 |
+
"success": True,
|
| 53 |
+
"fileId": file_id,
|
| 54 |
+
"filename": file.filename,
|
| 55 |
+
"status": "uploaded"
|
| 56 |
+
}
|
backend/app/storage/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Placeholder
|
backend/app/utils/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Placeholder
|
backend/requirements.txt
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Backend dependencies
|
| 2 |
+
fastapi>=0.100.0
|
| 3 |
+
uvicorn>=0.23.0
|
| 4 |
+
pydantic>=2.0.0
|
| 5 |
+
python-multipart>=0.0.6
|
| 6 |
+
xarray>=2023.0.0
|
| 7 |
+
netCDF4>=1.6.0
|
| 8 |
+
h5py>=3.8.0
|
| 9 |
+
h5netcdf>=1.1.0
|
| 10 |
+
pytest>=7.0.0
|
backend/tests/test_metadata.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import tempfile
|
| 3 |
+
import pytest
|
| 4 |
+
import h5py
|
| 5 |
+
import netCDF4 as nc
|
| 6 |
+
import numpy as np
|
| 7 |
+
from fastapi.testclient import TestClient
|
| 8 |
+
|
| 9 |
+
from app.main import app
|
| 10 |
+
from app.services.scientific.metadata_service import MetadataService
|
| 11 |
+
from app.schemas.metadata import MetadataResponse
|
| 12 |
+
|
| 13 |
+
client = TestClient(app)
|
| 14 |
+
|
| 15 |
+
@pytest.fixture
|
| 16 |
+
def mock_netcdf():
|
| 17 |
+
fd, path = tempfile.mkstemp(suffix=".nc")
|
| 18 |
+
os.close(fd)
|
| 19 |
+
|
| 20 |
+
# Create a mock netcdf file
|
| 21 |
+
ds = nc.Dataset(path, 'w', format='NETCDF4')
|
| 22 |
+
ds.title = "Mock NetCDF"
|
| 23 |
+
ds.projection = "mercator"
|
| 24 |
+
|
| 25 |
+
ds.createDimension('time', None) # unlimited
|
| 26 |
+
ds.createDimension('lat', 10)
|
| 27 |
+
ds.createDimension('lon', 10)
|
| 28 |
+
|
| 29 |
+
times = ds.createVariable('time', 'f8', ('time',))
|
| 30 |
+
lats = ds.createVariable('lat', 'f4', ('lat',))
|
| 31 |
+
lons = ds.createVariable('lon', 'f4', ('lon',))
|
| 32 |
+
temp = ds.createVariable('temperature', 'f4', ('time', 'lat', 'lon',))
|
| 33 |
+
|
| 34 |
+
lats.units = 'degrees_north'
|
| 35 |
+
lons.units = 'degrees_east'
|
| 36 |
+
temp.units = 'K'
|
| 37 |
+
temp.grid_mapping = 'mercator'
|
| 38 |
+
|
| 39 |
+
times[:] = [1620000000, 1620003600] # two time steps
|
| 40 |
+
lats[:] = np.linspace(-90, 90, 10)
|
| 41 |
+
lons[:] = np.linspace(-180, 180, 10)
|
| 42 |
+
temp[:] = np.random.uniform(200, 300, size=(2, 10, 10))
|
| 43 |
+
|
| 44 |
+
ds.close()
|
| 45 |
+
|
| 46 |
+
yield path
|
| 47 |
+
os.remove(path)
|
| 48 |
+
|
| 49 |
+
@pytest.fixture
|
| 50 |
+
def mock_hdf5():
|
| 51 |
+
fd, path = tempfile.mkstemp(suffix=".h5")
|
| 52 |
+
os.close(fd)
|
| 53 |
+
|
| 54 |
+
with h5py.File(path, "w") as f:
|
| 55 |
+
f.attrs["title"] = "Mock HDF5"
|
| 56 |
+
f.attrs["projection"] = "geospatial"
|
| 57 |
+
|
| 58 |
+
# Dimensions
|
| 59 |
+
lat_dim = f.create_dataset("latitude", data=np.linspace(-90, 90, 10))
|
| 60 |
+
lat_dim.attrs["CLASS"] = "DIMENSION_SCALE"
|
| 61 |
+
|
| 62 |
+
lon_dim = f.create_dataset("longitude", data=np.linspace(-180, 180, 10))
|
| 63 |
+
lon_dim.attrs["CLASS"] = "DIMENSION_SCALE"
|
| 64 |
+
|
| 65 |
+
time_dim = f.create_dataset("time", data=np.array([1620000000, 1620003600]))
|
| 66 |
+
time_dim.attrs["CLASS"] = "DIMENSION_SCALE"
|
| 67 |
+
|
| 68 |
+
# Variable
|
| 69 |
+
temp = f.create_dataset("temperature", data=np.random.uniform(200, 300, size=(2, 10, 10)))
|
| 70 |
+
temp.dims[0].label = "time"
|
| 71 |
+
temp.dims[1].label = "lat"
|
| 72 |
+
temp.dims[2].label = "lon"
|
| 73 |
+
temp.attrs["units"] = "K"
|
| 74 |
+
|
| 75 |
+
yield path
|
| 76 |
+
os.remove(path)
|
| 77 |
+
|
| 78 |
+
def test_extract_netcdf_metadata(mock_netcdf):
|
| 79 |
+
metadata = MetadataService.extract_metadata("test_nc", mock_netcdf)
|
| 80 |
+
assert metadata.format == "nc"
|
| 81 |
+
assert metadata.summary.file_format == "nc"
|
| 82 |
+
assert metadata.coordinates.latitude == "lat"
|
| 83 |
+
assert metadata.coordinates.longitude == "lon"
|
| 84 |
+
assert metadata.coordinates.projection == "mercator"
|
| 85 |
+
assert metadata.temporal_info.time_steps == 2
|
| 86 |
+
assert "time" in [d.name for d in metadata.dimensions]
|
| 87 |
+
assert "temperature" in [v.name for v in metadata.variables]
|
| 88 |
+
|
| 89 |
+
def test_extract_hdf5_metadata(mock_hdf5):
|
| 90 |
+
metadata = MetadataService.extract_metadata("test_h5", mock_hdf5)
|
| 91 |
+
assert metadata.format == "h5"
|
| 92 |
+
assert metadata.coordinates.latitude == "latitude"
|
| 93 |
+
assert metadata.coordinates.longitude == "longitude"
|
| 94 |
+
assert metadata.coordinates.projection == "geospatial"
|
| 95 |
+
assert metadata.temporal_info.time_steps == 2
|
| 96 |
+
assert "temperature" in [v.name for v in metadata.variables]
|
| 97 |
+
|
| 98 |
+
def test_metadata_api_not_found():
|
| 99 |
+
response = client.get("/api/v1/metadata/non_existent_file")
|
| 100 |
+
assert response.status_code == 404
|
| 101 |
+
|
| 102 |
+
def test_metadata_service_invalid_ext():
|
| 103 |
+
with pytest.raises(ValueError):
|
| 104 |
+
MetadataService.get_parser("test.txt")
|