Monaljain commited on
Commit
4d6f282
·
1 Parent(s): 9f33565

added backend logic for upload and metadata module

Browse files
Files changed (43) hide show
  1. backend/app/api/__init__.py +1 -0
  2. backend/app/api/animation.py +14 -0
  3. backend/app/api/export.py +24 -0
  4. backend/app/api/health.py +16 -0
  5. backend/app/api/interpolation.py +28 -0
  6. backend/app/api/metadata.py +34 -0
  7. backend/app/api/metrics.py +14 -0
  8. backend/app/api/upload.py +10 -0
  9. backend/app/api/visualization.py +22 -0
  10. backend/app/core/__init__.py +1 -0
  11. backend/app/core/config.py +1 -0
  12. backend/app/core/exceptions.py +1 -0
  13. backend/app/jobs/__init__.py +1 -0
  14. backend/app/jobs/cleanup_job.py +1 -0
  15. backend/app/jobs/export_job.py +1 -0
  16. backend/app/jobs/interpolation_job.py +1 -0
  17. backend/app/main.py +32 -0
  18. backend/app/schemas/__init__.py +1 -0
  19. backend/app/schemas/animation.py +7 -0
  20. backend/app/schemas/common.py +18 -0
  21. backend/app/schemas/export.py +15 -0
  22. backend/app/schemas/interpolation.py +17 -0
  23. backend/app/schemas/metadata.py +44 -0
  24. backend/app/schemas/metrics.py +12 -0
  25. backend/app/schemas/upload.py +10 -0
  26. backend/app/schemas/visualization.py +7 -0
  27. backend/app/services/__init__.py +1 -0
  28. backend/app/services/inference/__init__.py +1 -0
  29. backend/app/services/inference/model_loader.py +1 -0
  30. backend/app/services/inference/onnx_runtime.py +1 -0
  31. backend/app/services/inference/rife.py +1 -0
  32. backend/app/services/scientific/__init__.py +1 -0
  33. backend/app/services/scientific/base_parser.py +11 -0
  34. backend/app/services/scientific/hdf_parser.py +156 -0
  35. backend/app/services/scientific/metadata_service.py +64 -0
  36. backend/app/services/scientific/metrics.py +1 -0
  37. backend/app/services/scientific/netcdf_parser.py +110 -0
  38. backend/app/services/scientific/visualization.py +1 -0
  39. backend/app/services/upload_service.py +56 -0
  40. backend/app/storage/__init__.py +1 -0
  41. backend/app/utils/__init__.py +1 -0
  42. backend/requirements.txt +10 -0
  43. backend/tests/test_metadata.py +104 -0
backend/app/api/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # Placeholder
backend/app/api/animation.py ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import APIRouter
2
+ from app.schemas.common import ApiResponse
3
+ from app.schemas.animation import AnimationSequenceResponse
4
+
5
+ router = APIRouter()
6
+
7
+ @router.get("/{file_id}/sequence", response_model=ApiResponse)
8
+ async def get_sequence(file_id: str, start_time: str = "", end_time: str = "", fps: int = 10):
9
+ # Placeholder
10
+ return ApiResponse(
11
+ success=True,
12
+ message="Animation sequence retrieved",
13
+ data=None
14
+ )
backend/app/api/export.py ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import APIRouter
2
+ from app.schemas.common import ApiResponse
3
+ from app.schemas.export import ExportRequest, ExportJobStatusResponse
4
+ from app.schemas.interpolation import JobResponse
5
+
6
+ router = APIRouter()
7
+
8
+ @router.post("/job", response_model=ApiResponse)
9
+ async def create_export_job(request: ExportRequest):
10
+ # Placeholder
11
+ return ApiResponse(
12
+ success=True,
13
+ message="Export job queued",
14
+ data=JobResponse(job_id="dummy-export-job", status="preparing")
15
+ )
16
+
17
+ @router.get("/status/{job_id}", response_model=ApiResponse)
18
+ async def get_export_status(job_id: str):
19
+ # Placeholder
20
+ return ApiResponse(
21
+ success=True,
22
+ message="Export status retrieved",
23
+ data=ExportJobStatusResponse(status="processing", progress=10.0)
24
+ )
backend/app/api/health.py ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import APIRouter
2
+ from app.schemas.common import ApiResponse
3
+
4
+ router = APIRouter()
5
+
6
+ @router.get("/")
7
+ async def health_check():
8
+ return ApiResponse(success=True, message="API is healthy", data=None)
9
+
10
+ @router.get("/gpu")
11
+ async def health_gpu():
12
+ return ApiResponse(success=True, message="GPU status", data={"cuda_available": False})
13
+
14
+ @router.get("/models")
15
+ async def health_models():
16
+ return ApiResponse(success=True, message="Models status", data={"rife": "not_loaded"})
backend/app/api/interpolation.py ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import APIRouter
2
+ from app.schemas.common import ApiResponse
3
+ from app.schemas.interpolation import InterpolationRequest, JobResponse, JobStatusResponse
4
+
5
+ router = APIRouter()
6
+
7
+ @router.post("/generate", response_model=ApiResponse)
8
+ async def generate_interpolation(request: InterpolationRequest):
9
+ # Placeholder
10
+ return ApiResponse(
11
+ success=True,
12
+ message="Job queued",
13
+ data=JobResponse(job_id="dummy-job", status="queued")
14
+ )
15
+
16
+ @router.get("/status/{job_id}", response_model=ApiResponse)
17
+ async def get_status(job_id: str):
18
+ # Placeholder
19
+ return ApiResponse(
20
+ success=True,
21
+ message="Status retrieved",
22
+ data=JobStatusResponse(status="processing", progress=50.0)
23
+ )
24
+
25
+ @router.get("/events/{job_id}")
26
+ async def get_events(job_id: str):
27
+ # Placeholder for SSE
28
+ return {"message": "SSE endpoint placeholder"}
backend/app/api/metadata.py ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import APIRouter, HTTPException
2
+ from app.schemas.common import ApiResponse
3
+ from app.services.scientific.metadata_service import MetadataService
4
+ import os
5
+
6
+ router = APIRouter()
7
+
8
+ # Note: Depending on where uploads are stored, we assume a storage path.
9
+ # Assuming standard backend/storage/uploads based on previous contexts.
10
+ STORAGE_DIR = os.path.join(os.getcwd(), "storage", "uploads")
11
+
12
+ @router.get("/{file_id}", response_model=ApiResponse)
13
+ async def get_metadata(file_id: str):
14
+ # Search for the file in the storage directory
15
+ file_path = None
16
+ for ext in [".nc", ".h5", ".hdf5"]:
17
+ possible_path = os.path.join(STORAGE_DIR, f"{file_id}{ext}")
18
+ if os.path.exists(possible_path):
19
+ file_path = possible_path
20
+ break
21
+
22
+ if not file_path:
23
+ raise HTTPException(status_code=404, detail="File not found")
24
+
25
+ try:
26
+ metadata_response = MetadataService.extract_metadata(file_id, file_path)
27
+ return ApiResponse(
28
+ success=True,
29
+ message="Metadata retrieved successfully",
30
+ data=metadata_response
31
+ )
32
+ except Exception as e:
33
+ raise HTTPException(status_code=500, detail=f"Failed to extract metadata: {str(e)}")
34
+
backend/app/api/metrics.py ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import APIRouter
2
+ from app.schemas.common import ApiResponse
3
+ from app.schemas.metrics import MetricsRequest, MetricsResponse
4
+
5
+ router = APIRouter()
6
+
7
+ @router.post("/compare", response_model=ApiResponse)
8
+ async def compare_frames(request: MetricsRequest):
9
+ # Placeholder
10
+ return ApiResponse(
11
+ success=True,
12
+ message="Metrics computed",
13
+ data=None
14
+ )
backend/app/api/upload.py ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import APIRouter, UploadFile, File
2
+ from app.schemas.upload import UploadResponse
3
+ from app.services.upload_service import UploadService
4
+
5
+ router = APIRouter()
6
+
7
+ @router.post("", response_model=UploadResponse)
8
+ async def upload_file(file: UploadFile = File(...)):
9
+ result = await UploadService.process_upload(file)
10
+ return result
backend/app/api/visualization.py ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import APIRouter
2
+ from app.schemas.common import ApiResponse
3
+
4
+ router = APIRouter()
5
+
6
+ @router.get("/{file_id}/frame", response_model=ApiResponse)
7
+ async def get_frame(file_id: str, time_index: int = 0, variable: str = ""):
8
+ # Placeholder
9
+ return ApiResponse(
10
+ success=True,
11
+ message="Frame retrieved",
12
+ data=None
13
+ )
14
+
15
+ @router.get("/{file_id}/thumbnail", response_model=ApiResponse)
16
+ async def get_thumbnail(file_id: str):
17
+ # Placeholder
18
+ return ApiResponse(
19
+ success=True,
20
+ message="Thumbnail retrieved",
21
+ data={"url": "dummy_url"}
22
+ )
backend/app/core/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # Placeholder
backend/app/core/config.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # Placeholder
backend/app/core/exceptions.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # Placeholder
backend/app/jobs/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # Placeholder
backend/app/jobs/cleanup_job.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # Placeholder
backend/app/jobs/export_job.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # Placeholder
backend/app/jobs/interpolation_job.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # Placeholder
backend/app/main.py ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import FastAPI
2
+ from fastapi.middleware.cors import CORSMiddleware
3
+ from .api import upload, metadata, visualization, interpolation, metrics, animation, export, health
4
+
5
+ app = FastAPI(
6
+ title="Fill the Frames API",
7
+ description="Scientific backend for satellite frame interpolation and visualization",
8
+ version="0.1.0"
9
+ )
10
+
11
+ # CORS configuration
12
+ app.add_middleware(
13
+ CORSMiddleware,
14
+ allow_origins=["*"], # Update for production
15
+ allow_credentials=True,
16
+ allow_methods=["*"],
17
+ allow_headers=["*"],
18
+ )
19
+
20
+ # Register routers
21
+ app.include_router(health.router, tags=["Health"])
22
+ app.include_router(upload.router, prefix="/api/v1/upload", tags=["Upload"])
23
+ app.include_router(metadata.router, prefix="/api/v1/metadata", tags=["Metadata"])
24
+ app.include_router(visualization.router, prefix="/api/v1/visualization", tags=["Visualization"])
25
+ app.include_router(interpolation.router, prefix="/api/v1/interpolation", tags=["Interpolation"])
26
+ app.include_router(metrics.router, prefix="/api/v1/metrics", tags=["Metrics"])
27
+ app.include_router(animation.router, prefix="/api/v1/animation", tags=["Animation"])
28
+ app.include_router(export.router, prefix="/api/v1/export", tags=["Export"])
29
+
30
+ @app.get("/")
31
+ def read_root():
32
+ return {"message": "Welcome to Fill the Frames API"}
backend/app/schemas/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # Placeholder
backend/app/schemas/animation.py ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ from pydantic import BaseModel
2
+ from typing import List
3
+ from .visualization import FrameDataResponse
4
+
5
+ class AnimationSequenceResponse(BaseModel):
6
+ frames: List[FrameDataResponse]
7
+ total_frames: int
backend/app/schemas/common.py ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from pydantic import BaseModel
2
+ from typing import Any
3
+ from datetime import datetime
4
+
5
+ class ApiResponse(BaseModel):
6
+ success: bool
7
+ message: str
8
+ data: Any
9
+
10
+ class FrameData(BaseModel):
11
+ frame_id: str
12
+ timestamp: datetime
13
+ variable: str
14
+ width: int
15
+ height: int
16
+ min_value: float
17
+ max_value: float
18
+ source: str
backend/app/schemas/export.py ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from pydantic import BaseModel
2
+ from typing import Optional
3
+
4
+ class ExportRequest(BaseModel):
5
+ target_id: str
6
+ format: str
7
+ resolution: str
8
+ include_metadata: bool
9
+ include_metrics: bool
10
+ include_animation: bool = False
11
+
12
+ class ExportJobStatusResponse(BaseModel):
13
+ status: str
14
+ progress: float
15
+ download_url: Optional[str] = None
backend/app/schemas/interpolation.py ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from pydantic import BaseModel
2
+ from typing import Optional
3
+ from .visualization import FrameDataResponse
4
+
5
+ class InterpolationRequest(BaseModel):
6
+ file_id: str
7
+ time_ratio: float
8
+ model: str
9
+
10
+ class JobResponse(BaseModel):
11
+ job_id: str
12
+ status: str
13
+
14
+ class JobStatusResponse(BaseModel):
15
+ status: str
16
+ progress: float
17
+ result_frame: Optional[FrameDataResponse] = None
backend/app/schemas/metadata.py ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from pydantic import BaseModel
2
+ from typing import List, Dict, Any, Optional
3
+
4
+ class DimensionInfo(BaseModel):
5
+ name: str
6
+ size: int
7
+
8
+ class VariableInfo(BaseModel):
9
+ name: str
10
+ datatype: str
11
+ dimensions: List[str]
12
+ shape: List[int]
13
+ attributes: Dict[str, Any]
14
+ min_value: Optional[float] = None
15
+ max_value: Optional[float] = None
16
+
17
+ class CoordinateInfo(BaseModel):
18
+ latitude: Optional[str] = None
19
+ longitude: Optional[str] = None
20
+ projection: Optional[str] = None
21
+
22
+ class TemporalInfo(BaseModel):
23
+ start_time: Optional[str] = None
24
+ end_time: Optional[str] = None
25
+ time_steps: Optional[int] = None
26
+
27
+ class DatasetSummary(BaseModel):
28
+ file_format: str
29
+ variable_count: int
30
+ dimension_count: int
31
+ coordinate_count: int
32
+ dataset_size: int
33
+
34
+ class MetadataResponse(BaseModel):
35
+ file_id: str
36
+ filename: str
37
+ size: int
38
+ format: str
39
+ global_attributes: Dict[str, Any]
40
+ dimensions: List[DimensionInfo]
41
+ variables: List[VariableInfo]
42
+ coordinates: CoordinateInfo
43
+ temporal_info: TemporalInfo
44
+ summary: DatasetSummary
backend/app/schemas/metrics.py ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from pydantic import BaseModel
2
+ from typing import List, Dict
3
+ from .visualization import FrameDataResponse
4
+
5
+ class MetricsRequest(BaseModel):
6
+ frame_a_id: str
7
+ frame_b_id: str
8
+ metrics: List[str]
9
+
10
+ class MetricsResponse(BaseModel):
11
+ scores: Dict[str, float]
12
+ difference_map: FrameDataResponse
backend/app/schemas/upload.py ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ from pydantic import BaseModel, Field
2
+
3
+ class UploadResponse(BaseModel):
4
+ success: bool = True
5
+ fileId: str = Field(..., alias="fileId")
6
+ filename: str
7
+ status: str
8
+
9
+ class Config:
10
+ populate_by_name = True
backend/app/schemas/visualization.py ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ from pydantic import BaseModel
2
+ from typing import List
3
+ from .common import FrameData
4
+
5
+ class FrameDataResponse(BaseModel):
6
+ frame_metadata: FrameData
7
+ z: List[List[float]]
backend/app/services/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # Placeholder
backend/app/services/inference/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # Placeholder
backend/app/services/inference/model_loader.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # Placeholder
backend/app/services/inference/onnx_runtime.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # Placeholder
backend/app/services/inference/rife.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # Placeholder
backend/app/services/scientific/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # Placeholder
backend/app/services/scientific/base_parser.py ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from abc import ABC, abstractmethod
2
+ from typing import Dict, Any
3
+
4
+ class BaseDatasetParser(ABC):
5
+ @abstractmethod
6
+ def load_dataset(self, file_path: str):
7
+ pass
8
+
9
+ @abstractmethod
10
+ def extract_metadata(self) -> Dict[str, Any]:
11
+ pass
backend/app/services/scientific/hdf_parser.py ADDED
@@ -0,0 +1,156 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import h5py
2
+ import numpy as np
3
+ from typing import Dict, Any, List
4
+ from .base_parser import BaseDatasetParser
5
+
6
+ class HDFParser(BaseDatasetParser):
7
+ def __init__(self):
8
+ self.file = None
9
+
10
+ def load_dataset(self, file_path: str):
11
+ self.file = h5py.File(file_path, "r")
12
+
13
+ def _convert_type(self, val):
14
+ if isinstance(val, (np.integer, int)):
15
+ return int(val)
16
+ if isinstance(val, (np.floating, float)):
17
+ return float(val)
18
+ if isinstance(val, np.ndarray):
19
+ if val.size == 1:
20
+ return self._convert_type(val.item())
21
+ return [self._convert_type(v) for v in val.tolist()]
22
+ if isinstance(val, bytes):
23
+ try:
24
+ return val.decode('utf-8')
25
+ except Exception:
26
+ return str(val)
27
+ return str(val)
28
+
29
+ def extract_metadata(self) -> Dict[str, Any]:
30
+ if self.file is None:
31
+ raise ValueError("Dataset not loaded")
32
+
33
+ # Global Attributes
34
+ global_attributes = {k: self._convert_type(v) for k, v in self.file.attrs.items()}
35
+
36
+ variables = []
37
+ dimensions_map = {}
38
+ coordinate_names = []
39
+
40
+ # Helper to traverse HDF5 recursively
41
+ def visit_func(name, node):
42
+ if isinstance(node, h5py.Dataset):
43
+ var_attrs = {k: self._convert_type(v) for k, v in node.attrs.items()}
44
+
45
+ # Check if it's a dimension scale (often used as coordinates in HDF5/netCDF4-hdf5)
46
+ is_scale = False
47
+ if "CLASS" in var_attrs and var_attrs["CLASS"] == "DIMENSION_SCALE":
48
+ is_scale = True
49
+ coordinate_names.append(name.split('/')[-1])
50
+
51
+ # Extract dimension sizes
52
+ dims = []
53
+ for i, dim in enumerate(node.dims):
54
+ if len(dim) > 0 and dim[0].name:
55
+ dim_name = dim[0].name.split('/')[-1]
56
+ dims.append(dim_name)
57
+ dimensions_map[dim_name] = node.shape[i]
58
+ else:
59
+ dim_name = f"{name}_dim_{i}"
60
+ dims.append(dim_name)
61
+ dimensions_map[dim_name] = node.shape[i]
62
+
63
+ if not dims:
64
+ # If dims aren't formally defined, make placeholder names
65
+ for i, s in enumerate(node.shape):
66
+ dim_name = f"dim_{i}"
67
+ dims.append(dim_name)
68
+ dimensions_map[dim_name] = s
69
+
70
+ min_val = None
71
+ max_val = None
72
+ if node.size > 0 and node.size < 100000:
73
+ try:
74
+ # Only read small datasets to avoid memory issues
75
+ data = node[:]
76
+ min_val = float(np.min(data))
77
+ max_val = float(np.max(data))
78
+ except Exception:
79
+ pass
80
+
81
+ variables.append({
82
+ "name": name,
83
+ "datatype": str(node.dtype),
84
+ "dimensions": dims,
85
+ "shape": list(node.shape),
86
+ "attributes": var_attrs,
87
+ "min_value": min_val,
88
+ "max_value": max_val
89
+ })
90
+
91
+ self.file.visititems(visit_func)
92
+
93
+ dimensions = [{"name": str(k), "size": int(v)} for k, v in dimensions_map.items()]
94
+
95
+ # Coordinates
96
+ lat_names = ["lat", "latitude", "y", "geolocation/latitude"]
97
+ lon_names = ["lon", "longitude", "x", "geolocation/longitude"]
98
+
99
+ lat_coord = next((c for c in coordinate_names if c.lower() in lat_names), None)
100
+ lon_coord = next((c for c in coordinate_names if c.lower() in lon_names), None)
101
+
102
+ # Sometimes lat/lon are just variables not marked as scales
103
+ if not lat_coord:
104
+ lat_coord = next((v["name"] for v in variables if v["name"].split('/')[-1].lower() in lat_names), None)
105
+ if not lon_coord:
106
+ lon_coord = next((v["name"] for v in variables if v["name"].split('/')[-1].lower() in lon_names), None)
107
+
108
+ projection = global_attributes.get("projection") or global_attributes.get("grid_mapping_name")
109
+ for var in variables:
110
+ if "grid_mapping" in var["attributes"] or "grid_mapping_name" in var["attributes"]:
111
+ projection = var["attributes"].get("grid_mapping_name") or "mapped"
112
+ break
113
+
114
+ coordinates = {
115
+ "latitude": lat_coord,
116
+ "longitude": lon_coord,
117
+ "projection": str(projection) if projection else None
118
+ }
119
+
120
+ # Temporal Information
121
+ time_names = ["time", "valid_time", "timestamp", "forecast_time"]
122
+ time_coord_name = next((c for c in coordinate_names if c.lower() in time_names), None)
123
+ if not time_coord_name:
124
+ time_coord_name = next((v["name"] for v in variables if v["name"].split('/')[-1].lower() in time_names), None)
125
+
126
+ temporal_info = {
127
+ "start_time": None,
128
+ "end_time": None,
129
+ "time_steps": None
130
+ }
131
+
132
+ if time_coord_name and time_coord_name in self.file:
133
+ t_var = self.file[time_coord_name]
134
+ temporal_info["time_steps"] = int(t_var.size)
135
+ if t_var.size > 0:
136
+ try:
137
+ data = t_var[:]
138
+ temporal_info["start_time"] = str(data[0])
139
+ temporal_info["end_time"] = str(data[-1])
140
+ except Exception:
141
+ pass
142
+
143
+ return {
144
+ "global_attributes": global_attributes,
145
+ "dimensions": dimensions,
146
+ "variables": variables,
147
+ "coordinates": coordinates,
148
+ "temporal_info": temporal_info,
149
+ "variable_count": len(variables),
150
+ "dimension_count": len(dimensions),
151
+ "coordinate_count": len(coordinate_names)
152
+ }
153
+
154
+ def close(self):
155
+ if self.file is not None:
156
+ self.file.close()
backend/app/services/scientific/metadata_service.py ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import logging
3
+ from typing import Dict, Any
4
+
5
+ from app.schemas.metadata import MetadataResponse, DimensionInfo, VariableInfo, CoordinateInfo, TemporalInfo, DatasetSummary
6
+ from .netcdf_parser import NetCDFParser
7
+ from .hdf_parser import HDFParser
8
+
9
+ logger = logging.getLogger(__name__)
10
+
11
+ class MetadataService:
12
+ @staticmethod
13
+ def get_parser(file_path: str):
14
+ ext = os.path.splitext(file_path)[1].lower()
15
+ if ext == ".nc":
16
+ return NetCDFParser()
17
+ elif ext in [".h5", ".hdf5"]:
18
+ return HDFParser()
19
+ else:
20
+ raise ValueError(f"Unsupported file extension: {ext}")
21
+
22
+ @staticmethod
23
+ def extract_metadata(file_id: str, file_path: str) -> MetadataResponse:
24
+ logger.info(f"Opening dataset {file_id} at {file_path}")
25
+ parser = None
26
+ try:
27
+ parser = MetadataService.get_parser(file_path)
28
+ parser.load_dataset(file_path)
29
+
30
+ raw_metadata = parser.extract_metadata()
31
+
32
+ file_size = os.path.getsize(file_path)
33
+ file_format = os.path.splitext(file_path)[1].lower().strip(".")
34
+
35
+ summary = DatasetSummary(
36
+ file_format=file_format,
37
+ variable_count=raw_metadata["variable_count"],
38
+ dimension_count=raw_metadata["dimension_count"],
39
+ coordinate_count=raw_metadata["coordinate_count"],
40
+ dataset_size=file_size
41
+ )
42
+
43
+ response = MetadataResponse(
44
+ file_id=file_id,
45
+ filename=os.path.basename(file_path),
46
+ size=file_size,
47
+ format=file_format,
48
+ global_attributes=raw_metadata["global_attributes"],
49
+ dimensions=[DimensionInfo(**d) for d in raw_metadata["dimensions"]],
50
+ variables=[VariableInfo(**v) for v in raw_metadata["variables"]],
51
+ coordinates=CoordinateInfo(**raw_metadata["coordinates"]),
52
+ temporal_info=TemporalInfo(**raw_metadata["temporal_info"]),
53
+ summary=summary
54
+ )
55
+
56
+ logger.info(f"Metadata extracted successfully for {file_id}")
57
+ return response
58
+
59
+ except Exception as e:
60
+ logger.error(f"Metadata extraction failed for {file_id}: {str(e)}")
61
+ raise e
62
+ finally:
63
+ if parser is not None:
64
+ parser.close()
backend/app/services/scientific/metrics.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # Placeholder
backend/app/services/scientific/netcdf_parser.py ADDED
@@ -0,0 +1,110 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import xarray as xr
2
+ import numpy as np
3
+ from typing import Dict, Any
4
+ from .base_parser import BaseDatasetParser
5
+
6
+ class NetCDFParser(BaseDatasetParser):
7
+ def __init__(self):
8
+ self.ds = None
9
+
10
+ def load_dataset(self, file_path: str):
11
+ self.ds = xr.open_dataset(file_path, engine="netcdf4")
12
+
13
+ def _convert_type(self, val):
14
+ if isinstance(val, (np.integer, int)):
15
+ return int(val)
16
+ if isinstance(val, (np.floating, float)):
17
+ return float(val)
18
+ if isinstance(val, np.ndarray):
19
+ return val.tolist()
20
+ return str(val)
21
+
22
+ def extract_metadata(self) -> Dict[str, Any]:
23
+ if self.ds is None:
24
+ raise ValueError("Dataset not loaded")
25
+
26
+ # Global Attributes
27
+ global_attributes = {k: self._convert_type(v) for k, v in self.ds.attrs.items()}
28
+
29
+ # Dimensions
30
+ dimensions = [{"name": str(k), "size": int(v)} for k, v in self.ds.sizes.items()]
31
+
32
+ # Variables
33
+ variables = []
34
+ for name, var in self.ds.variables.items():
35
+ # Skip if it's purely a coordinate (optional, but requested all variables? Usually we list all)
36
+ min_val = None
37
+ max_val = None
38
+ # Only calculate min/max if it's not a huge dataset or inexpensive. For metadata, we can sample or ignore.
39
+ # To be safe and inexpensive, we might just skip or do min/max on small arrays.
40
+ if var.size < 100000: # arbitrary small limit
41
+ try:
42
+ min_val = float(var.min().values)
43
+ max_val = float(var.max().values)
44
+ except Exception:
45
+ pass
46
+
47
+ variables.append({
48
+ "name": str(name),
49
+ "datatype": str(var.dtype),
50
+ "dimensions": list(var.dims),
51
+ "shape": list(var.shape),
52
+ "attributes": {k: self._convert_type(v) for k, v in var.attrs.items()},
53
+ "min_value": min_val,
54
+ "max_value": max_val
55
+ })
56
+
57
+ # Coordinates
58
+ lat_names = ["lat", "latitude", "y"]
59
+ lon_names = ["lon", "longitude", "x"]
60
+
61
+ lat_coord = next((c for c in self.ds.coords if c.lower() in lat_names), None)
62
+ lon_coord = next((c for c in self.ds.coords if c.lower() in lon_names), None)
63
+
64
+ # Look for projection attributes or standard grid mappings
65
+ projection = global_attributes.get("projection") or global_attributes.get("grid_mapping_name")
66
+ for var in variables:
67
+ if "grid_mapping" in var["attributes"] or "grid_mapping_name" in var["attributes"]:
68
+ projection = var["attributes"].get("grid_mapping_name") or var["attributes"].get("grid_mapping") or "mapped"
69
+ break
70
+
71
+ coordinates = {
72
+ "latitude": lat_coord,
73
+ "longitude": lon_coord,
74
+ "projection": str(projection) if projection else None
75
+ }
76
+
77
+ # Temporal Information
78
+ time_names = ["time", "valid_time", "timestamp", "forecast_time"]
79
+ time_coord = next((c for c in self.ds.coords if c.lower() in time_names), None)
80
+
81
+ temporal_info = {
82
+ "start_time": None,
83
+ "end_time": None,
84
+ "time_steps": None
85
+ }
86
+
87
+ if time_coord is not None:
88
+ t_var = self.ds[time_coord]
89
+ temporal_info["time_steps"] = int(t_var.size)
90
+ if t_var.size > 0:
91
+ try:
92
+ temporal_info["start_time"] = str(t_var.values[0])
93
+ temporal_info["end_time"] = str(t_var.values[-1])
94
+ except Exception:
95
+ pass
96
+
97
+ return {
98
+ "global_attributes": global_attributes,
99
+ "dimensions": dimensions,
100
+ "variables": variables,
101
+ "coordinates": coordinates,
102
+ "temporal_info": temporal_info,
103
+ "variable_count": len(variables),
104
+ "dimension_count": len(dimensions),
105
+ "coordinate_count": len(self.ds.coords)
106
+ }
107
+
108
+ def close(self):
109
+ if self.ds is not None:
110
+ self.ds.close()
backend/app/services/scientific/visualization.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # Placeholder
backend/app/services/upload_service.py ADDED
@@ -0,0 +1,56 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import uuid
3
+ import shutil
4
+ from pathlib import Path
5
+ from fastapi import UploadFile, HTTPException
6
+ from typing import Dict, Any
7
+
8
+ UPLOAD_DIR = Path("storage/uploads")
9
+
10
+ ALLOWED_EXTENSIONS = {".nc", ".h5", ".hdf5"}
11
+ ALLOWED_MIME_TYPES = {
12
+ "application/x-netcdf",
13
+ "application/netcdf",
14
+ "application/x-hdf5",
15
+ "application/x-hdf",
16
+ "application/octet-stream",
17
+ }
18
+
19
+ MAX_FILE_SIZE = 100 * 1024 * 1024 # 100 MB
20
+
21
+ class UploadService:
22
+ @staticmethod
23
+ async def process_upload(file: UploadFile) -> Dict[str, Any]:
24
+ if not file.filename:
25
+ raise HTTPException(status_code=400, detail="Filename missing")
26
+
27
+ ext = Path(file.filename).suffix.lower()
28
+ if ext not in ALLOWED_EXTENSIONS:
29
+ raise HTTPException(status_code=400, detail=f"File extension {ext} not allowed. Supported: .nc, .h5, .hdf5")
30
+
31
+ if file.content_type not in ALLOWED_MIME_TYPES:
32
+ raise HTTPException(status_code=400, detail=f"Invalid MIME type: {file.content_type}")
33
+
34
+ # File size validation
35
+ file.file.seek(0, os.SEEK_END)
36
+ file_size = file.file.tell()
37
+ file.file.seek(0)
38
+
39
+ if file_size > MAX_FILE_SIZE:
40
+ raise HTTPException(status_code=400, detail="File size exceeds maximum allowed limit (100MB)")
41
+
42
+ UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
43
+
44
+ file_id = str(uuid.uuid4())
45
+ safe_filename = f"{file_id}{ext}"
46
+ file_path = UPLOAD_DIR / safe_filename
47
+
48
+ with open(file_path, "wb") as buffer:
49
+ shutil.copyfileobj(file.file, buffer)
50
+
51
+ return {
52
+ "success": True,
53
+ "fileId": file_id,
54
+ "filename": file.filename,
55
+ "status": "uploaded"
56
+ }
backend/app/storage/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # Placeholder
backend/app/utils/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # Placeholder
backend/requirements.txt ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ # Backend dependencies
2
+ fastapi>=0.100.0
3
+ uvicorn>=0.23.0
4
+ pydantic>=2.0.0
5
+ python-multipart>=0.0.6
6
+ xarray>=2023.0.0
7
+ netCDF4>=1.6.0
8
+ h5py>=3.8.0
9
+ h5netcdf>=1.1.0
10
+ pytest>=7.0.0
backend/tests/test_metadata.py ADDED
@@ -0,0 +1,104 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import tempfile
3
+ import pytest
4
+ import h5py
5
+ import netCDF4 as nc
6
+ import numpy as np
7
+ from fastapi.testclient import TestClient
8
+
9
+ from app.main import app
10
+ from app.services.scientific.metadata_service import MetadataService
11
+ from app.schemas.metadata import MetadataResponse
12
+
13
+ client = TestClient(app)
14
+
15
+ @pytest.fixture
16
+ def mock_netcdf():
17
+ fd, path = tempfile.mkstemp(suffix=".nc")
18
+ os.close(fd)
19
+
20
+ # Create a mock netcdf file
21
+ ds = nc.Dataset(path, 'w', format='NETCDF4')
22
+ ds.title = "Mock NetCDF"
23
+ ds.projection = "mercator"
24
+
25
+ ds.createDimension('time', None) # unlimited
26
+ ds.createDimension('lat', 10)
27
+ ds.createDimension('lon', 10)
28
+
29
+ times = ds.createVariable('time', 'f8', ('time',))
30
+ lats = ds.createVariable('lat', 'f4', ('lat',))
31
+ lons = ds.createVariable('lon', 'f4', ('lon',))
32
+ temp = ds.createVariable('temperature', 'f4', ('time', 'lat', 'lon',))
33
+
34
+ lats.units = 'degrees_north'
35
+ lons.units = 'degrees_east'
36
+ temp.units = 'K'
37
+ temp.grid_mapping = 'mercator'
38
+
39
+ times[:] = [1620000000, 1620003600] # two time steps
40
+ lats[:] = np.linspace(-90, 90, 10)
41
+ lons[:] = np.linspace(-180, 180, 10)
42
+ temp[:] = np.random.uniform(200, 300, size=(2, 10, 10))
43
+
44
+ ds.close()
45
+
46
+ yield path
47
+ os.remove(path)
48
+
49
+ @pytest.fixture
50
+ def mock_hdf5():
51
+ fd, path = tempfile.mkstemp(suffix=".h5")
52
+ os.close(fd)
53
+
54
+ with h5py.File(path, "w") as f:
55
+ f.attrs["title"] = "Mock HDF5"
56
+ f.attrs["projection"] = "geospatial"
57
+
58
+ # Dimensions
59
+ lat_dim = f.create_dataset("latitude", data=np.linspace(-90, 90, 10))
60
+ lat_dim.attrs["CLASS"] = "DIMENSION_SCALE"
61
+
62
+ lon_dim = f.create_dataset("longitude", data=np.linspace(-180, 180, 10))
63
+ lon_dim.attrs["CLASS"] = "DIMENSION_SCALE"
64
+
65
+ time_dim = f.create_dataset("time", data=np.array([1620000000, 1620003600]))
66
+ time_dim.attrs["CLASS"] = "DIMENSION_SCALE"
67
+
68
+ # Variable
69
+ temp = f.create_dataset("temperature", data=np.random.uniform(200, 300, size=(2, 10, 10)))
70
+ temp.dims[0].label = "time"
71
+ temp.dims[1].label = "lat"
72
+ temp.dims[2].label = "lon"
73
+ temp.attrs["units"] = "K"
74
+
75
+ yield path
76
+ os.remove(path)
77
+
78
+ def test_extract_netcdf_metadata(mock_netcdf):
79
+ metadata = MetadataService.extract_metadata("test_nc", mock_netcdf)
80
+ assert metadata.format == "nc"
81
+ assert metadata.summary.file_format == "nc"
82
+ assert metadata.coordinates.latitude == "lat"
83
+ assert metadata.coordinates.longitude == "lon"
84
+ assert metadata.coordinates.projection == "mercator"
85
+ assert metadata.temporal_info.time_steps == 2
86
+ assert "time" in [d.name for d in metadata.dimensions]
87
+ assert "temperature" in [v.name for v in metadata.variables]
88
+
89
+ def test_extract_hdf5_metadata(mock_hdf5):
90
+ metadata = MetadataService.extract_metadata("test_h5", mock_hdf5)
91
+ assert metadata.format == "h5"
92
+ assert metadata.coordinates.latitude == "latitude"
93
+ assert metadata.coordinates.longitude == "longitude"
94
+ assert metadata.coordinates.projection == "geospatial"
95
+ assert metadata.temporal_info.time_steps == 2
96
+ assert "temperature" in [v.name for v in metadata.variables]
97
+
98
+ def test_metadata_api_not_found():
99
+ response = client.get("/api/v1/metadata/non_existent_file")
100
+ assert response.status_code == 404
101
+
102
+ def test_metadata_service_invalid_ext():
103
+ with pytest.raises(ValueError):
104
+ MetadataService.get_parser("test.txt")