Spaces:
Sleeping
Sleeping
Commit
·
fc60200
1
Parent(s):
1a4a684
Upd efficiency API with doc
Browse files- README.md +12 -0
- app.py +49 -1
- data/firebase_saver.py +112 -1
README.md
CHANGED
|
@@ -97,6 +97,7 @@ The application is structured into modular components:
|
|
| 97 |
- `GET /download/{filename}`: Download cleaned CSV
|
| 98 |
- `GET /events`: Get processing status
|
| 99 |
- `GET /predictions/latest`: Get latest driver behavior and fuel efficiency predictions
|
|
|
|
| 100 |
|
| 101 |
### MongoDB Operations
|
| 102 |
- `GET /mongo/status`: Check MongoDB connection
|
|
@@ -163,10 +164,21 @@ The application is structured into modular components:
|
|
| 163 |
}
|
| 164 |
```
|
| 165 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 166 |
### Firebase Storage
|
| 167 |
- Structured data storage with automatic versioning
|
| 168 |
- **`skyledge/raw/`**: Original OBD data files
|
| 169 |
- **`skyledge/processed/`**: Cleaned and processed data
|
|
|
|
| 170 |
- **`skyledge/labeled/`**: Human-labeled data for RLHF training
|
| 171 |
- **`skyledge/labeled/trained.txt`**: Tracks processed datasets to avoid retraining
|
| 172 |
|
|
|
|
| 97 |
- `GET /download/{filename}`: Download cleaned CSV
|
| 98 |
- `GET /events`: Get processing status
|
| 99 |
- `GET /predictions/latest`: Get latest driver behavior and fuel efficiency predictions
|
| 100 |
+
- `GET /efficiency/{filename}`: Get fuel efficiency prediction for specific processed file
|
| 101 |
|
| 102 |
### MongoDB Operations
|
| 103 |
- `GET /mongo/status`: Check MongoDB connection
|
|
|
|
| 164 |
}
|
| 165 |
```
|
| 166 |
|
| 167 |
+
### Efficiency Retrieval Response (`GET /efficiency/{filename}`)
|
| 168 |
+
```json
|
| 169 |
+
{
|
| 170 |
+
"filename": "001_2024-12-01_processed.csv",
|
| 171 |
+
"efficiency_score": 85.2,
|
| 172 |
+
"timestamp": "2024-12-01T14:30:22",
|
| 173 |
+
"status": "success"
|
| 174 |
+
}
|
| 175 |
+
```
|
| 176 |
+
|
| 177 |
### Firebase Storage
|
| 178 |
- Structured data storage with automatic versioning
|
| 179 |
- **`skyledge/raw/`**: Original OBD data files
|
| 180 |
- **`skyledge/processed/`**: Cleaned and processed data
|
| 181 |
+
- **`skyledge/processed/efficiency.json`**: Fuel efficiency predictions for each processed file
|
| 182 |
- **`skyledge/labeled/`**: Human-labeled data for RLHF training
|
| 183 |
- **`skyledge/labeled/trained.txt`**: Tracks processed datasets to avoid retraining
|
| 184 |
|
app.py
CHANGED
|
@@ -26,7 +26,7 @@ from data.drive_saver import DriveSaver, get_drive_service, upload_to_folder
|
|
| 26 |
|
| 27 |
# Database
|
| 28 |
from data.mongo_saver import MongoSaver, save_csv_to_mongo, save_dataframe_to_mongo, MONGODB_AVAILABLE
|
| 29 |
-
from data.firebase_saver import FirebaseSaver, save_csv_increment, save_dataframe_increment
|
| 30 |
|
| 31 |
# UL Model
|
| 32 |
from utils.dbehavior_labeler import ULLabeler
|
|
@@ -547,6 +547,21 @@ def _process_and_save(df, norm_ts):
|
|
| 547 |
# Save to Firebase Storage (incremented NNN_YYYY-MM-DD_processed.csv at fixed path)
|
| 548 |
if gs_url:
|
| 549 |
logger.info(f"✅ Saved to Firebase Storage: {gs_url}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 550 |
else:
|
| 551 |
logger.warning("⚠️ Firebase Storage upload returned empty URL")
|
| 552 |
else:
|
|
@@ -640,6 +655,39 @@ def get_latest_predictions():
|
|
| 640 |
"fuel_efficiency_count": len(FUEL_EFFICIENCY)
|
| 641 |
}
|
| 642 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 643 |
|
| 644 |
# ────── Delete event from dashboard ──────────────
|
| 645 |
@app.delete("/events/remove/{timestamp}")
|
|
|
|
| 26 |
|
| 27 |
# Database
|
| 28 |
from data.mongo_saver import MongoSaver, save_csv_to_mongo, save_dataframe_to_mongo, MONGODB_AVAILABLE
|
| 29 |
+
from data.firebase_saver import FirebaseSaver, save_csv_increment, save_dataframe_increment, save_efficiency_data, get_efficiency_by_filename
|
| 30 |
|
| 31 |
# UL Model
|
| 32 |
from utils.dbehavior_labeler import ULLabeler
|
|
|
|
| 547 |
# Save to Firebase Storage (incremented NNN_YYYY-MM-DD_processed.csv at fixed path)
|
| 548 |
if gs_url:
|
| 549 |
logger.info(f"✅ Saved to Firebase Storage: {gs_url}")
|
| 550 |
+
|
| 551 |
+
# Extract filename from gs_url for efficiency data storage
|
| 552 |
+
try:
|
| 553 |
+
filename = gs_url.split('/')[-1] # Get filename from gs://bucket/path/filename.csv
|
| 554 |
+
if FUEL_EFFICIENCY and len(FUEL_EFFICIENCY) > 0:
|
| 555 |
+
efficiency_score = FUEL_EFFICIENCY[0] # Get the first (and only) efficiency score
|
| 556 |
+
success = save_efficiency_data(filename, efficiency_score)
|
| 557 |
+
if success:
|
| 558 |
+
logger.info(f"✅ Efficiency data saved for {filename}: {efficiency_score}%")
|
| 559 |
+
else:
|
| 560 |
+
logger.warning(f"⚠️ Failed to save efficiency data for {filename}")
|
| 561 |
+
else:
|
| 562 |
+
logger.warning("⚠️ No fuel efficiency data available to save")
|
| 563 |
+
except Exception as e:
|
| 564 |
+
logger.error(f"❌ Error saving efficiency data: {e}")
|
| 565 |
else:
|
| 566 |
logger.warning("⚠️ Firebase Storage upload returned empty URL")
|
| 567 |
else:
|
|
|
|
| 655 |
"fuel_efficiency_count": len(FUEL_EFFICIENCY)
|
| 656 |
}
|
| 657 |
|
| 658 |
+
@app.get("/efficiency/{filename}")
|
| 659 |
+
def get_efficiency_by_filename(filename: str):
|
| 660 |
+
"""
|
| 661 |
+
Get fuel efficiency prediction for a specific processed file from Firebase.
|
| 662 |
+
|
| 663 |
+
Args:
|
| 664 |
+
filename: The processed filename (e.g., "001_2024-12-01_processed.csv")
|
| 665 |
+
|
| 666 |
+
Returns:
|
| 667 |
+
dict: Efficiency data or error message
|
| 668 |
+
"""
|
| 669 |
+
try:
|
| 670 |
+
if not firebase_saver.is_available():
|
| 671 |
+
raise HTTPException(status_code=503, detail="Firebase Storage not available")
|
| 672 |
+
|
| 673 |
+
efficiency_data = get_efficiency_by_filename(filename)
|
| 674 |
+
|
| 675 |
+
if efficiency_data is None:
|
| 676 |
+
raise HTTPException(status_code=404, detail=f"Efficiency data not found for {filename}")
|
| 677 |
+
|
| 678 |
+
return {
|
| 679 |
+
"filename": filename,
|
| 680 |
+
"efficiency_score": efficiency_data["efficiency_score"],
|
| 681 |
+
"timestamp": efficiency_data["timestamp"],
|
| 682 |
+
"status": "success"
|
| 683 |
+
}
|
| 684 |
+
|
| 685 |
+
except HTTPException:
|
| 686 |
+
raise
|
| 687 |
+
except Exception as e:
|
| 688 |
+
logger.error(f"❌ Error retrieving efficiency data for {filename}: {e}")
|
| 689 |
+
raise HTTPException(status_code=500, detail=f"Failed to retrieve efficiency data: {str(e)}")
|
| 690 |
+
|
| 691 |
|
| 692 |
# ────── Delete event from dashboard ──────────────
|
| 693 |
@app.delete("/events/remove/{timestamp}")
|
data/firebase_saver.py
CHANGED
|
@@ -294,6 +294,90 @@ class FirebaseSaver:
|
|
| 294 |
except Exception as e:
|
| 295 |
logger.error(f"❌ Firebase DF upload failed: {e}")
|
| 296 |
return ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 297 |
|
| 298 |
|
| 299 |
# ---------- Convenience free functions ----------
|
|
@@ -312,4 +396,31 @@ def save_dataframe_increment(df: pd.DataFrame, date_str: Optional[str] = None) -
|
|
| 312 |
Returns gs:// URL or "".
|
| 313 |
"""
|
| 314 |
saver = FirebaseSaver()
|
| 315 |
-
return saver.upload_dataframe_with_increment(df, date_str=date_str)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 294 |
except Exception as e:
|
| 295 |
logger.error(f"❌ Firebase DF upload failed: {e}")
|
| 296 |
return ""
|
| 297 |
+
|
| 298 |
+
def save_efficiency_data(self, filename: str, efficiency_score: float) -> bool:
|
| 299 |
+
"""
|
| 300 |
+
Save efficiency data to efficiency.json file in Firebase.
|
| 301 |
+
|
| 302 |
+
Args:
|
| 303 |
+
filename: The processed filename (e.g., "001_2024-12-01_processed.csv")
|
| 304 |
+
efficiency_score: The fuel efficiency score (0-100)
|
| 305 |
+
|
| 306 |
+
Returns:
|
| 307 |
+
bool: True if successful, False otherwise
|
| 308 |
+
"""
|
| 309 |
+
try:
|
| 310 |
+
# Load existing efficiency data
|
| 311 |
+
efficiency_data = self.load_efficiency_data()
|
| 312 |
+
|
| 313 |
+
# Add new entry
|
| 314 |
+
efficiency_data[filename] = {
|
| 315 |
+
"efficiency_score": efficiency_score,
|
| 316 |
+
"timestamp": datetime.now().isoformat(),
|
| 317 |
+
"filename": filename
|
| 318 |
+
}
|
| 319 |
+
|
| 320 |
+
# Convert to JSON
|
| 321 |
+
json_data = json.dumps(efficiency_data, indent=2)
|
| 322 |
+
|
| 323 |
+
# Upload to Firebase
|
| 324 |
+
dest_path = f"{FIXED_PREFIX}/efficiency.json"
|
| 325 |
+
self.client.upload_from_bytes(
|
| 326 |
+
json_data.encode("utf-8"),
|
| 327 |
+
dest_path,
|
| 328 |
+
"application/json"
|
| 329 |
+
)
|
| 330 |
+
|
| 331 |
+
logger.info(f"✅ Efficiency data saved for {filename}: {efficiency_score}%")
|
| 332 |
+
return True
|
| 333 |
+
|
| 334 |
+
except Exception as e:
|
| 335 |
+
logger.error(f"❌ Failed to save efficiency data: {e}")
|
| 336 |
+
return False
|
| 337 |
+
|
| 338 |
+
def load_efficiency_data(self) -> dict:
|
| 339 |
+
"""
|
| 340 |
+
Load efficiency data from efficiency.json file in Firebase.
|
| 341 |
+
|
| 342 |
+
Returns:
|
| 343 |
+
dict: Efficiency data or empty dict if file doesn't exist
|
| 344 |
+
"""
|
| 345 |
+
try:
|
| 346 |
+
dest_path = f"{FIXED_PREFIX}/efficiency.json"
|
| 347 |
+
|
| 348 |
+
# Try to download the file
|
| 349 |
+
blob = self.bucket.blob(dest_path)
|
| 350 |
+
if not blob.exists():
|
| 351 |
+
logger.info("📄 efficiency.json not found, returning empty data")
|
| 352 |
+
return {}
|
| 353 |
+
|
| 354 |
+
# Download and parse JSON
|
| 355 |
+
json_data = blob.download_as_text()
|
| 356 |
+
efficiency_data = json.loads(json_data)
|
| 357 |
+
|
| 358 |
+
logger.info(f"✅ Loaded efficiency data: {len(efficiency_data)} entries")
|
| 359 |
+
return efficiency_data
|
| 360 |
+
|
| 361 |
+
except Exception as e:
|
| 362 |
+
logger.warning(f"⚠️ Failed to load efficiency data: {e}")
|
| 363 |
+
return {}
|
| 364 |
+
|
| 365 |
+
def get_efficiency_by_filename(self, filename: str) -> Optional[dict]:
|
| 366 |
+
"""
|
| 367 |
+
Get efficiency data for a specific filename.
|
| 368 |
+
|
| 369 |
+
Args:
|
| 370 |
+
filename: The processed filename
|
| 371 |
+
|
| 372 |
+
Returns:
|
| 373 |
+
dict: Efficiency data or None if not found
|
| 374 |
+
"""
|
| 375 |
+
try:
|
| 376 |
+
efficiency_data = self.load_efficiency_data()
|
| 377 |
+
return efficiency_data.get(filename)
|
| 378 |
+
except Exception as e:
|
| 379 |
+
logger.error(f"❌ Failed to get efficiency data for {filename}: {e}")
|
| 380 |
+
return None
|
| 381 |
|
| 382 |
|
| 383 |
# ---------- Convenience free functions ----------
|
|
|
|
| 396 |
Returns gs:// URL or "".
|
| 397 |
"""
|
| 398 |
saver = FirebaseSaver()
|
| 399 |
+
return saver.upload_dataframe_with_increment(df, date_str=date_str)
|
| 400 |
+
|
| 401 |
+
def save_efficiency_data(filename: str, efficiency_score: float) -> bool:
|
| 402 |
+
"""
|
| 403 |
+
Save efficiency data to Firebase efficiency.json file.
|
| 404 |
+
|
| 405 |
+
Args:
|
| 406 |
+
filename: The processed filename
|
| 407 |
+
efficiency_score: The fuel efficiency score (0-100)
|
| 408 |
+
|
| 409 |
+
Returns:
|
| 410 |
+
bool: True if successful, False otherwise
|
| 411 |
+
"""
|
| 412 |
+
saver = FirebaseSaver()
|
| 413 |
+
return saver.save_efficiency_data(filename, efficiency_score)
|
| 414 |
+
|
| 415 |
+
def get_efficiency_by_filename(filename: str) -> Optional[dict]:
|
| 416 |
+
"""
|
| 417 |
+
Get efficiency data for a specific filename from Firebase.
|
| 418 |
+
|
| 419 |
+
Args:
|
| 420 |
+
filename: The processed filename
|
| 421 |
+
|
| 422 |
+
Returns:
|
| 423 |
+
dict: Efficiency data or None if not found
|
| 424 |
+
"""
|
| 425 |
+
saver = FirebaseSaver()
|
| 426 |
+
return saver.get_efficiency_by_filename(filename)
|