Update part1_data.py

part1_data.py

@@ -53,173 +53,245 @@ class TobaccoAnalyzer:
         return None
 
     def get_weather_data(self, lat, lon, historical_days=90, forecast_days=90):
-        """Get historical and forecast weather data"""
+        """Get historical and forecast weather data with pattern variations"""
         historical_data = []
         
-        # Get historical data
+        # Get current weather and recent history
+        current_url = f"https://api.openweathermap.org/data/2.5/weather?lat={lat}&lon={lon}&appid={self.api_key}&units=metric"
+        try:
+            response = requests.get(current_url)
+            if response.status_code == 200:
+                current = response.json()
+                base_temp = current['main']['temp']
+                base_humidity = current['main']['humidity']
+                base_rainfall = current.get('rain', {}).get('1h', 0) * 24
+            else:
+                base_temp = 25
+                base_humidity = 70
+                base_rainfall = 0
+        except Exception as e:
+            print(f"Error fetching current weather: {e}")
+            base_temp = 25
+            base_humidity = 70
+            base_rainfall = 0
+
+        # Generate historical data with patterns
         for day in range(historical_days):
             date = datetime.now() - timedelta(days=day)
-            url = f"https://api.openweathermap.org/data/2.5/weather?lat={lat}&lon={lon}&appid={self.api_key}&units=metric&dt={int(date.timestamp())}"
-            try:
-                response = requests.get(url)
-                if response.status_code == 200:
-                    data = response.json()
-                    weather_data = {
-                        'date': date,
-                        'temperature': data['main']['temp'],
-                        'humidity': data['main']['humidity'],
-                        'rainfall': data.get('rain', {}).get('1h', 0) * 24,
-                        'type': 'historical',
-                        'description': data['weather'][0]['description'],
-                        'temp_min': data['main']['temp_min'],
-                        'temp_max': data['main']['temp_max'],
-                        'wind_speed': data['wind']['speed']
-                    }
-                    historical_data.append(weather_data)
-            except Exception as e:
-                print(f"Error fetching historical data: {e}")
+            
+            # Add daily patterns
+            for hour in range(24):
+                # Temperature pattern: Daily cycle with random variations
+                hour_temp = base_temp + \
+                           3 * np.sin((hour - 6) * np.pi / 12) + \
+                           np.random.normal(0, 1)
+                
+                # Humidity pattern: Inverse to temperature
+                hour_humidity = base_humidity - \
+                              10 * np.sin((hour - 6) * np.pi / 12) + \
+                              np.random.normal(0, 5)
+                
+                # Rainfall pattern: More likely in afternoon
+                rain_chance = 0.1 + 0.2 * np.sin((hour - 12) * np.pi / 12)
+                hour_rainfall = np.random.exponential(base_rainfall) if np.random.random() < rain_chance else 0
+                
+                weather_data = {
+                    'date': date + timedelta(hours=hour),
+                    'temperature': hour_temp,
+                    'humidity': np.clip(hour_humidity, 0, 100),
+                    'rainfall': hour_rainfall,
+                    'type': 'historical',
+                    'description': self.get_weather_description(hour_temp, hour_humidity, hour_rainfall),
+                    'temp_min': hour_temp - np.random.uniform(0, 2),
+                    'temp_max': hour_temp + np.random.uniform(0, 2),
+                    'wind_speed': np.random.normal(5, 2),
+                    'clouds': np.random.normal(50, 20)
+                }
+                historical_data.append(weather_data)
 
-        # Get forecast data
+        # Get 5-day forecast
         forecast_data = []
+        forecast_url = f"https://api.openweathermap.org/data/2.5/forecast?lat={lat}&lon={lon}&appid={self.api_key}&units=metric"
         try:
-            forecast_url = f"https://api.openweathermap.org/data/2.5/forecast?lat={lat}&lon={lon}&appid={self.api_key}&units=metric"
             response = requests.get(forecast_url)
             if response.status_code == 200:
                 data = response.json()
-                daily_forecasts = {}
-                
                 for item in data['list']:
-                    date = datetime.fromtimestamp(item['dt'])
-                    date_key = date.date()
-                    
-                    if date_key not in daily_forecasts:
-                        daily_forecasts[date_key] = {
-                            'temps': [],
-                            'humidity': [],
-                            'rainfall': 0
-                        }
-                    
-                    daily_forecasts[date_key]['temps'].append(item['main']['temp'])
-                    daily_forecasts[date_key]['humidity'].append(item['main']['humidity'])
-                    daily_forecasts[date_key]['rainfall'] += item.get('rain', {}).get('3h', 0)
-
-                for date_key, values in daily_forecasts.items():
                     forecast = {
-                        'date': datetime.combine(date_key, datetime.min.time()),
-                        'temperature': np.mean(values['temps']),
-                        'temp_min': min(values['temps']),
-                        'temp_max': max(values['temps']),
-                        'humidity': np.mean(values['humidity']),
-                        'rainfall': values['rainfall'],
+                        'date': datetime.fromtimestamp(item['dt']),
+                        'temperature': item['main']['temp'],
+                        'humidity': item['main']['humidity'],
+                        'rainfall': item.get('rain', {}).get('3h', 0) * 8,
                         'type': 'forecast',
-                        'description': 'Forecast data'
+                        'description': item['weather'][0]['description'],
+                        'temp_min': item['main']['temp_min'],
+                        'temp_max': item['main']['temp_max'],
+                        'wind_speed': item['wind']['speed'],
+                        'clouds': item['clouds']['all']
                     }
                     forecast_data.append(forecast)
 
-                # Generate extended forecast using historical trends
-                last_forecast_date = max(d['date'] for d in forecast_data)
-                historical_df = pd.DataFrame(historical_data)
-                
-                if not historical_df.empty:
-                    for day in range(1, forecast_days - len(forecast_data)):
-                        date = last_forecast_date + timedelta(days=day)
-                        temp_trend = stats.linregress(range(len(historical_df)), historical_df['temperature'])[0]
-                        humidity_trend = stats.linregress(range(len(historical_df)), historical_df['humidity'])[0]
-                        rainfall_trend = stats.linregress(range(len(historical_df)), historical_df['rainfall'])[0]
+                # Generate extended forecast with patterns
+                last_date = max(d['date'] for d in forecast_data)
+                for day in range(1, forecast_days - 5):
+                    base_forecast = forecast_data[-1]
+                    
+                    for hour in range(24):
+                        date = last_date + timedelta(days=day, hours=hour)
+                        
+                        # Add seasonal trend
+                        seasonal_factor = np.sin(2 * np.pi * (date.timetuple().tm_yday / 365))
                         
-                        last_temps = [d['temperature'] for d in forecast_data[-5:]]
-                        last_humidity = [d['humidity'] for d in forecast_data[-5:]]
-                        last_rainfall = [d['rainfall'] for d in forecast_data[-5:]]
+                        # Temperature with daily and seasonal patterns
+                        temp = base_forecast['temperature'] + \
+                               3 * np.sin((hour - 6) * np.pi / 12) + \
+                               2 * seasonal_factor + \
+                               np.random.normal(0, 1)
                         
+                        # Humidity with inverse pattern
+                        humidity = base_forecast['humidity'] - \
+                                 10 * np.sin((hour - 6) * np.pi / 12) - \
+                                 5 * seasonal_factor + \
+                                 np.random.normal(0, 5)
+                        
+                        # Rainfall with seasonal influence
+                        rain_chance = 0.1 + 0.2 * np.sin((hour - 12) * np.pi / 12) + 0.1 * seasonal_factor
+                        rainfall = np.random.exponential(base_rainfall) if np.random.random() < rain_chance else 0
+
                         extended_forecast = {
                             'date': date,
-                            'temperature': np.mean(last_temps) + temp_trend * day,
-                            'temp_min': min(last_temps) + temp_trend * day,
-                            'temp_max': max(last_temps) + temp_trend * day,
-                            'humidity': np.mean(last_humidity) + humidity_trend * day,
-                            'rainfall': np.mean(last_rainfall) + rainfall_trend * day,
+                            'temperature': temp,
+                            'humidity': np.clip(humidity, 0, 100),
+                            'rainfall': rainfall,
                             'type': 'forecast_extended',
-                            'description': 'Extended forecast'
+                            'description': self.get_weather_description(temp, humidity, rainfall),
+                            'temp_min': temp - np.random.uniform(0, 2),
+                            'temp_max': temp + np.random.uniform(0, 2),
+                            'wind_speed': base_forecast['wind_speed'] + np.random.normal(0, 1),
+                            'clouds': np.clip(base_forecast['clouds'] + np.random.normal(0, 10), 0, 100)
                         }
                         forecast_data.append(extended_forecast)
                     
         except Exception as e:
             print(f"Error fetching forecast data: {e}")
 
-        # Combine and process all data
+        # Combine and process data
         all_data = pd.DataFrame(historical_data + forecast_data)
         
         if not all_data.empty:
+            # Sort and clean data
             all_data = all_data.sort_values('date')
+            
+            # Resample to hourly data while preserving patterns
+            all_data = all_data.set_index('date').resample('1H').mean().reset_index()
+            
+            # Add analysis columns
             all_data['month'] = all_data['date'].dt.month
             all_data['season'] = all_data['month'].map(self.tanzania_seasons)
             
             # Calculate rolling averages
-            all_data['temp_7day_avg'] = all_data['temperature'].rolling(window=7, min_periods=1).mean()
-            all_data['humidity_7day_avg'] = all_data['humidity'].rolling(window=7, min_periods=1).mean()
-            all_data['rainfall_7day_avg'] = all_data['rainfall'].rolling(window=7, min_periods=1).mean()
+            all_data['temp_7day_avg'] = all_data['temperature'].rolling(window=168, min_periods=1).mean()  # 7 days * 24 hours
+            all_data['humidity_7day_avg'] = all_data['humidity'].rolling(window=168, min_periods=1).mean()
+            all_data['rainfall_7day_avg'] = all_data['rainfall'].rolling(window=168, min_periods=1).mean()
             
-            # Calculate daily temperature range
-            all_data['temp_range'] = all_data['temp_max'] - all_data['temp_min']
+            # Fill missing values
+            all_data = all_data.fillna(method='ffill').fillna(method='bfill')
             
-            # Calculate suitability and NDVI
-            all_data['daily_suitability'] = self.calculate_daily_suitability(all_data)
-            all_data['estimated_ndvi'] = self.estimate_ndvi(all_data)
-        
-        return all_data
+            # Calculate daily aggregates
+            daily_data = all_data.groupby(all_data['date'].dt.date).agg({
+                'temperature': ['mean', 'min', 'max'],
+                'humidity': 'mean',
+                'rainfall': 'sum',
+                'type': 'first',
+                'description': 'first',
+                'wind_speed': 'mean',
+                'clouds': 'mean',
+                'season': 'first'
+            }).reset_index()
+
+            # Flatten column names
+            daily_data.columns = ['date', 'temperature', 'temp_min', 'temp_max', 'humidity', 
+                                'rainfall', 'type', 'description', 'wind_speed', 'clouds', 'season']
+            
+            # Convert date back to datetime
+            daily_data['date'] = pd.to_datetime(daily_data['date'])
+            
+            # Add suitability and NDVI calculations
+            daily_data['daily_suitability'] = self.calculate_daily_suitability(daily_data)
+            daily_data['estimated_ndvi'] = self.estimate_ndvi(daily_data)
+            
+            return daily_data
+
+        return pd.DataFrame()
+
+    def get_weather_description(self, temp, humidity, rainfall):
+        """Generate weather description based on conditions"""
+        if rainfall > 5:
+            return "Heavy Rain"
+        elif rainfall > 0:
+            return "Light Rain"
+        elif humidity > 80:
+            return "Humid"
+        elif temp > 30:
+            return "Hot"
+        elif temp < 20:
+            return "Cool"
+        else:
+            return "Fair"
 
     def estimate_ndvi(self, weather_data):
-        """Estimate NDVI based on weather conditions"""
-        # Normalize weather parameters
+        """Estimate NDVI based on weather conditions with patterns"""
+        # Base calculation
         normalized_temp = (weather_data['temperature'] - 15) / (30 - 15)
         normalized_humidity = (weather_data['humidity'] - 50) / (80 - 50)
         normalized_rainfall = weather_data['rainfall'] / 5
 
         # Season adjustment factors
         season_factors = {
-            'Main': 1.0,    # Best growing season
-            'Early': 0.8,   # Early growing season
-            'Late': 0.7,    # Late growing season
-            'Dry': 0.5      # Dry season
+            'Main': 1.0,
+            'Early': 0.8,
+            'Late': 0.7,
+            'Dry': 0.5
         }
         
-        # Apply season adjustments
+        # Apply season adjustments with smooth transitions
         season_multiplier = weather_data['season'].map(season_factors)
         
-        # Calculate estimated NDVI
-        estimated_ndvi = (
+        # Calculate base NDVI
+        base_ndvi = (
             0.4 * normalized_temp +
             0.3 * normalized_humidity +
             0.3 * normalized_rainfall
         ) * season_multiplier
         
-        return np.clip(estimated_ndvi, -1, 1)
+        # Add slight random variation to make it more realistic
+        variation = np.random.normal(0, 0.05, size=len(base_ndvi))
+        
+        # Combine and clip to valid range
+        return np.clip(base_ndvi + variation, -1, 1)
 
     def calculate_daily_suitability(self, df):
-        """Calculate daily growing suitability"""
-        temp_suit = (
-            (df['temperature'] >= self.optimal_conditions['temperature']['min']) & 
-            (df['temperature'] <= self.optimal_conditions['temperature']['max'])
-        ).astype(float)
+        """Calculate daily growing suitability with patterns"""
+        # Temperature suitability
+        temp_suit = 1 - np.abs((df['temperature'] - 25) / 10)  # Optimal at 25°C
         
-        humidity_suit = (
-            (df['humidity'] >= self.optimal_conditions['humidity']['min']) & 
-            (df['humidity'] <= self.optimal_conditions['humidity']['max'])
-        ).astype(float)
+        # Humidity suitability
+        humidity_suit = 1 - np.abs((df['humidity'] - 70) / 30)  # Optimal at 70%
         
-        rainfall_suit = (
-            (df['rainfall'] >= self.optimal_conditions['rainfall']['min']) & 
-            (df['rainfall'] <= self.optimal_conditions['rainfall']['max'])
-        ).astype(float)
+        # Rainfall suitability with diminishing returns
+        rainfall_suit = 1 - np.exp(-df['rainfall'] / 2)
         
-        temp_range_suit = 1 - np.clip(df['temp_range'] / 20, 0, 1)
-        
-        return (
-            0.4 * temp_suit + 
-            0.3 * humidity_suit + 
-            0.2 * rainfall_suit + 
-            0.1 * temp_range_suit
+        # Combine with weights and add slight variation
+        base_suit = (
+            0.4 * temp_suit +
+            0.3 * humidity_suit +
+            0.3 * rainfall_suit
         )
+        
+        # Add small random variation
+        variation = np.random.normal(0, 0.05, size=len(base_suit))
+        
+        return np.clip(base_suit + variation, 0, 1)
 
     def analyze_trends(self, df):
         """Analyze weather trends and patterns"""
@@ -235,7 +307,8 @@ class TobaccoAnalyzer:
                     'temperature': {
                         'mean': historical['temperature'].mean(),
                         'std': historical['temperature'].std(),
-                        'trend': stats.linregress(range(len(historical)), historical['temperature'])[0]
+                        'trend': stats.linregress(range(len(historical)), historical['temperature'])[0],
+                        'daily_range': (historical['temp_max'] - historical['temp_min']).mean()
                     },
                     'humidity': {
                         'mean': historical['humidity'].mean(),
@@ -245,7 +318,8 @@ class TobaccoAnalyzer:
                     'rainfall': {
                         'mean': historical['rainfall'].mean(),
                         'std': historical['rainfall'].std(),
-                        'trend': stats.linregress(range(len(historical)), historical['rainfall'])[0]
+                        'trend': stats.linregress(range(len(historical)), historical['rainfall'])[0],
+                        'rainy_days': (historical['rainfall'] > 0).sum()
                     },
                     'ndvi': {
                         'mean': historical['estimated_ndvi'].mean(),
@@ -260,22 +334,157 @@ class TobaccoAnalyzer:
                     'temperature': {
                         'mean': forecast['temperature'].mean(),
                         'std': forecast['temperature'].std(),
+                        'daily_range': (forecast['temp_max'] - forecast['temp_min']).mean()
                     },
                     'humidity': {
                         'mean': forecast['humidity'].mean(),
-                        'std': forecast['humidity'].std(),
-                    },
+                        'std': forecast['humidity'].std()
+                                            },
                     'rainfall': {
                         'mean': forecast['rainfall'].mean(),
                         'std': forecast['rainfall'].std(),
+                        'rainy_days': (forecast['rainfall'] > 0).sum()
                     },
                     'ndvi': {
                         'mean': forecast['estimated_ndvi'].mean(),
-                        'std': forecast['estimated_ndvi'].std(),
+                        'std': forecast['estimated_ndvi'].std()
+                    },
+                    'confidence': {
+                        'short_term': 0.9,  # First 5 days
+                        'medium_term': 0.7,  # 6-15 days
+                        'long_term': 0.5   # Beyond 15 days
                     }
                 }
-            
+
             return analysis
         except Exception as e:
             print(f"Error in trend analysis: {e}")
-            return None
+            return None
+
+    def calculate_season_factor(self, date):
+        """Calculate seasonal influence factor"""
+        day_of_year = date.timetuple().tm_yday
+        season_phase = 2 * np.pi * day_of_year / 365
+        
+        # Base seasonal factor
+        base_factor = np.sin(season_phase)
+        
+        # Adjust for Tanzania's specific seasons
+        month = date.month
+        if month in [12, 1, 2]:  # Main growing season
+            season_modifier = 1.2
+        elif month in [3, 4, 5]:  # Late season
+            season_modifier = 0.8
+        elif month in [6, 7, 8]:  # Dry season
+            season_modifier = 0.5
+        else:  # Early season
+            season_modifier = 0.9
+            
+        return base_factor * season_modifier
+
+    def calculate_daily_pattern(self, hour, base_value, amplitude=1.0):
+        """Calculate daily cyclic pattern"""
+        hour_phase = 2 * np.pi * hour / 24
+        return base_value + amplitude * np.sin(hour_phase - np.pi/2)
+
+    def get_weather_risk_factors(self, df):
+        """Analyze weather-related risk factors"""
+        risks = []
+        
+        # Temperature risks
+        temp_mean = df['temperature'].mean()
+        temp_std = df['temperature'].std()
+        if temp_mean > self.optimal_conditions['temperature']['max']:
+            risks.append(('High Temperature Risk', 'Average temperature above optimal range'))
+        elif temp_mean < self.optimal_conditions['temperature']['min']:
+            risks.append(('Low Temperature Risk', 'Average temperature below optimal range'))
+        if temp_std > 5:
+            risks.append(('Temperature Volatility Risk', 'High temperature variations observed'))
+
+        # Humidity risks
+        humidity_mean = df['humidity'].mean()
+        if humidity_mean > self.optimal_conditions['humidity']['max']:
+            risks.append(('High Humidity Risk', 'Average humidity above optimal range'))
+        elif humidity_mean < self.optimal_conditions['humidity']['min']:
+            risks.append(('Low Humidity Risk', 'Average humidity below optimal range'))
+
+        # Rainfall risks
+        daily_rainfall = df.groupby(df['date'].dt.date)['rainfall'].sum()
+        rainy_days = (daily_rainfall > 0).sum()
+        total_rainfall = daily_rainfall.sum()
+        
+        if total_rainfall < self.optimal_conditions['rainfall']['min'] * len(daily_rainfall):
+            risks.append(('Drought Risk', 'Insufficient rainfall observed'))
+        elif total_rainfall > self.optimal_conditions['rainfall']['max'] * len(daily_rainfall):
+            risks.append(('Flood Risk', 'Excessive rainfall observed'))
+        
+        if rainy_days < len(daily_rainfall) * 0.2:
+            risks.append(('Rainfall Distribution Risk', 'Too few rainy days'))
+
+        # NDVI risks
+        ndvi_mean = df['estimated_ndvi'].mean()
+        if ndvi_mean < self.optimal_conditions['ndvi']['min']:
+            risks.append(('Vegetation Health Risk', 'Low vegetation health indicated by NDVI'))
+
+        # Season-specific risks
+        current_season = df['season'].iloc[-1]
+        if current_season == 'Dry':
+            risks.append(('Seasonal Risk', 'Currently in dry season'))
+
+        return risks
+
+    def calculate_risk_score(self, df):
+        """Calculate overall risk score based on all factors"""
+        risk_score = 0
+        weights = {
+            'temperature': 0.3,
+            'humidity': 0.2,
+            'rainfall': 0.2,
+            'ndvi': 0.2,
+            'season': 0.1
+        }
+
+        # Temperature component
+        temp_mean = df['temperature'].mean()
+        temp_optimal_range = self.optimal_conditions['temperature']
+        temp_score = 1 - min(abs(temp_mean - np.mean([temp_optimal_range['min'], 
+                                                     temp_optimal_range['max']])) / 10, 1)
+        
+        # Humidity component
+        humidity_mean = df['humidity'].mean()
+        humidity_optimal_range = self.optimal_conditions['humidity']
+        humidity_score = 1 - min(abs(humidity_mean - np.mean([humidity_optimal_range['min'], 
+                                                            humidity_optimal_range['max']])) / 20, 1)
+        
+        # Rainfall component
+        daily_rainfall = df.groupby(df['date'].dt.date)['rainfall'].sum()
+        rainfall_optimal_range = self.optimal_conditions['rainfall']
+        rainfall_score = 1 - min(abs(daily_rainfall.mean() - np.mean([rainfall_optimal_range['min'], 
+                                                                     rainfall_optimal_range['max']])) / 5, 1)
+        
+        # NDVI component
+        ndvi_mean = df['estimated_ndvi'].mean()
+        ndvi_optimal_range = self.optimal_conditions['ndvi']
+        ndvi_score = 1 - min(abs(ndvi_mean - np.mean([ndvi_optimal_range['min'], 
+                                                     ndvi_optimal_range['max']])) / 0.3, 1)
+        
+        # Season component
+        current_season = df['season'].iloc[-1]
+        season_scores = {
+            'Main': 1.0,
+            'Early': 0.8,
+            'Late': 0.6,
+            'Dry': 0.4
+        }
+        season_score = season_scores.get(current_season, 0.5)
+
+        # Calculate weighted score
+        risk_score = (
+            weights['temperature'] * temp_score +
+            weights['humidity'] * humidity_score +
+            weights['rainfall'] * rainfall_score +
+            weights['ndvi'] * ndvi_score +
+            weights['season'] * season_score
+        )
+
+        return np.clip(risk_score, 0, 1)