update of api connection

app.py

@@ -426,8 +426,10 @@ def create_interface():
                 
                 ### 📊 Sales Analytics
                 - **Week Analysis**: Shows sales data for the last 7 days
-                - **Month Analysis**: Shows sales data for the last 30 days
-                - View total revenue, order count, and top-performing products
+                - **Month Analysis**: Shows sales data for the last 30 days  
+                - **Enhanced Metrics**: Commission analysis, net revenue, platform fees
+                - **Commission Dashboard**: Detailed commission breakdown by products
+                - **Pagination**: Automatically handles large datasets (80,000+ records)
                 
                 ### 📦 Inventory Forecasting
                 Choose from multiple forecasting methods:
@@ -442,8 +444,17 @@ def create_interface():
                 - 🟡 **Warning** (7-14 days): Monitor closely
                 - 🟢 **Safe** (> 14 days): Adequate stock levels
                 
+                ### 🔍 Data Validation
+                - **Consistency Checks**: Automatic validation of data quality
+                - **Duplicate Detection**: Identifies duplicate sales records
+                - **Data Aggregation**: Performance optimization for large datasets
+                
                 ### 🔗 API Information
-                This dashboard uses the [Wildberries API](https://dev.wildberries.ru/en/openapi/api-information) with respect for rate limits (300 requests/minute).
+                This dashboard uses the [Wildberries API](https://dev.wildberries.ru/en/openapi/api-information):
+                - **Sales Endpoint**: `/api/v1/supplier/sales` (with automatic pagination)
+                - **Stocks Endpoint**: `/api/v1/supplier/stocks`  
+                - **Rate Limits**: 300 requests/minute (respected automatically)
+                - **Data Retention**: Sales data available for 90 days
                 
                 ### 🛠️ Technical Details
                 - **Framework**: Gradio + FastMCP

config.py

@@ -74,8 +74,8 @@ class Config:
     def get_endpoints(self) -> Dict[str, str]:
         """Get API endpoint configurations based on working API calls"""
         return {
-            # Statistics API endpoints - Updated to working v5 version
-            "sales": f"{self.wildberries_base_url}/api/v5/supplier/reportDetailByPeriod",
+            # Statistics API endpoints - Correct sales endpoint
+            "sales": f"{self.wildberries_base_url}/api/v1/supplier/sales",
             "orders": f"{self.wildberries_base_url}/api/v5/supplier/reportDetailByPeriod", 
             "stocks": f"{self.wildberries_base_url}/api/v1/supplier/stocks",
             "incomes": f"{self.wildberries_base_url}/api/v5/supplier/reportDetailByPeriod",

wildberries_client.py

@@ -46,26 +46,13 @@ def validate_wb_data(weekly_data: pd.DataFrame, monthly_data: pd.DataFrame) -> D
                     f"Data inconsistency: Weekly revenue (₽{weekly_revenue:,.0f}) exceeds monthly revenue (₽{monthly_revenue:,.0f})"
                 )
         
-        # Check for empty doc_type_name (logistics entries)
+        # Check for duplicate sales IDs
         for df_name, df in [("weekly", weekly_data), ("monthly", monthly_data)]:
-            if not df.empty and 'document_type' in df.columns:
-                logistics_count = df[df['document_type'].isna() | (df['document_type'] == '')].shape[0]
-                if logistics_count > 0:
+            if not df.empty and 'srid' in df.columns:
+                duplicate_count = df.duplicated('srid').sum()
+                if duplicate_count > 0:
                     validation_results["warnings"].append(
-                        f"{df_name.title()} data: {logistics_count} logistics entries found (empty doc_type_name)"
-                    )
-        
-        # Check for zero quantities in sales records
-        for df_name, df in [("weekly", weekly_data), ("monthly", monthly_data)]:
-            if not df.empty and 'quantity' in df.columns and 'document_type' in df.columns:
-                zero_qty_sales = df[
-                    (df['quantity'] == 0) & 
-                    (df['document_type'].notna()) & 
-                    (df['document_type'] != '')
-                ].shape[0]
-                if zero_qty_sales > 0:
-                    validation_results["warnings"].append(
-                        f"{df_name.title()} data: {zero_qty_sales} sales records with zero quantity"
+                        f"{df_name.title()} data: {duplicate_count} duplicate sale IDs found"
                     )
     
     except Exception as e:
@@ -95,12 +82,10 @@ def aggregate_wb_data(df: pd.DataFrame, period: str = 'daily') -> pd.DataFrame:
         }
         
         # Add optional columns if they exist
-        if 'sales_commission' in df.columns:
-            agg_functions['sales_commission'] = 'sum'
         if 'amount_for_pay' in df.columns:
             agg_functions['amount_for_pay'] = 'sum'
-        if 'delivery_cost' in df.columns:
-            agg_functions['delivery_cost'] = 'sum'
+        if 'payment_sale_amount' in df.columns:
+            agg_functions['payment_sale_amount'] = 'sum'
         
         if period == 'daily':
             grouped = df.groupby(df['sale_date'].dt.date)
@@ -245,46 +230,84 @@ class WildberriesAPI:
             logger.error(f"Request failed: {str(e)}")
             raise WildberriesAPIError(f"Request failed: {str(e)}")
     
-    def get_sales(self, date_from: str, date_to: str = None) -> pd.DataFrame:
+    def get_sales(self, date_from: str, date_to: str = None, flag: int = 0) -> pd.DataFrame:
         """
-        Get sales data from Wildberries API
+        Get sales data from Wildberries API with automatic pagination
         
         Args:
-            date_from: Start date in YYYY-MM-DD format
-            date_to: End date in YYYY-MM-DD format (optional)
+            date_from: Start date in YYYY-MM-DD format or lastChangeDate for pagination
+            date_to: End date in YYYY-MM-DD format (optional, not used by sales API)
+            flag: 0 for sales and returns, 1 for only sales (optional)
         
         Returns:
             pandas.DataFrame with sales data
+            
+        Note: API limit is 80,000 rows per request. This method handles pagination automatically.
         """
         endpoint = self.config.get_endpoints()["sales"]
         
-        # Add automatic dateTo defaulting to today's date when not provided
-        if not date_to:
-            date_to = datetime.now().strftime("%Y-%m-%d")
-        
-        params = {"dateFrom": date_from, "dateTo": date_to, "limit": 100}
+        all_sales_data = []
+        current_date_from = date_from
+        max_iterations = 50  # Safety limit to prevent infinite loops
+        iteration = 0
         
         try:
-            response = self._make_request("GET", endpoint, params=params)
+            while iteration < max_iterations:
+                # Build parameters for sales API
+                params = {"dateFrom": current_date_from}
+                if flag is not None:
+                    params["flag"] = flag
+                
+                logger.info(f"Fetching sales data from {current_date_from} (iteration {iteration + 1})")
+                
+                response = self._make_request("GET", endpoint, params=params)
+                
+                if not response:
+                    logger.warning("No sales data returned from API")
+                    break
+                
+                # Sales API returns direct array
+                if isinstance(response, list):
+                    batch_data = response
+                else:
+                    logger.warning("Unexpected API response format for sales")
+                    break
+                
+                if not batch_data:
+                    logger.info("Empty response received - all sales data retrieved")
+                    break
+                
+                logger.info(f"Retrieved {len(batch_data)} sales records")
+                all_sales_data.extend(batch_data)
+                
+                # Check if we need pagination (response has 80,000 rows)
+                if len(batch_data) < 80000:
+                    logger.info("Received less than 80,000 rows - all data retrieved")
+                    break
+                
+                # Get lastChangeDate from the last record for next request
+                last_record = batch_data[-1]
+                if 'lastChangeDate' in last_record:
+                    current_date_from = last_record['lastChangeDate']
+                    logger.info(f"Next pagination starts from: {current_date_from}")
+                else:
+                    logger.warning("No lastChangeDate found in response - stopping pagination")
+                    break
+                
+                iteration += 1
             
-            if not response:
-                logger.warning("No sales data returned from API")
-                return pd.DataFrame()
+            if iteration >= max_iterations:
+                logger.warning(f"Maximum iterations ({max_iterations}) reached - there may be more data")
             
-            # Handle direct array response (v5 API format)
-            if isinstance(response, list):
-                sales_data = pd.DataFrame(response)
-            elif isinstance(response, dict) and "data" in response:
-                sales_data = pd.DataFrame(response["data"])
-            else:
-                logger.warning("Unexpected API response format")
+            if not all_sales_data:
                 return pd.DataFrame()
             
-            if sales_data.empty:
-                return sales_data
+            # Convert to DataFrame and process
+            sales_data = pd.DataFrame(all_sales_data)
+            logger.info(f"Total sales records retrieved: {len(sales_data)}")
             
-            # Process and clean the data
-            sales_data = self._process_reportdetail_data(sales_data)
+            # Process and clean the data using the correct sales data processor
+            sales_data = self._process_sales_data(sales_data)
             
             return sales_data
             
@@ -294,41 +317,80 @@ class WildberriesAPI:
     
     def get_stocks(self, date_from: str = None, date_to: str = None) -> pd.DataFrame:
         """
-        Get current stock levels from Wildberries API
+        Get current stock levels from Wildberries API with automatic pagination
         
         Args:
-            date_from: Date to get stock levels for (optional, defaults to yesterday)
+            date_from: Date to get stock levels for (optional, defaults to 2019-06-20 for total stock)
             date_to: Not used for stocks endpoint (stocks API uses only dateFrom)
         
         Returns:
             pandas.DataFrame with stock data
+            
+        Note: API limit is 60,000 rows per request. This method handles pagination automatically.
         """
         endpoint = self.config.get_endpoints()["stocks"]
         
+        # Use early date to get total stock if no date specified
         if not date_from:
-            date_from = (datetime.now() - timedelta(days=1)).strftime("%Y-%m-%d")
+            date_from = "2019-06-20"  # Early date to get all stocks
         
-        # Stocks API uses only dateFrom parameter
-        params = {"dateFrom": date_from}
+        all_stock_data = []
+        current_date_from = date_from
+        max_iterations = 30  # Safety limit for stocks (should be less than sales)
+        iteration = 0
         
         try:
-            response = self._make_request("GET", endpoint, params=params)
+            while iteration < max_iterations:
+                # Stocks API uses only dateFrom parameter (RFC3339 format)
+                params = {"dateFrom": current_date_from}
+                
+                logger.info(f"Fetching stock data from {current_date_from} (iteration {iteration + 1})")
+                
+                response = self._make_request("GET", endpoint, params=params)
+                
+                if not response:
+                    logger.warning("No stock data returned from API")
+                    break
+                
+                # Stocks API returns direct array response
+                if isinstance(response, list):
+                    batch_data = response
+                else:
+                    logger.warning("Unexpected API response format for stocks")
+                    break
+                
+                if not batch_data:
+                    logger.info("Empty response received - all stock data retrieved")
+                    break
+                
+                logger.info(f"Retrieved {len(batch_data)} stock records")
+                all_stock_data.extend(batch_data)
+                
+                # Check if we need pagination (response has 60,000 rows)
+                if len(batch_data) < 60000:
+                    logger.info("Received less than 60,000 rows - all data retrieved")
+                    break
+                
+                # Get lastChangeDate from the last record for next request
+                last_record = batch_data[-1]
+                if 'lastChangeDate' in last_record:
+                    current_date_from = last_record['lastChangeDate']
+                    logger.info(f"Next pagination starts from: {current_date_from}")
+                else:
+                    logger.warning("No lastChangeDate found in response - stopping pagination")
+                    break
+                
+                iteration += 1
             
-            if not response:
-                logger.warning("No stock data returned from API")
-                return pd.DataFrame()
+            if iteration >= max_iterations:
+                logger.warning(f"Maximum iterations ({max_iterations}) reached - there may be more data")
             
-            # Stocks API returns direct array response
-            if isinstance(response, list):
-                stock_data = pd.DataFrame(response)
-            elif isinstance(response, dict) and "data" in response:
-                stock_data = pd.DataFrame(response["data"])
-            else:
-                logger.warning("Unexpected API response format")
+            if not all_stock_data:
                 return pd.DataFrame()
             
-            if stock_data.empty:
-                return stock_data
+            # Convert to DataFrame and process
+            stock_data = pd.DataFrame(all_stock_data)
+            logger.info(f"Total stock records retrieved: {len(stock_data)}")
             
             # Process and clean the stock data using specific stock processing
             stock_data = self._process_stock_data(stock_data)
@@ -387,36 +449,38 @@ class WildberriesAPI:
             raise WildberriesAPIError(f"Failed to fetch orders data: {str(e)}")
     
     def _process_sales_data(self, df: pd.DataFrame) -> pd.DataFrame:
-        """Process and clean sales data from API response"""
+        """Process and clean sales data from API response (v1 sales endpoint)"""
         
-        # Rename columns to standardized names
+        # Column mapping based on actual sales API response structure
         column_mapping = {
-            'srid': 'sale_id',
-            'supplierArticle': 'article',
-            'nmId': 'product_id',
             'date': 'sale_date',
             'lastChangeDate': 'last_change_date',
             'warehouseName': 'warehouse',
+            'warehouseType': 'warehouse_type',
             'countryName': 'country',
             'oblastOkrugName': 'region',
             'regionName': 'city',
+            'supplierArticle': 'supplier_article',
+            'nmId': 'product_id',
+            'barcode': 'barcode',
+            'category': 'category',
+            'subject': 'subject',
+            'brand': 'brand',
+            'techSize': 'tech_size',
             'incomeID': 'income_id',
             'isSupply': 'is_supply',
             'isRealization': 'is_realization',
-            'totalPrice': 'total_price',
+            'totalPrice': 'total_price',  # Already total price per item
             'discountPercent': 'discount_percent',
             'spp': 'spp_discount',
-            'paymentSaleAmount': 'sale_amount',
-            'forPay': 'amount_to_pay',
+            'paymentSaleAmount': 'payment_sale_amount',
+            'forPay': 'amount_for_pay',  # What seller receives
             'finishedPrice': 'finished_price',
             'priceWithDisc': 'price_with_discount',
-            'saleID': 'external_sale_id',
-            'orderType': 'order_type',
+            'saleID': 'sale_id',
             'sticker': 'sticker',
             'gNumber': 'g_number',
-            'techSize': 'tech_size',
-            'brand': 'brand',
-            'subject': 'category'
+            'srid': 'unique_id'  # Unique identifier for the sale
         }
         
         # Rename columns that exist
@@ -431,32 +495,60 @@ class WildberriesAPI:
                 df[col] = pd.to_datetime(df[col], errors='coerce')
         
         # Convert numeric columns
-        numeric_columns = ['total_price', 'discount_percent', 'sale_amount', 'amount_to_pay', 'finished_price', 'price_with_discount']
+        numeric_columns = [
+            'total_price', 'discount_percent', 'spp_discount', 'payment_sale_amount',
+            'amount_for_pay', 'finished_price', 'price_with_discount', 'income_id'
+        ]
         for col in numeric_columns:
             if col in df.columns:
                 df[col] = pd.to_numeric(df[col], errors='coerce')
         
-        # Add calculated fields
-        if 'total_price' in df.columns:
-            df['quantity'] = 1  # Each row represents one sale
-        
-        # Add product name (if not available, use article)
+        # Add product name (use supplier_article as primary)
         if 'product_name' not in df.columns:
-            if 'article' in df.columns:
-                df['product_name'] = df['article']
+            if 'supplier_article' in df.columns:
+                df['product_name'] = df['supplier_article']
+            elif 'category' in df.columns:
+                df['product_name'] = df['category']
             else:
                 df['product_name'] = 'Unknown Product'
         
+        # Add quantity (each row represents 1 item sale/return)
+        df['quantity'] = 1
+        
+        # Calculate commission (difference between total_price and amount_for_pay)
+        if 'total_price' in df.columns and 'amount_for_pay' in df.columns:
+            df['sales_commission'] = df['total_price'] - df['amount_for_pay']
+            # Handle negative commissions (returns)
+            df['sales_commission'] = df['sales_commission'].fillna(0)
+        
+        # Add sale_amount for compatibility (use amount_for_pay as seller's net amount)
+        if 'sale_amount' not in df.columns:
+            if 'amount_for_pay' in df.columns:
+                df['sale_amount'] = df['amount_for_pay']
+            else:
+                df['sale_amount'] = df['total_price']
+        
+        # Add current_stock for inventory forecasting (default to 0)
+        if 'current_stock' not in df.columns:
+            df['current_stock'] = 0
+        
+        # Filter out negative total_price (returns) if needed for analysis
+        # Note: Keep returns for complete data, but mark them
+        if 'total_price' in df.columns:
+            df['is_return'] = df['total_price'] < 0
+        
+        logger.info(f"Processed {len(df)} sales records")
+        
         return df
     
     def _process_stock_data(self, df: pd.DataFrame) -> pd.DataFrame:
-        """Process and clean stock data from API response"""
+        """Process and clean stock data from API response (v1 stocks endpoint)"""
         
-        # Rename columns to standardized names
+        # Column mapping based on actual stocks API response structure
         column_mapping = {
             'lastChangeDate': 'last_change_date',
             'warehouseName': 'warehouse',
-            'supplierArticle': 'article',
+            'supplierArticle': 'supplier_article',
             'nmId': 'product_id',
             'barcode': 'barcode',
             'quantity': 'current_stock',
@@ -468,7 +560,7 @@ class WildberriesAPI:
             'brand': 'brand',
             'techSize': 'tech_size',
             'Price': 'price',
-            'Discount': 'discount',
+            'Discount': 'discount_percent',
             'isSupply': 'is_supply',
             'isRealization': 'is_realization',
             'SCCode': 'sc_code'
@@ -484,27 +576,38 @@ class WildberriesAPI:
             df['last_change_date'] = pd.to_datetime(df['last_change_date'], errors='coerce')
         
         # Convert numeric columns
-        numeric_columns = ['current_stock', 'in_way_to_client', 'in_way_from_client', 'quantity_full', 'price', 'discount']
+        numeric_columns = [
+            'current_stock', 'in_way_to_client', 'in_way_from_client', 
+            'quantity_full', 'price', 'discount_percent', 'product_id'
+        ]
         for col in numeric_columns:
             if col in df.columns:
                 df[col] = pd.to_numeric(df[col], errors='coerce')
         
-        # Add product name (if not available, use article)
+        # Add product name (use supplier_article as primary)
         if 'product_name' not in df.columns:
-            if 'article' in df.columns:
-                df['product_name'] = df['article']
+            if 'supplier_article' in df.columns:
+                df['product_name'] = df['supplier_article']
+            elif 'subject' in df.columns:
+                df['product_name'] = df['subject']
+            elif 'category' in df.columns:
+                df['product_name'] = df['category']
             else:
                 df['product_name'] = 'Unknown Product'
         
-        # Add missing columns that dashboard expects for inventory data
-        # Create total_price from price field
+        # Calculate discounted price
+        if 'price_with_discount' not in df.columns and 'price' in df.columns:
+            if 'discount_percent' in df.columns:
+                df['price_with_discount'] = df['price'] * (1 - df['discount_percent'] / 100)
+            else:
+                df['price_with_discount'] = df['price']
+        
+        # Add total_price for compatibility (use price_with_discount)
         if 'total_price' not in df.columns:
-            if 'price' in df.columns:
-                # Calculate discounted price: price * (1 - discount/100)
-                if 'discount' in df.columns:
-                    df['total_price'] = df['price'] * (1 - df['discount'] / 100)
-                else:
-                    df['total_price'] = df['price']
+            if 'price_with_discount' in df.columns:
+                df['total_price'] = df['price_with_discount']
+            elif 'price' in df.columns:
+                df['total_price'] = df['price']
             else:
                 df['total_price'] = 0
         
@@ -526,6 +629,29 @@ class WildberriesAPI:
         if 'sale_amount' not in df.columns:
             df['sale_amount'] = df['total_price'] * df['quantity']
         
+        # Calculate total inventory value
+        if 'inventory_value' not in df.columns:
+            df['inventory_value'] = df['total_price'] * df['current_stock']
+        
+        # Add article field for backward compatibility
+        if 'article' not in df.columns and 'supplier_article' in df.columns:
+            df['article'] = df['supplier_article']
+        
+        # Mark low stock items (less than 5)
+        if 'is_low_stock' not in df.columns:
+            if 'current_stock' in df.columns:
+                df['is_low_stock'] = df['current_stock'] < 5
+            else:
+                df['is_low_stock'] = True
+        
+        # Calculate pipeline stock (items in transit)
+        if 'pipeline_stock' not in df.columns:
+            in_way_to_client = df['in_way_to_client'] if 'in_way_to_client' in df.columns else 0
+            in_way_from_client = df['in_way_from_client'] if 'in_way_from_client' in df.columns else 0
+            df['pipeline_stock'] = in_way_to_client + in_way_from_client
+        
+        logger.info(f"Processed {len(df)} stock records")
+        
         return df
     
     def _process_reportdetail_data(self, df: pd.DataFrame) -> pd.DataFrame: