File size: 15,828 Bytes
c534334 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 | import sqlite3
import pandas as pd
from datetime import datetime
from pathlib import Path
import json
from typing import List, Dict, Any, Optional
class DatabaseManager:
def __init__(self, db_path: str = "business_database.db"):
self.db_path = db_path
self.init_database()
def init_database(self):
"""Initialize database with required tables"""
with sqlite3.connect(self.db_path) as conn:
# Companies table
conn.execute("""
CREATE TABLE IF NOT EXISTS companies (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
address TEXT,
phone TEXT,
website TEXT,
email TEXT,
rating TEXT,
reviews TEXT,
category TEXT,
hours TEXT,
price TEXT,
description TEXT,
dataset_name TEXT NOT NULL,
business_type TEXT,
location TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
UNIQUE(name, address, dataset_name)
)
""")
# Business categories table (for multi-category support)
conn.execute("""
CREATE TABLE IF NOT EXISTS company_categories (
id INTEGER PRIMARY KEY AUTOINCREMENT,
company_id INTEGER,
category TEXT,
FOREIGN KEY (company_id) REFERENCES companies (id),
UNIQUE(company_id, category)
)
""")
# Contact collection status
conn.execute("""
CREATE TABLE IF NOT EXISTS contact_collection_status (
id INTEGER PRIMARY KEY AUTOINCREMENT,
company_id INTEGER,
website_scraped BOOLEAN DEFAULT FALSE,
email_found BOOLEAN DEFAULT FALSE,
phone_found BOOLEAN DEFAULT FALSE,
last_scraped TIMESTAMP,
FOREIGN KEY (company_id) REFERENCES companies (id)
)
""")
# Datasets table
conn.execute("""
CREATE TABLE IF NOT EXISTS datasets (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT UNIQUE NOT NULL,
description TEXT,
business_type TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
total_companies INTEGER DEFAULT 0
)
""")
conn.commit()
def create_dataset(self, name: str, description: str = "", business_type: str = "") -> bool:
"""Create a new dataset"""
try:
with sqlite3.connect(self.db_path) as conn:
conn.execute("""
INSERT INTO datasets (name, description, business_type)
VALUES (?, ?, ?)
""", (name, description, business_type))
conn.commit()
return True
except sqlite3.IntegrityError:
return False # Dataset already exists
def get_datasets(self) -> List[Dict]:
"""Get all datasets"""
with sqlite3.connect(self.db_path) as conn:
conn.row_factory = sqlite3.Row
cursor = conn.execute("""
SELECT d.*, COUNT(c.id) as company_count
FROM datasets d
LEFT JOIN companies c ON d.name = c.dataset_name
GROUP BY d.id
ORDER BY d.created_at DESC
""")
return [dict(row) for row in cursor.fetchall()]
def add_companies(self, companies: List[Dict], dataset_name: str, business_type: str = "") -> int:
"""Add companies to database, return number of new companies added"""
added_count = 0
with sqlite3.connect(self.db_path) as conn:
for company in companies:
try:
# Insert company
cursor = conn.execute("""
INSERT INTO companies (
name, address, phone, website, email, rating, reviews,
category, hours, price, description, dataset_name, business_type, location
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""", (
company.get('name', ''),
company.get('address', ''),
company.get('phone', ''),
company.get('website', ''),
company.get('email', ''),
company.get('rating', ''),
company.get('reviews', ''),
company.get('category', ''),
company.get('hours', ''),
company.get('price', ''),
company.get('description', ''),
dataset_name,
business_type,
company.get('location', '')
))
company_id = cursor.lastrowid
added_count += 1
# Add categories if multiple
categories = company.get('categories', [])
if not categories and company.get('category'):
categories = [company.get('category')]
for category in categories:
if category:
conn.execute("""
INSERT OR IGNORE INTO company_categories (company_id, category)
VALUES (?, ?)
""", (company_id, category))
# Initialize contact collection status
conn.execute("""
INSERT OR IGNORE INTO contact_collection_status (company_id)
VALUES (?)
""", (company_id,))
except sqlite3.IntegrityError:
# Company already exists, skip
continue
# Update dataset company count
conn.execute("""
UPDATE datasets
SET total_companies = (
SELECT COUNT(*) FROM companies WHERE dataset_name = ?
)
WHERE name = ?
""", (dataset_name, dataset_name))
conn.commit()
return added_count
def get_companies(self, dataset_name: str = None, business_type: str = None) -> pd.DataFrame:
"""Get companies from database"""
with sqlite3.connect(self.db_path) as conn:
query = """
SELECT c.*,
GROUP_CONCAT(cc.category, '; ') as all_categories,
ccs.website_scraped, ccs.email_found, ccs.phone_found, ccs.last_scraped
FROM companies c
LEFT JOIN company_categories cc ON c.id = cc.company_id
LEFT JOIN contact_collection_status ccs ON c.id = ccs.company_id
"""
params = []
conditions = []
if dataset_name:
conditions.append("c.dataset_name = ?")
params.append(dataset_name)
if business_type:
conditions.append("c.business_type = ?")
params.append(business_type)
if conditions:
query += " WHERE " + " AND ".join(conditions)
query += " GROUP BY c.id ORDER BY c.created_at DESC"
return pd.read_sql_query(query, conn, params=params)
def update_company(self, company_id: int, updates: Dict) -> bool:
"""Update company information"""
try:
with sqlite3.connect(self.db_path) as conn:
if not updates:
return True
# Add updated_at to updates
updates['updated_at'] = datetime.now().isoformat()
# Build the SET clause and values
set_clause = ", ".join([f"{key} = ?" for key in updates.keys()])
values = list(updates.values()) + [company_id]
query = f"""
UPDATE companies
SET {set_clause}
WHERE id = ?
"""
conn.execute(query, values)
conn.commit()
return True
except Exception as e:
print(f"Error updating company {company_id}: {e}")
print(f"Updates: {updates}")
print(f"Values: {values}")
return False
def delete_company(self, company_id: int) -> bool:
"""Delete a company from database"""
try:
with sqlite3.connect(self.db_path) as conn:
# Delete from contact_collection_status first (foreign key constraint)
conn.execute("DELETE FROM contact_collection_status WHERE company_id = ?", (company_id,))
# Delete from company_categories
conn.execute("DELETE FROM company_categories WHERE company_id = ?", (company_id,))
# Delete the company
conn.execute("DELETE FROM companies WHERE id = ?", (company_id,))
conn.commit()
return True
except Exception as e:
print(f"Error deleting company {company_id}: {e}")
return False
def mark_contact_collected(self, company_id: int, email_found: bool = False, phone_found: bool = False):
"""Mark contact collection status"""
with sqlite3.connect(self.db_path) as conn:
conn.execute("""
UPDATE contact_collection_status
SET website_scraped = TRUE,
email_found = ?,
phone_found = ?,
last_scraped = CURRENT_TIMESTAMP
WHERE company_id = ?
""", (email_found, phone_found, company_id))
conn.commit()
def get_companies_for_contact_collection(self, dataset_name: str = None, search_type: str = "both") -> List[Dict]:
"""Get companies that need contact information collection - only those missing emails or phones"""
with sqlite3.connect(self.db_path) as conn:
conn.row_factory = sqlite3.Row
# Build conditions based on search type
conditions = ["c.website IS NOT NULL AND c.website != ''"]
if search_type == "emails_only":
conditions.append("(c.email IS NULL OR c.email = '')")
elif search_type == "phones_only":
conditions.append("(c.phone IS NULL OR c.phone = '')")
else: # both or all
conditions.append("""
(
(c.email IS NULL OR c.email = '') OR
(c.phone IS NULL OR c.phone = '') OR
(ccs.website_scraped IS NULL OR ccs.website_scraped = 0)
)
""")
query = f"""
SELECT c.*, ccs.website_scraped, ccs.email_found, ccs.phone_found
FROM companies c
LEFT JOIN contact_collection_status ccs ON c.id = ccs.company_id
WHERE {' AND '.join(conditions)}
"""
params = []
if dataset_name:
query += " AND c.dataset_name = ?"
params.append(dataset_name)
query += " ORDER BY ccs.last_scraped ASC NULLS FIRST"
cursor = conn.execute(query, params)
return [dict(row) for row in cursor.fetchall()]
def delete_dataset(self, dataset_name: str) -> bool:
"""Delete a dataset and all its companies"""
try:
with sqlite3.connect(self.db_path) as conn:
# Get company IDs for this dataset
cursor = conn.execute("SELECT id FROM companies WHERE dataset_name = ?", (dataset_name,))
company_ids = [row[0] for row in cursor.fetchall()]
if company_ids:
# Delete related records
conn.execute("DELETE FROM company_categories WHERE company_id IN ({})".format(
','.join('?' * len(company_ids))), company_ids)
conn.execute("DELETE FROM contact_collection_status WHERE company_id IN ({})".format(
','.join('?' * len(company_ids))), company_ids)
conn.execute("DELETE FROM companies WHERE dataset_name = ?", (dataset_name,))
# Delete dataset
conn.execute("DELETE FROM datasets WHERE name = ?", (dataset_name,))
conn.commit()
return True
except Exception as e:
print(f"Error deleting dataset: {e}")
return False
def get_dataset_stats(self, dataset_name: str) -> Dict:
"""Get statistics for a dataset"""
with sqlite3.connect(self.db_path) as conn:
conn.row_factory = sqlite3.Row
# Basic stats
cursor = conn.execute("""
SELECT
COUNT(*) as total_companies,
COUNT(CASE WHEN email IS NOT NULL AND email != '' THEN 1 END) as with_email,
COUNT(CASE WHEN phone IS NOT NULL AND phone != '' THEN 1 END) as with_phone,
COUNT(CASE WHEN website IS NOT NULL AND website != '' THEN 1 END) as with_website
FROM companies
WHERE dataset_name = ?
""", (dataset_name,))
basic_stats = dict(cursor.fetchone())
# Contact collection stats
cursor = conn.execute("""
SELECT
COUNT(*) as scraped_count,
COUNT(CASE WHEN email_found = 1 THEN 1 END) as email_found_count,
COUNT(CASE WHEN phone_found = 1 THEN 1 END) as phone_found_count
FROM companies c
JOIN contact_collection_status ccs ON c.id = ccs.company_id
WHERE c.dataset_name = ? AND ccs.website_scraped = 1
""", (dataset_name,))
contact_stats = dict(cursor.fetchone())
# Companies needing contact collection
cursor = conn.execute("""
SELECT COUNT(*) as need_contact_collection
FROM companies c
LEFT JOIN contact_collection_status ccs ON c.id = ccs.company_id
WHERE c.dataset_name = ?
AND c.website IS NOT NULL AND c.website != ''
AND (
(c.email IS NULL OR c.email = '') OR
(c.phone IS NULL OR c.phone = '') OR
(ccs.website_scraped IS NULL OR ccs.website_scraped = 0)
)
""", (dataset_name,))
need_collection = dict(cursor.fetchone())
return {**basic_stats, **contact_stats, **need_collection}
|