#!/usr/bin/env python3 """ 🚀 M365 COMPLETE LOCAL HARVESTER ================================ Harvester for ALLE Microsoft 365 tjenester via lokale metoder: - Outlook (COM) - Teams (Local Cache + LevelDB) - SharePoint (OneDrive Sync) - OneDrive (Local Sync Folder) - Loop (Local Cache) Ingen Azure AD App nødvendig - bruger lokale filer og COM! """ import os import sys import json import sqlite3 import hashlib import shutil import struct from pathlib import Path from datetime import datetime, timedelta from dataclasses import dataclass, asdict from typing import List, Dict, Optional, Any from concurrent.futures import ThreadPoolExecutor import re # Neo4j from neo4j import GraphDatabase # ============================================================ # CONFIGURATION # ============================================================ NEO4J_URI = "neo4j+s://054eff27.databases.neo4j.io" NEO4J_USER = "neo4j" NEO4J_PASSWORD = "Qrt37mkb0xBZ7_ts5tG1J70K2mVDGPMF2L7Njlm7cg8" # Lokale paths (Windows) USER_HOME = Path(os.environ.get("USERPROFILE", os.path.expanduser("~"))) APPDATA_LOCAL = Path(os.environ.get("LOCALAPPDATA", USER_HOME / "AppData" / "Local")) APPDATA_ROAMING = Path(os.environ.get("APPDATA", USER_HOME / "AppData" / "Roaming")) # M365 Local Paths PATHS = { "teams_cache": APPDATA_ROAMING / "Microsoft" / "Teams", "teams_new": APPDATA_LOCAL / "Packages" / "MSTeams_8wekyb3d8bbwe" / "LocalCache" / "Microsoft" / "MSTeams", "onedrive": USER_HOME / "OneDrive - TDC", "onedrive_business": USER_HOME / "OneDrive - TDC Holding A_S", "sharepoint_sync": USER_HOME / "TDC Holding A_S", "loop_cache": APPDATA_LOCAL / "Microsoft" / "Loop", "outlook_cache": APPDATA_LOCAL / "Microsoft" / "Outlook", "edge_profile": APPDATA_LOCAL / "Microsoft" / "Edge" / "User Data" / "Default", } # Søgetermer SEARCH_KEYWORDS = [ "strategi", "cyber", "NIS2", "SOC", "MDR", "cloud", "Azure", "AI", "Copilot", "Columbus", "ERP", "budget", "forecast", "kunde", "kontrakt", "rammeaftale", "SKI", "produkt", "CloudKey", "arkitektur", "roadmap", "projekt", "meeting", "beslutning", "action", "deadline" ] # ============================================================ # DATA CLASSES # ============================================================ @dataclass class HarvestedItem: """Generisk harvested item""" id: str source: str # outlook, teams, sharepoint, onedrive, loop item_type: str # email, message, file, document, note title: str content_preview: str author: str timestamp: str path: str keywords: List[str] metadata: Dict[str, Any] # ============================================================ # BASE HARVESTER # ============================================================ class BaseHarvester: """Base class for alle harvesters""" def __init__(self, neo4j_driver): self.neo4j = neo4j_driver self.items: List[HarvestedItem] = [] self.stats = {"found": 0, "matched": 0, "errors": 0} def match_keywords(self, text: str) -> List[str]: """Match keywords i tekst""" if not text: return [] text_lower = text.lower() return [kw for kw in SEARCH_KEYWORDS if kw.lower() in text_lower] def save_to_neo4j(self, item: HarvestedItem): """Gem item i Neo4j""" content_hash = hashlib.md5(f"{item.source}:{item.id}".encode()).hexdigest() with self.neo4j.session() as session: session.run(""" MERGE (i:M365Item {contentHash: $hash}) ON CREATE SET i.itemId = $id, i.source = $source, i.itemType = $itemType, i.title = $title, i.contentPreview = $preview, i.author = $author, i.timestamp = $timestamp, i.path = $path, i.keywords = $keywords, i.harvestedAt = datetime() ON MATCH SET i.lastSeen = datetime() MERGE (ds:DataSource {name: $dsName}) ON CREATE SET ds.type = 'local_m365' MERGE (i)-[:HARVESTED_FROM]->(ds) """, hash=content_hash, id=item.id, source=item.source, itemType=item.item_type, title=item.title[:500] if item.title else "", preview=item.content_preview[:2000] if item.content_preview else "", author=item.author, timestamp=item.timestamp, path=item.path, keywords=item.keywords, dsName=f"TDC_M365_{item.source.capitalize()}" ) # Keyword relationships for kw in item.keywords: session.run(""" MERGE (k:SearchKeyword {name: $kw}) WITH k MATCH (i:M365Item {contentHash: $hash}) MERGE (i)-[:MATCHES_KEYWORD]->(k) """, kw=kw, hash=content_hash) # ============================================================ # OUTLOOK HARVESTER (COM) # ============================================================ class OutlookHarvester(BaseHarvester): """Harvest Outlook via COM""" def __init__(self, neo4j_driver): super().__init__(neo4j_driver) self.outlook = None self.namespace = None def connect(self) -> bool: """Forbind til Outlook via COM""" try: import win32com.client import pythoncom pythoncom.CoInitialize() print(" 🔌 Forbinder til Outlook...") self.outlook = win32com.client.Dispatch("Outlook.Application") self.namespace = self.outlook.GetNamespace("MAPI") accounts = self.namespace.Accounts print(f" ✅ Outlook forbundet! ({accounts.Count} konti)") return True except Exception as e: print(f" ❌ Outlook fejl: {e}") return False def harvest(self, days_back: int = 90) -> List[HarvestedItem]: """Harvest emails""" if not self.connect(): return [] print(" 📧 Scanner emails...") cutoff = datetime.now() - timedelta(days=days_back) # Scan Inbox og Sent folders_to_scan = ["Inbox", "Sent Items", "Sendt post"] for folder_name in folders_to_scan: try: folder = self.namespace.GetDefaultFolder(6 if "Inbox" in folder_name else 5) items = folder.Items items.Sort("[ReceivedTime]", True) count = 0 for item in items: try: if item.Class != 43: # MailItem continue received = item.ReceivedTime if hasattr(received, 'year'): item_date = datetime(received.year, received.month, received.day) if item_date < cutoff: break subject = str(item.Subject or "") body = str(item.Body or "")[:2000] sender = str(item.SenderEmailAddress or "") keywords = self.match_keywords(f"{subject} {body}") if keywords: harvested = HarvestedItem( id=item.EntryID, source="outlook", item_type="email", title=subject, content_preview=body[:500], author=sender, timestamp=received.strftime("%Y-%m-%d %H:%M") if hasattr(received, 'strftime') else str(received), path=folder_name, keywords=keywords, metadata={"has_attachments": item.Attachments.Count > 0} ) self.items.append(harvested) self.save_to_neo4j(harvested) self.stats["matched"] += 1 self.stats["found"] += 1 count += 1 if count >= 500: break except Exception as e: self.stats["errors"] += 1 continue except Exception as e: print(f" ⚠️ Folder fejl: {e}") print(f" 📧 Outlook: {self.stats['matched']}/{self.stats['found']} matched") return self.items # ============================================================ # TEAMS HARVESTER (Local Cache) # ============================================================ class TeamsHarvester(BaseHarvester): """Harvest Teams via local cache files""" def __init__(self, neo4j_driver): super().__init__(neo4j_driver) self.cache_path = None def find_cache(self) -> Optional[Path]: """Find Teams cache location""" # Prøv ny Teams først (Store version) if PATHS["teams_new"].exists(): self.cache_path = PATHS["teams_new"] return self.cache_path # Fallback til gammel Teams if PATHS["teams_cache"].exists(): self.cache_path = PATHS["teams_cache"] return self.cache_path return None def harvest(self, days_back: int = 90) -> List[HarvestedItem]: """Harvest Teams messages fra local cache""" print(" 💬 Scanner Teams cache...") if not self.find_cache(): print(" ⚠️ Teams cache ikke fundet") return [] print(f" 📁 Cache: {self.cache_path}") # Find IndexedDB / LevelDB files leveldb_paths = list(self.cache_path.rglob("*.ldb")) + list(self.cache_path.rglob("*.log")) for db_file in leveldb_paths[:20]: # Begrænset antal try: self._parse_leveldb_file(db_file) except Exception as e: self.stats["errors"] += 1 # Find JSON cache files json_files = list(self.cache_path.rglob("*.json")) for json_file in json_files[:50]: try: self._parse_json_cache(json_file) except Exception as e: self.stats["errors"] += 1 print(f" 💬 Teams: {self.stats['matched']}/{self.stats['found']} matched") return self.items def _parse_leveldb_file(self, filepath: Path): """Parse LevelDB fil for Teams data""" try: with open(filepath, 'rb') as f: content = f.read() # Søg efter JSON-lignende strukturer json_pattern = rb'\{[^{}]{50,5000}\}' matches = re.findall(json_pattern, content) for match in matches: try: text = match.decode('utf-8', errors='ignore') data = json.loads(text) # Check for message-lignende struktur if any(key in data for key in ['content', 'message', 'body', 'text']): content_text = data.get('content') or data.get('message') or data.get('body') or data.get('text', '') if isinstance(content_text, str) and len(content_text) > 20: keywords = self.match_keywords(content_text) if keywords: item = HarvestedItem( id=hashlib.md5(content_text[:100].encode()).hexdigest(), source="teams", item_type="message", title=content_text[:100], content_preview=content_text[:500], author=data.get('from', {}).get('user', {}).get('displayName', 'Unknown') if isinstance(data.get('from'), dict) else str(data.get('from', 'Unknown')), timestamp=data.get('createdDateTime', data.get('timestamp', '')), path=str(filepath), keywords=keywords, metadata={"channel": data.get('channelId', '')} ) self.items.append(item) self.save_to_neo4j(item) self.stats["matched"] += 1 self.stats["found"] += 1 except (json.JSONDecodeError, UnicodeDecodeError): continue except Exception as e: pass def _parse_json_cache(self, filepath: Path): """Parse JSON cache fil""" try: with open(filepath, 'r', encoding='utf-8', errors='ignore') as f: data = json.load(f) # Rekursivt søg efter messages self._extract_messages(data, str(filepath)) except Exception as e: pass def _extract_messages(self, data: Any, filepath: str, depth: int = 0): """Rekursivt udtræk messages fra nested data""" if depth > 5: return if isinstance(data, dict): # Check for message content content = data.get('content') or data.get('body') or data.get('message') if content and isinstance(content, str) and len(content) > 30: keywords = self.match_keywords(content) if keywords: item = HarvestedItem( id=hashlib.md5(content[:100].encode()).hexdigest(), source="teams", item_type="message", title=content[:100], content_preview=content[:500], author=str(data.get('from', 'Unknown')), timestamp=str(data.get('createdDateTime', '')), path=filepath, keywords=keywords, metadata={} ) self.items.append(item) self.save_to_neo4j(item) self.stats["matched"] += 1 self.stats["found"] += 1 # Recurse into dict values for value in data.values(): self._extract_messages(value, filepath, depth + 1) elif isinstance(data, list): for item in data[:100]: # Limit self._extract_messages(item, filepath, depth + 1) # ============================================================ # ONEDRIVE / SHAREPOINT HARVESTER (Local Sync) # ============================================================ class OneDriveSharePointHarvester(BaseHarvester): """Harvest OneDrive og SharePoint via local sync folders""" def __init__(self, neo4j_driver): super().__init__(neo4j_driver) self.sync_paths: List[Path] = [] def find_sync_folders(self) -> List[Path]: """Find alle OneDrive/SharePoint sync folders""" self.sync_paths = [] # Check standard paths for key in ["onedrive", "onedrive_business", "sharepoint_sync"]: path = PATHS.get(key) if path and path.exists(): self.sync_paths.append(path) print(f" 📁 Fundet: {path}") # Søg efter andre OneDrive folders for item in USER_HOME.iterdir(): if item.is_dir() and "OneDrive" in item.name and item not in self.sync_paths: self.sync_paths.append(item) print(f" 📁 Fundet: {item}") return self.sync_paths def harvest(self, days_back: int = 90) -> List[HarvestedItem]: """Harvest filer fra sync folders""" print(" 📂 Scanner OneDrive/SharePoint...") self.find_sync_folders() if not self.sync_paths: print(" ⚠️ Ingen sync folders fundet") return [] cutoff = datetime.now() - timedelta(days=days_back) # Filtyper at scanne extensions = {'.docx', '.xlsx', '.pptx', '.pdf', '.txt', '.md', '.doc', '.xls', '.ppt'} for sync_path in self.sync_paths: source = "sharepoint" if "SharePoint" in str(sync_path) or "TDC Holding" in str(sync_path) else "onedrive" for filepath in sync_path.rglob("*"): try: if not filepath.is_file(): continue if filepath.suffix.lower() not in extensions: continue # Check modificeret tid mtime = datetime.fromtimestamp(filepath.stat().st_mtime) if mtime < cutoff: continue # Extract content preview content_preview = self._extract_content(filepath) keywords = self.match_keywords(f"{filepath.name} {content_preview}") if keywords: item = HarvestedItem( id=hashlib.md5(str(filepath).encode()).hexdigest(), source=source, item_type="document", title=filepath.name, content_preview=content_preview[:500], author="", # Kan udtrækkes fra metadata timestamp=mtime.strftime("%Y-%m-%d %H:%M"), path=str(filepath.relative_to(sync_path)), keywords=keywords, metadata={ "size": filepath.stat().st_size, "extension": filepath.suffix, "full_path": str(filepath) } ) self.items.append(item) self.save_to_neo4j(item) self.stats["matched"] += 1 self.stats["found"] += 1 except Exception as e: self.stats["errors"] += 1 print(f" 📂 OneDrive/SharePoint: {self.stats['matched']}/{self.stats['found']} matched") return self.items def _extract_content(self, filepath: Path) -> str: """Udtræk tekst content fra fil""" try: suffix = filepath.suffix.lower() if suffix in ['.txt', '.md']: with open(filepath, 'r', encoding='utf-8', errors='ignore') as f: return f.read()[:2000] elif suffix == '.docx': return self._extract_docx(filepath) elif suffix == '.xlsx': return self._extract_xlsx(filepath) elif suffix == '.pptx': return self._extract_pptx(filepath) elif suffix == '.pdf': return self._extract_pdf(filepath) except Exception as e: pass return "" def _extract_docx(self, filepath: Path) -> str: """Udtræk tekst fra DOCX""" try: import zipfile with zipfile.ZipFile(filepath, 'r') as z: xml_content = z.read('word/document.xml').decode('utf-8', errors='ignore') # Strip XML tags text = re.sub(r'<[^>]+>', ' ', xml_content) text = re.sub(r'\s+', ' ', text) return text[:2000] except: return "" def _extract_xlsx(self, filepath: Path) -> str: """Udtræk tekst fra XLSX""" try: import zipfile with zipfile.ZipFile(filepath, 'r') as z: if 'xl/sharedStrings.xml' in z.namelist(): xml_content = z.read('xl/sharedStrings.xml').decode('utf-8', errors='ignore') text = re.sub(r'<[^>]+>', ' ', xml_content) text = re.sub(r'\s+', ' ', text) return text[:2000] except: pass return "" def _extract_pptx(self, filepath: Path) -> str: """Udtræk tekst fra PPTX""" try: import zipfile texts = [] with zipfile.ZipFile(filepath, 'r') as z: for name in z.namelist(): if name.startswith('ppt/slides/slide') and name.endswith('.xml'): xml_content = z.read(name).decode('utf-8', errors='ignore') text = re.sub(r'<[^>]+>', ' ', xml_content) texts.append(text) return ' '.join(texts)[:2000] except: return "" def _extract_pdf(self, filepath: Path) -> str: """Udtræk tekst fra PDF (basic)""" try: with open(filepath, 'rb') as f: content = f.read() # Simple PDF text extraction text_matches = re.findall(rb'\(([^)]+)\)', content) texts = [m.decode('utf-8', errors='ignore') for m in text_matches[:100]] return ' '.join(texts)[:2000] except: return "" # ============================================================ # LOOP HARVESTER (Local Cache) # ============================================================ class LoopHarvester(BaseHarvester): """Harvest Microsoft Loop via local cache""" def __init__(self, neo4j_driver): super().__init__(neo4j_driver) def harvest(self, days_back: int = 90) -> List[HarvestedItem]: """Harvest Loop notes fra local cache""" print(" 🔄 Scanner Loop cache...") loop_path = PATHS.get("loop_cache") if not loop_path or not loop_path.exists(): # Søg alternative paths alt_paths = [ APPDATA_LOCAL / "Microsoft" / "Loop", APPDATA_LOCAL / "Packages" / "Microsoft.MicrosoftLoop_8wekyb3d8bbwe", USER_HOME / ".loop", ] for alt in alt_paths: if alt.exists(): loop_path = alt break if not loop_path or not loop_path.exists(): print(" ⚠️ Loop cache ikke fundet") return [] print(f" 📁 Loop path: {loop_path}") # Scan for Loop files for filepath in loop_path.rglob("*"): try: if not filepath.is_file(): continue # Loop gemmer som JSON eller SQLite if filepath.suffix in ['.json', '.db', '.sqlite']: self._parse_loop_file(filepath) except Exception as e: self.stats["errors"] += 1 print(f" 🔄 Loop: {self.stats['matched']}/{self.stats['found']} matched") return self.items def _parse_loop_file(self, filepath: Path): """Parse Loop fil""" try: if filepath.suffix == '.json': with open(filepath, 'r', encoding='utf-8', errors='ignore') as f: data = json.load(f) self._extract_loop_content(data, str(filepath)) elif filepath.suffix in ['.db', '.sqlite']: self._parse_sqlite(filepath) except Exception as e: pass def _extract_loop_content(self, data: Any, filepath: str, depth: int = 0): """Rekursivt udtræk Loop content""" if depth > 5: return if isinstance(data, dict): # Loop component content content = data.get('content') or data.get('text') or data.get('title') if content and isinstance(content, str) and len(content) > 20: keywords = self.match_keywords(content) if keywords: item = HarvestedItem( id=hashlib.md5(content[:100].encode()).hexdigest(), source="loop", item_type="note", title=content[:100], content_preview=content[:500], author=str(data.get('author', 'Unknown')), timestamp=str(data.get('modifiedTime', '')), path=filepath, keywords=keywords, metadata={"type": data.get('type', 'unknown')} ) self.items.append(item) self.save_to_neo4j(item) self.stats["matched"] += 1 self.stats["found"] += 1 for value in data.values(): self._extract_loop_content(value, filepath, depth + 1) elif isinstance(data, list): for item in data[:100]: self._extract_loop_content(item, filepath, depth + 1) def _parse_sqlite(self, filepath: Path): """Parse SQLite database""" try: # Kopier til temp for at undgå lock temp_path = filepath.parent / f"{filepath.name}.tmp" shutil.copy2(filepath, temp_path) conn = sqlite3.connect(temp_path) cursor = conn.cursor() # Find tabeller med content cursor.execute("SELECT name FROM sqlite_master WHERE type='table'") tables = cursor.fetchall() for (table_name,) in tables: try: cursor.execute(f"SELECT * FROM {table_name} LIMIT 100") rows = cursor.fetchall() for row in rows: row_text = ' '.join(str(cell) for cell in row if cell) keywords = self.match_keywords(row_text) if keywords and len(row_text) > 50: item = HarvestedItem( id=hashlib.md5(row_text[:100].encode()).hexdigest(), source="loop", item_type="data", title=row_text[:100], content_preview=row_text[:500], author="", timestamp="", path=f"{filepath}:{table_name}", keywords=keywords, metadata={"table": table_name} ) self.items.append(item) self.save_to_neo4j(item) self.stats["matched"] += 1 self.stats["found"] += 1 except Exception as e: continue conn.close() temp_path.unlink(missing_ok=True) except Exception as e: pass # ============================================================ # MAIN M365 HARVESTER # ============================================================ class M365CompleteHarvester: """Komplet M365 harvester der kører alle sub-harvesters""" def __init__(self): self.neo4j = GraphDatabase.driver(NEO4J_URI, auth=(NEO4J_USER, NEO4J_PASSWORD)) self.harvesters = {} self.total_stats = { "outlook": {"found": 0, "matched": 0}, "teams": {"found": 0, "matched": 0}, "onedrive": {"found": 0, "matched": 0}, "sharepoint": {"found": 0, "matched": 0}, "loop": {"found": 0, "matched": 0}, } self.all_items: List[HarvestedItem] = [] # Output self.output_dir = Path("data/m365_harvest") self.output_dir.mkdir(parents=True, exist_ok=True) def run(self, days_back: int = 90, sources: List[str] = None): """Kør komplet harvest""" if sources is None: sources = ["outlook", "teams", "onedrive", "loop"] print("\n" + "=" * 60) print("🚀 M365 COMPLETE LOCAL HARVESTER") print("=" * 60) print(f" 📅 Periode: Sidste {days_back} dage") print(f" 🔍 Keywords: {len(SEARCH_KEYWORDS)}") print(f" 📦 Sources: {', '.join(sources)}") print("=" * 60) # Outlook if "outlook" in sources: print("\n📧 OUTLOOK") harvester = OutlookHarvester(self.neo4j) items = harvester.harvest(days_back) self.all_items.extend(items) self.total_stats["outlook"] = harvester.stats # Teams if "teams" in sources: print("\n💬 TEAMS") harvester = TeamsHarvester(self.neo4j) items = harvester.harvest(days_back) self.all_items.extend(items) self.total_stats["teams"] = harvester.stats # OneDrive/SharePoint if "onedrive" in sources or "sharepoint" in sources: print("\n📂 ONEDRIVE / SHAREPOINT") harvester = OneDriveSharePointHarvester(self.neo4j) items = harvester.harvest(days_back) self.all_items.extend(items) # Split stats for item in items: self.total_stats[item.source]["matched"] += 1 # Loop if "loop" in sources: print("\n🔄 LOOP") harvester = LoopHarvester(self.neo4j) items = harvester.harvest(days_back) self.all_items.extend(items) self.total_stats["loop"] = harvester.stats # Gem output self._save_results() # Print summary self._print_summary() # Cleanup self.neo4j.close() def _save_results(self): """Gem resultater til JSON""" output_file = self.output_dir / f"m365_harvest_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" with open(output_file, 'w', encoding='utf-8') as f: json.dump({ "timestamp": datetime.now().isoformat(), "stats": self.total_stats, "keywords": SEARCH_KEYWORDS, "items": [asdict(item) for item in self.all_items] }, f, indent=2, ensure_ascii=False) print(f"\n💾 Saved: {output_file}") def _print_summary(self): """Print harvest summary""" print("\n" + "=" * 60) print("📊 HARVEST SUMMARY") print("=" * 60) total_matched = 0 total_found = 0 for source, stats in self.total_stats.items(): matched = stats.get("matched", 0) found = stats.get("found", 0) total_matched += matched total_found += found icon = {"outlook": "📧", "teams": "💬", "onedrive": "☁️", "sharepoint": "📁", "loop": "🔄"}.get(source, "📦") print(f" {icon} {source.capitalize():12} {matched:5} / {found:5} matched") print(" " + "-" * 40) print(f" {'TOTAL':15} {total_matched:5} / {total_found:5} matched") print("=" * 60) # Keyword stats if self.all_items: print("\n🏷️ TOP KEYWORDS:") keyword_counts = {} for item in self.all_items: for kw in item.keywords: keyword_counts[kw] = keyword_counts.get(kw, 0) + 1 for kw, count in sorted(keyword_counts.items(), key=lambda x: -x[1])[:10]: print(f" • {kw}: {count}") print("=" * 60) # ============================================================ # CLI # ============================================================ def main(): import argparse parser = argparse.ArgumentParser(description="M365 Complete Local Harvester") parser.add_argument("--days", type=int, default=90, help="Dage tilbage (default: 90)") parser.add_argument("--sources", nargs="+", default=["outlook", "teams", "onedrive", "loop"], choices=["outlook", "teams", "onedrive", "sharepoint", "loop"], help="Sources at harveste") args = parser.parse_args() harvester = M365CompleteHarvester() harvester.run(days_back=args.days, sources=args.sources) if __name__ == "__main__": main()