Rename configuration and content files for improved project structure

docs/sample_urlscan_result.json

@@ -1,488 +0,0 @@
-{
-  "data": {
-    "requests": [
-      {
-        "request": {
-          "requestId": "FE77961E144ADAA2A7D7760E8CE7EB6A",
-          "loaderId": "FE77961E144ADAA2A7D7760E8CE7EB6A",
-          "documentURL": "https://example.com/",
-          "request": {
-            "url": "https://example.com/",
-            "method": "GET",
-            "headers": {
-              "Upgrade-Insecure-Requests": "1",
-              "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36"
-            },
-            "mixedContentType": "none",
-            "initialPriority": "VeryHigh",
-            "referrerPolicy": "strict-origin-when-cross-origin",
-            "isSameSite": true
-          },
-          "timestamp": 34065601.895598,
-          "wallTime": 1766933427.494116,
-          "initiator": {
-            "type": "other"
-          },
-          "redirectHasExtraInfo": false,
-          "type": "Document",
-          "frameId": "61B9C1B13CBD0AC8E19A4CB13D24DE8E",
-          "hasUserGesture": false,
-          "primaryRequest": true
-        },
-        "response": {
-          "encodedDataLength": 536,
-          "dataLength": 513,
-          "requestId": "FE77961E144ADAA2A7D7760E8CE7EB6A",
-          "type": "Document",
-          "response": {
-            "url": "https://example.com/",
-            "status": 200,
-            "statusText": "",
-            "headers": {
-              "age": "13121",
-              "allow": "GET, HEAD",
-              "cf-cache-status": "HIT",
-              "cf-ray": "9b51dec2092c3738-FRA",
-              "content-encoding": "gzip",
-              "content-type": "text/html",
-              "date": "Sun, 28 Dec 2025 14:50:27 GMT",
-              "last-modified": "Sat, 20 Dec 2025 05:43:40 GMT",
-              "server": "cloudflare",
-              "vary": "Accept-Encoding"
-            },
-            "mimeType": "text/html",
-            "charset": "",
-            "remoteIPAddress": "104.18.27.120",
-            "remotePort": 443,
-            "encodedDataLength": 536,
-            "timing": {
-              "requestTime": 34065601.895873,
-              "proxyStart": -1,
-              "proxyEnd": -1,
-              "dnsStart": 0,
-              "dnsEnd": 5.906,
-              "connectStart": 5.942,
-              "connectEnd": 25.448,
-              "sslStart": 11.119,
-              "sslEnd": 25.442,
-              "workerStart": -1,
-              "workerReady": -1,
-              "workerFetchStart": -1,
-              "workerRespondWithSettled": -1,
-              "sendStart": 25.509,
-              "sendEnd": 25.562,
-              "pushStart": 0,
-              "pushEnd": 0,
-              "receiveHeadersStart": 83.246,
-              "receiveHeadersEnd": 84.37
-            },
-            "responseTime": 1766933427577.645,
-            "protocol": "h2",
-            "alternateProtocolUsage": "unspecifiedReason",
-            "securityState": "secure",
-            "securityDetails": {
-              "protocol": "TLS 1.3",
-              "keyExchange": "",
-              "keyExchangeGroup": "X25519MLKEM768",
-              "cipher": "AES_128_GCM",
-              "certificateId": 0,
-              "subjectName": "example.com",
-              "sanList": [
-                "example.com",
-                "*.example.com"
-              ],
-              "issuer": "Cloudflare TLS Issuing ECC CA 3",
-              "validFrom": 1765913972,
-              "validTo": 1773685964,
-              "signedCertificateTimestampList": [
-                {
-                  "status": "Verified",
-                  "origin": "Embedded in certificate",
-                  "logDescription": "DigiCert 'Wyvern2026h1'",
-                  "logId": "6411C46CA412ECA7891CA2022E00BCAB4F2807D41E3527ABEAFED503C97DCDF0",
-                  "timestamp": 1765914573355,
-                  "hashAlgorithm": "SHA-256",
-                  "signatureAlgorithm": "ECDSA",
-                  "signatureData": "304502207EAD9214E5E3A94FD2FFFC3AACE657456FE8357607018CE2D8B75E80D14359C5022100F3A4736BB259749EB4EBF22271E5F29F15688F2193F659FC0AE7A3C2A9C4BF41"
-                },
-                {
-                  "status": "Verified",
-                  "origin": "Embedded in certificate",
-                  "logDescription": "Cloudflare 'Nimbus2026'",
-                  "logId": "CB38F715897C84A1445F5BC1DDFBC96EF29A59CD470A690585B0CB14C31458E7",
-                  "timestamp": 1765914573388,
-                  "hashAlgorithm": "SHA-256",
-                  "signatureAlgorithm": "ECDSA",
-                  "signatureData": "304402207CB577326AA9CEE312D5B03DE9930181A98958F77CB911850DF269CD39D1A09802201A31F3B9B9AA2A3010320E38E619B3EE5D1F998FED2CAEC05CB9EBFEE5C1C36A"
-                },
-                {
-                  "status": "Verified",
-                  "origin": "TLS extension",
-                  "logDescription": "Google 'Xenon2026h1' log",
-                  "logId": "969764BF555897ADF743876837084277E9F03AD5F6A4F3366E46A43F0FCAA9C6",
-                  "timestamp": 1765914576543,
-                  "hashAlgorithm": "SHA-256",
-                  "signatureAlgorithm": "ECDSA",
-                  "signatureData": "304402203CB5BE0CE4168D2F0F81BC896F518D7FF7EFF8B7BA72448C6B7C46DC6056CB15022055BAA26FBD349EADFA9B314204B5BA049E669BDDF386CA4F2B126BE27EDA183A"
-                },
-                {
-                  "status": "Verified",
-                  "origin": "TLS extension",
-                  "logDescription": "Google 'Argon2026h1' log",
-                  "logId": "0E5794BCF3AEA93E331B2C9907B3F790DF9BC23D713225DD21A925AC61C54E21",
-                  "timestamp": 1765914576444,
-                  "hashAlgorithm": "SHA-256",
-                  "signatureAlgorithm": "ECDSA",
-                  "signatureData": "3046022100A9914FC1A2559979D8351623D4CD7C8B3D52F29F9C3E816E59E0E7617DF21E9F022100B34E0A80A3547FF2DEEBE9A3A3E33ED211C23255A03A2F415750DCB2B02BC424"
-                }
-              ],
-              "certificateTransparencyCompliance": "compliant",
-              "serverSignatureAlgorithm": 1027,
-              "encryptedClientHello": false
-            },
-            "isIpProtectionUsed": false
-          },
-          "hasExtraInfo": true,
-          "hash": "6f5635035f36ad500b4fc4bb7816bb72ef5594e1bcae44fa074c5e988fc4c0fe",
-          "size": 513,
-          "asn": {
-            "ip": "104.18.27.120",
-            "asn": "13335",
-            "country": "US",
-            "description": "CLOUDFLARENET, US",
-            "name": "CLOUDFLARENET",
-            "route": "104.16.0.0/14"
-          },
-          "geoip": {
-            "country": "",
-            "region": "",
-            "timezone": "",
-            "city": "",
-            "ll": [
-              null,
-              null
-            ],
-            "country_name": "Ascension Island"
-          }
-        }
-      }
-    ],
-    "cookies": [],
-    "console": [],
-    "links": [
-      {
-        "href": "https://iana.org/domains/example",
-        "text": "Learn more",
-        "frameId": "61B9C1B13CBD0AC8E19A4CB13D24DE8E"
-      }
-    ],
-    "timing": {
-      "beginNavigation": "2025-12-28T14:50:27.492Z",
-      "frameStartedLoading": "2025-12-28T14:50:27.493Z",
-      "frameNavigated": "2025-12-28T14:50:27.581Z",
-      "domContentEventFired": "2025-12-28T14:50:27.593Z",
-      "frameStoppedLoading": "2025-12-28T14:50:27.595Z"
-    },
-    "globals": []
-  },
-  "lists": {
-    "ips": [
-      "104.18.27.120"
-    ],
-    "countries": [],
-    "asns": [
-      "13335"
-    ],
-    "domains": [
-      "example.com"
-    ],
-    "servers": [
-      "cloudflare"
-    ],
-    "urls": [
-      "https://example.com/"
-    ],
-    "linkDomains": [
-      "iana.org"
-    ],
-    "certificates": [
-      {
-        "subjectName": "example.com",
-        "issuer": "Cloudflare TLS Issuing ECC CA 3",
-        "validFrom": 1765913972,
-        "validTo": 1773685964
-      }
-    ],
-    "hashes": [
-      "6f5635035f36ad500b4fc4bb7816bb72ef5594e1bcae44fa074c5e988fc4c0fe"
-    ]
-  },
-  "meta": {
-    "processors": {
-      "umbrella": {
-        "data": [
-          {
-            "hostname": "example.com",
-            "rank": 6940
-          }
-        ]
-      },
-      "geoip": {
-        "data": [
-          {
-            "ip": "104.18.27.120",
-            "geoip": {
-              "country": "",
-              "region": "",
-              "timezone": "",
-              "city": "",
-              "ll": [
-                null,
-                null
-              ],
-              "country_name": "Ascension Island"
-            }
-          }
-        ]
-      },
-      "asn": {
-        "data": [
-          {
-            "ip": "104.18.27.120",
-            "asn": "13335",
-            "country": "US",
-            "description": "CLOUDFLARENET, US",
-            "name": "CLOUDFLARENET",
-            "route": "104.16.0.0/14"
-          }
-        ]
-      },
-      "rdns": {
-        "data": []
-      },
-      "wappa": {
-        "data": []
-      }
-    }
-  },
-  "page": {
-    "asn": "AS13335",
-    "asnname": "CLOUDFLARENET, US",
-    "city": "",
-    "country": "",
-    "domain": "example.com",
-    "ip": "104.18.27.120",
-    "server": "cloudflare",
-    "url": "https://example.com/",
-    "apexDomainAgeDays": 4034,
-    "language": "en",
-    "mimeType": "text/html",
-    "title": "Example Domain",
-    "tlsValidDays": 89,
-    "tlsAgeDays": 11,
-    "domainAgeDays": 4034,
-    "tlsValidFrom": "2025-12-16T19:39:32.000Z",
-    "umbrellaRank": 6940,
-    "apexDomain": "example.com",
-    "tlsIssuer": "Cloudflare TLS Issuing ECC CA 3",
-    "status": "200"
-  },
-  "scanner": {
-    "country": "de"
-  },
-  "stats": {
-    "IPv6Percentage": 0,
-    "adBlocked": 0,
-    "domainStats": [
-      {
-        "count": 1,
-        "ips": [
-          "104.18.27.120"
-        ],
-        "domain": "example.com",
-        "size": 513,
-        "encodedSize": 536,
-        "countries": [],
-        "index": 0,
-        "initiators": [],
-        "redirects": 0
-      }
-    ],
-    "ipStats": [
-      {
-        "requests": 1,
-        "domains": [
-          "example.com"
-        ],
-        "ip": "104.18.27.120",
-        "asn": {
-          "ip": "104.18.27.120",
-          "asn": "13335",
-          "country": "US",
-          "description": "CLOUDFLARENET, US",
-          "name": "CLOUDFLARENET",
-          "route": "104.16.0.0/14"
-        },
-        "dns": {},
-        "geoip": {
-          "country": "",
-          "region": "",
-          "timezone": "",
-          "city": "",
-          "ll": [
-            null,
-            null
-          ],
-          "country_name": "Ascension Island"
-        },
-        "size": 513,
-        "encodedSize": 536,
-        "countries": [],
-        "index": 0,
-        "ipv6": false,
-        "redirects": 0,
-        "count": null
-      }
-    ],
-    "malicious": 0,
-    "protocolStats": [
-      {
-        "count": 1,
-        "size": 513,
-        "encodedSize": 536,
-        "ips": [
-          "104.18.27.120"
-        ],
-        "countries": [],
-        "securityState": {},
-        "protocol": "h2"
-      }
-    ],
-    "regDomainStats": [
-      {
-        "count": 1,
-        "ips": [
-          "104.18.27.120"
-        ],
-        "regDomain": "example.com",
-        "size": 513,
-        "encodedSize": 536,
-        "countries": [],
-        "index": 0,
-        "subDomains": [
-          {
-            "domain": "",
-            "country": ""
-          }
-        ],
-        "redirects": 0
-      }
-    ],
-    "resourceStats": [
-      {
-        "count": 1,
-        "size": 513,
-        "encodedSize": 536,
-        "latency": 0,
-        "countries": [],
-        "ips": [
-          "104.18.27.120"
-        ],
-        "type": "Document",
-        "compression": "1.0",
-        "percentage": 100
-      }
-    ],
-    "securePercentage": 100,
-    "secureRequests": 1,
-    "serverStats": [
-      {
-        "count": 1,
-        "size": 513,
-        "encodedSize": 536,
-        "ips": [
-          "104.18.27.120"
-        ],
-        "countries": [],
-        "server": "cloudflare"
-      }
-    ],
-    "tlsStats": [
-      {
-        "count": 1,
-        "size": 513,
-        "encodedSize": 536,
-        "ips": [
-          "104.18.27.120"
-        ],
-        "countries": [],
-        "protocols": {
-          "TLS 1.3 /  / AES_128_GCM": 1
-        },
-        "securityState": "secure"
-      }
-    ],
-    "totalLinks": 1,
-    "uniqCountries": 1
-  },
-  "submitter": {
-    "country": "SE"
-  },
-  "task": {
-    "method": "api",
-    "source": "b1c2f290",
-    "tags": [],
-    "time": "2025-12-28T14:50:32.844Z",
-    "url": "https://example.com/",
-    "uuid": "019b6570-74f7-7269-aa5a-48da029c142c",
-    "visibility": "public",
-    "reportURL": "https://urlscan.io/result/019b6570-74f7-7269-aa5a-48da029c142c/",
-    "screenshotURL": "https://urlscan.io/screenshots/019b6570-74f7-7269-aa5a-48da029c142c.png",
-    "domURL": "https://urlscan.io/dom/019b6570-74f7-7269-aa5a-48da029c142c/",
-    "domain": "example.com",
-    "apexDomain": "example.com"
-  },
-  "verdicts": {
-    "overall": {
-      "score": 0,
-      "categories": [],
-      "brands": [],
-      "tags": [],
-      "malicious": false,
-      "hasVerdicts": true
-    },
-    "urlscan": {
-      "score": 0,
-      "categories": [],
-      "brands": [],
-      "tags": [],
-      "malicious": false,
-      "hasVerdicts": false
-    },
-    "engines": {
-      "score": -100,
-      "malicious": false,
-      "categories": [],
-      "enginesTotal": 0,
-      "maliciousTotal": 0,
-      "benignTotal": 0,
-      "maliciousVerdicts": [],
-      "benignVerdicts": [],
-      "hasVerdicts": true,
-      "tags": [
-        "urlscan-ml",
-        "urlscan-ml-0.1.0-1-d9114fd"
-      ]
-    },
-    "community": {
-      "score": 0,
-      "categories": [],
-      "brands": [],
-      "votesTotal": 0,
-      "votesMalicious": 0,
-      "votesBenign": 0,
-      "malicious": false,
-      "hasVerdicts": false
-    }
-  }
-}

docs/urlscan_api.md

@@ -1,169 +0,0 @@
-# URLScan.io API Integration
-
-This module provides a Python client for interacting with the URLScan.io API to analyze URLs for phishing detection.
-
-## Setup
-
-1. **Get an API key:**
-   - Sign up at https://urlscan.io/user/signup
-   - Get your API key from your account settings
-
-2. **Set environment variable:**
-   ```bash
-   export URLSCAN_API_KEY="your_api_key_here"
-   ```
-
-   Or create a `.env` file:
-   ```bash
-   cp .env.example .env
-   # Edit .env and add your API key
-   ```
-
-## Basic Usage
-
-### Initialize Client
-
-```python
-from phising_detection.api import URLScanClient
-
-# Using environment variable
-client = URLScanClient()
-
-# Or pass API key directly
-client = URLScanClient(api_key="your_api_key")
-```
-
-### Submit a URL for Scanning
-
-```python
-# Submit URL
-result = client.submit_url(
-    url="https://suspicious-site.com",
-    visibility="public",  # or "unlisted" or "private"
-    tags=["phishing", "test"]
-)
-
-uuid = result["uuid"]
-print(f"Scan UUID: {uuid}")
-```
-
-### Retrieve Results
-
-```python
-# Get results by UUID
-scan_result = client.get_result(uuid)
-
-# Access scan data
-page_title = scan_result["page"]["title"]
-screenshot = scan_result["task"]["screenshotURL"]
-```
-
-### Submit and Wait for Results
-
-```python
-# Submit and automatically wait for completion
-result = client.submit_and_wait(
-    url="https://example.com",
-    max_wait=60,  # seconds
-    poll_interval=5  # seconds between checks
-)
-```
-
-### Get Verdict
-
-```python
-# Get simple verdict (malicious/safe)
-verdict = client.get_verdict(uuid)
-print(f"Verdict: {verdict}")  # "malicious" or "safe"
-```
-
-### Search Existing Scans
-
-```python
-# Search for scans by domain
-results = client.search(
-    query="domain:example.com",
-    size=10
-)
-
-for scan in results["results"]:
-    print(scan["task"]["url"])
-```
-
-## API Response Examples
-
-### Submission Response
-
-```json
-{
-  "uuid": "abc123...",
-  "result": "https://urlscan.io/result/abc123.../",
-  "api": "https://urlscan.io/api/v1/result/abc123.../"
-}
-```
-
-### Result Response
-
-```json
-{
-  "page": {
-    "url": "https://example.com",
-    "title": "Example Domain",
-    "status": "200"
-  },
-  "verdicts": {
-    "overall": {
-      "score": 0,
-      "malicious": false
-    }
-  },
-  "task": {
-    "uuid": "abc123...",
-    "time": "2024-01-01T12:00:00.000Z",
-    "screenshotURL": "https://..."
-  }
-}
-```
-
-## Error Handling
-
-```python
-from phising_detection.api import URLScanClient, URLScanError
-
-try:
-    client = URLScanClient()
-    result = client.submit_url("https://example.com")
-except URLScanError as e:
-    print(f"Error: {e}")
-```
-
-## Rate Limits
-
-- Free tier: 50 submissions per day
-- Paid tier: Higher limits available
-- The client handles rate limit errors automatically
-
-## Integration with Phishing Detection
-
-```python
-from phising_detection.api import URLScanClient
-from phising_detection.data import load_phishing_urls
-
-# Load your phishing URLs
-df = load_phishing_urls()
-
-# Analyze URLs
-client = URLScanClient()
-
-for idx, row in df.head(10).iterrows():  # Sample first 10
-    try:
-        result = client.submit_and_wait(row['url'])
-        verdict = client.get_verdict(result['task']['uuid'])
-        print(f"{row['url']}: {verdict}")
-    except URLScanError as e:
-        print(f"Error scanning {row['url']}: {e}")
-```
-
-## API Documentation
-
-Full API documentation: https://urlscan.io/docs/api/