web_scrapping.py

# -*- coding: utf-8 -*-
"""Web Scrapping.ipynb

Automatically generated by Colab.

Original file is located at
    https://colab.research.google.com/drive/1OLoBK18jpB685Ivi8Zi3SzuVYiXJ9jRa
"""

!pip install selenium
!pip install webdriver-manager

# Detik.com

import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
from datetime import datetime

def scrape_detik_search(keyword, max_pages=1):
    base_search_url = "https://www.detik.com/search/searchall"
    results = []

    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0 Safari/537.36'
    }

    for page in range(1, max_pages + 1):
        params = {
            'query': keyword,
            'siteid': '2',
            'sortby': 'time',
            'page': page
        }
        print(f"Scraping page {page}...")
        r = requests.get(base_search_url, params=params, headers=headers)
        if r.status_code != 200:
            print(f"Gagal akses halaman (status {r.status_code}), hentikan scraping.")
            break

        soup = BeautifulSoup(r.text, 'html.parser')

        news_list = soup.find_all('div', class_='media')

        if not news_list:
            print("Tidak ada berita ditemukan di halaman ini, hentikan scraping.")
            break

        for news in news_list:
            try:
                title_tag = news.find('h3', class_='media__title')
                if not title_tag:
                    continue
                link_tag = title_tag.find('a', class_='media__link')
                if not link_tag or not link_tag.has_attr('href'):
                    continue
                link = link_tag['href']
                title = link_tag.text.strip()

                date_tag = news.find('div', class_='media__date')
                if date_tag:
                    span_tag = date_tag.find('span')
                    if span_tag and span_tag.has_attr('d-time'):
                        timestamp = span_tag['d-time']
                        news_date = datetime.fromtimestamp(int(timestamp))
                    else:
                        news_date = None
                else:
                    news_date = None

                # if news_date and news_date < cutoff_date:
                #     print("Berita sudah melewati batas waktu 3 tahun, hentikan scraping.")
                #     return pd.DataFrame(results)

                # Ambil halaman detail berita dengan header
                news_resp = requests.get(link, headers=headers)
                if news_resp.status_code != 200:
                    print(f"Gagal akses detail berita: {link} (status {news_resp.status_code}), skip berita ini.")
                    continue

                news_soup = BeautifulSoup(news_resp.text, 'html.parser')

                content_div = news_soup.find('div', class_='detail__body-text') or \
                              news_soup.find('div', class_='detail_text')

                if content_div:
                    content_parts = []
                    for tag in content_div.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p']):
                        text = tag.get_text(strip=True)
                        if text:
                            prefix = tag.name.upper() if tag.name.startswith('h') else ''
                            if prefix:
                                content_parts.append(f"{prefix}: {text}")
                            else:
                                content_parts.append(text)
                    content = '\n'.join(content_parts)
                else:
                    content = ''

                # Ambil tag dari elemen nav > a.nav__item
                nav_div = news_soup.find('div', class_='nav')

                tags = []
                if nav_div:
                    tags = [a.text.strip() for a in nav_div.find_all('a', class_='nav__item')]

                results.append({
                    'judul': title,
                    'tanggal': news_date.strftime('%Y-%m-%d %H:%M') if news_date else '',
                    'tag': ', '.join(tags),
                    'isi_berita': content,
                    'link': link
                })

                print(f"Berhasil scrape berita: {title}")

                time.sleep(1)

            except Exception as e:
                print(f"Error saat memproses berita: {e}")
                continue

        time.sleep(2)

    return pd.DataFrame(results)

if __name__ == "__main__":
    keyword = "Kabupaten Cirebon"
    df = scrape_detik_search(keyword)
    if not df.empty:
        df.to_csv("detik_berita_cirebonnn.csv", index=False, encoding='utf-8-sig')
        print("Selesai menyimpan data berita ke detik_berita_cirebon.csv")
    else:
        print("Tidak ada data yang berhasil di-scrape.")

# Radar Cirebon KW

import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
from urllib.parse import quote_plus

BASE_HOST = "https://radarcirebon.disway.id"
BASE_SEARCH = BASE_HOST + "/search/kata/"

def make_search_url(keyword, page, per_page=30):
    q = quote_plus(keyword)
    if page == 1:
        return f"{BASE_SEARCH}?c={q}&num="
    else:
        offset = (page - 1) * per_page
        return f"{BASE_SEARCH}{offset}/{offset}/?c={q}&num="

def absolute_url(href):
    if not href:
        return None
    href = href.strip()
    if href.startswith("http://") or href.startswith("https://"):
        return href
    if href.startswith("/"):
        return BASE_HOST + href
    return BASE_HOST + "/" + href

def scrape_radar_cirebon(keyword, max_pages=100, per_page=30, delay_between_items=1.0, delay_between_pages=2.0):
    sess = requests.Session()
    sess.headers.update({
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0 Safari/537.36'
    })

    results = []
    seen_links = set()

    for page in range(1, max_pages + 1):
        url = make_search_url(keyword, page, per_page)
        print(f"\nScraping page {page} -> {url}")
        try:
            r = sess.get(url, timeout=15)
        except Exception as e:
            print(f"  ERROR: Gagal request halaman search: {e}")
            break

        if r.status_code != 200:
            print(f"  ERROR: status code {r.status_code}, hentikan scraping.")
            break

        soup = BeautifulSoup(r.text, "html.parser")

        # Ambil daftar berita
        news_list = soup.find_all(class_='media-heading')
        if not news_list:
            news_list = soup.find_all('div', class_='media')
        if not news_list:
            news_list = soup.find_all('article')
        if not news_list:
            news_list = soup.select('ul.search-results li') or soup.select('div.search-result') or []

        if not news_list:
            print("  Tidak ada berita ditemukan di halaman ini.")
            continue

        print(f"  Ketemu {len(news_list)} item.")

        for item in news_list:
            try:
                a = item.find('a', href=True) or item.select_one('a[href]')
                if not a:
                    continue

                link = absolute_url(a.get('href'))
                if not link or link in seen_links:
                    continue
                seen_links.add(link)

                title = a.get_text(strip=True)

                # Ambil halaman detail
                try:
                    detail_r = sess.get(link, timeout=15)
                except Exception as e:
                    print(f"    ERROR request detail {link}: {e}")
                    continue
                if detail_r.status_code != 200:
                    print(f"    ERROR status {detail_r.status_code} for {link}")
                    continue

                detail_soup = BeautifulSoup(detail_r.text, "html.parser")

                # Judul detail
                h1 = detail_soup.find('h1', class_='text-black') or detail_soup.find('h1')
                title_detail = h1.get_text(strip=True) if h1 else title

                # Tanggal detail
                date_text = None  # Inisialisasi variabel

                # Opsi 1: Cari tag dengan class 'date' secara langsung
                date_detail_tag = detail_soup.find('span', class_='date') or detail_soup.find(class_='date')
                if date_detail_tag:
                    print("Ditemukan dengan Target Langsung")
                    # PERBAIKAN: Gunakan variabel 'date_detail_tag', bukan 'tag'
                    date_text = date_detail_tag.get_text(strip=True)

                # Opsi 2: Jika Opsi 1 gagal, cari di dalam kontainer 'post-info'
                if not date_text:
                    post_info_div = detail_soup.find('div', class_='post-info')
                    if post_info_div:
                        tag_tanggal = post_info_div.find('span', class_='date')
                        if tag_tanggal:
                            print("Ditemukan dengan Target Kontainer")
                            date_text = tag_tanggal.get_text(strip=True)

                # Opsi 3: Jika masih gagal, gunakan Regex sebagai usaha terakhir
                if not date_text:
                    # Pola Regex untuk format seperti "Rabu 22-08-2024" atau "Selasa, 21 Agustus 2024"
                    date_pattern = re.compile(r'\w+,\s*\d{1,2}\s+\w+\s+\d{4}|\w+\s+\d{2}-\d{2}-\d{4}')
                    found_text = detail_soup.find(string=date_pattern)
                    if found_text:
                        print("Ditemukan dengan Target Pola Teks (Regex)")
                        date_text = found_text.strip()

                # Isi berita
                content_container = None
                for cls in ('entry-content', 'post-content', 'article-body', 'detail__body-text', 'detail_text', 'content', 'article__content'):
                    content_container = detail_soup.find('div', class_=cls)
                    if content_container:
                        break
                if not content_container:
                    content_container = detail_soup.find('article')

                content_parts = []
                search_scope = content_container if content_container else detail_soup
                for p in search_scope.find_all('p'):
                    text = p.get_text(strip=True)
                    if text and 'Baca Juga:' not in text:
                        content_parts.append(text)
                content = "\n".join(content_parts)

                tags = []
                try:
                    # 1. Cari SEMUA tag <a> yang tautannya (href) mengandung '/listtag/'
                    #    Ini adalah pola unik untuk tag di situs tersebut.
                    tag_links = detail_soup.find_all('a', href=lambda href: href and '/listtag/' in href)

                    # 2. Loop melalui setiap tautan tag yang ditemukan
                    for a_tag in tag_links:
                        # 3. Ambil teks dari atribut 'title', karena itu berisi nama tag yang bersih
                        tag_text = a_tag.get('title', '').strip()

                        # 4. Pastikan teks tidak kosong sebelum menambahkannya ke list
                        if tag_text:
                            tags.append(tag_text)

                    # Jika tidak ada tag yang ditemukan, list akan tetap kosong, yang mana sudah benar.

                except Exception as e:
                    # Menjaga agar program tidak berhenti jika ada error tak terduga
                    print(f"    Terjadi error saat mencari tag: {e}")

                # Gabungkan hasil tag menjadi satu string untuk disimpan
                final_tags = ", ".join(tags) if tags else "-"

                results.append({
                    "judul": title_detail,
                    "tanggal": date_text,
                    "tag": final_tags,  # INI BAGIAN YANG DIPERBAIKI
                    "isi_berita": content,
                    "link": link
                })

                print(f"    Berhasil: {title_detail} | Tags: {', '.join(tags) if tags else '-'}")

                time.sleep(delay_between_items)

            except Exception as e:
                print(f"    Error saat memproses item: {e}")
                continue

        time.sleep(delay_between_pages)

    df = pd.DataFrame(results)
    return df

if __name__ == "__main__":
    keyword = "kabupaten cirebon"
    df = scrape_radar_cirebon(keyword, max_pages=100)
    if not df.empty:
        df.to_csv("/content/drive/MyDrive/Machine Learning/Sentiment Analysis/radarcirebondisway_berita.csv", index=False, encoding="utf-8-sig")
        print("\nSelesai menyimpan data berita ke radarcirebon_berita.csv")
    else:
        print("\nTidak ada data yang berhasil di-scrape.")

# Antara News

import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
import re
import random
from urllib.parse import quote_plus, urlparse, urlunparse

BASE_HOST = "https://www.antaranews.com"
BASE_SEARCH = BASE_HOST + "/search"

def make_search_url(keyword, page):
    q = quote_plus(keyword)
    if page == 1:
        return f"{BASE_SEARCH}?q={q}"
    else:
        return f"{BASE_SEARCH}?q={q}&page={page}"

def absolute_url(href):
    if not href:
        return None
    href = href.strip()
    if href.startswith("http://") or href.startswith("https://"):
        return href
    if href.startswith("/"):
        return BASE_HOST + href
    return BASE_HOST + "/" + href

def normalize_url(href):
    """Buat URL konsisten: absolut + buang query/fragment + hapus trailing slash."""
    if not href:
        return None
    href = absolute_url(href)
    parsed = urlparse(href)
    clean = parsed._replace(query="", fragment="")
    return urlunparse(clean).rstrip("/")

def get_with_retry(sess, url, max_retries=3, delay_range=(2, 5)):
    """Request dengan retry & delay acak."""
    for attempt in range(max_retries):
        try:
            r = sess.get(url, timeout=15)
            r.raise_for_status()
            return r
        except Exception as e:
            print(f"    Percobaan {attempt+1} gagal: {e}")
            if attempt < max_retries - 1:
                time.sleep(random.uniform(*delay_range))
    return None

def scrape_antaranews(keyword, max_pages=5, delay_between_items=(1, 2), delay_between_pages=(2, 4)):
    sess = requests.Session()
    sess.headers.update({
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
                      'AppleWebKit/537.36 (KHTML, like Gecko) '
                      'Chrome/115.0 Safari/537.36'
    })

    results = []
    seen_links = set()

    for page in range(1, max_pages + 1):
        url = make_search_url(keyword, page)
        print(f"\nScraping page {page} -> {url}")

        r = get_with_retry(sess, url)
        if not r:
            print(f"  ERROR: Gagal request halaman search setelah retry.")
            continue

        soup = BeautifulSoup(r.text, "html.parser")

        # Ambil semua anchor yang mengarah ke artikel berita (biasanya /berita/...)
        anchors = soup.select('a[href*="/berita/"]')
        all_links_in_page = {normalize_url(a.get('href')) for a in anchors if a.get('href')}
        all_links_in_page = {l for l in all_links_in_page if l}
        new_links = all_links_in_page - seen_links
        print(f"  Ketemu {len(all_links_in_page)} link artikel di halaman ini, {len(new_links)} link baru.")

        seen_links.update(all_links_in_page)

        for link in sorted(new_links):
            detail_r = get_with_retry(sess, link)
            if not detail_r:
                print(f"    ERROR: Gagal request detail {link}")
                continue

            detail_soup = BeautifulSoup(detail_r.text, "html.parser")

            # Judul
            h1 = detail_soup.select_one('div.wrap__article-detail-title h1') or detail_soup.find('h1')
            title_detail = h1.get_text(strip=True) if h1 else ""

            # Waktu / tanggal
            date_detail = ""
            cal_icon = detail_soup.select_one('i.fa-calendar') or detail_soup.select_one('i.fas.fa-calendar')
            if cal_icon:
                parent_li = cal_icon.find_parent('li') or cal_icon.find_parent()
                if parent_li:
                    date_detail = parent_li.get_text(" ", strip=True)
            if not date_detail:
                text_all = detail_soup.get_text(" ", strip=True)
                m = re.search(r'\b(?:[A-Za-z]+,\s*\d{1,2}\s+[A-Za-z]+ \d{4}\s*\d{1,2}:\d{2}\s*WIB|\d+\s+jam lalu|\bWIB\b)', text_all)
                if m:
                    date_detail = m.group(0)

            # Isi berita
            content_parts = []
            article_body = detail_soup.find('div', class_='wrap__article-detail-content') \
                           or detail_soup.find('div', class_='detail__body-text') \
                           or detail_soup.find('article')
            search_scope = article_body if article_body else detail_soup
            for p in search_scope.find_all('p'):
                text = p.get_text(strip=True)
                if text and not text.lower().startswith("baca juga"):
                    content_parts.append(text)
            content = "\n".join(content_parts)

            # Ambil tag
            tags = []
            found = False
            for ul in detail_soup.find_all('ul', class_='list-inline'):
                if ul.find('i', class_='fa-tags') or ul.find('i', class_='fas fa-tags'):
                    for a in ul.find_all('a', href=True):
                        if '/tag/' in a['href']:
                            tag_text = a.get('title') if a.get('title') else a.get_text(strip=True)
                            if tag_text:
                                tags.append(tag_text)
                    if tags:
                        found = True
                        break
            if not found:
                for a in detail_soup.select('a[href*="/tag/"]'):
                    tag_text = a.get('title') if a.get('title') else a.get_text(strip=True)
                    if tag_text:
                        tags.append(tag_text)
            tags = list(dict.fromkeys(tags))

            results.append({
                "judul": title_detail,
                "tanggal": date_detail,
                "tag": ", ".join(tags) if tags else "-",
                "isi_berita": content,
                "link": link
            })

            print(f"    Berhasil: {title_detail} | Tanggal: {date_detail if date_detail else '-'} | Tags: {', '.join(tags) if tags else '-'}")

            time.sleep(random.uniform(*delay_between_items))

        time.sleep(random.uniform(*delay_between_pages))

    df = pd.DataFrame(results)
    return df

if __name__ == "__main__":
    keyword = "kabupaten cirebon"
    df = scrape_antaranews(keyword, max_pages=100)
    if not df.empty:
        df.to_csv("antaranews_berita.csv", index=False, encoding="utf-8-sig")
        print(f"\nSelesai menyimpan {len(df)} data berita ke antaranews_berita.csv")
    else:
        print("\nTidak ada data yang berhasil di-scrape.")

# Jalanin di IDE lokal karena butuh chrome (CNN)

import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
import random
from urllib.parse import quote, urlparse, urlunparse
import re

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from webdriver_manager.chrome import ChromeDriverManager
from selenium.common.exceptions import TimeoutException

BASE_HOST = "https://www.cnnindonesia.com"

# <<< DIUBAH: Fungsi ini dimodifikasi untuk menangani nomor halaman >>>
def make_search_url(keyword, page):
    """
    Membuat URL pencarian yang benar untuk setiap halaman.
    """
    q = quote(keyword)
    base_url = f"{BASE_HOST}/search?query={q}&result_type=latest"
    if page == 1:
        return base_url
    else:
        return f"{base_url}&page={page}"

# --- Fungsi-fungsi pembantu lainnya tidak ada perubahan ---
def absolute_url(href):
    if not href: return None
    href = href.strip()
    if href.startswith("http://") or href.startswith("https://"): return href
    if href.startswith("/"): return BASE_HOST + href
    return BASE_HOST + "/" + href

def normalize_url(href):
    if not href: return None
    href = absolute_url(href)
    parsed = urlparse(href)
    clean = parsed._replace(query="", fragment="")
    return urlunparse(clean).rstrip("/")

def parse_cnn_date(raw_date):
    if not raw_date: return "-"
    if '|' in raw_date: raw_date = raw_date.split('|')[1]
    raw = raw_date.replace(" WIB", "").strip()
    try:
        from datetime import datetime
        import locale
        try: locale.setlocale(locale.LC_TIME, 'id_ID.UTF-8')
        except locale.Error: locale.setlocale(locale.LC_TIME, '')
        dt = datetime.strptime(raw, "%A, %d %b %Y %H:%M")
        return dt.strftime("%Y-%m-%d %H:%M")
    except Exception: return raw_date.strip()

def looks_like_article_href(href):
    if not href: return False
    parsed = urlparse(href.strip())
    path = parsed.path
    if any(skip in path for skip in ['/search', '/tag', '/kategori', '/author', '/channel', '/indeks', '/video', '/foto']): return False
    if re.search(r'/\d{14}-\d{2,3}-\d{6,}', path): return True
    return False

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0 Safari/537.36"}

def fetch_article_detail(url, retries=3, delay=3):
    for attempt in range(1, retries + 1):
        try:
            resp = requests.get(url, headers=HEADERS, timeout=15)
            if resp.status_code == 200: return resp.text
            else: print(f"  WARNING: HTTP {resp.status_code} saat akses {url}")
        except Exception as e: print(f"  WARNING: Gagal akses {url} ({attempt}/{retries}): {e}")
        time.sleep(delay)
    return None

def scrape_cnn_with_selenium(keyword, max_pages=3, delay_between_items=(1,2)):
    results = []
    seen_links = set()

    print("Menginisialisasi browser Chrome...")
    service = Service(ChromeDriverManager().install())
    options = webdriver.ChromeOptions()
    options.add_argument("--headless")
    options.add_argument("--disable-blink-features=AutomationControlled")
    options.add_experimental_option("excludeSwitches", ["enable-automation"])
    options.add_experimental_option('useAutomationExtension', False)

    driver = webdriver.Chrome(service=service, options=options)
    driver.set_page_load_timeout(30)

    # <<< DIUBAH: Logika perulangan kembali menggunakan nomor halaman (bukan scroll) >>>
    for page in range(1, max_pages + 1):
        # Membuat URL untuk halaman yang dituju
        url = make_search_url(keyword, page)
        print(f"\nMembuka halaman {page} -> {url}")
        driver.get(url)

        # Penanganan cookie hanya perlu saat pertama kali halaman dimuat (page 1)
        if page == 1:
            try:
                print("Mencari pop-up cookie...")
                cookie_agree_button = WebDriverWait(driver, 10).until(
                    EC.element_to_be_clickable((By.XPATH, "//button[text()='AGREE']"))
                )
                cookie_agree_button.click()
                print("  Pop-up cookie ditemukan dan ditutup.")
                time.sleep(2)
            except TimeoutException:
                print("  Pop-up cookie tidak ditemukan, melanjutkan proses.")

        print(f"Mengambil data dari halaman {page}...")

        try:
            # Menunggu konten dimuat di setiap halaman baru
            WebDriverWait(driver, 15).until(
                EC.presence_of_element_located((By.CSS_SELECTOR, "div.nhl-list article a"))
            )
        except TimeoutException:
            print(f"  WARNING: Waktu habis menunggu konten di halaman {page}. Mungkin halaman ini kosong.")
            continue # Lanjut ke halaman berikutnya jika ada

        page_html = driver.page_source
        soup = BeautifulSoup(page_html, "html.parser")

        link_elements = soup.select('div.nhl-list article a[href]')

        all_links_in_page = {normalize_url(a['href']) for a in link_elements if looks_like_article_href(a['href'])}
        new_links = all_links_in_page - seen_links

        if not new_links:
            print("  Tidak ada link baru yang ditemukan di halaman ini.")
            # Tidak perlu berhenti, karena halaman berikutnya mungkin punya link baru

        print(f"  Ditemukan {len(new_links)} link baru.")
        seen_links.update(new_links)

        # Proses scrape detail artikel tidak ada perubahan
        for link in sorted(new_links):
            print(f"   -> Memproses: {link}")
            html_detail = fetch_article_detail(link)
            if not html_detail: continue
            detail_soup = BeautifulSoup(html_detail, "html.parser")

            title_el = detail_soup.select_one('h1')
            title_text = title_el.get_text(strip=True) if title_el else "-"

            date_el = detail_soup.select_one('div.text-cnn_grey.text-sm')
            date_text = parse_cnn_date(date_el.get_text(strip=True)) if date_el else "-"

            tags_list = []
            topik_terkait_header = detail_soup.find('div', class_='title-box', text=re.compile(r'\s*TOPIK TERKAIT\s*'))
            if topik_terkait_header:
                tags_container = topik_terkait_header.find_next_sibling('div')
                if tags_container:
                    tags_elements = tags_container.select('a')
                    tags_list = [tag.get_text(strip=True) for tag in tags_elements]

            content_parts = []
            content_container = detail_soup.select_one("div.detail-text")
            if content_container:
                for p in content_container.find_all('p'):
                    text = p.get_text(" ", strip=True)
                    if text and not text.lower().startswith("lihat juga") and not text.lower().startswith("scroll to continue"):
                        content_parts.append(text)

            results.append({
                "judul": title_text, "tanggal": date_text,
                "tag": ", ".join(tags_list) if tags_list else "-",
                "isi_berita": "\n".join(content_parts) if content_parts else "-", "link": link
            })
            print(f"    Berhasil: {title_text} | Tanggal: {date_text}")
            time.sleep(random.uniform(*delay_between_items))

    print("\nMenutup browser...")
    driver.quit()
    return pd.DataFrame(results)

if __name__ == "__main__":
    keyword = "kabupaten cirebon"
    df = scrape_cnn_with_selenium(keyword, max_pages=100)
    if not df.empty:
        df.to_csv("cnnindonesia_berita_final.csv", index=False, encoding="utf-8-sig")
        print(f"\nSelesai menyimpan {len(df)} data berita ke cnnindonesia_berita_final.csv")
    else:
        print("\nTidak ada data yang berhasil di-scrape.")

# Radar Cirebon ID

import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
import random
from urllib.parse import quote, urlparse, urlunparse
import re

# Mengganti BASE_HOST ke situs target yang baru
BASE_HOST = "https://radarcirebon.id"

def make_search_url(keyword, page):
    """
    Membuat URL pencarian sesuai format radarcirebon.id.
    Contoh: https://radarcirebon.id/search/kabupaten+cirebon/page/2/
    """
    # Mengganti spasi dengan '+' sesuai format URL situs
    q = quote(keyword).replace('%20', '+')
    if page == 1:
        return f"{BASE_HOST}/search/{q}/"
    else:
        return f"{BASE_HOST}/search/{q}/page/{page}/"

def normalize_url(href):
    """
    Memastikan URL dalam format absolut dan bersih (tanpa parameter).
    """
    if not href:
        return None
    href = href.strip()
    # Membuat URL absolut jika hanya berupa path
    if href.startswith("//"):
        href = "https:" + href
    elif href.startswith("/"):
        href = BASE_HOST + href
    elif not href.startswith("http"):
        return None # Mengabaikan link yang tidak valid

    parsed = urlparse(href)
    clean = parsed._replace(query="", fragment="")
    return urlunparse(clean).rstrip("/")

def parse_radarcirebon_date(raw_date):
    """
    Mengubah format tanggal dari 'Selasa, 12 Agu 2025 - 11:01'
    menjadi format standar 'YYYY-MM-DD HH:MM'.
    """
    if not raw_date:
        return "-"
    try:
        # Pemetaan manual untuk nama bulan 3 huruf dalam Bahasa Indonesia
        month_map = {
            'Jan': '01', 'Feb': '02', 'Mar': '03', 'Apr': '04', 'Mei': '05', 'Jun': '06',
            'Jul': '07', 'Agu': '08', 'Sep': '09', 'Okt': '10', 'Nov': '11', 'Des': '12'
        }
        # Membersihkan hari dan memisahkan bagian-bagian tanggal
        date_part = raw_date.split(', ')[1] # -> "12 Agu 2025 - 11:01"
        parts = date_part.replace(' - ', ' ').split() # -> ['12', 'Agu', '2025', '11:01']

        day = parts[0].zfill(2) # zfill(2) untuk memastikan format '01', '02', dst.
        month_abbr = parts[1]
        year = parts[2]
        time_str = parts[3]

        # Mengambil angka bulan dari pemetaan
        month = month_map.get(month_abbr, '00')

        return f"{year}-{month}-{day} {time_str}"
    except Exception:
        return raw_date.strip()

def looks_like_article_href(href):
    """
    Memfilter URL agar hanya mengambil link artikel yang valid.
    Contoh URL artikel: /2025/08/12/nama-artikel/
    """
    if not href:
        return False
    # Pola URL artikel di radarcirebon.id selalu mengandung /YYYY/MM/DD/
    return bool(re.search(r'/\d{4}/\d{2}/\d{2}/', href))

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                  "AppleWebKit/537.36 (KHTML, like Gecko) "
                  "Chrome/126.0.0.0 Safari/537.36"
}

def fetch_url(url, retries=3, delay=3):
    """
    Fungsi untuk mengambil konten dari sebuah URL dengan mekanisme coba lagi (retry).
    """
    for attempt in range(1, retries + 1):
        try:
            resp = requests.get(url, headers=HEADERS, timeout=15)
            if resp.status_code == 200:
                return resp.text
            else:
                print(f"   WARNING: HTTP {resp.status_code} saat akses {url}")
        except Exception as e:
            print(f"   WARNING: Gagal akses {url} ({attempt}/{retries}): {e}")
        time.sleep(delay)
    return None

def scrape_radarcirebon(keyword, max_pages=3, delay_between_items=(1, 2), delay_between_pages=(2, 4)):
    """
    Fungsi utama untuk melakukan scraping dari situs radarcirebon.id.
    """
    results = []
    seen_links = set()

    for page in range(1, max_pages + 1):
        url = make_search_url(keyword, page)
        print(f"\nScraping halaman {page} -> {url}")

        html = fetch_url(url)
        if not html:
            print(f"   ERROR: Gagal mengambil halaman pencarian {page}")
            continue

        soup = BeautifulSoup(html, "html.parser")

        # Selektor CSS baru untuk menemukan link artikel di halaman pencarian
        link_elements = soup.select('article .wp-block-latest-posts__post-title a')
        print(f"   DEBUG: Ditemukan {len(link_elements)} elemen link di halaman {page}")

        all_links_in_page = set()
        for a in link_elements:
            href_raw = a.get('href')
            if href_raw and looks_like_article_href(href_raw):
                norm = normalize_url(href_raw)
                if norm:
                    all_links_in_page.add(norm)

        new_links = all_links_in_page - seen_links
        print(f"   Menemukan {len(all_links_in_page)} link artikel di halaman ini, {len(new_links)} link baru.")
        seen_links.update(all_links_in_page)

        for link in sorted(list(new_links)):
            html_detail = fetch_url(link)
            if not html_detail:
                print(f"     ERROR: Gagal mengambil artikel {link}")
                continue

            detail_soup = BeautifulSoup(html_detail, "html.parser")

            # Selektor baru untuk judul artikel
            title_el = detail_soup.select_one('h1.entry-title')
            title_detail = title_el.get_text(strip=True) if title_el else "-"

            # Selektor baru untuk tanggal
            date_el = detail_soup.select_one('time.entry-date')
            date_detail = parse_radarcirebon_date(date_el.get_text(strip=True)) if date_el else "-"

            # Selektor baru untuk isi berita
            content_parts = []
            content_container = detail_soup.select_one('div.entry-content')
            if content_container:
                for p in content_container.select('p'):
                    # Mengabaikan paragraf yang berisi link "Baca Juga"
                    if not p.find_parent(class_='read-also'):
                        text = p.get_text(" ", strip=True)
                        if text:
                            content_parts.append(text)
            content = "\n".join(content_parts)

            # Selektor baru untuk tag
            tags_container = detail_soup.select_one('div.wp-block-tag-cloud')
            tags = [a.get_text(strip=True) for a in tags_container.select('a')] if tags_container else []
            tags = list(dict.fromkeys(tags)) # Menghapus duplikat

            results.append({
                "judul": title_detail,
                "tanggal": date_detail,
                "tag": ", ".join(tags) if tags else "-",
                "isi_berita": content if content else "-",
                "link": link
            })
            print(f"     Berhasil: {title_detail} | Tanggal: {date_detail}")
            time.sleep(random.uniform(*delay_between_items))

        # Beri jeda antar halaman untuk tidak membebani server
        time.sleep(random.uniform(*delay_between_pages))

    return pd.DataFrame(results)

if __name__ == "__main__":
    keyword = "kabupaten cirebon"
    # Batasi max_pages sesuai kebutuhan Anda, misalnya 3 halaman
    df = scrape_radarcirebon(keyword, max_pages=3)
    if not df.empty:
        # Menyimpan ke file CSV baru
        output_filename = "radarcirebon_berita.csv"
        df.to_csv(output_filename, index=False, encoding="utf-8-sig")
        print(f"\nSelesai menyimpan {len(df)} data berita ke {output_filename}")
    else:
        print("\nTidak ada data yang berhasil di-scrape.")

# Download html

import requests

url = "https://radarcirebon.id/2025/08/12/warga-resah-dprd-cirebon-panggil-dpkpp-untuk-tuntaskan-masalah-psu-di-dua-perumahan/"
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0 Safari/537.36"
}
resp = requests.get(url, headers=headers)
with open("detail.html", "w", encoding="utf-8") as f:
    f.write(resp.text)
print("HTML halaman disimpan ke page.html")

# Detik.com memiliki batas waktu

import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
from datetime import datetime

def scrape_detik_search(keyword, max_years=3, max_pages=100):
    base_search_url = "https://www.detik.com/search/searchall"
    results = []

    cutoff_date = datetime.now().replace(year=datetime.now().year - max_years)

    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0 Safari/537.36'
    }

    for page in range(1, max_pages + 1):
        params = {
            'query': keyword,
            'siteid': '2',
            'sortby': 'time',
            'page': page
        }
        print(f"Scraping page {page}...")
        r = requests.get(base_search_url, params=params, headers=headers)
        if r.status_code != 200:
            print(f"Gagal akses halaman (status {r.status_code}), hentikan scraping.")
            break

        soup = BeautifulSoup(r.text, 'html.parser')

        news_list = soup.find_all('div', class_='media')

        if not news_list:
            print("Tidak ada berita ditemukan di halaman ini, hentikan scraping.")
            break

        for news in news_list:
            try:
                title_tag = news.find('h3', class_='media__title')
                if not title_tag:
                    continue
                link_tag = title_tag.find('a', class_='media__link')
                if not link_tag or not link_tag.has_attr('href'):
                    continue
                link = link_tag['href']
                title = link_tag.text.strip()

                date_tag = news.find('div', class_='media__date')
                if date_tag:
                    span_tag = date_tag.find('span')
                    if span_tag and span_tag.has_attr('d-time'):
                        timestamp = span_tag['d-time']
                        news_date = datetime.fromtimestamp(int(timestamp))
                    else:
                        news_date = None
                else:
                    news_date = None

                if news_date and news_date < cutoff_date:
                    print("Berita sudah melewati batas waktu 3 tahun, hentikan scraping.")
                    return pd.DataFrame(results)

                # Ambil halaman detail berita dengan header
                news_resp = requests.get(link, headers=headers)
                if news_resp.status_code != 200:
                    print(f"Gagal akses detail berita: {link} (status {news_resp.status_code}), skip berita ini.")
                    continue

                news_soup = BeautifulSoup(news_resp.text, 'html.parser')

                content_div = news_soup.find('div', class_='detail__body-text') or \
                              news_soup.find('div', class_='detail_text')

                if content_div:
                    content_parts = []
                    for tag in content_div.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p']):
                        text = tag.get_text(strip=True)
                        if text:
                            prefix = tag.name.upper() if tag.name.startswith('h') else ''
                            if prefix:
                                content_parts.append(f"{prefix}: {text}")
                            else:
                                content_parts.append(text)
                    content = '\n'.join(content_parts)
                else:
                    content = ''

                tag_list_div = news_soup.find('div', class_='tag__list') or \
                               news_soup.find('div', class_='detail_tag')

                tags = []
                if tag_list_div:
                    tags = [t.text.strip() for t in tag_list_div.find_all('a')]

                results.append({
                    'judul': title,
                    'tanggal': news_date.strftime('%Y-%m-%d %H:%M') if news_date else '',
                    'tag': ', '.join(tags),
                    'isi_berita': content,
                    'link': link
                })

                print(f"Berhasil scrape berita: {title}")

                time.sleep(1)

            except Exception as e:
                print(f"Error saat memproses berita: {e}")
                continue

        time.sleep(2)

    return pd.DataFrame(results)

if __name__ == "__main__":
    keyword = "Kabupaten Cirebon"
    df = scrape_detik_search(keyword)
    if not df.empty:
        df.to_csv("detik_berita_cirebonnn.csv", index=False, encoding='utf-8-sig')
        print("Selesai menyimpan data berita ke detik_berita_cirebon.csv")
    else:
        print("Tidak ada data yang berhasil di-scrape.")