Update app.py

app.py

@@ -96,6 +96,42 @@ def get_azure_response(messages, api_key, azure_api_base):
     if finish_reason == 'length' or finish_reason == 'stop':
         return response['choices'][0]['message']['content']
         
+def cek_url(url):
+    if not os.path.exists("log_url.txt"):
+        with open("log_url.txt", 'w') as file:
+            pass
+
+    with open("log_url.txt", 'r') as file:
+        scraped_urls = set(url.strip() for url in file.readlines())
+
+    if url in scraped_urls:
+        return True
+    else:
+        scraped_urls.add(url)
+        return False
+
+def scrap_portal(query):
+    api_key = 'AIzaSyDJUWVZG2oHkHSsYoqdqgUZwQC2Aa2kSok'
+    search_engine_id = 'a0dc878459ceb4811'
+    num_pages = 5
+    link = []
+
+    for page in range(num_pages):
+        start_index = page * 10 + 1
+        url = f'https://www.googleapis.com/customsearch/v1?key={api_key}&cx={search_engine_id}&q={query}&start={start_index}'
+        response = requests.get(url)
+        if response.status_code == 200:
+            data = response.json()
+
+            for item in data['items']:
+                url = item['link']
+                link.append(url)
+        else:
+            print(f"Permintaan halaman {page + 1} gagal. Kode status:", response.status_code)
+    filter_link1 = [url for url in link if "categories" not in url and "tags" not in url]
+    filter_link2 = [url for url in filter_link1 if "help" not in url]
+    return filter_link2
+        
 def clean_scrap(artikel,link,models,api_key,azure_api_base):
     new_artikel = []
     article = []
@@ -170,7 +206,7 @@ def clean_scrap(artikel,link,models,api_key,azure_api_base):
 
     return title, judul, link, contents
 
-def scrap_artikel(link_scrap,models,api_key,azure_api_base):
+def scrap_artikel(source_type,source,models,api_key,azure_api_base):
     options = webdriver.ChromeOptions()
     options.add_argument('--headless')
     options.add_argument('--no-sandbox')
@@ -178,35 +214,69 @@ def scrap_artikel(link_scrap,models,api_key,azure_api_base):
     
     user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
     options.add_argument(f"user-agent={user_agent}")
-
     wd = webdriver.Chrome(options=options)
-    wd.get(link_scrap)
-
-    wd.find_element(By.CSS_SELECTOR, 'body').send_keys(Keys.CONTROL, Keys.END)
-    time.sleep(1)
-
-    raw_html = wd.find_element(By.TAG_NAME, 'body').get_attribute('innerHTML')
-    wd.quit()
-
-    soup_html = BeautifulSoup(raw_html, "html.parser")
-    containers = soup_html.findAll('p')
 
-    artikel =[]
-    for paragraph in containers:
-        artic=paragraph.get_text()
-        artikel.append(artic)
-
-    paragraf = ' '.join(artikel)
-    if len(paragraf)>= 18000:
-        part1, part2, part3, part4 = split_article(paragraf)
-        artikels = [part1, part2, part3, part4]
-    else :
-        artikels = [paragraf]
-    title, judul, url, contents = clean_scrap(artikels,link_scrap,models,api_key,azure_api_base)
-    return title, judul, url, contents
-
-def artikel_processing(link_scrap,backlink,keyword,models,api_key,azure_api_base,replicate_key):
-    title, judul, url, artikel= scrap_artikel(link_scrap, models, api_key,azure_api_base)
+    
+    if source_type == "input topic":
+        link = scrap_portal(source)
+        for url in link:
+            if cek_url(url):
+                continue
+            else:
+                if len(artikel) >=1:
+                    continue
+            wd.get(url)
+            wd.find_element(By.CSS_SELECTOR, 'body').send_keys(Keys.CONTROL, Keys.END)
+            time.sleep(1)
+        
+            raw_html = wd.find_element(By.TAG_NAME, 'body').get_attribute('innerHTML')
+            wd.quit()
+        
+            soup_html = BeautifulSoup(raw_html, "html.parser")
+            containers = soup_html.findAll('p')
+        
+            artikel =[]
+            for paragraph in containers:
+                artic=paragraph.get_text()
+                artikel.append(artic)
+        
+            paragraf = ' '.join(artikel)
+            if len(paragraf)>= 18000:
+                part1, part2, part3, part4 = split_article(paragraf)
+                artikels = [part1, part2, part3, part4]
+            else :
+                artikels = [paragraf]
+            title, judul, url, contents = clean_scrap(artikels,url,models,api_key,azure_api_base)
+            return title, judul, url, contents
+    
+    else:
+        wd.get(source)
+    
+        wd.find_element(By.CSS_SELECTOR, 'body').send_keys(Keys.CONTROL, Keys.END)
+        time.sleep(1)
+    
+        raw_html = wd.find_element(By.TAG_NAME, 'body').get_attribute('innerHTML')
+        wd.quit()
+    
+        soup_html = BeautifulSoup(raw_html, "html.parser")
+        containers = soup_html.findAll('p')
+    
+        artikel =[]
+        for paragraph in containers:
+            artic=paragraph.get_text()
+            artikel.append(artic)
+    
+        paragraf = ' '.join(artikel)
+        if len(paragraf)>= 18000:
+            part1, part2, part3, part4 = split_article(paragraf)
+            artikels = [part1, part2, part3, part4]
+        else :
+            artikels = [paragraf]
+        title, judul, url, contents = clean_scrap(artikels,source,models,api_key,azure_api_base)
+        return title, judul, url, contents
+
+def artikel_processing(source_type,source,backlink,keyword,models,api_key,azure_api_base,replicate_key):
+    title, judul, url, artikel= scrap_artikel(source_type,source, models, api_key,azure_api_base)
     teks_to_tags = artikel[0][:500]
     translated = []
     optimized = []
@@ -447,69 +517,87 @@ def artikel_processing(link_scrap,backlink,keyword,models,api_key,azure_api_base
             base64_string = base64.b64decode(base64_string)
         image_data= base64_string
         os.remove(tmp_path)
-    return judul,content,image,image_data
+    return judul,content,image,image_data,url
+
+def scrap(source_type,source,backlink,keyword,version,api_key,azure_api_base,replicate_key):
+    # try:
+    judul,kontent,gambar, image_data,url= artikel_processing(source_type,source,backlink,keyword,version,api_key,azure_api_base,replicate_key)
+    title = '<h1>'+judul+'</h1>'
+    desired_timezone = pytz.timezone('Asia/Jakarta')
+    current_time = datetime.datetime.now(desired_timezone)
+    Timestamp = current_time.strftime('%Y-%m-%d %H:%M:%S')
+    
+    with open('log_activity.txt', 'r') as file:
+        existing_data = file.read()
+    
+    log = 'Source:' + url + '\nGenerated_Title:' + judul + '\nTimestamp:' + Timestamp +  "\n\n"
+    
+    combined_data = existing_data + log
+    
+    with tempfile.NamedTemporaryFile(mode='w', delete=False) as temp_file:
+        temp_file.write(combined_data)
 
-def scrap(link_scrap,backlink,keyword,version,api_key,azure_api_base,replicate_key):
-    try:
-        judul,kontent,gambar, image_data= artikel_processing(link_scrap,backlink,keyword,version,api_key,azure_api_base,replicate_key)
-        title = '<h1>'+judul+'</h1>'
-        desired_timezone = pytz.timezone('Asia/Jakarta')
-        current_time = datetime.datetime.now(desired_timezone)
-        Timestamp = current_time.strftime('%Y-%m-%d %H:%M:%S')
-        
-        with open('log_activity.txt', 'r') as file:
-            existing_data = file.read()
-        
-        log = 'Generated_Title:' + judul + '\nTimestamp:' + Timestamp + '\nLink:' + link_scrap + "\n\n"
-        
-        combined_data = existing_data + log
-        
-        with tempfile.NamedTemporaryFile(mode='w', delete=False) as temp_file:
-            temp_file.write(combined_data)
-        
-        with open("judul.txt", "w") as file:
-            file.write(judul)
-        
-        with open("kontent.txt", "w") as file:
-            file.write(kontent)
+    with open('log_url.txt', 'r') as file:
+        existing_data = file.read()
+
+    log = 'Source:' + url + "\n"
+    combined_data = existing_data + log
+
+    with tempfile.NamedTemporaryFile(mode='w', delete=False) as temp_file:
+        temp_file.write(combined_data)
+    
+    with open("judul.txt", "w") as file:
+        file.write(judul)
+    
+    with open("kontent.txt", "w") as file:
+        file.write(kontent)
+    
+    repo_name = get_full_repo_name(model_id="Article_Gen3", token="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")
+    file_url = upload_file(
+        path_or_fileobj=temp_file.name,  # Use the temporary file name
+        path_in_repo="log_activity.txt",
+        repo_id=repo_name,
+        repo_type="space",
+        token="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",
+    )
+    file_url = upload_file(
+        path_or_fileobj=image_data,
+        path_in_repo="image_data.txt",
+        repo_id=repo_name,
+        repo_type="space",
+        token="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",
+    )
+    file_url = upload_file(
+        path_or_fileobj='judul.txt',
+        path_in_repo="judul.txt",
+        repo_id=repo_name,
+        repo_type="space",
+        token="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",
+    )
+    file_url = upload_file(
+        path_or_fileobj='kontent.txt',
+        path_in_repo="kontent.txt",
+        repo_id=repo_name,
+        repo_type="space",
+        token="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",
+    )
+    if kontent:
+        status = "<h3>Berhasil Generate Artikel</h3>"
+        time.sleep(60)
+        return status,gambar
+    # except:
+    #     with open('log_url.txt', 'r') as file:
+    #         existing_data = file.read()
+
+    #     log = 'Source:' + source + "\n"
+    #     combined_data = existing_data + log
+
+    #     with tempfile.NamedTemporaryFile(mode='w', delete=False) as temp_file:
+    #         temp_file.write(combined_data)
         
-        repo_name = get_full_repo_name(model_id="Article_Gen3", token="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")
-        file_url = upload_file(
-            path_or_fileobj=temp_file.name,  # Use the temporary file name
-            path_in_repo="log_activity.txt",
-            repo_id=repo_name,
-            repo_type="space",
-            token="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",
-        )
-        file_url = upload_file(
-            path_or_fileobj=image_data,
-            path_in_repo="image_data.txt",
-            repo_id=repo_name,
-            repo_type="space",
-            token="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",
-        )
-        file_url = upload_file(
-            path_or_fileobj='judul.txt',
-            path_in_repo="judul.txt",
-            repo_id=repo_name,
-            repo_type="space",
-            token="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",
-        )
-        file_url = upload_file(
-            path_or_fileobj='kontent.txt',
-            path_in_repo="kontent.txt",
-            repo_id=repo_name,
-            repo_type="space",
-            token="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",
-        )
-        if kontent:
-            status = "<h3>Berhasil Generate Artikel</h3>"
-            time.sleep(60)
-            return status,gambar
-    except:
-        status = "<h3>Gagal Generate Artikel</h3>"
-        gambar = Image.open('error.png')
-        return status,gamber
+    #     status = "<h3>Gagal Generate Artikel</h3>"
+    #     gambar = Image.open('error.png')
+    #     return status,gamber
 
 
 def post(endpoint,endpoint_media,username,password,tags,categories,metode):
@@ -614,7 +702,8 @@ with gr.Blocks(theme = "soft", title="Wordpress Article Generator") as article_g
             """)
     with gr.Row():
         with gr.Column():
-            link = gr.Textbox(placeholder="Masukkan Link Artikel Yang Akan di Scrap", label="Link")
+            source_type = gr.Radio(["input link", "input topic"], label="Source")
+            source = gr.Textbox(placeholder="Masukkan Source Link/Topik Artikel Yang Akan Digenerate", show_label=False)
             backlink = gr.Textbox(placeholder="Masukkan Backlink Yang Akan Diterapkan", label="Backlink")
             keyword = gr.Textbox(placeholder="Masukkan Keyword Artikel", label="Keyword")
             versi = gr.Radio(["openai", "azure"], label="Request Schema", info="Pilih Skema Untuk Request ke ChatGPT ")
@@ -624,7 +713,7 @@ with gr.Blocks(theme = "soft", title="Wordpress Article Generator") as article_g
             button_scrap = gr.Button("Scrap Article")
             output = gr.HTML("")
             img = gr.Image(label="Content Media")
-            button_scrap.click(fn=scrap, inputs=[link,backlink,keyword,versi,api_key,link_azure,replicate_token], outputs= [output,img])
+            button_scrap.click(fn=scrap, inputs=[source_type,source,backlink,keyword,versi,api_key,link_azure,replicate_token], outputs= [output,img])
             view_outputs = gr.Button("View Article")
             with gr.Tab("Raw Article"):
                 title = gr.Textbox("", label="Title", interactive=True)