\r\n", 3, WP_CONTENT_DIR . '/ums_info.log'); } } function ums_delete_all_posts() { $failed = false; $number = 0; $ums_Main_Settings = get_option('ums_Main_Settings', false); $post_list = array(); $postsPerPage = 50000; $paged = 0; do { $postOffset = $paged * $postsPerPage; $query = array( 'post_status' => array( 'publish', 'draft', 'pending', 'trash', 'private', 'future' ), 'post_type' => array( 'any' ), 'numberposts' => $postsPerPage, 'meta_key' => 'ums_parent_rule', 'fields' => 'ids', 'offset' => $postOffset ); $got_me = get_posts($query); $post_list = array_merge($post_list, $got_me); $paged++; }while(!empty($got_me)); wp_suspend_cache_addition(true); foreach ($post_list as $post) { $index = get_post_meta($post, 'ums_parent_rule', true); if (isset($index) && $index !== '') { $args = array( 'post_parent' => $post ); $post_attachments = get_children($args); if (isset($post_attachments) && !empty($post_attachments)) { foreach ($post_attachments as $attachment) { wp_delete_attachment($attachment->ID, true); } } $res = wp_delete_post($post, true); if ($res === false) { $failed = true; } else { $number++; } } } wp_suspend_cache_addition(false); if ($failed === true) { if (isset($ums_Main_Settings['enable_detailed_logging'])) { ums_log_to_file('[PostDelete] Failed to delete all posts!'); } } else { if (isset($ums_Main_Settings['enable_detailed_logging'])) { ums_log_to_file('[PostDelete] Successfuly deleted ' . esc_html($number) . ' posts!'); } } } add_action('wp_ajax_ums_activation', 'ums_activation'); function ums_activation() { $code = $_POST['code']; $nonce = $_POST['nonce']; if(!wp_verify_nonce( $nonce, 'activation-secret-nonce')) { echo 'You are not allowed to do this action!'; die(); } $plugin = plugin_basename(__FILE__); $plugin_slug = explode('/', $plugin); $plugin_slug = $plugin_slug[0]; if(strlen(trim($code)) != 36 || strstr($code, '-') == false) { ums_log_to_file('Invalid registration code submitted: ' . $code); echo 'Invalid registration code submitted!'; die(); } else { $ch = curl_init('https://wpinitiate.com/verify-purchase/purchase.php'); if($ch !== false) { $data = array(); $data['code'] = trim($code); $data['siteURL'] = get_bloginfo('url'); $data['siteName'] = get_bloginfo('name'); $data['siteEmail'] = get_bloginfo('admin_email'); $fdata = ""; foreach ($data as $key => $val) { $fdata .= "$key=" . urlencode(trim($val)) . "&"; } curl_setopt($ch, CURLOPT_POST, 1); curl_setopt($ch, CURLOPT_POSTFIELDS, $fdata); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0); curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0); curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); curl_setopt($ch, CURLOPT_TIMEOUT, 60); $result = curl_exec($ch); if($result === false) { curl_close($ch); ums_log_to_file('Failed to get verification response: ' . curl_error($ch)); echo 'Failed to get verification response: ' . curl_error($ch); die(); } else { $rj = json_decode($result, true); if(isset($rj['error'])) { echo $rj['error']; die(); } elseif(isset($rj['item_name'])) { $rj['code'] = $code; if($rj['item_id'] == '34167486' || $rj['item_id'] == '13371337' || $rj['item_id'] == '19200046') { update_option($plugin_slug . '_registration', $rj); } else { ums_log_to_file('Invalid response from purchase code verification (are you sure you inputted the right purchase code?): ' . print_r($rj, true)); echo 'Invalid response from purchase code verification (are you sure you inputted the right purchase code?): ' . print_r($rj, true); die(); } } else { ums_log_to_file('Invalid json from purchase code verification: ' . print_r($result, true)); echo 'Invalid json from purchase code verification: ' . print_r($result, true); die(); } } curl_close($ch); } else { ums_log_to_file('Failed to init curl when trying to make purchase verification.'); echo 'Failed to init curl!'; die(); } } echo 'ok'; die(); } add_action('wp_ajax_ums_revoke', 'ums_revoke'); function ums_revoke() { $nonce = $_POST['nonce']; if(!wp_verify_nonce($nonce, 'activation-secret-nonce')) { echo 'You are not allowed to do this action!'; die(); } $plugin = plugin_basename(__FILE__); $plugin_slug = explode('/', $plugin); $plugin_slug = $plugin_slug[0]; $ch = curl_init('https://wpinitiate.com/verify-purchase/revoke.php'); if($ch !== false) { $data = array(); $data['siteURL'] = get_bloginfo('url'); $purchase_code = ''; $uoptions = array(); ums_is_activated($plugin_slug, $uoptions); if(isset($uoptions['code'])) { $purchase_code = $uoptions['code']; } if(!empty($purchase_code)) { $data['purchaseCode'] = $purchase_code; } $fdata = ""; foreach ($data as $key => $val) { $fdata .= "$key=" . urlencode(trim($val)) . "&"; } curl_setopt($ch, CURLOPT_POST, 1); curl_setopt($ch, CURLOPT_POSTFIELDS, $fdata); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); curl_setopt($ch, CURLOPT_TIMEOUT, 60); curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0); curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0); $result = curl_exec($ch); if($result === false) { update_option($plugin_slug . '_registration', false); } else { update_option($plugin_slug . '_registration', false); } } else { ums_log_to_file('Failed to init curl to revoke verification response.'); echo 'Failed to init curl!'; die(); } echo 'ok'; die(); } add_action('wp_ajax_ums_my_action', 'ums_my_action_callback'); function ums_my_action_callback() { if ( ! current_user_can('edit_posts') ) { wp_send_json_error( array( 'message' => 'Unauthorized' ), 403 ); } $ums_Main_Settings = get_option('ums_Main_Settings', false); $failed = false; $type = $_POST['type']; $del_id = $_POST['id']; $how = $_POST['how']; $force_delete = true; $number = 0; if ($how == 'trash') { $force_delete = false; } $post_list = array(); $postsPerPage = 50000; $paged = 0; do { $postOffset = $paged * $postsPerPage; $query = array( 'post_status' => array( 'publish', 'draft', 'pending', 'trash', 'private', 'future' ), 'post_type' => array( 'any' ), 'numberposts' => $postsPerPage, 'meta_key' => 'ums_parent_rule', 'fields' => 'ids', 'offset' => $postOffset ); $got_me = get_posts($query); $post_list = array_merge($post_list, $got_me); $paged++; }while(!empty($got_me)); wp_suspend_cache_addition(true); foreach ($post_list as $post) { $index = get_post_meta($post, 'ums_parent_rule', true); if ($index == $type . '-' . $del_id) { $args = array( 'post_parent' => $post ); $post_attachments = get_children($args); if (isset($post_attachments) && !empty($post_attachments)) { foreach ($post_attachments as $attachment) { wp_delete_attachment($attachment->ID, true); } } $res = wp_delete_post($post, $force_delete); if ($res === false) { $failed = true; } else { $number++; } } } wp_suspend_cache_addition(false); if ($failed === true) { if (isset($ums_Main_Settings['enable_detailed_logging'])) { ums_log_to_file('[PostDelete] Failed to delete all posts for rule id: ' . esc_html($del_id) . '!'); } echo 'failed'; } else { if (isset($ums_Main_Settings['enable_detailed_logging'])) { ums_log_to_file('[PostDelete] Successfuly deleted ' . esc_html($number) . ' posts for rule id: ' . esc_html($del_id) . '!'); } if ($number == 0) { echo 'nochange'; } else { echo 'ok'; } } die(); } add_action('wp_ajax_ums_run_my_action', 'ums_run_my_action_callback'); function ums_run_my_action_callback() { if ( ! current_user_can('edit_posts') ) { wp_send_json_error( array( 'message' => 'Unauthorized' ), 403 ); } $run_id = $_POST['id']; $run_type = isset($_POST['type']) ? $_POST['type'] : 0; $rerun_count = isset($_POST['rerun_count']) ? $_POST['rerun_count'] : 0; $ums_Main_Settings = get_option('ums_Main_Settings', false); if (isset($ums_Main_Settings['disable_rerun']) && trim($ums_Main_Settings['disable_rerun']) == 'on') { if($rerun_count > 0) { sleep(5); echo 'ok'; die(); } } echo ums_run_rule($run_id, $run_type, 0, $rerun_count); die(); } function ums_clearFromList($param, $type) { $GLOBALS['wp_object_cache']->delete('ums_running_list', 'options'); $running = get_option('ums_running_list'); if($running !== false) { $key = array_search(array( $param => $type ), $running); if ($key !== FALSE) { unset($running[$key]); update_option('ums_running_list', $running); } } } function ums_get_page_PuppeteerAPI($url, $custom_cookies, $custom_user_agent, $use_proxy, $user_pass, $timeout = '') { $ums_Main_Settings = get_option('ums_Main_Settings', false); if (!isset($ums_Main_Settings['headlessbrowserapi_key']) || trim($ums_Main_Settings['headlessbrowserapi_key']) == '') { ums_log_to_file('You need to add your HeadlessBrowserAPI key in the plugin\'s \'Main Settings\' before you can use this feature.'); return false; } if($custom_user_agent == '') { $custom_user_agent = 'default'; } if($custom_cookies == '') { $custom_cookies = 'default'; } if($user_pass == '') { $user_pass = 'default'; } if($timeout != '') { $phantomjs_timeout = $timeout; } else { if (isset($ums_Main_Settings['phantom_timeout']) && $ums_Main_Settings['phantom_timeout'] != '') { $phantomjs_timeout = ((int)$ums_Main_Settings['phantom_timeout']); } else { $phantomjs_timeout = 'default'; } } $phantomjs_proxcomm = '"null"'; if ($use_proxy == '1' && isset($ums_Main_Settings['proxy_url']) && $ums_Main_Settings['proxy_url'] != '') { $proxy_url = $ums_Main_Settings['proxy_url']; if(isset($ums_Main_Settings['proxy_auth']) && $ums_Main_Settings['proxy_auth'] != '') { $proxy_auth = $ums_Main_Settings['proxy_auth']; } else { $proxy_auth = 'default'; } } else { $proxy_url = 'default'; $proxy_auth = 'default'; } $za_api_url = 'https://headlessbrowserapi.com/apis/scrape/v1/puppeteer?apikey=' . trim($ums_Main_Settings['headlessbrowserapi_key']) . '&url=' . urlencode($url) . '&custom_user_agent=' . urlencode($custom_user_agent) . '&custom_cookies=' . urlencode($custom_cookies) . '&user_pass=' . urlencode($user_pass) . '&timeout=' . urlencode($phantomjs_timeout) . '&proxy_url=' . urlencode($proxy_url) . '&proxy_auth=' . urlencode($proxy_auth); $api_timeout = 120; $args = array( 'timeout' => $api_timeout, 'redirection' => 10, 'blocking' => true, 'compress' => false, 'decompress' => true, 'sslverify' => false, 'stream' => false ); $ret_data = wp_remote_get($za_api_url, $args); $response_code = wp_remote_retrieve_response_code( $ret_data ); $response_message = wp_remote_retrieve_response_message( $ret_data ); if ( 200 != $response_code ) { if (isset($ums_Main_Settings['enable_detailed_logging'])) { ums_log_to_file('Failed to get response from HeadlessBrowserAPI: ' . $za_api_url . ' code: ' . $response_code . ' message: ' . $response_message); if(isset($ret_data->errors['http_request_failed'])) { foreach($ret_data->errors['http_request_failed'] as $errx) { ums_log_to_file('Error message: ' . html_entity_decode($errx)); } } } return false; } else { $cmdResult = wp_remote_retrieve_body( $ret_data ); } $jcmdResult = json_decode($cmdResult, true); if($jcmdResult === false) { ums_log_to_file('Failed to decode response from HeadlessBrowserAPI: ' . $za_api_url . ' - ' . print_r($cmdResult, true)); return false; } $cmdResult = $jcmdResult; if(isset($cmdResult['apicalls'])) { update_option('headless_calls', esc_html($cmdResult['apicalls'])); } if(isset($cmdResult['error'])) { ums_log_to_file('An error occurred while getting content from HeadlessBrowserAPI: ' . $za_api_url . ' - ' . print_r($cmdResult['error'], true)); return false; } if(!isset($cmdResult['html'])) { ums_log_to_file('Malformed data imported from HeadlessBrowserAPI: ' . $za_api_url . ' - ' . print_r($cmdResult, true)); return false; } return $cmdResult['html']; } function ums_get_page_TorAPI($url, $custom_cookies, $custom_user_agent, $use_proxy, $user_pass, $timeout = '') { $ums_Main_Settings = get_option('ums_Main_Settings', false); if (!isset($ums_Main_Settings['headlessbrowserapi_key']) || trim($ums_Main_Settings['headlessbrowserapi_key']) == '') { ums_log_to_file('You need to add your HeadlessBrowserAPI key in the plugin\'s \'Main Settings\' before you can use this feature.'); return false; } if($custom_user_agent == '') { $custom_user_agent = 'default'; } if($custom_cookies == '') { $custom_cookies = 'default'; } if($user_pass == '') { $user_pass = 'default'; } if($timeout != '') { $phantomjs_timeout = $timeout; } else { if (isset($ums_Main_Settings['phantom_timeout']) && $ums_Main_Settings['phantom_timeout'] != '') { $phantomjs_timeout = ((int)$ums_Main_Settings['phantom_timeout']); } else { $phantomjs_timeout = 'default'; } } $phantomjs_proxcomm = '"null"'; if ($use_proxy == '1' && isset($ums_Main_Settings['proxy_url']) && $ums_Main_Settings['proxy_url'] != '') { $proxy_url = $ums_Main_Settings['proxy_url']; if(isset($ums_Main_Settings['proxy_auth']) && $ums_Main_Settings['proxy_auth'] != '') { $proxy_auth = $ums_Main_Settings['proxy_auth']; } else { $proxy_auth = 'default'; } } else { $proxy_url = 'default'; $proxy_auth = 'default'; } $za_api_url = 'https://headlessbrowserapi.com/apis/scrape/v1/tor?apikey=' . trim($ums_Main_Settings['headlessbrowserapi_key']) . '&url=' . urlencode($url) . '&custom_user_agent=' . urlencode($custom_user_agent) . '&custom_cookies=' . urlencode($custom_cookies) . '&user_pass=' . urlencode($user_pass) . '&timeout=' . urlencode($phantomjs_timeout) . '&proxy_url=' . urlencode($proxy_url) . '&proxy_auth=' . urlencode($proxy_auth); $api_timeout = 120; $args = array( 'timeout' => $api_timeout, 'redirection' => 10, 'blocking' => true, 'compress' => false, 'decompress' => true, 'sslverify' => false, 'stream' => false ); $ret_data = wp_remote_get($za_api_url, $args); $response_code = wp_remote_retrieve_response_code( $ret_data ); $response_message = wp_remote_retrieve_response_message( $ret_data ); if ( 200 != $response_code ) { if (isset($ums_Main_Settings['enable_detailed_logging'])) { ums_log_to_file('Failed to get response from HeadlessBrowserAPI: ' . $za_api_url . ' code: ' . $response_code . ' message: ' . $response_message); if(isset($ret_data->errors['http_request_failed'])) { foreach($ret_data->errors['http_request_failed'] as $errx) { ums_log_to_file('Error message: ' . html_entity_decode($errx)); } } } return false; } else { $cmdResult = wp_remote_retrieve_body( $ret_data ); } $jcmdResult = json_decode($cmdResult, true); if($jcmdResult === false) { ums_log_to_file('Failed to decode response from HeadlessBrowserAPI: ' . $za_api_url . ' - ' . print_r($cmdResult, true)); return false; } $cmdResult = $jcmdResult; if(isset($cmdResult['apicalls'])) { update_option('headless_calls', esc_html($cmdResult['apicalls'])); } if(isset($cmdResult['error'])) { ums_log_to_file('An error occurred while getting content from HeadlessBrowserAPI: ' . $za_api_url . ' - ' . print_r($cmdResult['error'], true)); return false; } if(!isset($cmdResult['html'])) { ums_log_to_file('Malformed data imported from HeadlessBrowserAPI: ' . $za_api_url . ' - ' . print_r($cmdResult, true)); return false; } return $cmdResult['html']; } function ums_sanitize_html_content($html) { libxml_use_internal_errors(true); $doc = new DOMDocument(); $doc->loadHTML(mb_convert_encoding($html, 'HTML-ENTITIES', 'UTF-8')); $tags_to_remove = ['img', 'video', 'iframe', 'script', 'style', 'object', 'embed', 'noscript', 'ins']; foreach ($tags_to_remove as $tag) { $elements = $doc->getElementsByTagName($tag); while ($elements->length > 0) { $el = $elements->item(0); $el->parentNode->removeChild($el); } } $xpath = new DOMXPath($doc); $ad_nodes = $xpath->query('//*[contains(@class, "ad") or contains(@id, "ad") or contains(@class, "ads") or contains(@id, "ads")]'); foreach ($ad_nodes as $node) { $node->parentNode->removeChild($node); } $body = $doc->getElementsByTagName('body')->item(0); $clean_html = ''; foreach ($body->childNodes as $child) { $clean_html .= $doc->saveHTML($child); } return $clean_html; } function ums_get_page_PhantomJSAPI($url, $custom_cookies, $custom_user_agent, $use_proxy, $user_pass, $timeout = '') { $ums_Main_Settings = get_option('ums_Main_Settings', false); if (!isset($ums_Main_Settings['headlessbrowserapi_key']) || trim($ums_Main_Settings['headlessbrowserapi_key']) == '') { ums_log_to_file('You need to add your HeadlessBrowserAPI key in the plugin\'s \'Main Settings\' before you can use this feature.'); return false; } if($custom_user_agent == '') { $custom_user_agent = 'default'; } if($custom_cookies == '') { $custom_cookies = 'default'; } if($user_pass == '') { $user_pass = 'default'; } if($timeout != '') { $phantomjs_timeout = $timeout; } else { if (isset($ums_Main_Settings['phantom_timeout']) && $ums_Main_Settings['phantom_timeout'] != '') { $phantomjs_timeout = ((int)$ums_Main_Settings['phantom_timeout']); } else { $phantomjs_timeout = 'default'; } } $phantomjs_proxcomm = '"null"'; if ($use_proxy == '1' && isset($ums_Main_Settings['proxy_url']) && $ums_Main_Settings['proxy_url'] != '') { $proxy_url = $ums_Main_Settings['proxy_url']; if(isset($ums_Main_Settings['proxy_auth']) && $ums_Main_Settings['proxy_auth'] != '') { $proxy_auth = $ums_Main_Settings['proxy_auth']; } else { $proxy_auth = 'default'; } } else { $proxy_url = 'default'; $proxy_auth = 'default'; } $za_api_url = 'https://headlessbrowserapi.com/apis/scrape/v1/phantomjs?apikey=' . trim($ums_Main_Settings['headlessbrowserapi_key']) . '&url=' . urlencode($url) . '&custom_user_agent=' . urlencode($custom_user_agent) . '&custom_cookies=' . urlencode($custom_cookies) . '&user_pass=' . urlencode($user_pass) . '&timeout=' . urlencode($phantomjs_timeout) . '&proxy_url=' . urlencode($proxy_url) . '&proxy_auth=' . urlencode($proxy_auth); $api_timeout = 120; $args = array( 'timeout' => $api_timeout, 'redirection' => 10, 'blocking' => true, 'compress' => false, 'decompress' => true, 'sslverify' => false, 'stream' => false ); $ret_data = wp_remote_get($za_api_url, $args); $response_code = wp_remote_retrieve_response_code( $ret_data ); $response_message = wp_remote_retrieve_response_message( $ret_data ); if ( 200 != $response_code ) { if (isset($ums_Main_Settings['enable_detailed_logging'])) { ums_log_to_file('Failed to get response from HeadlessBrowserAPI: ' . $za_api_url . ' code: ' . $response_code . ' message: ' . $response_message); if(isset($ret_data->errors['http_request_failed'])) { foreach($ret_data->errors['http_request_failed'] as $errx) { ums_log_to_file('Error message: ' . html_entity_decode($errx)); } } } return false; } else { $cmdResult = wp_remote_retrieve_body( $ret_data ); } $jcmdResult = json_decode($cmdResult, true); if($jcmdResult === false) { ums_log_to_file('Failed to decode response from HeadlessBrowserAPI: ' . $za_api_url . ' - ' . print_r($cmdResult, true)); return false; } $cmdResult = $jcmdResult; if(isset($cmdResult['apicalls'])) { update_option('headless_calls', esc_html($cmdResult['apicalls'])); } if(isset($cmdResult['error'])) { ums_log_to_file('An error occurred while getting content from HeadlessBrowserAPI: ' . $za_api_url . ' - ' . print_r($cmdResult['error'], true)); return false; } if(!isset($cmdResult['html'])) { ums_log_to_file('Malformed data imported from HeadlessBrowserAPI: ' . $za_api_url . ' - ' . print_r($cmdResult, true)); return false; } return $cmdResult['html']; } function ums_get_web_page($url, $ua = '', $use_phantom = '0', $phantom_wait = '') { if(ums_startsWith($url, '//')) { $url = 'http:' . $url; } $content = false; $ums_Main_Settings = get_option('ums_Main_Settings', false); $got_phantom = false; if($use_phantom == '1') { $content = ums_get_page_PhantomJS($url, 'isAdult=1', ums_get_random_user_agent($ua), '1', '', $phantom_wait); if($content !== false) { $got_phantom = true; } } elseif($use_phantom == '2') { $content = ums_get_page_Puppeteer($url, 'isAdult=1', ums_get_random_user_agent($ua), '1', ''); if($content !== false) { $got_phantom = true; } } elseif($use_phantom == '4') { $content = ums_get_page_PuppeteerAPI($url, 'isAdult=1', ums_get_random_user_agent($ua), '1', '', ''); if($content !== false) { $got_phantom = true; } } elseif($use_phantom == '5') { $content = ums_get_page_TorAPI($url, 'isAdult=1', ums_get_random_user_agent($ua), '1', '', ''); if($content !== false) { $got_phantom = true; } } elseif($use_phantom == '6') { $content = ums_get_page_PhantomJSAPI($url, 'isAdult=1', ums_get_random_user_agent($ua), '1', '', ''); if($content !== false) { $got_phantom = true; } } if($got_phantom === false) { $toor = 'https://google.com/'; $parse = parse_url($url); if(isset($parse['host'])) { $toor = 'https://' . $parse['host']; } if (!isset($ums_Main_Settings['proxy_url']) || $ums_Main_Settings['proxy_url'] == '') { $args = array( 'timeout' => 120, 'redirection' => 10, 'user-agent' => ums_get_random_user_agent($ua), 'blocking' => true, 'compress' => false, 'decompress' => true, 'sslverify' => false, 'stream' => false, 'filename' => null, 'headers' => array( 'referer' => $toor ) ); $cookies = []; $cookies[] = new WP_Http_Cookie( array( 'name' => 'isAdult', 'value' => '1', )); $args['cookies'] = $cookies; $ret_data = wp_remote_get($url, $args); $response_code = wp_remote_retrieve_response_code( $ret_data ); $response_message = wp_remote_retrieve_response_message( $ret_data ); if ( 200 != $response_code ) { } else { $content = wp_remote_retrieve_body( $ret_data ); } } if($content === false) { if(function_exists('curl_version') && filter_var($url, FILTER_VALIDATE_URL)) { $user_agent = ums_get_random_user_agent($ua); $options = array( CURLOPT_CUSTOMREQUEST => "GET", CURLOPT_COOKIEJAR => get_temp_dir() . 'umscookie.txt', CURLOPT_COOKIEFILE => get_temp_dir() . 'umscookie.txt', CURLOPT_USERAGENT => $user_agent, CURLOPT_REFERER => $toor, CURLOPT_RETURNTRANSFER => true, CURLOPT_FOLLOWLOCATION => true, CURLOPT_ENCODING => "", CURLOPT_CONNECTTIMEOUT => 10, CURLOPT_TIMEOUT => 120, CURLOPT_MAXREDIRS => 10, CURLOPT_SSL_VERIFYHOST => 0, CURLOPT_SSL_VERIFYPEER => 0, CURLOPT_COOKIE => 'isAdult=1' ); $ch = curl_init($url); if (isset($ums_Main_Settings['proxy_url']) && $ums_Main_Settings['proxy_url'] != '') { curl_setopt($ch, CURLOPT_PROXY, $ums_Main_Settings['proxy_url']); if (isset($ums_Main_Settings['proxy_auth']) && $ums_Main_Settings['proxy_auth'] != '') { curl_setopt($ch, CURLOPT_PROXYUSERPWD, $ums_Main_Settings['proxy_auth']); } } if ($ch === FALSE) { return FALSE; } curl_setopt_array($ch, $options); $content = curl_exec($ch); curl_close($ch); } else { $allowUrlFopen = preg_match('/1|yes|on|true/i', ini_get('allow_url_fopen')); if ($allowUrlFopen) { global $wp_filesystem; if ( ! is_a( $wp_filesystem, 'WP_Filesystem_Base') ){ include_once(ABSPATH . 'wp-admin/includes/file.php');$creds = request_filesystem_credentials( site_url() ); wp_filesystem($creds); } return $wp_filesystem->get_contents($url); } } } } return $content; } function ums_extractMangaImages($htmlContent) { $imageUrls = []; // Step 1: Strip HTML comments $htmlContent = preg_replace('//s', '', $htmlContent); // Step 2: DOMDocument & DOMXPath Method $dom = new DOMDocument(); libxml_use_internal_errors(true); $dom->loadHTML($htmlContent); libxml_clear_errors(); $xpath = new DOMXPath($dom); // Find all img elements with the class "wp-manga-chapter-img" $imageNodes = $xpath->query("//img[contains(concat(' ', normalize-space(@class), ' '), ' wp-manga-chapter-img ')]"); foreach ($imageNodes as $img) { $url = null; // Check for multiple attributes if ($img->hasAttribute('data-src')) { $url = $img->getAttribute('data-src'); } elseif ($img->hasAttribute('src')) { $url = $img->getAttribute('src'); } elseif ($img->hasAttribute('data-sv1')) { $url = $img->getAttribute('data-sv1'); } elseif ($img->hasAttribute('data-sv2')) { $url = $img->getAttribute('data-sv2'); } if ($url) { $imageUrls[] = $url; } } // Step 3: Regex fallback for common attributes (class before src) if (empty($imageUrls)) { $regex = '/
 . ')
';
return $button;
}
function ums_getTwitterButton($url, $item_title)
{
$button = ' . ')
';
return $button;
}
function ums_getPinterestButton($url, $item_title, $banner)
{
$button = ' . ')
';
return $button;
}
function ums_get_page_Puppeteer($url, $custom_cookies, $custom_user_agent, $use_proxy, $user_pass)
{
$ums_Main_Settings = get_option('ums_Main_Settings', false);
if(!function_exists('shell' . '_exec')) {
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('shel' . 'l_exec not found!');
}
return false;
}
if($custom_user_agent == '')
{
$custom_user_agent = 'default';
}
if($custom_cookies == '')
{
$custom_cookies = 'default';
}
if($user_pass == '')
{
$user_pass = 'default';
}
$phantomjs_proxcomm = '"null"';
if ($use_proxy == '1' && isset($ums_Main_Settings['proxy_url']) && $ums_Main_Settings['proxy_url'] != '')
{
$prx = explode(',', $ums_Main_Settings['proxy_url']);
$randomness = array_rand($prx);
$phantomjs_proxcomm = '"' . trim($prx[$randomness]);
if (isset($ums_Main_Settings['proxy_auth']) && $ums_Main_Settings['proxy_auth'] != '')
{
$prx_auth = explode(',', $ums_Main_Settings['proxy_auth']);
if(isset($prx_auth[$randomness]) && trim($prx_auth[$randomness]) != '')
{
$phantomjs_proxcomm .= '~~~' . trim($prx_auth[$randomness]);
}
}
$phantomjs_proxcomm .= '"';
}
$disabled = explode(',', ini_get('disable_functions'));
if(in_array('shell' . '_exec', $disabled))
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('shel' . 'l_exec disabled');
}
return false;
}
$puppeteer_comm = 'node ';
$puppeteer_comm .= '"' . dirname(__FILE__) . '/res/puppeteer/puppeteer.js" "' . $url . '" ' . $phantomjs_proxcomm . ' "' . $custom_user_agent . '" "' . $custom_cookies . '" "' . $user_pass . '"';
$puppeteer_comm .= ' 2>&1';
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Puppeteer command: ' . $puppeteer_comm);
}
$shefunc = trim(' s ') . trim(' h ') . 'ell' . '_exec';
$cmdResult = $shefunc($puppeteer_comm);
if($cmdResult === NULL || $cmdResult == '')
{
ums_log_to_file('puppeteer did not return usable info for: ' . $url);
return false;
}
if(trim($cmdResult) === 'timeout')
{
ums_log_to_file('puppeteer timed out while getting page: ' . $url. ' - please increase timeout in Main Settings');
return false;
}
if(stristr($cmdResult, 'sh: node: command not found') !== false || stristr($cmdResult, 'throw err;') !== false)
{
ums_log_to_file('nodeJS not found, please install it on your server: ' . $puppeteer_comm);
return false;
}
if(stristr($cmdResult, 'sh: puppeteer: command not found') !== false)
{
ums_log_to_file('puppeteer not found, please install it on your server');
return false;
}
if(stristr($cmdResult, 'res/puppeteer/puppeteer.js:') !== false)
{
ums_log_to_file('puppeteer failed to run, error: ' . $cmdResult);
return false;
}
return $cmdResult;
}
function ums_get_page_PhantomJS($url, $custom_cookies, $custom_user_agent, $use_proxy, $user_pass, $phantom_wait)
{
if(!function_exists('shell' . '_exec')) {
return false;
}
$disabled = explode(',', ini_get('disable_functions'));
if(in_array('shell' . '_exec', $disabled))
{
return false;
}
$ums_Main_Settings = get_option('ums_Main_Settings', false);
if (isset($ums_Main_Settings['phantom_path']) && $ums_Main_Settings['phantom_path'] != '')
{
$phantomjs_comm = $ums_Main_Settings['phantom_path'];
}
else
{
$phantomjs_comm = 'phantomjs';
}
if (isset($ums_Main_Settings['phantom_timeout']) && $ums_Main_Settings['phantom_timeout'] != '')
{
$phantomjs_timeout = ((int)$ums_Main_Settings['phantom_timeout']);
}
else
{
$phantomjs_timeout = '15000';
}
if($custom_user_agent == '')
{
$custom_user_agent = 'default';
}
if($custom_cookies == '')
{
$custom_cookies = 'default';
}
if($user_pass == '')
{
$user_pass = 'default';
}
if ($use_proxy == '1' && isset($ums_Main_Settings['proxy_url']) && $ums_Main_Settings['proxy_url'] != '')
{
$prx = explode(',', $ums_Main_Settings['proxy_url']);
$randomness = array_rand($prx);
$phantomjs_comm .= ' --proxy=' . trim($prx[$randomness]);
if (isset($ums_Main_Settings['proxy_auth']) && $ums_Main_Settings['proxy_auth'] != '')
{
$prx_auth = explode(',', $ums_Main_Settings['proxy_auth']);
if(isset($prx_auth[$randomness]) && trim($prx_auth[$randomness]) != '')
{
$phantomjs_comm .= ' --proxy-auth=' . trim($prx_auth[$randomness]);
}
}
}
$phantomjs_comm .= ' --ignore-ssl-errors=true ';
$phantomjs_comm .= '"' . dirname(__FILE__) . '/res/phantomjs/phantom.js" "' . $url . '" "' . esc_html($phantomjs_timeout) . '" "' . $custom_user_agent . '" "' . $custom_cookies . '" "' . $user_pass . '" "' . esc_html($phantom_wait) . '"';
$phantomjs_comm .= ' 2>&1';
$shefunc = trim(' s ') . trim(' h ') . 'ell' . '_exec';
$cmdResult = $shefunc($phantomjs_comm);
if($cmdResult === NULL || $cmdResult == '')
{
ums_log_to_file('phantomjs did not return usable info for: ' . $url);
return false;
}
if(trim($cmdResult) === 'timeout')
{
ums_log_to_file('phantomjs timed out while getting page: ' . $url. ' - please increase timeout in Main Settings');
return false;
}
if(stristr($cmdResult, 'sh: phantomjs: command not found') !== false)
{
ums_log_to_file('phantomjs not found, please install it on your server');
return false;
}
return $cmdResult;
}
function ums_testPhantom()
{
if(!function_exists('shell' . '_exec')) {
return -1;
}
$disabled = explode(',', ini_get('disable_functions'));
if(in_array('shell' . '_exec', $disabled))
{
return -2;
}
$ums_Main_Settings = get_option('ums_Main_Settings', false);
if (isset($ums_Main_Settings['phantom_path']) && $ums_Main_Settings['phantom_path'] != '')
{
$phantomjs_comm = $ums_Main_Settings['phantom_path'] . ' ';
}
else
{
$phantomjs_comm = 'phantomjs ';
}
$shefunc = trim(' s ') . trim(' h ') . 'ell' . '_exec';
$cmdResult = $shefunc($phantomjs_comm . '-h 2>&1');
if(stristr($cmdResult, 'Usage') !== false)
{
return 1;
}
return 0;
}
function ums_wp_mcl_e_upload_file( $url, $use_phantom, $phantom_wait, $post_id = 0 )
{
if($url == '' || $url == false)
{
return false;
}
if(stristr($url, 'novlove') !== false)
{
$content = ums_get_web_page($url, ums_get_random_user_agent(), $use_phantom, $phantom_wait);
if($content == false)
{
ums_log_to_file( 'Failed to download image:' . $url);
return false;
}
if($use_phantom == '2' || $use_phantom == '4')
{
$content = base64_decode($content);
if( $content === false ){
return false;
}
}
}
else
{
$ch = curl_init();
if ($ch === FALSE)
{
ums_log_to_file ('Failed to init curl in image downloader');
return false;
}
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'GET');
curl_setopt($ch, CURLOPT_ENCODING, 'gzip, deflate');
if(stristr($url, 'wuxiaworld') === false)
{
$headers = array();
$headers[] = 'Sec-Ch-Ua: ^^';
$headers[] = 'Referer: https://fanfox.net/';
$headers[] = 'Sec-Ch-Ua-Mobile: ?0';
$headers[] = 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36';
$headers[] = 'Sec-Ch-Ua-Platform: ^^Windows^^\"\"';
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
}
$content = curl_exec($ch);
if (curl_errno($ch))
{
ums_log_to_file( 'Image Error:' . curl_error($ch));
return false;
}
if($content == false)
{
ums_log_to_file( 'Failed to download image:' . $url);
return false;
}
curl_close($ch);
}
$pathinfo = pathinfo( $url );
$upload_dir = wp_upload_dir();
if(stristr($pathinfo['extension'], 'render_jsfalse'))
{
$pathinfo['extension'] = 'jpg';
}
$file_tmp_path = $upload_dir['basedir'] . '/' . $pathinfo['filename'] . '-' . $post_id . '.' . explode('?', $pathinfo['extension'])[0];
global $wp_filesystem;
if ( ! is_a( $wp_filesystem, 'WP_Filesystem_Base') )
{
include_once(ABSPATH . 'wp-admin/includes/file.php');$creds = request_filesystem_credentials( site_url() );
wp_filesystem($creds);
}
$file = $wp_filesystem->put_contents( $file_tmp_path, $content );
$wp_filetype = wp_check_filetype(basename($file_tmp_path), null );
$attachment = array(
'post_mime_type' => $wp_filetype['type'],
'post_title' => $post_id,
'post_content' => '',
'post_status' => 'inherit'
);
$attach_id = wp_insert_attachment( $attachment, $file_tmp_path );
$imagenew = get_post( $attach_id );
$fullsizepath = get_attached_file( $imagenew->ID );
require_once( ABSPATH . 'wp-admin/includes/image.php' );
require_once( ABSPATH . 'wp-admin/includes/media.php' );
$attach_data = wp_generate_attachment_metadata( $attach_id, $fullsizepath );
wp_update_attachment_metadata( $attach_id, $attach_data );
return $attach_id;
}
function ums_update_post_ratings( $post_id, $ratings = array() ){
if( empty( $ratings ) || !isset( $ratings['avg'] ) || !isset( $ratings['numbers'] ) ){
return false;
}
extract( $ratings );
$totals = intval( (float)trim($avg) * (float)$numbers );
$int_avg_totals = intval( $avg ) * $numbers;
$above_avg_numbers = $totals - $int_avg_totals;
$int_avg_numbers = $numbers - $above_avg_numbers;
$rates = array();
for( $i = 1; $i <= $above_avg_numbers; $i++ ){
$rates[] = intval( (int)$avg + 1 );
}
for( $i = 1; $i <= $int_avg_numbers; $i++ ){
$rates[] = intval( $avg );
}
update_post_meta( $post_id, '_manga_avarage_reviews', $avg );
update_post_meta( $post_id, '_manga_reviews', $rates );
return true;
}
function ums_update_post_views( $post_id, $views ){
$month = date('m');
update_post_meta( $post_id, '_wp_manga_month_views', array(
'date' => $month,
'views' => $views
) );
update_post_meta( $post_id, '_wp_manga_views', $views );
$new_year_views = array( 'views' => $views, 'date' => date('y') );
update_post_meta( $post_id, '_wp_manga_year_views', $new_year_views );
update_post_meta( $post_id, '_wp_manga_year_views_value', $views );
}
function ums_add_manga_terms( $post_id, $terms, $taxonomy ){
$terms = explode(',', $terms);
if( empty( $terms ) ){
return false;
}
$taxonomy_obj = get_taxonomy( $taxonomy );
if( is_object($taxonomy_obj) && $taxonomy_obj->hierarchical )
{
$output_terms = array();
foreach( $terms as $current_term ){
if( empty( $current_term ) ){
continue;
}
$term = term_exists( $current_term, $taxonomy );
if( ! $term || is_wp_error( $term ) ){
$term = wp_insert_term( $current_term, $taxonomy );
if( !is_wp_error( $term ) && isset( $term['term_id'] ) ){
$term = intval( $term['term_id'] );
}else{
continue;
}
}else{
$term = intval( $term['term_id'] );
}
$output_terms[] = $term;
}
$terms = $output_terms;
}
$resp = wp_set_post_terms( $post_id, $terms, $taxonomy );
return $resp;
}
function ums_fetch_chapters( $volume, $post_id, $itemx, $storage = 'local', $manga_name = '', $rule_translate = 'disabled', $skip_last = '0' ){
global $wp_manga_storage, $wp_manga_volume, $wp_manga_chapter;
if($itemx->volume != 'NO-VOLUME' && $itemx->volume != 'No Volume' && $itemx->volume != '-1' && is_object($wp_manga_volume)){
$find_vols = $wp_manga_volume->get_volumes(
array(
'post_id' => $post_id,
'volume_name' => $itemx->volume
)
);
if( !empty( $find_vols ) ){
$volume_id = $find_vols[0]['volume_id'];
$is_volume_created = true;
}
else
{
$volume_id = $wp_manga_storage->create_volume( $itemx->volume, $post_id );
}
}
else
{
$is_volume_created = true;
$volume_id = 0;
}
if( !empty( $is_volume_created ) && is_object($wp_manga_volume) )
{
$vol_created_chaps = $wp_manga_volume->get_volume_chapters( $post_id, $volume_id );
if( !empty( $vol_created_chaps ) ){
$vol_created_chaps = array_column( $vol_created_chaps, 'chapter_name' );
if(isset($volume['chapters']))
{
foreach( $volume['chapters'] as $index => $chapter ){
if( in_array( $chapter['name'], $vol_created_chaps ) === false ){
$cur_chap_index = $index;
break;
}
}
}
}
else
{
$cur_chap_index = 0;
}
}
else
{
$cur_chap_index = 0;
}
$resp = ums_fetch_single_chapter( $volume, $volume_id, $post_id, $itemx, $storage, $manga_name, $rule_translate, $skip_last );
if( is_wp_error( $resp ) ){
return $resp;
}
elseif($resp === false || $resp == null)
{
return false;
}
elseif($resp === 'CloudFlare')
{
return 'CloudFlare';
}
elseif($resp === 'fail')
{
return 'fail';
}
elseif($resp !== true)
{
return 'ok';
}
return true;
}
function ums_manga_url_filter( $url ){
$url = str_replace( 'https://', '', $url );
$url = str_replace( 'http://', '', $url );
$url = str_replace( '//', '', $url );
$url = str_replace( 'http:', '', $url );
return "http://{$url}";
}
function ums_get_chapter_images( $url )
{
$url = ums_manga_url_filter( $url );
$images_url = array();
$page_html = ums_get_web_page($url);
if( strpos( $page_html, 'Sorry, its licensed, and not available.') !== false || strpos( $page_html, 'has been licensed, it is not available in') !== false ){
ums_log_to_file( "URL not available in your country: $url");
return false;
}
if( strpos( $page_html, 'licensed and not available.') !== false ){
ums_log_to_file( "URL not available in your country (licensed): $url");
return false;
}
$total_pages = 1;
preg_match('/var imagecount=(\d+);/', $page_html, $matches);
if($matches){
$total_pages = $matches[1];
}
$chapterid = '';
preg_match('/var chapterid =(\d+);/', $page_html, $matches);
if($matches){
$chapterid = $matches[1];
}
require_once (dirname(__FILE__) . "/res/UMSJavaScriptUnpacker.php");
$dm5_key = '';
preg_match('/eval\(.*(dm5_key).*\)/', $page_html, $matches);
$unpacker = new UMSJavaScriptUnpacker();
if($matches){
$eval = $matches[0];
$js = $unpacker->unpack($eval);
$dm5_key = str_replace(array('\'','+',' '),array('','',''),str_replace(array('var guidkey=',';$("#dm5_key").val(guidkey);'), array('',''), $js));
}
$newImgs = '';
preg_match('/eval\(.*(newImgs).*\)/', $page_html, $matches);
if($matches){
$eval = $matches[0];
$js = $unpacker->unpack($eval);
$newImgs = explode(',', str_replace('var newImgs=[', '', substr($js, 0, strpos($js, '];var newImginfos'))));
}
if($newImgs)
{
foreach($newImgs as $img){
$url = 'http:' . str_replace("'", "", $img);
array_push($images_url, substr($url, 0, strpos($url, '?')));
}
if(count($images_url) > 0)
{
return $images_url;
}
}
$base_url = str_replace( '1.html', '', $url );
$base_url = str_replace( '1.htm', '', $base_url );
$base_url = ums_manga_url_filter( $base_url );
for( $i = 1; $i <= $total_pages; $i++ ){
$data_url = $base_url . 'chapterfun.ashx?cid=' . $chapterid . '&page=' . $i . '&key=' . ($i == 1 ? '' : $dm5_key);
$eval = ums_get_web_page($data_url);
if($eval != '' && strpos($eval, "Bad Request - Invalid URL") === false){
$js = $unpacker->unpack($eval);
$idx1 = strpos($js, 'var pix="') + 9;
$idx2 = strpos($js, '";');
$url_1 = substr($js, $idx1, $idx2 - $idx1);
$idx1 = strpos($js, 'var pvalue=["') + 13;
$idx2 = strpos($js, '","');
if($idx2 === false)
{
$idx2 = strpos($js, '"];');
}
$url_2 = substr($js, $idx1, $idx2 - $idx1);
$image = $url_1 . $url_2;
$image = ums_image_url_filter($image);
array_push( $images_url, substr($image, 0, strpos($image, '?')) );
}
}
if(count($images_url) == 0)
{
require_once (dirname(__FILE__) . "/res/simple_html_dom.php");
$html = ums_str_get_html( $page_html );
if( empty( $html ) ){
ums_log_to_file( "Cannot get chapter images from $url");
return false;
}
$pages = $html->find('#top_bar select.m > option');
if( empty( $pages ) && !isset( $pages[ count( $pages ) - 2 ] ) )
{
preg_match_all('#(?:\d*)<\/a>[\s\n]*\.\.\.<\/span>[\s\n]*><\/a>#', $page_html, $mxt);
if(isset($mxt[1][0]))
{
$total_pages = intval($mxt[1][0]);
}
else
{
ums_log_to_file( "Cannot get chapter total pages " . $page_html);
return false;
}
}
else
{
$total_pages = $pages[ count( $pages ) - 2 ]->plaintext;
}
$page_url = str_replace( '1.html', '', $url );
$page_url = str_replace( '1.htm', '', $page_url );
$page_url = ums_manga_url_filter( $page_url );
$images_url = array();
for( $i = 1; $i <= $total_pages; $i++ ){
$image = ums_get_page_images( "{$page_url}{$i}.html" );
if( empty( $image ) ){
ums_log_to_file( "Cannot get page image {$page_url}{$i}.html" );
return false;
}
$images_url = array_merge( $images_url, $image );
}
}
return $images_url;
}
function ums_get_page_images( $page_url ){
require_once (dirname(__FILE__) . "/res/simple_html_dom.php");
$page_html = ums_get_web_page($page_url);
$html = ums_str_get_html( $page_html );
if( empty( $html ) ){
ums_log_to_file("Cannot get page images from " . $page_url);
return false;
}
$images = $html->find( '#viewer .read_img a > img' );
if( empty( $images ) ){
return false;
}
$images_url = array();
foreach( $images as $image ){
$images_url[] = $image->src;
}
return $images_url;
}
$ums_chapter_images = array();
function ums_fetch_single_chapter( $chapter, $volume_id, $post_id, $itemx, $storage = 'local', $manga_name = '', $rule_translate = 'disabled', $skip_last = '0')
{
$ums_Main_Settings = get_option('ums_Main_Settings', false);
global $wp_manga, $wp_manga_storage;
$timestamp = time();
if($wp_manga == null)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('wp_manga global variable not found');
}
return false;
}
if($wp_manga_storage == null)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('wp_manga_storage global variable not found');
}
return false;
}
$uniqid = $wp_manga->get_uniqid( $post_id );
if(empty($uniqid))
{
$uniqid = uniqid( 'manga_' );
update_post_meta( $post_id, 'manga_unique_id', $uniqid );
}
$slugified_name = $wp_manga_storage->slugify( $itemx->title );
global $wp_manga_chapter;
$chapter_2 = $wp_manga_chapter->get_chapter_by_slug( $post_id, $slugified_name );
if($chapter_2 && $chapter_2['volume_id'] == $volume_id && $chapter_2['chapter_slug'] == $slugified_name){
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Chapter already existing: ' . $chapter_2['chapter_name']);
}
return true;
}
global $ums_chapter_images;
$ums_chapter_images = array();
$chapter->each(function($itempx, $keyx)
{
global $ums_chapter_images;
$ums_chapter_images[] = $itempx->scan;
});
$extract = WP_MANGA_DATA_DIR . $uniqid . '/' . $slugified_name;
$extract_uri = WP_MANGA_DATA_URL . $uniqid . '/';
global $wp_filesystem;
if ( ! is_a( $wp_filesystem, 'WP_Filesystem_Base') ){
include_once(ABSPATH . 'wp-admin/includes/file.php');$creds = request_filesystem_credentials( site_url() );
wp_filesystem($creds);
}
if (!$wp_filesystem->exists( $extract ) ){
if( ! wp_mkdir_p( $extract ) ){
ums_log_to_file("Cannot make dir $extract");
}
}
if(count($ums_chapter_images) == 0)
{
$ums_chapter_images = ums_get_chapter_images( $itemx->url );
if($ums_chapter_images === false)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('No images found for: ' . $itemx->title);
}
return 'fail';
}
}
$has_image = '';
$existing_images = get_post_meta($post_id, 'ums_' . $slugified_name . '_cnt', true);
if(!$existing_images)
{
$existing_images = 0;
}
$index = 1;
foreach( $ums_chapter_images as $image )
{
if($skip_last == '1')
{
if($index == count($ums_chapter_images) && $index > 1)
{
$index++;
continue;
}
}
if($index > $existing_images){
$data = ums_get_manga_page( $image );
if( $data === false ){
ums_log_to_file("Cannot get single chapter manga image from " . $image);
continue;
}
$pathinfo = pathinfo( $image );
$file_name = ums_url_file_name_filter( $pathinfo['basename'] );
$resp = $wp_filesystem->put_contents( "{$extract}/{$file_name}", $data );
if( $resp && empty( $has_image ) )
{
$has_image = '1';
}
if (isset($ums_Main_Settings['enable_cloudflare']) && $ums_Main_Settings['enable_cloudflare'] == 'on')
{
if(time() - $timestamp >= 93 && count($ums_chapter_images) > $index)
{
update_post_meta($post_id, 'ums_' . $slugified_name . '_cnt', $index);
ums_log_to_file('Your server is behind CloudFlare and scraping was stopped because of the 100 second limit CloudFlare is enforcing on sites.');
return 'CloudFlare';
}
}
}
$index++;
}
if( empty( $has_image ) ){
$wp_filesystem->copy( dirname(__FILE__) . "/images/image-placeholder.jpg", "{$extract}/image-placeholder.jpg" );
}
$zat = $itemx->title;
if(empty($zat))
{
$zat = 'Manga';
}
$z_arr = explode('-', $zat);
$ch_name = trim($z_arr[0]);
if(isset($z_arr[1]))
{
$ch_name_ext = trim($z_arr[1]);
}
else
{
$ch_name_ext = '';
}
$arr = ums_spin_and_translate($ch_name, $ch_name_ext, $rule_translate, 'en');
if($arr === false)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Failed to translate manga details: ' . print_r($ch_name, true));
}
}
else
{
$ch_name = $arr[0];
$ch_name_ext = $arr[1];
}
$chapter_args = array(
'post_id' => $post_id,
'volume_id' => $volume_id,
'chapter_name' => $ch_name,
'chapter_name_extend' => $ch_name_ext,
'chapter_slug' => $slugified_name,
);
$chapter_2 = $wp_manga_chapter->get_chapter_by_slug( $post_id, $slugified_name );
if($chapter_2 && $chapter_2['volume_id'] == $volume_id && $chapter_2['chapter_slug'] == $slugified_name){
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Chapter found to be already published: ' . $zat);
}
return true;
}
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Uploading chapter: ' . $zat);
}
$result = $wp_manga_storage->wp_manga_upload_single_chapter( $chapter_args, $extract, $extract_uri, $storage );
if($result == false || $result == null)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('wp_manga_upload_single_chapter failed, details: extract: ' . $extract . ' -- extract_uri: ' . $extract_uri . ' -- storage: ' . $storage . ' -- chapter_args: ' . print_r($chapter_args, true));
}
}
return $result;
}
function ums_url_file_name_filter( $name ){
$name = explode('?', $name);
return $name[0];
}
function ums_my_user_by_rand( $ua ) {
remove_action('pre_user_query', 'ums_my_user_by_rand');
$ua->query_orderby = str_replace( 'user_login ASC', 'RAND()', $ua->query_orderby );
}
function ums_get_upload_cloud_list($upload_cloud_file){
global $wp_filesystem;
if ( ! is_a( $wp_filesystem, 'WP_Filesystem_Base') ){
include_once(ABSPATH . 'wp-admin/includes/file.php');$creds = request_filesystem_credentials( site_url() );
wp_filesystem($creds);
}
if ($wp_filesystem->exists( $upload_cloud_file ) ){
$content = $wp_filesystem->get_contents( $upload_cloud_file );
return json_decode( $content, true );
}
return [];
}
function ums_fetch_chapters_generic( $scans, $post_id, $itemx, $storage = 'local', $manga_name = '', $use_phantom = '0' ){
global $wp_manga_storage, $wp_manga_volume, $wp_manga_chapter;
if(isset($itemx['volume']) && $itemx['volume'] != 'NO-VOLUME' && $itemx['volume'] != 'No Volume' && $itemx['volume'] != '-1' && is_object($wp_manga_volume)){
$find_vols = $wp_manga_volume->get_volumes(
array(
'post_id' => $post_id,
'volume_name' => $itemx['volume']
)
);
if( !empty( $find_vols ) ){
$volume_id = $find_vols[0]['volume_id'];
$is_volume_created = true;
}
else
{
$volume_id = $wp_manga_storage->create_volume( $itemx['volume'], $post_id );
}
}
else
{
$is_volume_created = true;
$volume_id = 0;
}
$cur_chap_index = 0;
$resp = ums_fetch_single_chapter_generic( $scans, $volume_id, $post_id, $itemx, $storage, $manga_name, $use_phantom );
if( is_wp_error( $resp ) ){
return $resp;
}
elseif($resp === false || $resp == null)
{
return false;
}
elseif($resp === 'CloudFlare')
{
return 'CloudFlare';
}
elseif($resp !== true)
{
return 'ok';
}
return true;
}
function ums_fetch_single_chapter_generic( $ums_chapter_images, $volume_id, $post_id, $itemx, $storage = 'local', $manga_name = '', $use_phantom = '0')
{
$ums_Main_Settings = get_option('ums_Main_Settings', false);
global $wp_manga, $wp_manga_storage;
if($wp_manga == null)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('wp_manga global variable not found');
}
return false;
}
if($wp_manga_storage == null)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('wp_manga_storage global variable not found');
}
return false;
}
$uniqid = $wp_manga->get_uniqid( $post_id );
if(empty($uniqid))
{
$uniqid = uniqid( 'manga_' );
update_post_meta( $post_id, 'manga_unique_id', $uniqid );
}
$clearstring = sanitize_title($itemx['title']);
$slugified_name = $wp_manga_storage->slugify( $clearstring );
$slugified_name = sanitize_title($slugified_name);
$slugified_name = preg_replace('/%[a-fA-F0-9]{2}/', '', $slugified_name);
if(empty($slugified_name))
{
$slugified_name = uniqid();
}
global $wp_manga_chapter;
$chapter_2 = $wp_manga_chapter->get_chapter_by_slug( $post_id, $slugified_name );
if($chapter_2 && $chapter_2['volume_id'] == $volume_id && $chapter_2['chapter_slug'] == $slugified_name){
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Chapter already existing: ' . $chapter_2['chapter_name']);
}
return true;
}
$extract = WP_MANGA_DATA_DIR . $uniqid . '/' . $slugified_name;
$extract_uri = WP_MANGA_DATA_URL;
global $wp_filesystem;
if ( ! is_a( $wp_filesystem, 'WP_Filesystem_Base') ){
include_once(ABSPATH . 'wp-admin/includes/file.php');$creds = request_filesystem_credentials( site_url() );
wp_filesystem($creds);
}
if (!$wp_filesystem->exists( $extract ) ){
if( ! wp_mkdir_p( $extract ) ){
ums_log_to_file("Cannot make dir $extract");
}
}
if(count($ums_chapter_images) == 0)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('No images found for: ' . $itemx['title']);
}
return true;
}
$has_image = '';
$existing_images = get_post_meta($post_id, 'ums_' . $slugified_name . '_cnt', true);
if(!$existing_images)
{
$existing_images = 0;
}
$index = 1;
foreach( $ums_chapter_images as $image )
{
if($index > $existing_images){
if($use_phantom == '2' || $use_phantom == '4')
{
$img_use_phantom = $use_phantom;
}
else
{
$img_use_phantom = '0';
}
$data = ums_get_web_page( $image, ums_get_random_user_agent(), $img_use_phantom );
if( $data === false ){
ums_log_to_file("Cannot get single chapter image from " . $image);
continue;
}
if($img_use_phantom == '2' || $img_use_phantom == '4')
{
$data = base64_decode($data);
if( $data === false ){
ums_log_to_file("Cannot base64decode for " . $image);
continue;
}
}
$pathinfo = pathinfo( $image );
$file_name = ums_url_file_name_filter( $pathinfo['basename'] );
$resp = $wp_filesystem->put_contents( "{$extract}/{$index}-{$file_name}", $data );
error_log("{$extract}/{$index}-{$file_name}");
if( $resp && empty( $has_image ) )
{
$has_image = '1';
}
}
$index++;
}
if( empty( $has_image ) ){
$wp_filesystem->copy( dirname(__FILE__) . "/images/image-placeholder.jpg", "{$extract}/image-placeholder.jpg" );
}
$ch_name = str_replace($manga_name, '', $itemx['title']);
$ch_name = trim(trim($ch_name, '- '));
$ch_name = trim($ch_name);
$chapter_args = array(
'post_id' => $post_id,
'volume_id' => $volume_id,
'chapter_name' => $ch_name,
'chapter_name_extend' => '',
'chapter_slug' => $slugified_name,
);
$chapter_2 = $wp_manga_chapter->get_chapter_by_slug( $post_id, $slugified_name );
if($chapter_2 && $chapter_2['volume_id'] == $volume_id && $chapter_2['chapter_slug'] == $slugified_name){
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Chapter found to be already published: ' . $itemx['title']);
}
return true;
}
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Uploading chapter: ' . $itemx['title']);
}
$result = $wp_manga_storage->wp_manga_upload_single_chapter( $chapter_args, $extract, $extract_uri, $storage );
if($result == false || $result == null)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('wp_manga_upload_single_chapter failed, details: extract: ' . $extract . ' -- extract_uri: ' . $extract_uri . ' -- storage: ' . $storage . ' -- chapter_args: ' . print_r($chapter_args, true));
}
}
return $result;
}
function ums_display_random_user(){
add_action('pre_user_query', 'ums_my_user_by_rand');
$args = array(
'orderby' => 'user_login', 'order' => 'ASC', 'number' => 1, 'role__in' => array( 'contributor','author','editor','administrator','super-admin' )
);
$user_query = new WP_User_Query( $args );
$user_query->query();
$results = $user_query->results;
if(empty($results))
{
return false;
}
shuffle($results);
return array_pop($results);
}
function ums_put_upload_cloud_list( $item, $upload_cloud_file ){
$list = ums_get_upload_cloud_list($upload_cloud_file);
if( ! isset( $list[ $item['id'] ] ) )
{
global $wp_filesystem;
if ( ! is_a( $wp_filesystem, 'WP_Filesystem_Base') ){
include_once(ABSPATH . 'wp-admin/includes/file.php');$creds = request_filesystem_credentials( site_url() );
wp_filesystem($creds);
}
$list[ $item['id'] ] = $item;
$wp_filesystem->put_contents( $upload_cloud_file, json_encode( $list, JSON_PRETTY_PRINT ) );
}
return true;
}
function ums_randomName() {
$firstname = array(
'Johnathon',
'Anthony',
'Erasmo',
'Raleigh',
'Nancie',
'Tama',
'Camellia',
'Augustine',
'Christeen',
'Luz',
'Diego',
'Lyndia',
'Thomas',
'Georgianna',
'Leigha',
'Alejandro',
'Marquis',
'Joan',
'Stephania',
'Elroy',
'Zonia',
'Buffy',
'Sharie',
'Blythe',
'Gaylene',
'Elida',
'Randy',
'Margarete',
'Margarett',
'Dion',
'Tomi',
'Arden',
'Clora',
'Laine',
'Becki',
'Margherita',
'Bong',
'Jeanice',
'Qiana',
'Lawanda',
'Rebecka',
'Maribel',
'Tami',
'Yuri',
'Michele',
'Rubi',
'Larisa',
'Lloyd',
'Tyisha',
'Samatha',
);
$lastname = array(
'Mischke',
'Serna',
'Pingree',
'Mcnaught',
'Pepper',
'Schildgen',
'Mongold',
'Wrona',
'Geddes',
'Lanz',
'Fetzer',
'Schroeder',
'Block',
'Mayoral',
'Fleishman',
'Roberie',
'Latson',
'Lupo',
'Motsinger',
'Drews',
'Coby',
'Redner',
'Culton',
'Howe',
'Stoval',
'Michaud',
'Mote',
'Menjivar',
'Wiers',
'Paris',
'Grisby',
'Noren',
'Damron',
'Kazmierczak',
'Haslett',
'Guillemette',
'Buresh',
'Center',
'Kucera',
'Catt',
'Badon',
'Grumbles',
'Antes',
'Byron',
'Volkman',
'Klemp',
'Pekar',
'Pecora',
'Schewe',
'Ramage',
);
$name = $firstname[rand ( 0 , count($firstname) -1)];
$name .= ' ';
$name .= $lastname[rand ( 0 , count($lastname) -1)];
return $name;
}
function ums_strip_links($content)
{
$content1 = preg_replace('~') !== false)
{
ums_log_to_file('CloudFlare protection active on wuxiaworld.site, access is limited! Please install puppeteer on your site and set the "Content Scraping Method To Use" to Puppeteer for scraping to work. Manga URL: ' . $current_manga);
sleep(1);
continue;
}
if(stristr($html_site, 'manga-title-badges custom comic') !== false)
{
ums_log_to_file('This URL is a web comic, not a web novel, skipping it: ' . $current_manga);
continue;
}
if (isset($ums_Main_Settings['request_timeout']) && $ums_Main_Settings['request_timeout'] != '') {
$timeout = intval($ums_Main_Settings['request_timeout']);
} else {
$timeout = 1;
}
sleep($timeout);
require_once (dirname(__FILE__) . "/res/simple_html_dom.php");
$html = ums_str_get_html( $html_site );
$tag = $html->find( '.profile-manga .post-title h1', 0 );
$name_str = '';
$my_slug = '';
$my_slug = str_replace( 'http://', '', $current_manga );
$my_slug = str_replace( 'https://', '', $my_slug );
$my_slug = str_replace( 'http:', '', $my_slug );
$my_slug = str_replace( 'wuxiaworld.site/novel/', '', $my_slug );
$my_slug = str_replace( '/', '', $my_slug );
$my_slug = str_replace( '.html', '', $my_slug );
$my_slug = str_replace( '.htm', '', $my_slug );
if($tag){
$name_str = trim($tag->plaintext);
$span = $tag->find('span', 0);
if($span){
$name_str = trim(str_replace($span->plaintext,'', $name_str));
}
}
if($name_str == '')
{
preg_match_all('#([^<]*?) - WuxiaWorld<\/title>#i', $html_site, $tmathc);
if(isset($tmathc[1][0]))
{
$name_str = $tmathc[1][0];
}
}
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Processing novel: ' . $name_str);
}
$find_post = new WP_Query( array(
'title' => $name_str,
'post_type' => 'wp-manga',
) );
$existing_post_id = false;
if( $find_post->have_posts() )
{
$existing_post_id = $find_post->posts[0]->ID;
}
if($existing_post_id == false)
{
$args = array(
'post_type' => 'wp-manga',
'meta_key' => '_manga_import_slug',
'meta_value' => $my_slug,
'post_status' => array('publish','draft','pending','trash','private','future')
);
$query = new WP_Query( $args );
if( $query->have_posts() ){
$existing_post_id = $query->posts[0]->ID;
}
}
if($existing_post_id == false)
{
$block_warning = $html->find('#page .warning', 0);
if( !empty( $block_warning ) && strpos( $block_warning->plaintext, 'has been licensed, it is not available' ) !== false ){
ums_log_to_file('Request blocked, please use a proxy!');
continue;
}
$desc = '';
$xdesc = $html->find( '.summary__content' );
$xd = '';
if(is_array($xdesc))
{
foreach($xdesc as $dd)
{
if(stristr($dd->plaintext, 'Description'))
{
$xd = trim($dd->plaintext);
$xprefix = 'Description';
if (substr($xd, 0, strlen($xprefix)) == $xprefix) {
$xd = substr($xd, strlen($xprefix));
}
$xd = trim($xd);
}
}
}
if($xd != '')
{
$desc = $xd;
}
$xdesc = $html->find( '.description-summary' );
$xd = '';
if(is_array($xdesc))
{
foreach($xdesc as $dd)
{
$xd .= trim($dd->plaintext);
}
}
if($xd != '')
{
$desc = $xd;
$desc = preg_replace('/Show more$/', '', $desc);
}
if($desc == '')
{
$xdesc = $html->find( '.summary__content .show-more' );
$xd = '';
if(is_array($xdesc))
{
$xd = '';
foreach($xdesc as $dd)
{
$xd .= $dd->plaintext;
}
}
if($xd != '')
{
$desc = $xd;
}
}
if($desc == '')
{
preg_match_all('#<div class="summary__content show-more">(?:<p>)?([\s\S]*?)(?:<\/p>)?<\/div>#i', $html_site, $dmathc);
if(isset($dmathc[1][0]))
{
$desc = $dmathc[1][0];
}
}
$desc = str_replace('wuxiaworld.site', '',$desc);
$thumb = $html->find( '.summary_image .img-responsive' );
$thumb = !empty( $thumb ) ? $thumb[0]->src : '';
if($thumb == '')
{
$thumb = $html->find( '.summary_image' );
$thumb = !empty( $thumb ) ? $thumb[0]->src : '';
}
$stx = '';
$status = $html->find( '.post-status .post-content_item' );
if(is_array($status))
{
foreach($status as $st)
{
$temp_stat = $st->plaintext;
if(stristr($temp_stat, 'status') !== false)
{
$stx = $temp_stat;
}
}
}
if($stx === '')
{
$status = '';
}
else
{
$status = $stx;
}
if(stristr($status, 'ongoing') !== false)
{
$status = 'on-going';
}
elseif(stristr($status, 'completed') !== false)
{
$status = 'end';
}
elseif(stristr($status, 'canceled') !== false)
{
$status = 'canceled';
}
elseif(stristr($status, 'on hold') !== false)
{
$status = 'on-hold';
}
elseif(stristr($status, 'upcoming') !== false)
{
$status = 'upcoming';
}
$alter_name = $html->find( '.summary_content .post-content .post-content_item' );
$xplain = '';
if( !empty( $alter_name ) )
{
foreach($alter_name as $an)
{
if(stristr($an->plaintext, 'Alternative '))
{
$xplain = trim($an->plaintext);
$xprefix = 'Alternative';
if (substr($xplain, 0, strlen($xprefix)) == $xprefix) {
$xplain = substr($xplain, strlen($xprefix));
}
$xplain = trim($xplain);
}
}
}
if($xplain != '')
{
$alter_name = $xplain;
}
else
{
$alter_name = '';
}
$data = $html->find( '.summary_content .post-content .post-content_item' );
$xtype = '';
if( !empty( $data ) ){
foreach($data as $item){
if(trim($item->find('.summary-heading h5',0)->plaintext) == 'Type'){
$xtype = $item->find('.summary-content', 0)->plaintext;
if(trim($xtype) == 'Updating')
{
$xtype = '';
}
}
}
}
$xrelease = '2021';
preg_match_all('#<a href="https:\/\/wuxiaworld\.site\/(?:manga-)?release\/(?:[^"]*?)" rel="tag">([^<]+?)<\/a>#i', $html_site, $xmathc);
if(isset($xmathc[1][0]))
{
$xrelease = trim($xmathc[1][0]);
}
$xauthor = '';
$author = '';
$data = $html->find( '.author-content a' );
if( !empty( $data ) ){
$authors = array();
foreach($data as $arxthor)
{
array_push($authors, $arxthor->plaintext);
if($author == '')
{
$author = $arxthor->plaintext;
}
}
$xauthor = implode(',', $authors);
}
$xartists = '';
$data = $html->find( '.artist-content a' );
if( !empty( $data ) ){
$artists = array();
foreach($data as $artist){
array_push($artists, $artist->plaintext);
}
$xartists = implode(',', $artists);
}
$xgenres = '';
$data = $html->find( '.genres-content a' );
if( !empty( $data ) ){
$genres = array();
foreach($data as $genre){
array_push($genres, $genre->plaintext);
}
$xgenres = implode(',', $genres);
}
$items = $html->find('.summary_content .post-content .post-content_item');
$views = '';
foreach($items as $item){
if(trim($item->find('.summary-heading', 0)->plaintext) == 'Rank'){
$str = $item->find('.summary-content',0)->plaintext;
preg_match_all('~\d+(?:\.\d+)?~', $str, $matches);
if($matches){
if(count($matches) == 3){
$views = $matches[0][1];
} else {
$views = $matches[0][0];
}
if(strpos($str, 'K monthly views') !== false){
$views = floatval($views) * 1000;
}
if(strpos($str, 'M monthly views') !== false){
$views = floatval($views) * 1000000;
}
break;
}
}
}
if(empty(trim($views)) || !is_numeric(strip_tags( $views )))
{
$viewsm = rand(100,500);
}
else
{
$viewsm = strip_tags(trim($views));
}
$average_vote = $html->find('.post-total-rating .total_votes', 0);
$str = $html->find('.vote-details',0);
$xrating = array();
if(!empty($str))
{
preg_match_all('!\d+!', $str, $matches);
$number_votes = 1;
if($matches && count($matches[0]) == 3)
{
$number_votes = $matches[0][2];
}
if( !empty( $average_vote ) && $average_vote->plaintext > 0 ){
$xrating = array(
'avg' => $average_vote->plaintext,
'numbers' => $number_votes
);
}
}
$xtags = '';
$data = $html->find( '.wp-manga-tags-list a' );
if( !empty( $data ) ){
$tags = array();
foreach($data as $tag){
$ttext = trim($tag->plaintext);
$ttext = str_ireplace('wuxiaworldsite', get_bloginfo('name'), $ttext);
$ttext = str_ireplace('daonovel', get_bloginfo('name'), $ttext);
array_push($tags, $ttext);
}
$xtags = implode(',', $tags);
}
$xtime_year = '';
$time_year = strtotime($xrelease);
if($time_year !== false)
{
$time_year = date("Y", $time_year);
if($time_year !== false)
{
$xtime_year = $time_year;
}
}
$post_args = array(
'manga_import_slug' => $my_slug,
'title' => $name_str,
'post_status' => $post_status,
'description' => $desc,
'thumb' => $thumb,
'status' => $status,
'altername' => strip_tags( $alter_name ),
'type' => strip_tags( $xtype ),
'release' => $xtime_year,
'authors' => strip_tags( $xauthor ),
'artists' => strip_tags( $xartists ),
);
$post_args['genres'] = '';
if($auto_categories == 'genre')
{
$post_args['genres'] = strip_tags( $xgenres );
}
if($default_category != '')
{
if($post_args['genres'] != '')
{
$post_args['genres'] .= ',';
}
$genreplus = get_term_by('id', $default_category, 'wp-manga-genre');
if($genreplus !== false)
{
$post_args['genres'] .= $genreplus->slug;
}
}
$post_args['views'] = strip_tags( $viewsm );
$post_args['ratings'] = $xrating;
if($can_create_tag == 'genre')
{
$post_args['tags'] = strip_tags( $xgenres );
}
elseif($can_create_tag == 'tags')
{
$post_args['tags'] = strip_tags( $xtags );
}
else
{
$post_args['tags'] = '';
}
if($item_create_tag != '')
{
$ctag = explode(',', $item_create_tag);
foreach($ctag as $ctap)
{
$ctap = trim($ctap);
if($post_args['tags'] != '')
{
$post_args['tags'] .= ',';
}
$genreplus = get_term_by('name', $ctap, 'wp-manga-tag');
if($genreplus !== false)
{
$post_args['tags'] .= $genreplus->slug;
}
else
{
$cat = wp_insert_term($ctap, 'wp-manga-tag');
if(!is_wp_error( $cat ))
{
$post_args['tags'] .= $cat['slug'];
}
}
}
}
$post_args['description'] = str_ireplace('wuxiaworld.site', $_SERVER['HTTP_HOST'], $post_args['description']);
$arr = ums_spin_and_translate($post_args['title'], $post_args['description'], $rule_translate, 'en');
if($arr === false)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Skipping web novel (because it failed to be translated): ' . print_r($post_args, true));
}
continue;
}
else
{
if($no_translate_title != '1')
{
$post_args['title'] = $arr[0];
}
$post_args['description'] = $arr[1];
}
$za_xpost_args = array(
'post_title' => !empty( $post_args['title'] ) ? $post_args['title'] : '',
'post_content' => !empty( $post_args['description'] ) ? $post_args['description'] : '',
'post_type' => 'wp-manga',
'post_status' => isset( $post_args['post_status'] ) ? $post_args['post_status'] : 'pending',
);
$existing_again = false;
if($za_xpost_args['post_title'] != '')
{
$xposts = get_posts(
array(
'post_type' => 'wp-manga',
'title' => html_entity_decode($za_xpost_args['post_title']),
'post_status' => 'all',
'numberposts' => 1,
'update_post_term_cache' => false,
'update_post_meta_cache' => false,
'orderby' => 'post_date ID',
'order' => 'ASC',
)
);
if ( ! empty( $xposts ) ) {
$ex_page = $xposts[0];
} else {
$ex_page = null;
}
if(isset($ex_page->ID))
{
$existing_again = true;
$existing_post_id = $ex_page->ID;
}
}
if($existing_again == false)
{
$accept_comments = 'closed';
if ($enable_comments == '1') {
$accept_comments = 'open';
}
$za_xpost_args['comment_status'] = $accept_comments;
if ($enable_pingback == '1')
{
$za_xpost_args['ping_status'] = 'open';
}
else
{
$za_xpost_args['ping_status'] = 'closed';
}
if($get_date == '1')
{
if(!empty($xrelease))
{
if(strlen($xrelease) == 4)
{
$xrelease .= '-01-01 00:00';
}
$postdatex = gmdate("Y-m-d H:i:s", strtotime($xrelease));
$za_xpost_args['post_date_gmt'] = $postdatex;
}
}
if($user_name_type == 'rand')
{
$randid = ums_display_random_user();
if($randid === false)
{
$za_xpost_args['post_author'] = ums_randomName();
}
else
{
$za_xpost_args['post_author'] = $randid->ID;
}
}
elseif($user_name_type == 'feed-news')
{
$sp_post_user_name = ums_randomName();
if($author == '' || $author == '1' || $author == 'null')
{
$author = ums_randomName();
}
if($author != '')
{
$xauthor = sanitize_user( $author );
$xauthor = apply_filters( 'pre_user_login', $xauthor );
$xauthor = trim( $xauthor );
if(username_exists( $xauthor ))
{
$user_id_t = get_user_by('login', $xauthor);
if($user_id_t)
{
$sp_post_user_name = $user_id_t->ID;
}
}
else
{
$palphabet = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890!@#$%^*()-+=_?><,.;:}{][';
$ppass = '';
$alphaLength = strlen($palphabet) - 1;
for ($ipass = 0; $ipass < 8; $ipass++)
{
$npass = rand(0, $alphaLength);
$ppass .= $palphabet[$npass];
}
$curr_id = wp_create_user($xauthor, $ppass, ums_generate_random_email());
if ( is_int($curr_id) )
{
$u = new WP_User($curr_id);
$u->remove_role('subscriber');
$u->add_role('author');
$sp_post_user_name = $curr_id;
}
}
}
$za_xpost_args['post_author'] = ums_utf8_encode($sp_post_user_name);
}
else
{
$za_xpost_args['post_author'] = ums_utf8_encode($post_user_name);
}
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Inserting new novel: ' . $za_xpost_args['post_title']);
}
$existing_post_id = wp_insert_post( $za_xpost_args );
if( ! $existing_post_id || is_wp_error( $existing_post_id ) ){
ums_log_to_file('Failed to insert novel into db: ' . $post_args['title']);
continue;
}
add_post_meta($existing_post_id, 'ums_parent_rule', $type . '-' . $param);
wp_set_object_terms( $existing_post_id, 'Ums_' . $type . '_' . $param, 'coderevolution_post_source', true);
if($thumb != '')
{
$thumb_id = ums_wp_mcl_e_upload_file( $post_args['thumb'], $use_phantom, $phantom_wait, $existing_post_id );
if($thumb_id === false)
{
include_once( ABSPATH . 'wp-admin/includes/image.php' );
$thcontent = ums_get_web_page($thumb);
if (isset($ums_Main_Settings['request_timeout']) && $ums_Main_Settings['request_timeout'] != '') {
$timeout = intval($ums_Main_Settings['request_timeout']);
} else {
$timeout = 1;
}
sleep($timeout);
$pathinfo = pathinfo( $thumb );
if( $thcontent != false ){
$upload_dir = wp_upload_dir();
$file_tmp_path = $upload_dir['basedir'] . '/' . $pathinfo['filename'] . '-' . $existing_post_id . '.' . explode('?',$pathinfo['extension'])[0];
$file = $wp_filesystem->put_contents( $file_tmp_path, $thcontent );
$wp_filetype = wp_check_filetype(basename($file_tmp_path), null );
$attachment = array(
'post_mime_type' => $wp_filetype['type'],
'post_title' => $existing_post_id,
'post_content' => '',
'post_status' => 'inherit'
);
$attach_id = wp_insert_attachment( $attachment, $file_tmp_path );
$thumb_id = $attach_id;
$imagenew = get_post( $attach_id );
$fullsizepath = get_attached_file( $imagenew->ID );
require_once( ABSPATH . 'wp-admin/includes/image.php' );
require_once( ABSPATH . 'wp-admin/includes/media.php' );
$attach_data = wp_generate_attachment_metadata( $attach_id, $fullsizepath );
wp_update_attachment_metadata( $attach_id, $attach_data );
}
}
}
else
{
$thumb_id = false;
}
$meta_data = array(
'_manga_import_slug' => $my_slug,
'_thumbnail_id' => $thumb_id,
'_wp_manga_alternative' => strip_tags( $alter_name ),
'_wp_manga_type' => strip_tags( $xtype ),
'_wp_manga_status' => $status,
'_wp_manga_chapter_type' => 'text',
'_wp_manga_chapters_warning'=> $chapter_warning,
);
foreach( $meta_data as $key => $value ){
if( !empty( $value ) ){
update_post_meta( $existing_post_id, $key, $value );
}
}
$manga_terms = array(
'wp-manga-release' => strip_tags( $xrelease ),
'wp-manga-author' => strip_tags( $xauthor ),
'wp-manga-artist' => strip_tags( $xartists ),
'wp-manga-genre' => isset( $post_args['genres'] ) ? $post_args['genres'] : '',
'wp-manga-tag' => isset( $post_args['tags'] ) ? $post_args['tags'] : null,
);
foreach( $manga_terms as $tax => $term ){
$resp = ums_add_manga_terms( $existing_post_id, $term, $tax );
}
ums_update_post_views( $existing_post_id, strip_tags( $viewsm ) );
ums_update_post_ratings( $existing_post_id, $xrating );
}
}
$find_chapters = $html->find('.listing-chapters_wrap li');
$output = array();
$current_vol = 'NO-VOLUME';
$chapters = array();
foreach( $find_chapters as $chapter )
{
$full_chapter_name = trim(preg_replace('/\t+/', '', $chapter->find('a',0)->plaintext));
$chapter_extend_name = '';
$names = explode(' - ', $full_chapter_name);
$chapter_name = $full_chapter_name;
if(count($names) == 2){
$chapter_extend_name = $names[1];
$chapter_name = $names[0];
}
$chapters[] = array(
'name' => $chapter_name,
'extend_name' => $chapter_extend_name,
'url' => $chapter->find('a',0)->href
);
}
$lazy_loaded = false;
if(count($chapters) == 0)
{
$latest_scraped = get_post_meta( $existing_post_id, 'ums_latest_scraped', true );
if(!empty($latest_scraped))
{
if(!stristr($latest_scraped, 'wuxiaworld'))
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Skipping, already scraped from another source: ' . $za_xpost_args['post_title'] . ' (' . $latest_scraped . ')');
}
continue;
}
$chap_url = $latest_scraped;
}
else
{
if($chapter_slug != '')
{
$chap_url = $current_manga . $chapter_slug . '/';
}
else
{
$chap_url = $current_manga . 'chapter-1/';
}
}
$lazy_loaded = true;
preg_match_all('#\/chapter-([^/]*?)\/#i', $chap_url, $chnamje);
if(isset($chnamje[1][0]))
{
$chap_name = 'Chapter ' . $chnamje[1][0];
}
else
{
$chap_name = 'Chapter ' . rand();
}
$chapters[] = array(
'name' => $chap_name,
'url' => $chap_url
);
}
$ums_max_chapters = $max;
if($reverse_chapters == '1')
{
$reverse = true;
}
else
{
$reverse = false;
}
if($reverse == true)
{
$chapters = array_reverse($chapters);
}
$local_imported = 0;
$new_chap = false;
//$chapters = array(array('url' => 'https://wuxiaworld.site/novel/the-demon-prince-goes-to-the-academy-pvt/chapter-219/', 'name' => 'ch 219'));
if($lazy_loaded == true)
{
$chapid = 1;
while(count($chapters) != 0)
{
$randsleep = rand(500,1000);
usleep($randsleep * 1000);
$chaps = $chapters[0];
if($ums_max_chapters <= $local_imported)
{
break;
}
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Processing novel chapter: ' . $chaps['url']);
}
$page_html = ums_get_web_page($chaps['url'], ums_get_random_user_agent(), $use_phantom, $phantom_wait);
if( $page_html == false ){
ums_log_to_file('Failed to download HTML for current chapter: ' . $chaps['name']);
break;;
}
if(stristr($page_html, 'Cloudflare Ray ID:') !== false)
{
ums_log_to_file('CloudFlare protection active on chapter, access is limited! Please install puppeteer on your site and set the "Content Scraping Method To Use" to Puppeteer for scraping to work. Chapter URL: ' . $chaps['url']);
sleep(1);
break;
}
$html = ums_str_get_html( $page_html );
if( empty( $html ) ){
ums_log_to_file('Failed to parse HTML for current chapter: ' . $chaps['name']);
break;
}
$chapid++;
preg_match_all('#<div class="cha-tit skiptranslate">\s*<h3>([^<]*?)<\/h3>#i', $page_html, $titli);
if(isset($titli[1][0]))
{
$chaps['name'] = $titli[1][0];
}
preg_match_all('#<a\s*href="([^"]*?)"\s*class="btn next_page"#i', $page_html, $zurli);
if(isset($zurli[1][0]))
{
preg_match_all('#https:\/\/wuxiaworld\.site\/novel\/(?:[^\/]*?)\/chapter-([^\/]*?)\/#i', $zurli[1][0], $xurli);
if(isset($xurli[1][0]))
{
$dummy_chap = array(
'name' => 'Chapter ' . $xurli[1][0],
'url' => $zurli[1][0]
);
}
else
{
$dummy_chap = array(
'name' => 'Chapter ' . $chapid,
'url' => $zurli[1][0]
);
}
update_post_meta($existing_post_id, 'ums_latest_scraped', $zurli[1][0]);
}
else
{
$dummy_chap = false;
}
if($dummy_chap !== false)
{
$chapters = array($dummy_chap);
}
else
{
$chapters = array();
}
$slugified_name = $wp_manga_storage->slugify( $chaps['name'] );
$chapter_2 = $wp_manga_chapter->get_chapter_by_slug( $existing_post_id, $slugified_name );
if($chapter_2 && strtolower($chapter_2['chapter_slug']) == strtolower($slugified_name))
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Chapter is already published, skipping it: ' . $chapter_2['chapter_name']);
}
continue;
}
$content = '';
$doc = new DOMDocument;
$internalErrors = libxml_use_internal_errors(true);
$doc->loadHTML('<?xml encoding="utf-8" ?>' . $page_html);
libxml_use_internal_errors($internalErrors);
$xpath = new \DOMXpath($doc);
$articles = $xpath->query('//*[contains(@class, "text-left")]');
if(empty($content))
{
$articles = $xpath->query('//*[contains(@class, "text-left")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
}
$content = trim($content);
}
}
if(empty($content))
{
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
break;
}
$content = trim($content);
$prfx1 = '<div class="text-left">';
$prfx2 = '<div class="new-content">';
$prfx3 = '<div class="chapter-entity">';
if (substr($content, 0, strlen($prfx1)) == $prfx1) {
$content = substr($content, strlen($prfx1));
$content = preg_replace('#<\/div>$#', '', $content);
}
$content = trim($content);
if (substr($content, 0, strlen($prfx2)) == $prfx2) {
$content = substr($content, strlen($prfx2));
$content = preg_replace('#<\/div>$#', '', $content);
}
$content = trim($content);
if (stristr($content, $prfx3) !== false) {
$content = str_ireplace($prfx3, '', $content);
$content = preg_replace('#<\/div>$#', '', $content);
}
$content = trim($content);
if(strstr($content, '<div class="entry-content">') !== false)
{
$content = str_replace('<div class="entry-content">', '', $content);
$content = preg_replace('#<\/div>$#', '', $content);
}
$content = trim($content);
if(preg_match('#<div(?:[^>]*?)id="chapterText">#', $content))
{
$content = preg_replace('#<div(?:[^>]*?)id="chapterText">#', '', $content);
$content = preg_replace('#<\/div>$#', '', $content);
}
}
}
if(empty($content))
{
$articles = $xpath->query('//*[contains(@class, "content-body")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
}
$content = trim($content);
}
}
if(empty($content))
{
$articles = $xpath->query('//*[contains(@class, "cha-tit")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
}
$content = trim($content);
}
$articles = $xpath->query('//*[contains(@class, "cha-content")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
}
}
}
if(empty($content))
{
$content = $html->find('.reading-content .text-left .cha-words .cha-words', 0);
if($content)
{
$other_ps = $content->find('p');
if(count($other_ps) > 0)
{
$text = '';
foreach($other_ps as $p)
{
$text .= '<p> ' . $p->plaintext . '</p>';
}
$content = $text;
}
else
{
$content = $content->innertext;
}
$content = trim($content);
}
else
{
if($html->find('.reading-content #content', 0))
{
$content = $html->find('.reading-content #content', 0)->innertext;
}
else
{
$content = $html->find('.reading-content .text-left .cha-content .cha-words', 0);
if($content)
{
$content = $content->innertext;
}
else
{
$content = $html->find('.reading-content .text-left', 0);
if($content)
{
$other_ps = $content->find('p');
if(count($other_ps) > 0)
{
$text = '';
foreach($other_ps as $p)
{
$text .= '<p> ' . $p->plaintext . '</p>';
}
$content = $text;
}
else
{
if($inner_content = $content->find('#chapter-content', 0))
{
$content = $inner_content->innertext;
}
else
{
if($inner_content = $content->find('.text-left', 0))
{
$content = $inner_content->innertext;
}
else
{
if($inner_content = $content->find('.fr-view', 0))
{
$content = $inner_content->innertext;
}
else
{
$content = $content->innertext;
}
}
}
}
}
}
}
if(!$content)
{
$content = '';
$ps = $html->find('.reading-content .text-left p');
foreach($ps as $p)
{
$content .= '<p>' . $p->plaintext . '</p>';
}
}
}
}
$content = preg_replace('#<a[^>]*href=["\']https?:\/\/wuxiaworld\.site[^"\']*["\'][^>]*>\s*<p[^>]*>.*?Wuxia.*?World\.Site.*?<\/p>\s*<\/a>#', '', $content);
$content = str_replace('wuxiaworld.site', '', $content);
$content = str_replace('wuxiaworldsite', '', $content);
if(stristr($content, '<article') !== false)
{
$html2 = ums_str_get_html( $content );
if( !empty( $html2 ) ){
$artcontent = $html2->find('//article', 0);
if($artcontent !== false)
{
$content = $artcontent->innertext;
}
}
}
$content = preg_replace('#<div class="wp-container-[a-zA-Z0-9]* entry-content wp-block-post-content">#i', '', $content);
$content = preg_replace('#<\/div>$#', '', $content);
$qtags = array('div', 'iframe', 'script', 'ins', 'h12');
foreach ($qtags as $qtag) {
$regex = '#<\s*' . $qtag . '[^>]*>.*?<\s*\/\s*'. $qtag . '>#msi';
$content = preg_replace($regex, '', $content);
}
$content = preg_replace('#<style([\s\S]+?)<\/style>#i', '', $content);
$content = preg_replace('#<\/?div([^>]*)>#i', '', $content);
if($strip_images == '1')
{
$content = ums_sanitize_html_content($content);
}
$chapter_content = trim($content);
$chapter_content = ums_strip_links($chapter_content);
if($chapter_content == '')
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Skipping current chapter (because it failed to parse first chapter content): ' . print_r($chaps['url'], true));
}
//new_chap true to prevent infinite running
$new_chap = true;
break;
}
$arr = ums_spin_and_translate($chaps['name'], $chapter_content, $rule_translate, 'en');
if($arr === false)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Skipping current chapter (because it failed to be translated): ' . print_r($chaps['url'], true));
}
}
elseif($no_translate_title != '1')
{
$chaps['name'] = $arr[0];
$chaps['extend_name'] = $arr[0];
$chapter_content = $arr[1];
}
global $wp_manga_text_type;
if(empty($chaps['name']))
{
$chaps['name'] = 'Novel';
}
$chapter_args = array(
'post_id' => $existing_post_id,
'chapter_name' => $chaps['name'],
'chapter_name_extend' => $name_str . ' ' . $chaps['extend_name'],
'volume_id' => '',
'chapter_content' => $chapter_content,
);
$slugified_name = $wp_manga_storage->slugify( $chaps['name'] );
$chapter_2 = $wp_manga_chapter->get_chapter_by_slug( $existing_post_id, $slugified_name );
if($chapter_2 && strtolower($chapter_2['chapter_slug']) == strtolower($slugified_name))
{
ums_log_to_file('Chapter name already published, we will skip: ' . $chapter_2['chapter_name']);
continue;
}
$chapter_id = $wp_manga_text_type->insert_chapter( $chapter_args );
if( $chapter_id ){
if( is_wp_error( $chapter_id ) ){
ums_log_to_file('Failed to insert new chapter: ' . $chaps['name'] . ' error: ' . $chapter_id->get_error_message());
continue;
}
else
{
$new_chap = true;
$ums_imported_chapters++;
$local_imported++;
}
}
if (isset($ums_Main_Settings['request_timeout']) && $ums_Main_Settings['request_timeout'] != '') {
$timeout = intval($ums_Main_Settings['request_timeout']);
} else {
$timeout = 1;
}
sleep($timeout);
}
}
else
{
foreach($chapters as $chaps)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Processing novel chapter (no lazyload): ' . $chaps['url']);
}
if($ums_max_chapters <= $local_imported)
{
break;
}
$slugified_name = $wp_manga_storage->slugify( $chaps['name'] );
$chapter_2 = $wp_manga_chapter->get_chapter_by_slug( $existing_post_id, $slugified_name );
if($chapter_2 && strtolower($chapter_2['chapter_slug']) == strtolower($slugified_name))
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Chapter already published, skipping it: ' . $chapter_2['chapter_name']);
}
continue;
}
$page_html = ums_get_web_page($chaps['url'], ums_get_random_user_agent(), $use_phantom, $phantom_wait);
if( $page_html == false ){
ums_log_to_file('Failed to download HTML for chapter: ' . $chaps['name']);
continue;
}
$html = ums_str_get_html( $page_html );
if( empty( $html ) ){
ums_log_to_file('Failed to parse HTML for chapter: ' . $chaps['name']);
continue;
}
$content = '';
$doc = new DOMDocument;
$internalErrors = libxml_use_internal_errors(true);
$doc->loadHTML('<?xml encoding="utf-8" ?>' . $page_html);
libxml_use_internal_errors($internalErrors);
$xpath = new \DOMXpath($doc);
$articles = $xpath->query('//*[contains(@class, "text-left")]');
if(empty($content))
{
$articles = $xpath->query('//*[contains(@class, "text-left")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
}
$content = trim($content);
}
}
if(empty($content))
{
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
break;
}
$content = trim($content);
$prfx1 = '<div class="text-left">';
$prfx2 = '<div class="new-content">';
if (substr($content, 0, strlen($prfx1)) == $prfx1) {
$content = substr($content, strlen($prfx1));
$content = preg_replace('#<\/div>$#', '', $content);
}
$content = trim($content);
if (substr($content, 0, strlen($prfx2)) == $prfx2) {
$content = substr($content, strlen($prfx2));
$content = preg_replace('#<\/div>$#', '', $content);
}
$content = trim($content);
if(strstr($content, '<div class="entry-content">') !== false)
{
$content = str_replace('<div class="entry-content">', '', $content);
$content = preg_replace('#<\/div>$#', '', $content);
}
}
}
if(empty($content))
{
$articles = $xpath->query('//*[contains(@class, "content-body")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
}
}
}
if(empty($content))
{
$articles = $xpath->query('//*[contains(@class, "cha-tit")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
}
}
$articles = $xpath->query('//*[contains(@class, "cha-content")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
}
}
}
if(empty($content))
{
$content = $html->find('.reading-content .text-left .cha-words .cha-words', 0);
if($content)
{
$other_ps = $content->find('p');
if(count($other_ps) > 0)
{
$text = '';
foreach($other_ps as $p)
{
$text .= '<p> ' . $p->plaintext . '</p>';
}
$content = $text;
}
else
{
$content = $content->innertext;
}
}
else
{
if($html->find('.reading-content #content', 0))
{
$content = $html->find('.reading-content #content', 0)->innertext;
}
else
{
$content = $html->find('.reading-content .text-left .cha-content .cha-words', 0);
if($content)
{
$content = $content->innertext;
}
else
{
$content = $html->find('.reading-content .text-left', 0);
if($content)
{
$other_ps = $content->find('p');
if(count($other_ps) > 0)
{
$text = '';
foreach($other_ps as $p)
{
$text .= '<p> ' . $p->plaintext . '</p>';
}
$content = $text;
}
else
{
if($inner_content = $content->find('#chapter-content', 0))
{
$content = $inner_content->innertext;
}
else
{
if($inner_content = $content->find('.text-left', 0))
{
$content = $inner_content->innertext;
}
else
{
if($inner_content = $content->find('.fr-view', 0))
{
$content = $inner_content->innertext;
}
else
{
$content = $content->innertext;
}
}
}
}
}
}
}
if(!$content)
{
$content = '';
$ps = $html->find('.reading-content .text-left p');
foreach($ps as $p)
{
$content .= '<p>' . $p->plaintext . '</p>';
}
}
}
}
$content = preg_replace('#<a[^>]*href=["\']https?:\/\/wuxiaworld\.site[^"\']*["\'][^>]*>\s*<p[^>]*>.*?Wuxia.*?World\.Site.*?<\/p>\s*<\/a>#', '', $content);
$content = str_replace('wuxiaworld.site', '', $content);
$content = str_replace('world.site', '', $content);
$content = str_replace('wuxia', '', $content);
$content = str_replace('wuxiaworldsite', '', $content);
if(stristr($content, '<article') !== false)
{
$html2 = ums_str_get_html( $content );
if( !empty( $html2 ) ){
$artcontent = $html2->find('//article', 0);
if($artcontent !== false)
{
$content = $artcontent->innertext;
}
}
}
$qtags = array('iframe', 'script', 'ins', 'h12');
foreach ($qtags as $qtag) {
$regex = '#<\s*' . $qtag . '[^>]*>.*?<\s*\/\s*'. $qtag . '>#msi';
$content = preg_replace($regex, '', $content);
}
$content = preg_replace('#<style([\s\S]+?)<\/style>#i', '', $content);
if($strip_images == '1')
{
$content = ums_sanitize_html_content($content);
}
$chapter_content = trim($content);
$chapter_content = ums_strip_links($chapter_content);
if($chapter_content == '')
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Skipping current chapter (because it failed to parse first chapter content): ' . print_r($chaps['url'], true));
}
//new_chap true to prevent infinite running
$new_chap = true;
break;
}
$arr = ums_spin_and_translate($chaps['name'], $chapter_content, $rule_translate, 'en');
if($arr === false)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Skipping chapter (because it failed to be translated): ' . print_r($post_args, true));
}
}
elseif($no_translate_title != '1')
{
$chaps['name'] = $arr[0];
$chaps['extend_name'] = $arr[0];
$chapter_content = $arr[1];
}
global $wp_manga_text_type;
if(empty($chaps['name']))
{
$chaps['name'] = 'Novel';
}
$chapter_args = array(
'post_id' => $existing_post_id,
'chapter_name' => $chaps['name'],
'chapter_name_extend' => $name_str . ' ' . $chaps['extend_name'],
'volume_id' => '',
'chapter_content' => $chapter_content,
);
$slugified_name = $wp_manga_storage->slugify( $chaps['name'] );
$chapter_2 = $wp_manga_chapter->get_chapter_by_slug( $existing_post_id, $slugified_name );
if($chapter_2 && strtolower($chapter_2['chapter_slug']) == strtolower($slugified_name))
{
ums_log_to_file('Chapter name already published, skipping it: ' . $chapter_2['chapter_name']);
continue;
}
$chapter_id = $wp_manga_text_type->insert_chapter( $chapter_args );
if( $chapter_id ){
if( is_wp_error( $chapter_id ) ){
ums_log_to_file('Failed to insert chapter: ' . $chaps['name'] . ' error: ' . $chapter_id->get_error_message());
continue;
}
else
{
$new_chap = true;
$ums_imported_chapters++;
$local_imported++;
}
}
if (isset($ums_Main_Settings['request_timeout']) && $ums_Main_Settings['request_timeout'] != '') {
$timeout = intval($ums_Main_Settings['request_timeout']);
} else {
$timeout = 1;
}
sleep($timeout);
}
}
if($new_chap == true)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Total novel chapters scraped: ' . $local_imported);
}
$scraped_manga++;
}
}
}
catch(Exception $e)
{
ums_log_to_file('Importing failed: ' . $e->getMessage());
if($auto == 1)
{
ums_clearFromList($param, $type);
}
return 'fail';
}
}
}
elseif($type == 2)
{
if(trim($max_manga) != '')
{
$get_max_manga = intval(trim($max_manga));
}
else
{
$get_max_manga = 999;
}
$items = array();
$page_increased = false;
{
$manga_arr = array();
try
{
global $wp_filesystem;
if ( ! is_a( $wp_filesystem, 'WP_Filesystem_Base') ){
include_once(ABSPATH . 'wp-admin/includes/file.php');$creds = request_filesystem_credentials( site_url() );
wp_filesystem($creds);
}
$prefixh = 'http://novlove.com/novel/';
$wprefixh = 'http://www.novlove.com/novel/';
$prefixs = 'https://novlove.com/novel/';
$wprefixs = 'https://www.novlove.com/novel/';
$manga_names = explode(',', $manga_name);
$manga_names = array_map('trim', $manga_names);
foreach($manga_names as $mngn)
{
if (substr($mngn, 0, strlen($prefixs)) == $prefixs) {
$manga_arr[] = $mngn;
}
elseif (substr($mngn, 0, strlen($prefixh)) == $prefixh) {
$manga_arr[] = $mngn;
}
elseif (substr($mngn, 0, strlen($wprefixh)) == $wprefixh) {
$manga_arr[] = $mngn;
}
elseif (substr($mngn, 0, strlen($wprefixs)) == $wprefixs) {
$manga_arr[] = $mngn;
}
else
{
if($wp_filesystem->exists(dirname(__FILE__) . "/sitemap_box.txt"))
{
$handle = fopen(dirname(__FILE__) . "/sitemap_box.txt", "r");
if ($handle)
{
while (!feof($handle))
{
$buffer = fgets($handle);
if($buffer !== false)
{
if($mngn == '*')
{
$manga_arr[] = 'https://novlove.com/novel/' . trim($buffer) . '/';
}
else
{
$xbuffer = str_replace('-', ' ', $buffer);
$mngnwords = explode(' ', $mngn);
$xbuffer_arr = explode(' ', $xbuffer);
foreach($mngnwords as $mangx)
{
$mangx = trim($mangx);
$xfound = false;
foreach($xbuffer_arr as $xbf)
{
$xbf = trim($xbf);
if($xbf == $mangx)
{
$xfound = true;
}
}
if($xfound == true)
{
$manga_arr[] = 'https://novlove.com/novel/' . trim($buffer) . '/';
break;
}
}
}
}
}
fclose($handle);
}
}
else
{
ums_log_to_file('Only URLs from https://novlove.com/novel/ are allowed, because sitemap file failed to be opened, skipping: ' . $mngn);
continue;
}
}
}
if(count($manga_arr) == 0)
{
ums_log_to_file('No Web Novel matched your query: ' . $manga_name);
return 'nochange';
}
$scraped_manga = 0;
foreach($manga_arr as $current_manga)
{
if($get_max_manga <= $scraped_manga)
{
break;
}
$current_manga = trim($current_manga, '/');
$current_manga = $current_manga . '/';
$html_site = ums_get_web_page($current_manga, ums_get_random_user_agent(), $use_phantom, $phantom_wait);
if($html_site == false)
{
ums_log_to_file('Failed to download manga chapter: ' . $current_manga);
continue;
}
if(stristr($html_site, 'Cloudflare Ray ID:') !== false)
{
ums_log_to_file('CloudFlare protection active on novlove.com, access is limited! Please install puppeteer on your site and set the "Content Scraping Method To Use" to Puppeteer for scraping to work. Manga URL: ' . $current_manga);
sleep(1);
continue;
}
if(stristr($html_site, 'manga-title-badges custom comic') !== false)
{
ums_log_to_file('This URL is a web comic, not a web novel, skipping it: ' . $current_manga);
continue;
}
if (isset($ums_Main_Settings['request_timeout']) && $ums_Main_Settings['request_timeout'] != '') {
$timeout = intval($ums_Main_Settings['request_timeout']);
} else {
$timeout = 1;
}
sleep($timeout);
require_once (dirname(__FILE__) . "/res/simple_html_dom.php");
$html = ums_str_get_html( $html_site );
$tag = $html->find( '.title', 0 );
$name_str = '';
$my_slug = '';
$my_slug = str_replace( 'http://', '', $current_manga );
$my_slug = str_replace( 'https://', '', $my_slug );
$my_slug = str_replace( 'http:', '', $my_slug );
$my_slug = str_replace( 'novlove.com/novel/', '', $my_slug );
$my_slug = str_replace( '/', '', $my_slug );
$my_slug = str_replace( '.html', '', $my_slug );
$my_slug = str_replace( '.htm', '', $my_slug );
if($tag){
$name_str = trim($tag->plaintext);
$span = $tag->find('span', 0);
if($span){
$name_str = trim(str_replace($span->plaintext,'', $name_str));
}
}
if($name_str == '')
{
preg_match_all('#<title>([^<]*?) - novlove<\/title>#i', $html_site, $tmathc);
if(isset($tmathc[1][0]))
{
$name_str = $tmathc[1][0];
}
}
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Processing novel: ' . $name_str);
}
$find_post = new WP_Query( array(
'title' => $name_str,
'post_type' => 'wp-manga',
) );
$existing_post_id = false;
if( $find_post->have_posts() )
{
$existing_post_id = $find_post->posts[0]->ID;
}
if($existing_post_id == false)
{
$args = array(
'post_type' => 'wp-manga',
'meta_key' => '_manga_import_slug',
'meta_value' => $my_slug,
'post_status' => array('publish','draft','pending','trash','private','future')
);
$query = new WP_Query( $args );
if( $query->have_posts() ){
$existing_post_id = $query->posts[0]->ID;
}
}
if($existing_post_id == false)
{
$block_warning = $html->find('#page .warning', 0);
if( !empty( $block_warning ) && strpos( $block_warning->plaintext, 'has been licensed, it is not available' ) !== false ){
ums_log_to_file('Request blocked, please use a proxy!');
continue;
}
$desc = $html->find( '.desc-text' );
$xd = '';
if(is_array($desc))
{
foreach($desc as $dd)
{
$xd = trim($dd->plaintext);
$xprefix = 'Description';
if (substr($xd, 0, strlen($xprefix)) == $xprefix) {
$xd = substr($xd, strlen($xprefix));
}
$xprefix = 'Summary';
if (substr($xd, 0, strlen($xprefix)) == $xprefix) {
$xd = substr($xd, strlen($xprefix));
}
$xd = trim($xd);
}
}
if($xd == '')
{
$doc = new DOMDocument;
$internalErrors = libxml_use_internal_errors(true);
$doc->loadHTML('<?xml encoding="utf-8" ?>' . $html_site);
libxml_use_internal_errors($internalErrors);
$xpath = new \DOMXpath($doc);
$articles = $xpath->query('//*[@class="g_txt_over mb48 fs16 j_synopsis _txtover _on"]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container) {
if(method_exists($container, 'saveHTML'))
{
$xd .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$xd .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$xd .= ' ' . $container->nodeValue;
}
}
$qtags = array('iframe', 'script');
foreach ($qtags as $qtag)
{
$regex = '#<\s*' . $qtag . '[^>]*>.*?<\s*/\s*'. $qtag . '>#msi';
$xd = preg_replace($regex, '', $xd);
}
}
else
{
$articles = $xpath->query('//*[@class="c_000"]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container) {
if(method_exists($container, 'saveHTML'))
{
$xd .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$xd .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$xd .= ' ' . $container->nodeValue;
}
}
}
}
}
if($xd == '')
{
$desc = '';
}
else
{
$desc = trim($xd);
}
if($desc == '')
{
preg_match_all('#<div class="summary__content show-more">(?:<p>)?([\s\S]*?)(?:<\/p>)?<\/?div>#i', $html_site, $dmathc);
if(isset($dmathc[1][0]))
{
$desc = $dmathc[1][0];
}
}
$desc = ums_strip_links($desc);
$uxrlparts = wp_parse_url(home_url());
$domainx = $uxrlparts['host'];
$desc = preg_replace('#<p><center><a href="https:\/\/bronovel\.com\/"><em><strong>Announcement<\/strong><\/em>: we are moving[\s\S]*?<\/center><\/p>#', '', $desc);
$desc = str_ireplace('novlove.com', $domainx, $desc);
$desc = str_ireplace('novlove', get_bloginfo('name'), $desc);
$desc = str_ireplace('listnovel.com', $domainx, $desc);
$desc = str_ireplace('bronovel.com', $domainx, $desc);
$desc = str_ireplace('bronovel', get_bloginfo('name'), $desc);
$thumb = '';
if($thumb == '')
{
preg_match_all('#<img class="lazy"(?:\s*title="[^"]*?"\s*)? (?:data-)?src="([^"]*?)"#is', $html_site, $xmathc);
if(isset($xmathc[1][0]))
{
$thumb = trim($xmathc[1][0]);
}
}
if($thumb == '')
{
preg_match_all('#<meta property="og:image" content="([^"]*?)"#is', $html_site, $xmathc);
if(isset($xmathc[1][0]))
{
$thumb = trim($xmathc[1][0]);
}
}
if(empty($thumb))
{
$thumb = $html->find( '.lazy' );
$thumb = !empty( $thumb ) ? $thumb[0]->{'data-src'} : '';
if($thumb == '')
{
$thumb = !empty( $thumb ) ? $thumb[0]->src : '';
}
if($thumb == '')
{
$thumb = $html->find( '.summary_image img' );
$thumb = !empty( $thumb ) ? $thumb[0]->src : '';
}
}
if($thumb == '')
{
preg_match_all('#<img class="lazy" data-src="([^"]*?)"#is', $html_site, $xmathc);
if(isset($xmathc[1][0]))
{
$thumb = trim($xmathc[1][0]);
}
}
$stx = '';
preg_match_all('#<h3>Status:<\/h3>[\s\n]*?<a href="[^"]*?" class="text-primary">([^<]*?)<\/a>#is', $html_site, $xmathc);
if(isset($xmathc[1][0]))
{
$status = $xmathc[1][0];
}
else
{
$status = $html->find( '.text-primary' );
if(is_array($status))
{
foreach($status as $st)
{
$stx = $st->plaintext;
}
}
if($stx === '')
{
$status = '';
}
else
{
$status = $stx;
}
}
if(stristr($status, 'ongoing') !== false)
{
$status = 'on-going';
}
elseif(stristr($status, 'completed') !== false)
{
$status = 'end';
}
elseif(stristr($status, 'canceled') !== false)
{
$status = 'canceled';
}
elseif(stristr($status, 'on hold') !== false)
{
$status = 'on-hold';
}
elseif(stristr($status, 'upcoming') !== false)
{
$status = 'upcoming';
}
preg_match_all('#<h3>Alternative names:\s?<\/h3>[\s\n]*?([^<]*?)[\s\n]*?<\/li>#is', $html_site, $xmathc);
if(isset($xmathc[1][0]))
{
$alter_name = $xmathc[1][0];
}
else
{
$alter_name = $html->find( '.summary_content .post-content .post-content_item' );
$xplain = '';
if( !empty( $alter_name ) )
{
foreach($alter_name as $an)
{
if(stristr($an->plaintext, 'Alternative '))
{
$xplain = trim($an->plaintext);
$xprefix = 'Alternative';
if (substr($xplain, 0, strlen($xprefix)) == $xprefix) {
$xplain = substr($xplain, strlen($xprefix));
}
$xplain = trim($xplain);
}
}
}
if($xplain != '')
{
$alter_name = $xplain;
}
else
{
$alter_name = '';
}
}
$data = $html->find( '.summary_content .post-content .post-content_item' );
$xtype = '';
if( !empty( $data ) ){
foreach($data as $item){
if(trim($item->find('.summary-heading h5',0)->plaintext) == 'Type'){
$xtype = $item->find('.summary-content', 0)->plaintext;
if(trim($xtype) == 'Updating')
{
$xtype = '';
}
}
}
}
$xrelease = '2021';
preg_match_all('#href="https:\/\/novlove\.com\/year\/(\d+)"#i', $html_site, $xmathc);
if(isset($xmathc[1][0]))
{
$xrelease = trim($xmathc[1][0]);
}
$xauthor = '';
$author = '';
preg_match_all('#<h3>Author:<\/h3>[\n\s]*<a[\n\s]*href="[^"]*?">([^<]*?)<\/a>#i', $html_site, $xmathc);
if(isset($xmathc[1][0]))
{
$author = trim($xmathc[1][0]);
$xauthor = trim($xmathc[1][0]);
}
if(empty($author))
{
$data = $html->find( '.author-content a' );
if( !empty( $data ) ){
$authors = array();
foreach($data as $arxthor)
{
array_push($authors, $arxthor->plaintext);
if($author == '')
{
$author = $arxthor->plaintext;
}
}
$xauthor = implode(',', $authors);
}
}
$xartists = '';
$data = $html->find( '.artist-content a' );
if( !empty( $data ) ){
$artists = array();
foreach($data as $artist){
array_push($artists, $artist->plaintext);
}
$xartists = implode(',', $artists);
}
$xgenres = '';
preg_match_all('#<li>[\s\n]*?<h3>Genre:<\/h3>[\s\n]*?([\s\S]*?)[\s\n]*?<\/li>#i', $html_site, $xmathc);
if(isset($xmathc[1][0]))
{
preg_match_all('#>([^<]*?)</a>#i', $xmathc[1][0], $xamathc);
if(isset($xamathc[1][0]))
{
$xgenres = implode(',', $xamathc[1]);
}
}
if(empty($xgenres))
{
$data = $html->find( '.genres-content a' );
if( !empty( $data ) ){
$genres = array();
foreach($data as $genre){
array_push($genres, $genre->plaintext);
}
$xgenres = implode(',', $genres);
}
}
$items = $html->find('.summary_content .post-content .post-content_item');
$views = '';
foreach($items as $item){
if(trim($item->find('.summary-heading', 0)->plaintext) == 'Rank'){
$str = $item->find('.summary-content',0)->plaintext;
preg_match_all('~\d+(?:\.\d+)?~', $str, $matches);
if($matches){
if(count($matches) == 3){
$views = $matches[0][1];
} else {
$views = $matches[0][0];
}
if(strpos($str, 'K monthly views') !== false){
$views = floatval($views) * 1000;
}
if(strpos($str, 'M monthly views') !== false){
$views = floatval($views) * 1000000;
}
break;
}
}
}
if(empty(trim($views)) || !is_numeric(strip_tags( $views )))
{
$viewsm = rand(100,500);
}
else
{
$viewsm = strip_tags(trim($views));
}
$average_vote = $html->find('.post-total-rating .total_votes', 0);
$str = $html->find('.vote-details',0);
$xrating = array();
if(!empty($str))
{
preg_match_all('!\d+!', $str, $matches);
$number_votes = 1;
if($matches && count($matches[0]) == 3)
{
$number_votes = $matches[0][2];
}
if( !empty( $average_vote ) && $average_vote->plaintext > 0 ){
$xrating = array(
'avg' => $average_vote->plaintext,
'numbers' => $number_votes
);
}
}
$xtags = '';
$data = $html->find( '.tags-content a' );
if( !empty( $data ) ){
$tags = array();
foreach($data as $tag){
$ttext = trim($tag->plaintext);
$ttext = str_ireplace('novlove', get_bloginfo('name'), $ttext);
$ttext = str_ireplace('box novel', get_bloginfo('name'), $ttext);
array_push($tags, $ttext);
}
$xtags = implode(',', $tags);
}
$xtime_year = '';
$time_year = strtotime($xrelease);
if($time_year !== false)
{
$time_year = date("Y", $time_year);
if($time_year !== false)
{
$xtime_year = $time_year;
}
}
$post_args = array(
'manga_import_slug' => $my_slug,
'title' => $name_str,
'post_status' => $post_status,
'description' => $desc,
'thumb' => $thumb,
'status' => $status,
'altername' => strip_tags( $alter_name ),
'type' => strip_tags( $xtype ),
'release' => $xtime_year,
'authors' => strip_tags( $xauthor ),
'artists' => strip_tags( $xartists ),
);
$post_args['genres'] = '';
if($auto_categories == 'genre')
{
$post_args['genres'] = strip_tags( $xgenres );
}
if($default_category != '')
{
if($post_args['genres'] != '')
{
$post_args['genres'] .= ',';
}
$genreplus = get_term_by('id', $default_category, 'wp-manga-genre');
if($genreplus !== false)
{
$post_args['genres'] .= $genreplus->slug;
}
}
$post_args['views'] = strip_tags( $viewsm );
$post_args['ratings'] = $xrating;
if($can_create_tag == 'genre')
{
$post_args['tags'] = strip_tags( $xgenres );
}
elseif($can_create_tag == 'tags')
{
$post_args['tags'] = strip_tags( $xtags );
}
else
{
$post_args['tags'] = '';
}
if($item_create_tag != '')
{
$ctag = explode(',', $item_create_tag);
foreach($ctag as $ctap)
{
$ctap = trim($ctap);
if($post_args['tags'] != '')
{
$post_args['tags'] .= ',';
}
$genreplus = get_term_by('name', $ctap, 'wp-manga-tag');
if($genreplus !== false)
{
$post_args['tags'] .= $genreplus->slug;
}
else
{
$cat = wp_insert_term($ctap, 'wp-manga-tag');
if(!is_wp_error( $cat ))
{
$post_args['tags'] .= $cat['slug'];
}
}
}
}
$post_args['description'] = str_ireplace('novlove.com', $_SERVER['HTTP_HOST'], $post_args['description']);
$arr = ums_spin_and_translate($post_args['title'], $post_args['description'], $rule_translate, 'en');
if($arr === false)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Skipping web novel (because it failed to be translated): ' . print_r($post_args, true));
}
continue;
}
else
{
if($no_translate_title != '1')
{
$post_args['title'] = $arr[0];
}
$post_args['description'] = $arr[1];
}
$za_xpost_args = array(
'post_title' => !empty( $post_args['title'] ) ? $post_args['title'] : '',
'post_content' => !empty( $post_args['description'] ) ? $post_args['description'] : '',
'post_type' => 'wp-manga',
'post_status' => isset( $post_args['post_status'] ) ? $post_args['post_status'] : 'pending',
);
$existing_again = false;
if($za_xpost_args['post_title'] != '')
{
$xposts = get_posts(
array(
'post_type' => 'wp-manga',
'title' => html_entity_decode($za_xpost_args['post_title']),
'post_status' => 'all',
'numberposts' => 1,
'update_post_term_cache' => false,
'update_post_meta_cache' => false,
'orderby' => 'post_date ID',
'order' => 'ASC',
)
);
if ( ! empty( $xposts ) ) {
$ex_page = $xposts[0];
} else {
$ex_page = null;
}
if(isset($ex_page->ID))
{
$existing_again = true;
$existing_post_id = $ex_page->ID;
}
}
if($existing_again == false)
{
$accept_comments = 'closed';
if ($enable_comments == '1') {
$accept_comments = 'open';
}
$za_xpost_args['comment_status'] = $accept_comments;
if ($enable_pingback == '1')
{
$za_xpost_args['ping_status'] = 'open';
}
else
{
$za_xpost_args['ping_status'] = 'closed';
}
if($get_date == '1')
{
if(!empty($xrelease))
{
if(strlen($xrelease) == 4)
{
$xrelease .= '-01-01 00:00';
}
$postdatex = gmdate("Y-m-d H:i:s", strtotime($xrelease));
$za_xpost_args['post_date_gmt'] = $postdatex;
}
}
if($user_name_type == 'rand')
{
$randid = ums_display_random_user();
if($randid === false)
{
$za_xpost_args['post_author'] = ums_randomName();
}
else
{
$za_xpost_args['post_author'] = $randid->ID;
}
}
elseif($user_name_type == 'feed-news')
{
$sp_post_user_name = ums_randomName();
if($author == '' || $author == '1' || $author == 'null')
{
$author = ums_randomName();
}
if($author != '')
{
$xauthor = sanitize_user( $author );
$xauthor = apply_filters( 'pre_user_login', $xauthor );
$xauthor = trim( $xauthor );
if(username_exists( $xauthor ))
{
$user_id_t = get_user_by('login', $xauthor);
if($user_id_t)
{
$sp_post_user_name = $user_id_t->ID;
}
}
else
{
$palphabet = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890!@#$%^*()-+=_?><,.;:}{][';
$ppass = '';
$alphaLength = strlen($palphabet) - 1;
for ($ipass = 0; $ipass < 8; $ipass++)
{
$npass = rand(0, $alphaLength);
$ppass .= $palphabet[$npass];
}
$curr_id = wp_create_user($xauthor, $ppass, ums_generate_random_email());
if ( is_int($curr_id) )
{
$u = new WP_User($curr_id);
$u->remove_role('subscriber');
$u->add_role('author');
$sp_post_user_name = $curr_id;
}
}
}
$za_xpost_args['post_author'] = ums_utf8_encode($sp_post_user_name);
}
else
{
$za_xpost_args['post_author'] = ums_utf8_encode($post_user_name);
}
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Inserting new novel: ' . $za_xpost_args['post_title']);
}
$existing_post_id = wp_insert_post( $za_xpost_args );
if( ! $existing_post_id || is_wp_error( $existing_post_id ) ){
ums_log_to_file('Failed to insert novel into db: ' . $post_args['title']);
continue;
}
add_post_meta($existing_post_id, 'ums_parent_rule', $type . '-' . $param);
wp_set_object_terms( $existing_post_id, 'Ums_' . $type . '_' . $param, 'coderevolution_post_source', true);
if($thumb != '')
{
$thumb_id = ums_wp_mcl_e_upload_file( $post_args['thumb'], $use_phantom, $phantom_wait, $existing_post_id );
if($thumb_id === false)
{
include_once( ABSPATH . 'wp-admin/includes/image.php' );
$thcontent = ums_get_web_page($thumb);
if (isset($ums_Main_Settings['request_timeout']) && $ums_Main_Settings['request_timeout'] != '') {
$timeout = intval($ums_Main_Settings['request_timeout']);
} else {
$timeout = 1;
}
sleep($timeout);
$pathinfo = pathinfo( $thumb );
if( $thcontent != false ){
$upload_dir = wp_upload_dir();
$file_tmp_path = $upload_dir['basedir'] . '/' . $pathinfo['filename'] . '-' . $existing_post_id . '.' . explode('?',$pathinfo['extension'])[0];
$file = $wp_filesystem->put_contents( $file_tmp_path, $thcontent );
$wp_filetype = wp_check_filetype(basename($file_tmp_path), null );
$attachment = array(
'post_mime_type' => $wp_filetype['type'],
'post_title' => $existing_post_id,
'post_content' => '',
'post_status' => 'inherit'
);
$attach_id = wp_insert_attachment( $attachment, $file_tmp_path );
$thumb_id = $attach_id;
$imagenew = get_post( $attach_id );
$fullsizepath = get_attached_file( $imagenew->ID );
require_once( ABSPATH . 'wp-admin/includes/image.php' );
require_once( ABSPATH . 'wp-admin/includes/media.php' );
$attach_data = wp_generate_attachment_metadata( $attach_id, $fullsizepath );
wp_update_attachment_metadata( $attach_id, $attach_data );
}
}
}
else
{
$thumb_id = false;
}
$meta_data = array(
'_manga_import_slug' => $my_slug,
'_thumbnail_id' => $thumb_id,
'_wp_manga_alternative' => strip_tags( $alter_name ),
'_wp_manga_type' => strip_tags( $xtype ),
'_wp_manga_status' => $status,
'_wp_manga_chapter_type' => 'text',
'_wp_manga_chapters_warning'=> $chapter_warning,
);
foreach( $meta_data as $key => $value ){
if( !empty( $value ) ){
update_post_meta( $existing_post_id, $key, $value );
}
}
$manga_terms = array(
'wp-manga-release' => strip_tags( $xrelease ),
'wp-manga-author' => strip_tags( $xauthor ),
'wp-manga-artist' => strip_tags( $xartists ),
'wp-manga-genre' => isset( $post_args['genres'] ) ? $post_args['genres'] : '',
'wp-manga-tag' => isset( $post_args['tags'] ) ? $post_args['tags'] : null,
);
foreach( $manga_terms as $tax => $term ){
$resp = ums_add_manga_terms( $existing_post_id, $term, $tax );
}
ums_update_post_views( $existing_post_id, strip_tags( $viewsm ) );
ums_update_post_ratings( $existing_post_id, $xrating );
}
}
$chapters = array();
$lazy_loaded = false;
$ums_max_chapters = $max;
if($reverse_chapters == '1')
{
$reverse = true;
}
else
{
$reverse = false;
}
$first_chapter = '';
$latest_scraped = get_post_meta( $existing_post_id, 'ums_latest_scraped', true );
if(!empty($latest_scraped))
{
if(stristr($latest_scraped, 'novlove'))
{
$first_chapter = $latest_scraped;
}
}
if(empty($first_chapter))
{
libxml_use_internal_errors(true);
$dom = new DOMDocument();
$dom->loadHTML($html_site);
$xpath = new DOMXPath($dom);
$links = $xpath->query('//ul[contains(@class, "list-chapter")]//a');
$chapterUrls = [];
foreach ($links as $link) {
$href = $link->getAttribute('href');
if ($href) {
$chapterUrls[] = $href;
}
}
if($reverse == true)
{
$chapterUrls = array_reverse($chapterUrls);
}
if(!empty($chapterUrls))
{
$first_chapter = $chapterUrls[0];
}
}
if(empty($first_chapter))
{
preg_match_all('#<meta property="og:novel:read_url"[\s\n]*?content="([^"]*?)"#i', $html_site, $xmathc);
if(isset($xmathc[1][0]))
{
$first_chapter = $xmathc[1][0];
}
if(empty($first_chapter))
{
preg_match_all('#<a class="btn btn-danger btn-read-now" title="READ NOW"[\s\n]*?href="([^"]*?)">#i', $html_site, $xmathc);
if(isset($xmathc[1][0]))
{
$first_chapter = $xmathc[1][0];
}
}
}
if(empty($first_chapter))
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Skipping web novel (because first chapter URL was not detected): ' . print_r($post_args, true));
}
continue;
}
$local_imported = 0;
$new_chap = false;
{
$chapid = 1;
while(!empty($first_chapter))
{
$randsleep = rand(500,1000);
usleep($randsleep * 1000);
if($ums_max_chapters <= $local_imported)
{
break;
}
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Processing novel chapter: ' . $first_chapter);
}
$page_html = ums_get_web_page($first_chapter, ums_get_random_user_agent(), $use_phantom, $phantom_wait);
if( $page_html == false ){
ums_log_to_file('Failed to download HTML for current chapter: ' . $first_chapter);
break;;
}
if(stristr($page_html, 'Cloudflare Ray ID:') !== false)
{
ums_log_to_file('CloudFlare protection active on chapter, access is limited! Please install puppeteer on your site and set the "Content Scraping Method To Use" to Puppeteer for scraping to work. Chapter URL: ' . $chaps['url']);
sleep(1);
break;
}
$html = ums_str_get_html( $page_html );
if( empty( $html ) ){
ums_log_to_file('Failed to parse HTML for current chapter: ' . $first_chapter);
break;
}
$chapid++;
$ntitle = '';
preg_match_all('#<a class="chr-title" href="[^"]*?" title="([^"]*?)">#i', $page_html, $titli);
if(isset($titli[1][0]))
{
$ntitle = $titli[1][0];
}
else
{
preg_match_all('#const chapterInfo = "([^"]*?)"#i', $page_html, $titli);
if(isset($titli[1][0]))
{
$ntitle = $titli[1][0];
}
}
$slugified_name = $wp_manga_storage->slugify( $ntitle );
$chapter_2 = $wp_manga_chapter->get_chapter_by_slug( $existing_post_id, $slugified_name );
if($chapter_2 && strtolower($chapter_2['chapter_slug']) == strtolower($slugified_name))
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Chapter is already published, skipping it: ' . $chapter_2['chapter_name']);
}
preg_match_all('#title="[^"]*?"[\s\n]*href="([^"]*?)"[\s\n]*?class="btn btn-success"[\s\n]*id="next_chap">#i', $page_html, $titli);
if(isset($titli[1][0]) && !ums_str_ends_with( $titli[1][0], 'null' ))
{
$first_chapter = $titli[1][0];
update_post_meta($existing_post_id, 'ums_latest_scraped', $first_chapter);
}
else
{
$first_chapter = '';
}
continue;
}
$content = '';
$doc = new DOMDocument;
$internalErrors = libxml_use_internal_errors(true);
$doc->loadHTML('<?xml encoding="utf-8" ?>' . $page_html);
libxml_use_internal_errors($internalErrors);
$xpath = new \DOMXpath($doc);
$articles = $xpath->query('//*[contains(@class, "chr-c")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
break;
}
}
if(empty($content))
{
$articles = $xpath->query('//*[contains(@class, "content-body")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
}
}
}
if(empty($content))
{
$articles = $xpath->query('//*[contains(@class, "cha-tit")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
}
}
$articles = $xpath->query('//*[contains(@class, "cha-content")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
}
}
}
if(empty($content))
{
$ccontent = $html->find('.reading-content .text-left .cha-words .cha-words');
if($ccontent)
{
$content = '';
foreach($ccontent as $xct)
{
$other_ps = $xct->find('p');
if(count($other_ps) > 0)
{
$text = '';
foreach($other_ps as $p)
{
$text .= '<p> ' . $p->plaintext . '</p>';
}
$content .= $text . '<hr/>';
}
else
{
$content .= $xct->innertext . '<hr/>';
}
}
}
else
{
if($html->find('.reading-content #content', 0))
{
$content = $html->find('.reading-content #content', 0)->innertext;
}
else
{
$ccontent = $html->find('.reading-content .text-left .cha-content .cha-words');
if($ccontent)
{
$content = '';
foreach($ccontent as $xct)
{
$content .= $xct->innertext . '<hr/>';
}
}
else
{
$content = $html->find('.reading-content .text-left', 0);
if($content)
{
$other_ps = $content->find('p');
if(count($other_ps) > 0)
{
$text = '';
foreach($other_ps as $p)
{
$text .= '<p> ' . $p->plaintext . '</p>';
}
$content = $text;
}
else
{
if($inner_content = $content->find('#chapter-content', 0))
{
$content = $inner_content->innertext;
}
else
{
if($inner_content = $content->find('.text-left', 0))
{
$content = $inner_content->innertext;
}
else
{
if($inner_content = $content->find('.fr-view', 0))
{
$content = $inner_content->innertext;
}
else
{
$content = $content->innertext;
}
}
}
}
}
}
}
if(!$content)
{
$content = '';
$ps = $html->find('.reading-content .text-left p');
foreach($ps as $p)
{
$content .= '<p>' . $p->plaintext . '</p>';
}
}
}
}
$content = preg_replace('#(?:my)?[.]?b[.\s]?o[.\s]?x[.\s]?n[.\s]?o[.\s]?v[.\s]?e[.\s]?l[.]?(?:\s?.c[o0]m)?#i', get_bloginfo('name'), $content);
$content = preg_replace('#<p class="text-m">[\s\S]*?<strong>\$1<\/strong>![\s\S]*?<\/p>#i', '', $content);
$content = preg_replace('#<i class="glyphicon glyphicon-remove"><\/i> Remove Ads From \$1#i', '', $content);
$content = preg_replace('#<button[\S\s]*?<\/button>#i', '', $content);
$content = str_ireplace('novlove.com', get_bloginfo('name'), $content);
$content = str_ireplace('nov love', get_bloginfo('name'), $content);
$content = str_ireplace('novlove', get_bloginfo('name'), $content);
$content = str_ireplace('Ads by PubFuture', '', $content);
$qtags = array('iframe', 'script', 'ins');
foreach ($qtags as $qtag) {
$regex = '#<\s*' . $qtag . '[^>]*>.*?<\s*/\s*'. $qtag . '>#msi';
$content = preg_replace($regex, '', $content);
}
$content = preg_replace('#<style([\s\S]+?)<\/style>#i', '', $content);
$content = preg_replace('#<\/?div([^>]*)>#i', '', $content);
$content = preg_replace("/\\\\u([0-9a-f]{3,4})/i", "&#x\\1;", $content);
if($strip_images == '1')
{
$content = ums_sanitize_html_content($content);
}
$chapter_content = trim($content);
$chapter_content = ums_strip_links($chapter_content);
if($chapter_content == '')
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Skipping current chapter (because it failed to parse first chapter content): ' . print_r($first_chapter, true));
}
//new_chap true to prevent infinite running
$new_chap = true;
break;
}
$arr = ums_spin_and_translate($ntitle, $chapter_content, $rule_translate, 'en');
if($arr === false)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Skipping current chapter (because it failed to be translated): ' . print_r($first_chapter, true));
}
}
elseif($no_translate_title != '1' && !empty($arr[0]) && !empty($arr[1]))
{
$ntitle = $arr[0];
$chapter_content = $arr[1];
}
global $wp_manga_text_type;
if(empty($ntitle))
{
$ntitle = 'Chapter';
}
if($ntitle == '')
{
$xext_name = $name_str;
}
else
{
$xext_name = '';
}
$chapter_args = array(
'post_id' => $existing_post_id,
'chapter_name' => $ntitle,
'chapter_name_extend' => $xext_name,
'volume_id' => '',
'chapter_content' => $chapter_content,
);
$slugified_name = $wp_manga_storage->slugify( $ntitle );
$chapter_2 = $wp_manga_chapter->get_chapter_by_slug( $existing_post_id, $slugified_name );
if($chapter_2 && strtolower($chapter_2['chapter_slug']) == strtolower($slugified_name))
{
ums_log_to_file('Chapter name already published, we will skip: ' . $chapter_2['chapter_name']);
preg_match_all('#title="[^"]*?"[\s\n]*href="([^"]*?)"[\s\n]*?class="btn btn-success"[\s\n]*id="next_chap">#i', $page_html, $titli);
if(isset($titli[1][0]) && !ums_str_ends_with( $titli[1][0], 'null' ))
{
$first_chapter = $titli[1][0];
update_post_meta($existing_post_id, 'ums_latest_scraped', $first_chapter);
}
else
{
$first_chapter = '';
}
continue;
}
else
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Now publishing: ' . print_r($ntitle, true));
}
}
$chapter_id = $wp_manga_text_type->insert_chapter( $chapter_args );
if( $chapter_id ){
if( is_wp_error( $chapter_id ) ){
ums_log_to_file('Failed to insert new chapter: ' . $ntitle . ' error: ' . $chapter_id->get_error_message());
break;
}
else
{
$new_chap = true;
$ums_imported_chapters++;
$local_imported++;
}
}
preg_match_all('#title="[^"]*?"[\s\n]*href="([^"]*?)"[\s\n]*?class="btn btn-success"[\s\n]*id="next_chap">#i', $page_html, $titli);
if(isset($titli[1][0]) && !ums_str_ends_with( $titli[1][0], 'null' ))
{
$first_chapter = $titli[1][0];
update_post_meta($existing_post_id, 'ums_latest_scraped', $first_chapter);
}
else
{
$first_chapter = '';
}
if (isset($ums_Main_Settings['request_timeout']) && $ums_Main_Settings['request_timeout'] != '') {
$timeout = intval($ums_Main_Settings['request_timeout']);
} else {
$timeout = 1;
}
sleep($timeout);
}
}
if($new_chap == true)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Total novel chapters scraped: ' . $local_imported);
}
$scraped_manga++;
}
}
}
catch(Exception $e)
{
ums_log_to_file('Importing failed: ' . $e->getMessage());
if($auto == 1)
{
ums_clearFromList($param, $type);
}
return 'fail';
}
}
}
elseif($type == 3)
{
if(trim($max_manga) != '')
{
$get_max_manga = intval(trim($max_manga));
}
else
{
$get_max_manga = 999;
}
$items = array();
$page_increased = false;
{
$manga_arr = array();
try
{
global $wp_filesystem;
if ( ! is_a( $wp_filesystem, 'WP_Filesystem_Base') ){
include_once(ABSPATH . 'wp-admin/includes/file.php');$creds = request_filesystem_credentials( site_url() );
wp_filesystem($creds);
}
$prefixh = 'http://newnovel.org/novel/';
$wprefixh = 'http://www.newnovel.org/novel/';
$prefixs = 'https://newnovel.org/novel/';
$wprefixs = 'https://www.newnovel.org/novel/';
$prefixh2 = 'http://vipnovel.com/vipnovel/';
$wprefixh2 = 'http://www.vipnovel.com/vipnovel/';
$prefixs2 = 'https://vipnovel.com/vipnovel/';
$wprefixs2 = 'https://www.vipnovel.com/vipnovel/';
$manga_names = explode(',', $manga_name);
$manga_names = array_map('trim', $manga_names);
foreach($manga_names as $mngn)
{
if (substr($mngn, 0, strlen($prefixs)) == $prefixs) {
$manga_arr[] = $mngn;
}
elseif (substr($mngn, 0, strlen($prefixh)) == $prefixh) {
$manga_arr[] = $mngn;
}
elseif (substr($mngn, 0, strlen($wprefixh)) == $wprefixh) {
$manga_arr[] = $mngn;
}
elseif (substr($mngn, 0, strlen($wprefixs)) == $wprefixs) {
$manga_arr[] = $mngn;
}
elseif (substr($mngn, 0, strlen($prefixs2)) == $prefixs2) {
$mngn = str_replace('vipnovel.com/vipnovel', 'newnovel.org/novel', $mngn);
$manga_arr[] = $mngn;
}
elseif (substr($mngn, 0, strlen($prefixh2)) == $prefixh2) {
$mngn = str_replace('vipnovel.com/vipnovel', 'newnovel.org/novel', $mngn);
$manga_arr[] = $mngn;
}
elseif (substr($mngn, 0, strlen($wprefixh2)) == $wprefixh2) {
$mngn = str_replace('vipnovel.com/vipnovel', 'newnovel.org/novel', $mngn);
$manga_arr[] = $mngn;
}
elseif (substr($mngn, 0, strlen($wprefixs2)) == $wprefixs2) {
$mngn = str_replace('vipnovel.com/vipnovel', 'newnovel.org/novel', $mngn);
$manga_arr[] = $mngn;
}
else
{
if($wp_filesystem->exists(dirname(__FILE__) . "/sitemap_vip.txt"))
{
$handle = fopen(dirname(__FILE__) . "/sitemap_vip.txt", "r");
if ($handle)
{
while (!feof($handle))
{
$buffer = fgets($handle);
if($buffer !== false)
{
if($mngn == '*')
{
$manga_arr[] = 'https://newnovel.org/novel/' . trim($buffer) . '/';
}
else
{
$xbuffer = str_replace('-', ' ', $buffer);
$mngnwords = explode(' ', $mngn);
$xbuffer_arr = explode(' ', $xbuffer);
foreach($mngnwords as $mangx)
{
$mangx = trim($mangx);
$xfound = false;
foreach($xbuffer_arr as $xbf)
{
$xbf = trim($xbf);
if($xbf == $mangx)
{
$xfound = true;
}
}
if($xfound == true)
{
$manga_arr[] = 'https://newnovel.org/novel/' . trim($buffer) . '/';
break;
}
}
}
}
}
fclose($handle);
}
}
else
{
ums_log_to_file('Only URLs from https://newnovel.org/novel/ are allowed, because sitemap file failed to be opened, skipping: ' . $mngn);
continue;
}
}
}
if(count($manga_arr) == 0)
{
ums_log_to_file('No Web Novel matched your query: ' . $manga_name);
return 'nochange';
}
$scraped_manga = 0;
foreach($manga_arr as $current_manga)
{
if($get_max_manga <= $scraped_manga)
{
break;
}
$current_manga = trim($current_manga, '/');
$current_manga = $current_manga . '/';
$html_site = ums_get_web_page($current_manga, ums_get_random_user_agent(), $use_phantom, $phantom_wait);
if($html_site == false)
{
ums_log_to_file('Failed to download manga chapter: ' . $current_manga);
continue;
}
if(stristr($html_site, 'Cloudflare Ray ID:') !== false)
{
ums_log_to_file('CloudFlare protection active on newnovel.org, access is limited! Please install puppeteer on your site and set the "Content Scraping Method To Use" to Puppeteer for scraping to work. Manga URL: ' . $current_manga);
sleep(1);
continue;
}
if(stristr($html_site, 'manga-title-badges custom comic') !== false)
{
ums_log_to_file('This URL is a web comic, not a web novel, skipping it: ' . $current_manga);
continue;
}
if (isset($ums_Main_Settings['request_timeout']) && $ums_Main_Settings['request_timeout'] != '') {
$timeout = intval($ums_Main_Settings['request_timeout']);
} else {
$timeout = 1;
}
sleep($timeout);
require_once (dirname(__FILE__) . "/res/simple_html_dom.php");
$html = ums_str_get_html( $html_site );
$tag = $html->find( '.profile-manga .post-title h1', 0 );
$name_str = '';
$my_slug = '';
$my_slug = str_replace( 'http://', '', $current_manga );
$my_slug = str_replace( 'https://', '', $my_slug );
$my_slug = str_replace( 'http:', '', $my_slug );
$my_slug = str_replace( 'newnovel.org/novel/', '', $my_slug );
$my_slug = str_replace( '/', '', $my_slug );
$my_slug = str_replace( '.html', '', $my_slug );
$my_slug = str_replace( '.htm', '', $my_slug );
if($tag){
$name_str = trim($tag->plaintext);
$span = $tag->find('span', 0);
if($span){
$name_str = trim(str_replace($span->plaintext,'', $name_str));
}
}
if($name_str == '')
{
preg_match_all('#<title>([^<]*?) - NewNovel<\/title>#i', $html_site, $tmathc);
if(isset($tmathc[1][0]))
{
$name_str = $tmathc[1][0];
}
}
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Processing novel: ' . $name_str);
}
$find_post = new WP_Query( array(
'title' => $name_str,
'post_type' => 'wp-manga',
) );
$existing_post_id = false;
if( $find_post->have_posts() )
{
$existing_post_id = $find_post->posts[0]->ID;
}
if($existing_post_id == false)
{
$args = array(
'post_type' => 'wp-manga',
'meta_key' => '_manga_import_slug',
'meta_value' => $my_slug,
'post_status' => array('publish','draft','pending','trash','private','future')
);
$query = new WP_Query( $args );
if( $query->have_posts() ){
$existing_post_id = $query->posts[0]->ID;
}
}
if($existing_post_id == false)
{
$block_warning = $html->find('#page .warning', 0);
if( !empty( $block_warning ) && strpos( $block_warning->plaintext, 'has been licensed, it is not available' ) !== false ){
ums_log_to_file('Request blocked, please use a proxy!');
continue;
}
$desc = $html->find( '.description-summary' );
$xd = '';
if(is_array($desc))
{
foreach($desc as $dd)
{
$xd = trim($dd->plaintext);
$xprefix = 'Description';
if (substr($xd, 0, strlen($xprefix)) == $xprefix) {
$xd = substr($xd, strlen($xprefix));
}
$xd = trim($xd);
}
}
if($xd == '')
{
$desc = '';
}
else
{
$desc = $xd;
}
$desc = preg_replace('#(?:my)?[.]?v[.\s]?i[.\s]?p[.\s]?n[.\s]?o[.\s]?v[.\s]?e[.\s]?l[.]?(?:\s?.c[o0]m)?#i', get_bloginfo('name'), $desc);
$desc = str_ireplace('newnovel.org', get_bloginfo('name'),$desc);
$desc = str_ireplace('newnovel', get_bloginfo('name'), $desc);
$thumb = $html->find( '.summary_image .img-responsive' );
$thumb = !empty( $thumb ) ? $thumb[0]->src : '';
if($thumb == '')
{
$thumb = $html->find( '.summary_image' );
$thumb = !empty( $thumb ) ? $thumb[0]->src : '';
}
$stx = '';
$status = $html->find( '.post-status .post-content_item' );
if(is_array($status))
{
foreach($status as $st)
{
$temp_stat = $st->plaintext;
if(stristr($temp_stat, 'status') !== false)
{
$stx = $temp_stat;
}
}
}
if($stx === '')
{
$status = '';
}
else
{
$status = $stx;
}
if(stristr($status, 'ongoing') !== false)
{
$status = 'on-going';
}
elseif(stristr($status, 'completed') !== false)
{
$status = 'end';
}
elseif(stristr($status, 'canceled') !== false)
{
$status = 'canceled';
}
elseif(stristr($status, 'on hold') !== false)
{
$status = 'on-hold';
}
elseif(stristr($status, 'upcoming') !== false)
{
$status = 'upcoming';
}
$alter_name = $html->find( '.summary_content .post-content .post-content_item' );
$xplain = '';
if( !empty( $alter_name ) )
{
foreach($alter_name as $an)
{
if(stristr($an->plaintext, 'Alternative '))
{
$xplain = trim($an->plaintext);
$xprefix = 'Alternative';
if (substr($xplain, 0, strlen($xprefix)) == $xprefix) {
$xplain = substr($xplain, strlen($xprefix));
}
$xplain = trim($xplain);
}
}
}
if($xplain != '')
{
$alter_name = $xplain;
}
else
{
$alter_name = '';
}
$data = $html->find( '.summary_content .post-content .post-content_item' );
$xtype = '';
if( !empty( $data ) ){
foreach($data as $item){
if(trim($item->find('.summary-heading h5',0)->plaintext) == 'Type'){
$xtype = $item->find('.summary-content', 0)->plaintext;
if(trim($xtype) == 'Updating')
{
$xtype = '';
}
}
}
}
$xrelease = '2021';
preg_match_all('#<a href="https:\/\/newnovel\.org\/(?:manga-)?release\/(?:[^"]*?)" rel="tag">([^<]+?)<\/a>#i', $html_site, $xmathc);
if(isset($xmathc[1][0]))
{
$xrelease = trim($xmathc[1][0]);
}
$xauthor = '';
$author = '';
$data = $html->find( '.author-content a' );
if( !empty( $data ) ){
$authors = array();
foreach($data as $arxthor)
{
array_push($authors, $arxthor->plaintext);
if($author == '')
{
$author = $arxthor->plaintext;
}
}
$xauthor = implode(',', $authors);
}
$xartists = '';
$data = $html->find( '.artist-content a' );
if( !empty( $data ) ){
$artists = array();
foreach($data as $artist){
array_push($artists, $artist->plaintext);
}
$xartists = implode(',', $artists);
}
$xgenres = '';
$data = $html->find( '.genres-content a' );
if( !empty( $data ) ){
$genres = array();
foreach($data as $genre){
array_push($genres, $genre->plaintext);
}
$xgenres = implode(',', $genres);
}
$items = $html->find('.summary_content .post-content .post-content_item');
$views = '';
foreach($items as $item){
if(trim($item->find('.summary-heading', 0)->plaintext) == 'Rank'){
$str = $item->find('.summary-content',0)->plaintext;
preg_match_all('~\d+(?:\.\d+)?~', $str, $matches);
if($matches){
if(count($matches) == 3){
$views = $matches[0][1];
} else {
$views = $matches[0][0];
}
if(strpos($str, 'K monthly views') !== false){
$views = floatval($views) * 1000;
}
if(strpos($str, 'M monthly views') !== false){
$views = floatval($views) * 1000000;
}
break;
}
}
}
if(empty(trim($views)) || !is_numeric(strip_tags( $views )))
{
$viewsm = rand(100,500);
}
else
{
$viewsm = strip_tags(trim($views));
}
$average_vote = $html->find('.post-total-rating .total_votes', 0);
$str = $html->find('.vote-details',0);
$xrating = array();
if(!empty($str))
{
preg_match_all('!\d+!', $str, $matches);
$number_votes = 1;
if($matches && count($matches[0]) == 3)
{
$number_votes = $matches[0][2];
}
if( !empty( $average_vote ) && $average_vote->plaintext > 0 ){
$xrating = array(
'avg' => $average_vote->plaintext,
'numbers' => $number_votes
);
}
}
$xtags = '';
$data = $html->find( '.tags-content a' );
if( !empty( $data ) ){
$tags = array();
foreach($data as $tag){
$ttext = trim($tag->plaintext);
$ttext = preg_replace('#(?:my)?[.]?v[.\s]?i[.\s]?p[.\s]?n[.\s]?o[.\s]?v[.\s]?e[.\s]?l[.]?(?:\s?.c[o0]m)?#i', get_bloginfo('name'), $ttext);
$ttext = str_ireplace('newnovel', get_bloginfo('name'), $ttext);
$ttext = str_ireplace('new novel', get_bloginfo('name'), $ttext);
array_push($tags, $ttext);
}
$xtags = implode(',', $tags);
}
$xtime_year = '';
$time_year = strtotime($xrelease);
if($time_year !== false)
{
$time_year = date("Y", $time_year);
if($time_year !== false)
{
$xtime_year = $time_year;
}
}
$post_args = array(
'manga_import_slug' => $my_slug,
'title' => $name_str,
'post_status' => $post_status,
'description' => $desc,
'thumb' => $thumb,
'status' => $status,
'altername' => strip_tags( $alter_name ),
'type' => strip_tags( $xtype ),
'release' => $xtime_year,
'authors' => strip_tags( $xauthor ),
'artists' => strip_tags( $xartists ),
);
$post_args['genres'] = '';
if($auto_categories == 'genre')
{
$post_args['genres'] = strip_tags( $xgenres );
}
if($default_category != '')
{
if($post_args['genres'] != '')
{
$post_args['genres'] .= ',';
}
$genreplus = get_term_by('id', $default_category, 'wp-manga-genre');
if($genreplus !== false)
{
$post_args['genres'] .= $genreplus->slug;
}
}
$post_args['views'] = strip_tags( $viewsm );
$post_args['ratings'] = $xrating;
if($can_create_tag == 'genre')
{
$post_args['tags'] = strip_tags( $xgenres );
}
elseif($can_create_tag == 'tags')
{
$post_args['tags'] = strip_tags( $xtags );
}
else
{
$post_args['tags'] = '';
}
if($item_create_tag != '')
{
$ctag = explode(',', $item_create_tag);
foreach($ctag as $ctap)
{
$ctap = trim($ctap);
if($post_args['tags'] != '')
{
$post_args['tags'] .= ',';
}
$genreplus = get_term_by('name', $ctap, 'wp-manga-tag');
if($genreplus !== false)
{
$post_args['tags'] .= $genreplus->slug;
}
else
{
$cat = wp_insert_term($ctap, 'wp-manga-tag');
if(!is_wp_error( $cat ))
{
$post_args['tags'] .= $cat['slug'];
}
}
}
}
$post_args['description'] = str_ireplace('newnovel.org', $_SERVER['HTTP_HOST'], $post_args['description']);
$arr = ums_spin_and_translate($post_args['title'], $post_args['description'], $rule_translate, 'en');
if($arr === false)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Skipping web novel (because it failed to be translated): ' . print_r($post_args, true));
}
continue;
}
else
{
if($no_translate_title != '1')
{
$post_args['title'] = $arr[0];
}
$post_args['description'] = $arr[1];
}
$za_xpost_args = array(
'post_title' => !empty( $post_args['title'] ) ? $post_args['title'] : '',
'post_content' => !empty( $post_args['description'] ) ? $post_args['description'] : '',
'post_type' => 'wp-manga',
'post_status' => isset( $post_args['post_status'] ) ? $post_args['post_status'] : 'pending',
);
$existing_again = false;
if($za_xpost_args['post_title'] != '')
{
$xposts = get_posts(
array(
'post_type' => 'wp-manga',
'title' => html_entity_decode($za_xpost_args['post_title']),
'post_status' => 'all',
'numberposts' => 1,
'update_post_term_cache' => false,
'update_post_meta_cache' => false,
'orderby' => 'post_date ID',
'order' => 'ASC',
)
);
if ( ! empty( $xposts ) ) {
$ex_page = $xposts[0];
} else {
$ex_page = null;
}
if(isset($ex_page->ID))
{
$existing_again = true;
$existing_post_id = $ex_page->ID;
}
}
if($existing_again == false)
{
$accept_comments = 'closed';
if ($enable_comments == '1') {
$accept_comments = 'open';
}
$za_xpost_args['comment_status'] = $accept_comments;
if ($enable_pingback == '1')
{
$za_xpost_args['ping_status'] = 'open';
}
else
{
$za_xpost_args['ping_status'] = 'closed';
}
if($get_date == '1')
{
if(!empty($xrelease))
{
if(strlen($xrelease) == 4)
{
$xrelease .= '-01-01 00:00';
}
$postdatex = gmdate("Y-m-d H:i:s", strtotime($xrelease));
$za_xpost_args['post_date_gmt'] = $postdatex;
}
}
if($user_name_type == 'rand')
{
$randid = ums_display_random_user();
if($randid === false)
{
$za_xpost_args['post_author'] = ums_randomName();
}
else
{
$za_xpost_args['post_author'] = $randid->ID;
}
}
elseif($user_name_type == 'feed-news')
{
$sp_post_user_name = ums_randomName();
if($author == '' || $author == '1' || $author == 'null')
{
$author = ums_randomName();
}
if($author != '')
{
$xauthor = sanitize_user( $author );
$xauthor = apply_filters( 'pre_user_login', $xauthor );
$xauthor = trim( $xauthor );
if(username_exists( $xauthor ))
{
$user_id_t = get_user_by('login', $xauthor);
if($user_id_t)
{
$sp_post_user_name = $user_id_t->ID;
}
}
else
{
$palphabet = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890!@#$%^*()-+=_?><,.;:}{][';
$ppass = '';
$alphaLength = strlen($palphabet) - 1;
for ($ipass = 0; $ipass < 8; $ipass++)
{
$npass = rand(0, $alphaLength);
$ppass .= $palphabet[$npass];
}
$curr_id = wp_create_user($xauthor, $ppass, ums_generate_random_email());
if ( is_int($curr_id) )
{
$u = new WP_User($curr_id);
$u->remove_role('subscriber');
$u->add_role('author');
$sp_post_user_name = $curr_id;
}
}
}
$za_xpost_args['post_author'] = ums_utf8_encode($sp_post_user_name);
}
else
{
$za_xpost_args['post_author'] = ums_utf8_encode($post_user_name);
}
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Inserting new novel: ' . $za_xpost_args['post_title']);
}
$existing_post_id = wp_insert_post( $za_xpost_args );
if( ! $existing_post_id || is_wp_error( $existing_post_id ) ){
ums_log_to_file('Failed to insert novel into db: ' . $post_args['title']);
continue;
}
add_post_meta($existing_post_id, 'ums_parent_rule', $type . '-' . $param);
wp_set_object_terms( $existing_post_id, 'Ums_' . $type . '_' . $param, 'coderevolution_post_source', true);
if($thumb != '')
{
$thumb_id = ums_wp_mcl_e_upload_file( $post_args['thumb'], $use_phantom, $phantom_wait, $existing_post_id );
if($thumb_id === false)
{
include_once( ABSPATH . 'wp-admin/includes/image.php' );
$thcontent = ums_get_web_page($thumb);
if (isset($ums_Main_Settings['request_timeout']) && $ums_Main_Settings['request_timeout'] != '') {
$timeout = intval($ums_Main_Settings['request_timeout']);
} else {
$timeout = 1;
}
sleep($timeout);
$pathinfo = pathinfo( $thumb );
if( $thcontent != false ){
$upload_dir = wp_upload_dir();
$file_tmp_path = $upload_dir['basedir'] . '/' . $pathinfo['filename'] . '-' . $existing_post_id . '.' . explode('?',$pathinfo['extension'])[0];
$file = $wp_filesystem->put_contents( $file_tmp_path, $thcontent );
$wp_filetype = wp_check_filetype(basename($file_tmp_path), null );
$attachment = array(
'post_mime_type' => $wp_filetype['type'],
'post_title' => $existing_post_id,
'post_content' => '',
'post_status' => 'inherit'
);
$attach_id = wp_insert_attachment( $attachment, $file_tmp_path );
$thumb_id = $attach_id;
$imagenew = get_post( $attach_id );
$fullsizepath = get_attached_file( $imagenew->ID );
require_once( ABSPATH . 'wp-admin/includes/image.php' );
require_once( ABSPATH . 'wp-admin/includes/media.php' );
$attach_data = wp_generate_attachment_metadata( $attach_id, $fullsizepath );
wp_update_attachment_metadata( $attach_id, $attach_data );
}
}
}
else
{
$thumb_id = false;
}
$meta_data = array(
'_manga_import_slug' => $my_slug,
'_thumbnail_id' => $thumb_id,
'_wp_manga_alternative' => strip_tags( $alter_name ),
'_wp_manga_type' => strip_tags( $xtype ),
'_wp_manga_status' => $status,
'_wp_manga_chapter_type' => 'text',
'_wp_manga_chapters_warning'=> $chapter_warning,
);
foreach( $meta_data as $key => $value ){
if( !empty( $value ) ){
update_post_meta( $existing_post_id, $key, $value );
}
}
$manga_terms = array(
'wp-manga-release' => strip_tags( $xrelease ),
'wp-manga-author' => strip_tags( $xauthor ),
'wp-manga-artist' => strip_tags( $xartists ),
'wp-manga-genre' => isset( $post_args['genres'] ) ? $post_args['genres'] : '',
'wp-manga-tag' => isset( $post_args['tags'] ) ? $post_args['tags'] : null,
);
foreach( $manga_terms as $tax => $term ){
$resp = ums_add_manga_terms( $existing_post_id, $term, $tax );
}
ums_update_post_views( $existing_post_id, strip_tags( $viewsm ) );
ums_update_post_ratings( $existing_post_id, $xrating );
}
}
$find_chapters = $html->find('.listing-chapters_wrap li');
$output = array();
$current_vol = 'NO-VOLUME';
$chapters = array();
foreach( $find_chapters as $chapter )
{
$full_chapter_name = trim(preg_replace('/\t+/', '', $chapter->find('a',0)->plaintext));
$chapter_extend_name = '';
$names = explode(' - ', $full_chapter_name);
$chapter_name = $full_chapter_name;
if(count($names) == 2){
$chapter_extend_name = $names[1];
$chapter_name = $names[0];
}
$chapters[] = array(
'name' => $chapter_name,
'extend_name' => $chapter_extend_name,
'url' => $chapter->find('a',0)->href
);
}
$lazy_loaded = false;
if(count($chapters) == 0)
{
$latest_scraped = get_post_meta( $existing_post_id, 'ums_latest_scraped', true );
$latest_scraped = str_replace('vipnovel.com/vipnovel', 'newnovel.org/novel', $latest_scraped);
if(!empty($latest_scraped))
{
if(!stristr($latest_scraped, 'newnovel'))
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Skipping, already scraped from another source: ' . $za_xpost_args['post_title'] . ' (' . $latest_scraped . ')');
}
continue;
}
$chap_url = $latest_scraped;
}
else
{
if($chapter_slug != '')
{
$chap_url = $current_manga . $chapter_slug . '/';
}
else
{
$chap_url = $current_manga . 'chapter-1/';
}
}
$lazy_loaded = true;
preg_match_all('#\/chapter-([^/]*?)\/#i', $chap_url, $chnamje);
if(isset($chnamje[1][0]))
{
$chap_name = 'Chapter ' . $chnamje[1][0];
}
else
{
$chap_name = 'Chapter ' . rand();
}
$chapters[] = array(
'name' => $chap_name,
'url' => $chap_url
);
}
$ums_max_chapters = $max;
if($reverse_chapters == '1')
{
$reverse = true;
}
else
{
$reverse = false;
}
if($reverse == true)
{
$chapters = array_reverse($chapters);
}
$local_imported = 0;
$new_chap = false;
if($lazy_loaded == true)
{
$chapid = 1;
while(count($chapters) != 0)
{
$randsleep = rand(500,1000);
usleep($randsleep * 1000);
$chaps = $chapters[0];
if($ums_max_chapters <= $local_imported)
{
break;
}
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Processing novel chapter: ' . $chaps['url']);
}
$page_html = ums_get_web_page($chaps['url'], ums_get_random_user_agent(), $use_phantom, $phantom_wait);
if( $page_html == false ){
ums_log_to_file('Failed to download HTML for current chapter: ' . $chaps['name']);
break;;
}
if(stristr($page_html, 'Cloudflare Ray ID:') !== false)
{
ums_log_to_file('CloudFlare protection active on chapter, access is limited! Please install puppeteer on your site and set the "Content Scraping Method To Use" to Puppeteer for scraping to work. Chapter URL: ' . $chaps['url']);
sleep(1);
break;
}
$html = ums_str_get_html( $page_html );
if( empty( $html ) ){
ums_log_to_file('Failed to parse HTML for current chapter: ' . $chaps['name']);
break;
}
$chapid++;
preg_match_all('#<div class="cha-tit skiptranslate">\s*<h3>([^<]*?)<\/h3>#i', $page_html, $titli);
if(isset($titli[1][0]))
{
$chaps['name'] = $titli[1][0];
}
else
{
preg_match_all('#<li class="active">\s*?([^<]*?)\s*?<\/li>#i', $page_html, $titli);
if(isset($titli[1][0]))
{
$chaps['name'] = $titli[1][0];
}
}
preg_match_all('#<a\s*href="([^"]*?)"\s*class="btn next_page"#i', $page_html, $zurli);
if(isset($zurli[1][0]))
{
$dummy_chap = array(
'name' => 'Chapter ' . $chapid,
'url' => $zurli[1][0]
);
update_post_meta($existing_post_id, 'ums_latest_scraped', $zurli[1][0]);
}
else
{
$dummy_chap = false;
}
if($dummy_chap !== false)
{
$chapters = array($dummy_chap);
}
else
{
$chapters = array();
}
$slugified_name = $wp_manga_storage->slugify( $chaps['name'] );
$chapter_2 = $wp_manga_chapter->get_chapter_by_slug( $existing_post_id, $slugified_name );
if($chapter_2 && strtolower($chapter_2['chapter_slug']) == strtolower($slugified_name))
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Chapter is already published, skipping it: ' . $chapter_2['chapter_name']);
}
continue;
}
$content = '';
$doc = new DOMDocument;
$internalErrors = libxml_use_internal_errors(true);
$doc->loadHTML('<?xml encoding="utf-8" ?>' . $page_html);
libxml_use_internal_errors($internalErrors);
$xpath = new \DOMXpath($doc);
$articles = $xpath->query('//*[contains(@class, "text-left")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
break;
}
}
if(empty($content))
{
$articles = $xpath->query('//*[contains(@class, "content-body")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
}
}
}
if(empty($content))
{
$articles = $xpath->query('//*[contains(@class, "cha-tit")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
}
}
$articles = $xpath->query('//*[contains(@class, "cha-content")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
}
}
}
if(empty($content))
{
$ccontent = $html->find('.reading-content .text-left .cha-words .cha-words');
if($ccontent)
{
foreach($ccontent as $xct)
{
$other_ps = $xct->find('p');
if(count($other_ps) > 0)
{
$text = '';
foreach($other_ps as $p)
{
$text .= '<p> ' . $p->plaintext . '</p>';
}
$content .= $text . '<hr/>';
}
else
{
$content .= $xct->innertext . '<hr/>';
}
}
}
else
{
if($html->find('.reading-content #content', 0))
{
$content = $html->find('.reading-content #content', 0)->innertext;
}
else
{
$ccontent = $html->find('.reading-content .text-left .cha-content .cha-words');
if($ccontent)
{
$content = '';
foreach($ccontent as $xct)
{
$content .= $xct->innertext . '<hr/>';
}
}
else
{
$content = $html->find('.reading-content .text-left', 0);
if($content)
{
$other_ps = $content->find('p');
if(count($other_ps) > 0)
{
$text = '';
foreach($other_ps as $p)
{
$text .= '<p> ' . $p->plaintext . '</p>';
}
$content = $text;
}
else
{
if($inner_content = $content->find('#chapter-content', 0))
{
$content = $inner_content->innertext;
}
else
{
if($inner_content = $content->find('.text-left', 0))
{
$content = $inner_content->innertext;
}
else
{
if($inner_content = $content->find('.fr-view', 0))
{
$content = $inner_content->innertext;
}
else
{
$content = $content->innertext;
}
}
}
}
}
}
}
if(!$content)
{
$content = '';
$ps = $html->find('.reading-content .text-left p');
foreach($ps as $p)
{
$content .= '<p>' . $p->plaintext . '</p>';
}
}
}
}
if($use_phantom != '0')
{
$xdoc = new DOMDocument;
$internalErrors = libxml_use_internal_errors(true);
$xdoc->loadHTML('<?xml encoding="utf-8" ?>' . $content);
libxml_use_internal_errors($internalErrors);
$xxpath = new \DOMXpath($xdoc);
$ps = $xxpath->query('//p');
if($ps !== false && count($ps) > 0)
{
$tmpcont = '';
foreach($ps as $container)
{
if(method_exists($container, 'saveHTML'))
{
$tmpcont .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$tmpcont .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$tmpcont .= ' ' . $container->nodeValue;
}
}
if($tmpcont != '')
{
$content = $tmpcont;
}
}
$content = preg_replace('#(?:<p>)?<i(?:[^>]*?)>(?:[\s\S]*?)<\/i>(?:<\/p>)?#i', '', $content);
}
$content = preg_replace('#(?:my)?[.]?v[.\s]?i[.\s]?p[.\s]?n[.\s]?o[.\s]?v[.\s]?e[.\s]?l[.]?(?:\s?.c[o0]m)?#i', get_bloginfo('name'), $content);
$content = str_ireplace('newnovel.org', get_bloginfo('name'), $content);
$content = str_ireplace('new novel', get_bloginfo('name'), $content);
$content = str_ireplace('newnovel', get_bloginfo('name'), $content);
$qtags = array('iframe', 'script', 'ins');
foreach ($qtags as $qtag) {
$regex = '#<\s*' . $qtag . '[^>]*>.*?<\s*/\s*'. $qtag . '>#msi';
$content = preg_replace($regex, '', $content);
}
$content = preg_replace('#<style([\s\S]+?)<\/style>#i', '', $content);
$content = preg_replace('#<\/?div([^>]*)>#i', '', $content);
if($strip_images == '1')
{
$content = ums_sanitize_html_content($content);
}
$chapter_content = trim($content);
$chapter_content = ums_strip_links($chapter_content);
if($chapter_content == '')
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Skipping current chapter (because it failed to parse first chapter content): ' . print_r($chaps['url'], true));
}
//new_chap true to prevent infinite running
$new_chap = true;
break;
}
$arr = ums_spin_and_translate($chaps['name'], $chapter_content, $rule_translate, 'en');
if($arr === false)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Skipping current chapter (because it failed to be translated): ' . print_r($chaps['url'], true));
}
}
elseif($no_translate_title != '1')
{
$chaps['name'] = $arr[0];
$chaps['extend_name'] = $arr[0];
$chapter_content = $arr[1];
}
global $wp_manga_text_type;
if(empty($chaps['name']))
{
$chaps['name'] = 'Novel';
}
if($chaps['name'] == '')
{
$xext_name = $name_str . ' ' . $chaps['extend_name'];
}
else
{
$xext_name = '';
}
if($xext_name == '')
{
$chapsx = explode('-', $chaps['name']);
if(isset($chapsx[1]))
{
$xext_name = trim($chapsx[1]);
$chaps['name'] = trim($chapsx[0]);
}
}
$chapter_args = array(
'post_id' => $existing_post_id,
'chapter_name' => $chaps['name'],
'chapter_name_extend' => $xext_name,
'volume_id' => '',
'chapter_content' => $chapter_content,
);
$slugified_name = $wp_manga_storage->slugify( $chaps['name'] );
$chapter_2 = $wp_manga_chapter->get_chapter_by_slug( $existing_post_id, $slugified_name );
if($chapter_2 && strtolower($chapter_2['chapter_slug']) == strtolower($slugified_name))
{
ums_log_to_file('Chapter name already published, we will skip: ' . $chapter_2['chapter_name']);
continue;
}
$chapter_id = $wp_manga_text_type->insert_chapter( $chapter_args );
if( $chapter_id ){
if( is_wp_error( $chapter_id ) ){
ums_log_to_file('Failed to insert new chapter: ' . $chaps['name'] . ' error: ' . $chapter_id->get_error_message());
continue;
}
else
{
$new_chap = true;
$ums_imported_chapters++;
$local_imported++;
}
}
if (isset($ums_Main_Settings['request_timeout']) && $ums_Main_Settings['request_timeout'] != '') {
$timeout = intval($ums_Main_Settings['request_timeout']);
} else {
$timeout = 1;
}
sleep($timeout);
}
}
else
{
foreach($chapters as $chaps)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Processing novel chapter (no lazyload): ' . $chaps['url']);
}
if($ums_max_chapters <= $local_imported)
{
break;
}
$slugified_name = $wp_manga_storage->slugify( $chaps['name'] );
$chapter_2 = $wp_manga_chapter->get_chapter_by_slug( $existing_post_id, $slugified_name );
if($chapter_2 && strtolower($chapter_2['chapter_slug']) == strtolower($slugified_name))
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Chapter already published, skipping it: ' . $chapter_2['chapter_name']);
}
continue;
}
$page_html = ums_get_web_page($chaps['url'], ums_get_random_user_agent(), $use_phantom, $phantom_wait);
if( $page_html == false ){
ums_log_to_file('Failed to download HTML for chapter: ' . $chaps['name']);
continue;
}
$html = ums_str_get_html( $page_html );
if( empty( $html ) ){
ums_log_to_file('Failed to parse HTML for chapter: ' . $chaps['name']);
continue;
}
$content = '';
$doc = new DOMDocument;
$internalErrors = libxml_use_internal_errors(true);
$doc->loadHTML('<?xml encoding="utf-8" ?>' . $page_html);
libxml_use_internal_errors($internalErrors);
$xpath = new \DOMXpath($doc);
$articles = $xpath->query('//*[contains(@class, "text-left")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
break;
}
}
if(empty($content))
{
$articles = $xpath->query('//*[contains(@class, "content-body")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
}
}
}
if(empty($content))
{
$articles = $xpath->query('//*[contains(@class, "cha-tit")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
}
}
$articles = $xpath->query('//*[contains(@class, "cha-content")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
}
}
}
if(empty($content))
{
$content = $html->find('.reading-content .text-left .cha-words .cha-words', 0);
if($content)
{
$other_ps = $content->find('p');
if(count($other_ps) > 0)
{
$text = '';
foreach($other_ps as $p)
{
$text .= '<p> ' . $p->plaintext . '</p>';
}
$content = $text;
}
else
{
$content = $content->innertext;
}
}
else
{
if($html->find('.reading-content #content', 0))
{
$content = $html->find('.reading-content #content', 0)->innertext;
}
else
{
$content = $html->find('.reading-content .text-left .cha-content .cha-words', 0);
if($content)
{
$content = $content->innertext;
}
else
{
$content = $html->find('.reading-content .text-left', 0);
if($content)
{
$other_ps = $content->find('p');
if(count($other_ps) > 0)
{
$text = '';
foreach($other_ps as $p)
{
$text .= '<p> ' . $p->plaintext . '</p>';
}
$content = $text;
}
else
{
if($inner_content = $content->find('#chapter-content', 0))
{
$content = $inner_content->innertext;
}
else
{
if($inner_content = $content->find('.text-left', 0))
{
$content = $inner_content->innertext;
}
else
{
if($inner_content = $content->find('.fr-view', 0))
{
$content = $inner_content->innertext;
}
else
{
$content = $content->innertext;
}
}
}
}
}
}
}
if(!$content)
{
$content = '';
$ps = $html->find('.reading-content .text-left p');
foreach($ps as $p)
{
$content .= '<p>' . $p->plaintext . '</p>';
}
}
}
}
if($use_phantom != '0')
{
$xdoc = new DOMDocument;
$internalErrors = libxml_use_internal_errors(true);
$xdoc->loadHTML('<?xml encoding="utf-8" ?>' . $content);
libxml_use_internal_errors($internalErrors);
$xxpath = new \DOMXpath($xdoc);
$ps = $xxpath->query('//p');
if($ps !== false && count($ps) > 0)
{
$tmpcont = '';
foreach($ps as $container)
{
if(method_exists($container, 'saveHTML'))
{
$tmpcont .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$tmpcont .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$tmpcont .= ' ' . $container->nodeValue;
}
}
if($tmpcont != '')
{
$content = $tmpcont;
}
}
$content = preg_replace('#(?:<p>)?<i(?:[^>]*?)>(?:[\s\S]*?)<\/i>(?:<\/p>)?#i', '', $content);
}
$content = preg_replace('#(?:my)?[.]?v[.\s]?i[.\s]?p[.\s]?n[.\s]?o[.\s]?v[.\s]?e[.\s]?l[.]?(?:\s?.c[o0]m)?#i', get_bloginfo('name'), $content);
$content = str_ireplace('newnovel.org', get_bloginfo('name'), $content);
$content = str_ireplace('new novel', get_bloginfo('name'), $content);
$content = str_ireplace('newnovel', get_bloginfo('name'), $content);
$qtags = array('iframe', 'script', 'ins');
foreach ($qtags as $qtag) {
$regex = '#<\s*' . $qtag . '[^>]*>.*?<\s*/\s*'. $qtag . '>#msi';
$content = preg_replace($regex, '', $content);
}
$content = preg_replace('#<style([\s\S]+?)<\/style>#i', '', $content);
$content = preg_replace('#<\/?div([^>]*)>#i', '', $content);
if($strip_images == '1')
{
$content = ums_sanitize_html_content($content);
}
$chapter_content = trim($content);
$chapter_content = ums_strip_links($chapter_content);
$arr = ums_spin_and_translate($chaps['name'], $chapter_content, $rule_translate, 'en');
if($arr === false)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Skipping chapter (because it failed to be translated): ' . print_r($post_args, true));
}
}
elseif($no_translate_title != '1')
{
$chaps['name'] = $arr[0];
$chaps['extend_name'] = $arr[0];
$chapter_content = $arr[1];
}
global $wp_manga_text_type;
if(empty($chaps['name']))
{
$xext_name = $name_str . ' ' . $chaps['extend_name'];
$chaps['name'] = 'Novel';
}
else
{
$xext_name = '';
}
if($xext_name == '')
{
$chapsx = explode('-', $chaps['name']);
if(isset($chapsx[1]))
{
$xext_name = trim($chapsx[1]);
$chaps['name'] = trim($chapsx[0]);
}
}
$chapter_args = array(
'post_id' => $existing_post_id,
'chapter_name' => $chaps['name'],
'chapter_name_extend' => $xext_name,
'volume_id' => '',
'chapter_content' => $chapter_content,
);
$slugified_name = $wp_manga_storage->slugify( $chaps['name'] );
$chapter_2 = $wp_manga_chapter->get_chapter_by_slug( $existing_post_id, $slugified_name );
if($chapter_2 && strtolower($chapter_2['chapter_slug']) == strtolower($slugified_name))
{
ums_log_to_file('Chapter name already published, skipping it: ' . $chapter_2['chapter_name']);
continue;
}
$chapter_id = $wp_manga_text_type->insert_chapter( $chapter_args );
if( $chapter_id ){
if( is_wp_error( $chapter_id ) ){
ums_log_to_file('Failed to insert chapter: ' . $chaps['name'] . ' error: ' . $chapter_id->get_error_message());
continue;
}
else
{
$new_chap = true;
$ums_imported_chapters++;
$local_imported++;
}
}
if (isset($ums_Main_Settings['request_timeout']) && $ums_Main_Settings['request_timeout'] != '') {
$timeout = intval($ums_Main_Settings['request_timeout']);
} else {
$timeout = 1;
}
sleep($timeout);
}
}
if($new_chap == true)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Total novel chapters scraped: ' . $local_imported);
}
$scraped_manga++;
}
}
}
catch(Exception $e)
{
ums_log_to_file('Importing failed: ' . $e->getMessage());
if($auto == 1)
{
ums_clearFromList($param, $type);
}
return 'fail';
}
}
}
elseif($type == 4)
{
if(trim($max_manga) != '')
{
$get_max_manga = intval(trim($max_manga));
}
else
{
$get_max_manga = 999;
}
$items = array();
$page_increased = false;
{
$manga_arr = array();
try
{
global $wp_filesystem;
if ( ! is_a( $wp_filesystem, 'WP_Filesystem_Base') ){
include_once(ABSPATH . 'wp-admin/includes/file.php');$creds = request_filesystem_credentials( site_url() );
wp_filesystem($creds);
}
$manga_names = explode(',', $manga_name);
$manga_names = array_map('trim', $manga_names);
$manga_arr = $manga_names;
if(count($manga_arr) == 0)
{
ums_log_to_file('No Web Novel matched your query: ' . $manga_name);
return 'nochange';
}
$scraped_manga = 0;
foreach($manga_arr as $current_manga)
{
if($get_max_manga <= $scraped_manga)
{
break;
}
$parsedUrl = parse_url($current_manga);
if($parsedUrl === false)
{
ums_log_to_file('Cannot parse URL: ' . $current_manga);
continue;
}
if(!isset($parsedUrl['host']))
{
ums_log_to_file('Incorrect URL submitted: ' . $current_manga);
continue;
}
if(!isset($parsedUrl['path']) || empty($parsedUrl['path']))
{
ums_log_to_file('Incorrect URL format submitted: ' . $current_manga);
continue;
}
$domain = $parsedUrl['host'];
$path = trim($parsedUrl['path'], '/');
$pathParts = explode('/', $path);
$firstDirectory = isset($pathParts[0]) ? $pathParts[0] : '';
$current_manga = trim($current_manga, '/');
$current_manga = $current_manga . '/';
$html_site = ums_get_web_page($current_manga, ums_get_random_user_agent(), $use_phantom, $phantom_wait);
if($html_site == false)
{
ums_log_to_file('Failed to download manga chapter: ' . $current_manga);
continue;
}
if(stristr($html_site, 'Cloudflare Ray ID:') !== false)
{
ums_log_to_file('CloudFlare protection active on novel site, access is limited! Please install puppeteer on your site and set the "Content Scraping Method To Use" to Puppeteer for scraping to work. Manga URL: ' . $current_manga);
sleep(1);
continue;
}
if(stristr($html_site, 'manga-title-badges custom comic') !== false)
{
ums_log_to_file('This URL is a web comic, not a web novel, skipping it: ' . $current_manga);
continue;
}
if (isset($ums_Main_Settings['request_timeout']) && $ums_Main_Settings['request_timeout'] != '') {
$timeout = intval($ums_Main_Settings['request_timeout']);
} else {
$timeout = 1;
}
sleep($timeout);
require_once (dirname(__FILE__) . "/res/simple_html_dom.php");
$html = ums_str_get_html( $html_site );
$tag = $html->find( '.profile-manga .post-title h1', 0 );
$name_str = '';
$my_slug = '';
$my_slug = str_replace( 'http://', '', $current_manga );
$my_slug = str_replace( 'https://', '', $my_slug );
$my_slug = str_replace( 'http:', '', $my_slug );
$my_slug = str_replace( $domain . '/' . $firstDirectory . '/', '', $my_slug );
$my_slug = str_replace( '/', '', $my_slug );
$my_slug = str_replace( '.html', '', $my_slug );
$my_slug = str_replace( '.htm', '', $my_slug );
if($tag){
$name_str = trim($tag->plaintext);
$span = $tag->find('span', 0);
if($span){
$name_str = trim(str_replace($span->plaintext,'', $name_str));
}
}
if($name_str == '')
{
preg_match_all('#<title>([^<]*?)<\/title>#i', $html_site, $tmathc);
if(isset($tmathc[1][0]))
{
$name_str = $tmathc[1][0];
}
}
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Processing novel: ' . $name_str);
}
$find_post = new WP_Query( array(
'title' => $name_str,
'post_type' => 'wp-manga',
) );
$existing_post_id = false;
if( $find_post->have_posts() )
{
$existing_post_id = $find_post->posts[0]->ID;
}
if($existing_post_id == false)
{
$args = array(
'post_type' => 'wp-manga',
'meta_key' => '_manga_import_slug',
'meta_value' => $my_slug,
'post_status' => array('publish','draft','pending','trash','private','future')
);
$query = new WP_Query( $args );
if( $query->have_posts() ){
$existing_post_id = $query->posts[0]->ID;
}
}
if($existing_post_id == false)
{
$block_warning = $html->find('#page .warning', 0);
if( !empty( $block_warning ) && strpos( $block_warning->plaintext, 'has been licensed, it is not available' ) !== false ){
ums_log_to_file('Request blocked, please use a proxy!');
continue;
}
$desc = $html->find( '.description-summary' );
$xd = '';
if(is_array($desc))
{
foreach($desc as $dd)
{
if(stristr($dd->plaintext, 'Description') || stristr($dd->plaintext, 'Summary'))
{
$xd = trim($dd->plaintext);
$xprefix = 'Description';
if (substr($xd, 0, strlen($xprefix)) == $xprefix) {
$xd = substr($xd, strlen($xprefix));
}
$xprefix = 'Summary';
if (substr($xd, 0, strlen($xprefix)) == $xprefix) {
$xd = substr($xd, strlen($xprefix));
}
$xd = trim($xd);
}
}
}
if($xd == '')
{
$doc = new DOMDocument;
$internalErrors = libxml_use_internal_errors(true);
$doc->loadHTML('<?xml encoding="utf-8" ?>' . $html_site);
libxml_use_internal_errors($internalErrors);
$xpath = new \DOMXpath($doc);
$articles = $xpath->query('//*[@class="g_txt_over mb48 fs16 j_synopsis _txtover _on"]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container) {
if(method_exists($container, 'saveHTML'))
{
$xd .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$xd .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$xd .= ' ' . $container->nodeValue;
}
}
$qtags = array('iframe', 'script');
foreach ($qtags as $qtag)
{
$regex = '#<\s*' . $qtag . '[^>]*>.*?<\s*/\s*'. $qtag . '>#msi';
$xd = preg_replace($regex, '', $xd);
}
}
else
{
$articles = $xpath->query('//*[@class="c_000"]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container) {
if(method_exists($container, 'saveHTML'))
{
$xd .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$xd .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$xd .= ' ' . $container->nodeValue;
}
}
}
}
}
if($xd == '')
{
$desc = '';
}
else
{
$desc = trim($xd);
}
if($desc == '')
{
preg_match_all('#<div class="summary__content show-more">(?:<p>)?([\s\S]*?)(?:<\/p>)?<\/?div>#i', $html_site, $dmathc);
if(isset($dmathc[1][0]))
{
$desc = $dmathc[1][0];
}
}
$desc = ums_strip_links($desc);
$uxrlparts = wp_parse_url(home_url());
$domainx = $uxrlparts['host'];
$desc = preg_replace('#<p><center><a href="https:\/\/bronovel\.com\/"><em><strong>Announcement<\/strong><\/em>: we are moving[\s\S]*?<\/center><\/p>#', '', $desc);
$desc = str_ireplace($domain, $domainx, $desc);
$thumb = $html->find( '.summary_image .img-responsive' );
$thumb = !empty( $thumb ) ? $thumb[0]->{'data-src'} : '';
if($thumb == '')
{
$thumb = !empty( $thumb ) ? $thumb[0]->src : '';
}
if($thumb == '')
{
$thumb = $html->find( '.summary_image' );
$thumb = !empty( $thumb ) ? $thumb[0]->src : '';
}
if($thumb == '')
{
preg_match_all('#<div class="summary_image">[\s\n]*<a href="[^"]*?">[\s\n]*<img[^>]*?src="([^"]*?)"#i', $html_site, $xmathc);
if(isset($xmathc[1][0]))
{
$thumb = trim($xmathc[1][0]);
}
}
if(empty($thumb))
{
preg_match_all('#<meta\s*property="og:image"\s*content="([^"]*?)"\s*\/?>#i', $html_site, $imatch);
if(isset($imatch[1][0]))
{
$thumb = trim($imatch[1][0]);
}
}
$stx = '';
$status = $html->find( '.post-status .post-content_item' );
if(is_array($status))
{
foreach($status as $st)
{
$temp_stat = $st->plaintext;
if(stristr($temp_stat, 'status') !== false)
{
$stx = $temp_stat;
}
}
}
if($stx === '')
{
$status = '';
}
else
{
$status = $stx;
}
if(stristr($status, 'ongoing') !== false)
{
$status = 'on-going';
}
elseif(stristr($status, 'completed') !== false)
{
$status = 'end';
}
elseif(stristr($status, 'canceled') !== false)
{
$status = 'canceled';
}
elseif(stristr($status, 'on hold') !== false)
{
$status = 'on-hold';
}
elseif(stristr($status, 'upcoming') !== false)
{
$status = 'upcoming';
}
$alter_name = $html->find( '.summary_content .post-content .post-content_item' );
$xplain = '';
if( !empty( $alter_name ) )
{
foreach($alter_name as $an)
{
if(stristr($an->plaintext, 'Alternative '))
{
$xplain = trim($an->plaintext);
$xprefix = 'Alternative';
if (substr($xplain, 0, strlen($xprefix)) == $xprefix) {
$xplain = substr($xplain, strlen($xprefix));
}
$xplain = trim($xplain);
}
}
}
if($xplain != '')
{
$alter_name = $xplain;
}
else
{
$alter_name = '';
}
$data = $html->find( '.summary_content .post-content .post-content_item' );
$xtype = '';
if( !empty( $data ) ){
foreach($data as $item){
if($item->find('.summary-heading h5',0) !== null && trim($item->find('.summary-heading h5',0)->plaintext) == 'Type'){
$xtype = $item->find('.summary-content', 0)->plaintext;
if(trim($xtype) == 'Updating')
{
$xtype = '';
}
}
}
}
$xrelease = '2021';
preg_match_all('#<a href="https:\/\/' . addslashes($domain) . '\/(?:manga-)?release\/(?:[^"]*?)" rel="tag">([^<]+?)<\/a>#i', $html_site, $xmathc);
if(isset($xmathc[1][0]))
{
$xrelease = trim($xmathc[1][0]);
}
$xauthor = '';
$author = '';
$data = $html->find( '.author-content a' );
if( !empty( $data ) ){
$authors = array();
foreach($data as $arxthor)
{
array_push($authors, $arxthor->plaintext);
if($author == '')
{
$author = $arxthor->plaintext;
}
}
$xauthor = implode(',', $authors);
}
$xartists = '';
$data = $html->find( '.artist-content a' );
if( !empty( $data ) ){
$artists = array();
foreach($data as $artist){
array_push($artists, $artist->plaintext);
}
$xartists = implode(',', $artists);
}
$xgenres = '';
$data = $html->find( '.genres-content a' );
if( !empty( $data ) ){
$genres = array();
foreach($data as $genre){
array_push($genres, $genre->plaintext);
}
$xgenres = implode(',', $genres);
}
$items = $html->find('.summary_content .post-content .post-content_item');
$views = '';
foreach($items as $item){
if(trim($item->find('.summary-heading', 0)->plaintext) == 'Rank'){
$str = $item->find('.summary-content',0)->plaintext;
preg_match_all('~\d+(?:\.\d+)?~', $str, $matches);
if($matches){
if(count($matches) == 3){
$views = $matches[0][1];
} else {
$views = $matches[0][0];
}
if(strpos($str, 'K monthly views') !== false){
$views = floatval($views) * 1000;
}
if(strpos($str, 'M monthly views') !== false){
$views = floatval($views) * 1000000;
}
break;
}
}
}
if(empty(trim($views)) || !is_numeric(strip_tags( $views )))
{
$viewsm = rand(100,500);
}
else
{
$viewsm = strip_tags(trim($views));
}
$average_vote = $html->find('.post-total-rating .total_votes', 0);
$str = $html->find('.vote-details',0);
$xrating = array();
if(!empty($str))
{
preg_match_all('!\d+!', $str, $matches);
$number_votes = 1;
if($matches && count($matches[0]) == 3)
{
$number_votes = $matches[0][2];
}
if( !empty( $average_vote ) && $average_vote->plaintext > 0 ){
$xrating = array(
'avg' => $average_vote->plaintext,
'numbers' => $number_votes
);
}
}
$xtags = '';
$data = $html->find( '.tags-content a' );
if( !empty( $data ) ){
$tags = array();
foreach($data as $tag){
$ttext = trim($tag->plaintext);
array_push($tags, $ttext);
}
$xtags = implode(',', $tags);
}
$xtime_year = '';
$time_year = strtotime($xrelease);
if($time_year !== false)
{
$time_year = date("Y", $time_year);
if($time_year !== false)
{
$xtime_year = $time_year;
}
}
$post_args = array(
'manga_import_slug' => $my_slug,
'title' => $name_str,
'post_status' => $post_status,
'description' => $desc,
'thumb' => $thumb,
'status' => $status,
'altername' => strip_tags( $alter_name ),
'type' => strip_tags( $xtype ),
'release' => $xtime_year,
'authors' => strip_tags( $xauthor ),
'artists' => strip_tags( $xartists ),
);
$post_args['genres'] = '';
if($auto_categories == 'genre')
{
$post_args['genres'] = strip_tags( $xgenres );
}
if($default_category != '')
{
if($post_args['genres'] != '')
{
$post_args['genres'] .= ',';
}
$genreplus = get_term_by('id', $default_category, 'wp-manga-genre');
if($genreplus !== false)
{
$post_args['genres'] .= $genreplus->slug;
}
}
$post_args['views'] = strip_tags( $viewsm );
$post_args['ratings'] = $xrating;
if($can_create_tag == 'genre')
{
$post_args['tags'] = strip_tags( $xgenres );
}
elseif($can_create_tag == 'tags')
{
$post_args['tags'] = strip_tags( $xtags );
}
else
{
$post_args['tags'] = '';
}
if($item_create_tag != '')
{
$ctag = explode(',', $item_create_tag);
foreach($ctag as $ctap)
{
$ctap = trim($ctap);
if($post_args['tags'] != '')
{
$post_args['tags'] .= ',';
}
$genreplus = get_term_by('name', $ctap, 'wp-manga-tag');
if($genreplus !== false)
{
$post_args['tags'] .= $genreplus->slug;
}
else
{
$cat = wp_insert_term($ctap, 'wp-manga-tag');
if(!is_wp_error( $cat ))
{
$post_args['tags'] .= $cat['slug'];
}
}
}
}
$post_args['description'] = str_ireplace($domain, $_SERVER['HTTP_HOST'], $post_args['description']);
$arr = ums_spin_and_translate($post_args['title'], $post_args['description'], $rule_translate, 'en');
if($arr === false)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Skipping web novel (because it failed to be translated): ' . print_r($post_args, true));
}
continue;
}
else
{
if($no_translate_title != '1')
{
$post_args['title'] = $arr[0];
}
$post_args['description'] = $arr[1];
}
$za_xpost_args = array(
'post_title' => !empty( $post_args['title'] ) ? $post_args['title'] : '',
'post_content' => !empty( $post_args['description'] ) ? $post_args['description'] : '',
'post_type' => 'wp-manga',
'post_status' => isset( $post_args['post_status'] ) ? $post_args['post_status'] : 'pending',
);
$existing_again = false;
if($za_xpost_args['post_title'] != '')
{
$xposts = get_posts(
array(
'post_type' => 'wp-manga',
'title' => html_entity_decode($za_xpost_args['post_title']),
'post_status' => 'all',
'numberposts' => 1,
'update_post_term_cache' => false,
'update_post_meta_cache' => false,
'orderby' => 'post_date ID',
'order' => 'ASC',
)
);
if ( ! empty( $xposts ) ) {
$ex_page = $xposts[0];
} else {
$ex_page = null;
}
if(isset($ex_page->ID))
{
$existing_again = true;
$existing_post_id = $ex_page->ID;
}
}
if($existing_again == false)
{
$accept_comments = 'closed';
if ($enable_comments == '1') {
$accept_comments = 'open';
}
$za_xpost_args['comment_status'] = $accept_comments;
if ($enable_pingback == '1')
{
$za_xpost_args['ping_status'] = 'open';
}
else
{
$za_xpost_args['ping_status'] = 'closed';
}
if($get_date == '1')
{
if(!empty($xrelease))
{
if(strlen($xrelease) == 4)
{
$xrelease .= '-01-01 00:00';
}
$postdatex = gmdate("Y-m-d H:i:s", strtotime($xrelease));
$za_xpost_args['post_date_gmt'] = $postdatex;
}
}
if($user_name_type == 'rand')
{
$randid = ums_display_random_user();
if($randid === false)
{
$za_xpost_args['post_author'] = ums_randomName();
}
else
{
$za_xpost_args['post_author'] = $randid->ID;
}
}
elseif($user_name_type == 'feed-news')
{
$sp_post_user_name = ums_randomName();
if($author == '' || $author == '1' || $author == 'null')
{
$author = ums_randomName();
}
if($author != '')
{
$xauthor = sanitize_user( $author );
$xauthor = apply_filters( 'pre_user_login', $xauthor );
$xauthor = trim( $xauthor );
if(username_exists( $xauthor ))
{
$user_id_t = get_user_by('login', $xauthor);
if($user_id_t)
{
$sp_post_user_name = $user_id_t->ID;
}
}
else
{
$palphabet = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890!@#$%^*()-+=_?><,.;:}{][';
$ppass = '';
$alphaLength = strlen($palphabet) - 1;
for ($ipass = 0; $ipass < 8; $ipass++)
{
$npass = rand(0, $alphaLength);
$ppass .= $palphabet[$npass];
}
$curr_id = wp_create_user($xauthor, $ppass, ums_generate_random_email());
if ( is_int($curr_id) )
{
$u = new WP_User($curr_id);
$u->remove_role('subscriber');
$u->add_role('author');
$sp_post_user_name = $curr_id;
}
}
}
$za_xpost_args['post_author'] = ums_utf8_encode($sp_post_user_name);
}
else
{
$za_xpost_args['post_author'] = ums_utf8_encode($post_user_name);
}
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Inserting new novel: ' . $za_xpost_args['post_title']);
}
$existing_post_id = wp_insert_post( $za_xpost_args );
if( ! $existing_post_id || is_wp_error( $existing_post_id ) ){
ums_log_to_file('Failed to insert novel into db: ' . $post_args['title']);
continue;
}
add_post_meta($existing_post_id, 'ums_parent_rule', $type . '-' . $param);
wp_set_object_terms( $existing_post_id, 'Ums_' . $type . '_' . $param, 'coderevolution_post_source', true);
if($thumb != '')
{
$thumb_id = ums_wp_mcl_e_upload_file( $post_args['thumb'], $use_phantom, $phantom_wait, $existing_post_id );
if($thumb_id === false)
{
include_once( ABSPATH . 'wp-admin/includes/image.php' );
$thcontent = ums_get_web_page($thumb);
if (isset($ums_Main_Settings['request_timeout']) && $ums_Main_Settings['request_timeout'] != '') {
$timeout = intval($ums_Main_Settings['request_timeout']);
} else {
$timeout = 1;
}
sleep($timeout);
$pathinfo = pathinfo( $thumb );
if( $thcontent != false ){
$upload_dir = wp_upload_dir();
$file_tmp_path = $upload_dir['basedir'] . '/' . $pathinfo['filename'] . '-' . $existing_post_id . '.' . explode('?',$pathinfo['extension'])[0];
$file = $wp_filesystem->put_contents( $file_tmp_path, $thcontent );
$wp_filetype = wp_check_filetype(basename($file_tmp_path), null );
$attachment = array(
'post_mime_type' => $wp_filetype['type'],
'post_title' => $existing_post_id,
'post_content' => '',
'post_status' => 'inherit'
);
$attach_id = wp_insert_attachment( $attachment, $file_tmp_path );
$thumb_id = $attach_id;
$imagenew = get_post( $attach_id );
$fullsizepath = get_attached_file( $imagenew->ID );
require_once( ABSPATH . 'wp-admin/includes/image.php' );
require_once( ABSPATH . 'wp-admin/includes/media.php' );
$attach_data = wp_generate_attachment_metadata( $attach_id, $fullsizepath );
wp_update_attachment_metadata( $attach_id, $attach_data );
}
}
}
else
{
$thumb_id = false;
}
$meta_data = array(
'_manga_import_slug' => $my_slug,
'_thumbnail_id' => $thumb_id,
'_wp_manga_alternative' => strip_tags( $alter_name ),
'_wp_manga_type' => strip_tags( $xtype ),
'_wp_manga_status' => $status,
'_wp_manga_chapter_type' => 'text',
'_wp_manga_chapters_warning'=> $chapter_warning,
);
foreach( $meta_data as $key => $value ){
if( !empty( $value ) ){
update_post_meta( $existing_post_id, $key, $value );
}
}
$manga_terms = array(
'wp-manga-release' => strip_tags( $xrelease ),
'wp-manga-author' => strip_tags( $xauthor ),
'wp-manga-artist' => strip_tags( $xartists ),
'wp-manga-genre' => isset( $post_args['genres'] ) ? $post_args['genres'] : '',
'wp-manga-tag' => isset( $post_args['tags'] ) ? $post_args['tags'] : null,
);
foreach( $manga_terms as $tax => $term ){
$resp = ums_add_manga_terms( $existing_post_id, $term, $tax );
}
ums_update_post_views( $existing_post_id, strip_tags( $viewsm ) );
ums_update_post_ratings( $existing_post_id, $xrating );
}
}
$find_chapters = $html->find('.listing-chapters_wrap li');
$output = array();
$current_vol = 'NO-VOLUME';
$chapters = array();
foreach( $find_chapters as $chapter )
{
$full_chapter_name = trim(preg_replace('/\t+/', '', $chapter->find('a',0)->plaintext));
$chapter_extend_name = '';
$names = explode(' - ', $full_chapter_name);
$chapter_name = $full_chapter_name;
if(count($names) == 2){
$chapter_extend_name = $names[1];
$chapter_name = $names[0];
}
$chapters[] = array(
'name' => $chapter_name,
'extend_name' => $chapter_extend_name,
'url' => $chapter->find('a',0)->href
);
}
$lazy_loaded = false;
if(count($chapters) == 0)
{
if($always_first == '1' && $chapter_slug != '')
{
$chap_url = $current_manga . $chapter_slug . '/';
}
else
{
$latest_scraped = get_post_meta( $existing_post_id, 'ums_latest_scraped', true );
if(!empty($latest_scraped))
{
$chap_url = $latest_scraped;
}
else
{
if($chapter_slug != '')
{
$chap_url = $current_manga . $chapter_slug . '/';
}
else
{
$chap_url = $current_manga . 'chapter-1/';
}
}
}
$lazy_loaded = true;
preg_match_all('#\/chapter-([^/]*?)\/#i', $chap_url, $chnamje);
if(isset($chnamje[1][0]))
{
$chap_name = 'Chapter ' . $chnamje[1][0];
}
else
{
$chap_name = 'Chapter ' . rand();
}
$chapters[] = array(
'name' => $chap_name,
'url' => $chap_url
);
}
$ums_max_chapters = $max;
if($reverse_chapters == '1')
{
$reverse = true;
}
else
{
$reverse = false;
}
if($reverse == true)
{
$chapters = array_reverse($chapters);
}
$local_imported = 0;
$new_chap = false;
if($lazy_loaded == true)
{
$chapid = 1;
while(count($chapters) != 0)
{
$randsleep = rand(500,1000);
usleep($randsleep * 1000);
$chaps = $chapters[0];
if($ums_max_chapters <= $local_imported)
{
break;
}
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Processing novel chapter: ' . $chaps['url']);
}
$page_html = ums_get_web_page($chaps['url'], ums_get_random_user_agent(), $use_phantom, $phantom_wait);
if( $page_html == false ){
ums_log_to_file('Failed to download HTML for current chapter: ' . $chaps['name']);
break;;
}
if(stristr($page_html, 'Cloudflare Ray ID:') !== false)
{
ums_log_to_file('CloudFlare protection active on chapter, access is limited! Please install puppeteer on your site and set the "Content Scraping Method To Use" to Puppeteer for scraping to work. Chapter URL: ' . $chaps['url']);
sleep(1);
break;
}
$html = ums_str_get_html( $page_html );
if( empty( $html ) ){
ums_log_to_file('Failed to parse HTML for current chapter: ' . $chaps['name']);
break;
}
$chapid++;
preg_match_all('#<div class="cha-tit skiptranslate">\s*<h3>([^<]*?)<\/h3>#i', $page_html, $titli);
if(isset($titli[1][0]))
{
$chaps['name'] = $titli[1][0];
}
else
{
preg_match_all('#<li class="active">\s*?([^<]*?)\s*?<\/li>#i', $page_html, $titli);
if(isset($titli[1][0]))
{
$chaps['name'] = $titli[1][0];
}
}
preg_match_all('#<a\s*href="([^"]*?)"\s*class="btn next_page"#i', $page_html, $zurli);
if(isset($zurli[1][0]))
{
$dummy_chap = array(
'name' => 'Chapter ' . $chapid,
'url' => $zurli[1][0]
);
update_post_meta($existing_post_id, 'ums_latest_scraped', $zurli[1][0]);
}
else
{
$dummy_chap = false;
}
if($dummy_chap !== false)
{
$chapters = array($dummy_chap);
}
else
{
$chapters = array();
}
$slugified_name = $wp_manga_storage->slugify( $chaps['name'] );
$chapter_2 = $wp_manga_chapter->get_chapter_by_slug( $existing_post_id, $slugified_name );
if($chapter_2 && strtolower($chapter_2['chapter_slug']) == strtolower($slugified_name))
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Chapter is already published, skipping it: ' . $chapter_2['chapter_name']);
}
continue;
}
$content = '';
$doc = new DOMDocument;
$internalErrors = libxml_use_internal_errors(true);
$doc->loadHTML('<?xml encoding="utf-8" ?>' . $page_html);
libxml_use_internal_errors($internalErrors);
$xpath = new \DOMXpath($doc);
$articles = $xpath->query('//*[contains(@class, "text-left")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
break;
}
}
if(empty($content))
{
$articles = $xpath->query('//*[contains(@class, "content-body")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
}
}
}
if(empty($content))
{
$articles = $xpath->query('//*[contains(@class, "cha-tit")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
}
}
$articles = $xpath->query('//*[contains(@class, "cha-content")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
}
}
}
if(empty($content))
{
$ccontent = $html->find('.reading-content .text-left .cha-words .cha-words');
if($ccontent)
{
$content = '';
foreach($ccontent as $xct)
{
$other_ps = $xct->find('p');
if(count($other_ps) > 0)
{
$text = '';
foreach($other_ps as $p)
{
$text .= '<p> ' . $p->plaintext . '</p>';
}
$content .= $text . '<hr/>';
}
else
{
$content .= $xct->innertext . '<hr/>';
}
}
}
else
{
if($html->find('.reading-content #content', 0))
{
$content = $html->find('.reading-content #content', 0)->innertext;
}
else
{
$ccontent = $html->find('.reading-content .text-left .cha-content .cha-words');
if($ccontent)
{
$content = '';
foreach($ccontent as $xct)
{
$content .= $xct->innertext . '<hr/>';
}
}
else
{
$content = $html->find('.reading-content .text-left', 0);
if($content)
{
$other_ps = $content->find('p');
if(count($other_ps) > 0)
{
$text = '';
foreach($other_ps as $p)
{
$text .= '<p> ' . $p->plaintext . '</p>';
}
$content = $text;
}
else
{
if($inner_content = $content->find('#chapter-content', 0))
{
$content = $inner_content->innertext;
}
else
{
if($inner_content = $content->find('.text-left', 0))
{
$content = $inner_content->innertext;
}
else
{
if($inner_content = $content->find('.fr-view', 0))
{
$content = $inner_content->innertext;
}
else
{
$content = $content->innertext;
}
}
}
}
}
}
}
if(!$content)
{
$content = '';
$ps = $html->find('.reading-content .text-left p');
foreach($ps as $p)
{
$content .= '<p>' . $p->plaintext . '</p>';
}
}
}
}
$content = preg_replace('#(?:my)?[.]?b[.\s]?o[.\s]?x[.\s]?n[.\s]?o[.\s]?v[.\s]?e[.\s]?l[.]?(?:\s?.c[o0]m)?#i', get_bloginfo('name'), $content);
$qtags = array('iframe', 'script', 'ins');
foreach ($qtags as $qtag) {
$regex = '#<\s*' . $qtag . '[^>]*>.*?<\s*/\s*'. $qtag . '>#msi';
$content = preg_replace($regex, '', $content);
}
$content = preg_replace('#<style([\s\S]+?)<\/style>#i', '', $content);
$content = preg_replace('#<\/?div([^>]*)>#i', '', $content);
if($strip_images == '1')
{
$content = ums_sanitize_html_content($content);
}
$chapter_content = trim($content);
$chapter_content = ums_strip_links($chapter_content);
if($chapter_content == '')
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Skipping current chapter (because it failed to parse first chapter content): ' . print_r($chaps['url'], true));
}
//new_chap true to prevent infinite running
$new_chap = true;
break;
}
$arr = ums_spin_and_translate($chaps['name'], $chapter_content, $rule_translate, 'en');
if($arr === false)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Skipping current chapter (because it failed to be translated): ' . print_r($chaps['url'], true));
}
}
elseif($no_translate_title != '1')
{
$chaps['name'] = $arr[0];
$chaps['extend_name'] = $arr[0];
$chapter_content = $arr[1];
}
global $wp_manga_text_type;
if(empty($chaps['name']))
{
$chaps['name'] = 'Novel';
}
if($chaps['name'] == '')
{
$xext_name = $name_str . ' ' . $chaps['extend_name'];
}
else
{
$xext_name = '';
}
if($xext_name == '')
{
$chapsx = explode('-', $chaps['name']);
if(isset($chapsx[1]))
{
$xext_name = trim($chapsx[1]);
$chaps['name'] = trim($chapsx[0]);
}
}
$chapter_args = array(
'post_id' => $existing_post_id,
'chapter_name' => $chaps['name'],
'chapter_name_extend' => $xext_name,
'volume_id' => '',
'chapter_content' => $chapter_content,
);
$slugified_name = $wp_manga_storage->slugify( $chaps['name'] );
$chapter_2 = $wp_manga_chapter->get_chapter_by_slug( $existing_post_id, $slugified_name );
if($chapter_2 && strtolower($chapter_2['chapter_slug']) == strtolower($slugified_name))
{
ums_log_to_file('Chapter name already published, we will skip: ' . $chapter_2['chapter_name']);
continue;
}
else
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Now publishing: ' . print_r($chaps['name'], true));
}
}
$chapter_id = $wp_manga_text_type->insert_chapter( $chapter_args );
if( $chapter_id ){
if( is_wp_error( $chapter_id ) ){
ums_log_to_file('Failed to insert new chapter: ' . $chaps['name'] . ' error: ' . $chapter_id->get_error_message());
continue;
}
else
{
$new_chap = true;
$ums_imported_chapters++;
$local_imported++;
}
}
if (isset($ums_Main_Settings['request_timeout']) && $ums_Main_Settings['request_timeout'] != '') {
$timeout = intval($ums_Main_Settings['request_timeout']);
} else {
$timeout = 1;
}
sleep($timeout);
}
}
else
{
foreach($chapters as $chaps)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Processing novel chapter (no lazyload): ' . $chaps['url']);
}
if($ums_max_chapters <= $local_imported)
{
break;
}
$slugified_name = $wp_manga_storage->slugify( $chaps['name'] );
$chapter_2 = $wp_manga_chapter->get_chapter_by_slug( $existing_post_id, $slugified_name );
if($chapter_2 && strtolower($chapter_2['chapter_slug']) == strtolower($slugified_name))
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Chapter already published, skipping it: ' . $chapter_2['chapter_name']);
}
continue;
}
$page_html = ums_get_web_page($chaps['url'], ums_get_random_user_agent(), $use_phantom, $phantom_wait);
if( $page_html == false ){
ums_log_to_file('Failed to download HTML for chapter: ' . $chaps['name']);
continue;
}
$html = ums_str_get_html( $page_html );
if( empty( $html ) ){
ums_log_to_file('Failed to parse HTML for chapter: ' . $chaps['name']);
continue;
}
$content = '';
$doc = new DOMDocument;
$internalErrors = libxml_use_internal_errors(true);
$doc->loadHTML('<?xml encoding="utf-8" ?>' . $page_html);
libxml_use_internal_errors($internalErrors);
$xpath = new \DOMXpath($doc);
$articles = $xpath->query('//*[contains(@class, "text-left")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
break;
}
}
if(empty($content))
{
$articles = $xpath->query('//*[contains(@class, "content-body")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
}
}
}
if(empty($content))
{
$articles = $xpath->query('//*[contains(@class, "cha-tit")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
}
}
$articles = $xpath->query('//*[contains(@class, "cha-content")]');
if($articles !== false && count($articles) > 0)
{
foreach($articles as $container)
{
if(method_exists($container, 'saveHTML'))
{
$content .= ' ' . $container->saveHTML();
}
elseif(isset($container->ownerDocument) && method_exists($container->ownerDocument, 'saveHTML'))
{
$content .= ' ' . $container->ownerDocument->saveHTML($container);
}
elseif(isset($container->nodeValue))
{
$content .= ' ' . $container->nodeValue;
}
}
}
}
if(empty($content))
{
$content = $html->find('.reading-content .text-left .cha-words .cha-words', 0);
if($content)
{
$other_ps = $content->find('p');
if(count($other_ps) > 0)
{
$text = '';
foreach($other_ps as $p)
{
$text .= '<p> ' . $p->plaintext . '</p>';
}
$content = $text;
}
else
{
$content = $content->innertext;
}
}
else
{
if($html->find('.reading-content #content', 0))
{
$content = $html->find('.reading-content #content', 0)->innertext;
}
else
{
$content = $html->find('.reading-content .text-left .cha-content .cha-words', 0);
if($content)
{
$content = $content->innertext;
}
else
{
$content = $html->find('.reading-content .text-left', 0);
if($content)
{
$other_ps = $content->find('p');
if(count($other_ps) > 0)
{
$text = '';
foreach($other_ps as $p)
{
$text .= '<p> ' . $p->plaintext . '</p>';
}
$content = $text;
}
else
{
if($inner_content = $content->find('#chapter-content', 0))
{
$content = $inner_content->innertext;
}
else
{
if($inner_content = $content->find('.text-left', 0))
{
$content = $inner_content->innertext;
}
else
{
if($inner_content = $content->find('.fr-view', 0))
{
$content = $inner_content->innertext;
}
else
{
$content = $content->innertext;
}
}
}
}
}
}
}
if(!$content)
{
$content = '';
$ps = $html->find('.reading-content .text-left p');
foreach($ps as $p)
{
$content .= '<p>' . $p->plaintext . '</p>';
}
}
}
}
$content = preg_replace('#(?:my)?[.]?b[.\s]?o[.\s]?x[.\s]?n[.\s]?o[.\s]?v[.\s]?e[.\s]?l[.]?(?:\s?.c[o0]m)?#i', get_bloginfo('name'), $content);
$content = str_ireplace($domain, get_bloginfo('name'), $content);
$qtags = array('iframe', 'script', 'ins');
foreach ($qtags as $qtag) {
$regex = '#<\s*' . $qtag . '[^>]*>.*?<\s*/\s*'. $qtag . '>#msi';
$content = preg_replace($regex, '', $content);
}
$content = preg_replace('#<style([\s\S]+?)<\/style>#i', '', $content);
$content = preg_replace('#<\/?div([^>]*)>#i', '', $content);
if($strip_images == '1')
{
$content = ums_sanitize_html_content($content);
}
$chapter_content = trim($content);
$chapter_content = ums_strip_links($chapter_content);
$arr = ums_spin_and_translate($chaps['name'], $chapter_content, $rule_translate, 'en');
if($arr === false)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Skipping chapter (because it failed to be translated): ' . print_r($post_args, true));
}
}
elseif($no_translate_title != '1')
{
$chaps['name'] = $arr[0];
$chaps['extend_name'] = $arr[0];
$chapter_content = $arr[1];
}
global $wp_manga_text_type;
if(empty($chaps['name']))
{
$xext_name = $name_str . ' ' . $chaps['extend_name'];
$chaps['name'] = 'Novel';
}
else
{
$xext_name = '';
}
if($xext_name == '')
{
$chapsx = explode('-', $chaps['name']);
if(isset($chapsx[1]))
{
$xext_name = trim($chapsx[1]);
$chaps['name'] = trim($chapsx[0]);
}
}
$chapter_args = array(
'post_id' => $existing_post_id,
'chapter_name' => $chaps['name'],
'chapter_name_extend' => $xext_name,
'volume_id' => '',
'chapter_content' => $chapter_content,
);
$slugified_name = $wp_manga_storage->slugify( $chaps['name'] );
$chapter_2 = $wp_manga_chapter->get_chapter_by_slug( $existing_post_id, $slugified_name );
if($chapter_2 && strtolower($chapter_2['chapter_slug']) == strtolower($slugified_name))
{
ums_log_to_file('Chapter name already published, skipping it: ' . $chapter_2['chapter_name']);
continue;
}
$chapter_id = $wp_manga_text_type->insert_chapter( $chapter_args );
if( $chapter_id ){
if( is_wp_error( $chapter_id ) ){
ums_log_to_file('Failed to insert chapter: ' . $chaps['name'] . ' error: ' . $chapter_id->get_error_message());
continue;
}
else
{
$new_chap = true;
$ums_imported_chapters++;
$local_imported++;
}
}
if (isset($ums_Main_Settings['request_timeout']) && $ums_Main_Settings['request_timeout'] != '') {
$timeout = intval($ums_Main_Settings['request_timeout']);
} else {
$timeout = 1;
}
sleep($timeout);
}
}
if($new_chap == true)
{
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Total novel chapters scraped: ' . $local_imported);
}
$scraped_manga++;
}
}
}
catch(Exception $e)
{
ums_log_to_file('Importing failed: ' . $e->getMessage());
if($auto == 1)
{
ums_clearFromList($param, $type);
}
return 'fail';
}
}
}
elseif($type == 5)
{
if(trim($max_manga) != '')
{
$get_max_manga = intval(trim($max_manga));
}
else
{
$get_max_manga = 999;
}
$items = array();
$page_increased = false;
{
$manga_arr = array();
try
{
global $wp_filesystem;
if ( ! is_a( $wp_filesystem, 'WP_Filesystem_Base') ){
include_once(ABSPATH . 'wp-admin/includes/file.php');$creds = request_filesystem_credentials( site_url() );
wp_filesystem($creds);
}
$manga_names = explode(',', $manga_name);
$manga_names = array_map('trim', $manga_names);
$manga_arr = $manga_names;
if(count($manga_arr) == 0)
{
ums_log_to_file('No Manga matched your query: ' . $manga_name);
return 'nochange';
}
$scraped_manga = 0;
$user_name_type = $post_user_name;
foreach($manga_arr as $ind => $current_manga)
{
$parsedUrl = parse_url($current_manga);
if($parsedUrl === false)
{
ums_log_to_file('Cannot parse URL: ' . $current_manga);
continue;
}
if(!isset($parsedUrl['host']))
{
ums_log_to_file('Incorrect URL submitted: ' . $current_manga);
continue;
}
if(!isset($parsedUrl['path']) || empty($parsedUrl['path']))
{
ums_log_to_file('Incorrect URL format submitted: ' . $current_manga);
continue;
}
$domain = $parsedUrl['host'];
$path = trim($parsedUrl['path'], '/');
$pathParts = explode('/', $path);
$firstDirectory = isset($pathParts[0]) ? $pathParts[0] : '';
$html_site = false;
if($get_max_manga <= $scraped_manga)
{
break;
}
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Starting scraping for: (' . $ind . ') ' . $current_manga);
}
$is_url_search = true;
{
$html_site = ums_get_web_page($current_manga, ums_get_random_user_agent(), $use_phantom);
if($html_site == false)
{
ums_log_to_file('Failed to download manga pages: ' . $html_site);
usleep(rand(500000, 1000000));
continue;
}
else
{
if(strstr($html_site, '<html lang="en-US"><head><title>Just a moment... ') !== false)
{
ums_log_to_file('This page is protected by CloudFlare, please try scraping it using Puppeteer: ' . $current_manga);
usleep(rand(500000, 1000000));
continue;
}
preg_match_all('#]*?class="[^"]*?wp-manga-chapter[^"]*?"[^>]*?>[\s\n]*?]*?href="([^"]*?)"#i', $html_site, $azurli);
if(isset($azurli[1][0]))
{
$current_manga = end($azurli[1]);
}
else
{
preg_match_all('#]*?href="([^"]*?)"[^>]*?id="?btn-read-last"?#i', $html_site, $azurli);
if(isset($azurli[1][0]))
{
$current_manga = end($azurli[1]);
}
else
{
$is_url_search = false;
$possible = array('chapter-0', 'chapter-1', '0', 'n-a', '00', '1', '01');
$foundit = false;
foreach($possible as $pp)
{
$try_me = rtrim($current_manga, '/') . '/' . $pp;
$html_sitex = ums_get_web_page($try_me, ums_get_random_user_agent(), $use_phantom);
if($html_sitex == false)
{
ums_log_to_file('Failed to download guessing pages: ' . $html_sitex);
usleep(rand(500000, 1000000));
continue;
}
if(strstr($html_sitex, 'Just a moment... ') !== false)
{
ums_log_to_file('This page is protected by CloudFlare, please try scraping it using Puppeteer: ' . $try_me);
usleep(rand(500000, 1000000));
continue;
}
preg_match_all('#Just a moment... ') !== false)
{
ums_log_to_file('This page is protected by CloudFlare, please try scraping it using Puppeteer: ' . $current_manga);
usleep(rand(500000, 1000000));
continue;
}
}
require_once (dirname(__FILE__) . "/res/simple_html_dom.php");
$html = ums_str_get_html( $html_site );
$tag = $html->find( '.profile-manga .post-title h1', 0 );
$name_str = '';
$my_slug = '';
$my_slug = str_replace( 'http://', '', $current_manga );
$my_slug = str_replace( 'https://', '', $my_slug );
$my_slug = str_replace( 'http:', '', $my_slug );
$my_slug = str_replace( $domain . '/' . $firstDirectory . '/', '', $my_slug );
$my_slug = str_replace( '/', '', $my_slug );
$my_slug = str_replace( '.html', '', $my_slug );
$my_slug = str_replace( '.htm', '', $my_slug );
if($tag){
$name_str = trim($tag->plaintext);
$span = $tag->find('span', 0);
if($span){
$name_str = trim(str_replace($span->plaintext,'', $name_str));
}
}
if (isset($ums_Main_Settings['enable_detailed_logging'])) {
ums_log_to_file('Processing manga: ' . $name_str);
}
$find_post = new WP_Query( array(
'title' => $name_str,
'post_type' => 'wp-manga',
) );
$existing_post_id = false;
if( $find_post->have_posts() )
{
$existing_post_id = $find_post->posts[0]->ID;
}
if($existing_post_id == false)
{
$args = array(
'post_type' => 'wp-manga',
'meta_key' => '_manga_import_slug',
'meta_value' => $my_slug,
'post_status' => array('publish','draft','pending','trash','private','future')
);
$query = new WP_Query( $args );
if( $query->have_posts() ){
$existing_post_id = $query->posts[0]->ID;
}
}
if($existing_post_id == false)
{
$block_warning = $html->find('#page .warning', 0);
if( !empty( $block_warning ) && strpos( $block_warning->plaintext, 'has been licensed, it is not available' ) !== false ){
ums_log_to_file('Request blocked, please use a proxy!');
continue;
}
$desc = $html->find( '.description-summary' );
$desc = !empty( $desc ) ? trim($desc[0]->plaintext) : '';
$desc = str_replace($domain, '',$desc);
$thumb = '';
$thumb = $html->find( '.summary_image .img-responsive' );
$thumb = !empty( $thumb ) ? $thumb[0]->{'data-src'} : '';
if($thumb == '')
{
$thumb = !empty( $thumb ) ? $thumb[0]->src : '';
}
if($thumb == '')
{
$thumb = $html->find( '.summary_image' );
$thumb = !empty( $thumb ) ? $thumb[0]->src : '';
}
if($thumb == '')
{
preg_match_all('#