File size: 3,846 Bytes
73e9c25 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 |
import time
import os
from urllib.parse import urlparse, parse_qs
from datetime import datetime
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
# Specify the download directory
download_path = '/Users/noamcohen/Downloads/data collection/data'
ROADS = {'405': ['N', 'S'], 101: ['N', 'S'], 5: ['N', 'S'], 110: ['N', 'S'], 170: ['N', 'S'], 118: ['E', 'W'],
134: ['E', 'W'], 605: ['N', 'S'], 210: ['E', 'W']}
# Set up Chrome options
chrome_options = Options()
chrome_prefs = {
"download.default_directory": download_path, # Change the download directory
"download.prompt_for_download": False, # Disable download prompt
"download.directory_upgrade": True, # Automatically upgrade download path
}
chrome_options.add_experimental_option("prefs", chrome_prefs)
# Set up ChromeDriver with the path to your chromedriver and options
driver = webdriver.Chrome(service=Service('/opt/homebrew/bin/chromedriver'), options=chrome_options)
# Continue with your script as normal...
# Open the login page
url = "https://pems.dot.ca.gov/"
driver.get(url)
# Wait for the page to load
time.sleep(20)
# Find the username and password fields
username_field = driver.find_element(By.ID, "username")
password_field = driver.find_element(By.ID, "password")
# Enter your credentials
username_field.send_keys("amitomer1912@gmail.com")
password_field.send_keys("5^applel?X")
# Find and click the login button (using the 'login' name attribute)
login_button = driver.find_element(By.NAME, "login")
login_button.click()
# Wait for login to complete
time.sleep(20)
i = 1
s_time_id=1740787200
while(i <= 1):
# Once logged in, navigate to the report page
if i <= 9:
day = f"0{i}"
else:
day = i
for road in ROADS.keys():
for dir in ROADS.get(road):
report_url = f"https://pems.dot.ca.gov/?report_form=1&dnode=Freeway&content=spatial&tab=contours&export=&fwy={road}&dir={dir}&s_time_id={s_time_id}&s_time_id_f=03%2F{day}%2F2025&from_hh=0&to_hh=23&start_pm=.0&end_pm=1000.09&lanes=&station_type=ml&q=speed&colormap=30%2C31%2C32&sc=auto&ymin=&ymax=&view_d=2&chart.x=93&chart.y=20"
driver.get(report_url)
time.sleep(60)
# Find the "Export XLS" button by its 'name' or 'alt' attribute
export_button = driver.find_element(By.NAME, "xls")
# Click the "Export XLS" button
export_button.click()
# Wait for the download to complete (you can increase or decrease this based on your network speed)
time.sleep(100)
# Extract highway number and date from the report URL
parsed_url = urlparse(report_url)
query_params = parse_qs(parsed_url.query)
highway_number = query_params.get('fwy', ['unknown'])[0] # Default to 'unknown' if not found
date_taken = query_params.get('s_time_id_f', ['unknown_date'])[0] # Default to 'unknown_date' if not found
# Construct the new file name
new_file_name = rf"{highway_number}_{dir}_{date_taken.replace("/", "*")}.xlsx"
new_file_path = os.path.join(download_path, new_file_name)
# Original file path (before renaming)
original_file_path = os.path.join(download_path, 'pems_output.xlsx')
# Rename the file if it exists
if os.path.exists(original_file_path):
os.rename(original_file_path, new_file_path)
print(f"File renamed to: {new_file_path}")
else:
print("Original file not found!")
print(f"Download {i} to March complete!")
i+=1
s_time_id += 86400
driver.quit()
|