掌握Selenium与BeautifulSoup协同抓取动态加载数据

<div class="full-date">
 <span class="day-of-week">{{ WMService.auctionStartDate(lot.auction) | moment:'dddd' }}</span> |
 <span class="month">{{ WMService.auctionStartDate(lot.auction) | moment:'MMMM' }}</span>
 <span class="date">{{ WMService.auctionStartDate(lot.auction) | moment:'DD' }}, </span>
 <span class="year">{{ WMService.auctionStartDate(lot.auction) | moment:'YYYY' }}</span>
</div>

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager

# 初始化Chrome WebDriver
# 使用webdriver_manager自动管理驱动
service = Service(ChromeDriverManager().install())
driver = webdriver.Chrome(service=service)

# 访问目标URL
url = "你的目标网页URL" # 请替换为实际的URL
driver.get(url)

try:
    # 设置最大等待时间为20秒
    wait = WebDriverWait(driver, 20)

    # 等待并获取月份信息
    month_element = wait.until(EC.element_to_be_clickable((By.XPATH, "//span[@class='month ng-binding']")))
    month = month_element.text
    print(f"月份: {month}")

    # 等待并获取日期信息
    date_element = wait.until(EC.element_to_be_clickable((By.XPATH, "//span[@class='date ng-binding']")))
    date = date_element.text.strip(', ') # 清除可能的逗号
    print(f"日期: {date}")

    # 等待并获取年份信息
    year_element = wait.until(EC.element_to_be_clickable((By.XPATH, "//span[@class='year ng-binding']")))
    year = year_element.text
    print(f"年份: {year}")

    print(f"完整日期: {month} {date}, {year}")

except Exception as e:
    print(f"抓取过程中发生错误: {e}")

finally:
    # 关闭浏览器
    driver.quit()

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
import time

def scrape_dynamic_date(url):
    """
    从动态加载的网页中抓取日期信息。

    Args:
        url (str): 目标网页的URL。

    Returns:
        tuple: 包含月份、日期、年份的元组，如果失败则返回None。
    """
    driver = None
    try:
        # 配置Chrome浏览器为无头模式（不显示浏览器界面，提高效率）
        chrome_options = webdriver.ChromeOptions()
        chrome_options.add_argument("--headless")
        chrome_options.add_argument("--disable-gpu") # 某些Linux系统需要
        chrome_options.add_argument("--window-size=1920,1080") # 设置窗口大小，避免某些元素在小窗口不加载
        chrome_options.add_argument("--no-sandbox") # Docker等环境可能需要

        # 初始化Chrome WebDriver
        service = Service(ChromeDriverManager().install())
        driver = webdriver.Chrome(service=service, options=chrome_options)
        driver.get(url)

        # 设置显式等待对象
        wait = WebDriverWait(driver, 20) # 最长等待20秒

        print(f"正在访问: {url}")
        print("等待动态内容加载...")

        # 等待并获取月份信息
        month_element = wait.until(EC.element_to_be_clickable((By.XPATH, "//span[@class='month ng-binding']")))
        month = month_element.text

        # 等待并获取日期信息
        date_element = wait.until(EC.element_to_be_clickable((By.XPATH, "//span[@class='date ng-binding']")))
        date = date_element.text.strip(', ') # 清除可能的逗号

        # 等待并获取年份信息
        year_element = wait.until(EC.element_to_be_clickable((By.XPATH, "//span[@class='year ng-binding']")))
        year = year_element.text

        print("数据抓取成功！")
        print(f"月份: {month}")
        print(f"日期: {date}")
        print(f"年份: {year}")

        return month, date, year

    except Exception as e:
        print(f"抓取过程中发生错误: {e}")
        return None
    finally:
        if driver:
            driver.quit() # 确保关闭浏览器

if __name__ == "__main__":
    # 替换为你的实际目标URL
    target_url = "http://example.com" # 请务必替换为一个包含动态日期信息的实际URL进行测试

    # 假设你的目标页面URL是这样的，但你需要替换为实际可用的URL
    # 例如，如果有一个页面在加载后才显示日期，你可以尝试用它来测试
    # target_url = "https://www.some-dynamic-website.com/auction-details" 

    date_data = scrape_dynamic_date(target_url)
    if date_data:
        print(f"\n抓取到的完整日期: {date_data[0]} {date_data[1]}, {date_data[2]}")
    else:
        print("\n未能成功抓取日期信息。")