使用Selenium从Google Maps提取地点评分与评论数据教程

pip install selenium

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
import time
import re # 用于解析评分和评论数量

# 配置Chrome选项
chrome_options = Options()
chrome_options.add_experimental_option("detach", True) # 保持浏览器开启

# 初始化WebDriver
driver = webdriver.Chrome(options=chrome_options)
actionChains = ActionChains(driver) # 用于执行复杂的用户交互，如滚动
wait = WebDriverWait(driver, 20) # 显式等待，最长等待20秒

def wait_for_element_location_to_be_stable(element):
    """
    等待给定元素的位置稳定，避免因元素位置变化导致的StaleElementReferenceException。
    """
    initial_location = element.location
    previous_location = initial_location
    start_time = time.time()
    while time.time() - start_time < 1: # 在1秒内位置没有变化则认为稳定
        current_location = element.location
        if current_location != previous_location:
            previous_location = current_location
            start_time = time.time() # 位置变化，重置计时器
        time.sleep(0.4) # 短暂等待

# 访问Google主页并同意Cookies（如果出现）
driver.get("https://www.google.com/")
try:
    # 尝试查找并点击同意Cookies按钮，此ID可能因地区而异
    wait.until(EC.element_to_be_clickable((By.ID, "L2AGLb"))).click()
except:
    print("未找到或无需点击Cookies同意按钮。")

# 访问Google Maps
driver.get("https://www.google.com/maps")

# 等待页面加载并找到搜索框
time.sleep(3) # 简单等待，可替换为显式等待
search_box = wait.until(EC.presence_of_element_located((By.ID, "searchboxinput")))

# 输入搜索查询并回车
search_box.send_keys("jardins in toulouse")
search_box.send_keys(Keys.RETURN)

# 等待搜索结果加载
time.sleep(5) # 简单等待，可替换为更精确的等待条件

# 定位第一个结果列表中的元素，用于滚动操作
# hfpxzc 是每个地点结果卡片的链接元素的class
results = wait.until(EC.presence_of_all_elements_located((By.XPATH, "//a[@class='hfpxzc']")))

# 模拟滚动以加载更多结果
break_condition = False
# 找到一个可以作为滚动焦点的元素，通常是侧边栏的某个固定元素
focus_element = driver.find_element(By.ID, 'zero-input') # 这是一个常见的Google Maps侧边栏输入框ID

while not break_condition:
    # 记录当前列表中的最后一个元素
    temp_last_element = results[-1]

    # 滚动到最后一个元素
    actionChains.scroll_to_element(temp_last_element).perform()

    # 模拟点击焦点元素，确保滚动条激活
    actionChains.move_to_element(focus_element).click().perform()

    # 模拟按下几次向下箭头键，确保滚动发生
    for _ in range(3):
        actionChains.send_keys(Keys.ARROW_DOWN).perform()
        time.sleep(0.5)

    # 等待最后一个元素的位置稳定，确认滚动已完成
    wait_for_element_location_to_be_stable(temp_last_element)

    # 重新获取所有结果元素
    results = wait.until(EC.presence_of_all_elements_located((By.XPATH, "//a[@class='hfpxzc']")))

    # 如果新的结果列表的最后一个元素与之前的最后一个元素相同，说明已滚动到底部，没有更多新内容加载
    if results[-1] == temp_last_element:
        break_condition = True

# 循环遍历前100个结果（或所有已加载的结果）
for i, result in enumerate(results[:100], start=1):
    name = result.get_attribute('aria-label') # 获取地点的名称

    # 使用相对XPath定位评分和评论元素
    # ratings_elements 是一个列表，因为 find_elements 返回所有匹配的元素
    ratings_elements = result.find_elements(By.XPATH, "..//*[@class='MW4etd']")

    rating_value = "N/A"
    review_count = "N/A"

    if len(ratings_elements) > 0:
        full_text = ratings_elements[0].text # 获取包含评分和评论的完整文本

        # 使用正则表达式从文本中提取评分和评论数量
        # 模式解释:
        # (\d+\.?\d*) : 匹配一个或多个数字，可选的小数点和数字 (例如 4.1, 5) - 这是评分
        # \s* : 匹配零个或多个空格
        # \( : 匹配开括号 (需要转义)
        # (\d{1,3}(?:,\d{3})*) : 匹配数字，可能包含逗号 (例如 1,234) - 这是评论数量
        # \) : 匹配闭括号 (需要转义)
        match = re.search(r'(\d+\.?\d*)\s*\((\d{1,3}(?:,\d{3})*)\)', full_text)

        if match:
            rating_value = float(match.group(1)) # 提取评分
            # 提取评论数量，并移除逗号以便转换为整数
            review_count = int(match.group(2).replace(',', '')) 
        else:
            # 如果没有找到 (评分) 格式，尝试只提取评分数字
            single_rating_match = re.search(r'(\d+\.?\d*)', full_text)
            if single_rating_match:
                rating_value = float(single_rating_match.group(1))

    print(f"Result {i}: {name} - Rating: {rating_value} - Reviews: {review_count}")

# 关闭浏览器
driver.quit()

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
import time
import re

# --- 配置与初始化 ---
chrome_options = Options()
chrome_options.add_experimental_option("detach", True) # 保持浏览器开启，方便调试

driver = webdriver.Chrome(options=chrome_options)
actionChains = ActionChains(driver)
wait = WebDriverWait(driver, 20)

# --- 辅助函数 ---
def wait_for_element_location_to_be_stable(element):
    """
    等待给定元素的位置稳定，避免因元素位置变化导致的StaleElementReferenceException。
    """
    initial_location = element.location
    previous_location = initial_location
    start_time = time.time()
    while time.time() - start_time < 1:
        current_location = element.location
        if current_location != previous_location:
            previous_location = current_location
            start_time = time.time()
        time.sleep(0.4)

# --- 导航与搜索 ---
driver.get("https://www.google.com/")
try:
    wait.until(EC.element_to_be_clickable((By.ID, "L2AGLb"))).click()
except:
    print("未找到或无需点击Cookies同意按钮。")

driver.get("https://www.google.com/maps")
time.sleep(3) # 等待页面加载
search_box = wait.until(EC.presence_of_element_located((By.ID, "searchboxinput")))
search_box.send_keys("jardins in toulouse")
search_box.send_keys(Keys.RETURN)
time.sleep(5) # 等待搜索结果加载

# --- 动态加载与滚动 ---
results = wait.until(EC.presence_of_all_elements_located((By.XPATH, "//a[@class='hfpxzc']")))

break_condition = False
focus_element = driver.find_element(By.ID, 'zero-input') # 侧边栏的某个固定元素ID

while not break_condition:
    temp_last_element = results[-1]
    actionChains.scroll_to_element(temp_last_element).perform()
    actionChains.move_to_element(focus_element).click().perform()
    for _ in range(3):
        actionChains.send_keys(Keys.ARROW_DOWN).perform()
        time.sleep(0.5)
    wait_for_element_location_to_be_stable(temp_last_element)

    results = wait.until(EC.presence_of_all_elements_located((By.XPATH, "//a[@class='hfpxzc']")))
    if results[-1] == temp_last_element:
        break_condition = True

# --- 提取数据 ---
# 循环遍历前100个结果（或所有已加载的结果）
for i, result in enumerate(results[:100], start=1):
    name = result.get_attribute('aria-label')
    ratings_elements = result.find_elements(By.XPATH, "..//*[@class='MW4etd']")

    rating_value = "N/A"
    review_count = "N/A"

    if len(ratings_elements) > 0:
        full_text = ratings_elements[0].text
        match = re.search(r'(\d+\.?\d*)\s*\((\d{1,3}(?:,\d{3})*)\)', full_text)

        if match:
            rating_value = float(match.group(1))
            review_count = int(match.group(2).replace(',', '')) 
        else:
            single_rating_match = re.search(r'(\d+\.?\d*)', full_text)
            if single_rating_match:
                rating_value = float(single_rating_match.group(1))

    print(f"Result {i}: {name} - Rating: {rating_value} - Reviews: {review_count}")

# --- 清理 ---
driver.quit()