使用BeautifulSoup处理缺失元素：构建健壮的网页数据抓取教程

pip install requests beautifulsoup4 pandas numpy

import requests
import numpy as np
import pandas as pd
from bs4 import BeautifulSoup

# 目标URL
url = "https://www.comicshoplocator.com/StoreLocatorPremier?query=75077&showCsls=true"

# 发送请求并解析HTML
response = requests.get(url)
soup = BeautifulSoup(response.content, "html.parser")

# 用于存储所有商店数据的列表
all_data = []

for shop in soup.select(".CslsLocationItem"):
    # 1. 提取商店名称
    # 使用.select_one()确保只获取第一个匹配项
    name_tag = shop.select_one(".LocationName")
    name = name_tag.text if name_tag else np.nan # 确保元素存在

    # 2. 检查是否存在商店详情页链接
    profile_link_tag = shop.select_one(".LocationShopProfile a")
    website_url = np.nan # 默认网站链接为NaN

    if profile_link_tag:
        # 如果存在详情页链接，则构建完整URL并访问
        profile_page_url = "https://www.comicshoplocator.com" + profile_link_tag["href"]
        profile_response = requests.get(profile_page_url)
        profile_soup = BeautifulSoup(profile_response.content, "html.parser")

        # 3. 在详情页中提取网站链接
        web_link_tag = profile_soup.select_one(".StoreWeb a")
        if web_link_tag:
            website_url = web_link_tag["href"]
        # else: website_url 保持为 np.nan

    # 将名称和网站URL作为元组添加到列表中
    all_data.append((name, website_url))

# 使用抓取到的数据创建DataFrame
df = pd.DataFrame(all_data, columns=["Name", "Website"])

# 打印结果（以Markdown表格形式，便于查看）
print(df.to_markdown(index=False))

| Name                      | Website                          |
|:--------------------------|:---------------------------------|
| TWENTY ELEVEN COMICS      | http://WWW.TWENTYELEVENCOMICS.COM |
| READ COMICS               | nan                              |
| BOOMERANG COMICS          | http://www.boomerangcomics.com   |
| MORE FUN COMICS AND GAMES | http://www.facebook.com/morefuncomics |
| MADNESS COMICS & GAMES    | http://www.madnesscomicsandgames.com |
| SANCTUARY BOOKS AND GAMES | nan                              |

import requests
import numpy as np
import pandas as pd
from bs4 import BeautifulSoup

def scrape_comic_shops(query_zip_code="75077"):
    """
    抓取指定邮政编码区域的漫画店名称和网站。

    Args:
        query_zip_code (str): 查询的邮政编码。

    Returns:
        pd.DataFrame: 包含漫画店名称和网站的DataFrame。
                      如果网站不存在，则对应值为np.nan。
    """
    base_url = "https://www.comicshoplocator.com"
    search_url = f"{base_url}/StoreLocatorPremier?query={query_zip_code}&showCsls=true"

    try:
        response = requests.get(search_url)
        response.raise_for_status() # 检查HTTP请求是否成功
    except requests.exceptions.RequestException as e:
        print(f"请求主页面失败: {e}")
        return pd.DataFrame(columns=["Name", "Website"])

    soup = BeautifulSoup(response.content, "html.parser")
    all_data = []

    # 遍历每个商店的父容器
    for shop in soup.select(".CslsLocationItem"):
        name_tag = shop.select_one(".LocationName")
        name = name_tag.text.strip() if name_tag else np.nan

        profile_link_tag = shop.select_one(".LocationShopProfile a")
        website_url = np.nan

        if profile_link_tag and 'href' in profile_link_tag.attrs:
            profile_path = profile_link_tag["href"]
            profile_page_url = f"{base_url}{profile_path}"

            try:
                profile_response = requests.get(profile_page_url)
                profile_response.raise_for_status()
                profile_soup = BeautifulSoup(profile_response.content, "html.parser")

                web_link_tag = profile_soup.select_one(".StoreWeb a")
                if web_link_tag and 'href' in web_link_tag.attrs:
                    website_url = web_link_tag["href"].strip()
            except requests.exceptions.RequestException as e:
                print(f"请求商店详情页 {profile_page_url} 失败: {e}")
                # 如果请求失败，website_url 保持为 np.nan

        all_data.append((name, website_url))

    df = pd.DataFrame(all_data, columns=["Name", "Website"])
    return df

if __name__ == "__main__":
    df_result = scrape_comic_shops(query_zip_code="75077")
    print("抓取结果:")
    print(df_result.to_markdown(index=False))