Python实现HTML链接的迭代抓取与跟踪

pip install beautifulsoup4

import urllib.request, urllib.parse, urllib.error
from urllib.parse import urljoin
from bs4 import BeautifulSoup

# blanc list - 列表在外部定义
l = []

# starting url
url = input('Enter URL: ')
if len(url) < 1:
    url = 'http://py4e-data.dr-chuck.net/known_by_Fikret.html'

# loop for 4 iterations
for _ in range(4):
    html = urllib.request.urlopen(url).read()    # open url
    soup = BeautifulSoup(html, 'html.parser')    # parse through BeautifulSoup
    tags = soup('a')    # extract tags

    for tag in tags:
        # 链接提取和URL更新都在内层循环中
        url = tag.get('href', None)    # extract links from tags
        l.append(url)    # add the links to a list
        url = l[2:3]    # slice the list to extract the 3rd url
        url = ' '.join(str(e) for e in url)    # change the type to string
    print(url)

http://py4e-data.dr-chuck.net/known_by_Montgomery.html
http://py4e-data.dr-chuck.net/known_by_Montgomery.html
http://py4e-data.dr-chuck.net/known_by_Montgomery.html
http://py4e-data.dr-chuck.net/known_by_Montgomery.html

import urllib.request, urllib.parse, urllib.error
from urllib.parse import urljoin # 导入urljoin，用于处理相对URL
from bs4 import BeautifulSoup

# starting url
url = input('Enter URL: ')
if len(url) < 1:
    url = 'http://py4e-data.dr-chuck.net/known_by_Fikret.html'

# loop for 4 iterations
for i in range(4): # 使用i来表示当前是第几次迭代
    # 每次外层循环开始时，清空链接列表，确保只收集当前页面的链接
    l = [] 

    print(f"--- 访问第 {i+1} 次，当前URL: {url} ---")
    try:
        html = urllib.request.urlopen(url).read()    # open url
        soup = BeautifulSoup(html, 'html.parser')    # parse through BeautifulSoup
        tags = soup('a')    # extract tags

        # 遍历所有链接，将它们添加到当前页面的链接列表中
        for tag in tags:
            href = tag.get('href', None)
            if href: # 确保链接存在
                # 使用urljoin处理相对URL，生成绝对URL
                absolute_url = urljoin(url, href) 
                l.append(absolute_url)

        # 确保有足够的链接来选择第三个链接
        if len(l) > 2:
            # 更新url变量为下一个要访问的链接（列表的第3个元素，索引为2）
            # 这会在内层循环结束后执行，确保url被正确赋值给下一个迭代
            url = l[2] 
            print(f"找到第3个链接: {url}")
        else:
            print("当前页面链接不足3个，无法继续跟踪。")
            break # 退出循环

    except Exception as e:
        print(f"访问URL {url} 时发生错误: {e}")
        break # 发生错误时退出循环

print("\n--- 迭代抓取完成 ---")

--- 访问第 1 次，当前URL: http://py4e-data.dr-chuck.net/known_by_Fikret.html ---
找到第3个链接: http://py4e-data.dr-chuck.net/known_by_Montgomery.html
--- 访问第 2 次，当前URL: http://py4e-data.dr-chuck.net/known_by_Montgomery.html ---
找到第3个链接: http://py4e-data.dr-chuck.net/known_by_Mhairade.html
--- 访问第 3 次，当前URL: http://py4e-data.dr-chuck.net/known_by_Mhairade.html ---
找到第3个链接: http://py4e-data.dr-chuck.net/known_by_Butchi.html
--- 访问第 4 次，当前URL: http://py4e-data.dr-chuck.net/known_by_Butchi.html ---
找到第3个链接: http://py4e-data.dr-chuck.net/known_by_Anayah.html

--- 迭代抓取完成 ---