使用 Python 进行网页数据抓取：基础教程与最佳实践

pip install requests beautifulsoup4

import requests

url = 'https://example.com'
response = requests.get(url)

if response.status_code == 200:
    print("请求成功！")
else:
    print(f"请求失败，状态码：{response.status_code}")

from bs4 import BeautifulSoup

soup = BeautifulSoup(response.text, 'html.parser')

# 提取所有链接
links = soup.find_all('a')
for link in links:
    print(link.get('href'))

# 提取标题
title = soup.find('title').text
print(f"网页标题：{title}")

import requests
from bs4 import BeautifulSoup

def scrape_website(url):
    try:
        response = requests.get(url)
        response.raise_for_status()  # 检查请求是否成功

        soup = BeautifulSoup(response.text, 'html.parser')

        # 提取所有链接
        links = soup.find_all('a')
        print("链接：")
        for link in links:
            print(link.get('href'))

        # 提取标题
        title = soup.find('title').text
        print(f"\n网页标题：{title}")

    except requests.exceptions.RequestException as e:
        print(f"请求错误：{e}")
    except Exception as e:
        print(f"解析错误：{e}")

# 示例用法
url_to_scrape = 'https://example.com'
scrape_website(url_to_scrape)

pip install google-cloud-language

from google.cloud import language_v1

def analyze_entities(text):
    client = language_v1.LanguageServiceClient()
    document = language_v1.Document(content=text, type_=language_v1.Document.Type.PLAIN_TEXT)
    response = client.analyze_entities(document=document)

    entities = response.entities
    for entity in entities:
        print(f"实体名称：{entity.name}")
        print(f"实体类型：{language_v1.Entity.Type(entity.type_).name}")
        print(f"置信度：{entity.salience}")
        print("-" * 20)

# 示例用法
text_to_analyze = "Google, headquartered in Mountain View, unveiled the new Android phone at a conference. Sundar Pichai spoke."
analyze_entities(text_to_analyze)

headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}
response = requests.get(url, headers=headers)

import time

time.sleep(1)  # 延迟 1 秒