Python爬虫进阶：高效获取动态网站数据的API抓取策略

import requests
import pandas as pd

s = requests.Session() # 使用Session可以保持会话状态和重用连接

headers = {
    'content-type': 'application/json',
    'accept-language': 'en-US,en;q=0.9',
    'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36"
}

# 请求体，根据实际API需求构建JSON字符串
payload = '{"proMerchantAds":false,"page":1,"rows":10,"payTypes":[],"countries":[],"publisherType":null,"asset":"USDT","fiat":"RUB","tradeType":"BUY"}'

url = 'https://p2p.binance.com/bapi/c2c/v2/friendly/c2c/adv/search'

s.headers.update(headers) # 更新Session的默认请求头
r = s.post(url, data=payload) # 发送POST请求，data参数接受字符串或字典

# 检查请求是否成功
if r.status_code == 200:
    # 将JSON响应数据展平为Pandas DataFrame
    df = pd.json_normalize(r.json()['data'])
    print(df.head()) # 打印DataFrame的前几行
else:
    print(f"请求失败，状态码: {r.status_code}")
    print(r.text)

import requests
import pandas as pd

# 初始化一个请求会话，可以自动处理cookies和连接池
s = requests.Session()

# 模拟浏览器请求头
headers = {
    'content-type': 'application/json',
    'accept-language': 'en-US,en;q=0.9',
    'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36"
}

# API接口URL
url = 'https://p2p.binance.com/bapi/c2c/v2/friendly/c2c/adv/search'

# 请求体（Payload），以JSON字符串形式提供
# 示例：查询USDT/RUB的购买广告，第一页，每页10行
payload = '{"proMerchantAds":false,"page":1,"rows":10,"payTypes":[],"countries":[],"publisherType":null,"asset":"USDT","fiat":"RUB","tradeType":"BUY"}'

# 更新会话的请求头
s.headers.update(headers)

try:
    # 发送POST请求
    response = s.post(url, data=payload)
    response.raise_for_status() # 检查HTTP请求是否成功 (2xx状态码)

    # 解析JSON响应
    json_data = response.json()

    # 检查'data'键是否存在且不为空
    if 'data' in json_data and json_data['data']:
        # 使用pandas.json_normalize将嵌套的JSON数据展平为DataFrame
        df = pd.json_normalize(json_data['data'])
        print("成功获取数据并转换为DataFrame:")
        print(df.head()) # 打印DataFrame的前5行
        print(f"\nDataFrame的形状: {df.shape}")
        print(f"DataFrame的列名: {df.columns.tolist()}")
    else:
        print("API响应中没有找到'data'键或数据为空。")
        print(json_data) # 打印完整的JSON响应以供调试

except requests.exceptions.RequestException as e:
    print(f"请求过程中发生错误: {e}")
except ValueError as e:
    print(f"解析JSON响应时发生错误: {e}")
except KeyError as e:
    print(f"JSON响应结构不符合预期，缺少键: {e}")