从API正确解析Apache Parquet数据的实践指南

import requests
import io
import pyarrow.parquet as pq
import pandas as pd

def get_orders_data_incorrect(date):
    # 假设这是一个API接口，实际url需要替换
    url = "http://your-api-endpoint/orders"
    params = {"date": date}
    response = requests.get(url, params=params)

    if response.status_code == 200:
        # 错误示范：将二进制内容解码为字符串
        # 这会破坏Parquet文件的二进制结构
        data_str = response.text.strip() 
        return data_str
    else:
        print(f"Failed to fetch orders data: {response.status_code}")
        return None

# 调用函数获取数据（假设API返回Parquet）
date_to_fetch = "2023-12-08"
orders_info_str = get_orders_data_incorrect(date_to_fetch)

if orders_info_str:
    try:
        # 错误示范：尝试将已损坏的字符串重新编码为字节流
        # 原始二进制信息已丢失
        buffer = io.BytesIO(orders_info_str.encode())
        table = pq.read_table(buffer) # 这里会抛出错误
        df = table.to_pandas()
        print(df.head())
    except Exception as e:
        print(f"解码Parquet数据时发生错误: {e}")
        # 错误信息可能类似：'Parquet format error: Invalid Parquet file'
        # 或 'pyarrow.lib.ArrowInvalid: Parquet magic bytes not found'

import requests
import io
import pandas as pd # 确保安装了pandas和pyarrow/fastparquet

def get_orders_data_pandas(date: str) -> pd.DataFrame | None:
    # 假设这是一个API接口，实际url需要替换
    url = "http://your-api-endpoint/orders" 
    params = {"date": date}

    try:
        response = requests.get(url, params=params)
        response.raise_for_status() # 检查HTTP请求是否成功（状态码2xx）

        # 关键步骤：直接使用 response.content 获取原始字节流
        # 并通过 io.BytesIO 封装成文件对象
        df = pd.read_parquet(io.BytesIO(response.content))
        return df
    except requests.exceptions.RequestException as e:
        print(f"API请求失败: {e}")
        return None
    except Exception as e:
        print(f"解码Parquet数据时发生错误: {e}")
        return None

# 完整示例：
date_to_fetch = "2023-12-08"
orders_df = get_orders_data_pandas(date_to_fetch)

if orders_df is not None:
    print("成功获取并解码Parquet数据，前5行如下：")
    print(orders_df.head())
    # 进一步处理 orders_df ...
else:
    print("未能获取或解码订单数据。")

import requests
import io
import pyarrow.parquet as pq
import pandas as pd

def get_orders_data_pyarrow(date: str) -> pd.DataFrame | None:
    # 假设这是一个API接口，实际url需要替换
    url = "http://your-api-endpoint/orders"
    params = {"date": date}

    try:
        response = requests.get(url, params=params)
        response.raise_for_status() # 检查HTTP请求是否成功（状态码2xx）

        # 关键步骤：直接使用 response.content 获取原始字节流
        buffer = io.BytesIO(response.content)

        # 使用 pyarrow.parquet.read_table 读取数据为 PyArrow Table
        table = pq.read_table(buffer)

        # 将 PyArrow Table 转换为 Pandas DataFrame
        df = table.to_pandas()
        return df
    except requests.exceptions.RequestException as e:
        print(f"API请求失败: {e}")
        return None
    except Exception as e:
        print(f"解码Parquet数据时发生错误: {e}")
        return None

# 完整示例：
date_to_fetch = "2023-12-08"
orders_df_pyarrow = get_orders_data_pyarrow(date_to_fetch)

if orders_df_pyarrow is not None:
    print("成功获取并解码Parquet数据（PyArrow方法），前5行如下：")
    print(orders_df_pyarrow.head())
    # 进一步处理 orders_df_pyarrow ...
else:
    print("未能获取或解码订单数据。")