DynamoDB海量数据高效查询策略

import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import software.amazon.awssdk.services.dynamodb.model.QueryRequest;
import software.amazon.awssdk.services.dynamodb.model.QueryResponse;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class DynamoDBLargeDataFetcher {

    private final DynamoDbClient dynamoDbClient;
    private final String tableName;

    public DynamoDBLargeDataFetcher(DynamoDbClient dynamoDbClient, String tableName) {
        this.dynamoDbClient = dynamoDbClient;
        this.tableName = tableName;
    }

    /**
     * 示例：从DynamoDB分批查询所有满足条件的乘客数据
     * 注意：此方法用于演示服务器端分页，不建议直接暴露给API消费者返回海量数据。
     * 适用于内部数据处理或导出场景。
     *
     * @param partitionKeyValue 分区键值，例如航空公司名称
     * @param sortKeyCondition 排序键条件，例如预订日期范围或舱位
     * @return 满足条件的所有乘客数据列表
     */
    public List<Map<String, AttributeValue>> fetchAllPassengers(String partitionKeyValue, String sortKeyCondition) {
        List<Map<String, AttributeValue>> allItems = new ArrayList<>();
        Map<String, AttributeValue> lastEvaluatedKey = null;

        do {
            Map<String, AttributeValue> expressionAttributeValues = new HashMap<>();
            expressionAttributeValues.put(":pkVal", AttributeValue.builder().s(partitionKeyValue).build());
            // 假设sortKeyCondition是一个简单的字符串匹配，实际可能更复杂
            expressionAttributeValues.put(":skVal", AttributeValue.builder().s(sortKeyCondition).build());

            QueryRequest.Builder requestBuilder = QueryRequest.builder()
                    .tableName(tableName)
                    .keyConditionExpression("airlineName = :pkVal AND bookingClass = :skVal") // 示例条件
                    .expressionAttributeValues(expressionAttributeValues)
                    .limit(1000); // 每次请求的数据量，可根据需求调整，但仍受1MB限制

            if (lastEvaluatedKey != null) {
                requestBuilder.exclusiveStartKey(lastEvaluatedKey);
            }

            QueryResponse response = dynamoDbClient.query(requestBuilder.build());
            allItems.addAll(response.items());
            lastEvaluatedKey = response.lastEvaluatedKey();

            System.out.println("Fetched " + response.items().size() + " items. Total so far: " + allItems.size());

        } while (lastEvaluatedKey != null && !lastEvaluatedKey.isEmpty());

        return allItems;
    }

    // 实际使用时，可能需要将AttributeValue映射到Java对象
    // 例如使用DynamoDBMapper
}