深入解析：NumPy数组与Python列表存储大小差异及优化策略

import random
import pickle
import sys

# 假设 all_games 包含一些7元素的列表
# 为了演示，我们创建一个简单的 all_games
all_games = [[float(i), float(i+1), float(i+2), float(i+3), float(i+4), float(i+5), float(i+6)] for i in range(100)]

def sample_and_pickle_list(all_games, file_name, DRAW=10000, SAMPLE=10000):
    print(f"开始生成Python列表 (SAMPLE={SAMPLE}, DRAW={DRAW})...")
    # sampled_data 存储的是对 all_games 中现有子列表的引用
    sampled_data = [[random.choice(all_games) for _ in range(DRAW)] for _ in range(SAMPLE)]

    print(f"生成的列表内存大小估算 (sys.getsizeof): {sys.getsizeof(sampled_data) / (1024**2):.2f} MB (仅顶层列表)")

    with open(file_name, 'wb') as file:
        pickle.dump(sampled_data, file)
    print(f"Python列表已保存到 {file_name}，文件大小可能远小于NumPy数组。")

# 调用函数进行演示
# sample_and_pickle_list(all_games, 'sampled_list.pkl')

import numpy as np
import random
import os

# 假设 all_games 包含一些7元素的列表
# 为了演示，我们创建一个简单的 all_games
all_games_list = [[float(i), float(i+1), float(i+2), float(i+3), float(i+4), float(i+5), float(i+6)] for i in range(100)]
# 将 all_games 转换为 NumPy 数组以便后续采样
all_games_np = np.array(all_games_list, dtype=np.float16)


def sample_and_save_numpy(all_games_source, file_name, DRAW=10000, SAMPLE=10000):
    print(f"开始生成NumPy数组 (SAMPLE={SAMPLE}, DRAW={DRAW})...")
    # 生成随机索引
    rng = np.random.default_rng()
    sampled_indices = rng.choice(all_games_source.shape[0], size=(SAMPLE, DRAW), replace=True)
    # 根据索引获取采样数据
    sampled_data_np = all_games_source[sampled_indices]

    print(f"生成的NumPy数组形状: {sampled_data_np.shape}, dtype: {sampled_data_np.dtype}")
    print(f"NumPy数组原始内存大小估算: {sampled_data_np.nbytes / (1024**3):.2f} GB")

    # 1. 不压缩保存 (对应原始问题中的1.4GB情况)
    uncompressed_file_name = file_name.replace('.npz', '_uncompressed.npy')
    np.save(uncompressed_file_name, sampled_data_np)
    print(f"NumPy数组未压缩保存到 {uncompressed_file_name}，文件大小: {os.path.getsize(uncompressed_file_name) / (1024**3):.2f} GB")

    # 2. 压缩保存
    compressed_file_name = file_name
    np.savez_compressed(compressed_file_name, data=sampled_data_np)
    print(f"NumPy数组压缩保存到 {compressed_file_name}，文件大小: {os.path.getsize(compressed_file_name) / (1024**2):.2f} MB")

# 调用函数进行演示
# sample_and_save_numpy(all_games_np, 'sampled_numpy_compressed.npz')