优化Q-learning：解决FrozenLake环境中Q表不更新的常见问题

import gymnasium as gym
import numpy as np
import random

def run_problematic_q_learning():
    env = gym.make("FrozenLake-v1")
    Q = np.zeros((env.observation_space.n, env.action_space.n)) # 初始化Q表为全零

    alpha = 0.7  # 学习率
    gamma = 0.95 # 折扣因子
    epsilon = 0.9 # 探索率
    epsilon_decay = 0.005 # epsilon衰减率
    epsilon_min = 0.01 # 最小epsilon
    episodes = 10000 # 训练回合数

    print("训练前Q表:")
    print(Q)

    for episode in range(episodes):
        state, info = env.reset()
        terminated = False
        truncated = False

        while not terminated and not truncated:
            # Epsilon-greedy策略选择动作
            if random.random() < epsilon:
                action = env.action_space.sample() # 探索
            else:
                action = np.argmax(Q[state]) # 利用

            new_state, reward, terminated, truncated, info = env.step(action)

            # Q表更新公式
            Q[state, action] = Q[state, action] + alpha * (float(reward) + gamma * np.max(Q[new_state]) - Q[state, action])

            state = new_state

        # 衰减epsilon
        if epsilon > epsilon_min:
            epsilon -= epsilon_decay

    print("训练后Q表:")
    print(Q)
    env.close()

run_problematic_q_learning()

def argmax_random_tie_breaking(arr):
    arr_max = np.max(arr)
    # 找到所有等于最大值的索引
    max_indices = np.where(arr == arr_max)[0]
    # 从这些索引中随机选择一个
    return np.random.choice(max_indices)

epsilon = 1.0 # 初始epsilon可以设为1，确保初期完全探索
# 衰减率计算：确保epsilon在episodes / 2时达到epsilon_min
epsilon_decay = (epsilon - epsilon_min) / (episodes / 2) 
epsilon_min = 0.001 # 最小epsilon可以更小

import gymnasium as gym
import numpy as np
import random

# 改进的argmax函数，用于随机打破平局
def argmax_random_tie_breaking(arr):
    arr_max = np.max(arr)
    max_indices = np.where(arr == arr_max)[0]
    return np.random.choice(max_indices)

def run_optimized_q_learning():
    env = gym.make("FrozenLake-v1")
    Q = np.zeros((env.observation_space.n, env.action_space.n))

    alpha = 0.7
    gamma = 0.95

    # 优化后的epsilon衰减参数
    epsilon = 1.0 # 初始探索率为1
    epsilon_min = 0.001 # 最小探索率
    episodes = 10000
    # 确保epsilon在大约一半的训练回合中衰减到epsilon_min
    epsilon_decay = (epsilon - epsilon_min) / (episodes / 2) 

    print("训练前Q表:")
    print(Q)

    for episode in range(episodes):
        state, info = env.reset()
        terminated = False
        truncated = False

        while not terminated and not truncated:
            if random.random() < epsilon:
                action = env.action_space.sample() # 探索
            else:
                # 使用改进的argmax函数进行动作选择
                action = argmax_random_tie_breaking(Q[state]) # 利用

            new_state, reward, terminated, truncated, info = env.step(action)

            # Q表更新公式
            Q[state, action] = Q[state, action] + alpha * (float(reward) + gamma * np.max(Q[new_state]) - Q[state, action])

            state = new_state

        # 衰减epsilon
        if epsilon > epsilon_min:
            epsilon -= epsilon_decay

    print("训练后Q表:")
    print(Q)
    env.close()

run_optimized_q_learning()