import gym
import numpy as np

# 定义环境
env = gym.make('CartPole-v1')

# 初始化Q表
num_states = env.observation_space.shape[0]
num_actions = env.action_space.n
Q = np.zeros((num_states, num_actions))

# 定义超参数
num_episodes = 1000
max_steps_per_episode = 100
learning_rate = 0.1
discount_factor = 0.99
exploration_rate = 1.0
max_exploration_rate = 1.0
min_exploration_rate = 0.01
exploration_decay_rate = 0.01

# 训练智能体
for episode in range(num_episodes):
    state = env.reset()
    for step in range(max_steps_per_episode):
        # 选择动作
        exploration_rate_threshold = np.random.uniform(0, 1)
        if exploration_rate_threshold > exploration_rate:
            action = np.argmax(Q[state, :])
        else:
            action = env.action_space.sample()

        # 执行动作并观察下一个状态和奖励
        next_state和奖励
        next_state, reward, done, _ = env.step(action)

        # 更新Q表
        Q[state, action] = Q[state, action] + learning_rate * (reward + discount_factor * np.max(Q[next_state, :]) - Q[state, action])

        state = next_state

        if done:
            break

    # 衰减探索率
    exploration_rate = min_exploration_rate + (max_exploration_rate - min_exploration_rate) * np.exp(-exploration_decay_rate * episode)

# 测试智能体
total_rewards = 0
num_test_episodes = 10

for episode in range(num_test_episodes):
    state = env.reset()
    for step in range(max_steps_per_episode):
        env.render()
        action = np.argmax(Q[state, :])
        next_state, reward, done, _ = env.step(action)
        total_rewards += reward
        state = next_state
        if done:
            break

average_reward = total_rewards / num_test_episodes
print("平均奖励：", average_reward)

env.close()