Upload 32 files

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+Super-Mario-RL/mario1.gif filter=lfs diff=lfs merge=lfs -text
+Super-Mario-RL/mario1.mp4 filter=lfs diff=lfs merge=lfs -text
+Super-Mario-RL/mario14.gif filter=lfs diff=lfs merge=lfs -text
+Super-Mario-RL/mario14.mp4 filter=lfs diff=lfs merge=lfs -text

Super-Mario-RL-PyQt5/app.py

@@ -0,0 +1,673 @@
+import pickle
+import random
+import time
+from collections import deque
+
+import gym_super_mario_bros
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+from gym_super_mario_bros.actions import COMPLEX_MOVEMENT
+from nes_py.wrappers import JoypadSpace
+
+from PyQt5.QtWidgets import (QApplication, QMainWindow, QWidget, QVBoxLayout, 
+                             QHBoxLayout, QPushButton, QLabel, QComboBox, 
+                             QTextEdit, QProgressBar, QTabWidget, QFrame, QGroupBox)
+from PyQt5.QtCore import QTimer, Qt, pyqtSignal, QThread
+from PyQt5.QtGui import QImage, QPixmap, QFont
+import sys
+import cv2
+
+# Import your wrappers (make sure this module exists)
+try:
+    from wrappers import *
+except ImportError:
+    # Create a proper wrapper if the module doesn't exist
+    class SimpleWrapper:
+        def __init__(self, env):
+            self.env = env
+            self.action_space = env.action_space
+            self.observation_space = env.observation_space
+            
+        def reset(self):
+            return self.env.reset()
+            
+        def step(self, action):
+            return self.env.step(action)
+            
+        def render(self, mode='rgb_array'):
+            return self.env.render(mode)
+            
+        def close(self):
+            if hasattr(self.env, 'close'):
+                self.env.close()
+    
+    def wrap_mario(env):
+        return SimpleWrapper(env)
+
+
+class FrameStacker:
+    """Handles frame stacking and preprocessing"""
+    def __init__(self, frame_size=(84, 84), stack_size=4):
+        self.frame_size = frame_size
+        self.stack_size = stack_size
+        self.frames = deque(maxlen=stack_size)
+        
+    def preprocess_frame(self, frame):
+        """Convert frame to grayscale and resize"""
+        # Convert to grayscale
+        gray = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)
+        # Resize to 84x84
+        resized = cv2.resize(gray, self.frame_size, interpolation=cv2.INTER_AREA)
+        # Normalize to [0, 1]
+        normalized = resized.astype(np.float32) / 255.0
+        return normalized
+    
+    def reset(self, frame):
+        """Reset frame stack with initial frame"""
+        self.frames.clear()
+        processed_frame = self.preprocess_frame(frame)
+        for _ in range(self.stack_size):
+            self.frames.append(processed_frame)
+        return self.get_stacked_frames()
+    
+    def append(self, frame):
+        """Add new frame to stack"""
+        processed_frame = self.preprocess_frame(frame)
+        self.frames.append(processed_frame)
+        return self.get_stacked_frames()
+    
+    def get_stacked_frames(self):
+        """Get stacked frames as numpy array"""
+        stacked = np.array(self.frames)
+        return np.ascontiguousarray(stacked)
+
+
+class replay_memory(object):
+    def __init__(self, N):
+        self.memory = deque(maxlen=N)
+
+    def push(self, transition):
+        self.memory.append(transition)
+
+    def sample(self, n):
+        return random.sample(self.memory, n)
+
+    def __len__(self):
+        return len(self.memory)
+
+
+class DuelingDQNModel(nn.Module):
+    def __init__(self, n_frame, n_action, device):
+        super(DuelingDQNModel, self).__init__()
+        
+        # CNN layers for feature extraction
+        self.conv_layers = nn.Sequential(
+            nn.Conv2d(n_frame, 32, kernel_size=8, stride=4),
+            nn.ReLU(),
+            nn.Conv2d(32, 64, kernel_size=4, stride=2),
+            nn.ReLU(),
+            nn.Conv2d(64, 64, kernel_size=3, stride=1),
+            nn.ReLU()
+        )
+        
+        # Calculate conv output size
+        self.conv_out_size = self._get_conv_out((n_frame, 84, 84))
+        
+        # Advantage stream
+        self.advantage_stream = nn.Sequential(
+            nn.Linear(self.conv_out_size, 512),
+            nn.ReLU(),
+            nn.Linear(512, n_action)
+        )
+        
+        # Value stream
+        self.value_stream = nn.Sequential(
+            nn.Linear(self.conv_out_size, 512),
+            nn.ReLU(),
+            nn.Linear(512, 1)
+        )
+
+        self.device = device
+        self.apply(self.init_weights)
+
+    def _get_conv_out(self, shape):
+        with torch.no_grad():
+            x = torch.zeros(1, *shape)
+            x = self.conv_layers(x)
+            return int(np.prod(x.size()))
+
+    def init_weights(self, m):
+        if isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear):
+            torch.nn.init.xavier_uniform_(m.weight)
+            if m.bias is not None:
+                m.bias.data.fill_(0.01)
+
+    def forward(self, x):
+        if not isinstance(x, torch.Tensor):
+            x = torch.FloatTensor(x).to(self.device)
+        
+        # Forward through conv layers
+        x = self.conv_layers(x)
+        x = x.view(x.size(0), -1)
+        
+        # Forward through advantage and value streams
+        advantage = self.advantage_stream(x)
+        value = self.value_stream(x)
+        
+        # Combine value and advantage
+        q_values = value + (advantage - advantage.mean(dim=1, keepdim=True))
+        
+        return q_values
+
+
+def train(q, q_target, memory, batch_size, gamma, optimizer, device):
+    if len(memory) < batch_size:
+        return 0.0
+        
+    transitions = memory.sample(batch_size)
+    s, r, a, s_prime, done = list(map(list, zip(*transitions)))
+    
+    # Ensure positive strides for all arrays
+    s = np.array([np.ascontiguousarray(arr) for arr in s])
+    s_prime = np.array([np.ascontiguousarray(arr) for arr in s_prime])
+    
+    # Move computations to device
+    s_tensor = torch.FloatTensor(s).to(device)
+    s_prime_tensor = torch.FloatTensor(s_prime).to(device)
+    
+    # Get next Q values from target network
+    with torch.no_grad():
+        next_q_values = q_target(s_prime_tensor)
+        next_actions = next_q_values.max(1)[1].unsqueeze(1)
+        next_q_value = next_q_values.gather(1, next_actions)
+    
+    # Calculate target Q values
+    r = torch.FloatTensor(r).unsqueeze(1).to(device)
+    done = torch.FloatTensor(done).unsqueeze(1).to(device)
+    target_q_values = r + gamma * next_q_value * (1 - done)
+    
+    # Get current Q values
+    a_tensor = torch.LongTensor(a).unsqueeze(1).to(device)
+    current_q_values = q(s_tensor).gather(1, a_tensor)
+    
+    # Calculate loss
+    loss = F.smooth_l1_loss(current_q_values, target_q_values)
+    
+    # Optimize
+    optimizer.zero_grad()
+    loss.backward()
+    
+    # Gradient clipping
+    torch.nn.utils.clip_grad_norm_(q.parameters(), max_norm=10.0)
+    
+    optimizer.step()
+    return loss.item()
+
+
+def copy_weights(q, q_target):
+    q_dict = q.state_dict()
+    q_target.load_state_dict(q_dict)
+
+
+class MarioTrainingThread(QThread):
+    update_signal = pyqtSignal(dict)
+    frame_signal = pyqtSignal(np.ndarray)
+    
+    def __init__(self, device="cpu"):
+        super().__init__()
+        self.device = device
+        self.running = False
+        self.env = None
+        self.q = None
+        self.q_target = None
+        self.optimizer = None
+        self.frame_stacker = None
+        
+        # Training parameters
+        self.gamma = 0.99
+        self.batch_size = 32
+        self.memory_size = 10000
+        self.eps = 1.0  # Start with full exploration
+        self.eps_min = 0.01
+        self.eps_decay = 0.995
+        self.update_interval = 1000
+        self.save_interval = 100
+        self.print_interval = 10
+        
+        self.memory = None
+        self.t = 0
+        self.k = 0
+        self.total_score = 0.0
+        self.loss_accumulator = 0.0
+        self.best_score = -float('inf')
+        self.last_x_pos = 0
+        
+    def setup_training(self):
+        n_frame = 4  # Number of stacked frames
+        try:
+            self.env = gym_super_mario_bros.make("SuperMarioBros-v3")
+            self.env = JoypadSpace(self.env, COMPLEX_MOVEMENT)
+            self.env = wrap_mario(self.env)
+            
+            # Initialize frame stacker
+            self.frame_stacker = FrameStacker(frame_size=(84, 84), stack_size=n_frame)
+            
+            self.q = DuelingDQNModel(n_frame, self.env.action_space.n, self.device).to(self.device)
+            self.q_target = DuelingDQNModel(n_frame, self.env.action_space.n, self.device).to(self.device)
+            
+            copy_weights(self.q, self.q_target)
+            
+            # Set target network to eval mode
+            self.q_target.eval()
+            
+            # Optimizer
+            self.optimizer = optim.Adam(self.q.parameters(), lr=0.0001, weight_decay=1e-5)
+            
+            self.memory = replay_memory(self.memory_size)
+            
+            self.log_message(f"✅ Training setup complete - Actions: {self.env.action_space.n}, Device: {self.device}")
+            
+        except Exception as e:
+            self.log_message(f"❌ Error setting up training: {e}")
+            import traceback
+            traceback.print_exc()
+            self.running = False
+        
+    def run(self):
+        self.running = True
+        self.setup_training()
+        
+        if not self.running:
+            return
+            
+        start_time = time.perf_counter()
+        score_lst = []
+        
+        try:
+            for k in range(1000000):
+                if not self.running:
+                    break
+                    
+                # Reset environment and frame stacker
+                frame = self.env.reset()
+                s = self.frame_stacker.reset(frame)
+                done = False
+                episode_loss = 0.0
+                episode_steps = 0
+                episode_score = 0.0
+                self.last_x_pos = 0
+
+                while not done and self.running:
+                    # Ensure state has positive strides before processing
+                    s_processed = np.ascontiguousarray(s)
+                    
+                    # Epsilon-greedy action selection
+                    if np.random.random() <= self.eps:
+                        a = self.env.action_space.sample()
+                    else:
+                        with torch.no_grad():
+                            # Add batch dimension and create tensor
+                            state_tensor = torch.FloatTensor(s_processed).unsqueeze(0).to(self.device)
+                            q_values = self.q(state_tensor)
+                        
+                        if self.device == "cuda" or self.device == "mps":
+                            a = np.argmax(q_values.cpu().numpy())
+                        else:
+                            a = np.argmax(q_values.detach().numpy())
+                            
+                    # Take action
+                    frame, r, done, info = self.env.step(a)
+                    
+                    # Update frame stack
+                    s_prime = self.frame_stacker.append(frame)
+                    
+                    episode_score += r
+                    
+                    # Enhanced reward shaping
+                    reward = r  # Start with original reward
+                    
+                    # Bonus for x_pos progress
+                    if 'x_pos' in info:
+                        x_pos = info['x_pos']
+                        x_progress = x_pos - self.last_x_pos
+                        if x_progress > 0:
+                            reward += 0.1 * x_progress
+                        self.last_x_pos = x_pos
+                    
+                    # Large bonus for completing the level
+                    if done and info.get('flag_get', False):
+                        reward += 100.0
+                        self.log_message(f"🎉 LEVEL COMPLETED at episode {k}! 🎉")
+                    
+                    # Store transition with contiguous arrays
+                    s_contiguous = np.ascontiguousarray(s)
+                    s_prime_contiguous = np.ascontiguousarray(s_prime)
+                    self.memory.push((s_contiguous, float(reward), int(a), s_prime_contiguous, int(1 - done)))
+                    
+                    s = s_prime
+                    stage = info.get('stage', 1)
+                    world = info.get('world', 1)
+                    
+                    # Emit frame for display
+                    try:
+                        display_frame = self.env.render()
+                        if display_frame is not None:
+                            # Ensure frame has positive strides
+                            frame_contiguous = np.ascontiguousarray(display_frame)
+                            self.frame_signal.emit(frame_contiguous)
+                    except Exception as e:
+                        # Create a placeholder frame if rendering fails
+                        frame = np.zeros((240, 256, 3), dtype=np.uint8)
+                        self.frame_signal.emit(frame)
+                    
+                    # Train only if we have enough samples
+                    if len(self.memory) > self.batch_size:
+                        loss_val = train(self.q, self.q_target, self.memory, self.batch_size, 
+                                       self.gamma, self.optimizer, self.device)
+                        if loss_val > 0:
+                            self.loss_accumulator += loss_val
+                            episode_loss += loss_val
+                            self.t += 1
+                        
+                        # Update target network
+                        if self.t % self.update_interval == 0:
+                            copy_weights(self.q, self.q_target)
+                    
+                    episode_steps += 1
+
+                    # Emit training progress every 10 steps
+                    if episode_steps % 10 == 0:
+                        progress_data = {
+                            'episode': k,
+                            'total_reward': episode_score,
+                            'steps': episode_steps,
+                            'epsilon': self.eps,
+                            'world': world,
+                            'stage': stage,
+                            'loss': episode_loss / (episode_steps + 1e-8),
+                            'memory_size': len(self.memory),
+                            'x_pos': info.get('x_pos', 0),
+                            'score': info.get('score', 0),
+                            'coins': info.get('coins', 0),
+                            'time': info.get('time', 400),
+                            'flag_get': info.get('flag_get', False)
+                        }
+                        self.update_signal.emit(progress_data)
+
+                # Epsilon decay after each episode
+                if self.eps > self.eps_min:
+                    self.eps *= self.eps_decay
+
+                # Update total score
+                self.total_score += episode_score
+
+                # Save best model
+                if episode_score > self.best_score and k > 0:
+                    self.best_score = episode_score
+                    torch.save(self.q.state_dict(), "enhanced_mario_q_best.pth")
+                    torch.save(self.q_target.state_dict(), "enhanced_mario_q_target_best.pth")
+                    self.log_message(f"💾 New best model saved! Score: {self.best_score:.2f}")
+
+                # Save models periodically
+                if k % self.save_interval == 0 and k > 0:
+                    torch.save(self.q.state_dict(), "enhanced_mario_q.pth")
+                    torch.save(self.q_target.state_dict(), "enhanced_mario_q_target.pth")
+                    self.log_message(f"💾 Models saved at episode {k}")
+
+                # Print progress
+                if k % self.print_interval == 0 and k > 0:
+                    time_spent = time.perf_counter() - start_time
+                    start_time = time.perf_counter()
+                    
+                    avg_loss = self.loss_accumulator / (self.print_interval * max(episode_steps, 1))
+                    avg_score = self.total_score / self.print_interval
+                    
+                    log_msg = (
+                        f"{self.device} | Ep: {k} | Score: {avg_score:.2f} | Loss: {avg_loss:.4f} | "
+                        f"Stage: {world}-{stage} | Eps: {self.eps:.3f} | Time: {time_spent:.2f}s | "
+                        f"Mem: {len(self.memory)} | Steps: {episode_steps}"
+                    )
+                    self.log_message(log_msg)
+                    
+                    score_lst.append(avg_score)
+                    self.total_score = 0.0
+                    self.loss_accumulator = 0.0
+                    
+                    try:
+                        pickle.dump(score_lst, open("score.p", "wb"))
+                    except Exception as e:
+                        self.log_message(f"⚠️ Could not save scores: {e}")
+                    
+                self.k = k
+                
+        except Exception as e:
+            self.log_message(f"❌ Training error: {e}")
+            import traceback
+            traceback.print_exc()
+            
+    def log_message(self, message):
+        progress_data = {
+            'log_message': message
+        }
+        self.update_signal.emit(progress_data)
+        
+    def stop(self):
+        self.running = False
+        if self.env:
+            try:
+                self.env.close()
+            except:
+                pass
+
+
+class MarioRLApp(QMainWindow):
+    def __init__(self):
+        super().__init__()
+        self.training_thread = None
+        self.init_ui()
+        
+    def init_ui(self):
+        self.setWindowTitle('🎮 Super Mario Bros - Dueling DQN Training')
+        self.setGeometry(100, 100, 1200, 800)
+        
+        central_widget = QWidget()
+        self.setCentralWidget(central_widget)
+        layout = QVBoxLayout(central_widget)
+        
+        # Title
+        title = QLabel('🎮 Super Mario Bros - Enhanced Dueling DQN')
+        title.setFont(QFont('Arial', 16, QFont.Bold))
+        title.setAlignment(Qt.AlignCenter)
+        layout.addWidget(title)
+        
+        # Control Panel
+        control_layout = QHBoxLayout()
+        
+        self.device_combo = QComboBox()
+        self.device_combo.addItems(['cpu', 'cuda', 'mps'])
+        
+        self.start_btn = QPushButton('Start Training')
+        self.start_btn.clicked.connect(self.start_training)
+        
+        self.stop_btn = QPushButton('Stop Training')
+        self.stop_btn.clicked.connect(self.stop_training)
+        self.stop_btn.setEnabled(False)
+        
+        self.load_btn = QPushButton('Load Model')
+        self.load_btn.clicked.connect(self.load_model)
+        
+        control_layout.addWidget(QLabel('Device:'))
+        control_layout.addWidget(self.device_combo)
+        control_layout.addWidget(self.start_btn)
+        control_layout.addWidget(self.stop_btn)
+        control_layout.addWidget(self.load_btn)
+        control_layout.addStretch()
+        
+        layout.addLayout(control_layout)
+        
+        # Content Area
+        content_layout = QHBoxLayout()
+        
+        # Left side - Game Display
+        left_frame = QFrame()
+        left_frame.setFrameStyle(QFrame.Box)
+        left_layout = QVBoxLayout(left_frame)
+        
+        self.game_display = QLabel()
+        self.game_display.setMinimumSize(400, 300)
+        self.game_display.setAlignment(Qt.AlignCenter)
+        self.game_display.setText('Game display will appear here\nPress "Start Training" to begin')
+        self.game_display.setStyleSheet('border: 1px solid gray; background-color: black; color: white;')
+        
+        left_layout.addWidget(QLabel('Mario Game Display:'))
+        left_layout.addWidget(self.game_display)
+        
+        # Right side - Training Info
+        right_frame = QFrame()
+        right_frame.setFrameStyle(QFrame.Box)
+        right_layout = QVBoxLayout(right_frame)
+        
+        # Training stats
+        stats_group = QGroupBox("Training Statistics")
+        stats_layout = QVBoxLayout(stats_group)
+        
+        self.episode_label = QLabel('Episode: 0')
+        self.world_label = QLabel('World: 1-1')
+        self.score_label = QLabel('Score: 0')
+        self.reward_label = QLabel('Episode Reward: 0')
+        self.steps_label = QLabel('Steps: 0')
+        self.epsilon_label = QLabel('Epsilon: 1.000')
+        self.loss_label = QLabel('Loss: 0.0000')
+        self.memory_label = QLabel('Memory: 0')
+        self.xpos_label = QLabel('X Position: 0')
+        self.coins_label = QLabel('Coins: 0')
+        self.time_label = QLabel('Time: 400')
+        self.flag_label = QLabel('Flag: No')
+        
+        stats_layout.addWidget(self.episode_label)
+        stats_layout.addWidget(self.world_label)
+        stats_layout.addWidget(self.score_label)
+        stats_layout.addWidget(self.reward_label)
+        stats_layout.addWidget(self.steps_label)
+        stats_layout.addWidget(self.epsilon_label)
+        stats_layout.addWidget(self.loss_label)
+        stats_layout.addWidget(self.memory_label)
+        stats_layout.addWidget(self.xpos_label)
+        stats_layout.addWidget(self.coins_label)
+        stats_layout.addWidget(self.time_label)
+        stats_layout.addWidget(self.flag_label)
+        
+        right_layout.addWidget(stats_group)
+        
+        # Training log
+        right_layout.addWidget(QLabel('Training Log:'))
+        self.log_text = QTextEdit()
+        self.log_text.setMaximumHeight(300)
+        right_layout.addWidget(self.log_text)
+        
+        content_layout.addWidget(left_frame)
+        content_layout.addWidget(right_frame)
+        layout.addLayout(content_layout)
+        
+    def start_training(self):
+        device = self.device_combo.currentText()
+        
+        # Check device availability
+        if device == "cuda" and not torch.cuda.is_available():
+            self.log_text.append("❌ CUDA not available, using CPU instead")
+            device = "cpu"
+        elif device == "mps" and not torch.backends.mps.is_available():
+            self.log_text.append("❌ MPS not available, using CPU instead")
+            device = "cpu"
+            
+        self.training_thread = MarioTrainingThread(device)
+        self.training_thread.update_signal.connect(self.update_training_info)
+        self.training_thread.frame_signal.connect(self.update_game_display)
+        self.training_thread.start()
+        
+        self.start_btn.setEnabled(False)
+        self.stop_btn.setEnabled(True)
+        
+        self.log_text.append(f'🚀 Started Dueling DQN training on {device}...')
+        
+    def stop_training(self):
+        if self.training_thread:
+            self.training_thread.stop()
+            self.training_thread.wait()
+            
+        self.start_btn.setEnabled(True)
+        self.stop_btn.setEnabled(False)
+        self.log_text.append('⏹️ Training stopped.')
+        
+    def load_model(self):
+        # Placeholder for model loading functionality
+        self.log_text.append('📁 Load model functionality not implemented yet')
+        
+    def update_training_info(self, data):
+        if 'episode' in data:
+            self.episode_label.setText(f'Episode: {data["episode"]}')
+        if 'world' in data and 'stage' in data:
+            self.world_label.setText(f'World: {data["world"]}-{data["stage"]}')
+        if 'score' in data:
+            self.score_label.setText(f'Score: {data["score"]}')
+        if 'total_reward' in data:
+            self.reward_label.setText(f'Episode Reward: {data["total_reward"]:.2f}')
+        if 'steps' in data:
+            self.steps_label.setText(f'Steps: {data["steps"]}')
+        if 'epsilon' in data:
+            self.epsilon_label.setText(f'Epsilon: {data["epsilon"]:.3f}')
+        if 'loss' in data:
+            self.loss_label.setText(f'Loss: {data["loss"]:.4f}')
+        if 'memory_size' in data:
+            self.memory_label.setText(f'Memory: {data["memory_size"]}')
+        if 'x_pos' in data:
+            self.xpos_label.setText(f'X Position: {data["x_pos"]}')
+        if 'coins' in data:
+            self.coins_label.setText(f'Coins: {data["coins"]}')
+        if 'time' in data:
+            self.time_label.setText(f'Time: {data["time"]}')
+        if 'flag_get' in data:
+            flag_text = "Yes" if data["flag_get"] else "No"
+            self.flag_label.setText(f'Flag: {flag_text}')
+        if 'log_message' in data:
+            self.log_text.append(data['log_message'])
+            # Auto-scroll to bottom
+            self.log_text.verticalScrollBar().setValue(
+                self.log_text.verticalScrollBar().maximum()
+            )
+            
+    def update_game_display(self, frame):
+        if frame is not None:
+            try:
+                h, w, ch = frame.shape
+                bytes_per_line = ch * w
+                # Ensure contiguous array
+                frame_contiguous = np.ascontiguousarray(frame)
+                q_img = QImage(frame_contiguous.data, w, h, bytes_per_line, QImage.Format_RGB888)
+                pixmap = QPixmap.fromImage(q_img)
+                self.game_display.setPixmap(pixmap.scaled(400, 300, Qt.KeepAspectRatio))
+            except Exception as e:
+                print(f"Error updating display: {e}")
+            
+    def closeEvent(self, event):
+        self.stop_training()
+        event.accept()
+
+
+def main():
+    # Set random seeds for reproducibility
+    torch.manual_seed(42)
+    np.random.seed(42)
+    random.seed(42)
+    
+    app = QApplication(sys.argv)
+    window = MarioRLApp()
+    window.show()
+    sys.exit(app.exec_())
+
+
+if __name__ == '__main__':
+    main()

Super-Mario-RL-PyQt5/app_2.py

@@ -0,0 +1,676 @@
+import pickle
+import random
+import time
+from collections import deque
+
+import gym_super_mario_bros
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+from gym_super_mario_bros.actions import COMPLEX_MOVEMENT
+from nes_py.wrappers import JoypadSpace
+
+from PyQt5.QtWidgets import (QApplication, QMainWindow, QWidget, QVBoxLayout, 
+                             QHBoxLayout, QPushButton, QLabel, QComboBox, 
+                             QTextEdit, QProgressBar, QTabWidget, QFrame, QGroupBox)
+from PyQt5.QtCore import QTimer, Qt, pyqtSignal, QThread
+from PyQt5.QtGui import QImage, QPixmap, QFont
+import sys
+import cv2
+
+# Import your wrappers (make sure this module exists)
+try:
+    from wrappers import *
+except ImportError:
+    # Create a proper wrapper if the module doesn't exist
+    class SimpleWrapper:
+        def __init__(self, env):
+            self.env = env
+            self.action_space = env.action_space
+            self.observation_space = env.observation_space
+            
+        def reset(self):
+            return self.env.reset()
+            
+        def step(self, action):
+            return self.env.step(action)
+            
+        def render(self, mode='rgb_array'):
+            return self.env.render(mode)
+            
+        def close(self):
+            if hasattr(self.env, 'close'):
+                self.env.close()
+    
+    def wrap_mario(env):
+        return SimpleWrapper(env)
+
+
+class FrameStacker:
+    """Handles frame stacking and preprocessing for the neural network"""
+    def __init__(self, frame_size=(84, 84), stack_size=4):
+        self.frame_size = frame_size
+        self.stack_size = stack_size
+        self.frames = deque(maxlen=stack_size)
+        
+    def preprocess_frame(self, frame):
+        """Convert frame to grayscale and resize for the neural network"""
+        # Convert to grayscale
+        gray = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)
+        # Resize to 84x84
+        resized = cv2.resize(gray, self.frame_size, interpolation=cv2.INTER_AREA)
+        # Normalize to [0, 1]
+        normalized = resized.astype(np.float32) / 255.0
+        return normalized
+    
+    def reset(self, frame):
+        """Reset frame stack with initial frame"""
+        self.frames.clear()
+        processed_frame = self.preprocess_frame(frame)
+        for _ in range(self.stack_size):
+            self.frames.append(processed_frame)
+        return self.get_stacked_frames()
+    
+    def append(self, frame):
+        """Add new frame to stack"""
+        processed_frame = self.preprocess_frame(frame)
+        self.frames.append(processed_frame)
+        return self.get_stacked_frames()
+    
+    def get_stacked_frames(self):
+        """Get stacked frames as numpy array for the neural network"""
+        stacked = np.array(self.frames)
+        return np.ascontiguousarray(stacked)
+
+
+class replay_memory(object):
+    def __init__(self, N):
+        self.memory = deque(maxlen=N)
+
+    def push(self, transition):
+        self.memory.append(transition)
+
+    def sample(self, n):
+        return random.sample(self.memory, n)
+
+    def __len__(self):
+        return len(self.memory)
+
+
+class DuelingDQNModel(nn.Module):
+    def __init__(self, n_frame, n_action, device):
+        super(DuelingDQNModel, self).__init__()
+        
+        # CNN layers for feature extraction
+        self.conv_layers = nn.Sequential(
+            nn.Conv2d(n_frame, 32, kernel_size=8, stride=4),
+            nn.ReLU(),
+            nn.Conv2d(32, 64, kernel_size=4, stride=2),
+            nn.ReLU(),
+            nn.Conv2d(64, 64, kernel_size=3, stride=1),
+            nn.ReLU()
+        )
+        
+        # Calculate conv output size
+        self.conv_out_size = self._get_conv_out((n_frame, 84, 84))
+        
+        # Advantage stream
+        self.advantage_stream = nn.Sequential(
+            nn.Linear(self.conv_out_size, 512),
+            nn.ReLU(),
+            nn.Linear(512, n_action)
+        )
+        
+        # Value stream
+        self.value_stream = nn.Sequential(
+            nn.Linear(self.conv_out_size, 512),
+            nn.ReLU(),
+            nn.Linear(512, 1)
+        )
+
+        self.device = device
+        self.apply(self.init_weights)
+
+    def _get_conv_out(self, shape):
+        with torch.no_grad():
+            x = torch.zeros(1, *shape)
+            x = self.conv_layers(x)
+            return int(np.prod(x.size()))
+
+    def init_weights(self, m):
+        if isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear):
+            torch.nn.init.xavier_uniform_(m.weight)
+            if m.bias is not None:
+                m.bias.data.fill_(0.01)
+
+    def forward(self, x):
+        if not isinstance(x, torch.Tensor):
+            x = torch.FloatTensor(x).to(self.device)
+        
+        # Forward through conv layers
+        x = self.conv_layers(x)
+        x = x.view(x.size(0), -1)
+        
+        # Forward through advantage and value streams
+        advantage = self.advantage_stream(x)
+        value = self.value_stream(x)
+        
+        # Combine value and advantage
+        q_values = value + (advantage - advantage.mean(dim=1, keepdim=True))
+        
+        return q_values
+
+
+def train(q, q_target, memory, batch_size, gamma, optimizer, device):
+    if len(memory) < batch_size:
+        return 0.0
+        
+    transitions = memory.sample(batch_size)
+    s, r, a, s_prime, done = list(map(list, zip(*transitions)))
+    
+    # Ensure positive strides for all arrays
+    s = np.array([np.ascontiguousarray(arr) for arr in s])
+    s_prime = np.array([np.ascontiguousarray(arr) for arr in s_prime])
+    
+    # Move computations to device
+    s_tensor = torch.FloatTensor(s).to(device)
+    s_prime_tensor = torch.FloatTensor(s_prime).to(device)
+    
+    # Get next Q values from target network
+    with torch.no_grad():
+        next_q_values = q_target(s_prime_tensor)
+        next_actions = next_q_values.max(1)[1].unsqueeze(1)
+        next_q_value = next_q_values.gather(1, next_actions)
+    
+    # Calculate target Q values
+    r = torch.FloatTensor(r).unsqueeze(1).to(device)
+    done = torch.FloatTensor(done).unsqueeze(1).to(device)
+    target_q_values = r + gamma * next_q_value * (1 - done)
+    
+    # Get current Q values
+    a_tensor = torch.LongTensor(a).unsqueeze(1).to(device)
+    current_q_values = q(s_tensor).gather(1, a_tensor)
+    
+    # Calculate loss
+    loss = F.smooth_l1_loss(current_q_values, target_q_values)
+    
+    # Optimize
+    optimizer.zero_grad()
+    loss.backward()
+    
+    # Gradient clipping
+    torch.nn.utils.clip_grad_norm_(q.parameters(), max_norm=10.0)
+    
+    optimizer.step()
+    return loss.item()
+
+
+def copy_weights(q, q_target):
+    q_dict = q.state_dict()
+    q_target.load_state_dict(q_dict)
+
+
+class MarioTrainingThread(QThread):
+    update_signal = pyqtSignal(dict)
+    frame_signal = pyqtSignal(np.ndarray)
+    
+    def __init__(self, device="cpu"):
+        super().__init__()
+        self.device = device
+        self.running = False
+        self.env = None
+        self.q = None
+        self.q_target = None
+        self.optimizer = None
+        self.frame_stacker = None
+        
+        # Training parameters
+        self.gamma = 0.99
+        self.batch_size = 32
+        self.memory_size = 10000
+        self.eps = 1.0  # Start with full exploration
+        self.eps_min = 0.01
+        self.eps_decay = 0.995
+        self.update_interval = 1000
+        self.save_interval = 100
+        self.print_interval = 10
+        
+        self.memory = None
+        self.t = 0
+        self.k = 0
+        self.total_score = 0.0
+        self.loss_accumulator = 0.0
+        self.best_score = -float('inf')
+        self.last_x_pos = 0
+        
+    def setup_training(self):
+        n_frame = 4  # Number of stacked frames
+        try:
+            self.env = gym_super_mario_bros.make("SuperMarioBros-v3")
+            self.env = JoypadSpace(self.env, COMPLEX_MOVEMENT)
+            self.env = wrap_mario(self.env)
+            
+            # Initialize frame stacker
+            self.frame_stacker = FrameStacker(frame_size=(84, 84), stack_size=n_frame)
+            
+            self.q = DuelingDQNModel(n_frame, self.env.action_space.n, self.device).to(self.device)
+            self.q_target = DuelingDQNModel(n_frame, self.env.action_space.n, self.device).to(self.device)
+            
+            copy_weights(self.q, self.q_target)
+            
+            # Set target network to eval mode
+            self.q_target.eval()
+            
+            # Optimizer
+            self.optimizer = optim.Adam(self.q.parameters(), lr=0.0001, weight_decay=1e-5)
+            
+            self.memory = replay_memory(self.memory_size)
+            
+            self.log_message(f"✅ Training setup complete - Actions: {self.env.action_space.n}, Device: {self.device}")
+            
+        except Exception as e:
+            self.log_message(f"❌ Error setting up training: {e}")
+            import traceback
+            traceback.print_exc()
+            self.running = False
+        
+    def run(self):
+        self.running = True
+        self.setup_training()
+        
+        if not self.running:
+            return
+            
+        start_time = time.perf_counter()
+        score_lst = []
+        
+        try:
+            for k in range(1000000):
+                if not self.running:
+                    break
+                    
+                # Reset environment and frame stacker
+                frame = self.env.reset()
+                s = self.frame_stacker.reset(frame)
+                done = False
+                episode_loss = 0.0
+                episode_steps = 0
+                episode_score = 0.0
+                self.last_x_pos = 0
+
+                while not done and self.running:
+                    # Ensure state has positive strides before processing
+                    s_processed = np.ascontiguousarray(s)
+                    
+                    # Epsilon-greedy action selection
+                    if np.random.random() <= self.eps:
+                        a = self.env.action_space.sample()
+                    else:
+                        with torch.no_grad():
+                            # Add batch dimension and create tensor
+                            state_tensor = torch.FloatTensor(s_processed).unsqueeze(0).to(self.device)
+                            q_values = self.q(state_tensor)
+                        
+                        if self.device == "cuda" or self.device == "mps":
+                            a = np.argmax(q_values.cpu().numpy())
+                        else:
+                            a = np.argmax(q_values.detach().numpy())
+                            
+                    # Take action
+                    next_frame, r, done, info = self.env.step(a)
+                    
+                    # Update frame stack for neural network
+                    s_prime = self.frame_stacker.append(next_frame)
+                    
+                    episode_score += r
+                    
+                    # Enhanced reward shaping
+                    reward = r  # Start with original reward
+                    
+                    # Bonus for x_pos progress
+                    if 'x_pos' in info:
+                        x_pos = info['x_pos']
+                        x_progress = x_pos - self.last_x_pos
+                        if x_progress > 0:
+                            reward += 0.1 * x_progress
+                        self.last_x_pos = x_pos
+                    
+                    # Large bonus for completing the level
+                    if done and info.get('flag_get', False):
+                        reward += 100.0
+                        self.log_message(f"🎉 LEVEL COMPLETED at episode {k}! 🎉")
+                    
+                    # Store transition with contiguous arrays
+                    s_contiguous = np.ascontiguousarray(s)
+                    s_prime_contiguous = np.ascontiguousarray(s_prime)
+                    self.memory.push((s_contiguous, float(reward), int(a), s_prime_contiguous, int(1 - done)))
+                    
+                    s = s_prime
+                    stage = info.get('stage', 1)
+                    world = info.get('world', 1)
+                    
+                    # Emit ORIGINAL COLOR FRAME for display (not preprocessed)
+                    try:
+                        # Get the original color frame for display
+                        display_frame = self.env.render()
+                        if display_frame is not None:
+                            # Ensure frame has positive strides and emit original color frame
+                            frame_contiguous = np.ascontiguousarray(display_frame)
+                            self.frame_signal.emit(frame_contiguous)
+                    except Exception as e:
+                        # Create a placeholder frame if rendering fails
+                        frame = np.zeros((240, 256, 3), dtype=np.uint8)
+                        self.frame_signal.emit(frame)
+                    
+                    # Train only if we have enough samples
+                    if len(self.memory) > self.batch_size:
+                        loss_val = train(self.q, self.q_target, self.memory, self.batch_size, 
+                                       self.gamma, self.optimizer, self.device)
+                        if loss_val > 0:
+                            self.loss_accumulator += loss_val
+                            episode_loss += loss_val
+                            self.t += 1
+                        
+                        # Update target network
+                        if self.t % self.update_interval == 0:
+                            copy_weights(self.q, self.q_target)
+                            self.log_message(f"🔄 Target network updated at step {self.t}")
+                    
+                    episode_steps += 1
+
+                    # Emit training progress every 5 steps for more frequent updates
+                    if episode_steps % 5 == 0:
+                        progress_data = {
+                            'episode': k,
+                            'total_reward': episode_score,
+                            'steps': episode_steps,
+                            'epsilon': self.eps,
+                            'world': world,
+                            'stage': stage,
+                            'loss': episode_loss / (episode_steps + 1e-8),
+                            'memory_size': len(self.memory),
+                            'x_pos': info.get('x_pos', 0),
+                            'score': info.get('score', 0),
+                            'coins': info.get('coins', 0),
+                            'time': info.get('time', 400),
+                            'flag_get': info.get('flag_get', False)
+                        }
+                        self.update_signal.emit(progress_data)
+
+                # Epsilon decay after each episode
+                if self.eps > self.eps_min:
+                    self.eps *= self.eps_decay
+
+                # Update total score
+                self.total_score += episode_score
+
+                # Save best model
+                if episode_score > self.best_score and k > 0:
+                    self.best_score = episode_score
+                    torch.save(self.q.state_dict(), "enhanced_mario_q_best.pth")
+                    torch.save(self.q_target.state_dict(), "enhanced_mario_q_target_best.pth")
+                    self.log_message(f"💾 New best model saved! Score: {self.best_score:.2f}")
+
+                # Save models periodically
+                if k % self.save_interval == 0 and k > 0:
+                    torch.save(self.q.state_dict(), "enhanced_mario_q.pth")
+                    torch.save(self.q_target.state_dict(), "enhanced_mario_q_target.pth")
+                    self.log_message(f"💾 Models saved at episode {k}")
+
+                # Print progress
+                if k % self.print_interval == 0 and k > 0:
+                    time_spent = time.perf_counter() - start_time
+                    start_time = time.perf_counter()
+                    
+                    avg_loss = self.loss_accumulator / (self.print_interval * max(episode_steps, 1))
+                    avg_score = self.total_score / self.print_interval
+                    
+                    log_msg = (
+                        f"{self.device} | Ep: {k} | Score: {avg_score:.2f} | Loss: {avg_loss:.4f} | "
+                        f"Stage: {world}-{stage} | Eps: {self.eps:.3f} | Time: {time_spent:.2f}s | "
+                        f"Mem: {len(self.memory)} | Steps: {episode_steps}"
+                    )
+                    self.log_message(log_msg)
+                    
+                    score_lst.append(avg_score)
+                    self.total_score = 0.0
+                    self.loss_accumulator = 0.0
+                    
+                    try:
+                        pickle.dump(score_lst, open("score.p", "wb"))
+                    except Exception as e:
+                        self.log_message(f"⚠️ Could not save scores: {e}")
+                    
+                self.k = k
+                
+        except Exception as e:
+            self.log_message(f"❌ Training error: {e}")
+            import traceback
+            traceback.print_exc()
+            
+    def log_message(self, message):
+        progress_data = {
+            'log_message': message
+        }
+        self.update_signal.emit(progress_data)
+        
+    def stop(self):
+        self.running = False
+        if self.env:
+            try:
+                self.env.close()
+            except:
+                pass
+
+
+class MarioRLApp(QMainWindow):
+    def __init__(self):
+        super().__init__()
+        self.training_thread = None
+        self.init_ui()
+        
+    def init_ui(self):
+        self.setWindowTitle('🎮 Super Mario Bros - Dueling DQN Training')
+        self.setGeometry(100, 100, 1200, 800)
+        
+        central_widget = QWidget()
+        self.setCentralWidget(central_widget)
+        layout = QVBoxLayout(central_widget)
+        
+        # Title
+        title = QLabel('🎮 Super Mario Bros - Enhanced Dueling DQN')
+        title.setFont(QFont('Arial', 16, QFont.Bold))
+        title.setAlignment(Qt.AlignCenter)
+        layout.addWidget(title)
+        
+        # Control Panel
+        control_layout = QHBoxLayout()
+        
+        self.device_combo = QComboBox()
+        self.device_combo.addItems(['cpu', 'cuda', 'mps'])
+        
+        self.start_btn = QPushButton('Start Training')
+        self.start_btn.clicked.connect(self.start_training)
+        
+        self.stop_btn = QPushButton('Stop Training')
+        self.stop_btn.clicked.connect(self.stop_training)
+        self.stop_btn.setEnabled(False)
+        
+        self.load_btn = QPushButton('Load Model')
+        self.load_btn.clicked.connect(self.load_model)
+        
+        control_layout.addWidget(QLabel('Device:'))
+        control_layout.addWidget(self.device_combo)
+        control_layout.addWidget(self.start_btn)
+        control_layout.addWidget(self.stop_btn)
+        control_layout.addWidget(self.load_btn)
+        control_layout.addStretch()
+        
+        layout.addLayout(control_layout)
+        
+        # Content Area
+        content_layout = QHBoxLayout()
+        
+        # Left side - Game Display
+        left_frame = QFrame()
+        left_frame.setFrameStyle(QFrame.Box)
+        left_layout = QVBoxLayout(left_frame)
+        
+        self.game_display = QLabel()
+        self.game_display.setMinimumSize(400, 300)
+        self.game_display.setAlignment(Qt.AlignCenter)
+        self.game_display.setText('Game display will appear here\nPress "Start Training" to begin')
+        self.game_display.setStyleSheet('border: 1px solid gray; background-color: black; color: white;')
+        
+        left_layout.addWidget(QLabel('Mario Game Display:'))
+        left_layout.addWidget(self.game_display)
+        
+        # Right side - Training Info
+        right_frame = QFrame()
+        right_frame.setFrameStyle(QFrame.Box)
+        right_layout = QVBoxLayout(right_frame)
+        
+        # Training stats
+        stats_group = QGroupBox("Training Statistics")
+        stats_layout = QVBoxLayout(stats_group)
+        
+        self.episode_label = QLabel('Episode: 0')
+        self.world_label = QLabel('World: 1-1')
+        self.score_label = QLabel('Score: 0')
+        self.reward_label = QLabel('Episode Reward: 0')
+        self.steps_label = QLabel('Steps: 0')
+        self.epsilon_label = QLabel('Epsilon: 1.000')
+        self.loss_label = QLabel('Loss: 0.0000')
+        self.memory_label = QLabel('Memory: 0')
+        self.xpos_label = QLabel('X Position: 0')
+        self.coins_label = QLabel('Coins: 0')
+        self.time_label = QLabel('Time: 400')
+        self.flag_label = QLabel('Flag: No')
+        
+        stats_layout.addWidget(self.episode_label)
+        stats_layout.addWidget(self.world_label)
+        stats_layout.addWidget(self.score_label)
+        stats_layout.addWidget(self.reward_label)
+        stats_layout.addWidget(self.steps_label)
+        stats_layout.addWidget(self.epsilon_label)
+        stats_layout.addWidget(self.loss_label)
+        stats_layout.addWidget(self.memory_label)
+        stats_layout.addWidget(self.xpos_label)
+        stats_layout.addWidget(self.coins_label)
+        stats_layout.addWidget(self.time_label)
+        stats_layout.addWidget(self.flag_label)
+        
+        right_layout.addWidget(stats_group)
+        
+        # Training log
+        right_layout.addWidget(QLabel('Training Log:'))
+        self.log_text = QTextEdit()
+        self.log_text.setMaximumHeight(300)
+        right_layout.addWidget(self.log_text)
+        
+        content_layout.addWidget(left_frame)
+        content_layout.addWidget(right_frame)
+        layout.addLayout(content_layout)
+        
+    def start_training(self):
+        device = self.device_combo.currentText()
+        
+        # Check device availability
+        if device == "cuda" and not torch.cuda.is_available():
+            self.log_text.append("❌ CUDA not available, using CPU instead")
+            device = "cpu"
+        elif device == "mps" and not torch.backends.mps.is_available():
+            self.log_text.append("❌ MPS not available, using CPU instead")
+            device = "cpu"
+            
+        self.training_thread = MarioTrainingThread(device)
+        self.training_thread.update_signal.connect(self.update_training_info)
+        self.training_thread.frame_signal.connect(self.update_game_display)
+        self.training_thread.start()
+        
+        self.start_btn.setEnabled(False)
+        self.stop_btn.setEnabled(True)
+        
+        self.log_text.append(f'🚀 Started Dueling DQN training on {device}...')
+        
+    def stop_training(self):
+        if self.training_thread:
+            self.training_thread.stop()
+            self.training_thread.wait()
+            
+        self.start_btn.setEnabled(True)
+        self.stop_btn.setEnabled(False)
+        self.log_text.append('⏹️ Training stopped.')
+        
+    def load_model(self):
+        # Placeholder for model loading functionality
+        self.log_text.append('📁 Load model functionality not implemented yet')
+        
+    def update_training_info(self, data):
+        if 'episode' in data:
+            self.episode_label.setText(f'Episode: {data["episode"]}')
+        if 'world' in data and 'stage' in data:
+            self.world_label.setText(f'World: {data["world"]}-{data["stage"]}')
+        if 'score' in data:
+            self.score_label.setText(f'Score: {data["score"]}')
+        if 'total_reward' in data:
+            self.reward_label.setText(f'Episode Reward: {data["total_reward"]:.2f}')
+        if 'steps' in data:
+            self.steps_label.setText(f'Steps: {data["steps"]}')
+        if 'epsilon' in data:
+            self.epsilon_label.setText(f'Epsilon: {data["epsilon"]:.3f}')
+        if 'loss' in data:
+            self.loss_label.setText(f'Loss: {data["loss"]:.4f}')
+        if 'memory_size' in data:
+            self.memory_label.setText(f'Memory: {data["memory_size"]}')
+        if 'x_pos' in data:
+            self.xpos_label.setText(f'X Position: {data["x_pos"]}')
+        if 'coins' in data:
+            self.coins_label.setText(f'Coins: {data["coins"]}')
+        if 'time' in data:
+            self.time_label.setText(f'Time: {data["time"]}')
+        if 'flag_get' in data:
+            flag_text = "Yes" if data["flag_get"] else "No"
+            self.flag_label.setText(f'Flag: {flag_text}')
+        if 'log_message' in data:
+            self.log_text.append(data['log_message'])
+            # Auto-scroll to bottom
+            self.log_text.verticalScrollBar().setValue(
+                self.log_text.verticalScrollBar().maximum()
+            )
+            
+    def update_game_display(self, frame):
+        if frame is not None:
+            try:
+                h, w, ch = frame.shape
+                bytes_per_line = ch * w
+                # Ensure contiguous array and display original color frame
+                frame_contiguous = np.ascontiguousarray(frame)
+                q_img = QImage(frame_contiguous.data, w, h, bytes_per_line, QImage.Format_RGB888)
+                pixmap = QPixmap.fromImage(q_img)
+                # Scale to fit the display while maintaining aspect ratio
+                self.game_display.setPixmap(pixmap.scaled(400, 300, Qt.KeepAspectRatio, Qt.SmoothTransformation))
+            except Exception as e:
+                print(f"Error updating display: {e}")
+            
+    def closeEvent(self, event):
+        self.stop_training()
+        event.accept()
+
+
+def main():
+    # Set random seeds for reproducibility
+    torch.manual_seed(42)
+    np.random.seed(42)
+    random.seed(42)
+    
+    app = QApplication(sys.argv)
+    window = MarioRLApp()
+    window.show()
+    sys.exit(app.exec_())
+
+
+if __name__ == '__main__':
+    main()

Super-Mario-RL-PyQt5/enhanced_mario_q_best.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c82b8bb39904cf745061a6ba1ca2a207977f22f13fb0e72f1141c3f85045eb0
+size 13193617

Super-Mario-RL-PyQt5/enhanced_mario_q_target_best.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9f6060fca857b4335781a219ffa7450baa20eb8efb9287e9561087978c1a1e0
+size 13193949

Super-Mario-RL-PyQt5/requirements.txt

@@ -0,0 +1,9 @@
+numpy==1.26.4
+torch>=1.6.0
+torchvision
+gym==0.23
+nes-py
+gym-super-mario-bros==7.2.3
+opencv-python
+matplotlib
+pyqt5

Super-Mario-RL/README.md

@@ -0,0 +1,85 @@
+# :mushroom: Super-Mario-RL 
+
+This is a private project to make Super Mario Agent.
+
+It consists of training an agent to clear Super Mario Bros with deep reinforcement learning methods.
+
+Here are my super mario agents with dueling network. ( trained 7,000 epoch )
+
+**(25-05-20) SuperMario with PPO has been updated!**
+
+<p float="center">
+  <img src="/mario1.gif" width="350" />
+  <img src="/mario14.gif" width="350" /> 
+</p>
+
+# Get started
+
+## Cloning git
+
+```
+git clone https://github.com/jiseongHAN/Super-Mario-RL.git
+cd Super-Mario-RL
+```
+
+## Install Requirements
+```
+pip install -r requirements.txt
+```
+
+## Or Install Manually
+* Install [openAI gym](http://gym.openai.com/)
+```
+pip install 'gym'
+```
+* Install [Pytorch](https://pytorch.org/)
+```
+pip install torch torchvision
+```
+* Install [nes-py](https://pypi.org/project/nes-py/)
+```
+pip install nes-py
+```
+* Install [gym-super-mario-bros](https://pypi.org/project/gym-super-mario-bros/)
+```
+pip install gym-super-mario-bros
+```
+
+# Running
+
+## Train
+
+* Train with dueling dqn.
+```
+python duel_dqn.py
+```
+
+* Train with PPO.
+
+```
+python ppo.py
+```
+
+### Result
+* score.p : save total score every 50 episode
+* *.pth : save weight of q, q_target every 50 training
+
+
+## Evaluate
+* (Now, pre-trained agent has been corrupted😢)
+* Test and render trained agent.
+* To test our agent, we need 'q_target.pth' that generated at the training step.
+* (eval.py with PPO is not supported now)
+```
+python eval.py
+```
+* Or you can use your own agent.
+```
+python eval.py your_own_agent.pth
+```
+
+## Reference
+[Wang, Ziyu, et al. "Dueling network architectures for deep reinforcement learning." International conference on machine learning. PMLR, 2016.](https://arxiv.org/pdf/1511.06581.pdf)
+
+[Schulman, J., Wolski, F., Dhariwal, P., Radford, A. & Klimov, O. Proximal policy optimization
+algorithms. arXiv preprint arXiv:1707.06347 (2017).](https://arxiv.org/pdf/1707.06347)

Super-Mario-RL/duel_dqn.py

@@ -0,0 +1,178 @@
+import pickle
+import random
+import time
+from collections import deque
+
+import gym_super_mario_bros
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+from gym_super_mario_bros.actions import COMPLEX_MOVEMENT
+from nes_py.wrappers import JoypadSpace
+
+from wrappers import *
+
+
+def arrange(s):
+    if not type(s) == "numpy.ndarray":
+        s = np.array(s)
+    assert len(s.shape) == 3
+    ret = np.transpose(s, (2, 0, 1))
+    return np.expand_dims(ret, 0)
+
+
+class replay_memory(object):
+    def __init__(self, N):
+        self.memory = deque(maxlen=N)
+
+    def push(self, transition):
+        self.memory.append(transition)
+
+    def sample(self, n):
+        return random.sample(self.memory, n)
+
+    def __len__(self):
+        return len(self.memory)
+
+
+class model(nn.Module):
+    def __init__(self, n_frame, n_action, device):
+        super(model, self).__init__()
+        self.layer1 = nn.Conv2d(n_frame, 32, 8, 4)
+        self.layer2 = nn.Conv2d(32, 64, 3, 1)
+        self.fc = nn.Linear(20736, 512)
+        self.q = nn.Linear(512, n_action)
+        self.v = nn.Linear(512, 1)
+
+        self.device = device
+        self.seq = nn.Sequential(self.layer1, self.layer2, self.fc, self.q, self.v)
+
+        self.seq.apply(init_weights)
+
+    def forward(self, x):
+        if type(x) != torch.Tensor:
+            x = torch.FloatTensor(x).to(self.device)
+        x = torch.relu(self.layer1(x))
+        x = torch.relu(self.layer2(x))
+        x = x.view(-1, 20736)
+        x = torch.relu(self.fc(x))
+        adv = self.q(x)
+        v = self.v(x)
+        q = v + (adv - 1 / adv.shape[-1] * adv.sum(-1, keepdim=True))
+
+        return q
+
+
+def init_weights(m):
+    if type(m) == nn.Conv2d:
+        torch.nn.init.xavier_uniform_(m.weight)
+        m.bias.data.fill_(0.01)
+
+
+def train(q, q_target, memory, batch_size, gamma, optimizer, device):
+    s, r, a, s_prime, done = list(map(list, zip(*memory.sample(batch_size))))
+    s = np.array(s).squeeze()
+    s_prime = np.array(s_prime).squeeze()
+    a_max = q(s_prime).max(1)[1].unsqueeze(-1)
+    r = torch.FloatTensor(r).unsqueeze(-1).to(device)
+    done = torch.FloatTensor(done).unsqueeze(-1).to(device)
+    with torch.no_grad():
+        y = r + gamma * q_target(s_prime).gather(1, a_max) * done
+    a = torch.tensor(a).unsqueeze(-1).to(device)
+    q_value = torch.gather(q(s), dim=1, index=a.view(-1, 1).long())
+
+    loss = F.smooth_l1_loss(q_value, y).mean()
+    optimizer.zero_grad()
+    loss.backward()
+    optimizer.step()
+    return loss
+
+
+def copy_weights(q, q_target):
+    q_dict = q.state_dict()
+    q_target.load_state_dict(q_dict)
+
+
+def main(env, q, q_target, optimizer, device):
+    t = 0
+    gamma = 0.99
+    batch_size = 256
+
+    N = 50000
+    eps = 0.001
+    memory = replay_memory(N)
+    update_interval = 50
+    print_interval = 10
+
+    score_lst = []
+    total_score = 0.0
+    loss = 0.0
+    start_time = time.perf_counter()
+
+    for k in range(1000000):
+        s = arrange(env.reset())
+        done = False
+
+        while not done:
+            if eps > np.random.rand():
+                a = env.action_space.sample()
+            else:
+                if device == "cpu":
+                    a = np.argmax(q(s).detach().numpy())
+                else:
+                    a = np.argmax(q(s).cpu().detach().numpy())
+            s_prime, r, done, _ = env.step(a)
+            s_prime = arrange(s_prime)
+            total_score += r
+            r = np.sign(r) * (np.sqrt(abs(r) + 1) - 1) + 0.001 * r
+            memory.push((s, float(r), int(a), s_prime, int(1 - done)))
+            s = s_prime
+            stage = env.unwrapped._stage
+            if len(memory) > 2000:
+                loss += train(q, q_target, memory, batch_size, gamma, optimizer, device)
+                t += 1
+            if t % update_interval == 0:
+                copy_weights(q, q_target)
+                torch.save(q.state_dict(), "mario_q.pth")
+                torch.save(q_target.state_dict(), "mario_q_target.pth")
+
+        if k % print_interval == 0:
+            time_spent, start_time = (
+                time.perf_counter() - start_time,
+                time.perf_counter(),
+            )
+            print(
+                "%s |Epoch : %d | score : %f | loss : %.2f | stage : %d | time spent: %f"
+                % (
+                    device,
+                    k,
+                    total_score / print_interval,
+                    loss / print_interval,
+                    stage,
+                    time_spent,
+                )
+            )
+            score_lst.append(total_score / print_interval)
+            total_score = 0
+            loss = 0.0
+            pickle.dump(score_lst, open("score.p", "wb"))
+
+
+if __name__ == "__main__":
+    n_frame = 4
+    env = gym_super_mario_bros.make("SuperMarioBros-v0")
+    env = JoypadSpace(env, COMPLEX_MOVEMENT)
+    env = wrap_mario(env)
+    device = "cpu"
+    if torch.cuda.is_available():
+        device = "cuda"
+    elif torch.backends.mps.is_available():
+        device = "mps"
+    q = model(n_frame, env.action_space.n, device).to(device)
+    q_target = model(n_frame, env.action_space.n, device).to(device)
+    optimizer = optim.Adam(q.parameters(), lr=0.0001)
+    print(device)
+
+    main(env, q, q_target, optimizer, device)

Super-Mario-RL/duel_dqn_2.py

@@ -0,0 +1,237 @@
+import pickle
+import random
+import time
+from collections import deque
+
+import gym_super_mario_bros
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+from gym_super_mario_bros.actions import COMPLEX_MOVEMENT
+from nes_py.wrappers import JoypadSpace
+
+from wrappers import *
+
+
+def arrange(s):
+    if not type(s) == "numpy.ndarray":
+        s = np.array(s)
+    assert len(s.shape) == 3
+    ret = np.transpose(s, (2, 0, 1))
+    return np.expand_dims(ret, 0)
+
+
+class replay_memory(object):
+    def __init__(self, N):
+        self.memory = deque(maxlen=N)
+
+    def push(self, transition):
+        self.memory.append(transition)
+
+    def sample(self, n):
+        return random.sample(self.memory, n)
+
+    def __len__(self):
+        return len(self.memory)
+
+
+class model(nn.Module):
+    def __init__(self, n_frame, n_action, device):
+        super(model, self).__init__()
+        self.layer1 = nn.Conv2d(n_frame, 32, 8, 4)
+        self.layer2 = nn.Conv2d(32, 64, 3, 1)
+        self.fc = nn.Linear(20736, 512)
+        self.q = nn.Linear(512, n_action)
+        self.v = nn.Linear(512, 1)
+
+        self.device = device
+        self.seq = nn.Sequential(self.layer1, self.layer2, self.fc, self.q, self.v)
+
+        self.seq.apply(init_weights)
+
+    def forward(self, x):
+        if type(x) != torch.Tensor:
+            x = torch.FloatTensor(x).to(self.device)
+        x = torch.relu(self.layer1(x))
+        x = torch.relu(self.layer2(x))
+        x = x.view(-1, 20736)
+        x = torch.relu(self.fc(x))
+        adv = self.q(x)
+        v = self.v(x)
+        q = v + (adv - 1 / adv.shape[-1] * adv.sum(-1, keepdim=True))
+
+        return q
+
+
+def init_weights(m):
+    if type(m) == nn.Conv2d:
+        torch.nn.init.xavier_uniform_(m.weight)
+        m.bias.data.fill_(0.01)
+
+
+def train(q, q_target, memory, batch_size, gamma, optimizer, device):
+    s, r, a, s_prime, done = list(map(list, zip(*memory.sample(batch_size))))
+    s = np.array(s).squeeze()
+    s_prime = np.array(s_prime).squeeze()
+    
+    # Move computations to device
+    s_tensor = torch.FloatTensor(s).to(device)
+    s_prime_tensor = torch.FloatTensor(s_prime).to(device)
+    
+    a_max = q(s_prime_tensor).max(1)[1].unsqueeze(-1)
+    r = torch.FloatTensor(r).unsqueeze(-1).to(device)
+    done = torch.FloatTensor(done).unsqueeze(-1).to(device)
+    
+    with torch.no_grad():
+        y = r + gamma * q_target(s_prime_tensor).gather(1, a_max) * done
+    
+    a = torch.tensor(a).unsqueeze(-1).to(device)
+    q_value = torch.gather(q(s_tensor), dim=1, index=a.view(-1, 1).long())
+
+    loss = F.smooth_l1_loss(q_value, y).mean()
+    optimizer.zero_grad()
+    loss.backward()
+    
+    # Gradient clipping to prevent explosion
+    torch.nn.utils.clip_grad_norm_(q.parameters(), max_norm=1.0)
+    
+    optimizer.step()
+    return loss.item()  # Use .item() to get scalar value
+
+
+def copy_weights(q, q_target):
+    q_dict = q.state_dict()
+    q_target.load_state_dict(q_dict)
+
+
+def main(env, q, q_target, optimizer, device):
+    t = 0
+    gamma = 0.99
+    batch_size = 256
+
+    N = 50000
+    eps = 0.1  # Increased exploration
+    eps_min = 0.01
+    eps_decay = 0.999
+    memory = replay_memory(N)
+    update_interval = 50  # How often to update target network
+    save_interval = 100   # How often to save models (in episodes)
+    print_interval = 10
+
+    score_lst = []
+    total_score = 0.0
+    loss_accumulator = 0.0
+    start_time = time.perf_counter()
+
+    for k in range(1000000):
+        s = arrange(env.reset())
+        done = False
+
+        while not done:
+            # Epsilon decay
+            if eps > eps_min:
+                eps *= eps_decay
+                
+            if eps > np.random.rand():
+                a = env.action_space.sample()
+            else:
+                # Get action with proper device handling
+                with torch.no_grad():
+                    q_values = q(torch.FloatTensor(s).to(device))
+                
+                # Move to CPU for numpy conversion regardless of device
+                if device == "cuda" or device == "mps":
+                    a = np.argmax(q_values.cpu().numpy())
+                else:
+                    a = np.argmax(q_values.detach().numpy())
+                    
+            s_prime, r, done, info = env.step(a)
+            s_prime = arrange(s_prime)
+            total_score += r
+            
+            # Enhanced reward shaping
+            reward = np.sign(r) * (np.sqrt(abs(r) + 1) - 1) + 0.001 * r
+            
+            # Bonus for x_pos progress
+            if 'x_pos' in info:
+                x_pos = info['x_pos']
+                if hasattr(main, 'last_x_pos'):
+                    x_progress = x_pos - main.last_x_pos
+                    if x_progress > 0:
+                        reward += 0.1 * x_progress  # Small bonus for moving right
+                main.last_x_pos = x_pos
+            
+            memory.push((s, float(reward), int(a), s_prime, int(1 - done)))
+            s = s_prime
+            stage = env.unwrapped._stage
+            
+            if len(memory) > 2000:
+                loss_val = train(q, q_target, memory, batch_size, gamma, optimizer, device)
+                loss_accumulator += loss_val
+                t += 1
+                
+                # Update target network (but don't save every time)
+                if t % update_interval == 0:
+                    copy_weights(q, q_target)
+
+        # Save models less frequently (every save_interval episodes)
+        if k % save_interval == 0 and k > 0:
+            torch.save(q.state_dict(), "mario_q.pth")
+            torch.save(q_target.state_dict(), "mario_q_target.pth")
+            print(f"Models saved at episode {k}")
+
+        if k % print_interval == 0:
+            time_spent, start_time = (
+                time.perf_counter() - start_time,
+                time.perf_counter(),
+            )
+            
+            # Fixed: Use loss_accumulator instead of loss and ensure proper formatting
+            avg_loss = loss_accumulator / print_interval if print_interval > 0 else 0.0
+            avg_score = total_score / print_interval if print_interval > 0 else 0.0
+            
+            print(
+                "%s | Epoch : %d | score : %.2f | loss : %.2f | stage : %d | eps : %.3f | time: %.2fs | memory: %d"
+                % (
+                    device,
+                    k,
+                    avg_score,
+                    avg_loss,
+                    stage,
+                    eps,
+                    time_spent,
+                    len(memory)
+                )
+            )
+            score_lst.append(avg_score)
+            total_score = 0.0
+            loss_accumulator = 0.0
+            pickle.dump(score_lst, open("score.p", "wb"))
+
+
+if __name__ == "__main__":
+    n_frame = 4
+    env = gym_super_mario_bros.make("SuperMarioBros-v3")
+    env = JoypadSpace(env, COMPLEX_MOVEMENT)
+    env = wrap_mario(env)
+    
+    # Device detection with MPS support
+    device = "cpu"
+    if torch.cuda.is_available():
+        device = "cuda"
+    elif torch.backends.mps.is_available():
+        device = "mps"
+    
+    print(f"Using device: {device}")
+    
+    q = model(n_frame, env.action_space.n, device).to(device)
+    q_target = model(n_frame, env.action_space.n, device).to(device)
+    
+    # Copy weights initially
+    copy_weights(q, q_target)
+    
+    optimizer = optim.Adam(q.parameters(), lr=0.0001, weight_decay=1e-5)  # Added weight decay
+
+    main(env, q, q_target, optimizer, device)

Super-Mario-RL/enhanced_duel_dqn.py

@@ -0,0 +1,257 @@
+import pickle
+import random
+import time
+from collections import deque
+
+import gym_super_mario_bros
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+from gym_super_mario_bros.actions import COMPLEX_MOVEMENT
+from nes_py.wrappers import JoypadSpace
+
+from wrappers import *
+
+
+def arrange(s):
+    if not type(s) == "numpy.ndarray":
+        s = np.array(s)
+    assert len(s.shape) == 3
+    ret = np.transpose(s, (2, 0, 1))
+    return np.expand_dims(ret, 0)
+
+
+class replay_memory(object):
+    def __init__(self, N):
+        self.memory = deque(maxlen=N)
+
+    def push(self, transition):
+        self.memory.append(transition)
+
+    def sample(self, n):
+        return random.sample(self.memory, n)
+
+    def __len__(self):
+        return len(self.memory)
+
+
+class model(nn.Module):
+    def __init__(self, n_frame, n_action, device):
+        super(model, self).__init__()
+        self.layer1 = nn.Conv2d(n_frame, 32, 8, 4)
+        self.layer2 = nn.Conv2d(32, 64, 3, 1)
+        self.fc = nn.Linear(20736, 512)
+        self.q = nn.Linear(512, n_action)
+        self.v = nn.Linear(512, 1)
+
+        self.device = device
+        self.seq = nn.Sequential(self.layer1, self.layer2, self.fc, self.q, self.v)
+
+        self.seq.apply(init_weights)
+
+    def forward(self, x):
+        if type(x) != torch.Tensor:
+            x = torch.FloatTensor(x).to(self.device)
+        x = torch.relu(self.layer1(x))
+        x = torch.relu(self.layer2(x))
+        x = x.view(-1, 20736)
+        x = torch.relu(self.fc(x))
+        adv = self.q(x)
+        v = self.v(x)
+        q = v + (adv - 1 / adv.shape[-1] * adv.sum(-1, keepdim=True))
+
+        return q
+
+
+def init_weights(m):
+    if type(m) == nn.Conv2d:
+        torch.nn.init.xavier_uniform_(m.weight)
+        m.bias.data.fill_(0.01)
+
+
+def train(q, q_target, memory, batch_size, gamma, optimizer, device):
+    s, r, a, s_prime, done = list(map(list, zip(*memory.sample(batch_size))))
+    s = np.array(s).squeeze()
+    s_prime = np.array(s_prime).squeeze()
+    
+    # Move computations to device
+    s_tensor = torch.FloatTensor(s).to(device)
+    s_prime_tensor = torch.FloatTensor(s_prime).to(device)
+    
+    a_max = q(s_prime_tensor).max(1)[1].unsqueeze(-1)
+    r = torch.FloatTensor(r).unsqueeze(-1).to(device)
+    done = torch.FloatTensor(done).unsqueeze(-1).to(device)
+    
+    with torch.no_grad():
+        y = r + gamma * q_target(s_prime_tensor).gather(1, a_max) * done
+    
+    a = torch.tensor(a).unsqueeze(-1).to(device)
+    q_value = torch.gather(q(s_tensor), dim=1, index=a.view(-1, 1).long())
+
+    loss = F.smooth_l1_loss(q_value, y).mean()
+    optimizer.zero_grad()
+    loss.backward()
+    
+    # Gradient clipping to prevent explosion
+    torch.nn.utils.clip_grad_norm_(q.parameters(), max_norm=10.0)  # Increased clipping
+    
+    optimizer.step()
+    return loss.item()
+
+
+def copy_weights(q, q_target):
+    q_dict = q.state_dict()
+    q_target.load_state_dict(q_dict)
+
+
+def main(env, q, q_target, optimizer, device):
+    t = 0
+    gamma = 0.99
+    batch_size = 256
+
+    N = 50000
+    eps = 0.3  # Higher initial exploration
+    eps_min = 0.05  # Higher minimum exploration
+    eps_decay = 0.995  # Slower decay
+    memory = replay_memory(N)
+    update_interval = 100  # Less frequent target updates
+    save_interval = 100
+    print_interval = 10
+
+    score_lst = []
+    total_score = 0.0
+    loss_accumulator = 0.0
+    start_time = time.perf_counter()
+    
+    # Track best score for saving
+    best_score = -float('inf')
+
+    for k in range(1000000):
+        s = arrange(env.reset())
+        done = False
+        episode_loss = 0.0
+        episode_steps = 0
+
+        while not done:
+            # Epsilon decay per step
+            if eps > eps_min:
+                eps *= eps_decay
+                
+            if eps > np.random.rand():
+                a = env.action_space.sample()
+            else:
+                with torch.no_grad():
+                    q_values = q(torch.FloatTensor(s).to(device))
+                
+                if device == "cuda" or device == "mps":
+                    a = np.argmax(q_values.cpu().numpy())
+                else:
+                    a = np.argmax(q_values.detach().numpy())
+                    
+            s_prime, r, done, info = env.step(a)
+            s_prime = arrange(s_prime)
+            total_score += r
+            
+            # Enhanced reward shaping
+            reward = np.sign(r) * (np.sqrt(abs(r) + 1) - 1) + 0.001 * r
+            
+            # Bonus for x_pos progress and stage completion
+            if 'x_pos' in info:
+                x_pos = info['x_pos']
+                if hasattr(main, 'last_x_pos'):
+                    x_progress = x_pos - main.last_x_pos
+                    if x_progress > 0:
+                        reward += 0.05 * x_progress  # Reduced bonus to prevent over-optimization
+                main.last_x_pos = x_pos
+            
+            # Large bonus for completing the level
+            if done and info.get('flag_get', False):
+                reward += 50.0
+                print(f"🎉 LEVEL COMPLETED at episode {k}! 🎉")
+            
+            memory.push((s, float(reward), int(a), s_prime, int(1 - done)))
+            s = s_prime
+            stage = info.get('stage', 1)
+            world = info.get('world', 1)
+            
+            # Train only if we have enough samples
+            if len(memory) > 5000:  # Increased minimum buffer size
+                loss_val = train(q, q_target, memory, batch_size, gamma, optimizer, device)
+                loss_accumulator += loss_val
+                episode_loss += loss_val
+                episode_steps += 1
+                t += 1
+                
+                # Update target network less frequently
+                if t % update_interval == 0:
+                    copy_weights(q, q_target)
+
+        # Save best model
+        current_avg_score = total_score / print_interval if k % print_interval == 0 else total_score
+        if current_avg_score > best_score and k > 0:
+            best_score = current_avg_score
+            torch.save(q.state_dict(), "enhanced_mario_q_best.pth")
+            torch.save(q_target.state_dict(), "enhanced_mario_q_target_best.pth")
+            print(f"💾 New best model saved! Score: {best_score:.2f}")
+
+        # Save models periodically
+        if k % save_interval == 0 and k > 0:
+            torch.save(q.state_dict(), "enhanced_mario_q.pth")
+            torch.save(q_target.state_dict(), "enhanced_mario_q_target.pth")
+            print(f"Models saved at episode {k}")
+
+        if k % print_interval == 0:
+            time_spent, start_time = (
+                time.perf_counter() - start_time,
+                time.perf_counter(),
+            )
+            
+            avg_loss = loss_accumulator / (print_interval * episode_steps) if episode_steps > 0 else 0.0
+            avg_score = total_score / print_interval
+            
+            print(
+                "%s | Ep: %d | Score: %.2f | Loss: %.2f | Stage: %d-%d | Eps: %.3f | Time: %.2fs | Mem: %d | Steps: %d"
+                % (
+                    device,
+                    k,
+                    avg_score,
+                    avg_loss,
+                    world,
+                    stage,
+                    eps,
+                    time_spent,
+                    len(memory),
+                    episode_steps
+                )
+            )
+            score_lst.append(avg_score)
+            total_score = 0.0
+            loss_accumulator = 0.0
+            pickle.dump(score_lst, open("score.p", "wb"))
+
+
+if __name__ == "__main__":
+    n_frame = 4
+    env = gym_super_mario_bros.make("SuperMarioBros-v3")
+    env = JoypadSpace(env, COMPLEX_MOVEMENT)
+    env = wrap_mario(env)
+    
+    device = "cpu"
+    if torch.cuda.is_available():
+        device = "cuda"
+    elif torch.backends.mps.is_available():
+        device = "mps"
+    
+    print(f"Using device: {device}")
+    
+    q = model(n_frame, env.action_space.n, device).to(device)
+    q_target = model(n_frame, env.action_space.n, device).to(device)
+    
+    copy_weights(q, q_target)
+    
+    # Lower learning rate for stability
+    optimizer = optim.Adam(q.parameters(), lr=0.00005, weight_decay=1e-5)
+
+    main(env, q, q_target, optimizer, device)

Super-Mario-RL/enhanced_mario_q.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:034fc1bde429fd3e9bdde72fb16697707a604dad8db737f49f8e61ad5b442026
+size 42607893

Super-Mario-RL/enhanced_mario_q_best.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6874204a382dc834d328f078a05110c793ead351eee5034b9f37f09ed6c11b9
+size 42607973

Super-Mario-RL/enhanced_mario_q_target.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fa5448953e60e5be93f7a6a0843d11af0a279a6e13adafd36cb31f025fdf914
+size 42608069

Super-Mario-RL/enhanced_mario_q_target_best.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a5f341b08473c536af3398f2b984006c55d9e96952b0b0f5263bf1cdd7f7917
+size 42608213

Super-Mario-RL/eval.py

@@ -0,0 +1,111 @@
+import sys
+import time
+
+import gym_super_mario_bros
+import torch
+import torch.nn as nn
+from gym_super_mario_bros.actions import COMPLEX_MOVEMENT
+from nes_py.wrappers import JoypadSpace
+
+from wrappers import *
+
+# Device detection
+device = "cpu"
+if torch.cuda.is_available():
+    device = "cuda"
+elif torch.backends.mps.is_available():
+    device = "mps"
+
+print(f"Using device: {device}")
+
+
+# Same as duel_dqn.mlp (you can make model.py to avoid duplication.)
+class model(nn.Module):
+    def __init__(self, n_frame, n_action, device):
+        super(model, self).__init__()
+        self.layer1 = nn.Conv2d(n_frame, 32, 8, 4)
+        self.layer2 = nn.Conv2d(32, 64, 3, 1)
+        self.fc = nn.Linear(20736, 512)
+        self.q = nn.Linear(512, n_action)
+        self.v = nn.Linear(512, 1)
+
+        self.device = device
+        self.seq = nn.Sequential(self.layer1, self.layer2, self.fc, self.q, self.v)
+
+        self.seq.apply(init_weights)
+
+    def forward(self, x):
+        if type(x) != torch.Tensor:
+            x = torch.FloatTensor(x).to(self.device)
+        x = torch.relu(self.layer1(x))
+        x = torch.relu(self.layer2(x))
+        x = x.view(-1, 20736)
+        x = torch.relu(self.fc(x))
+        adv = self.q(x)
+        v = self.v(x)
+        q = v + (adv - 1 / adv.shape[-1] * adv.max(-1, True)[0])
+        return q
+
+
+def init_weights(m):
+    if type(m) == nn.Conv2d:
+        torch.nn.init.xavier_uniform_(m.weight)
+        m.bias.data.fill_(0.01)
+
+
+def arange(s):
+    if not type(s) == "numpy.ndarray":
+        s = np.array(s)
+    assert len(s.shape) == 3
+    ret = np.transpose(s, (2, 0, 1))
+    return np.expand_dims(ret, 0)
+
+
+if __name__ == "__main__":
+    ckpt_path = sys.argv[1] if len(sys.argv) > 1 else "mario_q_target.pth"
+    print(f"Load ckpt from {ckpt_path}")
+    n_frame = 4
+    env = gym_super_mario_bros.make("SuperMarioBros-v0")
+    env = JoypadSpace(env, COMPLEX_MOVEMENT)
+    env = wrap_mario(env)
+    
+    q = model(n_frame, env.action_space.n, device).to(device)
+
+    # Load model with proper device mapping
+    try:
+        q.load_state_dict(torch.load(ckpt_path, map_location=torch.device(device)))
+        print(f"Model loaded successfully on {device}")
+    except Exception as e:
+        print(f"Error loading model with {device}: {e}")
+        print("Trying to load with CPU mapping...")
+        q.load_state_dict(torch.load(ckpt_path, map_location="cpu"))
+        q = q.to(device)
+        print(f"Model loaded with CPU mapping and moved to {device}")
+    
+    total_score = 0.0
+    done = False
+    s = arange(env.reset())
+    i = 0
+    
+    # Evaluation loop
+    while not done:
+        env.render()
+        
+        # Get Q-values and action
+        with torch.no_grad():
+            q_values = q(s)
+            
+        # Move to CPU for numpy conversion regardless of device
+        if device == "cuda" or device == "mps":
+            a = np.argmax(q_values.cpu().numpy())
+        else:
+            a = np.argmax(q_values.detach().numpy())
+            
+        s_prime, r, done, _ = env.step(a)
+        s_prime = arange(s_prime)
+        total_score += r
+        s = s_prime
+        time.sleep(0.001)
+
+    stage = env.unwrapped._stage
+    print("Total score : %f | stage : %d" % (total_score, stage))

Super-Mario-RL/mario1.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c3e5f49d302a68348659adba9a7f9f1be4d57fd3204214a191a47234aea0cd0
+size 643908

Super-Mario-RL/mario14.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52efcbda754d016c56b6cf67d50da683988061b31ae7d43217f995a5db89474a
+size 547827

Super-Mario-RL/mario_q.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e1d46a2e2822ff25428906b6964f578bcf2f16526cf4edecef0ded6350499d6
+size 42607237

Super-Mario-RL/mario_q_best.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d286a832aaeae13128121500fe34759a50f0e587df5cfd9ac83d780b181ed340
+size 42607829

Super-Mario-RL/mario_q_target.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7811192034bc02b5236043c3bfc8e982038c970bcd6e948b82cdac18706a964
+size 39059456

Super-Mario-RL/mario_q_target_best.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c358203988162769284fe455cdc9fc047e69e6672d4ec05066f79a20dd54404c
+size 42607941

Super-Mario-RL/ppo.py

@@ -0,0 +1,272 @@
+from collections import Counter
+
+import gym_super_mario_bros
+import gymnasium as gym
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from gym_super_mario_bros.actions import COMPLEX_MOVEMENT
+from nes_py.wrappers import JoypadSpace
+
+from wrappers import *
+
+device = "cpu"
+if torch.cuda.is_available():
+    device = "cuda"
+elif torch.backends.mps.is_available():
+    device = "mps"
+
+print(f"Using device: {device}")
+
+
+def make_env():
+    env = gym_super_mario_bros.make("SuperMarioBros-v0")
+    env = JoypadSpace(env, COMPLEX_MOVEMENT)
+    env = wrap_mario(env)
+    return env
+
+
+def get_reward(r):
+    r = np.sign(r) * (np.sqrt(abs(r) + 1) - 1) + 0.001 * r
+    return r
+
+
+class ActorCritic(nn.Module):
+    def __init__(self, n_frame, act_dim):
+        super().__init__()
+        self.net = nn.Sequential(
+            nn.Conv2d(n_frame, 32, 8, 4),
+            nn.ReLU(),
+            nn.Conv2d(32, 64, 3, 1),
+            nn.ReLU(),
+        )
+        self.linear = nn.Linear(20736, 512)
+        self.policy_head = nn.Linear(512, act_dim)
+        self.value_head = nn.Linear(512, 1)
+
+    def forward(self, x):
+        if x.dim() == 4:
+            x = x.permute(0, 3, 1, 2)
+        elif x.dim() == 3:
+            x = x.permute(2, 0, 1)
+        x = self.net(x)
+        x = x.reshape(-1, 20736)
+        x = torch.relu(self.linear(x))
+
+        return self.policy_head(x), self.value_head(x).squeeze(-1)
+
+    def act(self, obs):
+        logits, value = self.forward(obs)
+        dist = torch.distributions.Categorical(logits=logits)
+        action = dist.sample()
+        logprob = dist.log_prob(action)
+        return action, logprob, value
+
+
+def compute_gae_batch(rewards, values, dones, gamma=0.99, lam=0.95):
+    T, N = rewards.shape
+    advantages = torch.zeros_like(rewards)
+    gae = torch.zeros(N, device=device)
+
+    for t in reversed(range(T)):
+        not_done = 1.0 - dones[t]
+        delta = rewards[t] + gamma * values[t + 1] * not_done - values[t]
+        gae = delta + gamma * lam * not_done * gae
+        advantages[t] = gae
+
+    returns = advantages + values[:-1]
+    return advantages, returns
+
+
+def rollout_with_bootstrap(envs, model, rollout_steps, init_obs):
+    obs = init_obs
+    obs = torch.tensor(obs, dtype=torch.float32).to(device)
+    obs_buf, act_buf, rew_buf, done_buf, val_buf, logp_buf = [], [], [], [], [], []
+
+    for _ in range(rollout_steps):
+        obs_buf.append(obs)
+
+        with torch.no_grad():
+            action, logp, value = model.act(obs)
+
+        val_buf.append(value)
+        logp_buf.append(logp)
+        act_buf.append(action)
+
+        actions = action.cpu().numpy()
+        next_obs, reward, done, infos = envs.step(actions)
+
+        reward = [get_reward(r) for r in reward]
+        # done = np.logical_or(terminated)
+
+        rew_buf.append(torch.tensor(reward, dtype=torch.float32).to(device))
+        done_buf.append(torch.tensor(done, dtype=torch.float32).to(device))
+
+        for i, d in enumerate(done):
+            if d:
+                print(f"Env {i} done. Resetting. (info: {infos[i]})")
+                next_obs[i] = envs.envs[i].reset()
+
+        obs = torch.tensor(next_obs, dtype=torch.float32).to(device)
+        max_stage = max([i["stage"] for i in infos])
+
+    with torch.no_grad():
+        _, last_value = model.forward(obs)
+
+    obs_buf = torch.stack(obs_buf)
+    act_buf = torch.stack(act_buf)
+    rew_buf = torch.stack(rew_buf)
+    done_buf = torch.stack(done_buf)
+    val_buf = torch.stack(val_buf)
+    val_buf = torch.cat([val_buf, last_value.unsqueeze(0)], dim=0)
+    logp_buf = torch.stack(logp_buf)
+
+    adv_buf, ret_buf = compute_gae_batch(rew_buf, val_buf, done_buf)
+    adv_buf = (adv_buf - adv_buf.mean()) / (adv_buf.std() + 1e-8)
+
+    return {
+        "obs": obs_buf,  # [T, N, obs_dim]
+        "actions": act_buf,
+        "logprobs": logp_buf,
+        "advantages": adv_buf,
+        "returns": ret_buf,
+        "max_stage": max_stage,
+        "last_obs": obs,
+    }
+
+
+def evaluate_policy(env, model, episodes=5, render=False):
+    """
+    Function to evaluate the learned policy
+
+    Args:
+    env: gym.Env single environment (not vector!)
+
+    model: ActorCritic model
+
+    episodes: number of episodes to evaluate
+
+    render: whether to visualize (if True, display on screen)
+
+    Returns:
+    avg_return: average total reward
+    """
+    model.eval()
+    total_returns = []
+    actions = []
+    stages = []
+    for ep in range(episodes):
+        obs = env.reset()
+        done = False
+        total_reward = 0
+        if render:
+            env.render()
+        while not done:
+            obs_tensor = (
+                torch.tensor(np.array(obs), dtype=torch.float32).unsqueeze(0).to(device)
+            )
+            with torch.no_grad():
+                logits, _ = model(obs_tensor)
+                dist = torch.distributions.Categorical(logits=logits)
+                action = dist.probs.argmax(dim=-1).item()  # greedy action
+                actions.append(action)
+
+            obs, reward, done, info = env.step(action)
+            stages.append(info["stage"])
+            total_reward += reward
+
+        total_returns.append(total_reward)
+        info["action_count"] = Counter(actions)
+    model.train()
+    return np.mean(total_returns), info, max(stages)
+
+
+def train_ppo():
+    num_env = 8
+    envs = gym.vector.SyncVectorEnv([lambda: make_env() for _ in range(num_env)])
+    obs_dim = envs.single_observation_space.shape[-1]
+    act_dim = envs.single_action_space.n
+    print(f"{obs_dim=} {act_dim=}")
+    model = ActorCritic(obs_dim, act_dim).to(device)
+    
+    # Load model with proper device mapping
+    try:
+        # Try to load with current device first
+        model.load_state_dict(torch.load("mario_1_1.pt", map_location=device))
+        print("Model loaded successfully with current device mapping")
+    except:
+        try:
+            # If that fails, try loading with CPU and then moving to device
+            model.load_state_dict(torch.load("mario_1_1.pt", map_location="cpu"))
+            model = model.to(device)
+            print("Model loaded successfully with CPU mapping")
+        except Exception as e:
+            print(f"Failed to load model: {e}")
+            print("Starting with fresh model")
+    
+    optimizer = optim.Adam(model.parameters(), lr=2.5e-4)
+
+    rollout_steps = 128
+    epochs = 4
+    minibatch_size = 64
+    clip_eps = 0.2
+    vf_coef = 0.5
+    ent_coef = 0.01
+    eval_env = make_env()
+    eval_env.reset()
+
+    init_obs = envs.reset()
+    update = 0
+    while True:
+        update += 1
+        batch = rollout_with_bootstrap(envs, model, rollout_steps, init_obs)
+        init_obs = batch["last_obs"]
+
+        T, N = rollout_steps, envs.num_envs
+        total_size = T * N
+
+        obs = batch["obs"].reshape(total_size, *envs.single_observation_space.shape)
+        act = batch["actions"].reshape(total_size)
+        logp_old = batch["logprobs"].reshape(total_size)
+        adv = batch["advantages"].reshape(total_size)
+        ret = batch["returns"].reshape(total_size)
+
+        for _ in range(epochs):
+            idx = torch.randperm(total_size)
+            for start in range(0, total_size, minibatch_size):
+                i = idx[start : start + minibatch_size]
+                logits, value = model(obs[i])
+                dist = torch.distributions.Categorical(logits=logits)
+                logp = dist.log_prob(act[i])
+                ratio = torch.exp(logp - logp_old[i])
+                clipped = torch.clamp(ratio, 1 - clip_eps, 1 + clip_eps) * adv[i]
+                policy_loss = -torch.min(ratio * adv[i], clipped).mean()
+                value_loss = (ret[i] - value).pow(2).mean()
+                entropy = dist.entropy().mean()
+                loss = policy_loss + vf_coef * value_loss - ent_coef * entropy
+
+                optimizer.zero_grad()
+                loss.backward()
+                optimizer.step()
+
+        # logging
+        avg_return = batch["returns"].mean().item()
+        max_stage = batch["max_stage"]
+        print(f"Update {update}: avg return = {avg_return:.2f} {max_stage=}")
+
+        # eval and save
+        if update % 10 == 0:
+            avg_score, info, eval_max_stage = evaluate_policy(
+                eval_env, model, episodes=1, render=False
+            )
+            print(f"[Eval] Update {update}: avg return = {avg_score:.2f} info: {info}")
+            if eval_max_stage > 1:
+                torch.save(model.state_dict(), "mario_1_1_clear.pt")
+                break
+        if update > 0 and update % 50 == 0:
+            torch.save(model.state_dict(), "mario_1_1_ppo.pt")
+
+
+if __name__ == "__main__":
+    train_ppo()

Super-Mario-RL/requirements.txt

@@ -0,0 +1,8 @@
+numpy==1.26.4
+torch>=1.6.0
+torchvision
+gym==0.23
+nes-py
+gym-super-mario-bros==7.2.3
+opencv-python
+matplotlib

Super-Mario-RL/terminal.txt

@@ -0,0 +1,5 @@
+python enhanced_dual_dqn.py 
+python eval.py enhanced_mario_q_best.pth 
+python eval.py mario_q_target.pth
+python eval.py mario_q_best.pth
+python eval.py mario_q.pth

Super-Mario-RL/wrappers.py

@@ -0,0 +1,361 @@
+"""
+Code from OpenAI baseline
+https://github.com/openai/baselines/blob/master/baselines/common/atari_wrappers.py
+"""
+
+import os
+
+import numpy as np
+
+os.environ.setdefault("PATH", "")
+from collections import deque
+
+import cv2
+import gym
+from gym import spaces
+
+cv2.ocl.setUseOpenCL(False)
+from gym.wrappers import TimeLimit
+
+
+class NoopResetEnv(gym.Wrapper):
+    def __init__(self, env, noop_max=30):
+        """Sample initial states by taking random number of no-ops on reset.
+        No-op is assumed to be action 0.
+        """
+        gym.Wrapper.__init__(self, env)
+        self.noop_max = noop_max
+        self.override_num_noops = None
+        self.noop_action = 0
+        assert env.unwrapped.get_action_meanings()[0] == "NOOP"
+
+    def reset(self, **kwargs):
+        """Do no-op action for a number of steps in [1, noop_max]."""
+        self.env.reset(**kwargs)
+        if self.override_num_noops is not None:
+            noops = self.override_num_noops
+        else:
+            noops = self.unwrapped.np_random.randint(
+                1, self.noop_max + 1
+            )  # pylint: disable=E1101
+        assert noops > 0
+        obs = None
+        for _ in range(noops):
+            obs, _, done, _ = self.env.step(self.noop_action)
+            if done:
+                obs = self.env.reset(**kwargs)
+        return obs
+
+    def step(self, ac):
+        return self.env.step(ac)
+
+
+class FireResetEnv(gym.Wrapper):
+    def __init__(self, env):
+        """Take action on reset for environments that are fixed until firing."""
+        gym.Wrapper.__init__(self, env)
+        assert env.unwrapped.get_action_meanings()[1] == "FIRE"
+        assert len(env.unwrapped.get_action_meanings()) >= 3
+
+    def reset(self, **kwargs):
+        self.env.reset(**kwargs)
+        obs, _, done, _ = self.env.step(1)
+        if done:
+            self.env.reset(**kwargs)
+        obs, _, done, _ = self.env.step(2)
+        if done:
+            self.env.reset(**kwargs)
+        return obs
+
+    def step(self, ac):
+        return self.env.step(ac)
+
+
+class EpisodicLifeEnv(gym.Wrapper):
+    def __init__(self, env):
+        """Make end-of-life == end-of-episode, but only reset on true game over.
+        Done by DeepMind for the DQN and co. since it helps value estimation.
+        """
+        gym.Wrapper.__init__(self, env)
+        self.lives = 0
+        self.was_real_done = True
+
+    def step(self, action):
+        obs, reward, done, info = self.env.step(action)
+        self.was_real_done = done
+        # check current lives, make loss of life terminal,
+        # then update lives to handle bonus lives
+        lives = self.env.unwrapped.ale.lives()
+        if lives < self.lives and lives > 0:
+            # for Qbert sometimes we stay in lives == 0 condition for a few frames
+            # so it's important to keep lives > 0, so that we only reset once
+            # the environment advertises done.
+            done = True
+        self.lives = lives
+        return obs, reward, done, info
+
+    def reset(self, **kwargs):
+        """Reset only when lives are exhausted.
+        This way all states are still reachable even though lives are episodic,
+        and the learner need not know about any of this behind-the-scenes.
+        """
+        if self.was_real_done:
+            obs = self.env.reset(**kwargs)
+        else:
+            # no-op step to advance from terminal/lost life state
+            obs, _, _, _ = self.env.step(0)
+        self.lives = self.env.unwrapped.ale.lives()
+        return obs
+
+
+class MaxAndSkipEnv(gym.Wrapper):
+    def __init__(self, env, skip=4):
+        """Return only every `skip`-th frame"""
+        gym.Wrapper.__init__(self, env)
+        # most recent raw observations (for max pooling across time steps)
+        self._obs_buffer = np.zeros((2,) + env.observation_space.shape, dtype=np.uint8)
+        self._skip = skip
+
+    def step(self, action):
+        """Repeat action, sum reward, and max over last observations."""
+        total_reward = 0.0
+        done = None
+        for i in range(self._skip):
+            obs, reward, done, info = self.env.step(action)
+            if i == self._skip - 2:
+                self._obs_buffer[0] = obs
+            if i == self._skip - 1:
+                self._obs_buffer[1] = obs
+            total_reward += reward
+            if done:
+                break
+        # Note that the observation on the done=True frame
+        # doesn't matter
+        max_frame = self._obs_buffer.max(axis=0)
+
+        return max_frame, total_reward, done, info
+
+    def reset(self, **kwargs):
+        return self.env.reset(**kwargs)
+
+
+class ClipRewardEnv(gym.RewardWrapper):
+    def __init__(self, env):
+        gym.RewardWrapper.__init__(self, env)
+
+    def reward(self, reward):
+        """Bin reward to {+1, 0, -1} by its sign."""
+        return np.sign(reward)
+
+
+class WarpFrame(gym.ObservationWrapper):
+    def __init__(self, env, width=84, height=84, grayscale=True, dict_space_key=None):
+        """
+        Warp frames to 84x84 as done in the Nature paper and later work.
+        If the environment uses dictionary observations, `dict_space_key` can be specified which indicates which
+        observation should be warped.
+        """
+        super().__init__(env)
+        self._width = width
+        self._height = height
+        self._grayscale = grayscale
+        self._key = dict_space_key
+        if self._grayscale:
+            num_colors = 1
+        else:
+            num_colors = 3
+
+        new_space = gym.spaces.Box(
+            low=0,
+            high=255,
+            shape=(self._height, self._width, num_colors),
+            dtype=np.uint8,
+        )
+        if self._key is None:
+            original_space = self.observation_space
+            self.observation_space = new_space
+        else:
+            original_space = self.observation_space.spaces[self._key]
+            self.observation_space.spaces[self._key] = new_space
+        assert original_space.dtype == np.uint8 and len(original_space.shape) == 3
+
+    def observation(self, obs):
+        if self._key is None:
+            frame = obs
+        else:
+            frame = obs[self._key]
+
+        if self._grayscale:
+            frame = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)
+        frame = cv2.resize(
+            frame, (self._width, self._height), interpolation=cv2.INTER_AREA
+        )
+        if self._grayscale:
+            frame = np.expand_dims(frame, -1)
+
+        if self._key is None:
+            obs = frame
+        else:
+            obs = obs.copy()
+            obs[self._key] = frame
+        return obs
+
+
+class FrameStack(gym.Wrapper):
+    def __init__(self, env, k):
+        """Stack k last frames.
+        Returns lazy array, which is much more memory efficient.
+        See Also
+        --------
+        baselines.common.atari_wrappers.LazyFrames
+        """
+        gym.Wrapper.__init__(self, env)
+        self.k = k
+        self.frames = deque([], maxlen=k)
+        shp = env.observation_space.shape
+        self.observation_space = spaces.Box(
+            low=0,
+            high=255,
+            shape=(shp[:-1] + (shp[-1] * k,)),
+            dtype=env.observation_space.dtype,
+        )
+
+    def reset(self):
+        ob = self.env.reset()
+        for _ in range(self.k):
+            self.frames.append(ob)
+        return self._get_ob()
+
+    def step(self, action):
+        ob, reward, done, info = self.env.step(action)
+        self.frames.append(ob)
+        return self._get_ob(), reward, done, info
+
+    def _get_ob(self):
+        assert len(self.frames) == self.k
+        return LazyFrames(list(self.frames))
+
+
+class ScaledFloatFrame(gym.ObservationWrapper):
+    def __init__(self, env):
+        gym.ObservationWrapper.__init__(self, env)
+        self.observation_space = gym.spaces.Box(
+            low=0, high=1, shape=env.observation_space.shape, dtype=np.float32
+        )
+
+    def observation(self, observation):
+        # careful! This undoes the memory optimization, use
+        # with smaller replay buffers only.
+        return np.array(observation).astype(np.float32) / 255.0
+
+
+class LazyFrames(object):
+    def __init__(self, frames):
+        """This object ensures that common frames between the observations are only stored once.
+        It exists purely to optimize memory usage which can be huge for DQN's 1M frames replay
+        buffers.
+        This object should only be converted to numpy array before being passed to the model.
+        You'd not believe how complex the previous solution was."""
+        self._frames = frames
+        self._out = None
+
+    def _force(self):
+        if self._out is None:
+            self._out = np.concatenate(self._frames, axis=-1)
+            self._frames = None
+        return self._out
+
+    def __array__(self, dtype=None):
+        out = self._force()
+        if dtype is not None:
+            out = out.astype(dtype)
+        return out
+
+    def __len__(self):
+        return len(self._force())
+
+    def __getitem__(self, i):
+        return self._force()[i]
+
+    def count(self):
+        frames = self._force()
+        return frames.shape[frames.ndim - 1]
+
+    def frame(self, i):
+        return self._force()[..., i]
+
+
+def make_atari(env_id, max_episode_steps=None):
+    env = gym.make(env_id)
+    assert "NoFrameskip" in env.spec.id
+    env = NoopResetEnv(env, noop_max=30)
+    env = MaxAndSkipEnv(env, skip=4)
+    if max_episode_steps is not None:
+        env = TimeLimit(env, max_episode_steps=max_episode_steps)
+    return env
+
+
+def wrap_deepmind(
+    env, episode_life=True, clip_rewards=True, frame_stack=True, scale=True
+):
+    """Configure environment for DeepMind-style Atari."""
+    if episode_life:
+        env = EpisodicLifeEnv(env)
+    if "FIRE" in env.unwrapped.get_action_meanings():
+        env = FireResetEnv(env)
+    env = WarpFrame(env)
+    if scale:
+        env = ScaledFloatFrame(env)
+    if clip_rewards:
+        env = ClipRewardEnv(env)
+    if frame_stack:
+        env = FrameStack(env, 4)
+    return env
+
+
+class EpisodicLifeMario(gym.Wrapper):
+    def __init__(self, env):
+        """Make end-of-life == end-of-episode, but only reset on true game over.
+        Done by DeepMind for the DQN and co. since it helps value estimation.
+        """
+        gym.Wrapper.__init__(self, env)
+        self.lives = 0
+        self.was_real_done = True
+
+    def step(self, action):
+        obs, reward, done, info = self.env.step(action)
+        self.was_real_done = done
+        # check current lives, make loss of life terminal,
+        # then update lives to handle bonus lives
+        lives = self.env.unwrapped._life
+        if lives < self.lives and lives > 0:
+            # for Qbert sometimes we stay in lives == 0 condition for a few frames
+            # so it's important to keep lives > 0, so that we only reset once
+            # the environment advertises done.
+            done = True
+        self.lives = lives
+        return obs, reward, done, info
+
+    def reset(self, **kwargs):
+        """Reset only when lives are exhausted.
+        This way all states are still reachable even though lives are episodic,
+        and the learner need not know about any of this behind-the-scenes.
+        """
+        if self.was_real_done:
+            obs = self.env.reset(**kwargs)
+        else:
+            # no-op step to advance from terminal/lost life state
+            obs, _, _, _ = self.env.step(0)
+        self.lives = self.env.unwrapped._life
+        return obs
+
+
+def wrap_mario(env):
+    env = NoopResetEnv(env, noop_max=30)
+    env = MaxAndSkipEnv(env, skip=4)
+    env = EpisodicLifeMario(env)
+    env = WarpFrame(env)
+    env = ScaledFloatFrame(env)
+    # env = custom_reward(env)
+    env = FrameStack(env, 4)
+    return env

ale_pyqt5/app.py

@@ -0,0 +1,514 @@
+import sys
+import os
+import numpy as np
+import random
+from collections import deque
+import gymnasium as gym
+import ale_py
+
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import torch.nn.functional as F
+from torch.distributions import Categorical
+
+from PyQt5.QtWidgets import (QApplication, QMainWindow, QWidget, QVBoxLayout, 
+                             QHBoxLayout, QPushButton, QLabel, QComboBox, 
+                             QTextEdit, QProgressBar, QTabWidget, QFrame)
+from PyQt5.QtCore import QTimer, Qt, pyqtSignal, QThread
+from PyQt5.QtGui import QImage, QPixmap, QFont
+
+# Register ALE environments
+gym.register_envs(ale_py)
+
+# Environment setup
+def create_env(env_name='ALE/Breakout-v5'):
+    """
+    Create ALE environment with Gymnasium API
+    Available environments: 
+    - ALE/Breakout-v5, ALE/Pong-v5, ALE/SpaceInvaders-v5, 
+    - ALE/Assault-v5, ALE/BeamRider-v5, ALE/Enduro-v5
+    """
+    env = gym.make(env_name, render_mode='rgb_array')
+    return env
+
+# Neural Network for Dueling DQN
+class DuelingDQN(nn.Module):
+    def __init__(self, input_shape, n_actions):
+        super(DuelingDQN, self).__init__()
+        self.conv = nn.Sequential(
+            nn.Conv2d(input_shape[0], 32, kernel_size=8, stride=4),
+            nn.ReLU(),
+            nn.Conv2d(32, 64, kernel_size=4, stride=2),
+            nn.ReLU(),
+            nn.Conv2d(64, 64, kernel_size=3, stride=1),
+            nn.ReLU()
+        )
+        
+        conv_out_size = self._get_conv_out(input_shape)
+        
+        self.fc_advantage = nn.Sequential(
+            nn.Linear(conv_out_size, 512),
+            nn.ReLU(),
+            nn.Linear(512, n_actions)
+        )
+        
+        self.fc_value = nn.Sequential(
+            nn.Linear(conv_out_size, 512),
+            nn.ReLU(),
+            nn.Linear(512, 1)
+        )
+        
+    def _get_conv_out(self, shape):
+        o = self.conv(torch.zeros(1, *shape))
+        return int(np.prod(o.size()))
+    
+    def forward(self, x):
+        conv_out = self.conv(x).view(x.size()[0], -1)
+        advantage = self.fc_advantage(conv_out)
+        value = self.fc_value(conv_out)
+        return value + advantage - advantage.mean()
+
+# Neural Network for PPO
+class PPONetwork(nn.Module):
+    def __init__(self, input_shape, n_actions):
+        super(PPONetwork, self).__init__()
+        self.conv = nn.Sequential(
+            nn.Conv2d(input_shape[0], 32, kernel_size=8, stride=4),
+            nn.ReLU(),
+            nn.Conv2d(32, 64, kernel_size=4, stride=2),
+            nn.ReLU(),
+            nn.Conv2d(64, 64, kernel_size=3, stride=1),
+            nn.ReLU()
+        )
+        
+        conv_out_size = self._get_conv_out(input_shape)
+        
+        self.actor = nn.Sequential(
+            nn.Linear(conv_out_size, 512),
+            nn.ReLU(),
+            nn.Linear(512, n_actions),
+            nn.Softmax(dim=-1)
+        )
+        
+        self.critic = nn.Sequential(
+            nn.Linear(conv_out_size, 512),
+            nn.ReLU(),
+            nn.Linear(512, 1)
+        )
+        
+    def _get_conv_out(self, shape):
+        o = self.conv(torch.zeros(1, *shape))
+        return int(np.prod(o.size()))
+    
+    def forward(self, x):
+        conv_out = self.conv(x).view(x.size()[0], -1)
+        return self.actor(conv_out), self.critic(conv_out)
+
+# Dueling DQN Agent
+class DuelingDQNAgent:
+    def __init__(self, state_dim, action_dim, lr=1e-4, gamma=0.99, epsilon=1.0, 
+                 epsilon_min=0.01, epsilon_decay=0.995, memory_size=10000, batch_size=32):
+        self.state_dim = state_dim
+        self.action_dim = action_dim
+        self.lr = lr
+        self.gamma = gamma
+        self.epsilon = epsilon
+        self.epsilon_min = epsilon_min
+        self.epsilon_decay = epsilon_decay
+        self.batch_size = batch_size
+        
+        self.memory = deque(maxlen=memory_size)
+        self.model = DuelingDQN(state_dim, action_dim)
+        self.optimizer = optim.Adam(self.model.parameters(), lr=lr)
+        self.criterion = nn.MSELoss()
+        
+    def remember(self, state, action, reward, next_state, done):
+        self.memory.append((state, action, reward, next_state, done))
+        
+    def act(self, state):
+        if np.random.random() <= self.epsilon:
+            return random.randrange(self.action_dim)
+        
+        state = torch.FloatTensor(state).unsqueeze(0)
+        with torch.no_grad():
+            q_values = self.model(state)
+        return np.argmax(q_values.detach().numpy())
+    
+    def replay(self):
+        if len(self.memory) < self.batch_size:
+            return
+        
+        batch = random.sample(self.memory, self.batch_size)
+        states = torch.FloatTensor(np.array([e[0] for e in batch]))
+        actions = torch.LongTensor([e[1] for e in batch])
+        rewards = torch.FloatTensor([e[2] for e in batch])
+        next_states = torch.FloatTensor(np.array([e[3] for e in batch]))
+        dones = torch.BoolTensor([e[4] for e in batch])
+        
+        current_q_values = self.model(states).gather(1, actions.unsqueeze(1))
+        with torch.no_grad():
+            next_q_values = self.model(next_states).max(1)[0]
+        target_q_values = rewards + (self.gamma * next_q_values * ~dones)
+        
+        loss = self.criterion(current_q_values.squeeze(), target_q_values)
+        
+        self.optimizer.zero_grad()
+        loss.backward()
+        self.optimizer.step()
+        
+        if self.epsilon > self.epsilon_min:
+            self.epsilon *= self.epsilon_decay
+
+# PPO Agent
+class PPOAgent:
+    def __init__(self, state_dim, action_dim, lr=3e-4, gamma=0.99, epsilon=0.2, 
+                 entropy_coef=0.01, value_coef=0.5):
+        self.state_dim = state_dim
+        self.action_dim = action_dim
+        self.gamma = gamma
+        self.epsilon = epsilon
+        self.entropy_coef = entropy_coef
+        self.value_coef = value_coef
+        
+        self.model = PPONetwork(state_dim, action_dim)
+        self.optimizer = optim.Adam(self.model.parameters(), lr=lr)
+        
+        self.memory = []
+        
+    def remember(self, state, action, reward, value, log_prob):
+        self.memory.append((state, action, reward, value, log_prob))
+        
+    def act(self, state):
+        state = torch.FloatTensor(state).unsqueeze(0)
+        with torch.no_grad():
+            probs, value = self.model(state)
+        dist = Categorical(probs)
+        action = dist.sample()
+        return action.item(), dist.log_prob(action), value.squeeze()
+    
+    def train(self):
+        if not self.memory:
+            return
+        
+        states, actions, rewards, values, log_probs = zip(*self.memory)
+        
+        # Calculate returns and advantages
+        returns = []
+        R = 0
+        
+        for r in reversed(rewards):
+            R = r + self.gamma * R
+            returns.insert(0, R)
+        
+        returns = torch.FloatTensor(returns)
+        values = torch.FloatTensor(values)
+        advantages = returns - values
+        
+        # Normalize advantages
+        advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8)
+        
+        # Convert to tensors
+        states = torch.FloatTensor(np.array(states))
+        actions = torch.LongTensor(actions)
+        old_log_probs = torch.FloatTensor(log_probs)
+        
+        # Get new probabilities
+        new_probs, new_values = self.model(states)
+        dist = Categorical(new_probs)
+        new_log_probs = dist.log_prob(actions)
+        entropy = dist.entropy().mean()
+        
+        # PPO loss
+        ratio = (new_log_probs - old_log_probs).exp()
+        surr1 = ratio * advantages
+        surr2 = torch.clamp(ratio, 1 - self.epsilon, 1 + self.epsilon) * advantages
+        actor_loss = -torch.min(surr1, surr2).mean()
+        
+        critic_loss = F.mse_loss(new_values.squeeze(), returns)
+        
+        total_loss = actor_loss + self.value_coef * critic_loss - self.entropy_coef * entropy
+        
+        self.optimizer.zero_grad()
+        total_loss.backward()
+        self.optimizer.step()
+        
+        self.memory = []
+
+# Training Thread
+class TrainingThread(QThread):
+    update_signal = pyqtSignal(dict)
+    frame_signal = pyqtSignal(np.ndarray)
+    
+    def __init__(self, algorithm='dqn', env_name='ALE/Breakout-v5'):
+        super().__init__()
+        self.algorithm = algorithm
+        self.env_name = env_name
+        self.running = False
+        self.env = None
+        self.agent = None
+        
+    def preprocess_state(self, state):
+        # Convert to CHW format and normalize
+        state = state.transpose((2, 0, 1))
+        state = state / 255.0
+        return state
+        
+    def run(self):
+        self.running = True
+        try:
+            self.env = create_env(self.env_name)
+            state, info = self.env.reset()
+            state = self.preprocess_state(state)
+            
+            n_actions = self.env.action_space.n
+            state_dim = state.shape
+            
+            print(f"Environment: {self.env_name}")
+            print(f"State shape: {state_dim}, Actions: {n_actions}")
+            
+            if self.algorithm == 'dqn':
+                self.agent = DuelingDQNAgent(state_dim, n_actions)
+            else:
+                self.agent = PPOAgent(state_dim, n_actions)
+                
+            episode = 0
+            total_reward = 0
+            steps = 0
+            episode_rewards = []
+            
+            while self.running:
+                try:
+                    if self.algorithm == 'dqn':
+                        action = self.agent.act(state)
+                        next_state, reward, terminated, truncated, info = self.env.step(action)
+                        done = terminated or truncated
+                        next_state = self.preprocess_state(next_state)
+                        self.agent.remember(state, action, reward, next_state, done)
+                        self.agent.replay()
+                    else:
+                        action, log_prob, value = self.agent.act(state)
+                        next_state, reward, terminated, truncated, info = self.env.step(action)
+                        done = terminated or truncated
+                        next_state = self.preprocess_state(next_state)
+                        self.agent.remember(state, action, reward, value, log_prob)
+                        if done:
+                            self.agent.train()
+                    
+                    state = next_state
+                    total_reward += reward
+                    steps += 1
+                    
+                    # Emit frame for display
+                    try:
+                        frame = self.env.render()
+                        if frame is not None:
+                            self.frame_signal.emit(frame)
+                    except Exception as e:
+                        # Create a placeholder frame if rendering fails
+                        frame = np.zeros((210, 160, 3), dtype=np.uint8)
+                        self.frame_signal.emit(frame)
+                    
+                    # Emit training progress
+                    if steps % 10 == 0:
+                        progress_data = {
+                            'episode': episode,
+                            'total_reward': total_reward,
+                            'steps': steps,
+                            'epsilon': self.agent.epsilon if self.algorithm == 'dqn' else 0.2,
+                            'env_name': self.env_name,
+                            'lives': info.get('lives', 0) if isinstance(info, dict) else 0
+                        }
+                        self.update_signal.emit(progress_data)
+                    
+                    if terminated or truncated:
+                        episode_rewards.append(total_reward)
+                        avg_reward = np.mean(episode_rewards[-10:]) if episode_rewards else total_reward
+                        
+                        print(f"Episode {episode}: Total Reward: {total_reward:.2f}, "
+                              f"Steps: {steps}, Avg Reward (last 10): {avg_reward:.2f}")
+                        
+                        episode += 1
+                        state, info = self.env.reset()
+                        state = self.preprocess_state(state)
+                        total_reward = 0
+                        steps = 0
+                        
+                except Exception as e:
+                    print(f"Error in training loop: {e}")
+                    import traceback
+                    traceback.print_exc()
+                    break
+                    
+        except Exception as e:
+            print(f"Error setting up environment: {e}")
+            import traceback
+            traceback.print_exc()
+                
+    def stop(self):
+        self.running = False
+        if self.env:
+            self.env.close()
+
+# Main Application Window
+class ALE_RLApp(QMainWindow):
+    def __init__(self):
+        super().__init__()
+        self.training_thread = None
+        self.init_ui()
+        
+    def init_ui(self):
+        self.setWindowTitle('🎮 ALE Arcade RL Training')
+        self.setGeometry(100, 100, 1200, 800)
+        
+        central_widget = QWidget()
+        self.setCentralWidget(central_widget)
+        layout = QVBoxLayout(central_widget)
+        
+        # Title
+        title = QLabel('🎮 Arcade Reinforcement Learning (ALE)')
+        title.setFont(QFont('Arial', 16, QFont.Bold))
+        title.setAlignment(Qt.AlignCenter)
+        layout.addWidget(title)
+        
+        # Control Panel
+        control_layout = QHBoxLayout()
+        
+        self.algorithm_combo = QComboBox()
+        self.algorithm_combo.addItems(['Dueling DQN', 'PPO'])
+        
+        self.env_combo = QComboBox()
+        self.env_combo.addItems([
+            'ALE/Breakout-v5',
+            'ALE/Pong-v5', 
+            'ALE/SpaceInvaders-v5',
+            'ALE/Assault-v5',
+            'ALE/BeamRider-v5',
+            'ALE/Enduro-v5',
+            'ALE/Seaquest-v5',
+            'ALE/Qbert-v5'
+        ])
+        
+        self.start_btn = QPushButton('Start Training')
+        self.start_btn.clicked.connect(self.start_training)
+        
+        self.stop_btn = QPushButton('Stop Training')
+        self.stop_btn.clicked.connect(self.stop_training)
+        self.stop_btn.setEnabled(False)
+        
+        control_layout.addWidget(QLabel('Algorithm:'))
+        control_layout.addWidget(self.algorithm_combo)
+        control_layout.addWidget(QLabel('Environment:'))
+        control_layout.addWidget(self.env_combo)
+        control_layout.addWidget(self.start_btn)
+        control_layout.addWidget(self.stop_btn)
+        control_layout.addStretch()
+        
+        layout.addLayout(control_layout)
+        
+        # Content Area
+        content_layout = QHBoxLayout()
+        
+        # Left side - Game Display
+        left_frame = QFrame()
+        left_frame.setFrameStyle(QFrame.Box)
+        left_layout = QVBoxLayout(left_frame)
+        
+        self.game_display = QLabel()
+        self.game_display.setMinimumSize(400, 300)
+        self.game_display.setAlignment(Qt.AlignCenter)
+        self.game_display.setText('Game display will appear here\nPress "Start Training" to begin')
+        self.game_display.setStyleSheet('border: 1px solid gray; background-color: black; color: white;')
+        
+        left_layout.addWidget(QLabel('Game Display:'))
+        left_layout.addWidget(self.game_display)
+        
+        # Right side - Training Info
+        right_frame = QFrame()
+        right_frame.setFrameStyle(QFrame.Box)
+        right_layout = QVBoxLayout(right_frame)
+        
+        # Progress bars
+        self.env_label = QLabel('Environment: Not started')
+        self.episode_label = QLabel('Episode: 0')
+        self.reward_label = QLabel('Total Reward: 0')
+        self.steps_label = QLabel('Steps: 0')
+        self.epsilon_label = QLabel('Epsilon: 0')
+        self.lives_label = QLabel('Lives: 0')
+        
+        right_layout.addWidget(self.env_label)
+        right_layout.addWidget(self.episode_label)
+        right_layout.addWidget(self.reward_label)
+        right_layout.addWidget(self.steps_label)
+        right_layout.addWidget(self.epsilon_label)
+        right_layout.addWidget(self.lives_label)
+        
+        # Training log
+        right_layout.addWidget(QLabel('Training Log:'))
+        self.log_text = QTextEdit()
+        self.log_text.setMaximumHeight(200)
+        right_layout.addWidget(self.log_text)
+        
+        content_layout.addWidget(left_frame)
+        content_layout.addWidget(right_frame)
+        layout.addLayout(content_layout)
+        
+    def start_training(self):
+        algorithm = 'dqn' if self.algorithm_combo.currentText() == 'Dueling DQN' else 'ppo'
+        env_name = self.env_combo.currentText()
+        
+        self.training_thread = TrainingThread(algorithm, env_name)
+        self.training_thread.update_signal.connect(self.update_training_info)
+        self.training_thread.frame_signal.connect(self.update_game_display)
+        self.training_thread.start()
+        
+        self.start_btn.setEnabled(False)
+        self.stop_btn.setEnabled(True)
+        
+        self.log_text.append(f'Started {self.algorithm_combo.currentText()} training on {env_name}...')
+        
+    def stop_training(self):
+        if self.training_thread:
+            self.training_thread.stop()
+            self.training_thread.wait()
+            
+        self.start_btn.setEnabled(True)
+        self.stop_btn.setEnabled(False)
+        self.log_text.append('Training stopped.')
+        
+    def update_training_info(self, data):
+        self.env_label.setText(f'Environment: {data.get("env_name", "Unknown")}')
+        self.episode_label.setText(f'Episode: {data["episode"]}')
+        self.reward_label.setText(f'Total Reward: {data["total_reward"]:.2f}')
+        self.steps_label.setText(f'Steps: {data["steps"]}')
+        self.epsilon_label.setText(f'Epsilon: {data["epsilon"]:.3f}')
+        self.lives_label.setText(f'Lives: {data.get("lives", 0)}')
+        
+    def update_game_display(self, frame):
+        if frame is not None:
+            try:
+                h, w, ch = frame.shape
+                bytes_per_line = ch * w
+                q_img = QImage(frame.data, w, h, bytes_per_line, QImage.Format_RGB888)
+                pixmap = QPixmap.fromImage(q_img)
+                self.game_display.setPixmap(pixmap.scaled(400, 300, Qt.KeepAspectRatio))
+            except Exception as e:
+                print(f"Error updating display: {e}")
+            
+    def closeEvent(self, event):
+        self.stop_training()
+        event.accept()
+
+def main():
+    # Set random seeds for reproducibility
+    torch.manual_seed(42)
+    np.random.seed(42)
+    random.seed(42)
+    
+    app = QApplication(sys.argv)
+    window = ALE_RLApp()
+    window.show()
+    sys.exit(app.exec_())
+
+if __name__ == '__main__':
+    main()

ale_pyqt5/app_2.py

@@ -0,0 +1,559 @@
+import sys
+import os
+import numpy as np
+import random
+from collections import deque
+import gymnasium as gym
+import ale_py
+
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import torch.nn.functional as F
+from torch.distributions import Categorical
+
+from PyQt5.QtWidgets import (QApplication, QMainWindow, QWidget, QVBoxLayout, 
+                             QHBoxLayout, QPushButton, QLabel, QComboBox, 
+                             QTextEdit, QProgressBar, QTabWidget, QFrame)
+from PyQt5.QtCore import QTimer, Qt, pyqtSignal, QThread
+from PyQt5.QtGui import QImage, QPixmap, QFont
+
+# Register ALE environments
+gym.register_envs(ale_py)
+
+# Environment setup
+def create_env(env_name='ALE/SpaceInvaders-v5'):
+    """
+    Create ALE environment with Gymnasium API
+    """
+    env = gym.make(env_name, render_mode='rgb_array')
+    return env
+
+# Enhanced Neural Network for Dueling DQN
+class DuelingDQN(nn.Module):
+    def __init__(self, input_shape, n_actions):
+        super(DuelingDQN, self).__init__()
+        self.conv = nn.Sequential(
+            nn.Conv2d(input_shape[0], 32, kernel_size=8, stride=4),
+            nn.ReLU(),
+            nn.Conv2d(32, 64, kernel_size=4, stride=2),
+            nn.ReLU(),
+            nn.Conv2d(64, 64, kernel_size=3, stride=1),
+            nn.ReLU()
+        )
+        
+        conv_out_size = self._get_conv_out(input_shape)
+        
+        self.fc_advantage = nn.Sequential(
+            nn.Linear(conv_out_size, 256),
+            nn.ReLU(),
+            nn.Linear(256, n_actions)
+        )
+        
+        self.fc_value = nn.Sequential(
+            nn.Linear(conv_out_size, 256),
+            nn.ReLU(),
+            nn.Linear(256, 1)
+        )
+        
+    def _get_conv_out(self, shape):
+        o = self.conv(torch.zeros(1, *shape))
+        return int(np.prod(o.size()))
+    
+    def forward(self, x):
+        conv_out = self.conv(x).view(x.size()[0], -1)
+        advantage = self.fc_advantage(conv_out)
+        value = self.fc_value(conv_out)
+        return value + advantage - advantage.mean()
+
+# Enhanced Neural Network for PPO
+class PPONetwork(nn.Module):
+    def __init__(self, input_shape, n_actions):
+        super(PPONetwork, self).__init__()
+        self.conv = nn.Sequential(
+            nn.Conv2d(input_shape[0], 32, kernel_size=8, stride=4),
+            nn.ReLU(),
+            nn.Conv2d(32, 64, kernel_size=4, stride=2),
+            nn.ReLU(),
+            nn.Conv2d(64, 64, kernel_size=3, stride=1),
+            nn.ReLU()
+        )
+        
+        conv_out_size = self._get_conv_out(input_shape)
+        
+        self.actor = nn.Sequential(
+            nn.Linear(conv_out_size, 256),
+            nn.ReLU(),
+            nn.Linear(256, n_actions),
+            nn.Softmax(dim=-1)
+        )
+        
+        self.critic = nn.Sequential(
+            nn.Linear(conv_out_size, 256),
+            nn.ReLU(),
+            nn.Linear(256, 1)
+        )
+        
+    def _get_conv_out(self, shape):
+        o = self.conv(torch.zeros(1, *shape))
+        return int(np.prod(o.size()))
+    
+    def forward(self, x):
+        conv_out = self.conv(x).view(x.size()[0], -1)
+        return self.actor(conv_out), self.critic(conv_out)
+
+# Enhanced Dueling DQN Agent with better training
+class DuelingDQNAgent:
+    def __init__(self, state_dim, action_dim, lr=1e-4, gamma=0.99, epsilon=1.0, 
+                 epsilon_min=0.01, epsilon_decay=0.999, memory_size=50000, batch_size=32):
+        self.state_dim = state_dim
+        self.action_dim = action_dim
+        self.lr = lr
+        self.gamma = gamma
+        self.epsilon = epsilon
+        self.epsilon_min = epsilon_min
+        self.epsilon_decay = epsilon_decay
+        self.batch_size = batch_size
+        
+        self.memory = deque(maxlen=memory_size)
+        self.model = DuelingDQN(state_dim, action_dim)
+        self.optimizer = optim.Adam(self.model.parameters(), lr=lr, weight_decay=1e-5)
+        self.criterion = nn.SmoothL1Loss()  # Huber loss for better stability
+        
+        # Target network for stable training
+        self.target_model = DuelingDQN(state_dim, action_dim)
+        self.update_target_network()
+        self.target_update_frequency = 1000
+        self.train_step = 0
+        
+    def update_target_network(self):
+        self.target_model.load_state_dict(self.model.state_dict())
+        
+    def remember(self, state, action, reward, next_state, done):
+        self.memory.append((state, action, reward, next_state, done))
+        
+    def act(self, state):
+        if np.random.random() <= self.epsilon:
+            return random.randrange(self.action_dim)
+        
+        state = torch.FloatTensor(state).unsqueeze(0)
+        with torch.no_grad():
+            q_values = self.model(state)
+        return np.argmax(q_values.detach().numpy())
+    
+    def replay(self):
+        if len(self.memory) < self.batch_size:
+            return
+        
+        batch = random.sample(self.memory, self.batch_size)
+        states = torch.FloatTensor(np.array([e[0] for e in batch]))
+        actions = torch.LongTensor([e[1] for e in batch])
+        rewards = torch.FloatTensor([e[2] for e in batch])
+        next_states = torch.FloatTensor(np.array([e[3] for e in batch]))
+        dones = torch.BoolTensor([e[4] for e in batch])
+        
+        current_q_values = self.model(states).gather(1, actions.unsqueeze(1))
+        
+        with torch.no_grad():
+            next_actions = self.model(next_states).max(1)[1]
+            next_q_values = self.target_model(next_states).gather(1, next_actions.unsqueeze(1)).squeeze()
+        
+        target_q_values = rewards + (self.gamma * next_q_values * ~dones)
+        
+        loss = self.criterion(current_q_values.squeeze(), target_q_values)
+        
+        self.optimizer.zero_grad()
+        loss.backward()
+        # Gradient clipping
+        torch.nn.utils.clip_grad_norm_(self.model.parameters(), 1.0)
+        self.optimizer.step()
+        
+        # Update target network periodically
+        self.train_step += 1
+        if self.train_step % self.target_update_frequency == 0:
+            self.update_target_network()
+        
+        if self.epsilon > self.epsilon_min:
+            self.epsilon *= self.epsilon_decay
+
+# Enhanced PPO Agent
+class PPOAgent:
+    def __init__(self, state_dim, action_dim, lr=3e-4, gamma=0.99, epsilon=0.2, 
+                 entropy_coef=0.01, value_coef=0.5, ppo_epochs=4, batch_size=64):
+        self.state_dim = state_dim
+        self.action_dim = action_dim
+        self.gamma = gamma
+        self.epsilon = epsilon
+        self.entropy_coef = entropy_coef
+        self.value_coef = value_coef
+        self.ppo_epochs = ppo_epochs
+        self.batch_size = batch_size
+        
+        self.model = PPONetwork(state_dim, action_dim)
+        self.optimizer = optim.Adam(self.model.parameters(), lr=lr)
+        
+        self.memory = []
+        
+    def remember(self, state, action, reward, value, log_prob):
+        self.memory.append((state, action, reward, value, log_prob))
+        
+    def act(self, state):
+        state = torch.FloatTensor(state).unsqueeze(0)
+        with torch.no_grad():
+            probs, value = self.model(state)
+        dist = Categorical(probs)
+        action = dist.sample()
+        return action.item(), dist.log_prob(action), value.squeeze()
+    
+    def train(self):
+        if len(self.memory) < self.batch_size:
+            return
+        
+        states, actions, rewards, values, log_probs = zip(*self.memory)
+        
+        # Calculate returns and advantages
+        returns = []
+        R = 0
+        for r in reversed(rewards):
+            R = r + self.gamma * R
+            returns.insert(0, R)
+        
+        returns = torch.FloatTensor(returns)
+        old_values = torch.FloatTensor(values)
+        advantages = returns - old_values
+        
+        # Normalize advantages
+        advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8)
+        
+        # Convert to tensors
+        states_tensor = torch.FloatTensor(np.array(states))
+        actions_tensor = torch.LongTensor(actions)
+        old_log_probs = torch.FloatTensor(log_probs)
+        
+        # PPO epochs
+        for _ in range(self.ppo_epochs):
+            # Get new probabilities
+            new_probs, new_values = self.model(states_tensor)
+            dist = Categorical(new_probs)
+            new_log_probs = dist.log_prob(actions_tensor)
+            entropy = dist.entropy().mean()
+            
+            # PPO loss
+            ratio = (new_log_probs - old_log_probs).exp()
+            surr1 = ratio * advantages
+            surr2 = torch.clamp(ratio, 1 - self.epsilon, 1 + self.epsilon) * advantages
+            actor_loss = -torch.min(surr1, surr2).mean()
+            
+            critic_loss = F.mse_loss(new_values.squeeze(), returns)
+            
+            total_loss = actor_loss + self.value_coef * critic_loss - self.entropy_coef * entropy
+            
+            self.optimizer.zero_grad()
+            total_loss.backward()
+            torch.nn.utils.clip_grad_norm_(self.model.parameters(), 0.5)
+            self.optimizer.step()
+        
+        self.memory = []
+
+# Enhanced Training Thread with better state processing
+class TrainingThread(QThread):
+    update_signal = pyqtSignal(dict)
+    frame_signal = pyqtSignal(np.ndarray)
+    
+    def __init__(self, algorithm='dqn', env_name='ALE/Breakout-v5'):
+        super().__init__()
+        self.algorithm = algorithm
+        self.env_name = env_name
+        self.running = False
+        self.env = None
+        self.agent = None
+        
+    def preprocess_state(self, state):
+        # Convert to CHW format, normalize, and convert to grayscale
+        if len(state.shape) == 3:
+            # Convert to grayscale and resize for faster processing
+            state = state.mean(axis=2, keepdims=True)  # Convert to grayscale
+            state = state.transpose((2, 0, 1))
+            state = state / 255.0
+        return state
+        
+    def run(self):
+        self.running = True
+        try:
+            self.env = create_env(self.env_name)
+            state, info = self.env.reset()
+            state = self.preprocess_state(state)
+            
+            n_actions = self.env.action_space.n
+            state_dim = state.shape
+            
+            print(f"🎮 Training on: {self.env_name}")
+            print(f"📊 State shape: {state_dim}, Actions: {n_actions}")
+            print(f"🤖 Algorithm: {self.algorithm}")
+            
+            if self.algorithm == 'dqn':
+                self.agent = DuelingDQNAgent(state_dim, n_actions)
+            else:
+                self.agent = PPOAgent(state_dim, n_actions)
+                
+            episode = 0
+            total_reward = 0
+            steps = 0
+            episode_rewards = []
+            best_reward = -float('inf')
+            
+            while self.running:
+                try:
+                    if self.algorithm == 'dqn':
+                        action = self.agent.act(state)
+                        next_state, reward, terminated, truncated, info = self.env.step(action)
+                        done = terminated or truncated
+                        next_state = self.preprocess_state(next_state)
+                        self.agent.remember(state, action, reward, next_state, done)
+                        self.agent.replay()
+                    else:
+                        action, log_prob, value = self.agent.act(state)
+                        next_state, reward, terminated, truncated, info = self.env.step(action)
+                        done = terminated or truncated
+                        next_state = self.preprocess_state(next_state)
+                        self.agent.remember(state, action, reward, value, log_prob)
+                        if done:
+                            self.agent.train()
+                    
+                    state = next_state
+                    total_reward += reward
+                    steps += 1
+                    
+                    # Emit frame for display
+                    try:
+                        frame = self.env.render()
+                        if frame is not None:
+                            self.frame_signal.emit(frame)
+                    except Exception as e:
+                        # Create a placeholder frame if rendering fails
+                        frame = np.zeros((210, 160, 3), dtype=np.uint8)
+                        self.frame_signal.emit(frame)
+                    
+                    # Emit training progress more frequently for better feedback
+                    if steps % 5 == 0:
+                        avg_reward = np.mean(episode_rewards[-10:]) if episode_rewards else total_reward
+                        progress_data = {
+                            'episode': episode,
+                            'total_reward': total_reward,
+                            'steps': steps,
+                            'epsilon': self.agent.epsilon if self.algorithm == 'dqn' else 0.2,
+                            'env_name': self.env_name,
+                            'lives': info.get('lives', 0) if isinstance(info, dict) else 0,
+                            'avg_reward': avg_reward,
+                            'best_reward': best_reward
+                        }
+                        self.update_signal.emit(progress_data)
+                    
+                    if terminated or truncated:
+                        episode_rewards.append(total_reward)
+                        if total_reward > best_reward:
+                            best_reward = total_reward
+                        
+                        avg_reward = np.mean(episode_rewards[-10:]) if episode_rewards else total_reward
+                        
+                        print(f"🎯 Episode {episode}: Reward: {total_reward:.1f}, "
+                              f"Steps: {steps}, Avg (last 10): {avg_reward:.1f}, "
+                              f"Best: {best_reward:.1f}, Epsilon: {self.agent.epsilon:.3f}")
+                        
+                        episode += 1
+                        state, info = self.env.reset()
+                        state = self.preprocess_state(state)
+                        total_reward = 0
+                        steps = 0
+                        
+                except Exception as e:
+                    print(f"❌ Error in training loop: {e}")
+                    import traceback
+                    traceback.print_exc()
+                    break
+                    
+        except Exception as e:
+            print(f"❌ Error setting up environment: {e}")
+            import traceback
+            traceback.print_exc()
+                
+    def stop(self):
+        self.running = False
+        if self.env:
+            self.env.close()
+
+# Enhanced Main Application Window
+class ALE_RLApp(QMainWindow):
+    def __init__(self):
+        super().__init__()
+        self.training_thread = None
+        self.init_ui()
+        
+    def init_ui(self):
+        self.setWindowTitle('🎮 ALE Arcade RL Training - Enhanced')
+        self.setGeometry(100, 100, 1200, 800)
+        
+        central_widget = QWidget()
+        self.setCentralWidget(central_widget)
+        layout = QVBoxLayout(central_widget)
+        
+        # Title
+        title = QLabel('🎮 Arcade Reinforcement Learning (ALE) - Enhanced Training')
+        title.setFont(QFont('Arial', 16, QFont.Bold))
+        title.setAlignment(Qt.AlignCenter)
+        layout.addWidget(title)
+        
+        # Control Panel
+        control_layout = QHBoxLayout()
+        
+        self.algorithm_combo = QComboBox()
+        self.algorithm_combo.addItems(['Dueling DQN', 'PPO'])
+        
+        self.env_combo = QComboBox()
+        self.env_combo.addItems([
+            'ALE/Breakout-v5',
+            'ALE/Pong-v5', 
+            'ALE/SpaceInvaders-v5',
+            'ALE/Assault-v5',
+            'ALE/BeamRider-v5',
+            'ALE/Enduro-v5',
+            'ALE/Seaquest-v5',
+            'ALE/Qbert-v5'
+        ])
+        
+        self.start_btn = QPushButton('🚀 Start Training')
+        self.start_btn.clicked.connect(self.start_training)
+        
+        self.stop_btn = QPushButton('⏹️ Stop Training')
+        self.stop_btn.clicked.connect(self.stop_training)
+        self.stop_btn.setEnabled(False)
+        
+        control_layout.addWidget(QLabel('🤖 Algorithm:'))
+        control_layout.addWidget(self.algorithm_combo)
+        control_layout.addWidget(QLabel('🎮 Environment:'))
+        control_layout.addWidget(self.env_combo)
+        control_layout.addWidget(self.start_btn)
+        control_layout.addWidget(self.stop_btn)
+        control_layout.addStretch()
+        
+        layout.addLayout(control_layout)
+        
+        # Content Area
+        content_layout = QHBoxLayout()
+        
+        # Left side - Game Display
+        left_frame = QFrame()
+        left_frame.setFrameStyle(QFrame.Box)
+        left_layout = QVBoxLayout(left_frame)
+        
+        self.game_display = QLabel()
+        self.game_display.setMinimumSize(400, 300)
+        self.game_display.setAlignment(Qt.AlignCenter)
+        self.game_display.setText('Game display will appear here\nPress "🚀 Start Training" to begin')
+        self.game_display.setStyleSheet('border: 1px solid gray; background-color: black; color: white; font-size: 14px;')
+        
+        left_layout.addWidget(QLabel('🎮 Game Display:'))
+        left_layout.addWidget(self.game_display)
+        
+        # Right side - Training Info
+        right_frame = QFrame()
+        right_frame.setFrameStyle(QFrame.Box)
+        right_layout = QVBoxLayout(right_frame)
+        
+        # Progress bars with better styling
+        self.env_label = QLabel('🎯 Environment: Not started')
+        self.episode_label = QLabel('📈 Episode: 0')
+        self.reward_label = QLabel('🏆 Total Reward: 0')
+        self.avg_reward_label = QLabel('📊 Avg Reward (last 10): 0')
+        self.best_reward_label = QLabel('⭐ Best Reward: 0')
+        self.steps_label = QLabel('⏱️ Steps: 0')
+        self.epsilon_label = QLabel('🎲 Epsilon: 0')
+        self.lives_label = QLabel('❤️ Lives: 0')
+        
+        # Style the labels
+        for label in [self.env_label, self.episode_label, self.reward_label, 
+                     self.avg_reward_label, self.best_reward_label, self.steps_label,
+                     self.epsilon_label, self.lives_label]:
+            label.setStyleSheet('font-weight: bold; font-size: 12px;')
+        
+        right_layout.addWidget(self.env_label)
+        right_layout.addWidget(self.episode_label)
+        right_layout.addWidget(self.reward_label)
+        right_layout.addWidget(self.avg_reward_label)
+        right_layout.addWidget(self.best_reward_label)
+        right_layout.addWidget(self.steps_label)
+        right_layout.addWidget(self.epsilon_label)
+        right_layout.addWidget(self.lives_label)
+        
+        # Training log
+        right_layout.addWidget(QLabel('📝 Training Log:'))
+        self.log_text = QTextEdit()
+        self.log_text.setMaximumHeight(200)
+        self.log_text.setStyleSheet('font-family: monospace; font-size: 10px;')
+        right_layout.addWidget(self.log_text)
+        
+        content_layout.addWidget(left_frame)
+        content_layout.addWidget(right_frame)
+        layout.addLayout(content_layout)
+        
+    def start_training(self):
+        algorithm = 'dqn' if self.algorithm_combo.currentText() == 'Dueling DQN' else 'ppo'
+        env_name = self.env_combo.currentText()
+        
+        self.training_thread = TrainingThread(algorithm, env_name)
+        self.training_thread.update_signal.connect(self.update_training_info)
+        self.training_thread.frame_signal.connect(self.update_game_display)
+        self.training_thread.start()
+        
+        self.start_btn.setEnabled(False)
+        self.stop_btn.setEnabled(True)
+        
+        self.log_text.append(f'🚀 Started {self.algorithm_combo.currentText()} training on {env_name}...')
+        
+    def stop_training(self):
+        if self.training_thread:
+            self.training_thread.stop()
+            self.training_thread.wait()
+            
+        self.start_btn.setEnabled(True)
+        self.stop_btn.setEnabled(False)
+        self.log_text.append('⏹️ Training stopped.')
+        
+    def update_training_info(self, data):
+        self.env_label.setText(f'🎯 Environment: {data.get("env_name", "Unknown")}')
+        self.episode_label.setText(f'📈 Episode: {data["episode"]}')
+        self.reward_label.setText(f'🏆 Total Reward: {data["total_reward"]:.1f}')
+        self.avg_reward_label.setText(f'📊 Avg Reward (last 10): {data.get("avg_reward", 0):.1f}')
+        self.best_reward_label.setText(f'⭐ Best Reward: {data.get("best_reward", 0):.1f}')
+        self.steps_label.setText(f'⏱️ Steps: {data["steps"]}')
+        self.epsilon_label.setText(f'🎲 Epsilon: {data["epsilon"]:.3f}')
+        self.lives_label.setText(f'❤️ Lives: {data.get("lives", 0)}')
+        
+    def update_game_display(self, frame):
+        if frame is not None:
+            try:
+                h, w, ch = frame.shape
+                bytes_per_line = ch * w
+                q_img = QImage(frame.data, w, h, bytes_per_line, QImage.Format_RGB888)
+                pixmap = QPixmap.fromImage(q_img)
+                self.game_display.setPixmap(pixmap.scaled(400, 300, Qt.KeepAspectRatio))
+            except Exception as e:
+                print(f"Error updating display: {e}")
+            
+    def closeEvent(self, event):
+        self.stop_training()
+        event.accept()
+
+def main():
+    # Set random seeds for reproducibility
+    torch.manual_seed(42)
+    np.random.seed(42)
+    random.seed(42)
+    
+    app = QApplication(sys.argv)
+    window = ALE_RLApp()
+    window.show()
+    sys.exit(app.exec_())
+
+if __name__ == '__main__':
+    main()

ale_pyqt5/installed_packages_ale_py.txt

@@ -0,0 +1,30 @@
+ale-py==0.11.2
+cloudpickle==3.1.2
+contourpy==1.3.3
+cycler==0.12.1
+Farama-Notifications==0.0.4
+filelock==3.20.0
+fonttools==4.60.1
+fsspec==2025.10.0
+gym==0.26.2
+gym-notices==0.1.0
+gymnasium==1.2.2
+Jinja2==3.1.6
+kiwisolver==1.4.9
+MarkupSafe==3.0.3
+matplotlib==3.10.7
+mpmath==1.3.0
+networkx==3.5
+numpy==2.2.6
+opencv-python==4.12.0.88
+packaging==25.0
+pillow==12.0.0
+pyglet==1.5.11
+pyparsing==3.2.5
+python-dateutil==2.9.0.post0
+setuptools==80.9.0
+six==1.17.0
+sympy==1.14.0
+torch==2.9.0
+tqdm==4.67.1
+typing_extensions==4.15.0