Spaces:

arad1367
/

RL_Airplane_Simulator

Running

App Files Files Community

RL_Airplane_Simulator / index.html

arad1367

Update index.html

84bf0da verified 10 months ago

raw

history blame contribute delete

20.2 kB

	<!-- Airplane Simulator By Pejman Ebrahimi -->
	<!DOCTYPE html>
	<html lang="en">
	<head>
	<meta charset="UTF-8" />
	<meta name="viewport" content="width=device-width, initial-scale=1.0" />
	<title>Simple RL Airplane Simulator</title>
	<style>
	body {
	font-family: Arial, sans-serif;
	margin: 0;
	padding: 20px;
	background-color: #f0f8ff;
	display: flex;
	flex-direction: column;
	align-items: center;
	}

	canvas {
	background: linear-gradient(to bottom, #87ceeb, #e0f7ff);
	border: 1px solid #333;
	margin-bottom: 20px;
	}

	.controls {
	display: flex;
	flex-wrap: wrap;
	gap: 20px;
	max-width: 800px;
	margin-bottom: 20px;
	}

	.control-group {
	background-color: white;
	padding: 15px;
	border-radius: 8px;
	box-shadow: 0 2px 5px rgba(0, 0, 0, 0.1);
	min-width: 250px;
	}

	h1,
	h2 {
	color: #1e3a8a;
	}

	button {
	padding: 10px 15px;
	margin: 5px;
	background-color: #1e3a8a;
	color: white;
	border: none;
	border-radius: 4px;
	cursor: pointer;
	}

	button:hover {
	background-color: #162a61;
	}

	button:disabled {
	background-color: #ccc;
	cursor: not-allowed;
	}

	.param-group {
	margin-bottom: 10px;
	}

	label {
	display: block;
	margin-bottom: 5px;
	font-weight: bold;
	}

	.footer {
	margin-top: 30px;
	text-align: center;
	color: #555;
	}
	</style>
	</head>
	<body>
	<h1>Simple RL Airplane Simulator</h1>

	<canvas id="canvas" width="600" height="400"></canvas>

	<div class="controls">
	<div class="control-group">
	<h2>RL Parameters</h2>

	<div class="param-group">
	<label for="learningRate"
	>Learning Rate: <span id="learningRateValue">0.1</span></label
	>
	<input
	type="range"
	id="learningRate"
	min="0.01"
	max="0.5"
	step="0.01"
	value="0.1"
	/>
	</div>

	<div class="param-group">
	<label for="epsilon"
	>Exploration Rate: <span id="epsilonValue">0.3</span></label
	>
	<input
	type="range"
	id="epsilon"
	min="0.01"
	max="1"
	step="0.01"
	value="0.3"
	/>
	</div>

	<div class="param-group">
	<label for="gamma"
	>Discount Factor: <span id="gammaValue">0.9</span></label
	>
	<input
	type="range"
	id="gamma"
	min="0.5"
	max="0.99"
	step="0.01"
	value="0.9"
	/>
	</div>

	<button id="startBtn">Start Learning</button>
	<button id="resetBtn">Reset Agent</button>
	<button id="toggleHumanBtn">Toggle Human Control</button>
	</div>

	<div class="control-group">
	<h2>Statistics</h2>
	<p>Episode: <span id="episodeCounter">0</span></p>
	<p>Current Reward: <span id="currentReward">0</span></p>
	<p>Average Reward: <span id="avgReward">0</span></p>
	<p>Altitude: <span id="altitude">100</span> m</p>
	<p>Speed: <span id="speed">150</span> km/h</p>
	<p>Pitch: <span id="pitch">0</span>°</p>
	<p>Human Control: <span id="humanControlStatus">Off</span></p>
	</div>
	</div>

	<div class="control-group" style="width: 100%; max-width: 800px">
	<h2>Learning Progress</h2>
	<div
	id="infoPanel"
	style="
	height: 100px;
	overflow-y: auto;
	border: 1px solid #ccc;
	padding: 10px;
	background-color: #f9f9f9;
	"
	></div>
	</div>

	<div class="footer">
	<p>Flight Simulator Designed by Pejman</p>
	<p>© 2025 Pejman Ebrahimi - All Rights Reserved</p>
	</div>

	<script>
	// Get canvas and context
	const canvas = document.getElementById("canvas");
	const ctx = canvas.getContext("2d");

	// Game state
	let isLearning = false;
	let isHumanControl = false;
	let frameCount = 0;
	let episode = 0;
	let currentReward = 0;
	let rewardHistory = [];

	// Airplane state
	const airplane = {
	x: 300,
	y: 200,
	altitude: 100,
	speed: 150,
	pitch: 0,
	crashed: false,
	};

	// RL parameters
	let learningRate = 0.1;
	let epsilon = 0.3;
	let gamma = 0.9;

	// Q-learning variables
	const qTable = {};
	const actions = [
	"nothing",
	"pitch_up",
	"pitch_down",
	"throttle_up",
	"throttle_down",
	];

	// Key states for human control
	const keys = {
	up: false,
	down: false,
	left: false,
	right: false,
	};

	// UI elements
	const startBtn = document.getElementById("startBtn");
	const resetBtn = document.getElementById("resetBtn");
	const toggleHumanBtn = document.getElementById("toggleHumanBtn");
	const infoPanel = document.getElementById("infoPanel");

	// Initialize event listeners
	function init() {
	// Parameter sliders
	document
	.getElementById("learningRate")
	.addEventListener("input", function () {
	learningRate = parseFloat(this.value);
	document.getElementById("learningRateValue").textContent =
	learningRate.toFixed(2);
	});

	document
	.getElementById("epsilon")
	.addEventListener("input", function () {
	epsilon = parseFloat(this.value);
	document.getElementById("epsilonValue").textContent =
	epsilon.toFixed(2);
	});

	document.getElementById("gamma").addEventListener("input", function () {
	gamma = parseFloat(this.value);
	document.getElementById("gammaValue").textContent = gamma.toFixed(2);
	});

	// Buttons
	startBtn.addEventListener("click", function () {
	console.log("Start button clicked");
	if (!isLearning) {
	isLearning = true;
	isHumanControl = false;
	episode = 0;
	resetAirplane();
	startBtn.textContent = "Learning...";
	startBtn.disabled = true;
	document.getElementById("humanControlStatus").textContent = "Off";
	logInfo("Started learning process");
	}
	});

	resetBtn.addEventListener("click", function () {
	isLearning = false;
	resetAirplane();
	qTable = {};
	episode = 0;
	rewardHistory = [];
	startBtn.textContent = "Start Learning";
	startBtn.disabled = false;
	logInfo("Reset agent and Q-table");
	updateStats();
	});

	toggleHumanBtn.addEventListener("click", function () {
	isHumanControl = !isHumanControl;
	isLearning = false;
	document.getElementById("humanControlStatus").textContent =
	isHumanControl ? "On" : "Off";
	startBtn.disabled = isHumanControl;
	resetAirplane();
	logInfo(
	isHumanControl
	? "Switched to manual control"
	: "Switched to AI mode"
	);
	});

	// Keyboard controls
	window.addEventListener("keydown", function (e) {
	if (isHumanControl) {
	switch (e.key) {
	case "ArrowUp":
	keys.up = true;
	break;
	case "ArrowDown":
	keys.down = true;
	break;
	case "ArrowLeft":
	keys.left = true;
	break;
	case "ArrowRight":
	keys.right = true;
	break;
	}
	}
	});

	window.addEventListener("keyup", function (e) {
	switch (e.key) {
	case "ArrowUp":
	keys.up = false;
	break;
	case "ArrowDown":
	keys.down = false;
	break;
	case "ArrowLeft":
	keys.left = false;
	break;
	case "ArrowRight":
	keys.right = false;
	break;
	}
	});

	// Start game loop
	gameLoop();
	}

	// Helper: Log info to panel
	function logInfo(message) {
	const entry = document.createElement("div");
	entry.textContent = `Frame ${frameCount}: ${message}`;
	infoPanel.appendChild(entry);
	infoPanel.scrollTop = infoPanel.scrollHeight;
	console.log(message); // Also log to console for debugging
	}

	// Helper: Reset airplane
	function resetAirplane() {
	airplane.x = 300;
	airplane.y = 200;
	airplane.altitude = 100;
	airplane.speed = 150;
	airplane.pitch = 0;
	airplane.crashed = false;
	frameCount = 0;
	currentReward = 0;
	updateStats();
	}

	// Helper: Update stats display
	function updateStats() {
	document.getElementById("episodeCounter").textContent = episode;
	document.getElementById("currentReward").textContent =
	currentReward.toFixed(1);
	document.getElementById("altitude").textContent = Math.round(
	airplane.altitude
	);
	document.getElementById("speed").textContent = Math.round(
	airplane.speed
	);
	document.getElementById("pitch").textContent = Math.round(
	airplane.pitch
	);

	// Calculate average reward
	if (rewardHistory.length > 0) {
	const sum = rewardHistory.reduce((a, b) => a + b, 0);
	const avg = sum / rewardHistory.length;
	document.getElementById("avgReward").textContent = avg.toFixed(1);
	} else {
	document.getElementById("avgReward").textContent = "0";
	}
	}

	// RL: Get state key for Q-table
	function getStateKey() {
	// Discretize the continuous state
	const altBucket = Math.floor(airplane.altitude / 50);
	const speedBucket = Math.floor(airplane.speed / 20);
	const pitchBucket = Math.floor(airplane.pitch / 5);

	return `${altBucket}_${speedBucket}_${pitchBucket}`;
	}

	// RL: Choose action using epsilon-greedy policy
	function chooseAction(stateKey) {
	// Ensure state exists in Q-table
	if (!qTable[stateKey]) {
	qTable[stateKey] = {};
	actions.forEach((action) => {
	qTable[stateKey][action] = 0;
	});
	}

	// Exploration: choose random action
	if (Math.random() < epsilon) {
	const randomAction =
	actions[Math.floor(Math.random() * actions.length)];
	logInfo(`Exploring: Random action ${randomAction}`);
	return randomAction;
	}

	// Exploitation: choose best action
	let bestAction = actions[0];
	let bestValue = qTable[stateKey][bestAction];

	actions.forEach((action) => {
	if (qTable[stateKey][action] > bestValue) {
	bestValue = qTable[stateKey][action];
	bestAction = action;
	}
	});

	logInfo(`Exploiting: Best action ${bestAction}`);
	return bestAction;
	}

	// RL: Update Q-value
	function updateQValue(stateKey, action, reward, nextStateKey) {
	// Ensure states exist in Q-table
	if (!qTable[stateKey]) {
	qTable[stateKey] = {};
	actions.forEach((a) => (qTable[stateKey][a] = 0));
	}

	if (!qTable[nextStateKey]) {
	qTable[nextStateKey] = {};
	actions.forEach((a) => (qTable[nextStateKey][a] = 0));
	}

	// Find max Q-value for next state
	let maxNextQ = Math.max(...actions.map((a) => qTable[nextStateKey][a]));

	// Update rule: Q(s,a) = Q(s,a) + α * (r + γ * max(Q(s')) - Q(s,a))
	const oldValue = qTable[stateKey][action];
	const newValue =
	oldValue + learningRate * (reward + gamma * maxNextQ - oldValue);
	qTable[stateKey][action] = newValue;

	logInfo(
	`Updated Q(${stateKey}, ${action}): ${oldValue.toFixed(
	2
	)} → ${newValue.toFixed(2)}`
	);
	}

	// Apply the chosen action
	function applyAction(action) {
	switch (action) {
	case "pitch_up":
	airplane.pitch += 2;
	break;
	case "pitch_down":
	airplane.pitch -= 2;
	break;
	case "throttle_up":
	airplane.speed += 10;
	break;
	case "throttle_down":
	airplane.speed -= 10;
	break;
	// 'nothing' case does nothing
	}

	// Clamp values to reasonable ranges
	airplane.pitch = Math.max(-30, Math.min(30, airplane.pitch));
	airplane.speed = Math.max(50, Math.min(300, airplane.speed));
	}

	// Apply human controls
	function applyHumanControl() {
	if (keys.up) {
	airplane.pitch += 1;
	}
	if (keys.down) {
	airplane.pitch -= 1;
	}
	if (keys.left) {
	airplane.speed -= 2;
	}
	if (keys.right) {
	airplane.speed += 2;
	}

	// Clamp values
	airplane.pitch = Math.max(-30, Math.min(30, airplane.pitch));
	airplane.speed = Math.max(50, Math.min(300, airplane.speed));
	}

	// Update physics
	function updatePhysics() {
	// Update altitude based on pitch and speed
	const pitchFactor = Math.sin((airplane.pitch * Math.PI) / 180);
	airplane.altitude += pitchFactor * airplane.speed * 0.05;

	// Apply gravity
	airplane.altitude -= 0.5;

	// Add some turbulence
	airplane.pitch += Math.random() - 0.5;

	// Gradual return to level flight (stabilization)
	airplane.pitch *= 0.98;

	// Check for crash
	if (airplane.altitude <= 0) {
	airplane.altitude = 0;
	airplane.crashed = true;
	logInfo("CRASHED!");
	}

	updateStats();
	}

	// Calculate reward
	function calculateReward() {
	let reward = 0;

	// Base reward for staying airborne
	reward += 1;

	// Reward for being at the target altitude (100-200m)
	if (airplane.altitude > 100 && airplane.altitude < 200) {
	reward += 2;
	} else if (airplane.altitude < 50 \|\| airplane.altitude > 300) {
	reward -= 2;
	}

	// Reward for good speed
	if (airplane.speed > 100 && airplane.speed < 200) {
	reward += 1;
	} else if (airplane.speed < 80 \|\| airplane.speed > 220) {
	reward -= 1;
	}

	// Penalty for extreme pitch
	if (Math.abs(airplane.pitch) > 20) {
	reward -= 2;
	}

	// Big penalty for crashing
	if (airplane.crashed) {
	reward -= 50;
	}

	return reward;
	}

	// Draw the scene
	function draw() {
	// Clear canvas
	ctx.clearRect(0, 0, canvas.width, canvas.height);

	// Draw sky
	const skyGradient = ctx.createLinearGradient(0, 0, 0, canvas.height);
	skyGradient.addColorStop(0, "#1e90ff");
	skyGradient.addColorStop(1, "#87ceeb");
	ctx.fillStyle = skyGradient;
	ctx.fillRect(0, 0, canvas.width, canvas.height);

	// Draw ground
	const groundY = canvas.height - airplane.altitude * 0.8;
	if (groundY < canvas.height) {
	ctx.fillStyle = "#8B4513";
	ctx.fillRect(0, groundY, canvas.width, canvas.height);
	}

	// Draw airplane
	ctx.save();
	ctx.translate(airplane.x, airplane.y);
	ctx.rotate((airplane.pitch * Math.PI) / 180);

	// Draw airplane body
	ctx.fillStyle = airplane.crashed ? "red" : "white";
	ctx.beginPath();
	ctx.moveTo(20, 0); // nose
	ctx.lineTo(-20, 10); // bottom rear
	ctx.lineTo(-10, 0); // tail
	ctx.lineTo(-20, -10); // top rear
	ctx.closePath();
	ctx.fill();
	ctx.strokeStyle = "#000";
	ctx.lineWidth = 2;
	ctx.stroke();

	// Draw wings
	ctx.fillStyle = "#ccc";
	ctx.beginPath();
	ctx.moveTo(0, 0);
	ctx.lineTo(-5, 30);
	ctx.lineTo(5, 30);
	ctx.closePath();
	ctx.fill();
	ctx.stroke();

	ctx.beginPath();
	ctx.moveTo(0, 0);
	ctx.lineTo(-5, -30);
	ctx.lineTo(5, -30);
	ctx.closePath();
	ctx.fill();
	ctx.stroke();

	ctx.restore();

	// Draw HUD
	ctx.font = "14px Arial";
	ctx.fillStyle = "white";
	ctx.fillText(`Altitude: ${Math.round(airplane.altitude)}m`, 20, 30);
	ctx.fillText(`Speed: ${Math.round(airplane.speed)} km/h`, 20, 50);
	ctx.fillText(`Pitch: ${Math.round(airplane.pitch)}°`, 20, 70);
	ctx.fillText(
	`Mode: ${isHumanControl ? "Human" : isLearning ? "Learning" : "AI"}`,
	20,
	90
	);

	// Draw altitude indicator
	const altIndicatorX = canvas.width - 50;
	const altIndicatorTop = 50;
	const altIndicatorHeight = 300;

	ctx.fillStyle = "rgba(0,0,0,0.5)";
	ctx.fillRect(
	altIndicatorX - 15,
	altIndicatorTop,
	30,
	altIndicatorHeight
	);

	const markerPosition =
	altIndicatorTop +
	altIndicatorHeight -
	(airplane.altitude / 200) * altIndicatorHeight;
	ctx.fillStyle = "lime";
	ctx.beginPath();
	ctx.moveTo(altIndicatorX - 20, markerPosition);
	ctx.lineTo(altIndicatorX, markerPosition - 10);
	ctx.lineTo(altIndicatorX, markerPosition + 10);
	ctx.closePath();
	ctx.fill();
	}

	// Main game loop
	function gameLoop() {
	frameCount++;

	if (isHumanControl) {
	// Human control mode
	applyHumanControl();
	updatePhysics();
	} else if (isLearning) {
	// AI learning mode
	const currentState = getStateKey();

	// Choose and apply action
	const action = chooseAction(currentState);
	applyAction(action);

	// Update physics
	updatePhysics();

	// Calculate reward
	const reward = calculateReward();
	currentReward += reward;

	// Get next state
	const nextState = getStateKey();

	// Update Q-table
	updateQValue(currentState, action, reward, nextState);

	// Check if episode is complete
	if (airplane.crashed \|\| frameCount > 1000) {
	// Log episode result
	logInfo(
	`Episode ${episode} finished with reward: ${currentReward.toFixed(
	1
	)}`
	);
	rewardHistory.push(currentReward);

	// Move to next episode
	episode++;
	document.getElementById("episodeCounter").textContent = episode;

	// Reset for next episode
	resetAirplane();

	// Reduce exploration rate over time (optional)
	if (epsilon > 0.05) {
	epsilon *= 0.99;
	document.getElementById("epsilonValue").textContent =
	epsilon.toFixed(2);
	document.getElementById("epsilon").value = epsilon;
	}
	}
	} else {
	// Idle mode - just apply physics with small drone movement
	if (frameCount % 60 === 0) {
	// Make small random adjustments
	airplane.pitch += (Math.random() - 0.5) * 2;
	}
	updatePhysics();
	}

	// Draw everything
	draw();

	// Continue game loop
	requestAnimationFrame(gameLoop);
	}

	// Start everything
	window.onload = init;
	</script>
	</body>
	</html>