diff --git a/CHANGELOG.md b/CHANGELOG.md index c0693b260..09d96b2c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Changes: * Feat: enforce maximum evolved tree-GP size with `max_len` instead of just the initial size ([#245](https://github.com/xcsf-dev/xcsf/pull/245)) +* Fix: add cartpole example normalisation and fix truncation reward ([#246](https://github.com/xcsf-dev/xcsf/pull/246)) ## Version 1.5.1 (Aug 24, 2026) diff --git a/python/example_cartpole.py b/python/example_cartpole.py index 07cae850d..1e50a1d09 100644 --- a/python/example_cartpole.py +++ b/python/example_cartpole.py @@ -1,6 +1,5 @@ -#!/usr/bin/python3 # -# Copyright (C) 2020--2023 Richard Preen +# Copyright (C) 2020--2026 Richard Preen # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -25,8 +24,6 @@ Note: These hyperparameters do not result in consistently optimal performance. """ -from __future__ import annotations - import json import random from collections import deque @@ -44,9 +41,37 @@ # Initialise OpenAI Gym problem environment ############################################ + +class CartPoleNormalisation(gym.ObservationWrapper): + """Normalise CartPole-v1 observations linearly to [0, 1]. + + Explicit domain bounds: + - Cart Position: [-2.4, 2.4] + - Cart Velocity: [-2.0, 2.0] + - Pole Angle: [-0.2095, 0.2095] + - Pole Angular Velocity: [-4.0, 4.0] + """ + + def __init__(self, env: gym.Env) -> None: + """Initialise CartPole normalisation.""" + super().__init__(env) + self.l = np.array([-2.4, -2.0, -0.2095, -4.0], dtype=np.float32) + self.u = np.array([2.4, 2.0, 0.2095, 4.0], dtype=np.float32) + self.observation_space = gym.spaces.Box( + low=0.0, high=1.0, shape=env.observation_space.shape, dtype=np.float32 + ) + + def observation(self, obs: np.ndarray) -> np.ndarray: + """Return normalised observation.""" + norm_obs = (obs - self.l) / (self.u - self.l) + return np.clip(norm_obs, 0.0, 1.0) + + env = gym.make("CartPole-v1", render_mode="rgb_array") +env = CartPoleNormalisation(env) env.reset(seed=RANDOM_STATE) +MAX_STEPS: int = int(env.spec.max_episode_steps) X_DIM: int = int(env.observation_space.shape[0]) N_ACTIONS: int = int(env.action_space.n) @@ -58,7 +83,7 @@ x_dim=X_DIM, y_dim=N_ACTIONS, n_actions=1, - omp_num_threads=12, + omp_num_threads=20, random_state=RANDOM_STATE, pop_init=False, max_trials=1, # one trial per fit() @@ -67,6 +92,8 @@ e0=0.001, alpha=1, beta=0.05, + nu=5, + loss_func="mse", ea={ "select_type": "roulette", "theta_ea": 100, @@ -97,10 +124,10 @@ }, ) -GAMMA: float = 0.95 # discount rate for delayed reward +GAMMA: float = 0.99 # discount rate for delayed reward epsilon: float = 1 # initial probability of exploring -EPSILON_MIN: float = 0.1 # the minimum exploration rate -EPSILON_DECAY: float = 0.98 # the decay of exploration after each batch replay +EPSILON_MIN: float = 0.05 # the minimum exploration rate +EPSILON_DECAY: float = 0.99 # the decay of exploration after each batch replay REPLAY_TIME: int = 1 # perform replay update every n episodes print(json.dumps(xcs.internal_params(), indent=4)) @@ -120,11 +147,12 @@ def replay(replay_size: int = 5000) -> None: """Performs experience replay updates.""" batch_size: int = min(len(memory), replay_size) batch = random.sample(memory, batch_size) - for state, action, reward, next_state, done in batch: + for state, action, reward, next_state, terminated in batch: y_target = reward - if not done: + if not terminated: prediction_array = xcs.predict(next_state.reshape(1, -1))[0] y_target += GAMMA * np.max(prediction_array) + target = xcs.predict(state.reshape(1, -1))[0] target[action] = y_target xcs.fit( @@ -150,11 +178,10 @@ def episode() -> tuple[float, int]: while True: action = egreedy_action(state) next_state, reward, terminated, truncated, _ = env.step(action) - done = terminated or truncated episode_steps += 1 episode_score += reward - memory.append((state, action, reward, next_state, done)) - if done: + memory.append((state, action, reward, next_state, terminated)) + if terminated or truncated: break state = next_state return episode_score, episode_steps @@ -175,9 +202,12 @@ def episode() -> tuple[float, int]: f"episodes={ep} " f"steps={total_steps} " f"score={mean_score:.2f} " - f"epsilon={epsilon:.5f} " - f"error={xcs.error():.5f} " - f"msize={xcs.mset_size():.2f}" + f"epsilon={epsilon:.3f} " + f"error={xcs.error():.3f} " + f"iter={xcs.time():.0f} " + f"psize={xcs.pset_size():.0f} " + f"msize={xcs.mset_size():.0f} " + f"mfrac={xcs.mfrac():.2f}" ) # is the problem solved? if ep > N and mean_score > env.spec.reward_threshold: diff --git a/python/notebooks/example_cartpole.ipynb b/python/notebooks/example_cartpole.ipynb index d05d7fe14..ac8ab0ebe 100644 --- a/python/notebooks/example_cartpole.ipynb +++ b/python/notebooks/example_cartpole.ipynb @@ -12,11 +12,9 @@ "\n", "```\n", "\n", - "**Note:** \n", + "**Note:**\n", "\n", - "These hyperparameters do not result in consistently optimal performance.\n", - "\n", - "Normalising the inputs and reward, and using prioritised experience replay may increase performance." + "These hyperparameters do not result in consistently optimal performance." ] }, { @@ -60,9 +58,36 @@ }, "outputs": [], "source": [ + "class CartPoleNormalisation(gym.ObservationWrapper):\n", + " \"\"\"Normalise CartPole-v1 observations linearly to [0, 1].\n", + "\n", + " Explicit domain bounds:\n", + " - Cart Position: [-2.4, 2.4]\n", + " - Cart Velocity: [-2.0, 2.0]\n", + " - Pole Angle: [-0.2095, 0.2095]\n", + " - Pole Angular Velocity: [-4.0, 4.0]\n", + " \"\"\"\n", + "\n", + " def __init__(self, env: gym.Env) -> None:\n", + " \"Initialise CartPole normalisation.\"\n", + " super().__init__(env)\n", + " self.l = np.array([-2.4, -2.0, -0.2095, -4.0], dtype=np.float32)\n", + " self.u = np.array([2.4, 2.0, 0.2095, 4.0], dtype=np.float32)\n", + " self.observation_space = gym.spaces.Box(\n", + " low=0.0, high=1.0, shape=env.observation_space.shape, dtype=np.float32\n", + " )\n", + "\n", + " def observation(self, obs: np.ndarray) -> np.ndarray:\n", + " \"\"\"Return normalised observation.\"\"\"\n", + " norm_obs = (obs - self.l) / (self.u - self.l)\n", + " return np.clip(norm_obs, 0.0, 1.0)\n", + "\n", + "\n", "env = gym.make(\"CartPole-v1\", render_mode=\"rgb_array\")\n", + "env = CartPoleNormalisation(env)\n", "env.reset(seed=RANDOM_STATE)\n", "\n", + "MAX_STEPS: int = int(env.spec.max_episode_steps)\n", "X_DIM: int = int(env.observation_space.shape[0])\n", "N_ACTIONS: int = int(env.action_space.n)" ] @@ -84,7 +109,7 @@ "output_type": "stream", "text": [ "{\n", - " \"version\": \"1.5.0\",\n", + " \"version\": \"1.5.1\",\n", " \"x_dim\": 4,\n", " \"y_dim\": 2,\n", " \"n_actions\": 1,\n", @@ -95,7 +120,7 @@ " \"max_trials\": 1,\n", " \"perf_trials\": 1000,\n", " \"pop_size\": 200,\n", - " \"loss_func\": \"mae\",\n", + " \"loss_func\": \"mse\",\n", " \"set_subsumption\": false,\n", " \"theta_sub\": 100,\n", " \"e0\": 0.001,\n", @@ -174,6 +199,8 @@ " e0=0.001,\n", " alpha=1,\n", " beta=0.05,\n", + " nu=5,\n", + " loss_func=\"mse\",\n", " ea={\n", " \"select_type\": \"roulette\",\n", " \"theta_ea\": 100,\n", @@ -204,10 +231,10 @@ " },\n", ")\n", "\n", - "GAMMA: float = 0.95 # discount rate for delayed reward\n", + "GAMMA: float = 0.99 # discount rate for delayed reward\n", "epsilon: float = 1 # initial probability of exploring\n", - "EPSILON_MIN: float = 0.1 # the minimum exploration rate\n", - "EPSILON_DECAY: float = 0.98 # the decay of exploration after each batch replay\n", + "EPSILON_MIN: float = 0.05 # the minimum exploration rate\n", + "EPSILON_DECAY: float = 0.99 # the decay of exploration after each batch replay\n", "REPLAY_TIME: int = 1 # perform replay update every n episodes\n", "\n", "print(json.dumps(xcs.internal_params(), indent=4))" @@ -231,226 +258,239 @@ "name": "stdout", "output_type": "stream", "text": [ - "episodes=0 steps=29 score=29.00 epsilon=1.00000 error=0.77227 msize=0.99\n", - "episodes=1 steps=45 score=22.50 epsilon=0.98000 error=1.39253 msize=1.00\n", - "episodes=2 steps=75 score=25.00 epsilon=0.96040 error=1.70605 msize=3.00\n", - "episodes=3 steps=91 score=22.75 epsilon=0.94119 error=1.28962 msize=5.00\n", - "episodes=4 steps=132 score=26.40 epsilon=0.92237 error=1.61216 msize=7.00\n", - "episodes=5 steps=144 score=24.00 epsilon=0.90392 error=3.13718 msize=10.82\n", - "episodes=6 steps=155 score=22.14 epsilon=0.88584 error=1.86810 msize=13.00\n", - "episodes=7 steps=167 score=20.88 epsilon=0.86813 error=1.65494 msize=16.99\n", - "episodes=8 steps=202 score=22.44 epsilon=0.85076 error=1.39431 msize=21.00\n", - "episodes=9 steps=223 score=22.30 epsilon=0.83375 error=1.61573 msize=25.00\n", - "episodes=10 steps=241 score=21.91 epsilon=0.81707 error=1.97849 msize=29.45\n", - "episodes=11 steps=256 score=21.33 epsilon=0.80073 error=1.10901 msize=34.55\n", - "episodes=12 steps=274 score=21.08 epsilon=0.78472 error=1.49194 msize=40.64\n", - "episodes=13 steps=289 score=20.64 epsilon=0.76902 error=0.98999 msize=46.43\n", - "episodes=14 steps=302 score=20.13 epsilon=0.75364 error=1.05824 msize=52.47\n", - "episodes=15 steps=317 score=19.81 epsilon=0.73857 error=1.20864 msize=58.32\n", - "episodes=16 steps=335 score=19.71 epsilon=0.72380 error=1.32337 msize=64.52\n", - "episodes=17 steps=359 score=19.94 epsilon=0.70932 error=1.07952 msize=72.51\n", - "episodes=18 steps=369 score=19.42 epsilon=0.69514 error=1.16774 msize=80.23\n", - "episodes=19 steps=380 score=19.00 epsilon=0.68123 error=1.06811 msize=86.31\n", - "episodes=20 steps=392 score=18.67 epsilon=0.66761 error=1.18508 msize=94.60\n", - "episodes=21 steps=405 score=18.41 epsilon=0.65426 error=1.92067 msize=102.08\n", - "episodes=22 steps=421 score=18.30 epsilon=0.64117 error=1.36597 msize=112.50\n", - "episodes=23 steps=471 score=19.62 epsilon=0.62835 error=1.37105 msize=120.47\n", - "episodes=24 steps=482 score=19.28 epsilon=0.61578 error=1.56588 msize=130.14\n", - "episodes=25 steps=493 score=18.96 epsilon=0.60346 error=1.18533 msize=140.65\n", - "episodes=26 steps=502 score=18.59 epsilon=0.59140 error=1.05623 msize=150.45\n", - "episodes=27 steps=525 score=18.75 epsilon=0.57957 error=1.59909 msize=160.48\n", - "episodes=28 steps=557 score=19.21 epsilon=0.56798 error=1.45899 msize=172.39\n", - "episodes=29 steps=568 score=18.93 epsilon=0.55662 error=1.25364 msize=184.53\n", - "episodes=30 steps=580 score=18.71 epsilon=0.54548 error=1.71899 msize=194.63\n", - "episodes=31 steps=595 score=18.59 epsilon=0.53457 error=1.36862 msize=199.34\n", - "episodes=32 steps=646 score=19.58 epsilon=0.52388 error=1.11035 msize=199.18\n", - "episodes=33 steps=659 score=19.38 epsilon=0.51341 error=1.21698 msize=198.64\n", - "episodes=34 steps=669 score=19.11 epsilon=0.50314 error=1.18631 msize=199.57\n", - "episodes=35 steps=704 score=19.56 epsilon=0.49307 error=1.70460 msize=199.29\n", - "episodes=36 steps=718 score=19.41 epsilon=0.48321 error=1.46391 msize=199.44\n", - "episodes=37 steps=739 score=19.45 epsilon=0.47355 error=1.51883 msize=199.20\n", - "episodes=38 steps=749 score=19.21 epsilon=0.46408 error=1.06259 msize=199.59\n", - "episodes=39 steps=777 score=19.43 epsilon=0.45480 error=0.98910 msize=199.72\n", - "episodes=40 steps=810 score=19.76 epsilon=0.44570 error=0.94652 msize=199.69\n", - "episodes=41 steps=825 score=19.64 epsilon=0.43679 error=1.27681 msize=199.63\n", - "episodes=42 steps=852 score=19.81 epsilon=0.42805 error=1.85845 msize=195.53\n", - "episodes=43 steps=879 score=19.98 epsilon=0.41949 error=1.48093 msize=199.60\n", - "episodes=44 steps=926 score=20.58 epsilon=0.41110 error=1.88501 msize=198.91\n", - "episodes=45 steps=937 score=20.37 epsilon=0.40288 error=1.38547 msize=199.47\n", - "episodes=46 steps=950 score=20.21 epsilon=0.39482 error=1.90332 msize=195.23\n", - "episodes=47 steps=960 score=20.00 epsilon=0.38692 error=1.84686 msize=194.73\n", - "episodes=48 steps=986 score=20.12 epsilon=0.37919 error=2.39687 msize=189.44\n", - "episodes=49 steps=1016 score=20.32 epsilon=0.37160 error=2.48595 msize=181.82\n", - "episodes=50 steps=1027 score=20.14 epsilon=0.36417 error=2.06327 msize=186.03\n", - "episodes=51 steps=1059 score=20.37 epsilon=0.35689 error=1.89673 msize=184.57\n", - "episodes=52 steps=1079 score=20.36 epsilon=0.34975 error=1.75714 msize=180.28\n", - "episodes=53 steps=1146 score=21.22 epsilon=0.34275 error=1.47433 msize=184.26\n", - "episodes=54 steps=1167 score=21.22 epsilon=0.33590 error=2.12846 msize=180.18\n", - "episodes=55 steps=1206 score=21.54 epsilon=0.32918 error=2.74092 msize=165.71\n", - "episodes=56 steps=1236 score=21.68 epsilon=0.32260 error=1.82319 msize=172.20\n", - "episodes=57 steps=1278 score=22.03 epsilon=0.31614 error=4.42087 msize=142.45\n", - "episodes=58 steps=1291 score=21.88 epsilon=0.30982 error=1.35976 msize=174.11\n", - "episodes=59 steps=1318 score=21.97 epsilon=0.30363 error=2.23872 msize=170.85\n", - "episodes=60 steps=1515 score=24.84 epsilon=0.29755 error=2.07831 msize=176.87\n", - "episodes=61 steps=1555 score=25.08 epsilon=0.29160 error=2.39666 msize=173.18\n", - "episodes=62 steps=1587 score=25.19 epsilon=0.28577 error=2.50001 msize=178.24\n", - "episodes=63 steps=1657 score=25.89 epsilon=0.28005 error=1.45155 msize=184.10\n", - "episodes=64 steps=1701 score=26.17 epsilon=0.27445 error=1.97027 msize=172.20\n", - "episodes=65 steps=1730 score=26.21 epsilon=0.26896 error=1.56327 msize=177.29\n", - "episodes=66 steps=2230 score=33.28 epsilon=0.26359 error=2.05705 msize=178.81\n", - "episodes=67 steps=2730 score=40.15 epsilon=0.25831 error=1.39909 msize=175.16\n", - "episodes=68 steps=2742 score=39.74 epsilon=0.25315 error=0.78555 msize=181.24\n", - "episodes=69 steps=2772 score=39.60 epsilon=0.24808 error=1.32200 msize=187.24\n", - "episodes=70 steps=2899 score=40.83 epsilon=0.24312 error=0.83905 msize=182.82\n", - "episodes=71 steps=3075 score=42.71 epsilon=0.23826 error=0.90545 msize=187.04\n", - "episodes=72 steps=3297 score=45.16 epsilon=0.23349 error=1.52098 msize=187.12\n", - "episodes=73 steps=3368 score=45.51 epsilon=0.22883 error=1.27896 msize=188.30\n", - "episodes=74 steps=3485 score=46.47 epsilon=0.22425 error=1.02504 msize=191.07\n", - "episodes=75 steps=3501 score=46.07 epsilon=0.21976 error=0.82980 msize=192.98\n", - "episodes=76 steps=3688 score=47.90 epsilon=0.21537 error=0.79577 msize=194.31\n", - "episodes=77 steps=3899 score=49.99 epsilon=0.21106 error=1.15048 msize=192.36\n", - "episodes=78 steps=3937 score=49.84 epsilon=0.20684 error=1.05706 msize=197.51\n", - "episodes=79 steps=4215 score=52.69 epsilon=0.20270 error=1.49671 msize=195.26\n", - "episodes=80 steps=4263 score=52.63 epsilon=0.19865 error=0.57068 msize=195.73\n", - "episodes=81 steps=4659 score=56.82 epsilon=0.19468 error=0.55894 msize=197.85\n", - "episodes=82 steps=4846 score=58.39 epsilon=0.19078 error=0.56835 msize=198.38\n", - "episodes=83 steps=5109 score=60.82 epsilon=0.18697 error=0.96025 msize=192.00\n", - "episodes=84 steps=5328 score=62.68 epsilon=0.18323 error=1.70003 msize=177.66\n", - "episodes=85 steps=5385 score=62.62 epsilon=0.17956 error=0.38548 msize=197.49\n", - "episodes=86 steps=5605 score=64.43 epsilon=0.17597 error=0.83841 msize=188.36\n", - "episodes=87 steps=5940 score=67.50 epsilon=0.17245 error=0.45614 msize=196.51\n", - "episodes=88 steps=6128 score=68.85 epsilon=0.16900 error=0.25932 msize=194.84\n", - "episodes=89 steps=6307 score=70.08 epsilon=0.16562 error=0.63162 msize=196.29\n", - "episodes=90 steps=6406 score=70.40 epsilon=0.16231 error=0.73136 msize=175.13\n", - "episodes=91 steps=6584 score=71.57 epsilon=0.15906 error=0.23910 msize=195.59\n", - "episodes=92 steps=6681 score=71.84 epsilon=0.15588 error=0.28042 msize=181.10\n", - "episodes=93 steps=6762 score=71.94 epsilon=0.15277 error=0.35877 msize=177.68\n", - "episodes=94 steps=6836 score=71.96 epsilon=0.14971 error=0.26365 msize=175.06\n", - "episodes=95 steps=6915 score=72.03 epsilon=0.14672 error=0.63360 msize=183.27\n", - "episodes=96 steps=6992 score=72.08 epsilon=0.14378 error=0.21522 msize=188.24\n", - "episodes=97 steps=7081 score=72.26 epsilon=0.14091 error=0.45760 msize=178.67\n", - "episodes=98 steps=7178 score=72.51 epsilon=0.13809 error=0.27161 msize=186.19\n", - "episodes=99 steps=7318 score=73.18 epsilon=0.13533 error=0.32757 msize=162.36\n", - "episodes=100 steps=7399 score=73.70 epsilon=0.13262 error=0.13838 msize=180.18\n", - "episodes=101 steps=7482 score=74.37 epsilon=0.12997 error=0.67961 msize=173.93\n", - "episodes=102 steps=7622 score=75.47 epsilon=0.12737 error=0.11783 msize=174.24\n", - "episodes=103 steps=7712 score=76.21 epsilon=0.12482 error=0.25828 msize=180.81\n", - "episodes=104 steps=7817 score=76.85 epsilon=0.12232 error=0.12072 msize=189.46\n", - "episodes=105 steps=7897 score=77.53 epsilon=0.11988 error=0.14270 msize=177.82\n", - "episodes=106 steps=7980 score=78.25 epsilon=0.11748 error=0.61571 msize=174.46\n", - "episodes=107 steps=8181 score=80.14 epsilon=0.11513 error=0.13155 msize=183.23\n", - "episodes=108 steps=8312 score=81.10 epsilon=0.11283 error=0.14032 msize=163.98\n", - "episodes=109 steps=8481 score=82.58 epsilon=0.11057 error=0.69085 msize=181.22\n", - "episodes=110 steps=8714 score=84.73 epsilon=0.10836 error=0.18798 msize=127.59\n", - "episodes=111 steps=8817 score=85.61 epsilon=0.10619 error=0.51595 msize=152.29\n", - "episodes=112 steps=8932 score=86.58 epsilon=0.10407 error=0.17651 msize=197.06\n", - "episodes=113 steps=9195 score=89.06 epsilon=0.10199 error=0.16473 msize=178.62\n", - "episodes=114 steps=9449 score=91.47 epsilon=0.09995 error=0.26321 msize=181.75\n", - "episodes=115 steps=9718 score=94.01 epsilon=0.09995 error=0.25563 msize=151.52\n", - "episodes=116 steps=10072 score=97.37 epsilon=0.09995 error=0.17509 msize=177.99\n", - "episodes=117 steps=10350 score=99.91 epsilon=0.09995 error=0.17174 msize=162.91\n", - "episodes=118 steps=10650 score=102.81 epsilon=0.09995 error=0.33484 msize=189.72\n", - "episodes=119 steps=11072 score=106.92 epsilon=0.09995 error=0.18684 msize=179.07\n", - "episodes=120 steps=11362 score=109.70 epsilon=0.09995 error=0.13794 msize=195.28\n", - "episodes=121 steps=11862 score=114.57 epsilon=0.09995 error=0.11126 msize=183.33\n", - "episodes=122 steps=12098 score=116.77 epsilon=0.09995 error=0.24797 msize=187.03\n", - "episodes=123 steps=12467 score=119.96 epsilon=0.09995 error=0.25186 msize=143.56\n", - "episodes=124 steps=12723 score=122.41 epsilon=0.09995 error=0.11468 msize=175.40\n", - "episodes=125 steps=13223 score=127.30 epsilon=0.09995 error=0.10317 msize=184.58\n", - "episodes=126 steps=13552 score=130.50 epsilon=0.09995 error=0.07792 msize=182.68\n", - "episodes=127 steps=13876 score=133.51 epsilon=0.09995 error=0.10607 msize=167.65\n", - "episodes=128 steps=14375 score=138.18 epsilon=0.09995 error=0.28938 msize=194.95\n", - "episodes=129 steps=14729 score=141.61 epsilon=0.09995 error=0.07302 msize=183.70\n", - "episodes=130 steps=15088 score=145.08 epsilon=0.09995 error=0.11727 msize=183.77\n", - "episodes=131 steps=15395 score=148.00 epsilon=0.09995 error=0.07436 msize=194.18\n", - "episodes=132 steps=15873 score=152.27 epsilon=0.09995 error=0.11763 msize=192.46\n", - "episodes=133 steps=16373 score=157.14 epsilon=0.09995 error=0.08734 msize=197.07\n", - "episodes=134 steps=16873 score=162.04 epsilon=0.09995 error=0.07315 msize=195.17\n", - "episodes=135 steps=17251 score=165.47 epsilon=0.09995 error=0.10013 msize=193.64\n", - "episodes=136 steps=17751 score=170.33 epsilon=0.09995 error=0.11262 msize=198.01\n", - "episodes=137 steps=18251 score=175.12 epsilon=0.09995 error=0.24749 msize=194.95\n", - "episodes=138 steps=18751 score=180.02 epsilon=0.09995 error=0.06420 msize=194.39\n", - "episodes=139 steps=19251 score=184.74 epsilon=0.09995 error=0.07939 msize=191.47\n", - "episodes=140 steps=19751 score=189.41 epsilon=0.09995 error=0.07899 msize=194.52\n", - "episodes=141 steps=20251 score=194.26 epsilon=0.09995 error=0.07443 msize=194.00\n", - "episodes=142 steps=20751 score=198.99 epsilon=0.09995 error=0.08072 msize=191.52\n", - "episodes=143 steps=21251 score=203.72 epsilon=0.09995 error=0.07174 msize=190.51\n", - "episodes=144 steps=21751 score=208.25 epsilon=0.09995 error=0.05108 msize=196.63\n", - "episodes=145 steps=22251 score=213.14 epsilon=0.09995 error=0.05971 msize=192.30\n", - "episodes=146 steps=22751 score=218.01 epsilon=0.09995 error=0.07431 msize=196.32\n", - "episodes=147 steps=23251 score=222.91 epsilon=0.09995 error=0.05413 msize=198.17\n", - "episodes=148 steps=23751 score=227.65 epsilon=0.09995 error=0.07600 msize=192.06\n", - "episodes=149 steps=24251 score=232.35 epsilon=0.09995 error=0.09676 msize=197.50\n", - "episodes=150 steps=24751 score=237.24 epsilon=0.09995 error=0.09390 msize=197.64\n", - "episodes=151 steps=25251 score=241.92 epsilon=0.09995 error=0.05692 msize=198.76\n", - "episodes=152 steps=25751 score=246.72 epsilon=0.09995 error=0.07209 msize=197.75\n", - "episodes=153 steps=26251 score=251.05 epsilon=0.09995 error=0.05824 msize=198.49\n", - "episodes=154 steps=26751 score=255.84 epsilon=0.09995 error=0.23969 msize=193.34\n", - "episodes=155 steps=27251 score=260.45 epsilon=0.09995 error=0.05930 msize=195.79\n", - "episodes=156 steps=27751 score=265.15 epsilon=0.09995 error=0.10916 msize=175.87\n", - "episodes=157 steps=28251 score=269.73 epsilon=0.09995 error=0.05061 msize=198.05\n", - "episodes=158 steps=28751 score=274.60 epsilon=0.09995 error=0.04145 msize=196.37\n", - "episodes=159 steps=29251 score=279.33 epsilon=0.09995 error=0.05049 msize=193.79\n", - "episodes=160 steps=29743 score=282.28 epsilon=0.09995 error=0.04018 msize=193.92\n", - "episodes=161 steps=30243 score=286.88 epsilon=0.09995 error=0.06093 msize=199.68\n", - "episodes=162 steps=30743 score=291.56 epsilon=0.09995 error=0.11628 msize=197.04\n", - "episodes=163 steps=31243 score=295.86 epsilon=0.09995 error=0.04434 msize=198.07\n", - "episodes=164 steps=31743 score=300.42 epsilon=0.09995 error=0.04250 msize=199.43\n", - "episodes=165 steps=32243 score=305.13 epsilon=0.09995 error=0.08385 msize=197.17\n", - "episodes=166 steps=32743 score=305.13 epsilon=0.09995 error=0.10350 msize=195.94\n", - "episodes=167 steps=33243 score=305.13 epsilon=0.09995 error=0.04282 msize=185.47\n", - "episodes=168 steps=33743 score=310.01 epsilon=0.09995 error=0.07468 msize=195.98\n", - "episodes=169 steps=34243 score=314.71 epsilon=0.09995 error=0.09116 msize=182.41\n", - "episodes=170 steps=34743 score=318.44 epsilon=0.09995 error=0.07657 msize=174.51\n", - "episodes=171 steps=35243 score=321.68 epsilon=0.09995 error=0.07173 msize=186.95\n", - "episodes=172 steps=35743 score=324.46 epsilon=0.09995 error=0.03710 msize=196.04\n", - "episodes=173 steps=36243 score=328.75 epsilon=0.09995 error=0.06134 msize=187.94\n", - "episodes=174 steps=36743 score=332.58 epsilon=0.09995 error=0.08692 msize=180.14\n", - "episodes=175 steps=37243 score=337.42 epsilon=0.09995 error=0.08900 msize=178.12\n", - "episodes=176 steps=37743 score=340.55 epsilon=0.09995 error=0.05271 msize=189.40\n", - "episodes=177 steps=38243 score=343.44 epsilon=0.09995 error=0.03019 msize=187.97\n", - "episodes=178 steps=38743 score=348.06 epsilon=0.09995 error=0.03279 msize=179.09\n", - "episodes=179 steps=39243 score=350.28 epsilon=0.09995 error=0.04217 msize=191.36\n", - "episodes=180 steps=39276 score=350.13 epsilon=0.09995 error=0.09074 msize=191.17\n", - "episodes=181 steps=39776 score=351.17 epsilon=0.09995 error=0.04581 msize=184.57\n", - "episodes=182 steps=40276 score=354.30 epsilon=0.09995 error=0.05431 msize=174.96\n", - "episodes=183 steps=40776 score=356.67 epsilon=0.09995 error=0.10381 msize=191.67\n", - "episodes=184 steps=41276 score=359.48 epsilon=0.09995 error=0.25636 msize=195.41\n", - "episodes=185 steps=41776 score=363.91 epsilon=0.09995 error=0.14187 msize=161.67\n", - "episodes=186 steps=42276 score=366.71 epsilon=0.09995 error=0.05291 msize=183.51\n", - "episodes=187 steps=42776 score=368.36 epsilon=0.09995 error=0.04859 msize=181.77\n", - "episodes=188 steps=43276 score=371.48 epsilon=0.09995 error=0.06105 msize=196.93\n", - "episodes=189 steps=43776 score=374.69 epsilon=0.09995 error=0.03416 msize=194.86\n", - "episodes=190 steps=44276 score=378.70 epsilon=0.09995 error=0.07116 msize=190.63\n", - "episodes=191 steps=44776 score=381.92 epsilon=0.09995 error=0.02992 msize=192.82\n", - "episodes=192 steps=45276 score=385.95 epsilon=0.09995 error=0.03257 msize=174.61\n", - "episodes=193 steps=45776 score=390.14 epsilon=0.09995 error=0.02871 msize=177.69\n", - "episodes=194 steps=46276 score=394.40 epsilon=0.09995 error=0.03758 msize=177.77\n", - "episodes=195 steps=46776 score=398.61 epsilon=0.09995 error=0.04355 msize=173.98\n", - "episodes=196 steps=47276 score=402.84 epsilon=0.09995 error=0.08662 msize=184.84\n", - "episodes=197 steps=47776 score=406.95 epsilon=0.09995 error=0.03995 msize=161.56\n", - "episodes=198 steps=48276 score=410.98 epsilon=0.09995 error=0.06965 msize=195.64\n", - "episodes=199 steps=48776 score=414.58 epsilon=0.09995 error=0.04222 msize=188.29\n", - "episodes=200 steps=49276 score=418.77 epsilon=0.09995 error=0.03751 msize=175.51\n", - "episodes=201 steps=49776 score=422.94 epsilon=0.09995 error=0.06007 msize=173.06\n", - "episodes=202 steps=50276 score=426.54 epsilon=0.09995 error=0.08934 msize=178.47\n", - "episodes=203 steps=50776 score=430.64 epsilon=0.09995 error=0.03873 msize=169.02\n", - "episodes=204 steps=51276 score=434.59 epsilon=0.09995 error=0.03461 msize=184.38\n", - "episodes=205 steps=51776 score=438.79 epsilon=0.09995 error=0.03051 msize=189.60\n", - "episodes=206 steps=52276 score=442.96 epsilon=0.09995 error=0.03459 msize=191.66\n", - "episodes=207 steps=52776 score=445.95 epsilon=0.09995 error=0.03631 msize=171.10\n", - "episodes=208 steps=53276 score=449.64 epsilon=0.09995 error=0.03493 msize=176.55\n", - "episodes=209 steps=53776 score=452.95 epsilon=0.09995 error=0.04956 msize=183.14\n", - "episodes=210 steps=54276 score=455.62 epsilon=0.09995 error=0.04252 msize=189.74\n", - "episodes=211 steps=54776 score=459.59 epsilon=0.09995 error=0.04523 msize=189.02\n", - "episodes=212 steps=55276 score=463.44 epsilon=0.09995 error=0.03444 msize=187.42\n", - "episodes=213 steps=55776 score=465.81 epsilon=0.09995 error=0.03401 msize=168.78\n", - "episodes=214 steps=56276 score=468.27 epsilon=0.09995 error=0.02691 msize=191.72\n", - "episodes=215 steps=56776 score=470.58 epsilon=0.09995 error=0.02983 msize=193.18\n", - "episodes=216 steps=57276 score=472.04 epsilon=0.09995 error=0.04898 msize=177.08\n", - "episodes=217 steps=57776 score=474.26 epsilon=0.09995 error=0.02964 msize=188.71\n", - "episodes=218 steps=58276 score=476.26 epsilon=0.09995 error=0.03078 msize=180.95\n", - "solved after 218 episodes: mean score 476.26 > 475.00\n" + "episodes=0 steps=29 score=29.00 epsilon=1.000 error=2.507 iter=29 psize=1 msize=1 mfrac=0.69\n", + "episodes=1 steps=45 score=22.50 epsilon=0.990 error=1.568 iter=74 psize=1 msize=1 mfrac=1.00\n", + "episodes=2 steps=68 score=22.67 epsilon=0.980 error=2.979 iter=142 psize=3 msize=3 mfrac=1.00\n", + "episodes=3 steps=89 score=22.25 epsilon=0.970 error=1.259 iter=231 psize=5 msize=5 mfrac=1.00\n", + "episodes=4 steps=115 score=23.00 epsilon=0.961 error=0.840 iter=346 psize=7 msize=7 mfrac=1.00\n", + "episodes=5 steps=127 score=21.17 epsilon=0.951 error=1.279 iter=473 psize=9 msize=9 mfrac=1.00\n", + "episodes=6 steps=137 score=19.57 epsilon=0.941 error=0.559 iter=610 psize=13 msize=13 mfrac=1.00\n", + "episodes=7 steps=176 score=22.00 epsilon=0.932 error=0.625 iter=786 psize=15 msize=15 mfrac=1.00\n", + "episodes=8 steps=189 score=21.00 epsilon=0.923 error=1.037 iter=975 psize=19 msize=19 mfrac=1.00\n", + "episodes=9 steps=227 score=22.70 epsilon=0.914 error=0.585 iter=1202 psize=25 msize=23 mfrac=1.00\n", + "episodes=10 steps=241 score=21.91 epsilon=0.904 error=0.498 iter=1443 psize=29 msize=28 mfrac=1.00\n", + "episodes=11 steps=300 score=25.00 epsilon=0.895 error=0.361 iter=1743 psize=35 msize=34 mfrac=1.00\n", + "episodes=12 steps=343 score=26.38 epsilon=0.886 error=0.565 iter=2086 psize=41 msize=40 mfrac=1.00\n", + "episodes=13 steps=380 score=27.14 epsilon=0.878 error=1.589 iter=2466 psize=49 msize=48 mfrac=1.00\n", + "episodes=14 steps=390 score=26.00 epsilon=0.869 error=0.812 iter=2856 psize=57 msize=56 mfrac=1.00\n", + "episodes=15 steps=412 score=25.75 epsilon=0.860 error=1.574 iter=3268 psize=65 msize=64 mfrac=1.00\n", + "episodes=16 steps=432 score=25.41 epsilon=0.851 error=0.584 iter=3700 psize=75 msize=72 mfrac=1.00\n", + "episodes=17 steps=452 score=25.11 epsilon=0.843 error=2.294 iter=4152 psize=83 msize=81 mfrac=1.00\n", + "episodes=18 steps=483 score=25.42 epsilon=0.835 error=0.571 iter=4635 psize=93 msize=91 mfrac=1.00\n", + "episodes=19 steps=496 score=24.80 epsilon=0.826 error=1.503 iter=5131 psize=103 msize=101 mfrac=1.00\n", + "episodes=20 steps=514 score=24.48 epsilon=0.818 error=2.071 iter=5645 psize=113 msize=111 mfrac=1.00\n", + "episodes=21 steps=535 score=24.32 epsilon=0.810 error=0.524 iter=6180 psize=123 msize=121 mfrac=1.00\n", + "episodes=22 steps=580 score=25.22 epsilon=0.802 error=0.603 iter=6760 psize=135 msize=133 mfrac=1.00\n", + "episodes=23 steps=606 score=25.25 epsilon=0.794 error=1.000 iter=7366 psize=147 msize=145 mfrac=1.00\n", + "episodes=24 steps=620 score=24.80 epsilon=0.786 error=1.398 iter=7986 psize=159 msize=157 mfrac=1.00\n", + "episodes=25 steps=665 score=25.58 epsilon=0.778 error=0.498 iter=8651 psize=173 msize=171 mfrac=1.00\n", + "episodes=26 steps=693 score=25.67 epsilon=0.770 error=0.735 iter=9344 psize=187 msize=185 mfrac=1.00\n", + "episodes=27 steps=753 score=26.89 epsilon=0.762 error=0.639 iter=10097 psize=200 msize=199 mfrac=1.00\n", + "episodes=28 steps=762 score=26.28 epsilon=0.755 error=1.676 iter=10859 psize=200 msize=200 mfrac=1.00\n", + "episodes=29 steps=830 score=27.67 epsilon=0.747 error=1.051 iter=11689 psize=200 msize=200 mfrac=1.00\n", + "episodes=30 steps=889 score=28.68 epsilon=0.740 error=0.605 iter=12578 psize=200 msize=200 mfrac=1.00\n", + "episodes=31 steps=930 score=29.06 epsilon=0.732 error=0.766 iter=13508 psize=200 msize=200 mfrac=1.00\n", + "episodes=32 steps=997 score=30.21 epsilon=0.725 error=0.768 iter=14505 psize=200 msize=200 mfrac=1.00\n", + "episodes=33 steps=1062 score=31.24 epsilon=0.718 error=0.666 iter=15567 psize=200 msize=200 mfrac=1.00\n", + "episodes=34 steps=1105 score=31.57 epsilon=0.711 error=0.992 iter=16672 psize=200 msize=200 mfrac=1.00\n", + "episodes=35 steps=1140 score=31.67 epsilon=0.703 error=1.286 iter=17812 psize=200 msize=200 mfrac=1.00\n", + "episodes=36 steps=1155 score=31.22 epsilon=0.696 error=0.624 iter=18967 psize=200 msize=200 mfrac=1.00\n", + "episodes=37 steps=1168 score=30.74 epsilon=0.689 error=1.631 iter=20135 psize=200 msize=200 mfrac=1.00\n", + "episodes=38 steps=1243 score=31.87 epsilon=0.683 error=2.118 iter=21378 psize=200 msize=200 mfrac=1.00\n", + "episodes=39 steps=1298 score=32.45 epsilon=0.676 error=0.907 iter=22676 psize=200 msize=200 mfrac=1.00\n", + "episodes=40 steps=1362 score=33.22 epsilon=0.669 error=0.666 iter=24038 psize=200 msize=200 mfrac=1.00\n", + "episodes=41 steps=1410 score=33.57 epsilon=0.662 error=0.533 iter=25448 psize=200 msize=200 mfrac=1.00\n", + "episodes=42 steps=1445 score=33.60 epsilon=0.656 error=0.686 iter=26893 psize=200 msize=200 mfrac=1.00\n", + "episodes=43 steps=1591 score=36.16 epsilon=0.649 error=3.334 iter=28484 psize=200 msize=200 mfrac=1.00\n", + "episodes=44 steps=1614 score=35.87 epsilon=0.643 error=0.784 iter=30098 psize=200 msize=200 mfrac=1.00\n", + "episodes=45 steps=1653 score=35.93 epsilon=0.636 error=1.151 iter=31751 psize=200 msize=200 mfrac=1.00\n", + "episodes=46 steps=1707 score=36.32 epsilon=0.630 error=0.693 iter=33458 psize=200 msize=200 mfrac=1.00\n", + "episodes=47 steps=1732 score=36.08 epsilon=0.624 error=0.708 iter=35190 psize=200 msize=200 mfrac=1.00\n", + "episodes=48 steps=1762 score=35.96 epsilon=0.617 error=1.974 iter=36952 psize=200 msize=200 mfrac=1.00\n", + "episodes=49 steps=1847 score=36.94 epsilon=0.611 error=1.405 iter=38799 psize=200 msize=200 mfrac=1.00\n", + "episodes=50 steps=1923 score=37.71 epsilon=0.605 error=3.975 iter=40722 psize=200 msize=200 mfrac=1.00\n", + "episodes=51 steps=2013 score=38.71 epsilon=0.599 error=1.403 iter=42735 psize=200 msize=200 mfrac=1.00\n", + "episodes=52 steps=2099 score=39.60 epsilon=0.593 error=3.905 iter=44834 psize=200 msize=200 mfrac=1.00\n", + "episodes=53 steps=2164 score=40.07 epsilon=0.587 error=0.802 iter=46998 psize=200 msize=199 mfrac=1.00\n", + "episodes=54 steps=2291 score=41.65 epsilon=0.581 error=2.079 iter=49289 psize=200 msize=199 mfrac=0.97\n", + "episodes=55 steps=2322 score=41.46 epsilon=0.575 error=0.759 iter=51611 psize=200 msize=196 mfrac=0.95\n", + "episodes=56 steps=2353 score=41.28 epsilon=0.570 error=0.806 iter=53964 psize=200 msize=199 mfrac=1.00\n", + "episodes=57 steps=2364 score=40.76 epsilon=0.564 error=0.741 iter=56328 psize=200 msize=197 mfrac=1.00\n", + "episodes=58 steps=2456 score=41.63 epsilon=0.558 error=0.714 iter=58784 psize=200 msize=198 mfrac=1.00\n", + "episodes=59 steps=2589 score=43.15 epsilon=0.553 error=1.004 iter=61373 psize=200 msize=200 mfrac=1.00\n", + "episodes=60 steps=2706 score=44.36 epsilon=0.547 error=0.873 iter=64079 psize=200 msize=200 mfrac=1.00\n", + "episodes=61 steps=2836 score=45.74 epsilon=0.542 error=1.083 iter=66915 psize=200 msize=200 mfrac=1.00\n", + "episodes=62 steps=2909 score=46.17 epsilon=0.536 error=0.976 iter=69824 psize=200 msize=200 mfrac=1.00\n", + "episodes=63 steps=2958 score=46.22 epsilon=0.531 error=0.769 iter=72782 psize=200 msize=200 mfrac=1.00\n", + "episodes=64 steps=3305 score=50.85 epsilon=0.526 error=0.948 iter=76087 psize=200 msize=199 mfrac=1.00\n", + "episodes=65 steps=3378 score=51.18 epsilon=0.520 error=0.910 iter=79465 psize=200 msize=199 mfrac=1.00\n", + "episodes=66 steps=3472 score=51.82 epsilon=0.515 error=1.236 iter=82937 psize=200 msize=199 mfrac=1.00\n", + "episodes=67 steps=3536 score=52.00 epsilon=0.510 error=1.645 iter=86473 psize=200 msize=199 mfrac=1.00\n", + "episodes=68 steps=3670 score=53.19 epsilon=0.505 error=1.694 iter=90143 psize=200 msize=199 mfrac=1.00\n", + "episodes=69 steps=3735 score=53.36 epsilon=0.500 error=0.892 iter=93878 psize=200 msize=198 mfrac=1.00\n", + "episodes=70 steps=4018 score=56.59 epsilon=0.495 error=1.896 iter=97896 psize=200 msize=197 mfrac=0.76\n", + "episodes=71 steps=4262 score=59.19 epsilon=0.490 error=1.082 iter=102158 psize=200 msize=199 mfrac=1.00\n", + "episodes=72 steps=4456 score=61.04 epsilon=0.485 error=0.899 iter=106614 psize=200 msize=200 mfrac=1.00\n", + "episodes=73 steps=4572 score=61.78 epsilon=0.480 error=0.715 iter=111186 psize=200 msize=199 mfrac=0.87\n", + "episodes=74 steps=4597 score=61.29 epsilon=0.475 error=1.698 iter=115783 psize=200 msize=198 mfrac=0.78\n", + "episodes=75 steps=4725 score=62.17 epsilon=0.471 error=1.848 iter=120508 psize=200 msize=196 mfrac=0.74\n", + "episodes=76 steps=4790 score=62.21 epsilon=0.466 error=3.835 iter=125298 psize=200 msize=194 mfrac=0.78\n", + "episodes=77 steps=5116 score=65.59 epsilon=0.461 error=0.819 iter=130298 psize=200 msize=194 mfrac=0.87\n", + "episodes=78 steps=5178 score=65.54 epsilon=0.457 error=1.552 iter=135298 psize=200 msize=191 mfrac=0.91\n", + "episodes=79 steps=5306 score=66.33 epsilon=0.452 error=19.520 iter=140298 psize=200 msize=191 mfrac=0.87\n", + "episodes=80 steps=5449 score=67.27 epsilon=0.448 error=0.804 iter=145298 psize=200 msize=189 mfrac=0.94\n", + "episodes=81 steps=5701 score=69.52 epsilon=0.443 error=4.583 iter=150298 psize=200 msize=184 mfrac=0.99\n", + "episodes=82 steps=5894 score=71.01 epsilon=0.439 error=2.080 iter=155298 psize=200 msize=187 mfrac=0.95\n", + "episodes=83 steps=6100 score=72.62 epsilon=0.434 error=4.190 iter=160298 psize=200 msize=183 mfrac=0.90\n", + "episodes=84 steps=6179 score=72.69 epsilon=0.430 error=1.120 iter=165298 psize=200 msize=183 mfrac=0.79\n", + "episodes=85 steps=6638 score=77.19 epsilon=0.426 error=16.988 iter=170298 psize=200 msize=179 mfrac=0.92\n", + "episodes=86 steps=6917 score=79.51 epsilon=0.421 error=4.185 iter=175298 psize=200 msize=173 mfrac=0.99\n", + "episodes=87 steps=6989 score=79.42 epsilon=0.417 error=4.891 iter=180298 psize=200 msize=178 mfrac=0.80\n", + "episodes=88 steps=7170 score=80.56 epsilon=0.413 error=21.843 iter=185298 psize=200 msize=179 mfrac=0.96\n", + "episodes=89 steps=7560 score=84.00 epsilon=0.409 error=1.186 iter=190298 psize=200 msize=175 mfrac=0.83\n", + "episodes=90 steps=7862 score=86.40 epsilon=0.405 error=3.105 iter=195298 psize=200 msize=163 mfrac=0.49\n", + "episodes=91 steps=8061 score=87.62 epsilon=0.401 error=17.770 iter=200298 psize=200 msize=181 mfrac=0.57\n", + "episodes=92 steps=8342 score=89.70 epsilon=0.397 error=0.880 iter=205298 psize=200 msize=177 mfrac=0.88\n", + "episodes=93 steps=8561 score=91.07 epsilon=0.393 error=0.949 iter=210298 psize=200 msize=186 mfrac=0.91\n", + "episodes=94 steps=8832 score=92.97 epsilon=0.389 error=0.864 iter=215298 psize=200 msize=177 mfrac=0.78\n", + "episodes=95 steps=8882 score=92.52 epsilon=0.385 error=22.836 iter=220298 psize=200 msize=179 mfrac=0.78\n", + "episodes=96 steps=8964 score=92.41 epsilon=0.381 error=1.553 iter=225298 psize=200 msize=176 mfrac=0.93\n", + "episodes=97 steps=9301 score=94.91 epsilon=0.377 error=11.240 iter=230298 psize=200 msize=171 mfrac=0.95\n", + "episodes=98 steps=9585 score=96.82 epsilon=0.373 error=5.273 iter=235298 psize=200 msize=167 mfrac=0.94\n", + "episodes=99 steps=9743 score=97.43 epsilon=0.370 error=1.118 iter=240298 psize=200 msize=188 mfrac=0.86\n", + "episodes=100 steps=10121 score=100.92 epsilon=0.366 error=4.743 iter=245298 psize=200 msize=167 mfrac=0.88\n", + "episodes=101 steps=10407 score=103.62 epsilon=0.362 error=1.444 iter=250298 psize=200 msize=173 mfrac=0.81\n", + "episodes=102 steps=10739 score=106.71 epsilon=0.359 error=0.874 iter=255298 psize=200 msize=174 mfrac=0.82\n", + "episodes=103 steps=10893 score=108.04 epsilon=0.355 error=1.131 iter=260298 psize=200 msize=169 mfrac=0.91\n", + "episodes=104 steps=11220 score=111.05 epsilon=0.352 error=1.419 iter=265298 psize=200 msize=177 mfrac=0.76\n", + "episodes=105 steps=11531 score=114.04 epsilon=0.348 error=2.154 iter=270298 psize=200 msize=137 mfrac=0.96\n", + "episodes=106 steps=11806 score=116.69 epsilon=0.345 error=2.824 iter=275298 psize=200 msize=167 mfrac=0.85\n", + "episodes=107 steps=12073 score=118.97 epsilon=0.341 error=9.746 iter=280298 psize=200 msize=161 mfrac=0.16\n", + "episodes=108 steps=12509 score=123.20 epsilon=0.338 error=1.721 iter=285298 psize=200 msize=164 mfrac=0.94\n", + "episodes=109 steps=12805 score=125.78 epsilon=0.334 error=1.226 iter=290298 psize=200 msize=169 mfrac=0.92\n", + "episodes=110 steps=13128 score=128.87 epsilon=0.331 error=0.963 iter=295298 psize=200 msize=172 mfrac=1.00\n", + "episodes=111 steps=13418 score=131.18 epsilon=0.328 error=1.823 iter=300298 psize=200 msize=183 mfrac=0.94\n", + "episodes=112 steps=13762 score=134.19 epsilon=0.324 error=1.364 iter=305298 psize=200 msize=181 mfrac=0.90\n", + "episodes=113 steps=13953 score=135.73 epsilon=0.321 error=1.216 iter=310298 psize=200 msize=182 mfrac=0.86\n", + "episodes=114 steps=14278 score=138.88 epsilon=0.318 error=6.637 iter=315298 psize=200 msize=174 mfrac=0.89\n", + "episodes=115 steps=14349 score=139.37 epsilon=0.315 error=5.582 iter=320298 psize=200 msize=171 mfrac=0.99\n", + "episodes=116 steps=14723 score=142.91 epsilon=0.312 error=1.044 iter=325298 psize=200 msize=183 mfrac=0.92\n", + "episodes=117 steps=15200 score=147.48 epsilon=0.309 error=2.313 iter=330298 psize=200 msize=171 mfrac=0.93\n", + "episodes=118 steps=15700 score=152.17 epsilon=0.305 error=0.947 iter=335298 psize=200 msize=183 mfrac=0.80\n", + "episodes=119 steps=15945 score=154.49 epsilon=0.302 error=1.602 iter=340298 psize=200 msize=183 mfrac=0.87\n", + "episodes=120 steps=16317 score=158.03 epsilon=0.299 error=0.960 iter=345298 psize=200 msize=169 mfrac=0.73\n", + "episodes=121 steps=16602 score=160.67 epsilon=0.296 error=0.952 iter=350298 psize=200 msize=174 mfrac=0.85\n", + "episodes=122 steps=17003 score=164.23 epsilon=0.293 error=0.827 iter=355298 psize=200 msize=165 mfrac=0.86\n", + "episodes=123 steps=17310 score=167.04 epsilon=0.290 error=1.337 iter=360298 psize=200 msize=180 mfrac=0.93\n", + "episodes=124 steps=17810 score=171.90 epsilon=0.288 error=1.394 iter=365298 psize=200 msize=182 mfrac=0.84\n", + "episodes=125 steps=17846 score=171.81 epsilon=0.285 error=0.805 iter=370298 psize=200 msize=181 mfrac=0.96\n", + "episodes=126 steps=18346 score=176.53 epsilon=0.282 error=0.966 iter=375298 psize=200 msize=182 mfrac=0.88\n", + "episodes=127 steps=18699 score=179.46 epsilon=0.279 error=0.567 iter=380298 psize=200 msize=185 mfrac=0.88\n", + "episodes=128 steps=19117 score=183.55 epsilon=0.276 error=1.277 iter=385298 psize=200 msize=177 mfrac=0.86\n", + "episodes=129 steps=19512 score=186.82 epsilon=0.273 error=9.695 iter=390298 psize=200 msize=183 mfrac=0.97\n", + "episodes=130 steps=20012 score=191.23 epsilon=0.271 error=5.981 iter=395298 psize=200 msize=165 mfrac=0.57\n", + "episodes=131 steps=20327 score=193.97 epsilon=0.268 error=0.690 iter=400298 psize=200 msize=182 mfrac=0.89\n", + "episodes=132 steps=20747 score=197.50 epsilon=0.265 error=1.153 iter=405298 psize=200 msize=172 mfrac=0.93\n", + "episodes=133 steps=21247 score=201.85 epsilon=0.263 error=6.801 iter=410298 psize=200 msize=156 mfrac=0.75\n", + "episodes=134 steps=21627 score=205.22 epsilon=0.260 error=1.419 iter=415298 psize=200 msize=160 mfrac=0.83\n", + "episodes=135 steps=22046 score=209.06 epsilon=0.257 error=1.981 iter=420298 psize=200 msize=178 mfrac=0.97\n", + "episodes=136 steps=22546 score=213.91 epsilon=0.255 error=1.054 iter=425298 psize=200 msize=160 mfrac=0.80\n", + "episodes=137 steps=22910 score=217.42 epsilon=0.252 error=0.487 iter=430298 psize=200 msize=178 mfrac=0.95\n", + "episodes=138 steps=23340 score=220.97 epsilon=0.250 error=0.925 iter=435298 psize=200 msize=174 mfrac=0.91\n", + "episodes=139 steps=23704 score=224.06 epsilon=0.247 error=2.421 iter=440298 psize=200 msize=183 mfrac=0.65\n", + "episodes=140 steps=24204 score=228.42 epsilon=0.245 error=2.509 iter=445298 psize=200 msize=168 mfrac=0.94\n", + "episodes=141 steps=24483 score=230.73 epsilon=0.242 error=2.673 iter=450298 psize=200 msize=150 mfrac=0.90\n", + "episodes=142 steps=24983 score=235.38 epsilon=0.240 error=0.907 iter=455298 psize=200 msize=170 mfrac=0.92\n", + "episodes=143 steps=25483 score=238.92 epsilon=0.238 error=73.417 iter=460298 psize=200 msize=187 mfrac=0.92\n", + "episodes=144 steps=25902 score=242.88 epsilon=0.235 error=2.576 iter=465298 psize=200 msize=174 mfrac=0.82\n", + "episodes=145 steps=26402 score=247.49 epsilon=0.233 error=0.572 iter=470298 psize=200 msize=168 mfrac=0.99\n", + "episodes=146 steps=26902 score=251.95 epsilon=0.231 error=0.718 iter=475298 psize=200 msize=175 mfrac=0.87\n", + "episodes=147 steps=27402 score=256.70 epsilon=0.228 error=0.888 iter=480298 psize=200 msize=177 mfrac=0.69\n", + "episodes=148 steps=27902 score=261.40 epsilon=0.226 error=17.397 iter=485298 psize=200 msize=175 mfrac=0.81\n", + "episodes=149 steps=28402 score=265.55 epsilon=0.224 error=1.250 iter=490298 psize=200 msize=176 mfrac=0.83\n", + "episodes=150 steps=28902 score=269.79 epsilon=0.221 error=0.697 iter=495298 psize=200 msize=182 mfrac=0.77\n", + "episodes=151 steps=29402 score=273.89 epsilon=0.219 error=0.609 iter=500298 psize=200 msize=147 mfrac=0.95\n", + "episodes=152 steps=29902 score=278.03 epsilon=0.217 error=0.754 iter=505298 psize=200 msize=181 mfrac=0.96\n", + "episodes=153 steps=30402 score=282.38 epsilon=0.215 error=0.730 iter=510298 psize=200 msize=181 mfrac=0.92\n", + "episodes=154 steps=30902 score=286.11 epsilon=0.213 error=0.748 iter=515298 psize=200 msize=167 mfrac=0.94\n", + "episodes=155 steps=31402 score=290.80 epsilon=0.211 error=1.507 iter=520298 psize=200 msize=173 mfrac=0.70\n", + "episodes=156 steps=31902 score=295.49 epsilon=0.208 error=0.597 iter=525298 psize=200 msize=158 mfrac=0.84\n", + "episodes=157 steps=32402 score=300.38 epsilon=0.206 error=1.293 iter=530298 psize=200 msize=155 mfrac=0.90\n", + "episodes=158 steps=32902 score=304.46 epsilon=0.204 error=2.829 iter=535298 psize=200 msize=181 mfrac=1.00\n", + "episodes=159 steps=33402 score=308.13 epsilon=0.202 error=0.711 iter=540298 psize=200 msize=170 mfrac=0.90\n", + "episodes=160 steps=33902 score=311.96 epsilon=0.200 error=0.457 iter=545298 psize=200 msize=151 mfrac=0.74\n", + "episodes=161 steps=34402 score=315.66 epsilon=0.198 error=5.394 iter=550298 psize=200 msize=184 mfrac=0.57\n", + "episodes=162 steps=34902 score=319.93 epsilon=0.196 error=1.060 iter=555298 psize=200 msize=180 mfrac=1.00\n", + "episodes=163 steps=35402 score=324.44 epsilon=0.194 error=1.451 iter=560298 psize=200 msize=157 mfrac=0.95\n", + "episodes=164 steps=35902 score=325.97 epsilon=0.192 error=0.411 iter=565298 psize=200 msize=171 mfrac=0.84\n", + "episodes=165 steps=36402 score=330.24 epsilon=0.190 error=1.604 iter=570298 psize=200 msize=167 mfrac=0.79\n", + "episodes=166 steps=36902 score=334.30 epsilon=0.189 error=2.653 iter=575298 psize=200 msize=175 mfrac=0.67\n", + "episodes=167 steps=37402 score=338.66 epsilon=0.187 error=0.570 iter=580298 psize=200 msize=171 mfrac=0.96\n", + "episodes=168 steps=37902 score=342.32 epsilon=0.185 error=0.655 iter=585298 psize=200 msize=164 mfrac=0.83\n", + "episodes=169 steps=38402 score=346.67 epsilon=0.183 error=1.173 iter=590298 psize=200 msize=174 mfrac=0.79\n", + "episodes=170 steps=38902 score=348.84 epsilon=0.181 error=2.416 iter=595298 psize=200 msize=162 mfrac=0.81\n", + "episodes=171 steps=39402 score=351.40 epsilon=0.179 error=0.829 iter=600298 psize=200 msize=175 mfrac=0.72\n", + "episodes=172 steps=39902 score=354.46 epsilon=0.178 error=9.684 iter=605298 psize=200 msize=162 mfrac=0.92\n", + "episodes=173 steps=40402 score=358.30 epsilon=0.176 error=0.684 iter=610298 psize=200 msize=167 mfrac=0.77\n", + "episodes=174 steps=40902 score=363.05 epsilon=0.174 error=0.451 iter=615298 psize=200 msize=170 mfrac=0.82\n", + "episodes=175 steps=41402 score=366.77 epsilon=0.172 error=1.144 iter=620298 psize=200 msize=156 mfrac=0.72\n", + "episodes=176 steps=41902 score=371.12 epsilon=0.171 error=0.880 iter=625298 psize=200 msize=174 mfrac=0.80\n", + "episodes=177 steps=42402 score=372.86 epsilon=0.169 error=0.885 iter=630298 psize=200 msize=150 mfrac=0.95\n", + "episodes=178 steps=42902 score=377.24 epsilon=0.167 error=0.403 iter=635298 psize=200 msize=178 mfrac=0.88\n", + "episodes=179 steps=43402 score=380.96 epsilon=0.165 error=1.951 iter=640298 psize=200 msize=156 mfrac=0.79\n", + "episodes=180 steps=43902 score=384.53 epsilon=0.164 error=0.334 iter=645298 psize=200 msize=181 mfrac=0.83\n", + "episodes=181 steps=44402 score=387.01 epsilon=0.162 error=0.704 iter=650298 psize=200 msize=163 mfrac=0.88\n", + "episodes=182 steps=44902 score=390.08 epsilon=0.161 error=0.223 iter=655298 psize=200 msize=173 mfrac=0.94\n", + "episodes=183 steps=45402 score=393.02 epsilon=0.159 error=0.474 iter=660298 psize=200 msize=175 mfrac=0.94\n", + "episodes=184 steps=45902 score=397.23 epsilon=0.157 error=0.734 iter=665298 psize=200 msize=181 mfrac=0.68\n", + "episodes=185 steps=46402 score=397.64 epsilon=0.156 error=0.499 iter=670298 psize=200 msize=181 mfrac=0.91\n", + "episodes=186 steps=46902 score=399.85 epsilon=0.154 error=3.647 iter=675298 psize=200 msize=177 mfrac=0.93\n", + "episodes=187 steps=47402 score=404.13 epsilon=0.153 error=0.989 iter=680298 psize=200 msize=168 mfrac=0.82\n", + "episodes=188 steps=47902 score=407.32 epsilon=0.151 error=0.908 iter=685298 psize=200 msize=182 mfrac=0.88\n", + "episodes=189 steps=48402 score=408.42 epsilon=0.150 error=1.761 iter=690298 psize=200 msize=144 mfrac=0.85\n", + "episodes=190 steps=48902 score=410.40 epsilon=0.148 error=0.702 iter=695298 psize=200 msize=173 mfrac=0.76\n", + "episodes=191 steps=49402 score=413.41 epsilon=0.147 error=0.858 iter=700298 psize=200 msize=165 mfrac=0.96\n", + "episodes=192 steps=49902 score=415.60 epsilon=0.145 error=0.666 iter=705298 psize=200 msize=169 mfrac=0.90\n", + "episodes=193 steps=50402 score=418.41 epsilon=0.144 error=0.592 iter=710298 psize=200 msize=157 mfrac=0.89\n", + "episodes=194 steps=50902 score=420.70 epsilon=0.142 error=0.933 iter=715298 psize=200 msize=187 mfrac=0.95\n", + "episodes=195 steps=51402 score=425.20 epsilon=0.141 error=0.254 iter=720298 psize=200 msize=160 mfrac=0.89\n", + "episodes=196 steps=51902 score=429.38 epsilon=0.139 error=0.458 iter=725298 psize=200 msize=191 mfrac=0.92\n", + "episodes=197 steps=52402 score=431.01 epsilon=0.138 error=0.507 iter=730298 psize=200 msize=147 mfrac=0.64\n", + "episodes=198 steps=52902 score=433.17 epsilon=0.137 error=0.248 iter=735298 psize=200 msize=162 mfrac=0.82\n", + "episodes=199 steps=53402 score=436.59 epsilon=0.135 error=0.625 iter=740298 psize=200 msize=177 mfrac=0.76\n", + "episodes=200 steps=53902 score=437.81 epsilon=0.134 error=0.333 iter=745298 psize=200 msize=156 mfrac=0.90\n", + "episodes=201 steps=54402 score=439.95 epsilon=0.133 error=0.212 iter=750298 psize=200 msize=180 mfrac=0.74\n", + "episodes=202 steps=54902 score=441.63 epsilon=0.131 error=22.845 iter=755298 psize=200 msize=154 mfrac=0.82\n", + "episodes=203 steps=55402 score=445.09 epsilon=0.130 error=0.167 iter=760298 psize=200 msize=156 mfrac=0.76\n", + "episodes=204 steps=55902 score=446.82 epsilon=0.129 error=0.055 iter=765298 psize=200 msize=159 mfrac=0.70\n", + "episodes=205 steps=56402 score=448.71 epsilon=0.127 error=0.059 iter=770298 psize=200 msize=140 mfrac=0.81\n", + "episodes=206 steps=56902 score=450.96 epsilon=0.126 error=0.052 iter=775298 psize=200 msize=162 mfrac=0.58\n", + "episodes=207 steps=57402 score=453.29 epsilon=0.125 error=0.033 iter=780298 psize=200 msize=154 mfrac=0.87\n", + "episodes=208 steps=57692 score=451.83 epsilon=0.124 error=0.023 iter=785298 psize=200 msize=154 mfrac=0.84\n", + "episodes=209 steps=57947 score=451.42 epsilon=0.122 error=0.497 iter=790298 psize=200 msize=137 mfrac=0.85\n", + "episodes=210 steps=57959 score=448.31 epsilon=0.121 error=4.841 iter=795298 psize=200 msize=145 mfrac=0.88\n", + "episodes=211 steps=57973 score=445.55 epsilon=0.120 error=0.360 iter=800298 psize=200 msize=131 mfrac=0.80\n", + "episodes=212 steps=58473 score=447.11 epsilon=0.119 error=0.145 iter=805298 psize=200 msize=139 mfrac=0.55\n", + "episodes=213 steps=58973 score=450.20 epsilon=0.118 error=0.078 iter=810298 psize=200 msize=146 mfrac=0.71\n", + "episodes=214 steps=59473 score=451.95 epsilon=0.116 error=0.114 iter=815298 psize=200 msize=142 mfrac=0.48\n", + "episodes=215 steps=59973 score=456.24 epsilon=0.115 error=0.463 iter=820298 psize=200 msize=113 mfrac=0.66\n", + "episodes=216 steps=60473 score=457.50 epsilon=0.114 error=0.263 iter=825298 psize=200 msize=127 mfrac=0.55\n", + "episodes=217 steps=60973 score=457.73 epsilon=0.113 error=0.146 iter=830298 psize=200 msize=119 mfrac=0.71\n", + "episodes=218 steps=61473 score=457.73 epsilon=0.112 error=0.217 iter=835298 psize=200 msize=87 mfrac=0.69\n", + "episodes=219 steps=61973 score=460.28 epsilon=0.111 error=0.094 iter=840298 psize=200 msize=76 mfrac=0.78\n", + "episodes=220 steps=62473 score=461.56 epsilon=0.110 error=0.103 iter=845298 psize=200 msize=55 mfrac=0.72\n", + "episodes=221 steps=62973 score=463.71 epsilon=0.108 error=0.111 iter=850298 psize=200 msize=48 mfrac=0.50\n", + "episodes=222 steps=63473 score=464.70 epsilon=0.107 error=0.180 iter=855298 psize=200 msize=54 mfrac=0.50\n", + "episodes=223 steps=63973 score=466.63 epsilon=0.106 error=0.376 iter=860298 psize=200 msize=50 mfrac=0.48\n", + "episodes=224 steps=64473 score=466.63 epsilon=0.105 error=0.181 iter=865298 psize=200 msize=58 mfrac=0.53\n", + "episodes=225 steps=64973 score=471.27 epsilon=0.104 error=0.137 iter=870298 psize=200 msize=60 mfrac=0.80\n", + "episodes=226 steps=65473 score=471.27 epsilon=0.103 error=0.097 iter=875298 psize=200 msize=67 mfrac=0.64\n", + "episodes=227 steps=65973 score=472.74 epsilon=0.102 error=0.480 iter=880298 psize=200 msize=74 mfrac=0.53\n", + "episodes=228 steps=66473 score=473.56 epsilon=0.101 error=0.227 iter=885298 psize=200 msize=61 mfrac=0.75\n", + "episodes=229 steps=66973 score=474.61 epsilon=0.100 error=0.115 iter=890298 psize=200 msize=79 mfrac=0.39\n", + "episodes=230 steps=67473 score=474.61 epsilon=0.099 error=1.217 iter=895298 psize=200 msize=81 mfrac=0.69\n", + "episodes=231 steps=67973 score=476.46 epsilon=0.098 error=0.140 iter=900298 psize=200 msize=80 mfrac=0.20\n", + "solved after 231 episodes: mean score 476.46 > 475.00\n" ] } ], @@ -474,11 +514,12 @@ " \"\"\"Performs experience replay updates\"\"\"\n", " batch_size: int = min(len(memory), replay_size)\n", " batch = random.sample(memory, batch_size)\n", - " for state, action, reward, next_state, done in batch:\n", + " for state, action, reward, next_state, terminated in batch:\n", " y_target = reward\n", - " if not done:\n", + " if not terminated:\n", " prediction_array = xcs.predict(next_state.reshape(1, -1))[0]\n", " y_target += GAMMA * np.max(prediction_array)\n", + "\n", " target = xcs.predict(state.reshape(1, -1))[0]\n", " target[action] = y_target\n", " xcs.fit(\n", @@ -504,15 +545,14 @@ " while True:\n", " action = egreedy_action(state)\n", " next_state, reward, terminated, truncated, _ = env.step(action)\n", - " done = terminated or truncated\n", " episode_steps += 1\n", " episode_score += reward\n", - " memory.append((state, action, reward, next_state, done))\n", + " memory.append((state, action, reward, next_state, terminated))\n", " if create_gif:\n", " frames.append(env.render())\n", " fscore.append(episode_score)\n", " ftrial.append(episode_nr)\n", - " if done:\n", + " if terminated or truncated:\n", " if create_gif:\n", " for _ in range(100):\n", " frames.append(frames[-1])\n", @@ -541,9 +581,12 @@ " f\"episodes={ep} \"\n", " f\"steps={total_steps} \"\n", " f\"score={mean_score:.2f} \"\n", - " f\"epsilon={epsilon:.5f} \"\n", - " f\"error={xcs.error():.5f} \"\n", - " f\"msize={xcs.mset_size():.2f}\"\n", + " f\"epsilon={epsilon:.3f} \"\n", + " f\"error={xcs.error():.3f} \"\n", + " f\"iter={xcs.time():.0f} \"\n", + " f\"psize={xcs.pset_size():.0f} \"\n", + " f\"msize={xcs.mset_size():.0f} \"\n", + " f\"mfrac={xcs.mfrac():.2f}\"\n", " )\n", " # is the problem solved?\n", " if ep > N and mean_score > env.spec.reward_threshold:\n", @@ -603,7 +646,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2221/2221 [01:05<00:00, 34.01it/s]\n" + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2583/2583 [00:34<00:00, 75.19it/s]\n" ] } ], @@ -683,7 +726,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.10.19" } }, "nbformat": 4, diff --git a/python/notebooks/out.gif b/python/notebooks/out.gif index fd032c533..aed38d570 100644 Binary files a/python/notebooks/out.gif and b/python/notebooks/out.gif differ