MAML_with_evaluations.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "583e5492",
   "metadata": {},
   "outputs": [],
   "source": [
    "import carla\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.distributions import Normal\n",
    "import learn2learn as l2l\n",
    "import random\n",
    "import csv\n",
    "import time\n",
    "\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "class NeuralNetwork(nn.Module):\n",
    "    def __init__(self, state_dim, action_dim):\n",
    "        super(NeuralNetwork, self).__init__()\n",
    "\n",
    "        self.l1 = nn.Linear(state_dim, 64)\n",
    "        self.l2 = nn.Linear(64, 64)\n",
    "        self.l3 = nn.Linear(64, action_dim)\n",
    "\n",
    "    def forward(self, state):\n",
    "        x = F.relu(self.l1(state))\n",
    "        x = F.relu(self.l2(x))\n",
    "        #action_logits = self.l3(x)\n",
    "        actions = self.l3(x)\n",
    "        #ction_probs = F.softmax(action_logits, dim=1)\n",
    "\n",
    "        return actions\n",
    "\n",
    "\n",
    "class CarlaEnvironment:\n",
    "    def __init__(self, world, vehicle_blueprint, waypoints):\n",
    "        self.world = world\n",
    "        self.vehicle_blueprint = vehicle_blueprint\n",
    "        self.waypoints = waypoints\n",
    "        self.actorlist = []\n",
    "        self.vehicle = self.world.spawn_actor(self.vehicle_blueprint, carla.Transform(carla.Location(x=-23.6,y=137.5,z=1),carla.Rotation(yaw=0)))\n",
    "        \n",
    "    def reset(self):\n",
    "        self.vehicle.destroy()\n",
    "        self.vehicle = self.world.spawn_actor(self.vehicle_blueprint, carla.Transform(carla.Location(x=-23.6,y=137.5,z=1),carla.Rotation(yaw=0)))  \n",
    "        self.current_waypoint = 0\n",
    "        return self.get_state()\n",
    "\n",
    "    def step(self, action):\n",
    "        action = action.detach().numpy()\n",
    "        #print(action)\n",
    "        # Apply actions: Throttle, Steering, Brake\n",
    "        self.vehicle.apply_control(carla.VehicleControl(throttle = float(action[0]) , steer = float(action[1]) , brake = float(action[2])))\n",
    "       \n",
    "        #self.actorlist.append(self.vehicle)\n",
    "\n",
    "        # Get next state\n",
    "        next_state = self.get_state()\n",
    "\n",
    "        # Calculate reward\n",
    "        reward = self.calculate_reward(next_state) \n",
    "\n",
    "        # Check if the episode is done\n",
    "        done = self.is_done(next_state)\n",
    "\n",
    "        return next_state, reward, done\n",
    "\n",
    "#     def get_state(self):\n",
    "#         curr_vel = self.vehicle.get_velocity()\n",
    "#         curr_yaw = self.vehicle.get_transform().rotation.yaw\n",
    "#         waypoint = self.waypoints[self.current_waypoint]\n",
    "#         return np.array([curr_vel, curr_yaw - waypoint[1], curr_yaw - waypoint[2], waypoint[3]])\n",
    "    \n",
    "    def get_state(self):\n",
    "        state = []\n",
    "        curr_yaw = self.vehicle.get_transform().rotation.yaw\n",
    "        \n",
    "        \n",
    "        for i in range(10):\n",
    "            if self.current_waypoint + i < len(self.waypoints):\n",
    "                waypoint = self.waypoints[self.current_waypoint + i]\n",
    "                curr_vel = self.vehicle.get_velocity()\n",
    "                curr_loc = self.vehicle.get_location()\n",
    "                vx = curr_vel.x\n",
    "                vy = curr_vel.y\n",
    "                v = ((vx)**2+(vy)**2)**0.5\n",
    "                distance = curr_loc.distance(carla.Location(x=waypoint[4],y=waypoint[5]))\n",
    "                state.extend([v, curr_yaw - waypoint[1], curr_yaw - waypoint[2], distance ])\n",
    "            else:\n",
    "                state.extend([0, 0, 0, 0])\n",
    "        return np.array(state)\n",
    "\n",
    "    def generate_waypoints(self):\n",
    "        # Implement waypoint generation logic\n",
    "        pass\n",
    "\n",
    "    def calculate_reward(self, state):\n",
    "        reward = -(np.linalg.norm(state[0] - self.waypoints[self.current_waypoint][0]) - (np.linalg.norm (state[1])) - (np.linalg.norm ( state[2])) - (np.linalg.norm (state[3])) + self.current_waypoint)\n",
    "        return reward\n",
    "\n",
    "    def is_done(self, state):\n",
    "        if np.abs(state[3]) < 0.20:  # If the vehicle is close to the next waypoint\n",
    "            self.current_waypoint += 1\n",
    "\n",
    "        if self.current_waypoint >= len(self.waypoints):\n",
    "            return True\n",
    "\n",
    "        return False\n",
    "\n",
    "    def set_weather(self, weather):\n",
    "        self.world.set_weather(weather)\n",
    "        \n",
    "    def destroy(self):\n",
    "        for actor in self.actorlist:\n",
    "            actor.destroy()\n",
    "        print(\"All Cleared\")\n",
    "\n",
    "def set_conditions(env, condition):\n",
    "    \n",
    "        precipitation = condition[0]\n",
    "        precipitation_deposits = condition[1]\n",
    "        tire_friction = condition[2]\n",
    "        weather = carla.WeatherParameters( precipitation = precipitation,\n",
    "                                            precipitation_deposits = precipitation_deposits)\n",
    "        world.set_weather(weather)\n",
    "\n",
    "        tire_condition = carla.WheelPhysicsControl(tire_friction = tire_friction)\n",
    "\n",
    "        front_left_wheel  = carla.WheelPhysicsControl(tire_friction = tire_friction)\n",
    "        front_right_wheel = carla.WheelPhysicsControl(tire_friction = tire_friction)\n",
    "        rear_left_wheel   = carla.WheelPhysicsControl(tire_friction = tire_friction)\n",
    "        rear_right_wheel  = carla.WheelPhysicsControl(tire_friction = tire_friction)\n",
    "\n",
    "        wheels = [front_left_wheel, front_right_wheel, rear_left_wheel, rear_right_wheel]\n",
    "        \n",
    "        physics_control = env.vehicle.get_physics_control()\n",
    "        physics_control.wheels = wheels\n",
    "        env.vehicle.apply_physics_control(physics_control)\n",
    "    \n",
    "\n",
    "def train_ppo_maml(env, state_dim, action_dim, lr=1e-4, n_iters=1000000, inner_loop_steps=5):\n",
    "    \n",
    " \n",
    "    p_low = 0\n",
    "    p_mid = 50\n",
    "    p_high = 100\n",
    "\n",
    "    pd_low = 50\n",
    "    pd_high = 100\n",
    "\n",
    "    tf_low = 0.5\n",
    "    tf_mid = 1.5\n",
    "    tf_high = 2.5\n",
    "    \n",
    "    conditions = [[p_low, pd_low, tf_high],\n",
    "            [p_low, pd_low, tf_mid],\n",
    "            [p_low, pd_high, tf_low],\n",
    "            [p_mid, pd_low, tf_low],\n",
    "            [p_mid, pd_high, tf_low],\n",
    "            [p_mid, pd_low, tf_mid],\n",
    "            [p_high, pd_low, tf_high],\n",
    "            [p_high, pd_high, tf_low],\n",
    "            [p_low, pd_high, tf_mid],\n",
    "            [p_high, pd_low, tf_mid]]\n",
    "    \n",
    "    n_tasks = len(conditions)\n",
    "\n",
    "    \n",
    "    base_model = NeuralNetwork(state_dim, action_dim)\n",
    "    maml = l2l.algorithms.MAML(base_model, lr = lr)\n",
    "    optimizer = optim.Adam(maml.parameters(), lr = lr)\n",
    "    iteration = 1\n",
    "    for condition in conditions:\n",
    "        \n",
    "        set_conditions(env,condition)\n",
    "    \n",
    "        trajectories, state_tensors, action_tensors = sample_trajectories(env, maml)\n",
    "        state_tensors_stacked = torch.stack(state_tensors)\n",
    "        action_tensors_stacked = torch.stack(action_tensors)\n",
    "        \n",
    "        for _ in range(inner_loop_steps):\n",
    "        \n",
    "            learner = maml.clone()\n",
    "            error = nn.MSELoss()(learner(state_tensors_stacked), action_tensors_stacked)\n",
    "            learner.adapt(error)\n",
    "            error = nn.MSELoss()(learner(state_tensors_stacked), action_tensors_stacked)\n",
    "            #error.backward(retain_graph=True)\n",
    "\n",
    "        # Evaluating the model\n",
    "        p_evaluate = random.uniform(p_low, p_high)\n",
    "        pd_evaluate = random.uniform(pd_low, pd_high)\n",
    "        tf_evaluate = random.uniform(tf_low, tf_high)\n",
    "        evaluating_condition = [p_evaluate, pd_evaluate, tf_evaluate]\n",
    "        set_conditions(env, evaluating_condition)\n",
    "        \n",
    "        trajectories, state_tensors, action_tensors = sample_trajectories(env, maml)\n",
    "        state_tensors_stacked = torch.stack(state_tensors)\n",
    "        action_tensors_stacked = torch.stack(action_tensors)\n",
    "        error = nn.MSELoss()(learner(state_tensors_stacked), action_tensors_stacked)\n",
    "        error.backward(retain_graph=True)\n",
    "        \n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        optimizer.step()\n",
    "        \n",
    "        iteration += 1\n",
    "        print(\"Iteration for {condition}: \", iteration)\n",
    "\n",
    "        if iteration % 1000 == 0:\n",
    "            print(\"Iteration for {condition}: \", iteration)\n",
    "\n",
    "\n",
    "def sample_trajectories(env, model, num_episodes=10, max_episode_length=1000):\n",
    "#     model.to(\"cpu\")\n",
    "    trajectories = []\n",
    "    traj_states = []\n",
    "    traj_action = []\n",
    "    state_tensors = []\n",
    "    action_tensors = []\n",
    "       \n",
    "    for _ in range(num_episodes):\n",
    "        state = env.reset()                \n",
    "        trajectory = []\n",
    "\n",
    "        for j in range(max_episode_length):\n",
    "            \n",
    "#             state_tensor = torch.tensor(state).to(torch.float32)\n",
    "            state_tensor = torch.from_numpy(state.astype(np.float32))\n",
    "            state_tensors.append(state_tensor)\n",
    "            action_tensor = model(state_tensor)\n",
    "#             action_distribution = Normal(action_mean, torch.tensor(0.1))\n",
    "#             action = action_distribution.sample().detach().numpy().astype(float)\n",
    "            \n",
    "            action = action_tensor\n",
    "#             action_tensor = torch.tensor(action, dtype=torch.float32)\n",
    "            action_tensors.append(action)\n",
    "\n",
    "            next_state, reward, done = env.step(action)\n",
    "            trajectory.append((state, action, reward, next_state, done))\n",
    "\n",
    "            state = next_state\n",
    "            \n",
    "            if not j%200 : print(\"Reward: \", reward, '  J: ', j )\n",
    "            #state_tensors = torch.tensor(state_tensor, dtype=torch.float32)\n",
    "            if done or j==max_episode_length-1:\n",
    "                #env.vehicle.destroy()\n",
    "                break  \n",
    "\n",
    "        trajectories.append(trajectory)\n",
    "    return trajectories, state_tensors, action_tensors\n",
    "\n",
    "            \n",
    "def read_waypoints(file_path):\n",
    "    waypoints = []\n",
    "    with open(file_path, 'r') as csvfile:\n",
    "        csvreader = csv.reader(csvfile)\n",
    "        for row in csvreader:\n",
    "            v, yaw_c, yaw_n, r, x, y = map(float, row)\n",
    "            waypoints.append((v, yaw_c, yaw_n, r, x, y))\n",
    "            \n",
    "    return waypoints                        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0225a0b3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Version mismatch detected: You are trying to connect to a simulator that might be incompatible with this API \n",
      "WARNING: Client API version     = 0.9.14 \n",
      "WARNING: Simulator API version  = 0.9.14-4-gf14acb257 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.6699888110160828   J:  200\n",
      "Reward:  1.4970015287399292   J:  400\n",
      "Reward:  1.3412280082702637   J:  600\n",
      "Reward:  1.2657466530799866   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7433509826660156   J:  200\n",
      "Reward:  1.7132707834243774   J:  400\n",
      "Reward:  1.6200791597366333   J:  600\n",
      "Reward:  1.503084659576416   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7746806144714355   J:  200\n",
      "Reward:  1.7186057567596436   J:  400\n",
      "Reward:  1.621147871017456   J:  600\n",
      "Reward:  1.55790776014328   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7584391832351685   J:  200\n",
      "Reward:  1.7279086112976074   J:  400\n",
      "Reward:  1.632849097251892   J:  600\n",
      "Reward:  1.5005521178245544   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7436023950576782   J:  200\n",
      "Reward:  1.712868332862854   J:  400\n",
      "Reward:  1.6231322884559631   J:  600\n",
      "Reward:  1.4943248629570007   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7698469161987305   J:  200\n",
      "Reward:  1.7398407459259033   J:  400\n",
      "Reward:  1.7398407459259033   J:  600\n",
      "Reward:  1.5989875197410583   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7685843706130981   J:  200\n",
      "Reward:  1.6896512508392334   J:  400\n",
      "Reward:  1.5724045634269714   J:  600\n",
      "Reward:  1.442789912223816   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7734076976776123   J:  200\n",
      "Reward:  1.717820405960083   J:  400\n",
      "Reward:  1.6357215642929077   J:  600\n",
      "Reward:  1.5196374654769897   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7522213459014893   J:  200\n",
      "Reward:  1.6843091249465942   J:  400\n",
      "Reward:  1.587639570236206   J:  600\n",
      "Reward:  1.4694691896438599   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.773901343345642   J:  200\n",
      "Reward:  1.7215008735656738   J:  400\n",
      "Reward:  1.6814060807228088   J:  600\n",
      "Reward:  1.5832695960998535   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7758636474609375   J:  200\n",
      "Reward:  1.720279574394226   J:  400\n",
      "Reward:  1.6815918684005737   J:  600\n",
      "Reward:  1.5833873748779297   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7192012071609497   J:  200\n",
      "Reward:  1.6393504738807678   J:  400\n",
      "Reward:  1.5906885266304016   J:  600\n",
      "Reward:  1.468087375164032   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7498562335968018   J:  200\n",
      "Reward:  1.6877785921096802   J:  400\n",
      "Reward:  1.6433070302009583   J:  600\n",
      "Reward:  1.5331965684890747   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.756528377532959   J:  200\n",
      "Reward:  1.7209995985031128   J:  400\n",
      "Reward:  1.6432120203971863   J:  600\n",
      "Reward:  1.5355618596076965   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7759549617767334   J:  200\n",
      "Reward:  1.7259057760238647   J:  400\n",
      "Reward:  1.6840898394584656   J:  600\n",
      "Reward:  1.5869376063346863   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7747656106948853   J:  200\n",
      "Reward:  1.7213531732559204   J:  400\n",
      "Reward:  1.6388417482376099   J:  600\n",
      "Reward:  1.5249699354171753   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.77457857131958   J:  200\n",
      "Reward:  1.7210049629211426   J:  400\n",
      "Reward:  1.63694828748703   J:  600\n",
      "Reward:  1.5864851474761963   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7716810703277588   J:  200\n",
      "Reward:  1.7204710245132446   J:  400\n",
      "Reward:  1.6429210305213928   J:  600\n",
      "Reward:  1.5295193195343018   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.774780035018921   J:  200\n",
      "Reward:  1.724308729171753   J:  400\n",
      "Reward:  1.643919050693512   J:  600\n",
      "Reward:  1.5290435552597046   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7767137289047241   J:  200\n",
      "Reward:  1.7276536226272583   J:  400\n",
      "Reward:  1.644437313079834   J:  600\n",
      "Reward:  1.5305126309394836   J:  800\n",
      "Iteration for {condition}:  2\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.774747371673584   J:  200\n",
      "Reward:  1.7498644590377808   J:  400\n",
      "Reward:  1.6178264617919922   J:  600\n",
      "Reward:  1.6178264617919922   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7897270917892456   J:  200\n",
      "Reward:  1.7897270917892456   J:  400\n",
      "Reward:  1.7159448862075806   J:  600\n",
      "Reward:  1.7159448862075806   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7736201286315918   J:  200\n",
      "Reward:  1.6982236504554749   J:  400\n",
      "Reward:  1.596974492073059   J:  600\n",
      "Reward:  1.4666244387626648   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7827645540237427   J:  200\n",
      "Reward:  1.7827645540237427   J:  400\n",
      "Reward:  1.7827645540237427   J:  600\n",
      "Reward:  1.6921302676200867   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7706269025802612   J:  200\n",
      "Reward:  1.6211695075035095   J:  400\n",
      "Reward:  1.4399148225784302   J:  600\n",
      "Reward:  1.3075189590454102   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7602155208587646   J:  200\n",
      "Reward:  1.6387732028961182   J:  400\n",
      "Reward:  1.4910160303115845   J:  600\n",
      "Reward:  1.39655601978302   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7014038562774658   J:  200\n",
      "Reward:  1.6057470440864563   J:  400\n",
      "Reward:  1.4923489689826965   J:  600\n",
      "Reward:  1.426828682422638   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7533429861068726   J:  200\n",
      "Reward:  1.7241895198822021   J:  400\n",
      "Reward:  1.6400538086891174   J:  600\n",
      "Reward:  1.5884808897972107   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7556633949279785   J:  200\n",
      "Reward:  1.7246317863464355   J:  400\n",
      "Reward:  1.6530997157096863   J:  600\n",
      "Reward:  1.541484773159027   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7752282619476318   J:  200\n",
      "Reward:  1.7252886295318604   J:  400\n",
      "Reward:  1.6838680505752563   J:  600\n",
      "Reward:  1.5920982956886292   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.771416425704956   J:  200\n",
      "Reward:  1.7127811908721924   J:  400\n",
      "Reward:  1.672492504119873   J:  600\n",
      "Reward:  1.5625573992729187   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7742027044296265   J:  200\n",
      "Reward:  1.7192983627319336   J:  400\n",
      "Reward:  1.6763370037078857   J:  600\n",
      "Reward:  1.570890724658966   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7725448608398438   J:  200\n",
      "Reward:  1.7149418592453003   J:  400\n",
      "Reward:  1.6710458397865295   J:  600\n",
      "Reward:  1.5712074041366577   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7740793228149414   J:  200\n",
      "Reward:  1.7165507078170776   J:  400\n",
      "Reward:  1.6727752089500427   J:  600\n",
      "Reward:  1.6727752089500427   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7451032400131226   J:  200\n",
      "Reward:  1.6462063193321228   J:  400\n",
      "Reward:  1.6462063193321228   J:  600\n",
      "Reward:  1.5166344046592712   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7826366424560547   J:  200\n",
      "Reward:  1.7369123697280884   J:  400\n",
      "Reward:  1.6992815732955933   J:  600\n",
      "Reward:  1.6560617685317993   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7736222743988037   J:  200\n",
      "Reward:  1.7179666757583618   J:  400\n",
      "Reward:  1.6747925877571106   J:  600\n",
      "Reward:  1.5679619312286377   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7739027738571167   J:  200\n",
      "Reward:  1.7155570983886719   J:  400\n",
      "Reward:  1.6286952495574951   J:  600\n",
      "Reward:  1.568395733833313   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.774073600769043   J:  200\n",
      "Reward:  1.7159911394119263   J:  400\n",
      "Reward:  1.62469881772995   J:  600\n",
      "Reward:  1.4959377646446228   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7724957466125488   J:  200\n",
      "Reward:  1.715128779411316   J:  400\n",
      "Reward:  1.620128870010376   J:  600\n",
      "Reward:  1.5659045577049255   J:  800\n",
      "Iteration for {condition}:  3\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7746670246124268   J:  200\n",
      "Reward:  1.724421739578247   J:  400\n",
      "Reward:  1.6356545686721802   J:  600\n",
      "Reward:  1.524212121963501   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7726565599441528   J:  200\n",
      "Reward:  1.7470067739486694   J:  400\n",
      "Reward:  1.6731230020523071   J:  600\n",
      "Reward:  1.5692150592803955   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7736029624938965   J:  200\n",
      "Reward:  1.717355489730835   J:  400\n",
      "Reward:  1.6771443486213684   J:  600\n",
      "Reward:  1.567480206489563   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.743201494216919   J:  200\n",
      "Reward:  1.6719511151313782   J:  400\n",
      "Reward:  1.627178966999054   J:  600\n",
      "Reward:  1.5048144459724426   J:  800\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7137510776519775   J:  200\n",
      "Reward:  1.6254312992095947   J:  400\n",
      "Reward:  1.5030986070632935   J:  600\n",
      "Reward:  1.4329890608787537   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7738171815872192   J:  200\n",
      "Reward:  1.7494022846221924   J:  400\n",
      "Reward:  1.6748443245887756   J:  600\n",
      "Reward:  1.5742568373680115   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7720532417297363   J:  200\n",
      "Reward:  1.7178140878677368   J:  400\n",
      "Reward:  1.624607503414154   J:  600\n",
      "Reward:  1.5055365562438965   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7746049165725708   J:  200\n",
      "Reward:  1.7177900075912476   J:  400\n",
      "Reward:  1.6751168966293335   J:  600\n",
      "Reward:  1.5675711035728455   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7727339267730713   J:  200\n",
      "Reward:  1.7474547624588013   J:  400\n",
      "Reward:  1.6740996837615967   J:  600\n",
      "Reward:  1.6240941286087036   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7748045921325684   J:  200\n",
      "Reward:  1.7156554460525513   J:  400\n",
      "Reward:  1.6742994785308838   J:  600\n",
      "Reward:  1.5682203769683838   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7728463411331177   J:  200\n",
      "Reward:  1.7172561883926392   J:  400\n",
      "Reward:  1.6766655445098877   J:  600\n",
      "Reward:  1.569462537765503   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7716392278671265   J:  200\n",
      "Reward:  1.7478092908859253   J:  400\n",
      "Reward:  1.6716598868370056   J:  600\n",
      "Reward:  1.6236598491668701   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.750116229057312   J:  200\n",
      "Reward:  1.6653599739074707   J:  400\n",
      "Reward:  1.5716196298599243   J:  600\n",
      "Reward:  1.5083168745040894   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7477655410766602   J:  200\n",
      "Reward:  1.7172919511795044   J:  400\n",
      "Reward:  1.627497375011444   J:  600\n",
      "Reward:  1.5656529068946838   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7737441062927246   J:  200\n",
      "Reward:  1.714342474937439   J:  400\n",
      "Reward:  1.62722247838974   J:  600\n",
      "Reward:  1.5074427127838135   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7728875875473022   J:  200\n",
      "Reward:  1.7453259229660034   J:  400\n",
      "Reward:  1.6704789400100708   J:  600\n",
      "Reward:  1.6203216910362244   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7728480100631714   J:  200\n",
      "Reward:  1.715985894203186   J:  400\n",
      "Reward:  1.6693968176841736   J:  600\n",
      "Reward:  1.5636793971061707   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7728019952774048   J:  200\n",
      "Reward:  1.746880292892456   J:  400\n",
      "Reward:  1.6698569655418396   J:  600\n",
      "Reward:  1.5635422468185425   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7733824253082275   J:  200\n",
      "Reward:  1.719972848892212   J:  400\n",
      "Reward:  1.6797675490379333   J:  600\n",
      "Reward:  1.5688470602035522   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7734785079956055   J:  200\n",
      "Reward:  1.7143609523773193   J:  400\n",
      "Reward:  1.6763333082199097   J:  600\n",
      "Reward:  1.624383270740509   J:  800\n",
      "Iteration for {condition}:  4\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7746163606643677   J:  200\n",
      "Reward:  1.7187920808792114   J:  400\n",
      "Reward:  1.6235990524291992   J:  600\n",
      "Reward:  1.5713208317756653   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7739776372909546   J:  200\n",
      "Reward:  1.7174592018127441   J:  400\n",
      "Reward:  1.6744133234024048   J:  600\n",
      "Reward:  1.5675483345985413   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.772446870803833   J:  200\n",
      "Reward:  1.7178843021392822   J:  400\n",
      "Reward:  1.6194382309913635   J:  600\n",
      "Reward:  1.5627943277359009   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7717071771621704   J:  200\n",
      "Reward:  1.7453430891036987   J:  400\n",
      "Reward:  1.7453430891036987   J:  600\n",
      "Reward:  1.6090389490127563   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7669224739074707   J:  200\n",
      "Reward:  1.6796804666519165   J:  400\n",
      "Reward:  1.6796804666519165   J:  600\n",
      "Reward:  1.5613808035850525   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7590235471725464   J:  200\n",
      "Reward:  1.667548954486847   J:  400\n",
      "Reward:  1.5230932235717773   J:  600\n",
      "Reward:  1.428680419921875   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7646247148513794   J:  200\n",
      "Reward:  1.7252886295318604   J:  400\n",
      "Reward:  1.676975667476654   J:  600\n",
      "Reward:  1.5858457684516907   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7679897546768188   J:  200\n",
      "Reward:  1.7278491258621216   J:  400\n",
      "Reward:  1.6719226241111755   J:  600\n",
      "Reward:  1.5888761281967163   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7630257606506348   J:  200\n",
      "Reward:  1.738104224205017   J:  400\n",
      "Reward:  1.6796370148658752   J:  600\n",
      "Reward:  1.5927461385726929   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7759567499160767   J:  200\n",
      "Reward:  1.7301405668258667   J:  400\n",
      "Reward:  1.6981056928634644   J:  600\n",
      "Reward:  1.6112886667251587   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7764711380004883   J:  200\n",
      "Reward:  1.7326514720916748   J:  400\n",
      "Reward:  1.6578644514083862   J:  600\n",
      "Reward:  1.612501323223114   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.776155710220337   J:  200\n",
      "Reward:  1.7297804355621338   J:  400\n",
      "Reward:  1.6996917128562927   J:  600\n",
      "Reward:  1.5967691540718079   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7759541273117065   J:  200\n",
      "Reward:  1.7285364866256714   J:  400\n",
      "Reward:  1.6930816173553467   J:  600\n",
      "Reward:  1.5953229665756226   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.775282621383667   J:  200\n",
      "Reward:  1.730087161064148   J:  400\n",
      "Reward:  1.6499003767967224   J:  600\n",
      "Reward:  1.5992330312728882   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7738653421401978   J:  200\n",
      "Reward:  1.7233940362930298   J:  400\n",
      "Reward:  1.6911853551864624   J:  600\n",
      "Reward:  1.5955602526664734   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7451032400131226   J:  200\n",
      "Reward:  1.6092488765716553   J:  400\n",
      "Reward:  1.4161843061447144   J:  600\n",
      "Reward:  1.2913421392440796   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.775782823562622   J:  200\n",
      "Reward:  1.7233842611312866   J:  400\n",
      "Reward:  1.6890424489974976   J:  600\n",
      "Reward:  1.594660460948944   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7213785648345947   J:  200\n",
      "Reward:  1.6453964710235596   J:  400\n",
      "Reward:  1.5375048518180847   J:  600\n",
      "Reward:  1.4114932417869568   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7741930484771729   J:  200\n",
      "Reward:  1.7274256944656372   J:  400\n",
      "Reward:  1.6504395008087158   J:  600\n",
      "Reward:  1.5943078398704529   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7749049663543701   J:  200\n",
      "Reward:  1.7280285358428955   J:  400\n",
      "Reward:  1.6894423365592957   J:  600\n",
      "Reward:  1.5932039618492126   J:  800\n",
      "Iteration for {condition}:  5\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7770452499389648   J:  200\n",
      "Reward:  1.7246371507644653   J:  400\n",
      "Reward:  1.6430388689041138   J:  600\n",
      "Reward:  1.599306881427765   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.777335286140442   J:  200\n",
      "Reward:  1.7289369106292725   J:  400\n",
      "Reward:  1.6941568851470947   J:  600\n",
      "Reward:  1.6018509268760681   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7762582302093506   J:  200\n",
      "Reward:  1.722389578819275   J:  400\n",
      "Reward:  1.6854106783866882   J:  600\n",
      "Reward:  1.5935847759246826   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7577792406082153   J:  200\n",
      "Reward:  1.7285513877868652   J:  400\n",
      "Reward:  1.6523362398147583   J:  600\n",
      "Reward:  1.5443361401557922   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.775475263595581   J:  200\n",
      "Reward:  1.7547794580459595   J:  400\n",
      "Reward:  1.6820258498191833   J:  600\n",
      "Reward:  1.5932162404060364   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7759289741516113   J:  200\n",
      "Reward:  1.7523928880691528   J:  400\n",
      "Reward:  1.6856648325920105   J:  600\n",
      "Reward:  1.5923789143562317   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7741570472717285   J:  200\n",
      "Reward:  1.724097728729248   J:  400\n",
      "Reward:  1.6838906407356262   J:  600\n",
      "Reward:  1.5901918411254883   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7774975299835205   J:  200\n",
      "Reward:  1.730824589729309   J:  400\n",
      "Reward:  1.695283830165863   J:  600\n",
      "Reward:  1.6036130785942078   J:  800\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7758145332336426   J:  200\n",
      "Reward:  1.7250967025756836   J:  400\n",
      "Reward:  1.6884304881095886   J:  600\n",
      "Reward:  1.5934617519378662   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7755264043807983   J:  200\n",
      "Reward:  1.7274352312088013   J:  400\n",
      "Reward:  1.6864164471626282   J:  600\n",
      "Reward:  1.5869842171669006   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7475073337554932   J:  200\n",
      "Reward:  1.6943602561950684   J:  400\n",
      "Reward:  1.6024456024169922   J:  600\n",
      "Reward:  1.5520419478416443   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7764499187469482   J:  200\n",
      "Reward:  1.7285726070404053   J:  400\n",
      "Reward:  1.6922542452812195   J:  600\n",
      "Reward:  1.602317988872528   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7734977006912231   J:  200\n",
      "Reward:  1.72649085521698   J:  400\n",
      "Reward:  1.6470374464988708   J:  600\n",
      "Reward:  1.5431002378463745   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7775896787643433   J:  200\n",
      "Reward:  1.7302837371826172   J:  400\n",
      "Reward:  1.6951731443405151   J:  600\n",
      "Reward:  1.5974628329277039   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7755205631256104   J:  200\n",
      "Reward:  1.7256590127944946   J:  400\n",
      "Reward:  1.6893272399902344   J:  600\n",
      "Reward:  1.5982017517089844   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.777557611465454   J:  200\n",
      "Reward:  1.7265249490737915   J:  400\n",
      "Reward:  1.6568602919578552   J:  600\n",
      "Reward:  1.600528061389923   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7563402652740479   J:  200\n",
      "Reward:  1.691723346710205   J:  400\n",
      "Reward:  1.6014899611473083   J:  600\n",
      "Reward:  1.484407663345337   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.777026891708374   J:  200\n",
      "Reward:  1.7271214723587036   J:  400\n",
      "Reward:  1.6489726305007935   J:  600\n",
      "Reward:  1.601605772972107   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7739022970199585   J:  200\n",
      "Reward:  1.726933479309082   J:  400\n",
      "Reward:  1.6902093887329102   J:  600\n",
      "Reward:  1.596009612083435   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7766999006271362   J:  200\n",
      "Reward:  1.7283554077148438   J:  400\n",
      "Reward:  1.6909635066986084   J:  600\n",
      "Reward:  1.5912764072418213   J:  800\n",
      "Iteration for {condition}:  6\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7778295278549194   J:  200\n",
      "Reward:  1.7316943407058716   J:  400\n",
      "Reward:  1.652942955493927   J:  600\n",
      "Reward:  1.5522042512893677   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7753593921661377   J:  200\n",
      "Reward:  1.7284635305404663   J:  400\n",
      "Reward:  1.691049039363861   J:  600\n",
      "Reward:  1.6015160083770752   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7748725414276123   J:  200\n",
      "Reward:  1.7278083562850952   J:  400\n",
      "Reward:  1.6921666264533997   J:  600\n",
      "Reward:  1.5930520296096802   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.775587558746338   J:  200\n",
      "Reward:  1.750799298286438   J:  400\n",
      "Reward:  1.6876086592674255   J:  600\n",
      "Reward:  1.5962844491004944   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7760175466537476   J:  200\n",
      "Reward:  1.726987600326538   J:  400\n",
      "Reward:  1.6478796005249023   J:  600\n",
      "Reward:  1.5446126461029053   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7758700847625732   J:  200\n",
      "Reward:  1.7289122343063354   J:  400\n",
      "Reward:  1.654341459274292   J:  600\n",
      "Reward:  1.6042388081550598   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7744077444076538   J:  200\n",
      "Reward:  1.7463120222091675   J:  400\n",
      "Reward:  1.6796941757202148   J:  600\n",
      "Reward:  1.5811611413955688   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7553635835647583   J:  200\n",
      "Reward:  1.7289611101150513   J:  400\n",
      "Reward:  1.647001326084137   J:  600\n",
      "Reward:  1.5414116978645325   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7749660015106201   J:  200\n",
      "Reward:  1.7538695335388184   J:  400\n",
      "Reward:  1.7538695335388184   J:  600\n",
      "Reward:  1.5685368776321411   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7777994871139526   J:  200\n",
      "Reward:  1.7274678945541382   J:  400\n",
      "Reward:  1.689811646938324   J:  600\n",
      "Reward:  1.5963167548179626   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7503550052642822   J:  200\n",
      "Reward:  1.3837213516235352   J:  400\n",
      "Reward:  1.2485089302062988   J:  600\n",
      "Reward:  1.148986510756703   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7717453241348267   J:  200\n",
      "Reward:  1.7191979885101318   J:  400\n",
      "Reward:  1.6795055866241455   J:  600\n",
      "Reward:  1.5672892332077026   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.769590139389038   J:  200\n",
      "Reward:  1.7009342312812805   J:  400\n",
      "Reward:  1.654441237449646   J:  600\n",
      "Reward:  1.5940918326377869   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.767305850982666   J:  200\n",
      "Reward:  1.7405335903167725   J:  400\n",
      "Reward:  1.69858980178833   J:  600\n",
      "Reward:  1.5921549797058105   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7707988023757935   J:  200\n",
      "Reward:  1.7057374715805054   J:  400\n",
      "Reward:  1.656167984008789   J:  600\n",
      "Reward:  1.5954258441925049   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.682644546031952   J:  200\n",
      "Reward:  1.5667479634284973   J:  400\n",
      "Reward:  1.5021631121635437   J:  600\n",
      "Reward:  1.3502035737037659   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7734591960906982   J:  200\n",
      "Reward:  1.7097374200820923   J:  400\n",
      "Reward:  1.6697955131530762   J:  600\n",
      "Reward:  1.612912356853485   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7683581113815308   J:  200\n",
      "Reward:  1.7402032613754272   J:  400\n",
      "Reward:  1.659741759300232   J:  600\n",
      "Reward:  1.6004194021224976   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.768507480621338   J:  200\n",
      "Reward:  1.7042392492294312   J:  400\n",
      "Reward:  1.657054841518402   J:  600\n",
      "Reward:  1.531071960926056   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7714667320251465   J:  200\n",
      "Reward:  1.7466274499893188   J:  400\n",
      "Reward:  1.6664463877677917   J:  600\n",
      "Reward:  1.6079742908477783   J:  800\n",
      "Iteration for {condition}:  7\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7725270986557007   J:  200\n",
      "Reward:  1.707453727722168   J:  400\n",
      "Reward:  1.610148847103119   J:  600\n",
      "Reward:  1.5432765483856201   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.770255446434021   J:  200\n",
      "Reward:  1.7407281398773193   J:  400\n",
      "Reward:  1.659939467906952   J:  600\n",
      "Reward:  1.5991476774215698   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7705119848251343   J:  200\n",
      "Reward:  1.7435036897659302   J:  400\n",
      "Reward:  1.655918300151825   J:  600\n",
      "Reward:  1.5984576344490051   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7696913480758667   J:  200\n",
      "Reward:  1.7427620887756348   J:  400\n",
      "Reward:  1.6551952362060547   J:  600\n",
      "Reward:  1.60515558719635   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7726858854293823   J:  200\n",
      "Reward:  1.7139739990234375   J:  400\n",
      "Reward:  1.66563218832016   J:  600\n",
      "Reward:  1.546802043914795   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7712544202804565   J:  200\n",
      "Reward:  1.7450416088104248   J:  400\n",
      "Reward:  1.6615062952041626   J:  600\n",
      "Reward:  1.6102388501167297   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7690331935882568   J:  200\n",
      "Reward:  1.7403569221496582   J:  400\n",
      "Reward:  1.7021987438201904   J:  600\n",
      "Reward:  1.6011558771133423   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7711783647537231   J:  200\n",
      "Reward:  1.7436835765838623   J:  400\n",
      "Reward:  1.6585693955421448   J:  600\n",
      "Reward:  1.5992519855499268   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7701324224472046   J:  200\n",
      "Reward:  1.7113209962844849   J:  400\n",
      "Reward:  1.6611906290054321   J:  600\n",
      "Reward:  1.540496289730072   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7709821462631226   J:  200\n",
      "Reward:  1.742737054824829   J:  400\n",
      "Reward:  1.6589134335517883   J:  600\n",
      "Reward:  1.6033861637115479   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.768531084060669   J:  200\n",
      "Reward:  1.7048635482788086   J:  400\n",
      "Reward:  1.6021926403045654   J:  600\n",
      "Reward:  1.5361156463623047   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.771618127822876   J:  200\n",
      "Reward:  1.7430328130722046   J:  400\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reward:  1.6577802896499634   J:  600\n",
      "Reward:  1.6041189432144165   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7704294919967651   J:  200\n",
      "Reward:  1.741071343421936   J:  400\n",
      "Reward:  1.6613979935646057   J:  600\n",
      "Reward:  1.6011118292808533   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7467306852340698   J:  200\n",
      "Reward:  1.7088878154754639   J:  400\n",
      "Reward:  1.6071780920028687   J:  600\n",
      "Reward:  1.5380882024765015   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.773300290107727   J:  200\n",
      "Reward:  1.7115010023117065   J:  400\n",
      "Reward:  1.6618282198905945   J:  600\n",
      "Reward:  1.6038666367530823   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7726022005081177   J:  200\n",
      "Reward:  1.7405723333358765   J:  400\n",
      "Reward:  1.6577790975570679   J:  600\n",
      "Reward:  1.5945109724998474   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.772457242012024   J:  200\n",
      "Reward:  1.71019446849823   J:  400\n",
      "Reward:  1.664695680141449   J:  600\n",
      "Reward:  1.546335756778717   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7682393789291382   J:  200\n",
      "Reward:  1.7396353483200073   J:  400\n",
      "Reward:  1.6530855894088745   J:  600\n",
      "Reward:  1.5986239910125732   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7724452018737793   J:  200\n",
      "Reward:  1.7476527690887451   J:  400\n",
      "Reward:  1.6683358550071716   J:  600\n",
      "Reward:  1.6057698726654053   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.77066969871521   J:  200\n",
      "Reward:  1.7388581037521362   J:  400\n",
      "Reward:  1.6559271812438965   J:  600\n",
      "Reward:  1.6050025820732117   J:  800\n",
      "Iteration for {condition}:  8\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7736194133758545   J:  200\n",
      "Reward:  1.7121444940567017   J:  400\n",
      "Reward:  1.6063754558563232   J:  600\n",
      "Reward:  1.5468745827674866   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7680799961090088   J:  200\n",
      "Reward:  1.7001985311508179   J:  400\n",
      "Reward:  1.65128755569458   J:  600\n",
      "Reward:  1.526706337928772   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7658369541168213   J:  200\n",
      "Reward:  1.7045179605484009   J:  400\n",
      "Reward:  1.6092069149017334   J:  600\n",
      "Reward:  1.5456791520118713   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7720527648925781   J:  200\n",
      "Reward:  1.7094017267227173   J:  400\n",
      "Reward:  1.5404582023620605   J:  600\n",
      "Reward:  1.3840125799179077   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7658636569976807   J:  200\n",
      "Reward:  1.7092238664627075   J:  400\n",
      "Reward:  1.6545002460479736   J:  600\n",
      "Reward:  1.5371201634407043   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7673934698104858   J:  200\n",
      "Reward:  1.7387882471084595   J:  400\n",
      "Reward:  1.6506577134132385   J:  600\n",
      "Reward:  1.5965639352798462   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7499017715454102   J:  200\n",
      "Reward:  1.7145050764083862   J:  400\n",
      "Reward:  1.6729081273078918   J:  600\n",
      "Reward:  1.5572372078895569   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7729737758636475   J:  200\n",
      "Reward:  1.7471494674682617   J:  400\n",
      "Reward:  1.6635428071022034   J:  600\n",
      "Reward:  1.6098519563674927   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.76939058303833   J:  200\n",
      "Reward:  1.7434990406036377   J:  400\n",
      "Reward:  1.661105990409851   J:  600\n",
      "Reward:  1.605542778968811   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7727373838424683   J:  200\n",
      "Reward:  1.7015430927276611   J:  400\n",
      "Reward:  1.603629231452942   J:  600\n",
      "Reward:  1.4657193422317505   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.769280195236206   J:  200\n",
      "Reward:  1.707624912261963   J:  400\n",
      "Reward:  1.6083782315254211   J:  600\n",
      "Reward:  1.4677069187164307   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.770324468612671   J:  200\n",
      "Reward:  1.7409580945968628   J:  400\n",
      "Reward:  1.6621677875518799   J:  600\n",
      "Reward:  1.600532054901123   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7736012935638428   J:  200\n",
      "Reward:  1.7129374742507935   J:  400\n",
      "Reward:  1.6678615808486938   J:  600\n",
      "Reward:  1.6175658106803894   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7709728479385376   J:  200\n",
      "Reward:  1.74068021774292   J:  400\n",
      "Reward:  1.659273624420166   J:  600\n",
      "Reward:  1.602358877658844   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7744368314743042   J:  200\n",
      "Reward:  1.7511290311813354   J:  400\n",
      "Reward:  1.6710668206214905   J:  600\n",
      "Reward:  1.6198185682296753   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.77043616771698   J:  200\n",
      "Reward:  1.707047939300537   J:  400\n",
      "Reward:  1.6639547944068909   J:  600\n",
      "Reward:  1.5516924858093262   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7719768285751343   J:  200\n",
      "Reward:  1.7472965717315674   J:  400\n",
      "Reward:  1.6630987524986267   J:  600\n",
      "Reward:  1.6093058586120605   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7698789834976196   J:  200\n",
      "Reward:  1.7409635782241821   J:  400\n",
      "Reward:  1.66324383020401   J:  600\n",
      "Reward:  1.609253466129303   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.771230936050415   J:  200\n",
      "Reward:  1.7451359033584595   J:  400\n",
      "Reward:  1.659610629081726   J:  600\n",
      "Reward:  1.605654537677765   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.772944688796997   J:  200\n",
      "Reward:  1.746288776397705   J:  400\n",
      "Reward:  1.661568522453308   J:  600\n",
      "Reward:  1.6039742827415466   J:  800\n",
      "Iteration for {condition}:  9\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.770094633102417   J:  200\n",
      "Reward:  1.7049369812011719   J:  400\n",
      "Reward:  1.5957711338996887   J:  600\n",
      "Reward:  1.454856812953949   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.768437385559082   J:  200\n",
      "Reward:  1.741680383682251   J:  400\n",
      "Reward:  1.6541267037391663   J:  600\n",
      "Reward:  1.6029220819473267   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7703286409378052   J:  200\n",
      "Reward:  1.7440977096557617   J:  400\n",
      "Reward:  1.6620829105377197   J:  600\n",
      "Reward:  1.5437805652618408   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7728015184402466   J:  200\n",
      "Reward:  1.7468135356903076   J:  400\n",
      "Reward:  1.6579830646514893   J:  600\n",
      "Reward:  1.602666974067688   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7660342454910278   J:  200\n",
      "Reward:  1.7395373582839966   J:  400\n",
      "Reward:  1.6562871932983398   J:  600\n",
      "Reward:  1.6019354462623596   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7392733097076416   J:  200\n",
      "Reward:  1.701094388961792   J:  400\n",
      "Reward:  1.6114510893821716   J:  600\n",
      "Reward:  1.549073874950409   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7720574140548706   J:  200\n",
      "Reward:  1.7451115846633911   J:  400\n",
      "Reward:  1.666587769985199   J:  600\n",
      "Reward:  1.6107056140899658   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7701791524887085   J:  200\n",
      "Reward:  1.7106764316558838   J:  400\n",
      "Reward:  1.6695917248725891   J:  600\n",
      "Reward:  1.5522041320800781   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7695159912109375   J:  200\n",
      "Reward:  1.7103294134140015   J:  400\n",
      "Reward:  1.659510314464569   J:  600\n",
      "Reward:  1.5981059670448303   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7716412544250488   J:  200\n",
      "Reward:  1.711019515991211   J:  400\n",
      "Reward:  1.6641675233840942   J:  600\n",
      "Reward:  1.6094703078269958   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7708081007003784   J:  200\n",
      "Reward:  1.7085596323013306   J:  400\n",
      "Reward:  1.6633319854736328   J:  600\n",
      "Reward:  1.549139678478241   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7696009874343872   J:  200\n",
      "Reward:  1.740199089050293   J:  400\n",
      "Reward:  1.7023721933364868   J:  600\n",
      "Reward:  1.5953736901283264   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7714060544967651   J:  200\n",
      "Reward:  1.7057139873504639   J:  400\n",
      "Reward:  1.5975866913795471   J:  600\n",
      "Reward:  1.5382992029190063   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7659634351730347   J:  200\n",
      "Reward:  1.7113183736801147   J:  400\n",
      "Reward:  1.6124588251113892   J:  600\n",
      "Reward:  1.4858717918395996   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7691112756729126   J:  200\n",
      "Reward:  1.741576910018921   J:  400\n",
      "Reward:  1.6629499197006226   J:  600\n",
      "Reward:  1.6132161617279053   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7700071334838867   J:  200\n",
      "Reward:  1.7085095643997192   J:  400\n",
      "Reward:  1.602717101573944   J:  600\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reward:  1.5355716943740845   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7710862159729004   J:  200\n",
      "Reward:  1.7446426153182983   J:  400\n",
      "Reward:  1.5388233065605164   J:  600\n",
      "Reward:  1.4688900113105774   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7726202011108398   J:  200\n",
      "Reward:  1.7378228902816772   J:  400\n",
      "Reward:  1.6557130217552185   J:  600\n",
      "Reward:  1.608519971370697   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7725284099578857   J:  200\n",
      "Reward:  1.7485613822937012   J:  400\n",
      "Reward:  1.6694581508636475   J:  600\n",
      "Reward:  1.6203457117080688   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.770721673965454   J:  200\n",
      "Reward:  1.7449568510055542   J:  400\n",
      "Reward:  1.6632604002952576   J:  600\n",
      "Reward:  1.6079995036125183   J:  800\n",
      "Iteration for {condition}:  10\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.771622896194458   J:  200\n",
      "Reward:  1.711837649345398   J:  400\n",
      "Reward:  1.661904215812683   J:  600\n",
      "Reward:  1.5448241233825684   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.770617961883545   J:  200\n",
      "Reward:  1.7082713842391968   J:  400\n",
      "Reward:  1.6068644523620605   J:  600\n",
      "Reward:  1.5416052341461182   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7696983814239502   J:  200\n",
      "Reward:  1.7029030323028564   J:  400\n",
      "Reward:  1.6549853682518005   J:  600\n",
      "Reward:  1.5322790741920471   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7685681581497192   J:  200\n",
      "Reward:  1.74256432056427   J:  400\n",
      "Reward:  1.6566730737686157   J:  600\n",
      "Reward:  1.5961121916770935   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7706915140151978   J:  200\n",
      "Reward:  1.7082104682922363   J:  400\n",
      "Reward:  1.64799302816391   J:  600\n",
      "Reward:  1.5911096334457397   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7702232599258423   J:  200\n",
      "Reward:  1.6980646848678589   J:  400\n",
      "Reward:  1.6534598469734192   J:  600\n",
      "Reward:  1.5196818113327026   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7703335285186768   J:  200\n",
      "Reward:  1.71208655834198   J:  400\n",
      "Reward:  1.6089021563529968   J:  600\n",
      "Reward:  1.5398226976394653   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7712186574935913   J:  200\n",
      "Reward:  1.743099331855774   J:  400\n",
      "Reward:  1.6634501218795776   J:  600\n",
      "Reward:  1.608052670955658   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.772111177444458   J:  200\n",
      "Reward:  1.7420889139175415   J:  400\n",
      "Reward:  1.6602440476417542   J:  600\n",
      "Reward:  1.5979323387145996   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7716552019119263   J:  200\n",
      "Reward:  1.7425816059112549   J:  400\n",
      "Reward:  1.6593619585037231   J:  600\n",
      "Reward:  1.6050350069999695   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7703320980072021   J:  200\n",
      "Reward:  1.7067793607711792   J:  400\n",
      "Reward:  1.66028493642807   J:  600\n",
      "Reward:  1.601352870464325   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7701096534729004   J:  200\n",
      "Reward:  1.7140517234802246   J:  400\n",
      "Reward:  1.672373354434967   J:  600\n",
      "Reward:  1.5616668462753296   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7710405588150024   J:  200\n",
      "Reward:  1.7403446435928345   J:  400\n",
      "Reward:  1.6604378819465637   J:  600\n",
      "Reward:  1.541994571685791   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7695530652999878   J:  200\n",
      "Reward:  1.7424448728561401   J:  400\n",
      "Reward:  1.70440673828125   J:  600\n",
      "Reward:  1.5927002429962158   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.770380973815918   J:  200\n",
      "Reward:  1.7402111291885376   J:  400\n",
      "Reward:  1.7000207901000977   J:  600\n",
      "Reward:  1.593705415725708   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7706550359725952   J:  200\n",
      "Reward:  1.7460670471191406   J:  400\n",
      "Reward:  1.6597598791122437   J:  600\n",
      "Reward:  1.6042473912239075   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7713817358016968   J:  200\n",
      "Reward:  1.7402629852294922   J:  400\n",
      "Reward:  1.7024539113044739   J:  600\n",
      "Reward:  1.5948175191879272   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7687731981277466   J:  200\n",
      "Reward:  1.7422583103179932   J:  400\n",
      "Reward:  1.6541491150856018   J:  600\n",
      "Reward:  1.5996174216270447   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.7710485458374023   J:  200\n",
      "Reward:  1.7090569734573364   J:  400\n",
      "Reward:  1.6611611247062683   J:  600\n",
      "Reward:  1.6033235788345337   J:  800\n",
      "Reward:  139.78680419921875   J:  0\n",
      "Reward:  1.769480586051941   J:  200\n",
      "Reward:  1.6970415711402893   J:  400\n",
      "Reward:  1.5917468070983887   J:  600\n",
      "Reward:  1.4632230997085571   J:  800\n",
      "Iteration for {condition}:  11\n"
     ]
    }
   ],
   "source": [
    "if __name__ == '__main__':\n",
    "\n",
    "    client = carla.Client('localhost', 2000)\n",
    "    client.set_timeout(2.0)\n",
    "\n",
    "    world = client.get_world()\n",
    "    blueprint_library = world.get_blueprint_library()\n",
    "    vehicle_blueprint = blueprint_library.filter(\"wrangler_rubicon\")[0]\n",
    "\n",
    "    waypoints = read_waypoints(\"waypoint_with_xy.csv\")\n",
    "    env = CarlaEnvironment(world, vehicle_blueprint, waypoints)  # Implement CarlaEnvironment with the required state and action spaces\n",
    "\n",
    "    state_dim = 40\n",
    "    action_dim = 3\n",
    "    train_ppo_maml(env, state_dim, action_dim)\n",
    "    \n",
    "\n",
    "    \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5dd3e782",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}