{
  "grid": [
    [
      "S",
      "C",
      "#",
      "#"
    ],
    [
      "A",
      "B",
      "D",
      "G"
    ]
  ],
  "positions": {
    "S": [
      0,
      0
    ],
    "C": [
      0,
      1
    ],
    "A": [
      1,
      0
    ],
    "B": [
      1,
      1
    ],
    "D": [
      1,
      2
    ],
    "G": [
      1,
      3
    ]
  },
  "state_meanings": {
    "S": "起点",
    "C": "即时 +2，终止",
    "A": "长路线第一格",
    "B": "长路线第二格",
    "D": "长路线第三格",
    "G": "延迟 +10，终止"
  },
  "actions": {
    "up": [
      -1,
      0
    ],
    "right": [
      0,
      1
    ],
    "down": [
      1,
      0
    ],
    "left": [
      0,
      -1
    ]
  },
  "transitions": {
    "S": {
      "up": {
        "next_state": "S",
        "reward": 0,
        "done": false,
        "probability": 1.0
      },
      "right": {
        "next_state": "C",
        "reward": 2,
        "done": true,
        "probability": 1.0
      },
      "down": {
        "next_state": "A",
        "reward": 0,
        "done": false,
        "probability": 1.0
      },
      "left": {
        "next_state": "S",
        "reward": 0,
        "done": false,
        "probability": 1.0
      }
    },
    "C": {
      "up": {
        "next_state": "C",
        "reward": 0,
        "done": true,
        "probability": 1.0
      },
      "right": {
        "next_state": "C",
        "reward": 0,
        "done": true,
        "probability": 1.0
      },
      "down": {
        "next_state": "C",
        "reward": 0,
        "done": true,
        "probability": 1.0
      },
      "left": {
        "next_state": "C",
        "reward": 0,
        "done": true,
        "probability": 1.0
      }
    },
    "A": {
      "up": {
        "next_state": "S",
        "reward": 0,
        "done": false,
        "probability": 1.0
      },
      "right": {
        "next_state": "B",
        "reward": 0,
        "done": false,
        "probability": 1.0
      },
      "down": {
        "next_state": "A",
        "reward": 0,
        "done": false,
        "probability": 1.0
      },
      "left": {
        "next_state": "A",
        "reward": 0,
        "done": false,
        "probability": 1.0
      }
    },
    "B": {
      "up": {
        "next_state": "C",
        "reward": 2,
        "done": true,
        "probability": 1.0
      },
      "right": {
        "next_state": "D",
        "reward": 0,
        "done": false,
        "probability": 1.0
      },
      "down": {
        "next_state": "B",
        "reward": 0,
        "done": false,
        "probability": 1.0
      },
      "left": {
        "next_state": "A",
        "reward": 0,
        "done": false,
        "probability": 1.0
      }
    },
    "D": {
      "up": {
        "next_state": "D",
        "reward": 0,
        "done": false,
        "probability": 1.0
      },
      "right": {
        "next_state": "G",
        "reward": 10,
        "done": true,
        "probability": 1.0
      },
      "down": {
        "next_state": "D",
        "reward": 0,
        "done": false,
        "probability": 1.0
      },
      "left": {
        "next_state": "B",
        "reward": 0,
        "done": false,
        "probability": 1.0
      }
    },
    "G": {
      "up": {
        "next_state": "G",
        "reward": 0,
        "done": true,
        "probability": 1.0
      },
      "right": {
        "next_state": "G",
        "reward": 0,
        "done": true,
        "probability": 1.0
      },
      "down": {
        "next_state": "G",
        "reward": 0,
        "done": true,
        "probability": 1.0
      },
      "left": {
        "next_state": "G",
        "reward": 0,
        "done": true,
        "probability": 1.0
      }
    }
  },
  "discount_cases": [
    {
      "gamma": 0.5,
      "coin": {
        "actions": [
          "right"
        ],
        "trace": [
          {
            "t": 0,
            "state": "S",
            "action": "right",
            "next_state": "C",
            "reward": 2,
            "discounted": 2.0,
            "done": true
          }
        ],
        "discounted_return": 2.0,
        "end_state": "C"
      },
      "goal": {
        "actions": [
          "down",
          "right",
          "right",
          "right"
        ],
        "trace": [
          {
            "t": 0,
            "state": "S",
            "action": "down",
            "next_state": "A",
            "reward": 0,
            "discounted": 0.0,
            "done": false
          },
          {
            "t": 1,
            "state": "A",
            "action": "right",
            "next_state": "B",
            "reward": 0,
            "discounted": 0.0,
            "done": false
          },
          {
            "t": 2,
            "state": "B",
            "action": "right",
            "next_state": "D",
            "reward": 0,
            "discounted": 0.0,
            "done": false
          },
          {
            "t": 3,
            "state": "D",
            "action": "right",
            "next_state": "G",
            "reward": 10,
            "discounted": 1.25,
            "done": true
          }
        ],
        "discounted_return": 1.25,
        "end_state": "G"
      },
      "optimal_q_start": {
        "right": 2.0,
        "down": 1.25
      },
      "optimal_value_start": 2.0,
      "best_route": "coin"
    },
    {
      "gamma": 0.8,
      "coin": {
        "actions": [
          "right"
        ],
        "trace": [
          {
            "t": 0,
            "state": "S",
            "action": "right",
            "next_state": "C",
            "reward": 2,
            "discounted": 2.0,
            "done": true
          }
        ],
        "discounted_return": 2.0,
        "end_state": "C"
      },
      "goal": {
        "actions": [
          "down",
          "right",
          "right",
          "right"
        ],
        "trace": [
          {
            "t": 0,
            "state": "S",
            "action": "down",
            "next_state": "A",
            "reward": 0,
            "discounted": 0.0,
            "done": false
          },
          {
            "t": 1,
            "state": "A",
            "action": "right",
            "next_state": "B",
            "reward": 0,
            "discounted": 0.0,
            "done": false
          },
          {
            "t": 2,
            "state": "B",
            "action": "right",
            "next_state": "D",
            "reward": 0,
            "discounted": 0.0,
            "done": false
          },
          {
            "t": 3,
            "state": "D",
            "action": "right",
            "next_state": "G",
            "reward": 10,
            "discounted": 5.120000000000001,
            "done": true
          }
        ],
        "discounted_return": 5.120000000000001,
        "end_state": "G"
      },
      "optimal_q_start": {
        "right": 2.0,
        "down": 5.120000000000001
      },
      "optimal_value_start": 5.120000000000001,
      "best_route": "goal"
    },
    {
      "gamma": 0.95,
      "coin": {
        "actions": [
          "right"
        ],
        "trace": [
          {
            "t": 0,
            "state": "S",
            "action": "right",
            "next_state": "C",
            "reward": 2,
            "discounted": 2.0,
            "done": true
          }
        ],
        "discounted_return": 2.0,
        "end_state": "C"
      },
      "goal": {
        "actions": [
          "down",
          "right",
          "right",
          "right"
        ],
        "trace": [
          {
            "t": 0,
            "state": "S",
            "action": "down",
            "next_state": "A",
            "reward": 0,
            "discounted": 0.0,
            "done": false
          },
          {
            "t": 1,
            "state": "A",
            "action": "right",
            "next_state": "B",
            "reward": 0,
            "discounted": 0.0,
            "done": false
          },
          {
            "t": 2,
            "state": "B",
            "action": "right",
            "next_state": "D",
            "reward": 0,
            "discounted": 0.0,
            "done": false
          },
          {
            "t": 3,
            "state": "D",
            "action": "right",
            "next_state": "G",
            "reward": 10,
            "discounted": 8.573749999999999,
            "done": true
          }
        ],
        "discounted_return": 8.573749999999999,
        "end_state": "G"
      },
      "optimal_q_start": {
        "right": 2.0,
        "down": 8.57375
      },
      "optimal_value_start": 8.57375,
      "best_route": "goal"
    }
  ],
  "exploration_at_gamma_0_8": [
    {
      "epsilon": 0.0,
      "episodes": [
        {
          "episode": 1,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 2,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 3,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 4,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 5,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 6,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 7,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 8,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 9,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 10,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 11,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 12,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 13,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 14,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 15,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 16,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 17,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 18,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 19,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 20,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 21,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 22,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 23,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 24,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 25,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 26,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 27,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 28,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 29,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 30,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        }
      ],
      "counts": {
        "coin": 30,
        "goal": 0
      },
      "final_estimates": {
        "coin": 2.0,
        "goal": 0.0
      },
      "first_goal_episode": null
    },
    {
      "epsilon": 0.2,
      "episodes": [
        {
          "episode": 1,
          "route": "coin",
          "explore": true,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 2,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 3,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 4,
          "route": "coin",
          "explore": false,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 0.0
        },
        {
          "episode": 5,
          "route": "goal",
          "explore": true,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 6,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 7,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 8,
          "route": "coin",
          "explore": true,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 9,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 10,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 11,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 12,
          "route": "coin",
          "explore": true,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 13,
          "route": "coin",
          "explore": true,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 14,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 15,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 16,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 17,
          "route": "coin",
          "explore": true,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 18,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 19,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 20,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 21,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 22,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 23,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 24,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 25,
          "route": "goal",
          "explore": true,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 26,
          "route": "coin",
          "explore": true,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 27,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 28,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 29,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 30,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        }
      ],
      "counts": {
        "coin": 9,
        "goal": 21
      },
      "final_estimates": {
        "coin": 2.0,
        "goal": 5.120000000000001
      },
      "first_goal_episode": 5
    },
    {
      "epsilon": 0.5,
      "episodes": [
        {
          "episode": 1,
          "route": "goal",
          "explore": true,
          "observed_return": 5.120000000000001,
          "coin_estimate": 0.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 2,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 0.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 3,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 0.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 4,
          "route": "goal",
          "explore": true,
          "observed_return": 5.120000000000001,
          "coin_estimate": 0.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 5,
          "route": "coin",
          "explore": true,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 6,
          "route": "goal",
          "explore": true,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 7,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 8,
          "route": "coin",
          "explore": true,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 9,
          "route": "goal",
          "explore": true,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 10,
          "route": "goal",
          "explore": true,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 11,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 12,
          "route": "coin",
          "explore": true,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 13,
          "route": "goal",
          "explore": true,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 14,
          "route": "coin",
          "explore": true,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 15,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 16,
          "route": "coin",
          "explore": true,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 17,
          "route": "coin",
          "explore": true,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 18,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 19,
          "route": "goal",
          "explore": true,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 20,
          "route": "goal",
          "explore": true,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 21,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 22,
          "route": "goal",
          "explore": true,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 23,
          "route": "coin",
          "explore": true,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 24,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 25,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 26,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 27,
          "route": "coin",
          "explore": true,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 28,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 29,
          "route": "goal",
          "explore": false,
          "observed_return": 5.120000000000001,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        },
        {
          "episode": 30,
          "route": "coin",
          "explore": true,
          "observed_return": 2.0,
          "coin_estimate": 2.0,
          "goal_estimate": 5.120000000000001
        }
      ],
      "counts": {
        "coin": 9,
        "goal": 21
      },
      "final_estimates": {
        "coin": 2.0,
        "goal": 5.120000000000001
      },
      "first_goal_episode": 1
    }
  ],
  "threshold_gamma": 0.5848035476425733
}