From a291702af92b204701fb7a43472f922af1d4fea1 Mon Sep 17 00:00:00 2001 From: Constantin Ruhdorfer Date: Tue, 25 Jun 2024 16:22:33 +0200 Subject: [PATCH] Init --- .gitignore | 28 + .vscode/settings.json | 4 + LICENSE | 203 ++ README.md | 496 ++++ docs/envs/maze.md | 126 + docs/envs/overcooked.md | 110 + docs/evaluate_args.md | 37 + docs/images/OvercookedDCD.png | Bin 0 -> 160105 bytes docs/images/Training6x9SmallStylised.pdf | Bin 0 -> 23435 bytes docs/images/Training6x9SmallStylised.png | Bin 0 -> 219242 bytes docs/images/env_maze_overview.png | Bin 0 -> 135681 bytes docs/images/minimax_logo.png | Bin 0 -> 121469 bytes docs/images/minimax_speedups.png | Bin 0 -> 43148 bytes docs/images/minimax_speedups_darkmode.png | Bin 0 -> 37935 bytes docs/images/minimax_system_diagram.png | Bin 0 -> 235638 bytes docs/images/parallel_dcd_overview.png | Bin 0 -> 132713 bytes docs/make_cmd.md | 28 + docs/parsnip.md | 131 + docs/train_args.md | 125 + requirements.txt | 10 + src/__init__.py | 0 src/baseline_train__8_seeds.sh | 72 + src/baseline_train__all.sh | 8 + src/baseline_train__holdout_sp.sh | 71 + src/config/configs/maze/accel.json | 73 + src/config/configs/maze/dr.json | 59 + src/config/configs/maze/paccel.json | 73 + src/config/configs/maze/paired.json | 84 + src/config/configs/maze/plr.json | 69 + src/config/configs/maze/pplr.json | 69 + src/config/configs/maze/s5_accel.json | 78 + src/config/configs/maze/s5_dr.json | 63 + src/config/configs/maze/s5_paccel.json | 77 + src/config/configs/maze/s5_paired.json | 94 + src/config/configs/maze/s5_plr.json | 73 + src/config/configs/maze/s5_pplr.json | 73 + .../baseline__cnn_asymm_advantages.json | 63 + .../baseline__s5_asymm_advantages.json | 69 + .../overcooked/baseline__s5_coord_ring.json | 69 + .../baseline__s5_counter_circuit.json | 69 + .../overcooked/baseline__s5_cramped_room.json | 69 + .../overcooked/baseline__s5_forced_coord.json | 69 + .../configs/overcooked/baseline_dr_lstm.json | 64 + .../overcooked/baseline_dr_lstm5x5.json | 64 + .../configs/overcooked/baseline_dr_s5.json | 68 + .../configs/overcooked/baseline_dr_s55x5.json | 68 + .../overcooked/baseline_dr_softmoe_lstm.json | 67 + .../baseline_dr_softmoe_lstm5x5.json | 67 + .../overcooked/baseline_p_accel_lstm.json | 78 + .../overcooked/baseline_p_accel_lstm5x5.json | 78 + .../overcooked/baseline_p_accel_s5.json | 82 + .../overcooked/baseline_p_accel_s55x5.json | 82 + .../baseline_p_accel_softmoe_lstm.json | 81 + .../baseline_p_accel_softmoe_lstm5x5.json | 81 + .../overcooked/baseline_p_plr_lstm.json | 74 + .../overcooked/baseline_p_plr_lstm5x5.json | 74 + .../configs/overcooked/baseline_p_plr_s5.json | 78 + .../overcooked/baseline_p_plr_s55x5.json | 78 + .../baseline_p_plr_softmoe_lstm.json | 77 + .../baseline_p_plr_softmoe_lstm5x5.json | 77 + .../overcooked/baseline_pop_paired_lstm.json | 86 + .../baseline_pop_paired_lstm5x5.json | 86 + .../overcooked/baseline_pop_paired_s5.json | 90 + .../overcooked/baseline_pop_paired_s55x5.json | 90 + .../baseline_pop_paired_softmoe_lstm.json | 89 + .../baseline_pop_paired_softmoe_lstm5x5.json | 89 + src/config/configs/overcooked/paired.json | 83 + src/config/configs/overcooked/plr.json | 71 + src/config/configs/overcooked/plr_s5.json | 78 + ..._xpid_against_population_in_all_layouts.sh | 9 + src/eval_random_against_population.sh | 11 + src/eval_stay_against_population.sh | 10 + src/eval_xpid.sh | 6 + src/eval_xpid_against_population.sh | 10 + ..._xpid_against_population_in_all_layouts.sh | 14 + src/eval_xpid_all_cnn_lstm.sh | 19 + src/eval_xpid_all_cnn_s5.sh | 19 + src/eval_xpid_all_softmoe.sh | 19 + src/extract_fcp.sh | 14 + src/make_cmd.sh | 1 + src/minimax/__init__.py | 9 + src/minimax/agents/__init__.py | 15 + src/minimax/agents/agent.py | 40 + src/minimax/agents/mappo.py | 449 ++++ src/minimax/agents/ppo.py | 304 +++ src/minimax/arguments.py | 1023 +++++++ src/minimax/config/__init__.py | 0 src/minimax/config/configs/maze/accel.json | 73 + src/minimax/config/configs/maze/dr.json | 59 + src/minimax/config/configs/maze/paccel.json | 73 + src/minimax/config/configs/maze/paired.json | 84 + src/minimax/config/configs/maze/plr.json | 69 + src/minimax/config/configs/maze/pplr.json | 69 + src/minimax/config/configs/maze/s5_accel.json | 78 + src/minimax/config/configs/maze/s5_dr.json | 63 + .../config/configs/maze/s5_paccel.json | 77 + .../config/configs/maze/s5_paired.json | 94 + src/minimax/config/configs/maze/s5_plr.json | 73 + src/minimax/config/configs/maze/s5_pplr.json | 73 + src/minimax/config/make_cmd.py | 287 ++ src/minimax/config/xpid_maker.py | 328 +++ src/minimax/count_params.py | 133 + src/minimax/envs/__init__.py | 15 + src/minimax/envs/batch_env.py | 74 + src/minimax/envs/batch_env_ued.py | 134 + src/minimax/envs/environment.py | 201 ++ src/minimax/envs/environment_ued.py | 142 + src/minimax/envs/interactive/__init__.py | 0 .../envs/interactive/manual_ctrl_maze.py | 219 ++ src/minimax/envs/maze/__init__.py | 14 + src/minimax/envs/maze/common.py | 109 + src/minimax/envs/maze/maze.py | 521 ++++ src/minimax/envs/maze/maze_comparators.py | 34 + src/minimax/envs/maze/maze_mutators.py | 110 + src/minimax/envs/maze/maze_ood.py | 1111 ++++++++ src/minimax/envs/maze/maze_ued.py | 425 +++ src/minimax/envs/overcooked_proc/__init__.py | 14 + src/minimax/envs/overcooked_proc/common.py | 207 ++ .../envs/overcooked_proc/interactive.py | 290 ++ src/minimax/envs/overcooked_proc/layouts.py | 556 ++++ .../envs/overcooked_proc/overcooked.py | 1389 ++++++++++ .../overcooked_proc/overcooked_comparators.py | 40 + .../overcooked_proc/overcooked_mutators.py | 253 ++ .../envs/overcooked_proc/overcooked_ood.py | 405 +++ .../envs/overcooked_proc/overcooked_ued.py | 541 ++++ src/minimax/envs/registration.py | 149 ++ src/minimax/envs/spaces.py | 154 ++ src/minimax/envs/viz/__init__.py | 0 src/minimax/envs/viz/grid_rendering.py | 133 + src/minimax/envs/viz/grid_viz.py | 272 ++ src/minimax/envs/viz/overcooked_visualizer.py | 378 +++ src/minimax/envs/viz/window.py | 107 + src/minimax/envs/wrappers/__init__.py | 14 + src/minimax/envs/wrappers/env_wrapper.py | 114 + .../envs/wrappers/monitor_ep_metrics.py | 84 + src/minimax/envs/wrappers/monitor_return.py | 76 + src/minimax/envs/wrappers/ued_env_wrapper.py | 85 + .../envs/wrappers/world_state_wrapper.py | 160 ++ src/minimax/evaluate.py | 244 ++ src/minimax/evaluate_against_baseline.py | 276 ++ src/minimax/evaluate_against_population.py | 281 ++ .../evaluate_baseline_against_population.py | 319 +++ src/minimax/evaluate_from_pckl.py | 264 ++ src/minimax/extract_fcp.py | 276 ++ src/minimax/models/__init__.py | 33 + src/minimax/models/common.py | 383 +++ src/minimax/models/fast_attention.py | 711 +++++ src/minimax/models/maze/__init__.py | 12 + src/minimax/models/maze/gridworld_models.py | 277 ++ src/minimax/models/moe.py | 162 ++ src/minimax/models/overcooked/__init__.py | 14 + src/minimax/models/overcooked/models.py | 536 ++++ src/minimax/models/registration.py | 51 + src/minimax/models/rnn.py | 98 + src/minimax/models/s5.py | 706 +++++ src/minimax/models/transformer.py | 264 ++ src/minimax/runners/__init__.py | 22 + src/minimax/runners/dr_runner.py | 458 ++++ src/minimax/runners/eval_runner.py | 325 +++ src/minimax/runners/paired_runner.py | 604 +++++ src/minimax/runners/plr_runner.py | 549 ++++ src/minimax/runners/xp_runner.py | 310 +++ src/minimax/runners_ma/__init__.py | 24 + src/minimax/runners_ma/dr_runner.py | 569 ++++ src/minimax/runners_ma/eval_runner.py | 371 +++ .../runners_ma/eval_runner_heterogenous.py | 388 +++ src/minimax/runners_ma/paired_runner.py | 818 ++++++ src/minimax/runners_ma/plr_runner.py | 578 ++++ src/minimax/runners_ma/xp_runner.py | 377 +++ src/minimax/tests/__init__.py | 0 src/minimax/tests/base_req_rollout_storage.py | 116 + src/minimax/tests/dummy_test_envs.py | 130 + src/minimax/tests/test_rollout_storage.py | 163 ++ src/minimax/tests/test_ued_scores.py | 74 + src/minimax/tests/test_wrappers.py | 146 + src/minimax/train.py | 97 + src/minimax/util/__init__.py | 9 + src/minimax/util/args.py | 20 + src/minimax/util/checkpoint.py | 74 + src/minimax/util/dotdict.py | 68 + src/minimax/util/graph.py | 263 ++ src/minimax/util/loggers.py | 291 ++ src/minimax/util/parsnip.py | 329 +++ src/minimax/util/pytree.py | 37 + src/minimax/util/rl/__init__.py | 16 + src/minimax/util/rl/agent_pop.py | 119 + src/minimax/util/rl/agent_pop_heterogenous.py | 166 ++ src/minimax/util/rl/hl_gauss_transform.py | 39 + src/minimax/util/rl/plr.py | 466 ++++ src/minimax/util/rl/rolling_stats.py | 116 + src/minimax/util/rl/rollout_storage.py | 227 ++ .../util/rl/rollout_storage_seperate.py | 274 ++ src/minimax/util/rl/training.py | 201 ++ src/minimax/util/rl/ued_scores.py | 245 ++ ...ainst_population_in_all_69_layouts_out.txt | 54 + ..._against_population_in_all_layouts_out.txt | 138 + .../eval_xpid_all_cnn_lstm_out.txt | 2381 +++++++++++++++++ .../eval_xpid_all_cnn_s5_out.txt | 2290 ++++++++++++++++ .../eval_xpid_all_softmoe_out.txt | 2280 ++++++++++++++++ src/train_baseline_dr_lstm.sh | 67 + src/train_baseline_dr_s5.sh | 71 + src/train_baseline_dr_softmoe_lstm.sh | 70 + src/train_baseline_p_accel_lstm.sh | 81 + src/train_baseline_p_accel_s5.sh | 85 + src/train_baseline_p_accel_softmoe_lstm.sh | 84 + src/train_baseline_p_plr_lstm.sh | 77 + src/train_baseline_p_plr_s5.sh | 81 + src/train_baseline_p_plr_softmoe_lstm.sh | 80 + src/train_baseline_pop_paired_lstm.sh | 89 + src/train_baseline_pop_paired_s5.sh | 93 + src/train_baseline_pop_paired_softmoe_lstm.sh | 92 + src/train_baselines_lstm6x9.sh | 10 + src/train_baselines_s56x9.sh | 10 + src/train_baselines_softmoe_lstm6x9.sh | 10 + src/train_maze.sh | 67 + src/train_maze_s5.sh | 76 + 216 files changed, 39249 insertions(+) create mode 100644 .gitignore create mode 100644 .vscode/settings.json create mode 100644 LICENSE create mode 100644 README.md create mode 100644 docs/envs/maze.md create mode 100644 docs/envs/overcooked.md create mode 100644 docs/evaluate_args.md create mode 100644 docs/images/OvercookedDCD.png create mode 100644 docs/images/Training6x9SmallStylised.pdf create mode 100644 docs/images/Training6x9SmallStylised.png create mode 100644 docs/images/env_maze_overview.png create mode 100644 docs/images/minimax_logo.png create mode 100644 docs/images/minimax_speedups.png create mode 100644 docs/images/minimax_speedups_darkmode.png create mode 100644 docs/images/minimax_system_diagram.png create mode 100644 docs/images/parallel_dcd_overview.png create mode 100644 docs/make_cmd.md create mode 100644 docs/parsnip.md create mode 100644 docs/train_args.md create mode 100644 requirements.txt create mode 100644 src/__init__.py create mode 100755 src/baseline_train__8_seeds.sh create mode 100755 src/baseline_train__all.sh create mode 100755 src/baseline_train__holdout_sp.sh create mode 100644 src/config/configs/maze/accel.json create mode 100644 src/config/configs/maze/dr.json create mode 100644 src/config/configs/maze/paccel.json create mode 100644 src/config/configs/maze/paired.json create mode 100644 src/config/configs/maze/plr.json create mode 100644 src/config/configs/maze/pplr.json create mode 100644 src/config/configs/maze/s5_accel.json create mode 100644 src/config/configs/maze/s5_dr.json create mode 100644 src/config/configs/maze/s5_paccel.json create mode 100644 src/config/configs/maze/s5_paired.json create mode 100644 src/config/configs/maze/s5_plr.json create mode 100644 src/config/configs/maze/s5_pplr.json create mode 100644 src/config/configs/overcooked/baseline__cnn_asymm_advantages.json create mode 100644 src/config/configs/overcooked/baseline__s5_asymm_advantages.json create mode 100644 src/config/configs/overcooked/baseline__s5_coord_ring.json create mode 100644 src/config/configs/overcooked/baseline__s5_counter_circuit.json create mode 100644 src/config/configs/overcooked/baseline__s5_cramped_room.json create mode 100644 src/config/configs/overcooked/baseline__s5_forced_coord.json create mode 100644 src/config/configs/overcooked/baseline_dr_lstm.json create mode 100644 src/config/configs/overcooked/baseline_dr_lstm5x5.json create mode 100644 src/config/configs/overcooked/baseline_dr_s5.json create mode 100644 src/config/configs/overcooked/baseline_dr_s55x5.json create mode 100644 src/config/configs/overcooked/baseline_dr_softmoe_lstm.json create mode 100644 src/config/configs/overcooked/baseline_dr_softmoe_lstm5x5.json create mode 100644 src/config/configs/overcooked/baseline_p_accel_lstm.json create mode 100644 src/config/configs/overcooked/baseline_p_accel_lstm5x5.json create mode 100644 src/config/configs/overcooked/baseline_p_accel_s5.json create mode 100644 src/config/configs/overcooked/baseline_p_accel_s55x5.json create mode 100644 src/config/configs/overcooked/baseline_p_accel_softmoe_lstm.json create mode 100644 src/config/configs/overcooked/baseline_p_accel_softmoe_lstm5x5.json create mode 100644 src/config/configs/overcooked/baseline_p_plr_lstm.json create mode 100644 src/config/configs/overcooked/baseline_p_plr_lstm5x5.json create mode 100644 src/config/configs/overcooked/baseline_p_plr_s5.json create mode 100644 src/config/configs/overcooked/baseline_p_plr_s55x5.json create mode 100644 src/config/configs/overcooked/baseline_p_plr_softmoe_lstm.json create mode 100644 src/config/configs/overcooked/baseline_p_plr_softmoe_lstm5x5.json create mode 100644 src/config/configs/overcooked/baseline_pop_paired_lstm.json create mode 100644 src/config/configs/overcooked/baseline_pop_paired_lstm5x5.json create mode 100644 src/config/configs/overcooked/baseline_pop_paired_s5.json create mode 100644 src/config/configs/overcooked/baseline_pop_paired_s55x5.json create mode 100644 src/config/configs/overcooked/baseline_pop_paired_softmoe_lstm.json create mode 100644 src/config/configs/overcooked/baseline_pop_paired_softmoe_lstm5x5.json create mode 100644 src/config/configs/overcooked/paired.json create mode 100644 src/config/configs/overcooked/plr.json create mode 100644 src/config/configs/overcooked/plr_s5.json create mode 100755 src/eval_all_xpid_against_population_in_all_layouts.sh create mode 100755 src/eval_random_against_population.sh create mode 100755 src/eval_stay_against_population.sh create mode 100755 src/eval_xpid.sh create mode 100755 src/eval_xpid_against_population.sh create mode 100755 src/eval_xpid_against_population_in_all_layouts.sh create mode 100755 src/eval_xpid_all_cnn_lstm.sh create mode 100755 src/eval_xpid_all_cnn_s5.sh create mode 100755 src/eval_xpid_all_softmoe.sh create mode 100755 src/extract_fcp.sh create mode 100755 src/make_cmd.sh create mode 100644 src/minimax/__init__.py create mode 100644 src/minimax/agents/__init__.py create mode 100644 src/minimax/agents/agent.py create mode 100644 src/minimax/agents/mappo.py create mode 100644 src/minimax/agents/ppo.py create mode 100644 src/minimax/arguments.py create mode 100644 src/minimax/config/__init__.py create mode 100644 src/minimax/config/configs/maze/accel.json create mode 100644 src/minimax/config/configs/maze/dr.json create mode 100644 src/minimax/config/configs/maze/paccel.json create mode 100644 src/minimax/config/configs/maze/paired.json create mode 100644 src/minimax/config/configs/maze/plr.json create mode 100644 src/minimax/config/configs/maze/pplr.json create mode 100644 src/minimax/config/configs/maze/s5_accel.json create mode 100644 src/minimax/config/configs/maze/s5_dr.json create mode 100644 src/minimax/config/configs/maze/s5_paccel.json create mode 100644 src/minimax/config/configs/maze/s5_paired.json create mode 100644 src/minimax/config/configs/maze/s5_plr.json create mode 100644 src/minimax/config/configs/maze/s5_pplr.json create mode 100644 src/minimax/config/make_cmd.py create mode 100644 src/minimax/config/xpid_maker.py create mode 100644 src/minimax/count_params.py create mode 100644 src/minimax/envs/__init__.py create mode 100644 src/minimax/envs/batch_env.py create mode 100644 src/minimax/envs/batch_env_ued.py create mode 100644 src/minimax/envs/environment.py create mode 100644 src/minimax/envs/environment_ued.py create mode 100644 src/minimax/envs/interactive/__init__.py create mode 100644 src/minimax/envs/interactive/manual_ctrl_maze.py create mode 100644 src/minimax/envs/maze/__init__.py create mode 100644 src/minimax/envs/maze/common.py create mode 100644 src/minimax/envs/maze/maze.py create mode 100644 src/minimax/envs/maze/maze_comparators.py create mode 100644 src/minimax/envs/maze/maze_mutators.py create mode 100644 src/minimax/envs/maze/maze_ood.py create mode 100644 src/minimax/envs/maze/maze_ued.py create mode 100644 src/minimax/envs/overcooked_proc/__init__.py create mode 100644 src/minimax/envs/overcooked_proc/common.py create mode 100644 src/minimax/envs/overcooked_proc/interactive.py create mode 100644 src/minimax/envs/overcooked_proc/layouts.py create mode 100644 src/minimax/envs/overcooked_proc/overcooked.py create mode 100644 src/minimax/envs/overcooked_proc/overcooked_comparators.py create mode 100644 src/minimax/envs/overcooked_proc/overcooked_mutators.py create mode 100644 src/minimax/envs/overcooked_proc/overcooked_ood.py create mode 100644 src/minimax/envs/overcooked_proc/overcooked_ued.py create mode 100644 src/minimax/envs/registration.py create mode 100644 src/minimax/envs/spaces.py create mode 100644 src/minimax/envs/viz/__init__.py create mode 100644 src/minimax/envs/viz/grid_rendering.py create mode 100644 src/minimax/envs/viz/grid_viz.py create mode 100644 src/minimax/envs/viz/overcooked_visualizer.py create mode 100644 src/minimax/envs/viz/window.py create mode 100644 src/minimax/envs/wrappers/__init__.py create mode 100644 src/minimax/envs/wrappers/env_wrapper.py create mode 100644 src/minimax/envs/wrappers/monitor_ep_metrics.py create mode 100644 src/minimax/envs/wrappers/monitor_return.py create mode 100644 src/minimax/envs/wrappers/ued_env_wrapper.py create mode 100644 src/minimax/envs/wrappers/world_state_wrapper.py create mode 100644 src/minimax/evaluate.py create mode 100644 src/minimax/evaluate_against_baseline.py create mode 100644 src/minimax/evaluate_against_population.py create mode 100644 src/minimax/evaluate_baseline_against_population.py create mode 100644 src/minimax/evaluate_from_pckl.py create mode 100644 src/minimax/extract_fcp.py create mode 100644 src/minimax/models/__init__.py create mode 100644 src/minimax/models/common.py create mode 100644 src/minimax/models/fast_attention.py create mode 100644 src/minimax/models/maze/__init__.py create mode 100644 src/minimax/models/maze/gridworld_models.py create mode 100644 src/minimax/models/moe.py create mode 100644 src/minimax/models/overcooked/__init__.py create mode 100644 src/minimax/models/overcooked/models.py create mode 100644 src/minimax/models/registration.py create mode 100644 src/minimax/models/rnn.py create mode 100644 src/minimax/models/s5.py create mode 100644 src/minimax/models/transformer.py create mode 100644 src/minimax/runners/__init__.py create mode 100644 src/minimax/runners/dr_runner.py create mode 100644 src/minimax/runners/eval_runner.py create mode 100644 src/minimax/runners/paired_runner.py create mode 100644 src/minimax/runners/plr_runner.py create mode 100644 src/minimax/runners/xp_runner.py create mode 100644 src/minimax/runners_ma/__init__.py create mode 100644 src/minimax/runners_ma/dr_runner.py create mode 100644 src/minimax/runners_ma/eval_runner.py create mode 100644 src/minimax/runners_ma/eval_runner_heterogenous.py create mode 100644 src/minimax/runners_ma/paired_runner.py create mode 100644 src/minimax/runners_ma/plr_runner.py create mode 100644 src/minimax/runners_ma/xp_runner.py create mode 100644 src/minimax/tests/__init__.py create mode 100644 src/minimax/tests/base_req_rollout_storage.py create mode 100644 src/minimax/tests/dummy_test_envs.py create mode 100644 src/minimax/tests/test_rollout_storage.py create mode 100644 src/minimax/tests/test_ued_scores.py create mode 100644 src/minimax/tests/test_wrappers.py create mode 100644 src/minimax/train.py create mode 100644 src/minimax/util/__init__.py create mode 100644 src/minimax/util/args.py create mode 100644 src/minimax/util/checkpoint.py create mode 100644 src/minimax/util/dotdict.py create mode 100644 src/minimax/util/graph.py create mode 100644 src/minimax/util/loggers.py create mode 100644 src/minimax/util/parsnip.py create mode 100644 src/minimax/util/pytree.py create mode 100644 src/minimax/util/rl/__init__.py create mode 100644 src/minimax/util/rl/agent_pop.py create mode 100644 src/minimax/util/rl/agent_pop_heterogenous.py create mode 100644 src/minimax/util/rl/hl_gauss_transform.py create mode 100644 src/minimax/util/rl/plr.py create mode 100644 src/minimax/util/rl/rolling_stats.py create mode 100644 src/minimax/util/rl/rollout_storage.py create mode 100644 src/minimax/util/rl/rollout_storage_seperate.py create mode 100644 src/minimax/util/rl/training.py create mode 100644 src/minimax/util/rl/ued_scores.py create mode 100644 src/run_results_txt/al_all_xpid_against_population_in_all_69_layouts_out.txt create mode 100644 src/run_results_txt/eval_all_xpid_against_population_in_all_layouts_out.txt create mode 100644 src/run_results_txt/eval_xpid_all_cnn_lstm_out.txt create mode 100644 src/run_results_txt/eval_xpid_all_cnn_s5_out.txt create mode 100644 src/run_results_txt/eval_xpid_all_softmoe_out.txt create mode 100755 src/train_baseline_dr_lstm.sh create mode 100755 src/train_baseline_dr_s5.sh create mode 100755 src/train_baseline_dr_softmoe_lstm.sh create mode 100755 src/train_baseline_p_accel_lstm.sh create mode 100755 src/train_baseline_p_accel_s5.sh create mode 100755 src/train_baseline_p_accel_softmoe_lstm.sh create mode 100755 src/train_baseline_p_plr_lstm.sh create mode 100755 src/train_baseline_p_plr_s5.sh create mode 100755 src/train_baseline_p_plr_softmoe_lstm.sh create mode 100755 src/train_baseline_pop_paired_lstm.sh create mode 100755 src/train_baseline_pop_paired_s5.sh create mode 100755 src/train_baseline_pop_paired_softmoe_lstm.sh create mode 100755 src/train_baselines_lstm6x9.sh create mode 100755 src/train_baselines_s56x9.sh create mode 100755 src/train_baselines_softmoe_lstm6x9.sh create mode 100755 src/train_maze.sh create mode 100755 src/train_maze_s5.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ca27637 --- /dev/null +++ b/.gitignore @@ -0,0 +1,28 @@ +src/overcooked_teacher_layout_imgs + +*~ +.venv +venv +env +!.gitkeep +tmp +.DS_Store +.idea +*.log +*.map +*.pyc +*.h5 +__pycache__/ +.pytest_cache +dist/ +**/data/ +**/logs/ +**/results/ +**/images/ +**/wandb/ +**/figures/ +**/config/wandb.json +!docs/images +src/*.egg-info +**/.ipynb_checkpoints/ +!src/minimax/config diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..6a3f28b --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,4 @@ +{ + "editor.codeActionsOnSave": {}, + "git.ignoreLimitWarning": true +} \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..962fee0 --- /dev/null +++ b/LICENSE @@ -0,0 +1,203 @@ + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..3cb55f7 --- /dev/null +++ b/README.md @@ -0,0 +1,496 @@ +

The Overcooked Generalisation Challenge

+ +

+ + +This repository houses the Overcooked generalisation challange, a novel cooperative UED environment that explores the effect of generalisation on cooperative agents with a focus on zero-shot cooperation. +We built this work on top of [minimax](https://github.com/facebookresearch/minimax) (original README included below) and are inspired by many of their implementation details. + +We require Python to be above 3.9 and below 3.12, we use 3.10.12. +To install this research code use `pip install -r requirements.txt`. + +## Structure + +Our project inlcudes the following major components: + +- Overcooked UED +- Multi-Agent UED Runners +- Scripts for training and evaluations +- Holdout populations for evaluation (accesible [here](https://drive.google.com/drive/folders/11fxdhrRCSTmB7BvfqMGqdIhvJUDv_0zP?usp=share_link)) + +We highlight our additions to minimax below often with additional comments. +We choose minimax as the basis as it is tested and intended for this use case. +The project is structured as follows: + +``` +docs/ + envs/ + ... + overcooked.md (<- We document OvercookedUED here) + images/ + ... +examples/* +src/ + config/ + configs/ + maze/* + overcooked/* (<- Our configurations for all runs in the paper) + minimax/ + agents/ + ... + mappo.py (<- Our MAPPO interface for training) + config/* (<- logic related to configs, and getting commands, OvercookedUED included) + envs/ + ... + overcooked_proc/ (<- home of overcooked procedual content generation for UED) + ... + overcooked_mutators.py (<- For ACCEL) + overcooked_ood.py (<- Testing layouts (can be extended!)) + overcooked_ued.py (<- UED interface) + overcooked.py (<- Overcooked capable of being run in parallel across layouts) + models/ + ... + overcooked/ + ... + models.py (<- Models we use in the paper are defined here) + runners/* + runners_ma/* (<- multi-agent runners for Overcooked UED and potentially others) + tests/* + utils/* + arguments.py + count_params.py + evaluate_against_baseline.py + evaluate_against_population.py + evaluate_baseline_against_population.py + evaluate_from_pckl.py + evaluate.py + extract_fcp.py + train.py (<- minimax starting point, also for our work) + populations/ + fcp/* (see below) + baseline_train__${what} (Trains multiple self play agents across seeds) + eval_xpid_${what} (Evals populations, stay and random agents) + eval_xpid.sh (Evals a run based on its XPID) + extract_fcp.sh (Extracts FCP checkpoint from self-play agents) + make_cmd.sh (Extended with our work) + train_baseline_${method}_${architecture}.sh (Trains all methods in the paper) + train_maze_s5.sh + train_maze.sh +``` + +## Overcooked UED +We provide a detailed explanation of the environment in the paper. +OvercookedUED provides interfaces to both edit-based, generator-based and curator-based DCD methods. +For an overview see the figure above. + +## Mutli-Agent UED Runners +Multi-Agent runners are placed under `src/minimax/runners_ma`. +They extend the minimax runners by support for multiple agents, i.e. by carrying around hidden states etc. +Note: Our current implementation only features two agents. + +## Scripts + +Reproducability is important to us. +We thus store all important script in this repository that produce the policies discussed in the paper. +To generate a command, please use `make_cmd.sh` like so by specifying `overcooked` and the config file name: + +```bash +> ./make_cmd.sh overcooked baseline_dr_softmoe_lstm +python -m train \ +--seed=1 \ +--agent_rl_algo=ppo \ +--n_total_updates=30000 \ +--train_runner=dr \ +--n_devices=1 \ +--student_model_name=default_student_actor_cnn \ +--student_critic_model_name=default_student_critic_cnn \ +--env_name=Overcooked \ +--is_multi_agent=True \ +--verbose=False \ +--log_dir=~/logs/minimax \ +--log_interval=10 \ +--from_last_checkpoint=False \ +... +``` + +They are named `train_baseline_${method}_${architecture}.sh` and can be found in `src`. +`${method}` specifies the DCD method and can be from {`p_accel`, `dr`, `pop_paired`, `p_plr`} which correspond to parallel ACCEL (https://arxiv.org/abs/2203.01302 & https://arxiv.org/abs/2311.12716), domain randimisation (https://arxiv.org/abs/1703.06907), population paired (https://arxiv.org/abs/2012.02096) and parallel PLR (https://arxiv.org/abs/2010.03934 & https://arxiv.org/abs/2311.12716). +`${architecture}` on the other hand corresponds to the neural network architechture employed and can be from {`lstm`, `s5`, `softmoe`}. +To use them, please set the environment variable `${WANDB_ENTITY}` to your wandb user name or specify `wandb_mode=offline`. +The scripts can be called like this: + +```bash +./train_baseline_p_plr_s5.sh $device $seed +``` + +The scripts run `src/minimax/train.py` and store their results to the configured locations (see the config jsons and the `--log_dir` flag) but usually somewhere in your home directory `~/logs/`. +There are 12 train scripts and helper scripts that run multiple variations of these after the other, i.e. like in `train_baselines_s56x9.sh` that trains all 4 DCD methods with an S5 policy: + +```bash +DEFAULTVALUE=4 +DEFAULTSEED=1 +device="${1:-$DEFAULTVALUE}" +seed="${2:-$DEFAULTSEED}" +echo "Using device ${device} and seed ${seed}" + +./train_baseline_p_plr_s5.sh $device $seed +./train_baseline_p_accel_s5.sh $device $seed +./train_baseline_pop_paired_s5.sh $device $seed +./train_baseline_dr_s5.sh $device $seed +``` + +Evaluation is performed via scripts starting with `eval`. +One can evaluate against scripted agents `eval_stay_against_population.sh` and random ones via `eval_random_against_population.sh`. +To evaluate against a population using a trained agent use `eval_xpid_against_population.sh` with device 4 and the agents XPID `YOUR_XPID` you can use `./eval_xpid_against_population.sh 4 YOUR_XPID`. + +## Holdout populations for evaluation + +The populations can be accessed here: https://drive.google.com/drive/folders/11fxdhrRCSTmB7BvfqMGqdIhvJUDv_0zP?usp=share_link. +They need to be placed under `src/populations` to work with the provided scripts. +Alternatively -- if desired -- populations can be obtained by running `src/baseline_train__all.sh` or alternatively by using `src/baseline_train__8_seeds.sh` for the desired layout, i.e. via: + +```bash +./baseline_train__8_seeds.sh $device coord_ring_6_9 +``` + +We exclude the detailed calls here as they are too verbose. +The resulting directory structure for inlcuding the poppulations should look like the following: + +```txt +src/ + minimax + ... + populations/ + fcp/ + Overcooked-AsymmAdvantages6_9/ + 1/ + high.pkl + low.pkl + meta.json + mid.pkl + xpid.txt + 2/* + ... + 8/* + population.json + Overcooked-CoordRing6_9/* + Overcooked-CounterCircuit6_9/* + Overcooked-CrampedRoom6_9/* + Overcooked-ForcedCoord6_9/* +``` + +To work with these populations meta files point to the correct scripts. +These are included in the downloadable zip, called `population.json` (see above) and should look like this: + +```json +{ + "population_size": 24, + "1": "populations/fcp/Overcooked-AsymmAdvantages6_9/1/low.pkl", + "2": "populations/fcp/Overcooked-AsymmAdvantages6_9/1/mid.pkl", + ... + "24": "populations/fcp/Overcooked-AsymmAdvantages6_9/8/high.pkl", + "1_meta": "populations/fcp/Overcooked-AsymmAdvantages6_9/1/meta.json", + "2_meta": "populations/fcp/Overcooked-AsymmAdvantages6_9/1/meta.json", + ... + "24_meta": "populations/fcp/Overcooked-AsymmAdvantages6_9/8/meta.json" +} +``` + +They help our evaluation to keep track of the correct files to use. + +To check whether they work correctly use something along the lines of (compare the eval scripts): + +```bash +DEFAULTVALUE=4 +device="${1:-$DEFAULTVALUE}" + +for env in "Overcooked-CoordRing6_9" "Overcooked-ForcedCoord6_9" "Overcooked-CounterCircuit6_9" "Overcooked-AsymmAdvantages6_9" "Overcooked-CrampedRoom6_9"; +do + CUDA_VISIBLE_DEVICES=${device} LD_LIBRARY_PATH="" nice -n 5 python3 -m minimax.evaluate_baseline_against_population \ + --env_names=${env} \ + --population_json="populations/fcp/${env}/population.json" \ + --n_episodes=100 \ + --is_random=True +done +``` + +## Credit the minimax authors + +For attribution in academic contexts please also cite the original work on minimax: +``` +@article{jiang2023minimax, + title={minimax: Efficient Baselines for Autocurricula in JAX}, + author={Jiang, Minqi and Dennis, Michael and Grefenstette, Edward and Rocktäschel, Tim}, + booktitle={Agent Learning in Open-Endedness Workshop at NeurIPS}, + year={2023} +} +``` + +The original readme is included below. + +
+ +
+ +
+ +
+ +

Original Minimax Readme

+ +

+ +

Efficient baselines for autocurricula in JAX

+ +

+ + + + +

+ +## Contents +- [Why `minimax`?](#-why-minimax) + - [Hardware-accelerated baselines](#-hardware-accelerated-baselines) +- [Install](#%EF%B8%8F-install) +- [Quick start](#-quick-start) +- [Dive deeper](#-dive-deeper) + - [Training](#training) + - [Logging](#logging) + - [Checkpointing](#checkpointing) + - [Evaluating](#evaluating) +- [Environments](#%EF%B8%8F-environments) + - [Supported environments](#supported-environments) + - [Adding environments](#adding-environments) +- [Agents](#-agents) +- [Roadmap](#-roadmap) +- [License](#-license) +- [Citation](#-citation) + +## 🐢 Why `minimax`? + +Unsupervised Environment Design (UED) is a promising approach to generating autocurricula for training robust deep reinforcement learning (RL) agents. However, existing implementations of common baselines require excessive amounts of compute. In some cases, experiments can require more than a week to complete using V100 GPUs. **This long turn-around slows the rate of research progress in autocuriculum methods**. `minimax` provides fast, [JAX-based](https://github.com/google/jax) implementations of key UED baselines, which are based on the concept of _minimax_ regret. By making use of fully-tensorized environment implementations, `minimax` baselines are fully-jittable and thus take full advantage of the hardware acceleration offered by JAX. In timing studies done on V100 GPUs and Xeon E5-2698 v4 CPUs, we find `minimax` baselines can run **over 100x faster than previous reference implementations**, like those in [facebookresearch/dcd](https://github.com/facebookresearch/dcd). + +All autocurriculum algorithms implemented in `minimax` also support multi-device training, which can be activated through a [single command line flag](#multi-device-training). Using multiple devices for training can lead to further speed ups and allows scaling these autocurriculum methods to much larger batch sizes. + + + + Shows Anuraghazra's GitHub Stats. + + +### 🐇 Hardware-accelerated baselines + +`minimax` includes JAX-based implementations of + +- [Domain Randomization (DR)](https://arxiv.org/abs/1703.06907) + +- [Minimax adversary](https://arxiv.org/abs/2012.02096) + +- [PAIRED](https://arxiv.org/abs/2012.02096) + +- [Population PAIRED](https://arxiv.org/abs/2012.02096) + +- [Prioritized Level Replay (PLR)](https://arxiv.org/abs/2010.03934) + +- [Robust Prioritized Level Replay (PLR$`^{\perp}`$)](https://arxiv.org/abs/2110.02439) + +- [ACCEL](https://arxiv.org/abs/2203.01302) + +Additionally, `minimax` includes two new variants of PLR and ACCEL that further reduce wall time by better leveraging the massive degree of environment parallelism enabled by JAX: + +- Parallel PLR (PLR$`^{||}`$) + +- Parallel ACCEL (ACCEL$`^{||}`$) + +In brief, these two new algorithms collect rollouts for new level evaluation, level replay, and, in the case of Parallel ACCEL, mutation evaluation, all in parallel (i.e. rather than sequentially, as done by Robust PLR and ACCEL). As a simple example for why this parallelization improves wall time, consider how Robust PLR with replay probability of `0.5` would require approximately 2x as many rollouts in order to reach the same number of RL updates as a method like DR, because updates are only performed on rollouts based on level replay. Parallelizing level replay rollouts alongside new level evaluation rollouts by using 2x the environment parallelism reduces the total number of parallel rollouts to equal the total number of updates desired, thereby matching the 1:1 rollout to update ratio of DR. The diagram below summarizes this difference. + +![Parallel DCD overview](docs/images/parallel_dcd_overview.png) + +`minimax` includes a fully-tensorized implementation of a maze environment that we call [`AMaze`](docs/envs/maze.md). This environment exactly reproduces the MiniGrid-based mazes used in previous UED studies in terms of dynamics, reward function, observation space, and action space, while running many orders of magnitude faster in wall time, with increasing environment parallelism. + + +## 🛠️ Install + +1. Use a virtual environment manager like `conda` or `mamba` to create a new environment for your project: + +```bash +conda create -n minimax +conda activate minimax +``` + +2. Install `minimax` via either `pip install minimax-lib` or `pip install ued`. + +3. That's it! + +⚠️ Note that to enable hardware acceleration on GPU, you will need to make sure to install the latest version of `jax>=0.4.19` and `jaxlib>=0.4.19` that is compatible with your CUDA driver (requires minimum CUDA version of `11.8`). See [the official JAX installation guide](https://jax.readthedocs.io/en/latest/installation.html#pip-installation-gpu-cuda-installed-via-pip-easier) for detailed instructions. + +## 🏁 Quick start + +The easiest way to get started is to play with the Python notebooks in the [examples folder](examples) of this repository. We also host Colab versions of these notebooks: + +- DR [[IPython](examples/dr.ipynb), [Colab](https://colab.research.google.com/drive/1HhgQgcbt77uEtKnV1uSzDsWEMlqknEAM)] + +- PAIRED [[IPython](examples/paired.ipynb), [Colab](https://colab.research.google.com/drive/1NjMNbQ4dgn8f5rt154JKDnXmQ1yV0GbT?usp=drive_link)] + +- PLR and ACCEL*: [[IPython](examples/plr.ipynb), [Colab](https://colab.research.google.com/drive/1XqVRgcIXiMDrznMIQH7wEXjGZUdCYoG9?usp=drive_link)] + +*Depending on how the top-level flags are set, this notebook runs PLR, Robust PLR, Parallel PLR, ACCEL, or Parallel ACCEL. + +`minimax` comes with high-performing hyperparameter configurations for several algorithms, including domain randomization (DR), PAIRED, PLR, and ACCEL for 60-block mazes. You can train using these settings by first creating the training command for executing `minimax.train` using the convenience script [`minimax.config.make_cmd`](docs/make_cmd.md): + +`python -m minimax.config.make_cmd --config maze/[dr,paired,plr,accel] | pbcopy`, + +followed by pasting and executing the resulting command into the command line. + +[See the docs](docs/make_cmd.md) for `minimax.config.make_cmd` to learn more about how to use this script to generate training commands from JSON configurations. You can browse the available JSON configurations for various autocurriculum methods in the [configs folder](config/configs). + +Note that when logging and checkpointing are enabled, the main `minimax.train` script outputs this data as `logs.csv` and `checkpoint.pkl` respectively in an experiment directory located at `/`, where `log_dir` and `xpid` are arguments specified in the command. You can then evaluate the checkpoint by using `minimax.evaluate`: + +```bash +python -m minimax.evaluate \ +--seed 1 \ +--log_dir \ +--xpid_prefix \ +--env_names \ +--n_episodes \ +--results_path \ +--results_fname +``` + +Some behaviors of `minimax.evaluate` to be aware of: +- This command will search `log_dir` for all experiment directories with names matching `xpid_prefix` and evaluate the checkpoint named `.pkl`. +- `minimax.evaluate` assumes xpid values end with a unique index, so that they match the regex `.*_[0-9]+$`. +- The results will be averaged over all such checkpoints (at most one checkpoint per matching experiment folder). Using the `--xpid_prefix` argument can be useful for evaluating corresponding to the same experimental configuration with different training seeds (and thus share an xpid prefix, e.g. , , ). + +If you would like to evaluate a checkpoint for only a single experiment, specify the full experiment directory name using `--xpid` instead of using `--xpid_prefix`. + + +## All command-line arguments +| Argument | Description | +| ----------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| `seed` | Random seed for evaluation | +| `log_dir` | Directory containing experiment folders | +| `xpid` | Name of experiment folder, i.e. the experiment ID | +| `xpid_prefix` | Evaluate and average results over checkpoints for experiments with experiment IDs matching this prefix (ignores `--xpid` if set) | +| `checkpoint_name` | Name of checkpoint to evaluate (in each matching experiment folder) | +| `env_names` | Number of devices over which to shard the environment batch dimension | +| `n_episodes` | Number of students in the autocurriculum | +| `agent_idxs` | Indices of student agents to evaluate (csv of indices or `*` for all indices) | +| `results_path` | Number of parallel environments | +| `results_fname` | Number of parallel trials per environment (environment) | +| `render_mode` | If set, renders the evaluation episode. Requires disabling JIT. Use `'ipython'` if rendering inside an IPython notebook. | diff --git a/docs/images/OvercookedDCD.png b/docs/images/OvercookedDCD.png new file mode 100644 index 0000000000000000000000000000000000000000..89099ea4206767afd0cc58ba202c7ca21251ef97 GIT binary patch literal 160105 zcmd43Wmr^Q+dm8lh=2-+(jkpBL$^}WAX1Xj-Q9>FjWk0lNJvTz9ZI)IH-a$0Fd#WJ z|Bd%^-Nte2<(Kz3-VZaZ&7QUQTI>9seXbR%tSEgQlNb{T3F*4bLy1R7NEm%cNGJyw z=)ixfGn3VTKgdpxq#q!a43Mq?7ZN6#GNuX&NcVs?1`_fM3nbLbp8y|X;Ddy896K!gcrUzM|1@L5oF_Fpyh;3-adiHcewC$~d*u$H6fDv~|9`*tazGI# z0f>3O`I(s+?>vmKS`8sPoI#l~O=dghRQs*(-OdPb?sIYAat#}nY2MH*9z(S6RwXcL zki##qTfJzIL9HkIPhGzM{MXi=_QG5^Ctp;(zYeiv+#QN1#+YD2Q1nNhwY4`!zqFaZX#8DE}6@~GKYHKb=)cbUryuYCV z@D?LAG3$!QbRu646AulH5`a(2*nRJpcLNO-m!Z&1W>!{_q@*Nz(T@+!JC&ur5#igH zn35R!SY>r}QW(^@bX@xu1}dG^m2ydT*N3Qs)_2}di27}mb&iD|ha3yX`4cs2r1Shk z!|%`drxjf0kMWWtL=4Opjs$#~(e1?shX`HkXRk^c^lg`ym#gyg^HU$N`M%%~ zl@tG`XU7|Vj6@C|9-gSJ)z$p9_4O1M-yYoT7E?!+Sdbq&+JzE?E`Wm;-G)EN5rhh+58Q0qdgFI?xD;*I-xKu0{1J+2Fh;D! z{~`IG8T@eBmDoaJ*YNb#+pia&wh(@xJK+gEu;l)gfvl{oXeK5m#Fv#7&vF-=70K_N zU)@9w>=IH{)7SS)pQy8F>hd+mTQX5b(7jI9%ah#B7-uo2@?F!@%tmb zwO;w5SdA4O-HK6Jv6FZuKd286YaX+?HOap`hmulaq$ghGO^O6Vu(=%WLE5sUf zWHyjiD`}jmMO89ZVUs^=J5T5P;%xnvl8^)C3DfGuZH(l#=G)Y`|JUhz_kUV7mMClu zUq37wupd^s>rt~=Ry`qML+jZ95l3zpLU350e_!VoB(%EvdMCwn0T(ATGibTHknQd1 z5UCkk1ZdH1>cVfdKrH}SF)86QYM*gXg3RyVNto7B=K`DgY=1c4{sLPm#QDG!mvpH# zKi}PHs?uV*JtFjde~;D=FwyZrF3ZV3`S@BK5k7B2&{Y_2Bbl0}2+b_jYxL|N)mPYJ z!ebwv4kZ3Pn`MGqhGWBFzSfl)n!fE0%_wD{J>lbm}u+k z>z0;QRumBuR3?u1UbL1mva#7Mi{BI;@d#4AdF+g0imuQZ`okce;Oa@(BYm8Vn1rui>q zkWr;l#k{<{x<9wR$i$yC?YWLFE`i#A+PV}idS0z@B62Occ4|@Q=J%of(h876fsGZ{ z*B2IAMNT|>`_Y^Wy`Z2#FNMqE(^LO{fC=;=>_?%}>~LIY2^b8Y4=RmAtT_va;!kr4 zzq1@IC@FNANA1&Lu_y`rj;&u>{!#|;sVb#VmSkVN=+0u-`m>onFlhV>6zXOktSs@J zcMKk5r3raB-uLl2Yu#}(+~(u(Jw04@oA*Hk6*>$9ueokg67b8zzhC-GiJ|s-cP?$8 zZb?u*#b;ct9?TTI;D#7{LQ?tu5<(D4adB~uVR&w7n_(GBc>5>HXI*yMwT{a})>m-Y z<7}R;NPi#EFD)_Zcc%R*ez)Fp&fif81_TLWr9Tysn43!V138Tp#ZGisUxMe(({ z6T-!IZd6bEYyo6x;!ec@*hw|1_V?`c5km+u<1~2U-87oTsG_b8@f|DD(&n1Q|Dg+E z1&VL;{M~1!P52n71(F{ll#Ek6%mI!isX%+sA`dYJ6Z}H!A1)(Xb_Yc-GWALkBc47|l6?iO4I;myTkl$Y=l)7Xq?D1HhsROaRdRO&Gadts^>TAz!w=9i87DX)P(hQb$!H!6>xtT2P*3|iENdn&Dr`3G1UG( zuGD@T#XP6U_0d8lY;GK6kT^?^_wO0f997SAEvFPxU40?JMbsHdf|%M{8?OE~ixx`M z0dDji1Su?KX6`l%=5xamWxh>zFt$ql=&IY4LX*sr4wRt3k01IDOcj`%q56MdUjW?+ zk)TQ>6Lj@v!N*h!eFYXs>a9B5?xe6I?lHk-2gRrK1%1ccFD-wmg?0jc?^hKSyW^n} zA09n^ytmolu{$^9tSW&NfQ%8+Lb!X}-!FF~ZEtS%W=GhGz^H+;;j-jGb~Q7rx7~JM z!>&(c9z}q0fSZU-dIdsreDTXcvwQhUl2mr=LtBcs`)2y3^%YIrd|LIs6%!&~1(ilm z<)?EKn=kshOFNg)!=GqGEBV^a?2ayxl35B5Z&PFn?~mHcm(fRSxoEG2g9@CU#7_Gi z8p5g5XUlfJ95B5t)qMPQMF9&_$rAd|&2EZgs@$D~hWSq5yd@8M*shJj=n~v4X`oxw zh=*ip${`+C%)lNcK&*ukVs=c8E2L^#F&R}iyJ@9S{A5(PX7bS>XCBYcvtQQb$4iZS z?`&u5)}x}LpvLdyeSOdKPLB1-!3o%~J$@e^^Ui~XbFd4XXZhKHNkQNR*n=lBm&u=v=SO|NfVoVHaz5o|pR zrzGw{V<$k#mZzkr&$C+T0}qBxhaG!2d5xFoo|uWdgroLhC(!KvqRW1`w3iWk6L}%S zN19>8MO<$-m~r-uFDj*2&^-pXCItm1YMIgu7wZ(?e(u7~_Oc2hJ7^|WiE$NQ6|uCCImkPPAqh8aGTKzs&P)+Tn_nVRwCHS%camcIMvpm_KW zI%xm)g(F9DxZ{l<u|TnuoR~Cd)EsF_l#QVsruKT(;g$a=p~s?hg>R)7ms6irDdRrUS;wGcFGsa~qFgfo z=Q_Dsu?`+BsXI?It01gzewX#w_UbSUH=IP?rT_h>l_-0U7xX$qW4GU~aW6c|X=LI& z+V5uxCvV8H&eIbGZ1azP0Uo<6jpa7e5ZRc12SYEflikHWw!Mpuk(jz><2N_5(f%;Y zV?1*!tCLbq=0vDH-vFSqp|edL!_i16$0#Hswi#Pk9kP3#a+pP(%{yYtQ?V$in=ih& z7BU`;K$&=LXQrm!IY`5d$}~SD3JuOJW!%2W$~JFXMQqsZGXhAGFBr@zU?=Yf;NiH@ z+a&R;i<4;3{St@ttdRn^BNjjSwjA3>yXp4>^;=KW8V@1~HdB?{ z(ov*G>XbO{_Oo>^ZbeV3ESFl$>fF(hL4gTNLcixkE)+wi+PmJnlgm4^%wB!kert3vx!=dnXSS^9P??Y1@2C-OQy2%u3|cx`Y=ukS zoL6tsg$n{a%BKruQ-!S$t+Asg^|R!w!8|)V#@?NLw6HUA)VFwDWwjZfhSd!(C%>cR zd_W2-5c0dLIW$5}JiZyJ-ez* z0~yn1<_Vk==(N$j9fbc5g;(xm^heG)Zi=9#>Wrd>BgbzOnxvYsOD?a&FPv z3fjBHa~6|D<7rZjDZN-rv1ys~ z!WzOU5wh+>d#BC)@sTV2N-3qym1yyvPX@Q^_>W50(E3P4hINm3>La6FT%;d8vlbGO zw6ML&L_c^^;^$>m=(pBzdj00?qo(Zfwv?W{6G7;v$dpTIu1rjTB$dI6J|M}X%=AjV zq%>p5Qg$DgXYnQ6n#pf;aicCZNfL}V(<%o#^XJ@MkgPEDOMTfq^P z;<5cGXetUQb~=)3dc;O93}G>CqTC*Cr(d%+ISxC2%%&+NsGgWz_S~sABTD*wxpr8o zdxN(747~1m8m%vaFtK?jpSx{Fe2R9l*9+1o2oQC9gWmfl&B7coJn-&|&bynaE`5sE zQwU5hJ(X@2*`=q_G>tJ=yp}kG-KewYEZy07a2`}zJnjDZxHr-Qb23A;-$XUV@BH|v ze*c*XHae~E_9&k-7H1s@ZX_2^hu?f7`F7Q5P zzK^*7iwE<=WxyPY)p!ZZt+j=V$B-VsnxqT=c`sCMz#<80nZ!b#y_t}w;Jiur(3NOE z1u>^wK4{E^I$Gx$(Xn{oLto$d*2N=DXrSceJ|YG*J6yF?X;Mq3s^R9`08vPk!({FZ z+hO*#Ibj>iY3%{o~3gd$>}c{H?t3(bTdKE zD`#(IjAU+4ab%LOdM_;bki}KmSZbO2dYtzMU)+hJnX#NkiVC*5DoAk(UoizINFe}3 zyRa1uGdv4UBo!WRe2^zsyPfhO$xi=~GhZ#6_Eu1&wzgUGwnMM#kokug@BR_b4_~{R z2G}Y?7Rp9Mo}e??FL9&2*c-U5f0h5ey|q|?o#*byU?da*w1UFIn|$d5MpUiY!_eI` zV|Lan$4`ZlvF!h3yl4T=zYiOEv>@4y&Tg4FIx^CJL{H;gdu~*rbMvw}moUsC8&9|o zfM1Ye{o7Vpv^q_+jrH{qtL3q`a9);sQ$7oi5$Tq?!eiCXja&NAt4f4z1t;#FJmSpj zOOLIW6I8TDR&#$dR;>eA{w6%cd94$1vmF#6No}{@P8u5r-@#3GSou8N!)8aFlac9- z9h~JMJ>c$AnjoY;=67;`Nf(U593dsnrTa2k2X-8tl9|Taes+_;WDYryO0F+G=HX0^ zo5JMEJMa|C9r1V51Wd#AX8bF|K4gfwy8J_fsNQ6}h-CQD^8SgZcTJ(qRNhsArdPbS z@AqbS$#qm9pvxsk19PlppOGOLWT}?Vk{i$!W?vzQyIIh=je>2qs}QN1kYokMt;nW3 zr)|Qw?P!|yPXzLs&uGzjnkRZ#sLEBfJTv3ibVNsyvLNF zzLqyvn}Y_q@7J~Ayn3}Sgfc~MyntxSdT_8TO<|z#6FfSxmzi1Zc2vois?dcO@$lq6 zI-{?N?LpF&M*2RbD!dhaUu2DRJT&Iiw{0DoxG7Jr zyiWF5fBaGIn=zgO`TyHNN}2v~nckSNV7=Wl479#B8(%(Vy=)UghSLG@>ZjD8$^a|c|%dzgI0eNy&y~urQ zG-71()~2~D15XX6eE=Eff&`luy6p2cwKZ*}Mn7)MMepr8(5dF%iHTR>pgZ-}Qu9H>tTjw$>G(zu=wP9o=~=pkG>C+18Lboa**@Z)U#RW7L4i zqp9-vv_Hp-yxHqC8qf-~gbz)7EzGa4zK9piiQuPppzK{72XQ zCoipqHUQA?we9-lzB1~P@A`7(`blxb7BErU4orhJnV-sKJTR4xK_=;jpA|#-SjZd8 z-aoW_*Sdp-cCx%5+d0m2sET6hO13ROGTmeTK4M0AYF8<3S7~f)bVs~8vGH|cTy*rT z+TG(vfkcYqN4a#-qlMuuafkg_yZXn0C--<6C?L)_wz9Tw6TPH6*+TEyG(+%&#gPrL8l>{5Z^y{~J$rzF4IAMJ& zv+(hJsEL>FzQ9@()zZuTwebN_+AGPiogs*|7`vi^|zBuf|cK6VKA3gi(1vrq$WHGvQgO*ie;my{<2PfpR0HrjMvxky#-V+LI-?D&) z(BtJE47?Mc=6PApM(D5>S*;b(2%(QPGB!|@dCk3SRaZ;Rsb+n11XfXXme&A^gC^X+ zJ3h#?k>DU}z5N(a-6p(MkE5M(XH5&uuv&C4o#o}^` zJ-Jms8jly@yj>K2p<7mC9u6r<)E1;lg7zJpu5%BQ`JMJmj*{y;dpr8wFInE85Ph=W zpl19Xb5>B+`bEr~f|WgZPGoYfvi+v;+lI{7#)mHpX~hX!67f0YeKwxUW$r9_Q) zdUT<*4U9oai3S9&J)e!&*Vj|{?vHA)T{Nu9b2d|S6Iz>rA$4jsPq}qoMDBS z6x8|Zy(g+_Y+_7~l{DFLAJ6!L6tpSEMY}7_mtucbny2Kenl86R7}n-xBTm!#{6pEP zyAB79KK0RQcS1>aw@vr{@rqba-50{LoR3Fe%BGHY&?M=r_P;o)YQLeWRTyImgGYM8 zryy@{rS}tnU^Jd+JI=?x6zp;Eqbk@C6U<~VW{H{3NZHs~=Gx(>ftX^%gL-CUwU~#niH5K^!JIeh*ZxzRW~KVtXvc4d=IpFwm_3yMTc>TW=cT@#~+M*vegU z1YZEcn6N=iOG``F#s)4^rn??*z|PC(_yFcJ?b$wwI zR67=NAEo^$Q3Rw_=_2WT#Z*Tzu0&4XO7&z~;6a_=tiJk)o9lIOK!+uVduxnDi9n{% z@##>nlc|?4+~w*c+f57wz)9QPs83ADJ*#~QETCDnZ;QNSS@9qupo@4!07y&+>s?Fc z{6xFhcqNPT#=u99-o?}OTRmxY+9f7@EQPv`g`hHB4cT|8*r-fw5YGxv2Rdm3Y%KiS zx8d=~AgcT`mG>V@{lVvcH>@M#I(DiRW6vdFSAAk`GcKraI-Ad2MJB&lXmvwDW7?H;vqndKT4F3Lcv5vn z!BUr8cvdA9J8B4`Yr&5vlYE>)J!mo1pOGzGI$UVCUJiZ)DfpBzt}#gwwvXwm(h05I z2Oa01t*fB+y(tnVRnZxe(>4U;q6=9dzi(yrt}hY$Yd2fKQ<)W+9p}7Teeg*;9KzCR zF>)sGF={B0#WdUYojN6cm1U>Nz0Rd;;d;%(F?9+lb2Xo5AZG^9FA@)Aj)<#LuFv|| z$nd0Xg;(8WpSeskTm>>$nnJmyS{0_geq(Yi@_zAzzt1N9NB4jw3J`Vcz*?#QynOqj;zz(>w<;f925@pD1dc4S58Lypzmt+huIoLOkm3LCf^<2 zuk_(ptguCr(l&cc4+}o%3tv{IiQyc0%I%uc-%&#fcq6JB@g?mqA8z8l*9u~5@_8zG zXHyE`I6q)fv}yVlg%g9V>m zuZi&#dIU?&tv+}rBsaCi>$zkhjua4ZErjym-85Xpeyu1%o4f!7lj1E%2UN`ue%;Rj zy;s#@%y+yV3t_d1_Eblb@RmT_*Z?1|=25`pD1RP=Onz1}x0fX2>n0=-T&=I+XiPai zQC_|4+PE!wlsIg7l9z#m-FQw?(k;5*(aDlwPm@u&#%nD)!hbekBI%VX_(7E$9`^oK zo1KpGJB?|&-W_7PxV!k_wANEi4zv()zhn1(tSLyLE^A)}WKFkXngXV=5I9#`Ui{|c zl=p_UrM#q^7oWcKc%GmVB-{W~jt0+-Hrk(UDjZW&m2h|-&kEkDxh8Z4{ifdqLKi>t3|iyxUtLPiM0LFIxtqx~&!h&~mX zR}Zt?+9j}T^sy8Aj)yDyxy&>7M#zUmiNcN`5nx~*0X z15SrHJ>|1WH?PJyNkgmL!EmkSsqsGJhmMrE9n)VGs(a7oX=IX$k@(gE@tl5bUnAPh1iOJFES`*N;i=M2ZQlCXup9Y- zqU;TS#(|2A6y=pYn&>1KFd1!aV=)uw-CDTixa7wdwk`6ya_=~YpPZkyPbFOJg%9fw ziTHgrgfF+mgi(1_%pFxl2Hb0OwV5uVo(r9-dotjp%{mB%AB|e&3f_TEEBc<4$qdf= zIZbW~RUdip@PUPwm)%G=H%pba`gkbPXi3JJMx1*N_GNZ zlNHgibRRw-8o75|W#bj?(b0S)GrUZB!zQz9?{nitZfyD#PvcJPD6;sy#Z@H_ zQ@ZF(ZhkX2$J8fmjPKG_de?_2B5tPx@w2xsjZ2U#>2a5V)gckL@RLs@g2|8q%ol(b zFfveQCmrS%aW1bfI(T3p<0NffcR+Nblhy3w+qcdk3y~x6FvSc68+;?mbmkno8R=yHl z>~Gf?KGb{hME^pickviN4`@|LV?ke?M0BhJ`*+G`+JZR;nNi;tK9poHeV^-8?8y!@+%{ zxQVQMc`}kn+O#~_soZFt5D;25ZeM(MWs1k9)--GP`9yBcIDgjs*YJI4=3zK}q2Ikg zw#p@wqXdOWIg!cvGR|6`9rw<$pkXI{Ia>;OU1Q7bI?`NIeMn!aIm zCatiWI^7DYgm#tp4LCqu+LHp4%7y)Q_Tyl-c}F{Q(!DEpMtsXlm&C;7%x!mS6sD)i zYiw=LFNV-%0q20-%F#Bk|I-LrSljHB%i-zJC;dhhHyFi?My<>=;Plr3)>;X&iA^lD zSo3j%9rWoS^wzZ;4M%ja%Nmg8Y_hCv-suDt=rAG+>FKu}XD1E%WI?Pmuj;=D?oy6!D|Z0auMpff*Mk4Y$Xqq1B(F(SDENer{XS7CslpW+Ij59ahbi}$I{vFpc$4;i0?E3u`D87 zu3Hr0=rbS`4RJAa!kM}^tE0RkwjTG6#63LDZrB+w-WLsntX`-2TY|3g=VS6T` zqhRy;8eI3YY)oSg->g9U9RU|05MIZv>*wgz?+)Gcp>IH#PaADNSJS4@FceK*D%ZJ@ z_3@y4$7cKv*|D*mbZXzdY&dLBs^I)Ar$TAYO?3@@`@PR#qWF&Z;k@t12IujZfkBJT zg5{v$-GjHwOu{{4$hcp(59+72Gvza2=EZ(T!_E8znWDm5A3v<|yReYqLcIi^5Ri&O zn>jp++O7BIt169OK+ay@UYa}qjFnd!GkwcqdQcBT=cKf`fThOb{^wk+xb%J7dRoWB zh-&YX&jVIhQg^H;N_`y>? zZBBXaY-W9j+3C_ONw+7x+mFWLSjaxdV+(K^K*95AC#q#Z>bfdTIVF`J$7oPwc2cLs z6UgRPp96XBN6;a17E|$sm7}0512YH5SNrvc2S6Wp5X~I$(qqgFEFgd7ckUN`x+?}% z?_^5>BRWPxOVq0dfa;8LCdy}1O6h`bir3HsoyLl_&2)hp(5cKpplk(EW$@czO@Imw z<@uw1bq_(Hu8>}O$i{&Q16wXYyOc#_Y=(UD-I9IYW+G-;jp5p!1YLLD;n!w;gh3Ud zu)c@=2ev)Vx;Lo{D?yT8K-^(AOB4>_amq6uh`3wvmN`A?V$5rLTAf~B8uh5}^vG^t zuu?NZB)l63BX|*Q`}ujIs@n2FdT$9MU%F6d!U&KobhJNtJ>VJ5mQp#Y`oaE7#FqMz z_qGjfA}MxO<4S`?Z7Sw&F|y6{b58B?zV(^@7m8lC!;oG_D>L&Rt1W#TCEB~?b|DgC z#|Wm*9&EmwENlwh1Atw(Ty=7Sb6*WUT%bb%Lpu#INSKCcKdT?LemLpPCRCGUJ^g-o zmba^jv)d{uKz8TocqOBz=*uSZ_?G6Aow0B`Uiq=!MVe-wY2g^lqm8Zcl@CN7c3Q>3 z6aEC8eM$^$<>cNQC1KIQN1R&ZDA}@RJtE*5a)>NpA8Orh{))R-uu!qDL}%n^$czR#JkJOP-G zmD8a)f^I_*Gj$}TjPA6$7h+R|_eceSO!?4Ok94@Icv_cC_M?6RnZ}D{6Li zG#yrr!L4;F!8f)Lp3xwGBDH7Kzh5sR)HzRBF=ouu+>ZwA&%?sQNkG^+Sjecs$_rNd z$u;+_=PMM^m9Sl()LtJRrzkmo|6#3mKM-tWn&9PDw@a5Oi0Bauaz2|n$P>}8-CmKD zWD8@-6Ph{@>A6;x-n(+KeKqa^bXm5-;i^1a&CD#Y-37lb>Z++gb_FbkikE)ALOnXz z%~H8H5C=OuFd7w2Joh{UPs!7XuQsL1Wz;(Oetzq`1Zbf;*m_kjw%~5wazo*`TN7i= zf3hP!fMhAseC&MsTD$jL4&BkBG$}W&#?^HWwj>8ThYWhW3Jm~{K%33}yfKX<&%qR4>aTj%p>5DZ3M`;j;$&cZf)a#f#Q z*lmfrI?2b7ix}i9%G^_N*u49=->>0#E-2?D+HIa3(A40S#QXSEOI-t3Z5e&c9)K$b zTe7SNGcAg{eQwS^Y1)5S><4DUVUgLKKD2;c8SfuB2{`acE=GLV-^CojsHZsHaf4>A zew8S=48Zb)mz|6Oi#*>{dH6Mai)$vIH&@gKG5fgA>~r7Dtx_Ayx?OKM`;xx+k#Ubj z?x>JH3HnW)C*=<3lz|h+bseMYNn09a=_%5v9nQ;q3dQ5{gR>Rh_uq#$HL;eA9oO6d z3d||l*h-I1k9NZ_Tr-oqDdBqIV8yIIRC$)sy>o?6HRIg}&AZjldX+RDOc8WUojEmQ zU=sw0F__MIjexc%YB#&n=giA6TI_?Vuf32leub;WjY1_S>n8cO*miDYMwJTbjZICF zwjEh<`qR#Br>32{Tmug|>3p4;7rLnaTwyGZ#CdA9Z1T)*&gJbj8RWclg`ha`*# zoZYqQu5oUk_@G+2HBOxKHaARD=ymlWP$jq7*XDy`Uax_WxYxpSx5e5HF*SBlr&_X= zlI=y9(NR`bcA&>~(5#gZIqwGFm$!HU2BYW7ePr0}v#;$xSn!ZMa4mcftxTnQi#>8772aFPfQ3)?7EFaFg3T&krnfx(IOcT%J!8P|QY z7Mg)L@-A$5mEY=17u0Z@j^qe@ANfO+mpuZF$8PRa1K1c*`sU4-6kzMa=D2o&N@>g= zYWaZ_2<5qBlN6*zhqadW*3BOsPr_Ujp44Kf+rdMca`sugYsoV7;(yQ##6VP8Ktje#{H7 z_n}I($PaF9gi2&F&wj%Aw-F(4kH2-*s+xx1RR95=^7kPTnxdFi)I!}{w*?yn|NTKp zsDPyc#gUA2@4!YE#|8pIKRl`lg)UC!x}#f;HZ3w}&Di|+)x8U`Vw5-x@xcC>{PR{p zo}XqWkH&=zGEOGSEyeA9@O|liZV@7*uqH9-v&Q*@alaatN&I&l0UT_v(EDicu9*p7 zbN0Q7drkIR&ljl5j(PYT8`MDZ<0S5)+zHR22N?^I(;JvRZ!l%P^}7cc5MqWjD_8ZJ z5Mc+evg?q`{WRK0|HDRM*SD5KT!?$IPjd;b-u~|xfA_%uOfg`HRjj4Y+%KqB^ly9U zp`iBdEx!6wJ?KwMJcWjlOaw1wNBmyJ{H%LE%K7_)^++H8uu$LW{y$n$^+?E|?6cb| z=YQM7^BRz>2@k~l9q0c%9{(0|0!jnZ%H|I<6o1)6j13vZ)YC!v4`muZ(S;BWENaAo zM)JvjH|_k(TvlntTIAVo-1~8{A60+s=P)HdzID5JlFN+@C9z0E zrX_e>Nq>zRauh&xk(XX6{2yHLAKDO7qo{AQ?b1rNZke{KNLpB^;%DDixhJbNFNXG& zBwurDyXjSyvnwN}K1HMMu{O!3Z>x_F^)jrwdFyJ&w`JZ_^2Y|9IT_;rCG=YuZ~^&O4L0Z5rT!8(t@^KCy;`{O zqSc}EtNvP-_ERUP-Fb=Zp@lAOkM*!__BhK2SbsUU7KS^(YT!55%=}^7(+?P@^cO1t zCij-E@)s!o_A_`O^bjQ7|>{fVedIjPp1SH64^${F^qWh9Rt5A_FA z=-Co>ph1j@JJ@b~-*ove?;`XB$QC>oPxPb3_9xH>mjSqFJ#2q!@e^ErQB^-)in9T# z{bMTUAKc`h9@WBriB-a8KIXq8#HAZ}soOnOO~3EM`k7OIT?E1oK+j<>5xmIxTd-m` z`g^SA8B6^QG633Pp=>{tqZ&JlivW=ngEiPZPMny@iqQVJc1 z`S1Yd*rzq)bvd^Ivr)yCJ~P@um%pNgfj`Q&GS|9j`Wj7KyUwr%vpiQy7nfx7G4o1O z$){2asgYM=l=gJQk?7JRS;Ne()jUCjMXc5Cb~2PX7ry zKT^R!9~JM{&=?m2DeMdlnQ|m;nkcRN!HBCb-8T!b-_Exfs#Kk1epJ zvKJY;Wkd}^Jh4Nu!wIvjA}TF_^|L4?g#C73IAILR61}2rB35-65L99$_N}G_iB?uJ zhVkMM`-U4yX6v=Unh%?nU`EbOBb4ny0+yOR)|1 z`dfQZ?A4`P^5+EYPrx;JeC-|WaK0yhEzwW^{~ur=ltnHa&BDKZ=N>I>dsw&eI(Jp8 ztwu+5;i^Yn+KqN=9wx=43TSdjXMzF48`QwNw0FW8;bdezZ&l6kqpP(;CE7*YQ>1A~ z)h!3pHBbT!1~nB3zQqm#n6JHjN+>}X4sM86j$*v--!PkFJMQCZEdZJgXk-MYrg&W z2DN$%GeqwmGsCJ@bL(g24z+kDwOjOisEy`!J;q%3SaBa3;l5{=rMmm7xkT zzlu#>C%j8)= z%QA-)t}=B+FHi+}qh7(Go(bPbi7q94(tISqw^`r9!bYu#N?QRSEy{aJWYE1E(!e+< z*BN=ty?H5GQUiA> z=wgbc5|@V<%SZF~jfp(S_EWmGd1#|^fGw7j1nhO~W0qKToO-KO2wtYrFtglFy9+qQ z?P`n$IRtxh8v+xy?K~?CA1wA$J<4%6^*lYedhw>F9g$oaov|fzP&sQ<7U&NyJ1lS- z{7Dx7jJxk5@1tK(pePxJxrj3}VkHKO%_D}T zWb-Ul8NzIfP8uMpR@Qx5@CG{%`QIq>k6aFjVF9#9m+zUT@ZY*4NN5wW7XAjnM@aN_vI3ohg3}=PhpfIUmjkA( zzoh7R(&WEV^!c?3Yju-Hqt7$cw2Rf-;jWA^4>dzOjT$#;$fKEt%qyXcy_}R%!gN=M zyXuC_H#~?XkfIK7!`tIyV-0EPJ6Tr37|GJpdtOsZSBW!x%X4F28pq<;JjQW@NSve^puziQnTe}pi z6oe1bUL8fdHZb?L5H`NpsF<{Nd?0DNGom_1@{r_`oGmXeEj8nK`5#XA_}_@jzAuK} zoh+Z5p&j#{bDc-78?AZWhw&U z&tT0Kdv^q)jbOk1@gs2|gnimT@a+iFtjgp@}|4((T|pblkq|cKcUyZ=_QP7z~HWzuiK`>eM4RPw%;~R=&H*%`S$u2 z`;FPhegC>>{~)F1D*}2wJ)5}g>N+0MbF%sEjlg+@Pwesy4j+bfyD!A+w zwzC4&hj+`BPjSF(zV`iigKx8R7%~)(@SoinvLk@Is_>Yo#dm+3VmS*A2rPepTl6;P zhzKRs_E}N}JnQVaJNcdc#whXk;?yK>x0`b@ z+zFQ~d(y{vA7uF|oF`cd^7f`pJ#+5{=MMN3C)48IdxLy>bEyn+%%Ip8$`pYpF8c&w zPXwp1MzjZApzicaDHCtTM0<_o}f8xi};~4vPbddob@=MnDl$qVp+N0>(qU52OXB%el zML)wRU@POz_6z@p^UH&a)K31R?*FUHf1KD$MKN#mw{no9_LA%+y-CnAiBKj4vhP*CaqZ%^Hj+N z+poU4>wq2Fl#w@Ud$YT&^-S`q&$l_Pp|xEYC$|lvq#mM`K^e0)Y!;z`3Z%R386~uf znpEeVSrV1E!^~pbS#MQzL*0Cjhci9f2A>HpLR(K34|iWs_$EJu?v3pI&y@lcsFAuD z`n!<~;c=%6+a$TNTlHf~na`ChkW~Rhh$NK5i%e$CN%(lxDRtVcsIKQf#Vt?&5%bkvIe&{8Xkp+Q3)&~X#h_`p& zNS)r4F2N|!Vja-l-1lrg=>rXJ;O?IGz+FXsF3u2UX>7w)tG`Vz5bC0Yx}4`1Y*O9`?|c(wxv%U3Y|%%cK2y|XmiuDZc5r02v`Mt)f?QIh z^h*poVFNU+U9>iqUuwFxmDRxX@|=M1KIC~zbMqQC2|WRtn!^BGw^|+Pot>SiFeI>Q zdwY9-Hauv-_nWQ>1~B&M&Ye4uuDoQA`n1NXs;bARA4fvuO+XS0eaKSvD|vT$}GpVa`>}(im2NGht0G-0|7q%;k_PN zS8gsIHnnk8uY=PtP!;q*0}xXA5C;i^8cgg|e{hnXr$xdb^u%iW{)*0Rxi`ZFx3r91 zHPqDLxX2)ps`UR0;sxTqL{3V2##$Yg$zM`oHpq%IX?7><62Vt%FjC@_G9}o3ZI;O34>KlXCZvo>Ys;JI~I!P>4;2-t@;08`9 ziJzcb5`@Fy!!*wiAKVP11luGeBnV3KfO7}&URZsz*&_6ii9+~+^VC=tZ&ZgE^q1Gw z)wNT(uL$4@fi#5lkBzuJuoI+t05piQX2hc++sVXff_QmI~grk=Sa8 z#$t0ko-FKioY+s{=gDs%6*bieZP5izE3h~NnM)eDW9aPrgRAE}3$xz7U4t%`9&X)1 zMc;1yp!(=hxfW`VwR_JW?hPQLP)aqfdz%3^o<;?J&y)GNj+`!{)jed zIc@FDrZedkkNWy>343J+2Eu#z_a#(tzYeXvo37v4omIdp1=rKFu??2G`OVJyda;vk z+>dHsY%FWP_Mm@aYAz!?BQ-AW);ld`#`iRq&F{D}_*`RZW^xsBGfs%!=W0#!+fQ|R2> zPGs9)R?G}tg-ndB1%+U)GS=C%0%(sB10KitSjJX@t`tNGtmC6e{$kf?WslKJXj*{~u7>eV($EvQQbNW7l z2sC#~R8wlj?ez13W%;)=i~{Zg;v)@YHW2?lO*aE4++BX48n<|BxBC?#Ts+Fk(pCl7{RKn~ zB)pB!ZdKMu_vYRd)$={jnPs7xj#BfoD`$RYcJGa;cy<=4uaT5rzu;bWLz*F;JovDWkrz3sDX z%YpmLW9n?{gQKI%!8kF7Gw~awSm)>O18zT|o$#S;!SzFQWWwe$&L8V*58Y3~@|u&i zAE(JSJhl;QP%q;3`}+D~dwEwj)A=6ao#V%DWhP>;9!bFw%VJ{iF#D~E{W^?)vt(Imj`vD&Iig`NA>xb$ zg1x}J>N|%!LrR5wlahNzv8m3kp#95Dr26No|#A6GppMx6CsD) zlHKZ}`kUk2Zj~Hr)se0$`tCFwsvpKxpe@;0(bfalvFt_4dlWIhz54w9ShYQ{7tk5* zx0i!2A>I2MR=daH37N~@1f6Q-a9mA7oAB;xoxfjc$b25kmCF4b+_Opw=nwMgJS~kB zEJv;C>ZL2srW8)~rtrylE-fyS^d){X8tfikT@f&A{&Z`;6+Gdw!o6QvtmNXFoAcnM z)P@q@@W>EzUtf%&12rYBd%Y?Q*{&q#^GXwQpx*YjvLJH4hI70V5aJteHs8siPM#3Z zgL!#*IW3#uHG9^C9anSGzAuaaL85|k(|v;u01d#25mYRpa&VPT+S{JzXKOievnw9r z@am9bzw;!qr)0m3aZLR#fQG<2-f9P)6M~uZbiu1FROLet5|`{6)BBTqdiLb^rg@*- zENIR$9~_&lFjw4GNgeK4#OQBI1vt#apxi?&P*19ih&TOA3d~dFRLi2jCknMBMvq@h z7M{V^o7?cbe}P>|v-y%j=1VbkDUS1uXEmIW65?}-rW|FEJWXrXSW_CZ*zZ3l3u&)0 zQB0RnNf~uT4J;ZpHbIhiWIb$f;TUJ?Hylb%ae~QkPH<-~+N{!H9=1Con_;_**{(Dy zouPig6Ybu)d#zxsf4nvN$>aR#4@cX%B}(`YvSr-Ii~ehl69S#om8*B!otO@_gV^~9 zL|#klP^Tt1@+4z07?7Y!eIA}F6NrRU^h=FvbObLZ2V|Ma5w_Q^ME!>{*UgQ;jT>q33LV zO`t1bV8xpgn8`FL#4VOdu3*KP>7iP1kHSR}|FDhZ6k9XdkzqvOlP1QgT$FXxsMY%E zb3xgIjB&rc{i!CuxinLRfJi}+7nZ4R_MmxKVmu!Ty>QIyzS&%rT@mQ)#nh>p4#!OL zr(t7#oj6Ye15`W*AJbE;95rt=(6RgIjE#>}=4Mx{-z+p)SyDAUWRmgPnY_=dp*pTK zME1NFw@=}0h29-dP#_X<62DU1H<`wm>gec>3W0FLElSwpxd}RK#FhYgNQ5HwvmNxO z=o8Kzb9gY0ik)x2)->W1(#BfncOqD;56Pp07WKKsgBFds{eu?Gxr+tG4@`e8(h2S5 z|AzJ(P=nhAI(8>4eTo*yQKIMO;S*0bg*O%0S44A)%ny`OF`ouhMEg}qHk=I&y?LJ1 zfQ_}cenKBIz3heCMy$qK80aVEMYHEzWMK)eK95`q3ZF8E`|R#8A5DiK>O;|^5ZeB7 zw+5$0JR5dyztF;>!fsi|OQXkr$vmzucwqJ-2?mR4;!Q-?rS_S7l5_41EVW9y?6p;r!Lz(lY1EC{()j>t}KO$JnQnyhZ1`fCsly3>@mF#P*ixk|2ys@tS6Um z->3x$A$bem37rpZYETA3i*7rwV4$G)(Hi&M8@KQ7k%-lWR7CW;8vZH&P9>Rsga3^3@#l}3iZ6V(+0IyQw5+Z zWXP_%4aCs!@V-2*9hsb%=}DFj@%HkT=SlIGh_CGK=5S z(lrdIY_2{+w&C{s1>y${BBRJ=@K-HDE60xU(|O{$965^tByx{s}=h=_ z$Kk1`3bLAlf`-D~7kbw)F$`q^{DBrHhwt=y=OEUr5N;^F|GAt>NjfDN?&9Z@alg$8 zrz-GkoszS{06+0fM|m0QD?F@1BG(s2BpyRmNRVB`&*$DG*q+G%s4WhMV{se*G*90y z6ux4b=D;TLN0%rRO#AJ^pn96X`yrA#%=gyN;3DduU_WBazPIa%-(JAe)b$1qcHwnp}yEvOb9$IpaWoij6 zIl1o`7O=3eNEIH?$P*Fa3fRTHJx(X}YW800hLvpENK*w!bNi#@gi`m{77_RkWeoAj zS<+)Ko2B%JOg0Gv%{uwIk)s@)!S|zR?{(A;gUU@SHm8Ad9C6$HtNUm5gg)VulQT1E zY*|$WU?ZCeBWL&W1mD0F{!RaWb4(7}pG#t}KA6$Of*1zPufyK>S zyvWx7nUihvfKyZAt!dnB?Zg&0+jKxMJi9d-{P?Xf)A08|#{w}`|JTtIDT#}lYN#7^ zO$ecx(8F6oGX&R2kKG+jZ0KM8SO&c{KV$6iE|LQ`o%uPo^)kk=ZlxR5A&b`L4Q#VX?X1KoH ztuD{CDtaujHg0%aS0F}C*K$p4Apo>wvXAPjTg{4RaZAe#fBgv=K*aq!yd7UkfX}S= z-7I@1Ea zr%0amrP}6ZUXj6Uz%ds70}u^#$X$=KT8@Au51JFK$HMx;=i&?B4>6^!!7W+iMUo{? zE`;^?JQ|wqA8+7LPykh6j_g6VHBSz~hS=j1?i|dTpW6E(scTa#XDMXoIxlaxOQ!9) zs)n>Ue94a(Xw+S6N%|b4D9wd?f4{SgHYCVPGg}fxE;pA2^uy&)jk!o{A0> zs|RkC$LeyX#hPhbD#G~@V42^7Z_$u0cX|1=hf@sLTq6zngPUOafdD^yG^oU~e1OH4 z1M><5yIQ<^pHs;MSbe~F$^0tI{EYy{`MzDiqGEw`5JFPW6;z$|AH=jl=^Q?8ijuu(@9`;`HJhGc zKyEQzUN+?8(R$)CpR~K>Cg@29=;?0{>%(0=zuF2QKw*P@6Ut%^Wp?4CYG!4vb^J1Uw4=OysJACmvfYlY>% ze@(GKs6P|nT{N-Q&hMgQIoU*fiEGq0GMeR&qspBoE^lc;H!gN+&d`O$n(z2yLEL3NLZ?puX2odit6C+|tsV_a!bzt%_R}*T_z@E!)-v1R~m-d_2|V8_{0?G<&Vf zmpV02GSuP(pUqZ->U8fM+u!pF2VCMY5byhYS#sXEoX1qkp|8X#Lwo{F675}+m>*hb zn+3ZFx<0ftg~B?SA)C&^*N6Q;>?3bD)Ix8uA<5-l$#Ie}%8VLdru)Z^C4)|)l$;HO zXgo)20>m~tn~0kx-J2RWCm723Bqb&Tt1$JlH$)$1ze<=6d#2cZ+l4Lu&ikFW_{_kQ zqg~mA$9A>7EeE4Of#pc)b>8?d-He>29f@IyzD3x%oUd1LTySu+-OQw(qpWvRXzvP^ zL&&=>J@JZGcVC~YW*OIvpEz}G{!}@(52oW=S<&FDHsm=RRSw_%N-X0)hnU`MBI1~jgNG!XH1xi@kszhhnq>o<3L7vRR5Fuc3mfVTlw2%RnAr%1DXFG0_8ruP zH`jY^} zn!`$y7od>(nV1E9)a$WfwE1g>6{|N<53eeNM!IC;P;1LE)r{bU*A}Jb4V$KWpRw4@ zsXNEiZNiOUM@P>PuL+;2&z4=jdk3(Fb;YoW`p~UGHl;ZKfRNcYN8ZQNr(~IDeNl+8 ztx*OG!%vpx#WI{ZZE15M=k5HR8tHIexQmO~hU^U!mMtUvv89^x)QPQ{6Xjc)>zFaI zdwQA0#TJ9(Bf+kX-n>1%GDVppT(?+tjkSZ}0PZax97tcvF|fLC#Ibhm)m&CL`r^e* zLdeY6$%(WV5$xnc-(GDDw-DQYIV^N9F-!Uy=}Rx0LZC-{wBKl35i0`zF6Lh2GTngi zZo0>7+uy!>4+DO_`YsG|`0#t2tk^kjH(kl-XB=|xLdcF!)69M9R3ix?p&!xHCI%{z z=^nEV9*n=FoOCHeulFWr&Nc056?ZQ3$GN}nMDs(?8Q8Po2moF265lctEgqeb{mzde zCr6TmKa4%944SbLh0vk+#M7GR?5gt8w!R$npry`FYQ+4BC=$6ph0eSyu*zv#dkoeu zH^mRlUW{PTC@U^*f-LU~BH%+-cd=;{QJpM5=(>*&xIZ(6WKvd_7}ZGMCrL70|LR)U zX>2f8uPz(TEnGb%6#cyhWr~^n78}KDP+}HXe|Xfb*}L-1KBF&@jNH)uFHZOIOn&m=A`F2Nf!XKbOy#0xB3|6FKPhF4w6E46 zAK@t*kRo_pY||x;|8$A}er>SI6K?7||HwzzojrszzVjnYx)4`TR8}leh;vU!-zFoS78`(l7Z3!Z40*0CFe$qMf|efvhk?#dkQ~hR(AoCjaR{ zA8udIzfpc2=Ktg{f3o81Yyk3IQ{J-Y_+rIk54lb47U;VF79E=}wNn(saF`(;h^Gx> z)k@ty@;Kf;8R0Jnykcf-PN&8+$MDT%Dgyz>u-qWGQ7QlFphY!dB5k2{L%ez-x6c4m z-PsWe5wfnSfoR#qm+lmlg9{tHd8jZ4hy1a!v$KQutqDYYDPl-mQK#|T zMKYNxI;vl%;(TYcZ48p%Rh*^fm5(k|7i#shM;i$*{GeVh2TcJqAo?#k)z#G}%9>x< zS@tqY3h9qcU3Yyy#kT5_NY_>QR&}xns2YTYmgVghWBX2bn@J7aPj;Ibi^KLeo{{Yx z4aTD0mE*p9piaG{=i}lm?<@oz_N_3$z1%UpyQ`HyxlmwTxjCTTVp6^rAgzW0>2Ih> z$!Am=QaZ1U7i{dT;n3DM_F5}aXD~MrM>KM{ zg|!t8J5srvZ-$&#@3&yx%gWsu)mr3N)zS`K>#Z<@VzZ&xJfKAd*Bx4(ppz|rhcah&$Kf^3 z2i4#L8UCDV@;+)d3wXv386jWLy)459aLD$iDg~C$R9qHaVC`jQl z^?+!R?#5lA{a%Z+y=~x2DtGc*rEPC0=xjscBDC<@Js{}~jcP3{GWKp54 z+4OVD{Ntg{%iCj=7NZj#Q2y@a*sLAYurLRqos0}xz};#8I#QP+f|*1cM5V_C!0b8Lwkf=3m7g%4%`o*UJ3GidKO^nP7qBlvBd@;CQy)QQ6Op5NY(mheGePXVBAjx!{ zI;piAKQ5IvmRTw*WMP6_UO!429N+GXa-~}~0%DEUf7>+S@ukB* zOPKb!V^wPubD^nkt@uuDh6()6>JbJVsi8LM_vOyAjqz7}s|Z1gyxg1)6|?kFDHIBo zI%qj0vp8xaOId6*{hdGfU-{f0zd{SI09x-G=La)cLFW0EL1A99#DTh$K@6fu4KLC4b^`o*IbkByD%JU%z$7Z%!**7Ju( za|-f+Kn27WVoEJlR*+eAdi1Cd?^~?-<CG*k!3Rzzg52@FdrKRgr=lZJ8N(s_ zu9u%K3ig|ova3*s@m;yRn(lf)!~bq^Gjc1;-lB7qOMrZh#;L8+GIrnHT1SS1vA%wH zp~=;PM6I6vY<$|Q^8paB(IPa<5n&%8TX1WTf9KACr|~4a&^~r+CUi9}%;elsEyCS) zv%b&j;Om(6fY<&#(3$pd?M1F2S&r=FC)cVv`!U64$4vx!dHp$AliGOv21f%4_iU%R zOZ`6h^eH7}p4kc1<4pGCWHT&^sX0)m+uGm4B2$#x7`4C~fX8%5Z5THjsNx8nL*UKH zl&3*{SRz_wL&qr-7GKfX%7)Db^cY=c>}b}BG`)VeZ)HsD@|AB`-4J(IcX!0{LQ8Th za{1}%&0l&P9iI2?xAw$EC+G%;+4Yj%V%9nX&;rN-EKOOzZ1IR!Ab6${2=||df|p2p zlSeks9Oga)5JnmLs9VRtKFD&V9YiduqH|3qFe%2i<{;_q&K%ozg$XubwVr+4qqJ2FI|s~t9CEZ zXB+)|moVR0Zq5Yo&vVWgkbhCR(&;Z(>oRNn1VLK4ssERCpL?Bkl{v)_UjRROZ~_OL z$XXa={U8c%w;^?#sa!7wJ9BM4U^TKFziKDVJP&PSVQ-Ihz9(F7;+{aN(gE%soBXGxZskLx{XB+gmk% zz8jJCnE`q0=)i_#CE5XB-+W$g?&E_lVMIX1ZX5wrxt}B59sgY!d+OITalFpZgN?zQ zR2at82=v?%(gRZKqMeGqzCNsJ^Xi#RQJF%^-@Po3ApFTA{Zhlj$=TBg_zh=QcIvx_ zo3a5Iv%0wxRTm3R5R0UC(-$;`Gg#C}PjB~$k&22tXH8-)YrF>Q)@2b*8QhW|Vh#a@ z<7N>y-EXkN8C^1j`6ll>nLcWu!h3`vI*q3yfmtXzzK@`(I@fBMlf#TLbs&sN_w%+l zO4xejn2pIDexkwfiiEJetfi)<*Ba$H8GXS_UltkO%`(LT5ryfB`sR8M^k$@ zWzO>WY3xA zRSh2zgEvKUI;});R<5O>;qp~tupxOnP4j8dP^Vi{{(-{@)Iiq31h=<+D9E8XH8Q;d z<68DZS6ZANG^Z@9$d0gSrq6`JyVJA~eNG2w<+4VHOd--U9}28gWw`X+)`L+}S%|!i znxThjN;`?Dk-ZIKCTIVxog-7Pu+z^TvpVHsZpS?#qdBvti36j)1yw`GMpshZE`-F5 zVkZGEou*kcb~|-j^iKft-{a|jUQnqr7eGD8zNxZ6px$JDsxMCcIy94o^pY8%yYnOb z=&{|&gS?x@`Q;h-o**CCtT}e!NFPWpPU?V))9LSyUWZ7VKt#%Kvpy#$$GY@H@t{X+ zZmw@6e7J0a|Jy8+4+WQ&(MDHeE0QbZzCxMdyW^h5JY1(fe~)u)RuB)PJ}14eeLt5i zF={1AB9=YE_8?)ZOrUtiyL#1BNt1P+b8L{|8x*EuWIjF`l%joA$II400AN$iH?3?( z&k3^i5$-f)2cMsiD)kDQyByhh#Vf^q8Xn&c3DYRd5j<<2M@7`#JY$Sy^}rDwwCHa) zO$E!$8HZ5K!b6VLQV`co|6|_Uf31hLhT@Oi>g}%Vb zut9)AJvI8q#p~-%UDF|r?kM*=)4S`gZ;yv+Dw|L{sTS+v@xhyon6HC_R&TIq*D%&M z)Y5y$I#@tusyf7%|01q5R1P~IB~xva_kdQ6Nn6eu<6AW6}y}>NMPD!oN#$V^=0UT3+?QSg}Fwz zuO>2f_shB#^)eAhko7u5TEQmQG+7MAH`G4@&Oj=-(S931lHAr9F6-w#Nl!AfzX@7i z=P_;&!h$0Dq!-K<^8xk8P`qi#LrX=}=S0a1cs{`Wf#l0u8S0d7Os z1;|c|zh&dp7tF6qQgW7BmJ>toE(-T)=`=@7Nbjwh?b-_Zo>pbS(T~ef?ok=;6*nAZ z*yLCW#To}cDojxWsY?w&wrMUr!x5m<~<&@f!VP7c4P5CYm2S8B#3|nN?$j1 zO8f@w-9zOSqN37$Zy-Pv#A@nmcbR5YDjX*yVO8Y3Ps5jq-bkkh(L;Ehi@alFTBl5$Ih!nD6Ji43{NkM!VSb$&IGq{;Gc4B6L;csVc>R!#)1~9$n~**684>Q>?GNt<$S&9k`7Xmji0^+MWJ7 zN{F?C(>|@OwPk+ww@Fa4f#eNx2F>Q7;P#6BZGL9=j{$tEUHJPf8ri?&w*}O5n%S7 zbAs}h^u$CR-ng@gZ-IF6Dk2-2g#8#-cujYUIBEo8c(|oLD?4)lnGH$Px|T*k=@3- z5C#ntMRRRHrWyf5j$q8z?6kLfrh39F)f)|u6vaKuOFU~jP^=9p)lc47vL<=Bok&oB z@$B()vgCAC-?fi*51HGAJCXF7!h(}Woetd1Ob$+B7^32qR*k8t`L154ubt_}MEf_D zjE@DK1Uns;l@6CH{pNI|P$o{$hJvt7qTANX8|m78Jg&%^MtA@+B(@WnO(a3u3`eP1 z{MW_tuU|RYuaNcJ6BC=`$*D;Jrgh^<^t!r%rYkg2xC~#4dzF$>-W_gk2@qQEx74cy z2Ibu1JQ7Js<0^)VszJLgV`-g}uTy4Sn?xn}at2BTyqWKkvt@@yj4r7gltWn8lZ zq^7=eGi;CBY|HAQ5UlDU2@yM~`|em*-nlf!M#N7X-&|Mg9@-Wg#X!_BRt*HFK(J-^L2i?ov59Noi_AB>UmGr`jvTSA zMwU!VD0X`$ED!*_B76+(gwX{8z-G1(47MVHfwG}7Cl3zxIpEX?EtK91kWYlyH$SJ+3i-*{EpDJ$lG zNp-Pq5rt*iQHP20oE_Peiq+z^pWTtr@Sd9F2vlD$yGkA&6RP>~_Dn_hcwPM@ch2YB zOYbyBDO}e5s91!CP@z~$rAbt7^SEU8dIe{W#R#-j7w{;b;B;t6C0tB%x#r{VCFp;K zNB@pU0yjJBoy6{PN>_PSDZ&pr!rd&Q*!6Ak3=PGU?pT!X*ulai=BrdzU{0W(R|Mvc zi2DvY;C=R}&S}j664GyXx2@aOuX(Z>)1EVf1sGy0OV$Dr4c6%E3fu^mz4WmWiSVT* zAP=pZ_0sEpQZU09U5@@bbP^CG4>AHW4SkS^CDT%CWNE8ni~!lqVbc)2cNcto#1eq1(4tLYV`ip0x-*j6xfD1 z7lbLCNCcT_CiotY<{c+ut77D8N!B10Q(LnhyR*$wQ+TDYf0^=s|8zqI*#4gfSwIvs zq--k@-uj3zId~#l@m&y`gQ4EyBLR=y>DaPwuWC~JIQ2|Z_YyF#Ry{nt2tTkS6F>jF z=0N%K9g1KT1LnbmA}(~DT@4-x0zzq}<19Bb^MJ|=3X zA*t~vE^Z7u^#;K`R9hd_4GDoi4@{flqI{mj-dEe}Z`pmUq-x(7bdd~xeAEEY6YlLi zf9cKGw`vl6eU+glN&|&;PiSL1l}OV@JIFuHBt)>MlZGsVRPgL6-MU!k4Urjzz@!0&xiznMNs$B@%N> z6DL%%vIah<;R`5((fXTSOk=H`_QyQ3C-so_UGGz|++uXk+n41_%Z8`xu@zKM7vP4l zVoH5IlU7yee(-B)D(F1nuy6LS#l~T1)SGErtboYj(a4d>fk>a+-b8A-Xd(d-i?$$cqG>pgh?c z-R`C#!ld`*V0`E1*-47v{Y!n}q%KXBONCP9{0#4UoXlUg*e`!L$xs?Nn0@0xG2yUt z$RBvv#~JwX)f3@vORpkXgA?7TY{ZsXRmZf@#qLtab@}$8)605^J&yBcECud1YxRN9} z9k%=J8B0IEKVx&!hO2&}iK0^Ao@SjOe$-)gF2w9B^rg2cTey>z@7kaPJiuTyj^D1G zZc0pVAn0C-Yg%L$v)ZZf&FJ&7V%4zF(Tz5gcCQjxcxz;G>D1p;2R`5?(wK;itL480 zmxXu?QVoTND@quxp!a9Ip+HX)&71sVcBPApD)4`Iq1F zF8BP<0YU}XZOa7N_%U?HxJ&I5rMRx7jv-1yMe45J3OoYy<3p#6_#~zu&EhjXF?g-y zF^}+qxBzD&DR7BQp0*YgnG5vS1u7NEo{fvK82&gYH+Odv4=7pXjFUB`2?Liia9%g8 z<2xVSsx#QCR!~w@q3`0rC8%v%5MPHgE*|&I{t6$eFHA+XB z0(f4$2e_wZZek7xL6z{Al-{Ypc2f3KD?HZRvIf zrK(E94v7D9g??`MA<5c%OK%g^f(At?O+NzFWY>0s*oRSn8yGxh>b-mpw%SM zT~gDQ#-{()>-^^l@6hHXB&O$`43w7S~i&=@8p}z^+XUx#M5|s%1DHL_la;xmuIUb=ju)Y37^4NB$b{ zKACMQc-8fnp?`Gd3uO>gK)g1YK^d;|lfMQ5qm{mU5!nXISYXoJ;W+)Nqg+W%zKmH`*SvNvmSDa!y}}v|%KRU@K}8yJw{*P>}6qdW{VEqHi8f_O(xz zara@>=6gz{xNnxLC<7E`bht45djbYU(!82Uw8pL>T{uernpgr&AY6*_+(6w^FWjBidp@IdSV1FNRCGwJ;)tp_HdPYI{9~)!y zL2eo(5fs<|MRbVsecUPo#RzNxIiUyxsnfW%$LGxncScu1KFC4KF9Ii;d5)L zW;V~PB5!}iXWo3&`avOzG7|*xGXo_5N<$H{IZ9s}q=ij^NJCh!mHPD~#zs$1mVR3& z6Gyr~lIW_3tbUrYGdcDO1?%j8nO7U#20D61KfL4Uv8pI~N#>k1$MQH^>k)mpqiUPT zp3H}{bj?!J25kvO%H`dFkwTq<%14QA$VbrL>*+vHi7}~j3$fw6ISAwwUE~+k?$7}1 zwsc7nuIyr0ou&UrmKg8*hc#vRGUdwurIp)Mv<}(wj*uZBT$A!JKeL)z$Xk^*VVndf zN%alYJ^n;E4GqmlpflqUdZx}j>&=zVxfah!uihwcT_)|1t}{#e$=BdOkxgpZQYr~;IH|afRnu;Oq|HPuCQwL5c)J)fNfZ~yD{?}_sjt} z@#^~oEKN=hQd#c7giGzNC~WUCxbU~nFz)3%FCcn*`jis+>ECnVzw!pH?3YJI&9uUH zvk+xSyDH;Kn+E);;CXiB3wftjXYDgN@p)2OTyDPvNlt(Lj2osd?6|I|p}`2~`R6iy zPC+QF@bo)4znQuks6<*35ZYN$|M_-*^=Uu-|vSR7Yr@odlNoU9axOtGdMRS zlFxQ9?AXyPnSLV1E|ggB-fR-&Lm$_f6MD9vRlL7C5Qp{YVe^`QGh*D&cT;(|`E^3W z3~T(u-`-MrH!UqcQIc;6xBD2p*OM(DF(OX|%59t1%=aHcySln|D@-U_Dqg;l4!bHi zT;gIA7M6wo{@kBy+P|)8-z5~@E|m~@xP9ueTV~e8sr?eDrlx(vO#}!-VJ^XX>(~DB zu>HA=t<_`!E#{-&v{JssetKbl`$?2Az2Eg%@^JIEpf&o$Z1O5XySA$nf)~;>g+=KG zP_}J7T({E5=mgkDx-St-u*!LGGMFpipSB{Yfkmk@7c~BR`6;32L|~rx_$L2iMzsXe zuldeH-XekOJ#1{q8{tNZ7rG3%hwraTWUyOUQ)VO!E4E!05lMZX&lZ0h_@I|Bm;i%P z0PHURa3S;mPpmHXEUhdR@tf4~R1w+_@sFLA z)eo(!exibIUSSnaWGZKCkz-9Hx{S2hE0&-7bqxJ8XQ)(@8@j8XrarML+^+p47fI`V zk8hER@GY}P$0wttQMb6*+d1Uie4ROJLb(rx%@V)b^$Ab~J#W23KxnUcVL=E`Td({L z3RBBp{FhAjUq@*jT;H)%BMI$ly5_M8_iHg92b%jBI%SV`+fZV%TsYJABcga%=o#K zI$3|Tng8ny{nt-O9;jAV$xViyq+UmZ8hf^ln2-l=(L%+xfgDAXGMcz&4@RdLBQ;P5 zhY#w47G@;h3fihQk2+zR3G`40-3RJbq_O?CJsz{trYH1WeE;*GU%Lv}%2iPRFNNa% zNXjh93BtkUHeDa@+ErzStK{noKkTuhP#U-gyIm<;0$zz1Zo6eGoHV55A_{w>BDiOh z;v1!I=kP`o>tY9r(RuXn{#97Mv>WW3o*`nWopoAPpeB)q;&ROgDK^);I?c|=<$Osj_OxBznhEVwR%83q$!q3 z>TqkY$1HYn?)(gE@fN*C+{;dSB^CAvT=dtG!9Ta2)u@%7tykPt|83f$)`c)RB3s?m zSL~Lz@hMSiCqXV2;@bf}X4E43?OGBzzv~H-rhYOTQAWE!)(RZy)&QdVs91yOy+4ol zFWhppyqsp*$jB_ej*wB3%0nS&Z-{o3BA5_;pJ)*y;z-xLUb2}2M2AIQX<<- zYqIlIpWW4e%hlPg9XEANt8?I2@h8)HM)H;~f^{zJB?| zQoy78ewf1Z{j`~^BMrRiC!QOvG)G(f2liPqwVVE z)$$Kn7u557oLwa4tGgAlkLvj}scRjlS#x;o>&EH$`cCm^wp;T#S&>&i!P~H-SHhrU z87PdML-Y@ycB33{47-t!ng3}N|LNNP0a-xRXoO5Qi|hdW%UnO3AQVB`DmESLZ-(9%3 zM^4iFNLRr&q7j!&mGs2VyrehR%PZY!e}%utCHAF*k4;D6!OF`LZ0q%bJ?f{B^+h z&kg=QM0i`0jV;c>P?6KfYlUCen^yKCIGo+_LzGIs#hdlyO*M8#@4G^5T#vwhfi$Ml zJ624CK?0c_KW}%TNI5_nCmoOlVW1N*Ju-H5bVT3zDiPgUas}{qyykY2zf28(U;g;J z1d^M#bH36fS>(|2wW!D4Py9sqOc<-(cAMjNXy6O#!^)p~6|Kh#^rr!QSacktM5Tb3 z7?$-jtfJ?!K-$k6bH7*(XxI2lL;c5F3hT-##wM(%v}MEBK8ieN8?bTEaik@l=^U%9fJ%XWF$IK|F27|lIXxeSSCv$@RAFUORbVp%4;)g zA!Rlwy~@A~pJ$5C?sa^X1D7DbRZ&hT!_Lg}njdiZQZhaHX+Wv*KzvWK>$yk%^V*WP z5Pui*3y1#4l`SMf`0PTIq<)jK5y(3UgfQeEw|OiIv2nn2;NBZ!8svouW3*}8b4kgN zgOBZpFlEJhh5lzVxeuu`aRF4;$XpRCfC6mV2z;tq-69gk1=#LNi(U1Nf7!QI`aWoOz5k`Un9-fc@b4cIod8q2&7I@P`{~6CGC1`uuM0lEuPF2`St>Bp`)U)izvYFU`sAepenJQDgu)2SRs+2 z2F9O!r2L0Zzaw=&YOnf`AU9f(0|+d(PMz)~pbSz{2(gr27WAK(oi#6j+l7Y0SG_DC ze5n1POQwy}_;(W4+!`=?o*&0yc!To(Z~4P6f4u;ygwXVK#u`8{0CTz%W++t!f&Kp>7(}wTQr1>9F1)|9Fin*4BP?ZM1~e!Og9# zthDsB$|y#NOiI(kjuyUVG-692vS*b{YqP>TIh#0Mmgd%jyx=(-?CoJ02X?o}NAy<< z?yiQEm~*~3o~lO>C;+KOz|Qtpx3{>n^aRN>G9Cl@ zfZa&1g0&zJAI>ZNou;W>NvoMZpzyU{7k7u_LGmYQC1giWZ1A9aE}N zqfcSb3e5fk=4?$#R(tNKLyQx1Qx}-#`)447`5E-jrt<#bGXC=ZZKWYGMA|KTh_m0g z_V}}$oMKACEmfPl(VxjUH3#p}U$`mHQ)gW2MImp*7d#?-2UKCvG{cNRHbrphli0I) zye0^_2w#JV;aHh|39t&u5JT_e1V zWcNT$Q$&5$a@MCSY9mP7*xoH>Q)==#VYc-`m`wo80j-WltqIa)*_fFL0MatSwHWt~ow|J0;yyAz8?lB){6M?PpQ*n;H@iZwRt@WL>QJP*z zzQBR;_N-7RvgzuEn`e<8$@8$Ei^p@D3=2q(Sar!#`rMYW*r@Er38Zk`Oa{aanuCo} z><8rJ0;NB%$&x;Ng6j-FnQFC(<#Y+(=sP&1xS#aP8B)*^>TkDTQ0Unj8{GQ(b?>F# z_u-YPB|PlzE=mX+o}Jfe%M>2;&7ejmjpIHY+=rs(ifyJJg#fzp#h@E#w)=TRs}!HU zN0tN@+S@9TAd#O^Kr}bgZ+73Fr%lW?^r_FQ|8{!-e|Mbz)KuF+@9e1xPTKr>QgyQi zAR?8OJGa>6`erj~;*;ucTpvjH_;{4sghkU72<~aCYH%a#xSK{M@}BkVP;kGV{rD+& z)mLWDie2OI-TGG4j@X}x&wtw!AQUZl?CksKD~n9{PPt(JEw9kn zVZK$vq~5~SfSNAZE9?@2s6n=Y@ui!pMsHEPY)J8gD@j-Lh#N^>`uK1 z9}T)myY?gFF#>dM@wRx1XD5XUUd#k>H$hOD8*7(}x;C1@`d+iZx|)`RpFy-nrzlCZ zHlsIdKluM~e6A$T+gpl~u%B(Yi(-N=VfErptm@KqA3%{68^UUPAs&xzoYFyB);_73 zXX-uUw_C9U0*})>eXX3-s&ludb?qL?s6D)BB_3NAG|l)eUQXPw^}l z{cyzc%DCWLuy-nPhKZrE1;9z8dHriXh6F zu+R&AVSV#!4kC^4CtkvyZ(y_V`uxg2;e!A1@$sVsv&o+%_v@96wn9hO{ffsjnQiNg zb!{Ic^!SdC5}C9}nw&g~?7F;>ae7)*`u`Yv%dj}NY+E=X5ZooWySuv++}+)SI|O%k z4^GhF?oL5)cX#)~Zl!mh`}I9v?>^@z^#Jv*cg;2Cm}AOn;_&pgQ>>}QGIIesE8l9U zkxX&@fdYR`8bJlK`OaHs9T2FFO%i-+)1CNvoWvMKx^652AiWcl2Tf~ODvZjplZwT5 za)8PxbmacF$1;AEgOCs{M1d}#iYi(D!a*ZMTZJi6;Us-Iy_OL=jv>3*4b;*DeLHp z#Y(SzmIzw^<7JF7+|S0{>?~~TG|~M61{j|SJU3YcVm3HX=b=Z(g4Dy=4|gHmQRL`Y*SOF&Pe186I?b1;)CV!)>`=m zlZV|=_*-VpWHT!qI&DjTTb&C^76Jjqz%{2j@0M8;f-H*>T9T#{G5Hyj#*kLv2K`sxW?)D5pb zM(?YY^iDYi@)>=iWj8KR%%~%GCe@~BNGR{ zH4R=QSlri`rvY$X+={Sv$X%t`sROb5dIglj@)yVd>v5sRSUp&tvyVrgeKE|Lvo6+Ak8 zxia|gs`@y5d_FNlmXA%DjwE->9A7elCbu77O8InJ__UsHrO4QCd^Lw!<`ogyw>D&J<#z*^1q9iWhjD;nv6p|#9 z7#yC%!RZ-AU-Ax(gu_p+rq9V;IkPFrYKRq1PHHNy^*_IFLduIgw}%Ibw!A);CpmLx zEwYj*UgeC`lGAIO#7k*fw1N%Mg`PBA=*7bw&me5Tz|NbXpo=^2)a3Ygd-3sBrWt+l z<^OAypv;y(D6^HFF?v@Tj~|Orboet#cytejx{Z&xU|fR~$v8r;XZP;2DQQ|lENH@R6EL`;m7GI zQt~Zmuydpf$Hg{nNhUWWjvX`H#H4yl(UQn5_>c)Y+fJEj5Mw z%i=;2h7a9Z`lcA-5$<&$t*Ij-RFZi^VgNYnd?+bgTI#8?y#G>uCHPN3dTQ&WHH7XJ zCGAE@m@-bP)f%6AQh}5RfEIi3;x&X`INZU2p69&VxKhTzi;4flLiV>%s@d3+u~x+B&dRE zBrL=T2m~87gB|rGwF|=HQyMc!BWE91YW)>er5LL3jD;6|agAdxoZ za`H=t(bMKvXHR3{GuzBcYSsAZ-?rHd%lWs!TFr92U%VIVu(vIDnKyYJ3~W%|K;>ca z9m`mX1p`<65X&Q4oaR7d<5b^I{N67PpAVO)M63TefKv=keRX};>)EAbI@>d8A3c3T zLd@Z0fs=8JNK?V$!xI`>T22nXx78i>3N4Ri#wgZ4#Hck1u+(B7$k>XCa(kWQnuy)D;Xg3_q*W1*-FY{4g*$ZGca)IXN5_a4)@&;BRU(>E5AA=10qvDgLo$cj3y7 z%NHEnLZ@D(-S)0@jBl?^84D_U1c66hwS^?{Irk5Au`v-~mqR1~D&d+o!>rn#ytF7R671dp@=Z!aC zK-V`C#?ZB71(}z%nwT5cDGkNscDB`5>~6AUa&)SJdQwSRXjec@b$Z{sWjs@(HgItE zOc~QqWQTMdGa=vet3g(jmli&UZY^5e#4s(X=QA@*OV>6i70@FdAJOA`K3_{as%QkT^6Lve z@%f*ADv$tyfM5!A7N1+*bux#WFCQkEZmupa>KkqkXY$4(9Va-p_T!-f~ z*UX1MZAsC4-q2cFdcJp+0tn!N9>!GF5?M=Er?4LSzK1vO303Vm^G5^f>)r+xhUt&o zFaCl(une7Vkhxv&IesG(6LM8WW7mu$BDo~*G<`N|uA!<+xa?TrA+f5p2PHu{ zpp9&V8Kz`vibYGQS+dQp!Zj}#F~N2{jxHb|=vvviDwGLijY{_bJ?r9=U6a?A#4}=L zgzb<(*UfsUx{I?*b<{@5f@y1aN`@G$o^GAhz>GA;!b??qg|mGt|8SB5`ZEH)R;oAu zA1kKrh5}tr`5z{teDJqHnxw*djT^9GE|16>u)4{5PRjc%fIl_ExVMX57O}}}3WFG8 zcnigHAEvBH%+1E4A;hT_SfO7*(>n4;+WY{9Me(D=$jZ)kC8^4A&Z&HT$@P`38h!sj zLPGEA$r}T@vVzaK@ZO2JOj?*z9MWKY`R=We$G+`o&e;XNjzL9F%dt5>|E0gZMfL73 zT6Ru$yb`d&!@VF%jO=vH6&gG1dpzo5Z3Qo~{}!p8k)f^DZ!4}&wmb7WdH+lRD!PHb z0n6d_v0JdVKYt0q-y?s^+D~aTt$p=<;h#qqBvd8b;*dKwy#EhU(+4mXZ9)E06^uyL zO4=4V-MUM)q9$W$QWQ0!E;#1P5qiz!cau&3FiS>iR2l%wpMxN127CHuaG^Ub%qOVP zTWcdVe$%w(-S_)38-*Wwv|jp|<|_bXkvo<4qcuSafR-DVm1Q2D4F)>;^GE^v<~voI zl-K)WoW}DQJpKA_zIC8(;brX$9lw|UBQN7;3T;Fq-wu1rt z6Bp=hTqy*=;j8o)2AjpU_syD^-&Jg$JdMr|x3npWu5gh9z0tHGy)0msATM7iWcbg* zr+;#Y|Cdsjq=GVPK_nmdVBvpvBd4Zgomz|zl`qZtdHx|GCT)VCK4M>WdHLlj%|_j( zy|pbMsA0BuFi>Ljeq@+6a|dv!XpVsU=Dt?BQniBKC9vkHX*u%t_)Eq+UE99>E&Kld zH#T%bMa)L^f+={0*flE}C{yyf|6L$1RjI1zY?SXAA!qg5!12xqQQC_A`hAql)~)9? z*R=1s%?KrEWD=>M0XcNg-v-MdmNCr#5ijoQv3HapjIz&0i9QsRegSFm-y$)ar*jEVmkXgPQ6v1@_n=TQTprB#KZC`9W=K( zTbHq`wWzsyvXbCxPgCN9e+|grVr8Nh{d*Dj9|;W5+J+yFWtm$BCl^l0Fi}$^7)fVX zM!?aLpC{uLJxvq1xMc$0v5fi5lc>4=ky!_cj=I@=rbpCtH1yNTB+<=dnc8#I5Y9o& zTZq8UH-E)XEFnvm^m_Hb913Aol2T8A${5uXxkzbK5HL446OD}x6rgG;ijV*zbuX_W z?$^i1MJZFJ%#ME@lqgp+L+}q1kD@n-(hEZQw}$i2qtrcy95&1e*3%hw?HZ)|gQ0>L zB`%z91?Ku*fI*#w-{?UAAil zY^t%RMh#le2>_4wbgH@QLpn|2+Ei4NT(HQM8C$;3>Syt~Ow+AYc?lKk9?R2b z+VNn*aG8{qustK-`aYJ#o;{v?(f562=c`n1tp%|2V*QQaI27?s=j{D` zi3hQpvyA7xEB)+vu45DIbKpm1_#hNds5oK5LZ_CXwod@OFU>DII`qxoVN;m0L^y?o z5x|IuZV~(^d(dzKYaHSuE7sLk%-=RPD#SCgvvn&AbuGzW@hi;tA4c0BmZna}yfz-Z zVr}sHRho}wzC$zm=gAUcLZN9iFfwp7t&Q4&rZ_e>J8vm)a^NCR;DIKB*!VhW?i-`b zHtSS**5M5~n9f{({gUhV`+awB@4QYjcQR;$eJItdoZfRYuj}{E%1Q>Av^bKq52%Nv zNDq*USbaQyzUCu&H#q1s1F*mQ0W%0(5+^YIrI#K67MQfw_`RPMIFyw(TAK$)9VyC+ z`_~^6@<6*g^1`dJSzB3-H`Qj^<|%N}X9JEb0-91uJBK%?XwgR5QcYcO{W&n}*0!8k zwLMDGo%@5133R>YvRULg&*cZi#8%=FJ7<(&&-gbE5A;0mtm+OjW*wcV$r;m_v%C{W${^N6lRzye#E*u|LYHN%BrXcB_k+~&Ae)Scro-}cS;11NB zDb;Jp5oQ!O@dYlz=k5*+|FdX$mo^2MZ`T%W)z%GghzZ->3!wBF%79OO{@{b|M?YWk*rRJT6ir+~Io*{Cfn*xtz zCQW#H5uA3IClZg%?EAAm%q;M)(h*h_R=$tl#zhwsVI*8nSNVFV?d|MhOw$MJkxWPR zPhT8+nIs4rhAcSF{c5ZqIB{{%(x(=2V;j_QjrEoph5L z7GYNWFUQ0gh@L3rbF{HZ6q!rRA9N>$;dZnvb@6}Majk#%;#9#f4qOXbByub+rv_g+ zC^=Wu-kud?*Sp!Vr&G?3hK`Qk2r^FsyER`q=AJQ%6@H$Ttk9U7n`N zj!{gNq@;JD3%*{cz7yPj{o>KsQ$Z6ek`MQ=)tzt269O9^MNEiiUMuZV9g}G@H%zFQ zG2j}EEVx}Rb}awlEd6!L{0)T-R_0>|+^SktP&+3nt!fPdEk0wXr%g^zqueQ=8t;Aj z>-`L~gMs?W&I-g+x?)Wu<(TH_aKoE(eYW;!Ivt+K{A#q=kD{A4cxq}|o-B;@g>DXwVps?`SXuF(fteRsFUc%&UK_X%>nX7%3v(P}jG&7aV6vH=#AQ6&`j9U@ zS4MDTWfp-)2Z{1&Os|oc#zQ?zb#gwi0qNB{lV(v906;kl{tZ89Sy!MnqD}N z3y5?h5pCWyMB?!@BF4{uWBDgYVEH@!&bmA8^YhvUr%l{tEH1D5$#<70@uuhf5_QBN zm&JK5ST%$Sbd;YfopAMLa?+pkcZ#ySd$m-i5J=5Dfj%Buvnh(@f4Xf0F-YUsG)~{= zAO~Ya{HjX2rH-cK!!@saT|oeQ`b{hr+ws};b)9=Wsxi8TkHq)5$I|kPBe)FjiEU!|T69fMlNB`yJ{DE66A_YCYWnPEg zX1~)VZ5RJ1a9KJ!<@c+Jrs4=c7qhHSvC3=b0`7-2w^Bqt zCi6!W9aeH~w~7+yKJS6C*53T@P{vsk7Yu%T4IFmv2OV^wtN7{#pbL*~8lM5)XJ^L4 zfiX;UXDhbs?@1%5ZkcDVKC8#kyX;M$hs?LzYe|LEDcu-TG&gW*c~_CL?5e!9y&g5| zqGOFgNA@%CafWf8$;lLeK_EA!YzYT3D=j5}8zwK3QeCm%I(9wvx6y#MJ}y*YxuB|Q ze-&PCQ8{NN?u!DK<a zK~7b*DMHALlV8WZAEXFhOU(bm#6XS=5lo-&IZ7SB>?cpv+5t}brneGvO%$!vma-i8 zTeTX1Lo@l{yStdAFf@b+gxZx_qm+bPfXQ?n%dI-rJKV(3_V{vuQskdj3=G1WA$W2>xwkK9&xsK!OQeYL+nP<3TTChSZe;mLJ}LGc zg2g~lZ4rVr^Pc(asknkQQ)@*vwG~+9v)p!jqTY{PyL*^UL9k$zaCRrX@SvlVunt?? zNCXs1NxeOTwY3P-V$Zqim9kYMma_j0qmS!AZ ze%f1y70FZ3t7Vx)ktLBx>T>&?ULp``nyr;k)DV&BdmHbChp?h+-&wa8F+V>)qGp*z zZ$9H7L2x&VRg!z1r`kT{I$xpOB$LbTG9<9>eozjHsFoXkSI~;Hpbh|h72rQhN@u`j zGF&w@)@+{uPZKl(TYEJm`#r++zp%{iIO1;u=<26K<__%aRsiklXs^d6oP87HwPv{^ z62X9A`&>2>9|(lR1Pu#TnNABkvskFfgs%s=&&eFWL3eLy^SGYlcbV(Y#q;NOQKG@^ zQ%6`iah{&hHOhO+pHxXjp<;ptf>6Z8yJn9%H*Hd+`RZ4U9EpDDk3sN<5gBR*ic`O-eWB1 z;N|7T#5=Lbi!ZM%S5-nKz))>@dZ-9F6pd%=C>{4>)0Z{(o5@=$m)(lbh}_el1pF8F zrr;gkk9b@}4+P3X2fpc))n(LFbuucSSJFYJ<`Mh9+4jGEd7t4wQi!vU&YU3e(Ahp1 zr8WH2gsRfN`1ZL^c-E*?JtYV;JD8uf_VpRPk@5kVL?Zm+ZL@XV2O4Ya?&;okyQPHS z%aT@CM&I0a9_0BBDLG{gb*4h}U`PhtNH(*4(K^EdoTh^e+qm&*}b&b>?E(@exzoX5ti6<&(a zByC4G?7}LOnO65Yh8p44n%yWVkEFT)ano69h2v7nudzOfOKoohAA?jz>b2@9P0vsA zHET|vuf;}ND2Hz{(UIzG3n+LV;NP4sZ}cf~F}0J`dZ)H}5hH zY0T{$0bZBO09*Y=z3;Uyxu5XDg|ovq*5>nLXdUbXo(mJyf2S$7m#`2IWaB+AeYHw)1& zfV)L&qO_LEDyqYlhl}Ux;L<}(zIDq~?YmcJ8%@SUgb1C%@5tY}>qf>$h zZm5i}xk*&KIXtet2S3EPrFSDy{PKQsY^iE_CDoKn zylUn|i8xcL_^K7?H73W~e@aX1d?AwIt{*Lu6{&gyx)M|88(qf99R`rYUT@tlImGtg z9`dvgDKd^+Vk4!FOpd)8Om~k3!82#`zrbI8Yi@QX`nP-_MMYdHp*{oW1;#4U|0^K% zA464OOnN*xK=j#royh1arnG7ThJF(r_ix@i&VF8w&Gb4ee9o2|nO+&o3GB=l;CI?w z9^9YYRnhXVfgSSK+6aFTXm~HZ&}BBe37>&MARlddHt@R2&Edc82uayQ4xh$6o!$!B zbPn?`Ca8es^_+t!SZD4hTML#fy$p6c?nH&7f*3);FS{XVyN_+7sp>Gfy-4(^8$q)! zZ-odfJFAkoe+6Cr#yUmKuiI$Hkmj5n1~3HkA^u@a!_A6S*!Kfb?QyD6rO_8xNUQX+ zKGs!^uUZv?do)n)Q$TE~frz~LHhh~bN^=d&3jsWy=~1YsR$eZN_Q`_g)a~gcvkaQ# z?)DGU)Km6THXh)x2dua^1#DS9Zx`Qm;~76cm!7-E(t+tIaSzdKc+%z8=7`roZc zGx%}0YOW}i-%_FBbitgy(DC3sY~UPgzv%b97Oj2*OrAf?I<}$`@~hMs933=^tg2~k zK9TgZ0P#Eydyg!ZK!tLEfL=}i%##7e?C-ImY0cL!U^46~Rk~bFBvkhQ(YOEOq{2yo zJ@DzdTs1PR&>vdvlx;Pb%#e6>vRphT;xiwRT`)ZGtKB(O3n_`X}ZijURWIEbfjW641J&H8YVyJWIHY!nmyxpf*B8|0M#(143A4?~Mo0@3C7)O!1GPV{VS@rEy z&%GQz{`8=^cEVe0?PZBDHF*pdTswIcDq_qpm2>JiY%ZeX-M>S`DfZs$Pte_el&N9Q z4UKvA|F(6%R++Lbo_qqS|ryJbe~k0M>HK z)<*_RIM}X@65Fi=QA+3(i-d3LzeFC5h8mk0<#+eFf&i9*k|E%nWt8;2HL&?{t#MGm zUN9@c)zx2mxZo^}F;2&}6~!rod&)7}zA`iOIMjJ%03z^XSTMI)b)pSQyL+gO&@`?<#gj9Qol}x{?c-Lwn&@ z`Eku(w_6Ed4-KF8Y>V^enz;0n-^hx8$E;H0%7@k5Z3&_rbDPK$rW}d%WP$eo<3IL4 zz|0Z-ul@g#3)OwIPQ$4h^W@Afrc>wpI?89(Hg}Z;0muoQm?d_?ZTEe@{M|s~ns&D2 zu2b`EsroVtZ)PdX?Rhh#fo52n2=b3G(H%!3=o39C7>WX9aB0Is{a~BMMrGYtm6Q}? zgC=2lJCZhA@-shonvp{RFgv%fr26@S$-jbV(xK!xnW(Fqy7FN$nUltIQe1erwt_kZPU!%L*jQykk}{GurvnZ;I7E-w?)S^75!w1 zLwoHWYkNC6!*`2}&E?QSI)2{3?giH~dx4o%FxoxmEp+o(?~P{GnR|wglC+ulxV`O} zQT*n^s9M{>=7WtYzWpVR6W^#6tuX^jmr8ry=>aG+{-q^uPkWb56JuYW_hD$R>T%w( z%f!T3szvn;-0OCrP~X*s?=tQ}r-I~&kGVEr7gmnuRc0d7238-fr_L;e-}CCIh(Lt# z6H^Rv$BW)hJ&1ceA$!YD7Vf59_T29$R)Vvn8{s)!$9$Tlk((3&qfAw{8-!u9%*>K0 zG46?)anaV>Z;Nmi?sNBQn;%pb}8q7@))S zFKV61mygF`V=U@aHi=5U=lxE{GP?{csapz#oJ28Pvh%n!qt4E*{ZM9;RrRjS#nWo{ zq^U4uav-@;P#;=hgWiK`;gf9tOnXWjpKw5S44z!c@CX21t_ikDLMHzd32~BtiIN@- zDm;#&^p36^vjN&5!ECm`LtUQ*ug*f8VtmDh_=k}UDGGu`FJ`9enshHb!}H!=3-ORh zV(?Yd_mfV>4Whvq*T}}kxYBC-FB$=&sI8*kY_%aOX;IdGj7?AD#)pOvLr?~e<}*vg z-gZZdNY*!g7F9-#9`APQx_3jtbDIK;4FU}b)vEFBK{N!D6nq%Z{8=oO zn>|~k>9FtA(cTJ(I<`X+r&2jrgaT8Lpv{bW6RQ`z@pcFuII>6vciUet)MdSOF^onPGV;fc`~ke+k3J^M-W`|Kz_VtWsobVhg4 zk8c0Hv|m9=Ln3f|rF#cF4UnyZ-O)Z+i0*9L-Tr}>?16b=%9IQYNQo2&B-z?%7~G1D z$~2J?1?3Gann;j%q{7a(3|079v&hBS8zT7luI36%|ZD(duw$=SG&qOL(8c~62U7p4Q zeXeRog5EUT#kj`48@97VIvu6baGTz|UtvMQn3sr=gIaJep#?+S3XXAk;@i(Tckb)V z{{F-t-sziNUk}MqdJO}AKy*IyA=O9Wo%QY-UN_3)8?Gk-w|RS)XbfU{TvbdX zZmci6lhubZ;j5RO1S&@zp`4{+aJga+Fb_J2@YYb(t8g$#I zIo^-u3;6A*>8DtkgXmZz1^qS1 zEvVA+QgIg2^OnK!x}C+{J6yK18zKGqM})?6;gPvnd>aHB%mNuvcC{9Y* z&&%SK5gY{H=WR<_2zN+1bcgY{DXBf;b?slNuSqu$L_v03Fu)BM83kg|SS)$7gUlfg zltR}MamW#LT5DXEGIrcnCB$=KU~p|X@Z8m^W?=4J%*q&4D0wU{96ri?T_eF~8q_{( z@&sGP>MZ9@cp{f6sShs85pM<<|;fIV~k{L~V&xiEp{pf9b> zIp{=vesL8D?n&O~di>Zf4;RYRhBSs70-0$bs%YHL*=*`Y70csj65%2fECabC-YhU$ zuMr|Q5`cy3&Oc@bRxgu)x^A#ZJ+ri@IjFN!>V{cz|3d;mues`0PkSqvmbin_(qdOp zF13waPMC+=j=N_##ifmjm};vQ&TX@#*F~y&VVHD2C*jo{-B$VV>la$WjK?9Xh9+C^ zY*r8`1KK0{qiX4$WmS1tTSli_{{_{sRQMCq&Ob=dCsJ1;Xx$AAthLqQAo0#ti=lxz1rk)+2F(e zN)9zq4oI44Xp1#b3DRwyBgXu9H(tE1Tdp&trXLxexir3xFdz`{Q#RVauht5c=n_bfs~Pfj|;)>;HO8s@>`rxQ;eyBi>er=-MPdTynUEE~cfC#>Do1$B#q z(62T`Fq1*Ys!E2jV*d@8lg*KFHn<}w8J^jV10^0Ixf1$poP`v8O5nZDdk%O#+p)q1 zGtOKgCTZ_fa3Czi__3Nq0EaO9xP2^rhb=IzYg&=(@xGc~#53Ng5*vAAD;qw&?IG#H zvuE}vERzY`Bj5S_`#G|q1|P(>e9d@&&+@~XF^i=DWO+V%9_E`SMnvA!f5GO>8`^YAH#KTy%Wxc-8g|ikjr$|>$)OgB>lO7Qvjh;1I1 zS2u1cI_8;Hs$N}%xX@IK>a6ztvdD2&$)x+XFq)h#T<~<0DiY)El;Y*l94{+)NYn)5pdLf5(Nr@DuMD z(m@xuyP+0zLm_{o=iRYdN7Z2E6}J4)hC*xgl=BU9WA?-U6?gu51|=Cj-fCDGed$m^ z=>(;9L1Zw;cz7jB83(@3IG+^S_z?bUZmGm`hQ?Tut%r6Xk8*v7p|sa?0h* zH6^M8@*S(Q7|zLPoIKBGeTHJbmo3Lm{3|Q$fP>gLw>17>;C@_@0XfIe{+V)*6m#mA z7Oe!)65M3Vc{Ow@8b?yJXN{tZ%4i6d4zi&qVD^V_YT<9Nhk+;3S}9b=%)d^06Sm1+pLjd&F5LHA}|6I)cO1Q&9RD&&9p>f}Bfj)93$ z&pKo3d31Q(40EPYNnoulb=vK^>_@AsWrG+d@PlFZBs<5NR1q(+ap${6NYQX5<%xvz zGkb}@e?M=c!vg=OJ-RaMJ0DK-o^e?AijF%|C#@X^4Ge*Jw3=0mK(hnJ7>V(muCyST zhiz|`!-z?_2*3P|WM|N=EbsGNk#?QVeP>N}MK9+AE^zrIoc+VH*Vz_FmL|%&*=v71s!wA1-6eh2|5ed8sK?j z)r!riu@Jicoh30O$KLnBWE-c!0DO_q8^4VfySc`yphXs)9j5!Lz@)bjX4mNABm2^F zuJ)n-Mr;I&p)B&hcr94R!#}gPiJ*=wGD0g4hVh4kABtEW4HVt@U_P4^a31@qT_zY0 zvCHV0D};>WPX)1fuKm=wQ!-wI@oIn_I&9+KHYQ5T{O_!81d~%CbLl55zhdef|?ae)&kNxJ&FH!)=WY zWBAoTqMNg}{%XRjOWS2JmMLKO;{?26sO)v0lk8Zn)V_{v<$QjoPU?8`b^(`mZj6Ke zg{!c&8^ti|iRb~tm;-4RX9lIb{i&{ebjQfJN1C7j3ZX;r#K9LA7i3G5Hn4%xubDB@2i2aGg^uEw zHOhnQFsa?58b^&mn+Z!%v+F--n@l)Z9HE5!ZudK&XoH&So4(jVR5(Epq(xv|=bSu1 zGkZ$+j_4~FeTAr=U~BwZ(cBK1DmQX@OP2`>hXSF4VhtfgQOAX^wa!!R z9vu1>_nwJHSkQ^^{cOvqdeAid{lr?@+7kmTYcwJinwnF%yokA}xw-FFYAPW#J>Y5I z$v)B`Iv|U&uDAF?%gH-V-QTaX6sqDZ z4T8j^NfJSPQ}GyrwIxOh-Y}jldPN1D$t}Y{q3meP)6gAlEC92R!VhTkxKD`yqA&+5 zp8-Sn#JnYJ1oun_VT(-yKH(s+k9i)+N0hpX_mLlf6<5M zF4v#}<-Emx?F6QIKnup0u)aq`;yy&CR&L%ZReP{~Vhj>v%p`{?mna&1cm=ivynbmD z1y%Zk;sy%DYTDtN80;=_A};_u5V;vpxb$H+njG8wC3O`nVJO3*wUh#zdI+<$0?M;# z*ciU(j2>j?t>u2lSKTLx-5lHHDk`YW)^S z9AF_D;ou{~i~EcD*KqjoZwqa>#Qqz-x2S<03g_&skiN+g8saz0d7^saBmmDT1xI=% zF7vJ0eRylN7oxkDgil79ka1k$k41I$JJ;IYIK;Iw0m&-UBWv81hc-?H$+P;mHxK(K0#WoLrWG?iSjg!C|pyvx+-?ikcqV6|G z*7QLJ;AXelt&K8xWo;1mb?X!e7mfn*M{SJ$Q8(dFB-uSGf;(tn89q{HHlfe_DNw>~ zyXnTps`Z|{(#0&yeul+C!_i|4!k)f;^EGB$8Jc5EpBsIm2q{B)NGUgLy<|%9UX}7T z)&29cG~>!y?IFdy2c#vq1KRm40hd< z)3*o;s;(+B~58LZbD<~aSG<}_%$G_58svAg=nqQg}2N_>qQ@4v` z_y<0fh9=`fI7IiSC&T6!SfRWM!JjhL+%M@L8OHw>O32qRbK?`wB4q%o_f76s$om$lT+2+S(VcYN-sg?Pb%%=1O*;6xwZjP($JgM-v6V~>4o3SbtQD7PuKH zq`<&A-c+C78%~N}be1k(&DFr9O^7Id6mA4HpJN|9fWm&czMYQWx=UNBlUzI|(-W^# zMY;w~7x8iL4QwF@-m-X;CJk)^r2*W}V%O9v5o2MfE6=}0d~1zzpeyh3zApH$>VObb z|E0(`r-Z^FeZkDioGoXxmdgM)`GFU*)DYe2JGV-mrBVqETXP1TYc_YgXH;MFM843) zD>zqzcIsA$^$lhCrF}7kJN)|`BIVa}PCO8vDS|NrBb;Eac^nprzSkIYk(I0j(}kOJ zhlsIGeemckF4V1#nU$d3MDClPI=~Y0m?g+ajNLQsHT2G}UQYsSIZuaxID{5hyrk&t zJiF0qp{llcc#2Z(^&<*g$mUv?SbgL4Vg!Ss7f7ETwVoH{lryjSbge;Efl8~|pxm<@l`|Poc91(dxmY`^8&);#=toi*~jCEb}0ti#N8p+a)ApO<5WUabi1}%6YwXtOZ|4JdB z_o&f4!lm_ODf3!`aR$dK%e)(E(n*!EbbBkNeOUB|9tP5Q+{##KT!_u>)pG%XrmG;- z4AD8`T!tBvg68cd1{9_HuL4A}TFV?uixKi`P+K8xZfyHl+!t(YH)-734&QL=^*T(0 z*w0WJ#8%s(Yv=Aq1CuG!T2w1{tjv+7&S>qXU?KtE`AB;>YVY_|9mVg>TH_6+bOV1g zs=s&?wt?AJiDhyyfGXO-dE<7Ud#@`Ovu))fSLWaQ##WxMpw+50uVDLUB*rwGclA`0 z-v*N22vgqEZEB0Wn;p)+qL@xf7)X_B3!Zn4r+-3lncr6}<Za9csaFAWG7OkciLT zhS%EvERC;g9*;rp;5NN#%eNCNe*aqH3H_OP(a%&jr2*X@O`bBL zbqb@=r*x6H;L_TG&8A7CWWEBawDjh*?3UMe#oCAH$@$hR;g@AT7a};9Kw#xUKzd80 z9Y22lDnTY78o&JHs`M%~@VV35sW%9EKxodFt*XT(j78sVuXf0d{Zw8%b*5VF#!Z|e zwvBu^>?7X|o810)YN>^cRFyVR{t{rC_Xcsr#5Z!7#GlNpcnH{-{GeArbraE;@NW8- z3o!ipLsF^lmM2$+b|zRnq0<=AYWt+x1(aGfW6-&m4KqD#YPh-MWfvPmo-Tb4B)9-# zS_cH$+z?C5nM}k%chde5k>lb%KpSw2F-eLMW8tLePp9YwAbke(V_#!GjL4uqaOd;5 zlJ3T%YxRbQC?hSUe`sSeb=lp4M;#^xo4t_JyMOhiWeC|D%+Gl~B97HNnIb~O*3WZ5 z!A69(-XWPf;p-$~bx0eOJrwcCe+4GV_43=^)nG#$6gq3b>DDRhcn{wL2Nu%)l+c=> z`d=F@oU`kDMHv*Y5l~ff<5J=ZOeg=UemRrRh=CGIO8dBr8@uK=T_5miwlk2;zqxZ) zC$^G3UgI-O+^}Bax1orc{waLGL+$Ha=h2(NE?3hG2GhYxW1a18j!5Us*u7%W?@BXm z@>rGu?nD>41{+YZ&DV$-Q;3hA?)hX1heYKKY~La2aqg2b5m`vn^4z~l9HBx)MAEku z3ZLOl;-8e}$OKuq zh>Hu7?lil2DKaI8L9BmFPK1MpUDem&6{syRn@IHTA&k5+8_Uy_aBB{?yfNt!l^wI^ zNxe_2vF)$0qaW?CNT9!Q9111OYfy{WC<5$xDav2Yp=NNjBl{Rv^U+FFGx~zsmr)!& zl6s+1%UDKRQ&Y;0@}1RweLPp(WY6|=2!=GM`6wjQ4wx(3w5Db|mTK~z4$77`DU^JZ z$mm=2W%cT*1#|9G(b!KcBv7r=d9=s3SvliFv@y(RzBPhP>!h+3yy_hG@R`3f$nHsEMm~QzUQHT?Y!<(xBAoe3 z@4>7va5^MF#rL&stp2Rln>61x0C#Mw>o7j*^to8f*NZ@kmU^#{C`&p55W8b^hVJHp z184RmykOjQ3F)VE;Na&|tfg~dA}J8fG5ckI>XhQ1pwH}xn+gGLkzg+p7;uH=UlS8Y zNq1)rVBXsomJG4Z7!LHE+)d6D<7)! z+(M~t#dc=UqX&kPubrL23ukO@SMu1qcIcil{i9#wU+CF}L>MwJ&F&9r z*h&hjo3+;l4wc+!f;iZa7W!pM2^qNj+^@UI1m5R^O2!U+b`nDp$Jx2@IeI&WMsujN zi#5#4I{B!YRv4-%f*rKT<1E2izmef0SRuOY#*?;kBXu0drI7=Bg%J&Iw-aPGg&lU; zLW0Q58vXt?7bJr;GZ#$_|gB^Uid7C&3P^N$qu_M{1ro zOm?KcU?R~js70d`N>Z50|HDB3?_=LE5^j4I4Crqnah>5sdKCjG7+8!Od*W$45G z{0qcbW7kGBJWDoK-$3m271q@ll3c}s$R<#e@sm`k=h~o4`J3b!Dc}e@SwuK$W2=ZlyTUvUki`PdQIx6ip5_rue8R{r7lExU7dc-U0xT|FDNx_ zYFDy`F8L=RHxwicD%t4DquIuCCwGTE?P(Cw=dYNNg3>7sO*8HBrKx_S)=e{knZm8?ndb@LAo13>F$ya>F#dn?uN6m zUcc|0^XFNt1?=bEd(T``*UT&q01U$xed)cj{)2=8O0Q|l;U9K@_*V?&#}E1@DT>+E z7ZLCx%AR`*^;2XMMIr@&)9=6lWfuD?1p;O{6Vc(T5w+A#>4fWEXYCOG=fFyL1WDEk zlpU>bR~*xeg>QN)%FZPI`<7G5&EOK7TcF-zJ_{KG@<@+V;XrKFibT$>X}x%cI5`83v8A`{ltgM5mrF ztpnD4iP?3`*VYuv;RWPS1}^;|nQ7VV7~uzY2D*i$4q^z=xH6G$I34c}YTX8RcAUp^ zHmr_q?McSI*Fnd{67ONDzmDb8aGD1H?D}xKAvJT~*kDerUPaD|WS2?|R9nW>U9xsc zK3T6r-oBCgs8SoY0uC73XTX$%5)EN3(&B4Z<8lxFEJCpR@IvF17kxZReE2i`Ab0Y` z42Aq;V?WeRLYW%4!vTh1rUB{UOP_6JeQB{XKiAU{DuB}M z+%)>WM*%*_h@5MZKn}ntk^X&EJ+SWkLmQ=THtdRN zy%_gLZ~3A;jHEIs6p?Ss9P9pTv=`*mJoctuVO}|&k{S!8KOob&oICoDrjCGA#)D8H ztGsR5l$T)$E9X?=TjTTz{~X(1%ngm@*(rN`?sQ74~ytdc#fuD`w+|2VOn(arZO6Yt@>97GBFF3O6y0&pfCni9E zd`@4Qo|gOWrE?zMtzKj+&rfq*#k`^d)pn@xYOWGbX+7fMF`+t#8=ui(sOXp2?PMyI1U?Di8Mw|3pUohQkBJMrky9_w zSm~(%{PHjJz5N5o~XTa#2hl{#-u7_zrF; zm;U9lVy!3TQymWU+MQ)r69lgD0PU_hw4@0!?LvjXpyxw0$;%X#Vsq&-@C-l&~?1M>CDc=XR<0xA`*hB4^x3dqu` zju)(9Iwlyifb^|`phKwLuW&9eam;8kB z&ES+|x2(KtiOq3B7XX-Ce0;qHL|9**pzizQ*3-Q9@t7};+{f%O0QP~Pj@ z^==rXpVV2>uk;pB0-|8%BhsW91FMus+!7j+- zHx0UJ*1@}nwkwT2UyZk=b%!GtJp{0U?j<0he~xLIrpr#mj99zvmZkK{^`?rL@3W{r zSTtGAvE-DC`O;5fAYdw6!EXawS%)xB+EipHXD>(20^eS{+sw~8r*F#-Cl{=|?~_7L zm%F5cR*{tZ9H_&Qd5@Byb-PG~$E3Y#g{X09fwskQ>2X#_g!j%`+|yxlm6zZ<{pXvD z&(k$bnjT%D83#lXTH1@k^W))Ea~uwuL$_{t%U%~>>s#%**v^_e-qsGMDj(lZqKCVK zEgiT{rdz$Or0w1@UQDRZBBF&g9^;GZ*9GkeU43C zGm4A1cOjKK!I~~6Wp)fDOc@)}uKR)f31iJ#II(pe4h%Q`WJC@2plc%(cnVoS*@RmzlvY7;$f=EhpVW;Q)NtfQY{h+(ijD~b`!Y(sxdhZbVxD<=U)(^QRmREg z<@D7V0^+5kv9}BTO}q59MOlsX3tm`Hh_i*#95^-gk((WtjSlv{Oa9%$)EY@Q5cRbtq1;92<1_pU$svZGklwzwpU%xM&$eluA2CKl=n{jww3({RU|mX4_)6f zxxYkcI~L48-9}_uOE4GHYq6_6tymanXGfIZKQ5TGNqUWKexUau;D1s2QC(tNr{hkq zOFj29o0$FR?`KsIF{SEFl5mm@Aty(;0`rCe* zg8Ke~fVvz7$FF8_I<;T1@9#S!*7a7%rBQRX%&e{7B1iW##H5kp%-?QaD)e*5M1LmKXtWl8E==cE!rwi6SNpkXV}&O zc12}d2)0`QlPA^B21ps`ykJ89wvrnH;`pd?ZQoF%*<`Vt=4(W`0fcowkmM= zxY9sT;+>hew9sco`w1t{^;~j4q?hT7Y$P7HsG@oAR{{e=2 zOiuoYeTB3_oU10_47i4dK3B1RGd_3lU+}f2%MHngRKO^uyyw_g20KOT{g{{7a?SJ! z)>9u7(>KgM~1q-B_lIZq1YvrOdb`)pjNVX}H+1-ebVVo{$ zOEa!kdJ|MZ73^cVYeA);8scOiQWAQ$x>F)^IE%iwtTxQm&pG^89lpCYXh$q~@v z9L7UPr7I?9MTTUCiqgI@Otf6NQD~3VXYx_`h8PL`Y`gWOV2ELmRLANYsRh^YUx@01 z3O#|Zl{(GA#vmp3!*%g=!+Lb>O7(L;k(i>RQyV`ocKXYf6$5Hdj4KQU!mwe&Fh#lY zBDf4pC>0b)P8(wPfRPVIfW5zj%lD#c9Ap7S61mQoc0Szlys*4`4|hReC($}xEuA-L zJRKkFRHrF-#DFw%PAuyJ<4+H4bQu{tdF7luDgdrs{_ejg3cMRbG7!%09o*Sfwq^EX zzb0n%n*W;cFiB;l7*9{*D5MsG%n z(RPp`v)i`m9`X9{)2jhq6b6UKdyTFVjjI%}sc-(zg+CR-Xw|}7fkv1J z4%qu*FBz{7(qof}OKX|llS(F-N`vb8*JK6njI6# z5!U?5CI0+1J`uR$PK%uNeveys5V2TOt^VU!uC=B!Ec;cl01!_t!xn@`AaS&@3pKue z(4hUU@v;grDv*kjlA42Si(2!RQ7BGZ0B^pZ9#8CkAcLAJX1*`+pa$5n$h_iq-6X^X z`>~@10SuEKl#mO@gBIiSZL+833~u~tUemc5F0Z)x!xQiXh}e&xi@bX5CmuLRZ-m z+(DV4IXh!c_C`wq{5b&1T6uY;$_8>#$>vxLcNgMfWi#yaofE#kx2zs)CgOmtn(?8# zqfT5;6Hro`GZ43?NNhtoOva7Xmq2pRv7Hx6Pk+AwU<3q?I&V-LuLD894#}{g?rTbmK??{6IUfkzqDvj?@5g_7 z3oElZWG`B(OOay>Wdz|zkC#c8i>|am{TJq*PmD?Y0x@c?p@vut`Xyt$A2wzY3i=>G zazMgI@`wr@n(&4MIp}1_P;J1Q9N!)6f)`spSlBfqGVHB{YNUS5i2RU%gYazN~4j=5tqm z8{hi&M>;Y}ZZM-i*A*>5BWyN%xSdQ!0!`z`KD=M^>Mk}llAH?%Q;K@DG^#C&Yfl*( zXxk3!%9-(c6;C*wA%$TY1kF=ybBRX}5U!pMoCLWs)=#d@Q+&I7SvKyjwb8n=_Bmy} zzL|}m)(mw}&%AR!{!(sIMaJw~iM{i*DiYa>c<-r62SlLrur%9fT{G^C;kF!EZg9}j(p3-ZrXQOe|ym1qE#7gQB=ReBZ`KdEKBeSq+f)^Sj3f@;5v5UEL5%#C^FUW&M4Iwzg zoanzW02>2J<1L83=_-ScaH^&_d88a%A3VI@KU^~@Qfmfee}EAAVpK39R5w*obFDoB ze-lkb%nXqF=gWvk?t;EEkQ=x;OeS2Z#xr_%(HnFLy79Wn1@YZ_oFP0ryyOG>Q0e)Qk?cDYF{N6AJ1EHy z$3avnW5(p}`f?@k2Od@cA1h*5yUy@1^`(UJs`}HHOi$^MbHfVMqpNcp(>$F_=$Z1Q zUw()@@s|KYWHy1GH|VGI#}D@l-$HUbJ6mmms{I*Z#go}SuF30&HP(vgBFUjj*PNH63E)D3kUVkY z1Kj2ioWDsAD$or>Nh~H{9Iou@9ErvjrX8L$(RX8X;Pkfx@WEes(~OwP z;f!sz(@Zny?`eVW5%|~&gu~e19&a~P>Q5gK*GO%JZ00{|^zyvl-`!{~Qw(8<9=e<_ zvz|<+FXO>+@dW0wY}AFtk$oaMLyOQNh~jsbC2?Nn?Va-3-H1jtlI8N-n2A$rF1weS zcu?Il_KEjKjQideu^fW6-5{N6DBIQA8(AT!ypbz=C%9}zSPoSoIt-z)$4YAct>}|d zqTr)f&>GFYYR+rKikE+80q~i(T{wwLY9@Q}9uyoz=!vO?o+X+TaM*3=Ajr*cpm~7+ z0ra@RE{WjkT4%`Fdc4zi=Q5H`CTRsG=l7UJmqk=B!n9L-OG)gxMpb+2%Tao}HE?>z z6ynqOcdDYk6_t~rJR2^mHnZ}g3B?YZuJk^Z__BZ(U8CQ|07W|?LA}e!U*1b>?umDD zR9MjXT3eHJP3r=c;6_I1mLY`G(43memMaE-Xe=}^n@4xHZ`^LnRaMy%k{-t}fKV)9 z6{oJ9!4K)ms_vkMHq@?m*|P!Q1V(RMH!T}hS=>dRTT~e!;|ky*>zRf8P$XONyno$j z@r1qxyEdlr2I6}N_T=}mn1vTE0ae5u!lb&y-plJRZVUF?bVqVc$ZsjE92>jd9kl1jKV?t(~; z_lfPUW|Lv^f?LyYfu?YXrB%$$4+|f{kHRN_Ue$zQ zwK_}mLyp2oM|-Coj!_yWZ@YS7{#e7<-f{x0uj zprT|A6h2}h+e;wn$sM^p>Acd&!>q|{n4*i^Mf|O$=Udbu7)RpLBnJpLHL|N?^@}A}YzNMD!HoH1Z*;y)Q-#Vc%r^>e3SWsAkR&#Iq_I z@&e77298t)1$~cil~r#wJS8^VNsl@PH6gD9=@S$dwALEGna@`3v93t_LFPK)$7_l- zFi)5xN9S7M5Z6c9$bpcB55ZZ@_=TW{H(4RJ*044apP9w4sh7jZ-nlkgcUN{?9RIk2 zatTOf^y(c{g*^K@6n%kn3P;ha6DXR8hLJKdK{T={2x=`c9O)_tuwuAKyqA9%Lm9vK z;MU=@wzh6hD&V3?7|)G(;SzjOV;>geFF^1~hu`-*O;*TPqr|*)22d4c!!izAr>WPA ziH$&W(aY8BfOvQQilX`&1!1#g#+`U8 z6vl5LsFZVUn<<@9zGb~XDEJr`H>HosP8;ME>&=An`O?C6W->%L%8zZqWh?l040IY( zmlg5mS8&$fNU1rnU4MB@RM^ULUw8GC!v zwMTOyS>!RRN|di!iHd>k_34Yp6V;R0l81Q0|u`W3TeuR0&?GWcK% zV@6uJ_CtVCDb}xWWxhf6Zy?Rb&MI<{*5@AtbL9857W7DCajxG(K%R@e1N?p$EadTA z86OB!+1Ag;5cXG}$`xc{3@`ZMbR}Dz6d{}pm|Eo^fv8i-RRF$0rwB+KZ}`+BBI%6a zi&pqVCi1iQb>=G|*1c_xa@|eK)ok&6cXQVq{2dRZ*AAOB94`<$xI4qf0kmY<748Q% zcuIm_F+8o6^qNxF?f^#Y9o`JkS66qAYQHGid2|Hjj8qzDv~yNA{BV;a)jxmkeqGX1 za6~qrvz|OtX1&2Ut>g6&QsTQ7vTAP4?$0}Ze6=HDa;i;`Km2tGZ^r?@lxarhhGXww zSvIF+)c$x(?Ku9I&jJz93bb%}Mvz2pt8F5L`o&*nps6MZsB5Wji{B(=uVWJwU7R>j z$DE^uH=OZ8AyVxw0RH1nEsx$zo$+KP5MyY0FV6=?K@IncfrXt9eeC8TOA~805XU?d zPBWu@Z-2Pt9NiylB8mdbZ3s$%;vI;ITrhm<+N8ea)`$7-cdqQeoXCSXJ`$Aa@V=_} zg`XgqahoI|gC#Kj?RORnB9=EW@}qIPBL@}>&g&+N4RLjmAJyS9?7^rwX{MK%bhmIx zyi49AiL4yl0y$bcraj5b7Q3=TR5w>6e)xA_$UO8>qr(y^-%PICv5#H#v$aMl1AM>)&wAal2i9Jv)9k@gY zBbxc^J{$z&kgrz&L>d$Rq7QHS(m*2uwADn9#9!?^{L0@BkFK^I6wQX9M??|UsF6FN z#Sc^ab=QKaPHl6w)Sgf*R^l{{E%==7j9c|&p}%8yvEk}9YG*X8rDFp@DZ_u^bch%8 z4i%2Jz*rier%opo-PVA2cR3!3# z_;3bQfD6tjrZi5S*t71tDAIzeW%`5Au#*Ebzbd} zn*M(s@{c9fU4hng-N_F0yqjNU@$XoT2A0ao85tP`LNqID{+@yQpH~?Q4m7Pqd@;x! zzlj>PUNUKe`xHCcr=*7QYjPnhRv2Y=Kasd&&D#t0jm^zqXcSW2ZzluN@@@T1c?AW? zs@*ht8>b&M+w~+cxW2sl>W@%0rcjLc_`_s}f_yOh{^aT^>zm-)@W031hWBYWtlx`> zCgxJp(t3~$to2<&?Qr>3e)$6nuR!eUTLQVOLo6&Taw~L=b(`ySRX>4oX%CbF6{+r3IITPq%*Q_ZQF`6V*)$nPSF{1*%nm!rFX_-L5%CNB zt3{(U&}yhf1#kP43O{*D&yC$cbTFtUWww$0ldA;aL6q%`3~lH<8PDV=?)^ky2uu*? zIXBERHp61wHc19{>~8SeB=7MZmr7@$+@eQ`LuLvd+@L&;IB6C`Gi zn*h;*!gPcYCnjuMa(v~3+!tX~W^TUwy-grc5W~>`>Z5PQw|pTc=GsrTA~b;1EgzfqZs zNks5GrS+#i(wRbSwl5b%Kz%Xgm~!#28kfqS?5I1*gxU^ zbhqr4-ymM8o5R5mRM~H%c!`nG1_!0)FuBsOsc*pA;DU^{4GKh-vl&J~R~gW^EYWNU zEZD*DlA%%2P)$kI&CP!FFvw{uztzN5kz~L^8$JC|_pw7-f=;NV8b4kox&vno^Ok|R zp&*-ryr*O8En$ug+sYnNKfcBb30Aqh!oumAd&Ix8L`M$Dm$9s;m8RLV%<=AmKc`F~ z^LILYpy8ZAJNViBW(yDv^3SBU6{xtJk5^t9bcN0`F0?pX^ZdQYPhrI{1r0>RBjs!8 z>FvF7qjb<~lcbDggGa%50b1{b%x?r_Ad3gzUL$;P6h@5z1EXjw+2W#W&V>Dsz7b3_}k zUt9faDE)xBp(YS#wcQSXUg|T$PFEDxq~zOXa?iun@)K&~!2OHQr)0o`Crg0H{nZ`t zS2=dbUeGsG78hFyi}X6>yoUfd@BBgIbUq;X&jYEO+#Jo8jLye99AYuJTti#;Uk~CQ zs&)l}W8|A^i~57GD!SUu#e?HCqAI{e`$HmVoX0~aG_-HLH`m9E=tharM(F|fQw^Y~ z9|Of1=eDAVL=p#z!V-7qpXFVIePXr0j311}8^RcTqel2+H=pNawYDb_T*bZj`)fOV zz`+BGY#}5HZ1_{W0BC*B$-afxL0{^WwnC`;zeb4nF!?mOUEv1OtHCv~^E|a7dLHXh zZUI$`KQD#2Oca>KLEKBv|K^lW09PU-I&`~5SxH(sN6EDY7%%q+fq;-W6?nSTNVTj@ zW_kxnVhDXmWMo&9s+yrBvj0>EAB1#!piP>unvR-I8kQfdW#}+CGYSfJXL@7Eye4$$ zZaDwm+iwT$J91uWDbGi0Tjyh0OUX4Nz%&?l&`aXOLbqatzCkSjjUa;1f}_bpJ!B(2 zVjLApg@a~S&k&U>B~zQBD?$E8`~o;Y(1tKdybBOZ=&_BguQEd34r#7xGW`9M$WN`2 z$U|QkA3>mut>4*ikGN1Lik-(*e*T{(%P{^$yco{((h~RULRatwYbWu3dbu#u)X!kR zmHhl){9wqSQ6y3qn9S%rHiJ=wy{!Vb5acyS;Y;FPN|fO}4URO$Bc|Cf{|ESVXOIDa z)Bp_b;Sr5abL#SJz(IB^{2w```vDDrPXepeoc*D$6;1rSb`Z4XAktvyVy(?`Rofgw z3G3gR_@!pR_(AOPmx@n~jg2ktfDcq6R!y_7N2(;B8V+p;4I&nx;Q}o|H!7_0DJXSN zu&GzH0*-?lO`T1QoWDPXm1^S<*jMcJFubI> z%i|&R-+p-f>6hy2>c;067oRC-jeMg)PUkZ*5kVL;M>F1nDo}qh(TA}YjuSK13ic{AvmDi9(0hG&$n1(4o^GJJvOzLY8^1G$^&*p!iqA)l*xb^6Auw$2T^+E@kH{gk)Q+_ zCPX2@zy5R^rX4`MLWprli>OvCZyjZEEr05)-M+hPzW$!*ETq#Myz{%S&f5HB8^hNb z3!;}NZ49nD2>>CP$} z-6k>U{@F05fIk8(k|&wP9gfrpgoS|-j?)r$O7Kpia2~BA0bNHPQDE5FxF)SB^)o$J zxK^D)R#+f%;Zlkz8IF|m*x)d z>?Bevi}amd*Y6YkT8q?Lg*HQTgqav^rrz=$4&nEW1|aq1w_=8Lo@!Ojx~ELuEpuyBH-e=Gg_Uw`gwYz2pQzdz}omQXN;j-Raq#+-0vTE%-OjRwo%`A{I0Nd#aN zN)|FLby9=8vBo(kXv5(9>_hq%w$FKfC~(gKE&bXkPd7nP&19`KWO>?QLr?rZ-^b&@ z^+JTbQ$z$FwbGBb>*XPUGI5wK(5%l=^j?#k-xi?(QM;mRr_A*q;tag$R*Cl36Q)y}?LsYjKQE#E@Y{9Cn>O{KW5&zOyOrvcYaeSZCj zwf6bjjYQ@L2Y&r*j7=|z$CpBPxcG$wfg2Ci;bk~5+Fg|&dfE4#X}U-a#q+e6?C5ZN z+qt=^>0u3+O@Bp2MU`T(VfpWVL0x@t$AdYCIQK5~HCr|W(UDsZ1FJaTzWeNOKx4>} zV!yyO?RMWtC&`Ne^D0Bvo$-YVa#37w@h#-PAw-`*`o+wAu>Sb-!9Rb{g}(4%pzvm& zlaNp4vMVSj$*4IxKm_NpW3emUJJ)}0pFox-$!>lq!5(GrT=zCF!&qia4dM5J@C88M zVUeR8*^NuFK0HQ=;NEEI=qx+jT^;EtX=^_&pJLNKU?p5b%>1{(6IjSq1u7*aRY@ZH zFdofIsIIE&T#KM7`fzRc-o@LQa)ZFMG5jXy~6^9i-ifP?yY8!R__ z)b?#Hq1Dbmgo5`svem@jmJ;e3bGmAVv=YOr7AY$!tI;UXC&evqDKsr=A-{{^HWh}{ zj*X3@Xjm+{_iQn@G*AAaEt5Wfanb43P{*Syd`C*_3PaNbhyEq2bkf2yx%h`;``5_R zfG%vLqy_Bm-&h}s1L%rj_Z~0pAFsKj@AZMdNC@O9@mrLG@{fRQKZooUZ4K2-q0|+IYz<}V4=qV4 zfXA`SWE|Kws%yo{ME3gq#CdCAk;r2{mPU4NDt??i3*4WTyn}oF`JKv3zlWp#fCAI% zUEb}l;_rB3y}Q?M>}6zRj$}7ZwZG*8drkXgH&TBJOOKk586a9u!975fIaRFTq~zpu zm5#?|t7~Ouw(~@_v*xz8SN3;J@AFp^q0ay-4ql#8n|*Yz*-bDJE-V3ZG}o4pL^k9l zq#%rT82f~g)u5-xi-yX5)Rj-$D?z4AMdw6r53<~=ZH*_9+A>oUEf@~=k)a5UbdPvS zXoc3q8Ru2}-Rtxfjx)1U^y!A%qai{gG-!ntv(yxRTKXV#hKh6`lInSL$4;`MXJyG__x6oo&=)u5& zKr=jPvBUH(T>HeP$CMx#{S<(AB*z7uMBk3m+zwr6a$UK?hx%%@kD`l)(M9L3#~lwB zT>8IlrQLmihMJ7nSN$y_KQU4l6|}gxm~n4!uXH*A6&>B{E?&dh#S1k7Q4PQf0yFX+ z2cQ0g!Q#G;^!LVRL+3KfB)6TG;T{H)S@rAdC zP=!AA;vv%!@iDGG=KCR2lHrJQNhA#18&<21T`}&|_p;m1*n>yPtI*~R_3Skrd~^T< znCEp5J~0Yam5p6S3pevR4{I&ba_FQNHOo((rH|I~Ha&yC4vx_RAkjK&*2_P30vFT; zogBq&b_#&Tr82#a3mhCAJTc?yPK{dYrjr52;d_UoRGNSM1t>ULd^IC+$D<{Wz=5_+&W0kNX2oOf14;g5nn6Js-QZV~+Gq}dsF)b}FX2S%FSJXrh=q$4dcW3|c zuv-0LkjNRorZNTm#>scok+nVyC#6Z?gV^qCL1fJnv$Ho;YP2&oRvJgA8~sm6I)NU4 zOw;oV5T#kUa9u=pe%$&hn>PVi7CZLOthpucYAV48LI=YN-;3%xN;x`Ssw(KWShuWO z1d##TR7N4ajL(VZ?-CA586U9b5$#WpdS}#EDSuser6xDGcxttZclmDu|J>8_ogk4R!9iHzd0wyLR1}x17ds$R zf5->Vy6RHGym|}5xK5h&*S><-5*8?YIXC z{q8@8Np}tk(D=>l#+{rL&t~hzdd$Ve#bpaq)9XfH8L?KGZrdeOf~^1VhY6gBcL_#R z7_$<*e_`|LZcCy}GG+dtSu;mpTwILa6BqD0Y(!!IJ$Pg;kmUEcvT|X!4+#5nneX6G zu|Z;Rl}#5@8c#dr-MEE2J3BXBs~v&U!h}&|&r>@_=@R6Au1);qY4B=8`*-N=#nWh5 zAMEXEn6sp~6&4njep?N|;vNDA1z}8*&Hn$bn$&d#P>nHQ`l@E$`YuES*Ck64_i@?v zM^BW-53I_MxoNJuDF&v&68~snq9Z77M0Yj75TJEyT6&2Vl3iL=bqJyP zV`zr{KOXom*XH@rC+2`gn;lh{u~&s6?n&s}j#)8qDHKW$pAY8bVXj^81trq=vsJeWL~w9f$C-9)w(IH_o>HDh5vw?d}z z1e=mUj7-z1YYYrHk%6^VZRR{IIN@$_twtzI_d=p(;=u#f@Jt=r-!tmo!Z^=kO@`+_ zTJ?b7n(fFh6(~#=Oj-mMWy03G680I2YZ%S&=XPYJBAk_7-)YSCzT(9`Gzot^=Bzp3I+q#|E{&y*T=4(?c ztKy|K&g~NZ=nq3@!v43M{rrP_J*saC23gY z_&^@lOX7(TYYUf-u1{kxXYFehafJxg8ptcp?g~BSUs)+HYEP(M+^A|iG>w{s)6Oon zsPfDb{*sqxNuvYxu^q{+4?n=EFc#1%+=WGjSHq=(`UuhVA^dfibK!E#*8G}d!*Z-w zy8Xs{9p>*9$@D(JVIO&YjgwtnT@Mpllg^ert`;#R65eW&;UU<6-!1#cPV3gg?k~4I z4H$HX7e~`7&l=r0U+&K)Xac!v_!xR?P-9YoBsY9IexFFrvgr^6%W~C$Q+S2Kf;?NQ z-0=+T?iQgVcMm|+WO4%fiB)M@yj}UnU z_B~CpyN8#BX}B$;(rl}xKgvgLriK-wE@G4EJl4wTp%3gv__g9vb7b)D zN{a3mz|G374brG+^0L!1`hRMgPb!jXrSTvv7OS=9)WX8O26XbfBOnaYIo#iOUq2t0 zS^kbH_3i|DOzbbX`^donIIGf&Dp@NrG62`hi%tFZ?ISug=~K5H3*+G~Ac`rG5M$76 z`v}#xtNgEn`WEPqIT^Mh!6 z2kg@T7ilH6dP?zW%%e5Onb^V^t|j9-+&>zRp{)DtXv**01B~wn)g04XV-b<=&JI6IMWmUo}!RBi!<$u z}LWkd}+gicu+h z9nn5XMJZ$RGds_L2l?Nq6X2pvY1lmM0k7{g`|$8E@W*@&<3@+S+wR`pGRH=&LK-bO z`F2W=+bG&Upd3RXtD;g1Bn$-76Ry57-!?NmjP;X{kQCZ9o^%cM?u*w2|= z+Z$sx&EPl^HG~qA(laDW&cMqv8P?824sAN-V8(!#Bod2uW!RHq6i#wRY6&CCLN=&U z*GmiTs*;omiPQ)Zt9IqO19svR8Rlh}qy6HKnk;6b7sBbav((-5z&5Lxg`uEz%cJF> zw2J2d^tZeI-%q?H&^@<PgAGEvkx&isyJhcBWSBc);bT<>}EZ+HCVCMuz)F3sYup zWla-{_TfJSOI!jhU9Rob;bMBsRkg`Zl(=#dDG)8OB2zb87l=r=hBks0YbiW1NDe+| z80}_C(uBwq;F#8b{r0qMgS<}AhIuLifhIPcwKMuupP zenm)9mc3Z{_L-27G*M6IMg^7+6GZ_gAA^vPb+7U|%ca8P=+(qHcU)hstyN9yiCc~| zeayjw1T**0Kd|&4sz*wI>OAbKxMj6CaXwzJKW&`d;6`4A{siEOi}Zs0d>12NDz3)H z&C1Hn><6>R^VfexTvr`d^w?<>sP}M>&(6B&5hIuPZw<5om2w?%!q67=O5-qAEC`ua zNtQ;K@iIix@e_Q+pg8)%vL8IX?g5yCd4BKSdyMi4y?iO16rDfNLv2u=g^)O^W^9iX zihw4LPG?Z523R@Jq`7=qUp@J!J`2{`c|GvKZEwuQOIeAUE1Nve4}VqkKdAJ$)+Qr(LqiA@ln#?A zQ0$n+Qd5Ab5+Z;6aRq5K@G2jV<fC3ZM@DXVS~`*oLP4O~Hv4(_5bK2ogNXHfB4 zu8C;CA1PS=6(9c20#SpCEO{#LIB@pow%mlwya)VC25vIc^Dk&_~pz&c;0p<@hlFZif z0N8#wq={&X`S4;fa5+sHDzU(VaHY!ZA7ZnDk*qz>8fO6l3s;2SS520i|AM>h<}#88Ax z0=aSjjt4$mSM!{@>tq5E+8yDz`wkn~Z`?dMXk0k*Hl+yqqY4x+mBfN{Dh33J=R*e* z)5IWa&MzUdsk6dy58?*cJ(KG5B|pFK%kN_Dg|M#GVrKqGVi6`XcOvd4mx;5OmZqIw zN^1zd$i(EVBp^`w${z&=9-%R)8d%~QauM@Cg5$q%Kd#2x%$tX|_P}3?%%(szeEZnG zbyghpA~~j^p<#He&>i45$C_9~A<`hk#>XEfz%hvsM+!rZz&*UMWzVE%`NtK4?lC!L zNkXeqpZdr!IYwAI-5|@|?b!7u>MF|<;NsLvHk^l~*fhIzoLuFVuN5sL)5ToS${jk< zVZ6ca(Z9ElWJu5RH}n7DtQ%y+gUR-))*w2R!f!qls3xHQkrY~87D%F+>$K|&nZEe z(6?y``vNie3Mc0%@ywYd@WT*t%*y8vnksHL8q38a!=A3~c3oD+F9Z1hdz6;wkyn8% z&{Z}4Qv?bWn1zi__Tn!blZuool>6=Cz`J9`bsgIUSpE-VUmX=?+x4pm3I>fx2}&a+ zrR1PA3?(g6(hbr%A_5}a42_fwNH+{2(nEKvbW1Y>4Cfvn_3{1A`>k`<;*WvF+;Q!F z?fC8A-d7jHF#8jpovtG(kNJ}UfJPV^8dwb+;aElBrE-CrNFn(-xQp;zAe`;x`x~Bv zM0~_tS;iI&MH1*AIh%o;TB>?RC{9lyhEyaY^;ip!(dghewY$m#6V=aq+np8RaNh;( zgbmNS#zxN34?WIRNh+NTlNR36KC|KD)3kSn$H^nBoJ+vX7P{j!Fpk+q&Yop-4$oKW zzYGN)Lf?J8MtJjGYHk{Nh%O>krajnN@T44O!GF!BVcLMsdU5 z`$~XWVTGhK$tg^at`wlgPv~^b)dx--Wrjm$2>+N~IyY>F2_g0;qa4I{0Kvlyq7ErM znrfNdq$3;iuo2nEnS2shPYmGE*vwZvKrXV++2>lvnf-X9%miZyKaS0l;^X$}=E?3QQoo6n^Y`s7+GksB=1XH>sfu^><}_f8&!4R>H_WB zC7m_>t}<^??A*oqYnZ;v7?h+qaE`XY#OvbfaU$=(p7gIzi|Ng#33|7qRqcfuDKGK( zsO5OCI)-B-QttKRCv>}J+9)iAINRlajsBz%o?0ek5~h5^?nOWCzgHN3N@pUF>{I5s zRBn-8co!}|mMHB}T%+8#6RYaW(lO284>o)r`nZh_H&M>-bG{s5UspWM z(-SPzB*T-AtIh)@wbNdI5K#l-#B_#CdHO|b!0CCNA3iXZ?_F~d2Vc}s?>UUyRlz=j zWwMPeoQssxL9jX$>WW#RYI1ZUC4b-B2gBE*qW*qeaV6X{)Yp0WjGhx7nQ)T~POZ$k zgCTzUe0iEbTl9|_0?F~7{%fSnK3c}m>QX56!`zP|d}I6Uhg`%UdWDP9IsROm_sUFJ zF@38GCr&8h+yC_v{~DlQp8&|#B#{E=wnhi@SJw$fiBQF-FBw$Ks8zW%gOi}Dnk+56 zOkGMODtdP>PK4S=|6aR)RG~;_dLX!Zr9jl_nJ8|EX>T*bjQ_x5nS8Ip5 zs^K(gXU|yfTs-7|W1DRzdEXjqC_F8lmCp8eLFVTj#J6#6P*UFth@U{Z+&y1t&gB#} zE-Nosey%x!6%^fdqe4Foq1-lLgh1q%G9k0mBHLbr|I4F+Q!WA8WG<)BihJ_rNsbmg zmaHz!5^8h1yo~H|n3F)c;I}8`eUb|$x(tTxrm-D71CrSppT@0I7Jk>^{|=%5mhx}G zQ~-;J!Iwzx6UOm997KQN*;^W#sE}7R9rG|pyb76MoeU$0eTe^|)d%AAv`&`(|4TO9 zia5B5&|EH?DuRNNiWfTUnq>Gmv3b=Bb=J@B;eEhUa1KqgWzBH=9{+TB{{pCfOXk1bl%5=r)}rP}iumQ^+5AG+`s*mLyfWM*I+TZ)i+;_7!9tq2 z^Q{^-BJq=esZ*T>A7Ns1+5qVvdyZN1NF6^zWy!5`OH{^1d!JIyxO30iJ`mr4O@&8Z z3Y?Bc1*K8KCLV)`jvuZ;noIxQR(~WGxq#8blN*T&v*Z2hVI8o&AaExGWHmqT12KEn zI0^mW>GFiLVu$iZ2Bq$8edkF$#`4yW(pWPeHSs$u${Foz!$p)O?dnBG_*C+A( z*WLI^;`M9jjC*RX6qiB>81}IlH!8|rYF(VXoF|zcmeKqmijTRGQRk}~tn4$x6F%k- z7C`+`{*A>!J5vi!i1m#yg(=}Hu>3XG0ZYJVU9aHCup5S-n@H}I)&}It3JGdEg!;YdMY9k_Lb>-GWe0%{?lSz&y*X3#s zNw1$R8C10MYG}aE$bTFc3Y~XWeJ@6P_ zXFYyF35(qS{m*|kF-A|om4HCh&aW>Ho?r2;Usv0sXO7UY<)qImIWXSGn-Ei zWdC0qYq0=$L?o(Pxur$|sHopnW>fRNUkmO|1ClXIGcZ#*PtSJ+ZxZ$~8f~RvP;&y&C1_z3Q9^YC7z3Z3f#cNOrT6qRUl4x$KZ?D|MO`hA6~{X<8eO znO3fz#j^ibc>(MiL*OP?lO4>6u@6STLSEnz+;?(ynGaotgXt*r9{I|Ng>xAL?3Q+} z4BnpAHX}Edyy5&A96MuBqiMnD2WtF!llZ~mtgfAMQfUTW^@)*-mbkg`6=!Hs*2ZmF z;mqtp72jlyLyrc$tr$xt{iNSzP+3O8xEhopd$*Vdj$qyfOAlc{iuC{3pa1;o*8q4a zrX2`&?$xgor9wzL(L=cIRGg6KqT%|}zWc8M##47scc@p*3r^@O%fKZ5a>!K9303kE zc}+=)_*y=6ae<|al1vr-+5qZ*-ZUXZ5-XP1%W7iy>QBnts)2-5$C3!CwouM;(f*K;RfZ`mX-`*bjH599Q8aG}? zTUfc)6yz8Tb9ii1==q)>j%t{D%XT}Yt&=CKd@wd44hHMK&Z-<5yYDU{T>_>fRPi+6 z!baS=dFJ=8M1Ow*e{4n_Zai4>=$*_|u?P_9(V@I~mAt!(k2rYs8i;eUZgJvXf!Y`e zYcq2N`JeJM5T?gr@d!%}*a)1OQ0a=6?L-|1hiMkLq{nW>qJ4l&0aHUy>~2(QhvOwe zZ<=mnZoL@!de+rjo+(J8HU*zvTiQG(*kCa;cn|3gm@J-o87eBGmJ{C2rMIaN&nFj~id?2rr3h zuVv6JeR=LbFUJ@kv1svTdm-WDN_9}}{5PX+)?XL!yLr7~ulvE+^73*4Ba7M3Jg^Et zBpLjxvn~I4Y@oIf*}reIm*akJWT^Un8?CdM;+;#5M)pJIUY1W*^{jXoccKvMqXoxb zy7PJf-rd@sJP-b#@Ba+${8$g)ytDW{3xMbZ@QtxYjkBj?u<6DFxhWxDd4)KAUO0Hb zhtgoSc7F(b9C!4K?_}tVr7U_l0*$Ucb=BdT1Gu}Edu>eS-Z%C#WmFpZXV?s0SjFep z=pL}qdLMMJC+xSg(Y`y0pgo2>U~9bm`znCeztR`S%D3I~9lWO-gt(Mheq(BBg4H0F zE2dI@%u3LJQ3p@KG&NJJc-$-Q=GiIebh1BbDdfjamsdMT_G(fP;}u1%B7Z{en2ln( z^S`-O{}H`*DqMz|rO8V-ALZ46WG>ZvQq%QKI{?q!cf@QwQT_&ivuiz5y_FD4K@7}x+l z{6u;DsaFAWgG=QtOb`1 zA;zKR2W)DO&JPA_SQP)nOUAb*ym1v9qt0(7{)l2 zNMo$Z{S`{lcqB~2$WK?!lb_MjheSH_is7OzSRQ@lBg2Ogmf>k_!mzpV@ii&*6)T%V z+C4&ZLxY`Fp@0A!tD{0__YT1w)1Ag9#!UQczeL0h$43A79MFny(Y4g7Nm2FG-0i|E z%9*TyjcFzQOFvp^@tnTd3938#=DcBOues2^qG{V5;rMSoo!&7yorAnP5^FsH_g+kH1{U5o z+yroKK$+pDsB0JYjJt@6S%Feey8YeIe?b$vQyh7Dc_FPr?OiHp+S(G(spOqoUcUd- zL)oj*`{9Lu$6c?@guS$*Vn8n!dUK-y)1p{>)#nLds>;O08&gdVo+?vk_-lU{BvIpN zbF=5W%z3_*-A89E01eAedX~^WAN60HDnOaBnJCFr$!$R^BVD&`_e9#*S{|lBdCkQj z0M)hW0sYEu*MTpRg67X^a$vA|ufE3%pX*0=9Xddt%frL2FvB z8`rw6ViqFU`0mfM0>mZKgt&sH(-yd8$Nrc?@`x^%XnIwj37Id@Or$3`yPavz{A83| z&vkNODfU-t3-FcG0J@bmqssCLHIN%Z=li?b@P1$U_2$n$!ti2C3V2(x) z_k*WrMW@BUx?<1(h5pA9WTa%bJbI$A6$v~Z;`~zG$YmZuBh;oVO*e@*FMD;wU%tsz3DLbf!pEFSN$xR zGv?n0Qzmt7L8 zPK$CUXV?BQ@Sf~J8of4Co`I&&zbw;8=?tJDXX$Le|3VoWy++ejWD&zE*L3=$m7s}j zCrn}JMSSUf9)w9~d>~-<{ANg3*Vg!jidv1F>wIW7?-N*Ey~U)NTfKJM=Ul~hb>nGE zCIs9hx2@X$p%yXFKp8pKtsk8?8NQMwnPRabfiD8usUJ=+GviPN}D6vFLILsB}NS zk8$j=WA;Watk~c7{@}Zrb`$KCl$tl9zhswKgaty5h8*KRs%knL?+VV@+EfA!`}LYV z%W3n!5)`xnd4-&;zSvh(B@>PHPW#;w(5ZYGFCKXf`erSCsrFrMP0fj37u)_#9n-yi zb3fHTg%>1yiTmz%0!E9oMDXCjs5qU&Gd;c8Q%p!k1`Iu&cYp=r)TtWk;AJ<}E`f=yKG5X#!q*q+tv+C8holFmQCi#z;>1RrCr?MF!H~HST zTxlxyHwD^R(C$Z*<{+sO#7BzSe-+y>30GZHQ$XPo5+XMSZ8zk8TmbO)m5B{~znR>M zcn$4}UKRrcARdlOU0|=`X=`Vxp^s-WIQ&qMI{fV@Fwgb?Thn-S$uo=~Y4Egkztj_z zwCL(g<3r$6j0(JJ+)Kn%_aC6iKf(-^{|XT8J-|FY-U*!VPJFAu6pGQ=ls z1vGS2i%dAT-V&|m$G!|fRN8{`Ou)qqr=EiprZ1oHc^Uv!5rOaiR?ecL=W6!pGFYaR zrqg-KrHh%($`I7>bBd$trIiIOw^^6*5D(er$iG}aW?^${n~i}PHwL1jqSf6^KCMnG zeLk-MlS8Yxe~(w>&-TBJEzSPbwf?8c?8ndobx%>Kco%2qGgNic`IhOU8Hd(WdOMSw zWO#`bS11!OW_ef#Xf38_y%8Y6W`o_FconB^^?Fwe*BaP)b|Oi;Kij!EtkIlh0y6*Q zRd{Z>IJ=5;`-!QJZ5dix^A|k-O6hA*vb{R`$3RPyRMomfHHROfuldQ-%f?%`tDJmaQ97{HZU{?O<;MOdTha^VEh%u)g{hUPFwG zm4CV`VSpAYCOQK+Uv!`$RK(*O1=Am62zazYP3uK~QtgfS=0F|KHKXM&k_N)?GtIzZ zVP65077Cn5{L7kf`}&bmn6}Te{97>+WXZT`lJ#29lZ2n*4U$TAqX;rGGA<6CGLwDo1eM=Z49p&xy8$pwGO&9^ScZ9LX>ML!Iq`1P0_cVg z+dz%elbJH;H8l8k&Tlg#O9*s2`T1Y@r81qyev9HzkzB)7;8M*M7ulQdo2GC%aUU^^wjca14)hv5}BA~X|BdshuoSvMqd@bpoq1o#< zG(W$o za&wny{bJ=V$f`OWpmA@nKJK&xS^!?5ma;IQT6666LO8O{asHw_9lkv$ynTX_S+HFF zb`8?_{kREo#N1lxxIOImD%)l}-#B~s6IT5Lx^*lYATts-WcsLT$+=?vJ zs(Q1cub&-@nIM&r?M9}B{|lJK4Wau=YIXbUEz51#sSJm|zTz;_!uZxR7USoR*B<-$$^7eT5Sz{^hH z&^@ZLIc>6y{lJ;`J3PmjYC{XZ?(XhDopLi#T?2!Q0Dyq#q+OvICcnp+E(R( zs$?p8CFUV4xd!G2jY0?SUFH-*GqU(`;mbh2f^B|8AAx(51SKZ7@J$_sLSIuP4CgOi zj#QO0k6XKS$a>BvYcM}JnX^ts2~d~b^BdWmqk*NbfW^lPED@ZLbU*GC#RznVKkHAU z4!|BU;{N@ZiY$X8gE(Omq*!oMm)JDK%eNzY5TYl z=oAMxwhKM}?(VM#Xydn(`b zmkc@?{g!$XK<9nJJBX-$_7%C;DC41HF0kYU050K5P5pzb(Gg&7)k>av@23Md0cVWp zWGyGK(`udK2kDXp6m*ORHcn)t%CCmkXNp(>dM@HwO2G!`G7x2b`m|;K&7npzw@S=? z$}$fgo69BzEbYH(*Y(Twv)dqGna8*0;HO*-`a3Hab1@2-3w>%S64nBMY7!j~^Yc4G z3knj(F{Qaf?#FPBd?hDN0-$rC%by&2@8=?b>zXM>wF*M4iYYF9m>DRxPU@q$IDm!* zu429g3`ieH9+{sgHa$3qA+xhK05M3O>6DdRPtRYF>?JG`Min(Rr^3?GUBxbLCpmlj z9c4xfvtaDrw;jV`Ky$_5!t;OZ&z|gG?}~Y)Y%;){aS#MxbY~8))8A7807aCu20!M1 z`0zap(Dvm93B%^UULX9XIX^EA^?kWFSqm^+Qusj&VzoqJ&Z?THwB~LF#({!SZT8!MV+@T>`JI^(gLG&>0#ZcWbIRsV%LNZ$@Y@)%@<& z{iUaLYFN2oevxihgBVonQJbpaTC68ymQC`yARV`k9}m!#69x7X{Bh+8Igz-b{LhwJ zql;t_GMbw~t@9z0_w5pTDw_aidx!dgpVxivQ7joscnFUJ1>oEVD|0exQ7_9uqJ&oBl=a3}m;??JDJI^&0u$;W0m`XGC!!S5%*8hrjTY}+K1uR+-}jmH)g9yNv5JZN)1RFxxUe5S=RJ3HE}DLH@;Q!K{_2iTO||a% z@@U|X(CaH{3jM4X3xWy?;rK;u6xe~cPgbXy3ib6}A zHr)+Q4D?od@!Z1lY`M3xiL!{J8)LqAabpt_s)6=3`>pM5F|{{l2P@hJcNDyUa=cWS zHAZV3uXjF@jr&p#?*6dT_NLh#o9Tgr^qU6FIhS$0X;l!2);vLOBWf2 z1<)|POgqA99$@TR8nDBV1)86PM5cU+xp|*IT*nwxV^RkpJ*$?_ZQWLt`BPH?Y=rOF zJQi{nd+ns(n19$SR!QwOTN}!ezdOT%j0l)&f?VA3>Nmw@;n(rw)&7%Gsd3OfY1mPq z6_d$@=KC_%V47|s;h2n=Wpi`$tZX|I=yy$li{_Z)f(x4i6DYa7?PWS{5qT5}j>T{2 zp66YxmMX)Vyw7J0I1`uF?^p}J0I208%!I*`pLCB85shCDhALdPvW;B z>p#|8Z#BNPk;JdB-9Mez{!p$AUXNDvfsXh&@nSQnX24*uQlJ$y!6Z?KDcyg4-iiKn z!JX*0?9Zk$g;1KF% zH(4g7=R7F2aOZNu{Q)lVuQx$>|JX)w!|Q%pjTV*1wylg(W_T58*?a+(UeTa{7_!I4 zmCwzAVV$XpNctXGj2h06M}{_D&`p}kZ@Qj7M4D~P&Cn9jiyON8t;|_G-|agJrwS?- z^bXUvuUT$cS#R{*QQW^MXc*b1L0`85Y!WeAI@$^hpb#o;2psys zwKG*|tpoT9n85{V9C8@I<(=ulHy$B~<<9pglM@rzy?$nfbdWIjbnLWBKyabT@y{KD zN=HxSHY5|*2-Kb9DS}!v7ccio^2t%*-o#yPrqF3(VpfXR=+C60W--B}!Whk$W~7^6 zR%mvzh;lAfLa2m+}CeTlHjO8#$m zsp+^?u}o4hg?@>H$wSd-!_PxgYX)bV z*1apAYOR)+-(ukd+0_CTXHaeLMuhqoyOAlfk6~MRQ)AVr>kAtSsUN2;Essw=U$M=u z3wHt}2`B;pCd?hXi-a(KJZGvM2Zb;D0-Ud~+b$S2bHoKoEijje^Jkw7jx&H!TRNuk z%@pSxMl|dTyG%uX&5!Mz&z_~~Nuygc?)n_49yVVg_&9&I{Aj^G_BNp`y?8s6?=k7` zp$ZkmbPI?IPeF>CnANrNGvs!UX#7}H)$Y>`^K3=-OI;~r*8#c`-VZw8 zH|6_bL7DMWy_;9Do#{e)mZB%DgX5#xPM%8*xji^I0FzY8QkDf}(N6LpEgT>6V7 z<8PmO6(j7cT=yk@DmE+78hkaxB%xN*fN5b0>cdDQ4Vm_#G+6-)d+n|m1U98x#^zdXsQQ~fq7^GxQ_ed@^kEt>IM!PXT~o@hQnMeECb+wD;OkNg-#BQKy4AIE=qS#!>RnT9-8Xo39+B&M zQFxR#fRKG``dLmH zT(^58rpUKM2Ar(i3~HW%Xwdas9NyfT@mA9#%Uly~+4jqWW*3Mt*LlQU<(kiHB?qLn z95ihK@!?C%aho8QCq_;%>*3UCr;@z&R`{$!1RqLy@8oF?TBSmz-6qPiQUtKIGPz?y z6(&zw=0f^egy|+LP_!m_*4$E)CNuPHwqa^Qm#jNl(kSw~Lv9o{wmHq%=65r#n;U(s zI=;JJFzHrfZzquqvC~p6nX-IztQkMK!6jP%{T#`wr5l(uyjLG{^jwc zPf6ZUEE0#Kbm3&H<*KsnnWIMjxr&#YgX>vs=8+rC<|jLg-2~cs(6%Kq3M;8@zR-#4eE=NkU9)N;oT%LuGW1tZ;o5Prcw{7dtPOGkGT)B!N%g)pe=w`%~-gOU=QX3WZH1 z$okmero5NuCcvPa>#~Gi86_nSUdqb-lS2fe9K)&fTfk%>`nV4mpqTp!)*c5#B~rOK zKco+&G50a`q`EznqI{(Pu67<9TY)+qq3WkCBGY(>q|f)u%cjAl0L$MC7zi#<>ALlt zEI8%lIDgLGQVu!i+Ud&jlZ^B~Y!saX)3m`o6s8#qoC5GE8|B7$JXq!htj7u^NXB_M z?nS0z2~B@dnKbRwe}1qh`^NMo{b~7)2i5R;_Bw%+qlJ{)vZyv1XErF# z%7MeeK|USmv;r|7%{HL9TMf>LoC4qNSs}^YBR>6xx;?));S(-^vN8oN^?Rs!i6`7* zFSkySnW}r^bWJIH(!*F`8hIM-&Z$(y1Bu@V%GZz3hOf+9)zU{F2y@p@)oNgwjkT>8 zdZ8!MO(7?5uo>uA)s(O{rUtJn6vDx0oQjL{YNftqQgpi542eD=_=!>Z z(4qvfu*Zf>YoM}89%>YZy#cnroZg=oP9qEhx(~081qK?kQ9fiX=;FM17Dc!EuCN=1 zcxA&s9XiUsGkmr9nGBlzAZR3tUPc{&%$uJZ7!*6d#=n83#@ei|sFvRk2OJQ_a~ksC zB*H9tDr?Cx8-O2cFJ;NhXC*p1TJBuM$x6CMk0_E=`{0Lw_X-9a{POiF4a0{X;8$1m>hu>})T&2ruCDB|VzdzNiY_ntv(mXo$>47f`U=1( zhZoSa6Q9;L0x?}Tdzc|~QQ+yfkGf(E7vGyi_qu4ZQpM!QFV@s#8i{GyV`hBMc3V7^ zCkbf9+!L_TFZds8Gd1Rx+DFp`sxhvvq~#gdH5jT`&frza5YW4cK4?X?)3uyQR=(;I&vIj(X_S(5=fyHeng>LTWskXx9@NUe=bby|MI0N zv8<^gT2C;i>W7N@Q1O@b#q}c# zpYoz>86-ja^ms%vJg$l`kp-w{o9|R>v+Ds#DsYzZVM0U&Q2vr;lfuBCbgfuKWY2ER z(X3nS+;*w34}tC5sLGgJsEnVki+5aGS2@-kwJg%oi%BWu>tZOh`zr&A5LYB1#x~JC}`iIb|)C zCN=$HYok}_L2qP_$&7JlGA&nRM&|wz^(LHM1FoE|iXMqu(b$yd>{;yp9FnY?U>kjt zHB)6I&U?P4Lv=X4%GEbh>P^Sh1k-KA2f^*PuPIVh<&|;by=A%@c$mc!znJlRDM1Y$ zvy^Zk>uf+{ex6NdTZ6*Fb<#UUSJu)_BwEK0_+&@~`|?lpD0}DCNCY*fo*LfAaW=Gj z@}xy0Co=F+HdhA3PYrQ^!=Wj*{l*AC0id^HxGSj0f^_3vuJP_iQFb9n&5UDKI zg?-Ym)hgYsciEjJ)2*Uig$vd8mELZ`Pn=Bj)~9VM&yq;E)$Qtt`u?5Z`^E-AwHxhv zJp=lMje&WVV?9Ee@O1}O{(H$R|6H>{isua63twRm%9wH?e>j{ps8?<1k zCZEdP^0hY)5N2u?AL(dk$AR3b(FghcGlZ3`1n2v70{cU)YyMGBo)DG_aR%kgn(-n% zPe1eLw-;`};g!ZqOXgbJENtA&6*ShyJljO|TCgo^+KSY(qoZ)HY&Kyhx!v4;1I>Nk zS2|3P+s<_DRMSFeaM~mR3JTtqnJHx*-N?>UdSAesS#O@A?rqbma0?Hl1;n(l^UZ)! z;RSo=uOIUS&^3h?2`zP0&~r`0(cr^m?qWcmi~Du!TgQb$Nhy;s+V$?e{{^W6i+v8FDx9_7|A zyH158ub^=3=H{0EK*#K=A~0AKf|p1H9V^Oo&pk$tJ%p#G&M}4J`{k2zchw67_1r7) z!sYF-zIkb)y2o0^3evKeLcvJO zry7+<1>uQT*gv8Kh7%s;n12N{s~0E*{|NJx#S|00S81VUt;UaWvq^Drvi8hee>~G$ zUcKY(YGOW9G;*1jSd!O)3;ClKUeW3LSddqtn3+$)+z1L9=3kAW!SlVtM-wyp@<`89 z3Aufec8-b&W998(Pg$5dTk7vc>iPA&6v>KSdq*813fb%j@@CHWP5L{KlDv)X4bCE@ zxR|^0DU+TueVq3v{BFmIUOml_E?R>sA4or7+tPM1>2E zUa_sE0eY^e=*hxB%k7=*O?~$^4>GTpVXOIi_>WHdf@wPJUFr*p6OqqM^JfeCH!R)s zz4NvX9)WCh<7SS zD=%#m))J6zM!2IsOeejc&sm&znT^Oqrwy61Je#;&A&{A%$XK>p0LohWH(1ct`= zU8>DQ0aGPacywN&-{e$UDNE!r-@zwnP^GpXN6^eJ*uES%B-YEn9!7w?Mi#avNkn@V z(G?c*N|$bgN&`56T!Yh1*2NR0?&B3yCj0KFu+V5ntsuYK#-M%O+vqAAUcDo+GX|J`Sv@UKJdyz9)VwppSU>SyjuG)eY%m;SbK`2 zJ%_mO-@iXvIg(u!1;&5jzNKdcB`2!!AH9Y?O~@%{l_K|U{Nlxt&oQ$HfHcT1jrsY} zN=YoUxiC<1xd&h6sd6|d?|4OBxJoYmoyz7}GHoRnK$hbQq3Nh9FG$gC+HFJje+fUV zsu@^sS_#>Q?58VJm)*7cK{;I7)_v|LyGA=_se`T!DNs|QAuDiuY5krPl>dVuy4Scr z|HgYtHJm9M?{W`Qm<>Z^`OMBtr4`z#w91N&+Q+&msa7Yq>iUJHZpZora^gr_Z~nu{ zYYNNbDfO+i%Ip)jYJK6;!wDD9f=@*k;KQJOs}x0I_Jf2jzso|pNN zQ=Vjn9nZbnHVhsgk#(_s2eYeM*@f-Ll9( zP}EARxt*7$H_U2<`owZ7wr6QlPt&|hchl61H45!=t9WcDzn#2>E{Bt}@RoX29ls^^ zCAT)OckVt-(FZ`=_4QkQ@^{~2^KHrE#}B#`2{l<&HxvaiSa2A@>Z-Og@Drhtra=D1 z@MOFMqO90SXNe6kd;uWWZQ7|zFmaV0RX1P>A0X3{nyNW1Xh;q)zc~4s=NN8phX7kMVnQ0gs7t6C* zUua;0J2c;isP^AZcVLPfp)7t%6oH`!GYIPu3{!Ga!xVt~27My!ivaj_lj&oq)jOgZPnyWO)upf&o^kR?4xYNc(zGG8pk(ur}}G<>pJ#8 zXIH!E%VYUD`SyFKSNf^p64$sH6T13Wx4p&?7o#e}S55cKjW*ERI{J|8`?boR@6(HC z6NO%{Gz`lVKn@&_sk{gdeVBljZ`%IkZ$b^nKY|U`islAv&UqO1$T1;t&g-=Ia?0XXgStVB5mD->+rVfiC5qE_ z$1jEK9lY{W=K?v-L#nK9w;#vO+G=RVDCTwXzBD@+b}j3*c=Crm0pverT6DJ`$G}Ci z&3~fA5B*j#sP3m#B=Y4$iLYz+~a#F^g0L5`;`qO7S?ZZ z3+3JNMt%ITjTz+3gCgHNJi_!0$E+oE91eB%fEllaC8!8E=N~-#Cl7QrzV>&w$Et|NP{Jr`M2qdTRsfga%IVI0zOE64?)Xvj8C_)22G?H#n<(F*6PcI&V{~RK7KSoCfeQ zPyLD}?QOSCLcuhToN7?U9O>QjIZWvP2YCeZP9$lxA1I< z5e(S7C)doj6HeZjm}y2zPUXkxc|EuEY8|N{_4;Q<1$Fg3+_ANre8Ax_l3m)rQrv$K z;(uV`P&{WB7m7(!=l5>4UClv*6!rIR(MK|32_)BYZPKQ|7i43KZ3-{*iuz>6wqV=k zwZ({Rnuc5lf$@Vf&#&G_Fa3 z9_eNEp+mt77%!ZWot=$b9n6w1L-tX1i2}TfU}6^x;C>Ttb(2Q`Ngg+Bm%y5Eza3^z zb}_=JaY^y4R$Fg2zI6zc4#ods^b4*3RiXgC2z*Msb6IY!@kKQ-_Xrs{^d)jgJ{(sz zFkH;w{cC8hYr1!d{tHmN(E(D)n}rqvpEebH$c)&1)6<_naX59l)#$+BSeL7pe!w1# zgTR1I+1)6Ima?HIYIKJl^!_{uYzZPab7d1>Zv*Kv+F&*(bNgmheVgN)0s|eZ4X{>G z4*&tNUNO5q6aR#%wUW$WZtRQX9I%{{v^mwW(lw6xb1z;>r)J4{N_}GY% zYLu0+D@UrIpO257Qkc_o9K>4&<53cb#Tg>46;AH%^DVYT>0t=iE^p7xk^!P@`f9Tx ztXUG{M5%Vh84bbJ@Auj|lgZe$elawaRbb-Of&;V0n*w8C_zEVs8c;f)m+(p08g`QZQv@E6MhLrz$% zel6ngDAx-ZUXYhdtW;?% zvxlrRyNI5sJ$7}ux!KW2M*e&wfrHcLJ&S!lH|*wkH+{n6c}>tS6A-_brn=2$=^8{TYqd-oV7K-TAVvh;dhx zy+>gYLdE;2R#uX0CMfgFYObXXiqT=OTaN*7uah<%NB2`{REUo=t5b zR}Y)%dGj|1Z1&p<2^9~JIEbD=YtRyeC9wY8D@=gu7fdvuW4b@h2ZP+T9!j zY~tp@!E0^lO{sTQU#wmT4U;PAm6c&JsWRmn!H$p<A(Gmd*cmUsr377 z{QNm3Ht(*~4SPAI4CU%rdjgu{6r=6sN2joUrs-)jFA3x$5G&1htqShJ_w0^4otZkP zpQa+H;}6%;TDO}kEIb7jH%1nIINEzw_l3(Ri80y;<=%suLp z_^f%3)eAK7jP{a@w^GxTUYgt2L>WoYbtM@|)jYPVM_^kb)d<3J6^6X8jSIa`P7L-Z z(sp))zu&c7N`8YXSYxfC7u2b`0Il?_RZmLqyf@!UtBOYO{e`CdzC&8yI!IGt>oR(veXm;U)Nl=@23?op@V zS29VX#BAFx%@&c6%6W?d{*7`pHL+GLKH05l(5%|aqG3m7mdMzP5;#`zm!vCLu}F<_ zBUZTs+(A>hLg9UH#Ch>j=0L_lyZ7q0x77oxkF9B`seC6rJf@v>*{9tE(i;#keLLn; zB~LRfakwv>`9->8A!0h% zoBG{y(1-E~A2I&k7tErH_(b?sU&IhT+{>&ygIcYqT*%Q_ZrKji)_p}aH@&hA!`_Z_ zuk5#y^703@1FWsSXAb$}Gc>MWYx2}2-axx4n@q8Dxy>B~Lhr* zal2j*R%a-AABM6{<5@zTJzF&JL+Z71}uwB%+OVb5c=XA`!=dF73y zP_v!{gEFYo@>|7u(!tBu#V=w1`T73TaAE8dNtWOga#Pr6Eq7|$!sWYa?ofdnb5E91 z58{~`Lwr1?Z&T^R)Be6*QLu4i_Cs40@?RI261wbpLzHkm;XdNsk8N%7j8;OzVamY@ zSyP6z&W8jjI9E59b*N$B7oICU-SBgSuVf>nzvA6zswAZ+^ewGS#J-`tRkMy6o~s=O zLP|+E+^OZ%!k%u59FN9Ehwlr?n}!W(^oMYW#_rf6s-=@8Y2)8O&pT$|A3+9^>uO!b z_4Y8=abM7y3yY`x5<(YsuCQ3YQpYakTTd{4kn{bMCsn}~^E4aQ(DjW=0dT#%ZyM9*9*fYQB3{#P=p|-TNd%k?@Wn-+t4MkMDOTCv&M-Xl2Astz-8spETFU?&?cN z%f&rjjVE(aJIOxpL_|Lp*BpIsi1^g)JYAdTcAW%4Ew+8P%v8oBk$bt~GKJCb!!_T% zvebGuYna5{G>!VW86O!D9eBME{q^r+C*Ms#*Wvdze7^|rw=IV=`ErlhvdmU{y{|M} zcIdUVE8ctGb1!Ij7G-NXJ{x4!$A7vVzcKM)`s$H2cx1=Xa|0UTw;RBI75593kxo6&=l&zAA z`Ay%YS5zq6B38VDL4E;)93<@H*n^nZsmNkEm`u#tHX{P0YXmis6IK`4nXr~bI6@K+ z42tHcYb|8oIhb%KLy9Jcd+s5b#?G>)$#X@6$vAF21yn7cQ2saX40YK)!O!RX81ns) zk@`Ol=Y+ya%S6BOb^fH+(dCvY&lV9yZFTN@z$5ReGxn4BP{4N3Dvd#s5z#W!ZEu_V z$#v%!TfO+li5LF#qya|OWMkhLOMd8ad6;dspDS?#3RhKYCe!bVMe+sW$v1HYd3X%~ z3BI0+^|*_>l8>X+cV2)o{`cjKyP{$IC%3B(gx&|ElLgCM_62qiE2hT!2JyEl;Zh1l z!SCGV2~itGU6IFZ3EUok#H%bX-+?XT;#*WPXRMcG!BM-dOX->0Zk~IpN;5x44%()r zU|%emEDGFRMps8BxCLTcksMT<_0taTeeW}~3HHRgGLe`BnuC)if5=xNqx) z!99}=@pveC;wyqOF8OKvWDac^HiI0bT?7R7fq0gW`S-BJBg>&ryc`C1%aBmq>PQKv za6_i)?45G^aJT4uW{r?!z+5bDWQO>d5nUT4z>Q8Sw>78^DjgwtM};q8t%7~%2AX>L zEzLijaiv^jbMT4~{4u~;c5qrd92{Psbvf2T7zQP0OsLhJDM)&ZB5f<8+lKV2$nqyX z&X}g<6&P1m8d5T;W6TtwPi8*WS~yQHt@z(~(%?Ee+gx6^lH@Bv_D}9~O^_aL1(jYR zz>{(M-p?U$DkA+{PKi=>KHHrdaPAqu!>-0`5zY*%z-6FW7u2XX2lb|sR$g0#^V~RK zJOl^>!dVYfSr4_V&38VE;g0u->)ob_a@xuuq;92I)|XK0JW4xw@Od6~NHz4o8}!;{ zWnrBj32LPOaJ8hhfaW3E@%_fm5~Itq)_NXknit#1@~T zf=gj_&tWB}8N(=d0qM6>)zMx?qP-88DHosm;+%nZ<3JQe3jp_afSh)z&U>6(`_WY0 zj0JH&`F6*2fwqhz<)bo!n?r?ohJ}<^3O^N?Y5L>NQAFR;?eZpC9tk9@a^J|m+@`PX z^4@i=OFKib)Z1OCfdmd?#FCHaGS-T+oEhq+w%!ljO3QJ&(2_(%{aW9c5#W0?tS&Ei zR*k$noA9_B?iKsLy#S`F15K=;E~NX>kn&5>)5G?4ug#^3u2Gpqz2uZ?#L!)@SteU+ z&w#~a;~CxAw_E-|^X`%_MlP3Kqz*cQRzkbJj)_ZG8)YHZgkEm;zh|0@TMf^%3r4V) zcRtd#(pMt2AdS^=h8apm`y{#H0o4)u1F83}j417sjzgJKc^I^|v! z8^6gsy>Bni4nC@uxmmgltAAq^z@27%`0;4SKDThCa&_mZdGpaxKs+r&!X~r#Ajo&e zL+1;pbp6Lga!(zkZ4}S1-B@U*@B3#TRw94Cq)w;0x@b8XTdGgz=34*e#jh23g?Hk( zra+;t@Z!QFMIu`V0VQy1+&MdVetA@%;~f%6iETT* z|4vV6&F_}=y?=#uGP1k=0MevX64@f%-X~*#Y`DZ4Xw;f9(WhWo6I_%Za5eJ9VsH5L z27m^Xf#-_tcDj+z9g>2At?zQhe8G_ZfO8p-na{@} z!tn!CsN)6vXIj3X@2H*(i%hM1Z#11%bg01nNq01zpA8;HG^4wyH#z8|k5cwZKHGiH zV<%t-$mbt5OVt@hW?1hhfM?SBrR(wO%FnNu>iM0=d9N=bKfg&mkm+kqtLnmx>V7)u z@J)XDvXi9?$i{JzJ!~s)WjaC#*|s-(QmAjJ$J6qoI<8DalJ#8A8MkqFaj+^cRAfOS za6m$Q(t4db`VxQWR6x)07VIIFFcPy&X!$9t6m|3!Yf3)*bsE)gUkPaA_}dT9!IM>n zkPtbL_gx50yS<`&m4k=Y2${|k`+;L#RoP>9*okt3^9gn4x> z;tAukC+?KUDgm2#26WpX6j{qHh~Eeha~P#azX(eWL;UdR4?beDD%p`ZF=T0`-4Kq;0 zJf!k@UF`SgTsp0&lXfQPom?DlS=?PWyPv$2E`O?d=H|^t%o%Mmi~9!CB^%Ol4tUYU zcr&NRKv+^CMKHvRP>`3p?ePmWQ}+Y6YUkEI)7$M%^UL@Q5<(PU9`=ENphGlrujJP( zWPK~cTZ8jNx=eDz%&6DCbz;!&O>YDuAvm`;bvS_=gj&TR6~2G?4zi3F+CjnGEb31s z)w4))x*e^8_nhq)w~uFY$VXfp4V!B}cyc#+Eq(4_6Tl{lfRvy%AnFQeALgG1A?pA| zKDpto0L7=hZKGGhg7uv(OepxnH5%f9Qqy)>x=?YWm-1Pb{{V>pohA%x2;++=tJi;Z z%O?lC=3{_Emk}k-*KgYEZoNt_5D)HJId0v_6oZH(opB;W3+LMBJUp`*x9JLvkzCBqeGb=6^pTh3-_B z%m+UXH{?CN{Mw@I#+FLwFw%MSo8NV8j++-eWfANNSZLFsAlZkapLKSAo5dJpquF_982hzN-s|cOHH&u;d|q~)YWm6sut0=oPM%+Z0o)iRaS*Jw zt7wFCWW43<^?PxGS3OM=r(h?)10Wl|6IzMXcdE{E`4)t1W&yYD0etc>vnf169%KW+ zZ~Td&*WlPfii?=aHAF4g&!}pPRg3iBoiY0dmXVCV$9RU{|LHP&3(KiBezC4DxLo-3Zrds8&sr6P%U^3TTXK9TZ71^xv#=xQM%Z~2;FaQj~s6t z+vVOY@|Nh{in#|Xcvs|O23mH#yte3Zx;fM4ymVujrp^_{Vm@ zHgQf>l0iDoRR6N)BWRKCW>IpOfWk zA>fvzAIff3o*RzmEj?0#W7VcSFCNxs?jD@<7x!q$PPH0-4eo0LDX-cXc_#=If=D8I zrna^o4mJmxpGF`f^dQ`2LC^ZBM=jZ%T;wFRc=qmur-xR#Fn-p%KJ-8%>UTM3!<_bJ zn+T!SxD6%}ndjr(*BDNqU8a>Z!VhZvHM;Yf(XF&$0>O8fXbk;#QW_BYwR^QW?8_Fx zs~Qa&vRDk`aX z<|c)FeuQ(%>LI@Asx{j@ZC)u&;W8VIdTnDVzA&`N&1$n$ZQ4^hwwv|-ADW3nLX@xA zbk^NEGVL0N$?2`UytUU7|58@vn%CotIb@~)nkM!Q0Gm(B=f?FPYNy?e;H5Ph%9l@y z3Ep#Di2#rqT{bJaOgw;RBr!inC@;KLuO*LgSN2PH>z;y|7b?**tl3^;@>d8fl8(5! zO01lVcsAv+VO6d@l^vestEJ9k8uBs^HwqFQSGc}t^VC)5OaT|{jj`C&)6}=21=B)y zK7F~#5&*4a2!bafBovIp9?xrnemD$#stQM`DNgSL)%*cc9JXKRrzz$s4=o0#hD zaDhmbWl<4%NHm;T6*#79+TiYSpxl|!X}15)XnwtdYmq71;+lc5`S32gkMRhr=I{rz z%O>Um?d?9l*2K(#^7uH=vcP?O>s$Wo+s}d#^;%==_?0Q1Y?PVNCRb&2SOS6=gru*H z@j`0Lj;~UA14}}>$}5L|A31wGb)PH=TstlV>jd_HZ&YUg><$@FnW_$5pN(1uiHmw) z+=ki^UW@bN@2LdIuwx6s<# zxH$BX(IHK=Hp*uso<)x~TeH*B;P`YpFoMP@lX@q%a^2wgTJ|4!&j&upjCLzCz6Drf zg)L!}uGdxM-Kb2-)b1VqfMfVYco>8=zzF&oqU2#SP3+Pf#m+IiWcj2)bz8`3$ z2+xUJK$e*>8qe|+CTx@2hg{{Xt*x;bE(DA}W99MFlnGGlATy-Ddvwd4%d;jjW@Tuw zj9*DnsiuvXaID{SSZguf-8BGGB+_>!58(}KrL~L`knj``0@hhc)`CXRKYmmOoW^GA zCKWY66h#~JkAF-nsb=ul=x{Enq$fC^W_Jh=FecBRs6=jR@x< zTFg5fLaJtF%5{$>9@wG7iKo{bj3f~*rLGLOmXTD;wA{2t_7;k&y~h^1Co3BL8ao6V zk6Ova&)sjGp+~5&t=qm-W*}LUx4I04khiDe)b@9S)i+ z@P)s&V5>U?&hXFY28NY@6;W7OnQ=wKcj82uji{`204+JX zgoYMgkC>=4g_h6`e5PHS6usYs0?D%Mcecn#sF@Hm{LF;qZCB5~z9N;Ft%}Hf1rrVU zWk@Api*uK8J$utwZSLLg`K*+C%}=K5?>UWJ$d8<3aN5L7Sookt_T{!fksiDb%709KrHr0n;(o6v(5s_Q~n7 z1eNm$i+c1?L=9urRo1EA?AW>e6}{12Vq+^Z>6j~sZk+0YSGrt88;4p*@ka@0I4o@( z%5H2WmS|jN=`6>U(iP314CtmVxGQ@I%uS~pROK({+0pYERS#~;FDY-{PI>LTOL>2^ zV$z&%HqncLK3K?cAbdOS)u9!s6tzOS0EF>K!%Dr17PMIwGI&IuV?^!IU@Ft>4xtlG zHtxK|-gJ1#Wk8M4S#!aXKbDBYG5;_A}9@9%hERB8KoG9NGXWT+5xP5}s% zx)t$5{NJ`d6uR~KUGj$F&wvM6R)|k7_suz3Ky(rw{TFF%;EBAJHa`OtI?3DK=?o9$cFG|tjIX6OCz64?9ctIsF zm&d3+diEguelAptcr_2qhAhfz%CId$5*)Y#6e#nBxZAzE*uY4E9Gt3GW9ZhoU&IW# ze@d%M>t+J#&nup8nU5B9+ZPS%Zo@3*rb*9)lRds2mR!`+6nPb1#2C%BHS-!sB&qAY zr9dLhBogu=w0`|1ONPUcfcY7YsO8%yhFZ}6tG!!#3uU%EO4SX)2^|l%imkomr3aK? zIcv3+CMsfLV`^9hRdt1B5}gN}?E!zbxnHTQn{;^1CL2|9;K}d>?y0enSgJ4usDiwJ zeM+LY3Iq*gy+!iYn{$`PI0mk3N1sM#FbCCi?$YhvILo=H7o32vL^UaE~QYX*xw zO4gm!IeG^ni3VD_*8l)@p%*2h{QnbkN)W&z50Z^k3duo@0S^%j%7^?KNXQK+m2$#} z7_xDf3d15y1Y9p`X#-W;nG{SlT;F@u)z`!og>=YHO=5*cy@s;LvRP;|_l%11W(LXb zNNFoZXH||YTG}=xatdsiDG?ONc4Ylz=Iysi;&-G)i=V$GPzCqJQVu}E+l2LbJI|%U zH6;qc67zfK$L7*@Cspa;7I=-$k7=S;@_zBUVrBZ;!nzD?x;wjcMh>!$ai%yefGK&TlFStmFt5)N9 zlWBfxXLP@9tn$NpN-Fbq3~Apr>dm#tgmIK{5U|Ve_zpHX3!4iyEmyDo9j-?XB$?}8 zhXFL-#)|tRpgPC139j5fzg(6rXmZa-9vbCNCVAY;0`2)$S7LY@J^R$GqD- z4o2JG`}St8J`gEZF942NWlVn?u~1kJiP^&Uw~`>`(^=C(iQe z4?;W@>O*R#PSHX*PFah;23SH5YRhkj9xI2sf6JHABm>XFC4wBh#u6L+XTSP~T?~xq zk}Y>4GOQgK^oPB}*+nIL2hwm%HeuqpYLDU!F}`OhE1*R(!3A2w3C`{ z8M8ijiDEfVn{>}c^Kcbw-0D%uI}s?DpL8>L1;_C${bqr1+ealoHw{N7R?6Of_)yMJ zwd}ofYtB98#HN(l|1$c5EVxZF`eHaD+bb3V2QID@a%4{lVcG?Yt0FX)1}koUm$C2co{ zT!41i*l@+1bOD>b7IfQaP2QPLIui#UwE7rJ4r|~#^1K=UhzWgvQ;sGDa`y@1oFhZw zpy=CcYH+YZO>}R1sAuh!Xo8VR^+M9mRU_x-pY~2%cuEAV^9Rll8&k#i6Fjvp5mE?q_w7j|lvC0r_J6f$b4Rzv&eAUyh_6%`goFoIdqy^}B9vyWvWtYeg? zGh2DZfsdR@Dy4}%Re99e*42euV_cav&Qn#sd=NmuNWWs{t*|8RAPz z8O!COt&g9Q2{|Sc#pFOV9(|C2{r8?!?d--i$}L$#MH#r#z^^1LVOcVeF^X|EHyvp1 zb4cIv@Ef(R=-0kXC8H*m?FOWQAxC47R&&6vT865Yl?EH(gaE0r1zY%&w7xwd&nCZB z3MDWt5#k{~2+%j<6>;7^# z{6s2v+x&N_ZWzJEws{%Wo_Hw*$$%N;_OTqp_gfJX1f72?Y^sc zhM0Q8Mjs@pNZ)}bPqO#i`85nGLP@IdjXUzK@|vPaiC$|4lF_~5=p%q(HiGN@H!y`u z>_eSA-|H*P`H3hQ{=J|KpSY3b@vrv5J>}$;a%S~>P0sUkP85Wp#^C@YLfN~l6OOW~8colLV2C^!>6wytwUDXG1WSfzBx+5soJp^n$!QjKc!8@z9xoOy!D z92R2(F}W=wJIE^m?%lc93H$?Qg;Iiz&Ku(eL(asOeVHx4v{4_zIOF}{O{9&PO<#_E zR&o|mdpS%<>D#N6(^6Pi`1SOL)r3(Z8y-p`4p1#M*L3hF*F7=K;SSHoCnm4FOQ#_C zjMYV-BRvejDjUp_i5<>6KXcD`+)pz-;VfO$cGJQu&Q&Kv?OCk1pVM>?lyC*;_s@n- zAYcsVJeE%!w3l|@!cEx8j;21sAOM?KwpMd0XJuCO zPs6ybkdMkL8z!0-5i^UJ(UGd&03%-SbF*OGsRM5;t5UTPvtXNY^~U`Xa7K`&IBc5g zPTKdzvh>!s03oe?_IYE4rG|hT%#T4B?iA+vRw|{iFX)`+fM(L)cv~! zG2??7AUq9s`G!Fh^&JioPjb1nb}FOgzW)hU8&f!ev}X~P- z-S+fwMAn-P?atYtRd#(u_LlGVLEv(T^Y)=3;Q(2Hl70Z!L+Uf^Q}#BUSfzADEIR%A zC)6P=vDYa46h>oY)L7&>9+|$T6E459qTa=!Akj~kX$Z_%V%|6vCq6QI4|f)d%8>d-n-3@Y*3Cm#4k?{O^wyg`2?Bnr#O}5MXl(h~0 zt$f?<->ow(DN|?U0$i4XZ+QxMF5OVYumh$^D*~OhJ(w5VJx8y!B;HmU!1zuRMVV9r z2_NQ`dTK|DUgZF$o4e>RGB`ZO0JrS7a5sp_B!OPS5tXyL-=#^KG+O)fR1bA8?mS<; zdPObJ=R_VX(Zw#KmdjK1rW`kQ9)|fN+CzBTR8vdc{L!x~z6wiqq7F$fuU24n-m*kn z&J56#^~&0MilR);+wfpue12`cc3-rma_*rh+_HxhAmuo$-PT&6v2XnxVX}LH-je^! zuLkdVqtKt|YrKrc7d7JLIkGN0GO>a%z^)-Q@pr=qjqF{lbvK3#E9$UStt|8`c2u$HP7a(GukJ=s4aXmg2ZjH z+?-E~>F4e%7If}B?_UUGzk>)K?|r3BZbT~*WG?PQ0Ay>M&0RycdhB)XPOFmr58sy! z(!urq6MP{Lc->QTN#BZ>GTV-<+Ihn#kV zbr$|vYTp=)OOiyqDEvaCxoCY@k_qSy2WcRC@+aV!LNJj=n%GYAhqs1(SmnF^0akdB zk~el2x-uf0;oga0+{yk-9$yeRU8z*&rveg=07D1LPg+9f$M#6tIw$ovWABYFjOy>s z_(1{UIUh5NIL>qI924gA*+%nTW9m2NDGztuw!upg!mbs&;=Lr5;1bhqRVM7nZSzm2 z%oJ>*^kCXF`PE9Z;>t$yt^rk?rMS2_SE{G&67|qFC9;c`p94dJu~IpmmcI)wdmpu| z9XWoJvIyhUC`ef#gd%q_%s+*Ae@kN%joQHrF-_P-I0fQ^M#}oQ&+G8Ua`dL-UQ8Y? zc~tGNZZdZrnX(gv-`ro=R3l`Eg}3S}t|L2ji$xqZZV@>D`r-SVFX*p5Q>&E)ynyhB8%QDL>k^9 z2wzC}7Um$Kh`DG;M#o6D`_G-?`b=?SeyaC^{6?rF$*A=~W<2`BlHpqlooQF#L*oA2nk=A2k}i% zz8l`9gjI@R!I_@L%L60@w;;W4h!@F7QG|Pjnq9!3Ql}W77Z0FmYi-T%_easUF^{${q zL8#R-(Klx6UwqlLf8H5UKKPxZ@Rv06Z7al!Dy87RbJ-gMeHD$Hl}1YS+a`6xh$oaz z7#>A=pdj#+{LMY3KN9PThjsnzlYF_*vZU(0p>rirx>yWc`Y_Z-!h9IW)=;&F_Bl0{ zZGeGiw=POwl_deqW!WH7YK=Wl2qfhdD>@_q34tdh#w`3qY$u{2%7VkY$t&AJeFC-I zEI7#06gKqVA|^6M7h{~AXH||8dUesqX7%0>Lgx>#51zN7XWI?cQLVi5I|{%IPd95&{N+)`_C3P{-bxO| zqyw`Q>V8jb9SZ$Pa`$s_pFS|4a|38-ulS^eBMGNH&%`X&^E>P{ngPX=aaA{GTc(rO zl6Yu6Ye@wK+1vrI3vYqJJ66q+43m0^eY#)|1_)?G|oHo<+RVfYvDZ2pPZ^rp(8I5H;DW zGLiw`!NtiBVnei+Njs?I^4jCG355rR75Vqq9H#COQ44<3TL8gY83S7jxl$ILy)w9x zlWnAMe*H1jo%w1JF`+Q~Uim*v#oq;4QtHLm9-ZVDW3zXN} zL+qEG!19VR?dAz|nRm4GXK&cpv9axm*21#QXJ3etC5r?iR!T9p#EmW3wA;I{r{N3W zP1NkRb4Wn_MGZErcu@hHsN6QVhz>fJE4{h4VzjG&DWS4NkqficP@ShoF}QWd-LIbt zw$ypgs_1(+IBjc-aaS6gZ0OEkOuwWwQdNa(b^b$_hJV8=h#)RXt=k?&!M@J6%o75- z#}`%6iBewhJSYEBUF7_bSXV?eP4||?7C${l99Y|4#DMxbtjdZD?K}sjU2;1E-IaAf9)JjqT6eG)IhzInLsPE+VKg;D$ zqsDp(-3SjNA%$gw;8sj5^D4kJt3!c>e*-|3^)5VRiv%aE%Qy|)Ek$f8Ge}5ef+d)6 zir-OlDd7vp_nLm!97kFLCW$IaK|))YR4$>Wp$SoL7|ZOWLPYWHu&1mQ8#k=T(BDWq7`H;PSzM^q zQKE$C1+=2yrR1b+dtF%V6zz_kZ`>O4+sww;SXqb2c=Ypa-hX|C4$5@L*z#Aq*~t$5 zm8uUNi*_bxp+f&4m{UqVzL@`g5S`mdRcoDeb2vS9GaH^Y;HzhQQ9R#x;Gi|v6&r_r ze;l=Qhm%e(b)Wi9b_=f{)3p!_44lSVT8CSV&&xN7&ucOY2TKDSFaU{pe zqi7HH+pyuzOdhJtIK^bD13uN9lkFy#SDyJ%6=m;8Y8rk|AT4C&s22>f#0+QXhwA%3 zt}H_o__tm2l>rfBP>z%At4|OU(DT$&MY$1AA)O}OP*qzHDb>0VgbVnglNYc1|Vls@TqN2)z=N^{`1tZ5&oSccguf+~;Wo4vQuf5l=R-=#~ zoO=<P!Hj5SdK)$GCf7HTjM!AyWOy;4qV1$bHsvrXF`vyD^;=tm zlT@W+%WCTYr1&N~**o)~t80)YEscWwAv_Fv+Y~}YK^_7`L6PI%Ly{N1`6xnu$@z97 zw9t2ufZHw`^&Ro*=R5C}%2bx*OP)GX3;`u}M?Nv2X`Yb);c&!_YqRSbUq2V}9Yp$w z!M=VZ4YkwebU0jNJ<_cF?GRe!1DY=BfL5r=MamPPivg&Rn5EK&$bQBl=Mal@B&i2j zdd66^zkwzgS`|Rns?m#{LZ z2@OQeKl7DCFzyx^M&QvPv6q%t0!%OuF&#;*Cq?uPk}2F^@D$i)knO(c_^VPUT8Kq- zI&{4vW$Uw{t~&Rxs_DS}OpW**4&u>ERukqA)b-~@^%mutfI5qEag0A1>c20iZ#S9{ zR$w=)2m*91U&GJhId9tXBn@mB?d%lMg{!>DGj2}rQ!*)FNmT=;4JJfuG%Kde%;DNb z8bCkq3=le#jGVCT#O>H57g3&iZk;DMsYP@OEWpU8jky#3oT0BQOA;?$x*u(IzcpdX z_9J3W4{W zhwM6w`>-UjtmXBxBss(0xR^S{Mpc)e#A$bB?!0%kb*l zcge~RVsm6N@Wj%g96Xa*0q`ag?+r7|B>|``43KX(IvhBtadk#%S)Syj3mDu5^OtTh zH9d&UWE1?Q4K-*=KNy}QSpFCY9!{1;IPvQucu#?v$rmGN zy6Toou}oBS`pKV~RB5<8yXeK~hpDi>!0APgBj=?GgRT$=H0|_`8WI{rf!1XEsEH%) z{SNtzadRuRL1Zt7<$whB)so0ZRuNk9evmFOSn@@OL>TMSV$2#CDZJG0Ik zO5FP1AEQre_fEnAE@?Qq%=GgB|2cWthxeeI%CltpgJAXQ%@NI(CbLwkR2JiqrTA!1 zO#EhJVy8Wplzx9(hV3 zyJyG)*KQJZO9@v{ZoW4f!1-L5mUROcLtmwHCO-KEscQ3~ecSe^hPYS=$ zW{d?bfoY(=9hW={)X&JB*sX9?3bxFRc`!g_*Ba!ORPDsd*~JCCI>1CXS)x@8*HScl zbGA+0$rn%Sn@_eH`nw~CNj*qgIAL)xPB1>9xvNItr|>&-)_UNIjj1!vo5qa~NS*s@ z2-Uu)EbHS7P;%^x=*GN7Ot<@}NbEqLz&Nhxs+-l(bb3myp6{(+;aG2r{SEeGA za%~f(y2E~jfHCnc!5e#bnu0qu%N7J%%Pj^kH!goSmUN(hVg3Or9-YyXRakv+(?<_9 z+))qmPE_Yy%j8#5+eV~5o3b>xWV0*d65e+go5+}nJ(=eSu={>Fr*dEOhbxV(vmYFl z`@@i7`A&zHBvuf!&)7kK=EArU9>swE%ODrX($K!P|`5KrpB+ab`5QFYIlHJH7O=#(n~+5rTpz{ zDtp%eVS&x*3IG%J>zxWs>+4CiDVXKRny~7&$~6my?%xyw+A`L>SdfodAxeg25j1h(5dh8T{u55k z6T|R@TL-@RC;|&*#%#xe+AVi;rRGK`vshC)?1@>Fo>`JHJ2I@|@SXR6t4H@`?1Rk# zLr4V)fr=V~g(`!0^6jZM4WEHJ@uY-q!o0MM!I59e%f=|i@wK%T!mL>~AlJ<+kc>jL<03U!>v&niGG=y(LKeGdrtt)eq~6X0w>54a!x6S9h7G zqg;O%HE=0*PqDOqw@z_X63!ug%{}#cIF`eGd(`S#Pl70Y z&7v~6cqjB!A@f|7_4htJ}h2mH;6!EQw)sjxhxhtbwO@)N@(e-!IXUNo_LybfT;+Xg)6gy>xZoy^Re zhuSxzR4L5)7sK$rVXfOBlp+dB()c_cOS135=utv8{!khbTGt~FO%w?prtn@VH=9sK z2VU~yXzILQN3)r2g+Rt_Bo>cB?=R?!OXz8MxU4S017tFqGMu&Lbr3QFIy1<5j@qRN z@O_K6{Bfv)R~esdD?`PpcwJ5|!@auX$>|S~On4{ZK?_xS4+F9Gj^f(+`oche^l8)O zt(fyOcG?8XY=QG!fyuBtN5a$f#U8qsGNYO1r%sF`~i81W}WPlFHYb@wV$ z{LM$-^II{pgmY{yvL>j+C8fxVs7gocRZ~8GjU&fYJ_F^+*@v2y3DW&k27eNRu! zX#*g1LV%uIG*-K5Gh1k{Gh6Kr$dTcJr{jfaO{ZC#Bu-cY70E*M_5+K>(=-b5nJFU} zK4`vC#R?S_X3SjtzsiaKm1`1uLkulZ!{r`zR*%TU^VbyQwxD^Zth|w57e09ygYt4t zoEOop!4hpkP!IGHE+Q`7VBr$)U6Apuf07txSI}x9o4zZ< zTSGC#Z2~N9jCcfv@We^Q%tNkDJT~%)GEzGwr&^eRZ4xNAZ0N04xkhEz5W0Mw<_-pL z!Zw9^qs*wO4(BpB=5PldKB|t&>=~Yaw2vbx{IXeU+e!ilf1+1!JvDCAJHmcWK`jFU za29eCO6W={qplO$`Fq>1cuE-rW1;PbxV}tc!IaGuYLn-GE*4CbC@CKcZ=pUw6xwbwb6xzs)qCLvN5JGEn+c3IL^t~ zrLop)ixZyG??Y=95W;C=vLFgxVezSpiOK`R%(ubp>Aen+ypdS7o-p^nIDuNYPWJb~ zdIh6|v#$$Eb1_J&mP^k)6!UK*ld*QAF9=xX7A#fnJ;{8}xcd62sxmm5qF)we^#*X3 zXjfVqk}lLP>_QXMVcen4qc_!Quhl)ifmg?3y2WTsT6eodi$8;98atDMEDOx9G@8Ot zVWNkPY1^fRNr*HqKZyO)l0ZXWA~>EJz0ID9t*(cm%{trV#jD~u5rAN`v+VFPL6OG{ zKmCA12tkmEwjjtpmgn|H>M7_FLq7HkSiuN3^I&$QlO}q;39>4`u^8sBo_iunVv}lmg$HFe}DwVl)cXyI^Rv%RL#ER)R34<0QhZ1 zrj{3jl$_nDlox9C9KJMh`(V^?QHg`O)%<>33R7mHPCrSzG|2cv>Rm6cuNmv6doN|c zk1Gbe=*dhp=3R2{xzW`}v6ho=Tv3C4mlaP=PvMHr&=)eMH6i(|G)yH3~UFA5QcmiPX=iL_V{HL9wQ`hyyecC zbr`9s>1^T=>nhU~YdZI*!xCn@ZJi1a1)u|N5)+}bjMf}^u5@kNUs>=>{$f`9ha8z1 z!XXk0ELtqoS+U|F_8jsaHspBM94>%Ag8vju<46g?o0L6{(M^|Xx;VRzIA0zjJn6*@ z4I{>TsDo-F{g_q>9s~%R4NfD~Gkry7A78$~e!3HbH|bz*hy5oV{{j|2LP!cQZ80DZ zHK{_?Nh~P&AG0;qZHZ9k7!Z+HVgC)zAVaC-`O$V^)lXUU$jD!lUs|p>)t3)q<8k7gz__OQm{$ywDF<#j~-mTX7nhBvuw5`O+Eb z&oW0XK34-Du76wh6-p(xU67IG*s;egzLcQjjb&}itee}|A0H6Vi%G-5d-EV+gsFRv zPf>RZF5<{H8W>z3Q=1ut>S#d z46rLhL?_TUrFVwi-#7W|fQiUumPjN~foVUJ#TI^nZzPpsMds!?abN{@@8R}{V4io$ zjwp-yg1I|)bW}O*X6+Xa!{0^YKRwU?T@6r1l?bKw9C7yek261^gfDxyW$L_QW<)m* zi0*y;r8Mo7|bEbzB($^T0Z|NF~$n4n6=bKKrD3bnd$fH>?aT@>WO?Pyc*(FC#+ zg}Z3n|5AMVpAWhyIpR!0%i9Fx0Yax`}UW+PWm#ZQK8BL-u%^#~=8=zZKR9H3e znEyWn#779r_nZy4+18R^pwF&?v5m%bp_QlWy8wHM4&CcOZG(aj14cgsjVr5I7(K2Y z3`?r|4R&wT(x#>)FCW&XVt1F{6PJRP8`<2nbTuHdPAX^ME_f?JqoG0!-gi5@4^%r)bmw`4SN|W{W(4ucyxoj0QXo~HIhn|)Y z9F@T$gkzQQ37~*}a22lgAs@-(+l8@!V%PhoAH#S>7lK?}NWJ|u{?qe>$&GV$zmSx3 zC>6>BcE8_4Y0L9%s+DU5; zZw@)N zSYn~!En4?^)P#pI9!JU%&nB2Il$YVfNLFb`EsSSx&G z*g|o|mf-TR(d}xZ@sUCc{NL8)-<1xYlE_+kT+^*q7$bac$IHshQ_LoFH-$$9Biz%2RSjFB#%d?L!mVZi@ z7}(w2XA8dj;DLz(9i%$-BNc<8z!g^narl7NOG3r%%YJ#ex4D@w*?-=r7w`Z|p>{p0 zcVV65<75fRAky~GSU7qP)bSv*8FVL#u5K-cWIxt0H@+ga_?T3DZu=;!h0EH!^|N8z zjwYu86KrgT=|+clGJU__2if~E^1d~#s!C%^j}^Y+0QMe28%qnQC~r`v=JD>zQ1>qW z-(CPx_E#fKGZp``m_m0H&}i>YpcM-rzJ?1X*vpq^-}!cIJBu&`#S34t+|RcAQ?^uB zD*kPaG|PjzBNspo7!0186w3ykz~}}3YnlE>X#@0>a)b|qksVJLzgRLHjX`OoH9g5_ zlw-i0C8}sfB@lomxS_o1+)Ns#`~gsi;v|ZTqAaG{+3NQqz@Z8A^PPR8B$zD*xZPqyR@nX9*%g_xn zb|iTY27H7j;H6V*x4%e^A0pV44pXTspL;dn|8FNP%@*J~rCgJfL3ETolCT)!o$#|9 zsnTQmim(Pai30?7qrz0N$;qFijTK?PVi$+3d6|3;`9`vkt6+p3eCQOilU|}rB7OO& z9**>jUooZ=?>OSfpt13iElQwCcB`_^CnaaN zaQi$j+^QKHp^;#Oihc?@<(#i{h4=iQWE#|H&11lzz)6zT5nkVd*ey1P?a8U#VQySqU) z-6-ANCDJXO8{X|vkLNyrpL^UN-w!bM*s<2SX8dMc^O|#)tvfbU7b))z3q~kIQgOIC zY0iiU5_2qA+OY^^ezS4x2sU6T%kC<-`^kTRC6OddoXS+$=-9Ol@p4t+VXkK;3NFjv zzk?qucHIMKlq22Qey)7VPG~PlHe8zeBfFz_YI?Nr4S@$co#hqMHgKP1U$Bde@DP72L12whbZa^WFmT-fi#ceOwi3Hgv`XVv8TZhX_C7% zW;|F*^@An1<@*6mj-7$2t*~=^+P`509B%7Fk~BzbfEB|COE!PQsId{;JWs~Lf+K{D z6iS4}wq7V&f600@LrPWWF!$7&!>|iR+7d~eo99i(prR6cN#X#vv*FNg4RfM&fdmT) zP^MgBYb=|l&Q4DSTncpst<0715Yy_DkRFKo+ZS2j`i=xQ5Ix_pc-{ce!5V=>;yWrS z9H8)bqdAoDj#{M*S(OGK4gn_zej~+a!6thplZ*A|gviERN{vG$jvK&WSN>WO(aU)2 zhR3Y|bMHl?`2e2Ghwz?S`<^OP!s9caG^h9Dfo%VPbHR0(4=^kQkv#$C-MeMsB%~w* zFn*yTvxOvRVZH0fBL2l?$`Z=CG0U%nuoOS5(*b2>E2c?+VIMSe0SgP(V7n#q4jXm2 zN1WsR-nE4#rH`NpY0j94CU}X_w}?Ey?^I*%g>3)tlP-b#%@`S0g!rxRN&$P*6QV? zOzu0(RvQ^apH}%xJ3Yta*2Qq=@Lh{)OvGsHR0*%D23gpfujPOM2t)Z2$ zMbGrvoysni2rnHgY~xjq1wukY2g44*4H2%<#;&_~G8HljoLR8uqhi^p(fditvS7d8QS++PuLlKurQ3 z?k6Qb3;H{^A*hIk@(x`k-zN#BhYH=MAh5tRMiMI*i)84U3Nk#gi&aYBvS8tMrW4h~ zkcQ&*)fBZWk&Xs6$M9c~vnu*Twd-iQBO4N*T;q@*ZB*653p8Mgb6M1wb6(URV7Pr~n3i)ViN z$XaC}pMp|naizLF_kN7x;LOx1%6+HH{pi3cb-cih?~aNQ;GiB8^&~@-1WzwAC51ut2n3`b3uJ11wSwh5w+v)ln z(cg*+kwzov3{P}_EZV+;_c0Xk^AY)M2#%mMoUkc=hkfST03qA0`m-v-w}0C>@C(%M zdY2_{(RgRjRQZxY=tVWxd#eR{?XQWWDYe%ueD6Yv;$q`Z4K3V;C7Xz`(PmRTku&+P z5<#;S*UJpmtilg8!kY2~$7#{0B^l%OtXmBHos~3|ulY^g=tP}grrJigW$1l;BPMwQ ztZk~S1Cm$CAO-956D9Z?9#n@GxZhAO6Fz*e-QG?_e{mRDg7KD`0K>2GWMSlhL{lZzf;+xXSBK-CN>)7#(ZX;OdfWP)doj!%kg-9?6 zbxov+UKhWZg0Yu5^9=ST^LR|@gtPhW0DwEufX{N46L6j5{s!N#BKoHr{Xon0%CXQl zhPU}W^f-h0Ff^5z=yZ5W+AY?F<}%TiEjQJe52-N%tUR%7t0KVoq(hR-@5faO)L~y; zKRey!tb-LvcpcB#H#2V@$t+t51&qOV#7NhycGx9lu9FP3)Lt8De=TnKVsPpzxmtIK zv6a_Kb?tqr8fHuqEIbTbWQTLTKlL3iIJ)Gl4eox6V_KWRp5MW!y zV)-aPXB@ZR9#Ph{U3Y{Zi&F(*e_uvS;FFxwX@Zad(T(Rv>nQPz7IIilV8Z;v&Dw)8 zPf;weeCXhtl29PC-4{0@R%*lDhl3~>6@O%NteO|us4?ZX5FVBU?5m=0>enz#Du=0z z54@h?hgDx*vd$#!F8b&7=wO`=*>_akr-v2kgauk%Cgckb!>m}Qu_ch(?t*y69X!K= z`cdg~okgG>YZQL*&^ecXrx?3>JhH01#_UIn6hn=wS zZJ58?KP<}wnBRPB+ys2|HHC`#5EjT4=7QD~#=Paof^Rp0 zE-zHI+G$>R-cYOrKY85V{&B+COq*mJky=1zur20m{xOyf7|Uf)n8trp(M`} z(rMIs`Z`0`z+zmsatsTPdxcnqfuD)KO+IGos&9*JA^7-3T#!Bm0FFhLq@(sVMy8s; z_9d;}{dz6BN{_pn3n;IUatU-EFEhOYSDIY~|Jj=9JN3Y2a)QcjWZYOgaKDYo-(>Cb z17|DkePJ+vW7Jq6?3=wi4GfUa1YH8E^Gvzst@gtK)bWHC(z>jT5P+3U#ZT&jAlJQG z_99I&WSa%}9>%_?%^$#FnmLsg9XZx}nv#PNt;2HjqIN3*ol(!CCh?!g5mZ`QQEAQ% ztXIHR{Ne3m>lFbKQ;l~YiPc6JZz1!Ypk3N21vv#0(^3kp{5OLTCk-UqU z1wX>KM@%Ek?aa~V16{)SzQAJ#qMaV=X8)*UpG$#l>0$(m5+7*$(>|@p!`?k6OQT{k z4wcf|!@7!`xi{hcvoc;SskS8*72J&{*^5gg0tS1rosA?HvxDdKP=gEC2e($G7cFic z!|C$6-;*UqPKJ?%6Q17U1Y&K=` zO{&xucMh{>+;!%KHy!RZXEZ2GTqBBTz+9i2FAM7d(QRaIzuX8ceFN%_w%Cj?v)&02 zfSYcDVW&2{$g}_LQI`B~*8Pg0`z66?lnq{nofp4ZUIXK&Q4Lp(zJ-w!7bU8a4=sd1 zjGgg3rhIrK#payp{&kAdEM1a8OhJgndoYttHdo;LO*1kiKIjV-7?G?=yMhrj^-A>S zKMpK_w}S6^Ol&!)C3AyD4O4RNWG?dtl&Oz1#J9x4Ylqd=va?g5vjE3E0YKoBe~x_K(ssc$M0IK4@#$l z@`PJOP24ZwXO-Y``f3q=GAT*-yi9yDFS;#qxzA*Gc6RAv>X7l?V1o41XECyXCm-;+ zC2jIivK5yz#1u0v1Uxl4SMr@c>&TDzJRq-nFvWO?sxlVtVSvW+_6zwZ2P@5e@=uRb z7WDA2V7-dv8_srJlLxj09t&|r&`YE8HG1i~p0f}|r2$alE6c%XHgX2t`a9G0D+w&Y z51Gvg8C&VI^Uo%yf{?Ifx;k%TmL6NOb%xS;IEQoRMh(6fj+cI5Ghmo0)4qfaR&L9F zN8BUiUgm+g*&Ta%TR0%oSTbW=)IUE9Mi(f8w0%_pQPD3Y8KtWe=dbs_pHX8kc}FIB zRa|0;F{Dt0!(X#0`aDcchve`} zts11~AoPfFrvy=~T;zo|up5qF2T;q88pG*H=_OZS;cT&lKbm|^mzR~IK3fK#IvEC+ z{{Tv~S)GN|+6{d|SwK1U;|zI>Bb;Ddo0{7uOgfi5Z=wQscoh}YZdTWMN=&2I%sqsK z@LR}#;+mj12{>F9=U%}2hXc(f0SJ10GgWS!~hNdpmjL_G_o0B6(he>vzW$m0p3XI*;14JU5$sQeP zRcpSW!}W)v@P$%B$dYCQ9<%4kWd&*sKF3i$ekOnTvSXX+MOR?PI%^^CMvRifEbz(* z?nXD5NetYag|N7cO~@_D4NSLH;nnPRgmKKkX*f=W)L<2^I)T^y@}1J<^yuZ=1~j{7 zIyk1auhUAz85%Dtw|6G>-%tBO02Z*p#?{D%ghr2t=a32Sx5xT3Vd@8X4&DX&@OUKAkFSj_h);^YO%P^>lZ7LZ(sFs6``bj<7r z;WoBPkJxoa3^||1a`n$hPFhW4xfpY56uMG7jgJjzq;FwZA2VC;oG51x;JXu zc`R-{dT`Vy(pS$8uB^B^B`0sYZ6%xWd(=BN-Yg%Fue1Aau@j3+m~CUDaC$c{d&0VN zE3GuM;tG%4X6iaGXSURCKF(@l5Hv?n33tc|vhdN@9tex1oz}V*19Eh3*Djw9nCdkN zvJe)}nlRMWV=R01Y@ZI>ogrGg2EG|&vA8!;vI#C*FZ7&H3T5{>4hmC9_mW>$av$M} zE^0*%NH6J%MOW;N<^C( zfgl4a^Tc0Px)TCj z>Y{{1?+*)m4%_8E+N7^&+y{{Q)vH~jp&Ka&R~~m_q71x@>}1boCKeyh3u}ixWN`yu z;E*7%uZPBCn>+SAOP?p^*Dnx(M&LW?dVag+F~Cs2d>NE3(0IDAj?kIJ=+E1N|8zJe z%YRkMC4I88<#e`sd-$%s{P1N$0n8lS6auX$rH99+Pd*DrA4y%UF~6eYhLK5x%=0`` zy<4Y7U`C9LWKZg=pvG_S+tcf~>)Z}jK6{Kb=t`YS4>z3`LvJ{)f8D`2b#Ape;WG;h zX4vmPyd{~vLF+Ihd}D?s-)FXG6r*ca+^NuwCgBhQK;8Y4SXLJ?4wqctSC~$j-J1Pi z8ki>FWBWjT(R4H=ozaiTE$oA`ok><#>rLv%_h{ZbC4)H8yD1lRGQsMa-gc8lOE`Ky z)7VC>;Y5&d8c-EPXC!sbr3B&&C7@k|aq@ejantl9n&HRTmcM4V{Pk}~76Q`LT`AL% z)c5B!kyg4af0Kcd7UX_^RoH|o9Ve82vp@qcI;G$id?x@VkI=|sTz=tWu*SxGErR~kLN`8An(|9BVJN7A`im_6qfaj@^3ID^^QWIVoX z#*brh-POXy?Ym`>3~t+TgJ;JOIW@R_*A{Iv19OfE0X)4(bWPd+ImPw6Mr64HAlpK; zHeKmC+U?En-r|jvRbm%x7bQd#6~WcM#h7GMex>UXvsdR-H!?jvGsC|$H#e7N(m>j( zk?>-t&mJ%K0TM8j_vn#!4(EC#rx~yKVwCP}j_^8mxsT4!yPvG2hl^90mQUP`ZnK#z zQZ}UV&e*NNSn@m@zBRBubGV70l@fcXdrTcRNRE`u>dDOW-|;cPHtpcjdM86bS9-FS z6}dycp&StaUA^{9^$eYC{4jp)qnN~}Ugr1qJZidbCI$DUeWo<%q%7rI zOJH>hKZH=kg1N4==hnd-G#%QG1Mp-Jnb^Q2F=I$FMI4${8gS`WAvCYjYLt6zk>N6I z!kkNj8Cc989bYN$ZO?2DK-r1&-F-vTUo>CL-*o$>3E0~1DSvU|d{H||_Y1-BJz`VnN^I@sY=bdv8gmUyxJHfEVeTE;z(Eae$)`C};;K@y$?0T(6bO zb|m~Sz6t7~ew0~u*i7RTh0k0OK&;~rSkF&t_?en%5^#y)9Ne@ShjKznsVYH}DS8U2%VGaGHE zQEX^wlx3OSSMuXzu$c>u>LnsT8NRotCi0o%BcvYXji@YQi|N`An!C0)1Twd{g=I32 z7Y^FF%T?nhXn;hN4#<4jT|-QwwS6WK-a#W->djlI~e&wS)8pjh4$ zW*4#EIHITw$mlZ9)6ne%;3<7#nJ02c#}ygvs`-tz=du{-u35myK;oRYyehv= zMfEB9RDE1LW48UZu=@wuDxGMZW;wIX$p{Ca?Qox&D{~z2FL?WDcJwI)1F?DH*%+I@ z`q)jkw@BV?s;%<$US_K==)J`J@PiRN00fhf51HxW8DGnq4o7{irz&vy5dC=B(yi5I z6_a`Iy+j=9@Nk#K2p%%1Mx|AIGW9Ng}T$#E~8Ws zW5!u-uj3y5sC^8{X6otkn-p2AZ3%M$eE^27{j!e+i7 zr;PILK4b~}G;U%|I+h9O8z@{ooGbIB&WNrQ@N#n~|7GHT!Vy0D2e+}YPy80|Xr;_t%GYt*58+scFoOIpPD-J83=O-mPq0vtig zp5;0o11LLiew0TkO3gcVg?zLcX5KB|GFT~^(r2EyWx0oVt}T;+_AMsKwn%nk%#wNt zV`mB!-iJBXEWd$#_+r|8+5oKcM5ZuKisnD(eJWmaO7npyam6`<=_;5v^)!hj`3=M7 z(|E&YLr9Q(bb_k<_%4{K@s*Xv8~baFFXV!L67DA*$ASRXgjDC*C#=ffM_BzuL_axL z_0r96yY@OVE%yUfNUcUI!-OK>!M`Z51<9B?7`4lWmZ0|Kw?^HYYC2afH3IZ|Sgpy= ztL=&tk6O+mO_)vWc_O0DZhcXC4rD!#fSc<{80{;y^;D%hinqW}l^`2LXC8k=>$pjtnywqt zy-SpAXrbXGpCC-ne(*U7!FgNW`*KkF;WkmA?fJ5<9q!S=u|!rlPaChXI$?KWRivOR zEM3Ey3<%Wnao2gdjF69i4xy1DwsM#@kB+&a6An7idG<|Hg`PC@5LwrQ`l!I$2(OPi z=gIU|{+`|dhyOi1{ad2(-YaM0?N zm~2SrGIBCN{Vj`s5i~0Z0F$&!Y%cd}W;aY;Q3qVAJy)DT8~NmuR8ag(oKk>bBn(}f)d&*TIAY<$x8%Zi*-;OgVZ)w= zMbXth4Ikf(ugX>IL_>(Ci?Aps6WXfLKYKj1A+Y=y9L*}%bR7&aDo`YKRad>pa?r+^ zimMvgXVtf&JKsWBo9%z+R7VXS(Yb@@M=&(7eO}D7iI(fH%M%xWT&SjZ=Q_y35V0O& zyxFYGe+6mpOp_KrYN@5lpB^pmy(OW2X}sZanvHAwO-${mzpt=~gf#PL2F$7Rn>n!` z!u~g_ve^Ap39V~pATEAGjF(!61=qt1CZXBBRfJwM);H2G`})zC9DuUweV7j2vFjDw zk119a{jE1!sj>V+&Qz<85S!!3o<`LJ2Zu%cb5SQlE6W>N8Y5$}h!AH^7R=<5w4nQf z{4h9OSf-ot=kmO$G7c>)!g~~h=6t@CdvB9Rx{5FccVC+TQ^|lmEM(1*;YS(%e85wz zY=!G>XHds3rJ62JFH_i6J}-ZDgtu*byHjmDV6E>lh7s-K`Ev8){kBfOmV&?FJ|v}H z)0ek5171{Z`PpS|@W)(dGuIByAhSo8g~DnEoo_9ktr@s_go@#JQu7`sn}_p>mCiSp zvf9Q2WQ$X;n>J&HW8u?Ay6lK|dn;(~c0+kH)+SU2!Q;Kr<6Z(=$tzdiSnFTr!=Ia3 z9IYr~2V`gHyuHH46iciEyBvJff~sxTWuC1`(emvlbs9JBr?l@JiK8kYwK#3CiYkXG ziJi4v3yxIW_bo5l4Xg$STuOcTUkaE&@15N!x_c7eeBzH^-Bt#1mzCKlR5Iu#0`**@ z{a$RDBDZh=J8^z;%^C_Osmdf9hbcVAD6(x31cKP-`@LxC-JSr zF*eic1>B%YpSF_=V??K$!!uj=^V8A_J~;xV$E3K5KI|?j;_}Cr9{6Im+W!FLoE!>H11&P0}cBU*F^njwzyF zo)JjsOJ)7a^@e<9%MqDmFOxjOTf>-8?@^{OJx2QK}Gxdjin+AQPE}Q)We_0lm_zu*~X~e{Vs`hA?we|p+8QpdV#G1a8}zrNS#M-p6xQpLOqv%}~C9*_V^1OnQ8eU(ym-Yv%CK z^~l;WXzB;+$=1A|=N$%}9A31^&^y%0UPtICrTbZiM+k-LA#~mc^DeYXw+YzzcQ$vV?W8n$&rT#=dPV2ShLTlNVS;b%C8s)u)S;CX9$cu(bwvYMg zEl}ivZB*h<{M5m*#XOkRmtmf_Hs<20ZF8>Sd|(9q(D4z~=rcC;^67I?NqcekofRXe zo@lA``Zy>*bQG+Y<{!DQJ|7o!NEilvwAx0xnCnEQRx@?EzT>J(`2Krj$j5~mIWJwx zYDw2K7O&gvl8+zn*3~Fe5N+;4J8j!^o8;yFlJxTOx5*t;eu1dqBU*#kN;qTRi$&f6 z*`ut;WFFTe+KiUZc12mt8m6q#GVldz1ZkTaA*-F&h&7Yn z4a!ZjtkgncS1#H2BC?n$-eELedTe~SQPcKXb_(XlrbPY>SV+WUP>G;;Bc++*b*F9LMC!V7FI;Z`UkC& zYf;G8-scy`{MQ@%$T|n%6{*hi@2bhVHV1Zq33a%*z{JMtJ=cEhuaFmI!PN|2x3i(g zuiGO99;OKStJ^*G^a%BAo9)yY2fs0fma@{(ehUEL;@{olSL3Yj#Q|u&1dP?Mdi3gw zh0r>lBMpQK-@xsSSgGNqyTxxhumj}nwMv!3o&*KyKjf+VO#|7AIeUW2t>f0|J)Lh2-*qXQ(A&FM4Ce%X|yvZZ;j-rk66@Jw#M~@N86S|(K&xYW!y@(W9UY7`2sV24Z@w8t+6djh#k^)0s4+pUCm zDjtEUDwJ2>o2o7VtgsR$e($f+9+nQIGc}1In9SzFl)hS&o#oy0zJQao4kK2=679|* zqwt1xC8mF{0|QID`DA9-IM(J#vQl2Ug=Tgn`C>TI%hJTyk$8KMIN{^HmzhoUsl4%- zU@KvL8Wny9mSV)WE=iYU^REkwM>B}@4V)ex)AF0FHikk^g>^`vM72Y4jNN?5rzDg2!Fj_-F`S!p-1)#zt^eBQMi@Y>)YungUm#9cG)!d zH;Vq8LcKF}F{;be`atJcvhW+%7^8!bS&8h7IvaPV{I09H!0Vl?g!x215NolQ+yW)M zdhn5yoiqjGJ~k{HH%}Xhx6jG48L#)L2|R`LW$J3r?E9-Lw2+4wM6`N}-!lt^qYK~^g`~JN)PGK57Hr4-*#CO@SoX=^D@55L@ypb9 zUcNY8aJW&KRS6WD@i=|rs_r?PV6ud**TWA9lh@{(kwC>W=@TOotIe2OzBBg$vC=s3 zM#-qpr%q;`qEY<-lXSyfeWZH6ZKloXd*YGCkNRq1mLkp@4LC(dT`pT>k*yy+^1%ql z-I9h+RMRR{&eZX}_VFSIqK&@;#)?-DBFpP3(FyP7Vdnl2vL3Nz94C*6x0imCmND#m zN2d`jX3KmOx8hlhum}AEg{&Q245!so!d2u4|IVL)r-Fd}x!oeX-rt|p(CA~))Kg1P z2uQBWwlx8L$cuBaAOMQmUxM8nalpB`o?~I#QL6s5Q!VU@43Z*xX@z&<1Tawk>n1q) zR=ODig(dm^8L2)}t-TibWRj+~(L`8w(l)vK7|V{g+Jj6CvB1il!Q49B4adEOCSs@{ zUV+E3i`SbrE86GvldCKU+n-qKTx0RcGX0y$6t-$a&qlAcfE@h;Ad6}Ai}t|-tt6&VSr`B)=Kj6_vz_8F@2haT_P3$5h2A;fNEh||(8Q5; z;y012_!XjR-r|VAH9vc^B`6# zOQCi2$VF**r~#jA<*fqFzFIx~=)*;g&BTq8Y$b!hXij^2Sz()d>F)F<$W<)}-($4N zitRj9&=PK^%rl09#$a!tXgnsK5zb%s??eg&;I`D)fbAvx5-nPRUHB!|_7KgvX2woV zz+t3X`D;IYrfgots5Sdi;X=8HgM|w5YfBYdufgXFEqDDP7^6y2skxV#Oe1xV5^GUo z+rsdC^p7}2IVWR?(U)bY=8?YDeBl@cysSMX!eL))ONH< z?mKs%KQ`)4Pt8)=usZbVUwr=D8#@z3JJ6;mwf)rApmC!{qokyctI_e}77p`~-rv z-x%z(|NYi~eI+3c-dLMBwb{)1-<}8p5Ci)%Rz)G=w~YMx6-Z0JHv+#CJZa@-@}azA z`br!kvbWl#Ck(s(YNIxCxb23f6XtD9`lLcg02ar~b%%#<(y3p!a(q>4c1xQE32Yhk zpZZaNAMRRF!o#neXYv89Q4qjQ-*F{b2)p}V*96}9uy_kkrcaTplz!gn-(CK(dk~3T zxkg>yr2Pe0kyqO;wBOReq`7T)?@f<#Cen1$NxnZJ!xiq(Dr{r65dvBc{F`t8l!;%2 z-BAMXZGn4$g?@4o(VxqTz?%m3cA1@DUv@h-Kjz`T>=YnOBjl%<#>du!UeV?vJ9SRl z^Tu!Hju85 z;JYY^H*#VB8yWt*Twg&uFi=e{9zWUsUv+cm_4iHhxYXF+zRQ2*0L*zEm}&q2TZCC; zM)?1>2Elq5EH2RjJeIcZHesGw+81!-HqT?4LIz6mE_77Z`i2xWgs&{BcoHYe!!WcI zf(3iOdv^`^;{T3l{v>))ERbRqfFa{>D|_B-9X%;8D+tzU`pfYb@wT1$+M@ni2Ll?% zsx)(k&uLJ1K0Z-maZZYO`o93dckR5*`K~V~Iz~hP2mAjYl*Gr-mnuM-v?uz%s!5M1I|oOy>SqDU=-~KOG*|A|Bh;DxRjJ7=93<^V;z?!)}<|PYa_2!7`M$ zi#urYze>#B7Rfk>7DbJuzZ=xorEvg_;j)H@%a?;8D$QRjw)_$jSEfA zRud<${eCjeOHB}Z2(Y;sMfe0S&|t>~ynpc5?RP{WyyR`U`K~r~Jqcg3|ACo;$M8cb z&P`1x1&i@9f(cPAE=RV?azWV+)rgxXNlgvXnD|Vr-uG`Wrj+F0-?ctI+)YRN+^;k6 zL8pIbA^r=hGqX~?ij?WEYvPDlWzI>*DR%DrgJW{ zyz#&9PNSW&C7m)vpF>m7w_I^b${3(jf8h!krH-T`RMU_*Y@m7%Sv$M%ifI z@i?EO*vKEj8|;ww`P0QmM-%%oC%|2UVst1_2#PF@uAWpz*mjik025~YE2Ndvpa zz`&{d0exkdb9uiklM5zR5=@1@b@Wyg;3G+_DgVoh`=u~ zL`PMxcd4DI3=5NCuLcUGyItPISWpz<2&2)hFS1JTCHH0ouoeJ@oNT{v>x)4T+^-Wp zDrnW)9%`H6{ivw`hY<8%;+!1T@WjKz!_)6kN%AXWKmAbTF*5+k)kYs*(vV>*L^hS9TMZcblOQ0$ zj@>seX9k=J)b#WOJ`!p(CjP9g7=Z8NYZO5S?h>S*uABJ&i7+td3BIeA3X8a^MS%9@o_ms%K1xwMf(rA`vsdj z7=FMW#J&0Oc14YNhJK1_N+Jq0j9eq*XXwaklZ8IQ4=;zY9liS*bMM=H7rs6y50Hv? zu^B*vf4>6-QCM#=!?ozBB}D=eKk%{KTHGLEvUn&83IE{wYc#TkYcDb~A@*JW(#Z(j z;I7TCd8ZRcowB^nrb(Z6qy~rYT=GLl`>+&CdbNX&gCqRvPyQ}Z zJ^(?IGB!Fnn>U#OJCVGadDkr|PG;0tQX(v$C7)q3nmjTo9mrVC-v4-x7W2q8oaJEV za%peZ!1M53C7psh78dkC+4&J|*5-7|mUoO=mXirhZb9;~2b$^dNwAr{dXov0&K{|? zwe2^u^_ci7->;p(EA6SxYONT#Q(-|C# zT13_;%v;yMsX%d@xWRR$&DvT^HI?|&hBV7J-q{2F--=|9==TP5Lyj<}(rcVrV?4F*U?iLjl z&DWXX!T)K;1)Blqt?CaaM^R5^SiB-_XFsw1d%yX|BWA*0ThXapW;3Q~m|B^X-vT}B z2b>yJ^nJx&n5EL_yG{3evE4*|KQ$BLP+MCdWFiTg%kiUOq?JCbh&_NA>YEyEvux1* zp=*~g0cRtP+7;~X^<`z&Y)n7pf6N067WNw}OCF?ae|C8CDtwL25{R&cxs0VyJ4v6@ z_YX-SU1jWwp0mj!MS4?j#&N6FX}JNl2jbTd^r9-RHr^U@Qxs0|bb@x;oHFTt-N*;< zSpd6Fud1Cm(No04WK&(kXUvBollBMUh-v{TN8Kg6D`}ekMr?mn&OBeiz`^_;1f0BQ z8yf+!oiAmSpve7YnlX}dB%%9u_#@_A^)`_KQwa|Jf~1UpO9LU~8Uky91%D}HeRlRM z=%J54J#RrqSdpXdXb(q5ME}V@qHq)tk{dT{a7}@rUaQdrUn9j#HAX*a>u_se4k*z| z{D}iwu5jYjOYU!hzjAiBk%BKfhoCUEKW=1r92Fxy^LJ}@Jtlam-{GqR3i73ubp^-2 zcRep2&OY5x^NlIH8hfVSok}{lkLc*Ea%rymU$m;fGby=fZ9EQ3i~i$7jX6R>^iwEl zsA2MFBW3gt4k5^w#!m3Z}rb?F;yR0Vf5S!iqf@P-EI(n>Tf`~{% zWwVEi>v2T#^7&7zd*0NM(T`(-g^i6pxVOI_Nli<>Y$vYv{`E9xu5vNau<$;I>M$DlUez!UmDCI z*c2gP1ReKBQw4);9Foc{ci5KV82#i?PP+&;qhDqdB$36X-?Qt~MltIpqoOh`qH(5n z2-*oH(qX(nz7nvPZPLY^eGm2#sK=a;Rb{U>&RZlZ{+LtoV?h5%wv#`;m?ad$+_M%p z#2mv%F`u*eET4sJ-`6++R^m&zJa7O@v5vEbVtzdCdqMIpslqCXVjuvGLQAH2V8}S7 z$(TQ`FjCO5BkZ324#dvRJ}aG&?6tqQ2Q-)$+iT5Wzr}}$YBEmvvlVfjQYh^ z8E+ycPTQro3UFcJ#6M(x%nl4V;AaC%*J|wD@O#JXKPs4$5D`zMpe)ZEK-1# z^(+(Y^-R*z(iZj~fyY9mU~&8)hL>E{?h0-j=2hb-R{cMUkh?-hlmN7=8y*W(Z1$2y zAlx-h{OWF;MKa_-ybV>M$1xjeAe)OsI%>vSi%}<;Bkg*?3}EwRJ5WY?^MzTdf-$#fD&<(E~HkSE#r#LYhA3yl9sfk6y{v9gRq0Et)`k9I-mU!eDY{!V>gzT z=RJ{|k!qwwWvDxakl-D4E-tPs6*aY0s#mO?g98K4USMJtc0sEPc-b{r|78}vNwSU` zZ&$2RO#Q*ji5=Wx`D1{&6D9cY1>xf2I9y>pl1}yM5bQwgx?*5g9|G^^TqL~YQoI7K z+uFP#?HL(!Imlz39aZHTmUB zsi2;&S1{+UetUZhL*0z6x5DkKnV3~CH@-Z5jHZELeKp9EKzFl)mYhI;@43Vj7G=H4 zW&gQhIQj%A_Qdaja*UTew(VR=jw@dmIMh;5UvE9nqEo5y(;dn3fFillZiH?xIqqKe z;BfzAo+HZ+e&-985^(V4atxoCF1K4)#(QexYn2Xbb9cAS{jAACnMiU*E-_ssH|*1i z-oD59-8WS7f#8r~DJGa=t4d>4%3?#Vzi@Hf(e70&u*GN)6R#us=t;M_02CM zI@(@bM`xKX8Rsig9g@JG8HZbrNW7}!ZNp+k3lJvNd87??05h*CtF_gyTsQdPdAGB)U z6v|z>R{c?5u~K2dQjo@@n@7W6VZWaf=GfJO0#)4F$y?POk2g(vjI;l$>im&W5mbZk z>$KZ+YMRRXT3JjgmY#L)ZCq6r1$X;y*pI!5mzOyfRXL%GQkh~R0qoq@=w)SRk6(P6 zFfkJ+OpZ@(JnH}zltnO1E+PeWk|)oEZbXVppW8aMPMI_;A7wKnT79rc(NNqDD>KnD zl?URO9L76K06)hK66ShS@8jHn+=-e9=dUCkKFIQC%Gbx+J+w+27#J)^y&{*cKb&)Z zn|EFso*D@{EtSoTAYeBX&kF-T#W^9w;*!Js{CxW?IDp^Oi=EPN&-v~68A5nPPG4y* z+ia{Xb{A&x{1ISo-}@rHgVpe!#wt^4Q)n%_YVjnlgrjMP8*q}1R_lWa@Hf4MLck0L zLv72=(Vh6M=fmfn$uL}S11MORdE7<}o%zAVNu-5GdTgPyIkKXjstx~0d2lI`y*t?b zCie?KV0Fja2Alu&t5u)JN2!fvz|G)QC#7u4xhZ3KwY(xsmyHQ)?K4lO9Q@cV`luf!0{u4Wa|i z$N@$zp_= zdp5M=C(T`kORm4=H!(4h+Z&EsFEXKbbN3mFS?tH^Z{PVEC))3Nlo6D+ZC}1;X8rm8 zzv3iCF~N*+oYRKZo7d%lp1w1k7cc|oGAxrZn-&!oCVhdtwVaptqt^S(S%)l{KS3(_a1cVOC8|MYF+}MyH84ELN6E4+ci{9(aYACw3ja%ZhdzlU0_Z# z3oVjB9ee3@H|)A%g_JbDNyw;mA`6?;$T&fvGV5-cqiO-CQ^wi34F^ z+d>*1BA9+xVO9-b$p0Y++UpY(Chyx-uPYr!5HN{cTLS3YM6eCZz? zoPVvYUE_S(_Ize*dwWKFT;sd>6jX=1_G_yubUj`+0H{oUyyM>pZC)0a6f68-`eTqYG<8&Z?mecON(O4tgeqQIDY230_YfILf zatbG~m{2PYY3E>!|C}y&vESZ#!lxK8)R0e@c~7uH+YReViwZ?NmVw)0A#eZUo)0x)EVN#mHA>FHi>;84mD9ArGWySd%;evdRk5bv#i zguGciP#YPVImhJ=NYblT&1c%@zr5`K@c;fl_TDors%2>d6+}TnV1tTeketJiGa`}( zkeop!=bV#>fFL;yL(VWn$vKLWbIt-1hMdCy!(Hz1dCuAH^L>Br^V}boA3SSj*6P(= zUG-LVRdttG{XMSupfN~XNL|n zL16HQ_o_1Dt>iDa7Ll^^r(2gsjC3tCwVtl^O3v^TW}SnJ;|B&dLLBufdA(XpD4>>6 zzq__AXG_U}clW++AVOqws;w8@I|vFV`h*Uqvi8V@EjqCET8y_QmLWii`1bmG#1h9& zVF9hpT%#I`!GXK~@#WmG6xpegledxQg((xxY@_q6Qq*;q$G27#-UsA7+05nSE&7K` zBOhtfpC+eLE7PF}_W~fZ)P?jl@I1=4o^>VpZ92k0PE2aS@W%ZVA$?JK`^-}l^4`SF*VS27O@vmB&u3oa8ir)10?Cuq9eIc1G@t4bHt}a^imT5?C< z18P6UDnqFVGi;HO&hE2B$~LB@Rw~ktrw!1+8iaV!5!Wm1B>S`UmKE?D#D0Jg9IsTt|ry zEptbMqP|hj@tY}D!?`UdlF1APflQy@*8zYIho^21ooRZPTKnRV{N+RZcU}P8KoJA- z?k{&^$2bmOHL77;7YQGR*cM$nNR!u><^~-e$WPfyek+vMx!ZKQ*5J6}Z=8v9bb0%t zwCQB`jsR0I_MGN%d!feE9h>LjQGKjXCJ_Vv^(x*)8KPP|^CSz)?hcjx($3U+)py1w z-BWn}#>;n$ZVS5Xu?p_zXe^#D)jJzJ2rJw@F2P=KUp=Xb(7qZe6t(OYa>#r|kw7CZ zv@V3`7$uN%iO*iS0jz^d$?)JJ%)_IJp0jL)h1kR1$hO-Woj4Ai1A6x+A1(%Z1F3Fo ze(JsJ>&+8(Qv=+M^L}_5oW&$Ar;B;#eD?|YS zH5=wwM%No~<^^roQh*)L2KzYlN0 z8kyf+pUL_tW)~C|HteWTwNW1i=~#?p%S5TQz4L3TjF^#EZ8ti!sn{`xPPT19?m(r<)BY zUq8^_ORu$Ac?vz9*lbO76}>u-Y@(TOS#@dH6ee@6^Wi7SWkZHW2h%@P0c9@8RZgzuTkUY7nR167xQrkcwvaFq9G~<@$0@RF(B(AO z+%EU%7bx%Ri2f8Lrj1$25WW8nll6SwA4kMZ-_UTS;}Mg3vgbxdkO(!SjIF*o%hxY| z#sfG|Bo-xT0*q5TSG02WiV5`HUUzyd)BlRtB1{y7^l$um>JMa(>qfkzzyq@APwVE$ z9z}~0%}S=45gyWgw~2tw*{E=iY@6rgy^4Mnv|p*fX&ttc@vvGxz4p_q)%W)d7YVBd zMP?Zb3LI#%x{(Ifb6#HMS;n%(t=YjVY2#WV)NivD$~{8Q?cn7V#5A9!t~io-tcOvE zZ95^ZlgMpbvYX7~VuQ_Y)5rbOWfe{y$91|2ra%zJu`O)aIBKVq$2#1+u-7;wv-Q{< zUC67e^0r7s<>(fUsJ;l?KO$a_I*DVHEG|xCo61nOxtKBJ=4|zRl-ODFhZfUcqlil}F4*otskb7&~=_VE=r zGAMQ3T_`Ty3E1ESY%2=yJ9r`#JFIFVszp-d-pAmJw$0B#U3VD+uY)hczzO|6pG#Pb z{{)3d`t-dNKtl4~^z`%ys4VK>>nQydcluY{hzv(1B!pTVC)nd9FDYle2#bP)nIFaW z(dC*u$?b>wd?@mlb_#@shKAa`NOInq$e}7mTvaHJPn-5*p*L5aV)6!|^-G6>`!Ye- zxXkzxBfBZ@-d&BGrr94Z(wx(iIIF6vz7%vh?%Kgj^`tbMJtdWzapv1ezrOLFI`&t8 zzKK0$dpMK%uJIDUh*g8Ljd^b+csu02=trFR;YmtkCXHgP zOEqWbovDqjXn3+yD>x6bn&!))o~xFKw>bP3B{GPx0b${~pJt zvGO0C}6R&(utAp673Tw`bC@x)QUe{#lRu3v%-5 zc(7=dv-3&R{Qk`BH?5>ov`M>=%BolHt`0SMLS)^;n2(2gI%w0*8M%ozQOdz2Byx<7%62P*V8zqvoQPc zGSBXRuZV%rzS%1p2F_IK@A`m->tS6_^;WY5EaRn$)q@G z>CMeGR>*DXft<94A-$D#^d2F7c7u6M+NXy1q}~~44)VG$*BZN^fH=98jGaD#zbAN~ z66zOhd4)A6gpyPCI>_P%n7f8|mBBuh6c-QD6S9AcIWOB5b#~y>0ejUg;GFww9jyAa zh-NT1%j=xhSkLl*EtyvWVe43Fl=Lc`BPJi;Y?a;dxL=fD77y4@;gIsRw}b|UhRrtQ zP12F6dM<=CD4^)rPC{(z=t0puw5;!4+(a(o>;jrE$}4$I!-H+IYMfK_kE^j`hs8k$ zA7lBcUw+aYNMf?4cIl8t2+i#{%=UjaiG^sa^~B2G8|D6zR>*tmjjb^b(W3FSg3n=V z>;^w|d?ROFpQ)heIys}PyMfW7QCwvSLq_)K;Y?0U*qFf{IAb`eQozyQ>jiN-cU<3` zcZw>+wnU0*JdJ8md--yoAQ&U`K|_ULB}G_`{YIBf^xfY*1NbqG4yLH+z9bUKHvjYSb#HbiCbjnj(=+1buA81KaSZ@|WB8NG{&I)Tx5M7%ZzCp&vr zVpk-}?TMuvgSi1zMe8Wm@>fy9ihMm||o@cw~lx2%DeyT_j1{M`YIyTX( z>$&b;tI;JuXNS2w5QdV?9&YzB6J}gLwbw6O#69gtJcnW5*Q&7jyuTypg3VT%Po%D4 zAo?=*DdX|pJ)Yq1ZzH4x)NOqC(!L|bC@#q(gbr5X2iDIn99FijMST65$?l1iF)|gN zs7i&O^VsAu!Raco6Nh(laDqsPvEzfW-wrz;T-F!mNY>N{zUcYxv~JD}S6l5B{H!MXitVM>}O!r%47KByLW9b+^dc!~- zY3_J?>(Z7<<9A2h&@Aeqe+vw#ovzQ&f&q)%=A_&5G_#HPc=QXV#W-#hbHA`T*>53x z8BHl6FKQHs7@&c`-hx>&{f)isTR3|56Y$ta>o|HO-q5r_lz|^ub4O(#vuEAbXcy;0RLG<<2(I!wJ_{tA54zHy=fQX} zDaj`AGB==F;0fT5XqZ@!fmlB#oHuZB$+gzAw$|&bd^Nmo0ysTK&Zdff8ca? zcQ=#;=a#WcZao8FtO{1-oR0lzKP>{`4W&h@lP2Xw&`DE83!z=pPeJ~9RfdXKvNp-> z13$&_JjqPklvQeH(w)9@)7{gv`A&ZW)o+0y68Z(v@l>4T>$`P*KG%N4g9>fT_^69_ zsGQc?koluBB*}+_8)I#} z0#jdhb8f$$Dc^93N{3QkZ^f@aFOW1b#3&7Lgr7D)vbn7B@NK4*n|3wXR{S953zSC~W zyfON8ir72JXHY^y!J|WmYHVi0{Z+t<)Mz?H2dLk~b{04dvJ{^_!G|O#Z})U41DnPuNbd-cuxD|6 zNBPz+MdtpU)1BVUKuc@d9)ZsJ&Sd1E_x&G9FJ^^rvEzr&o?l0#&CSjA9ygU2caC8H zl~D0#UeAw@6xVke^-F=2Fvq@tK&R^Wkt&3_QUOITYgx082A^=k!X!{lo^f7(7-)>d zlm!1`6;!rF`|ZnyO(yhd~EG7`uD% z(8r5?aHlbOXysnJIXd_nTrR@>b<9@%ep%ELtMV4)_tniQg%(19EO~I9aTmIe$6Zt zeh-H~f{l%mXmT>zlDK-iCw3ZSGcPwbcUWd!Lo4B<0!vd_>k*{F4h`}t(Qm^t-ygo5 zG-^AjosS#2N#8&Mg-%X`7xucU!=3HxlE;D7EIY!jWf4N!L|#?wZ)tJdTts@A^BuP04ihvv`(f&<@3*0gGN(B&nYA4+0jZn6^ON*L zjJy5{_ntI;DgkQ8UftRqg&JeT%Q$7C{etaU^(m%m<*YBh>l8GLEgK{*qaMe67mkRC z>**d)3K~qvAi?7^tf}2(%+#K(O2_&^F+!E6i`Yw;-TSrH{uDh-jOfz-Qx6s7AxeA^RN&xhV-zT3<<@wRbXa>N{i7`U)*X{xKc zd_HRyJ=+=3zhbGGbGD-Rwx+8;RvkN!iiCOP2>rWwg>vpJOqXmh)#`pOS3F`L z52GL*Jo(+J!NYhe`JvweUgUa&v&*`E4YH6BT~Y8=K<8@6MHcc*RJ(dHzIZmGTrjuE znA19Ohd22uyq2$-4N5{!J49EHEOr&;1COLRZE9JUmu(zjkqQ{?RFgJfM~LKE5f9#+ zjoYSpPQcE9RQ@Ma^SeKs2~JB>*hQSYIESo12h70n-w7Y$modhIft^e~k=S~jt7%?jmq=e#n5b*sIIa9V(z?+!@P-+*{Op@vyERYWl35hJl?c@Q+C2zq`_aGiAI)Y7kam z_~MdwRA*%i$MT4I3bF1*^Sf-Cal6<>6~qL6wuN+OxoY_~o9B6WuF}Rah;xE)i1O|d ztilE-iI-XvMVz{^9{IW4la9BgY4NufG3nd*&A|JrJ7n#;>xI1aPS=f4)&6ps;frQb zvsc~WfO8*xXR6NfxY}CK>08&rhmTz-q?+icZOq;3AZCG;kX>3PP(@`)iI1BTgp)?O zyMTM^6w3En7d$~s+IsY*fIII+O_){h`{%(&Sdxq@6Ww1klSsY3`K0-tUTkvOX7SQS z=H*t~e2*Cvy4j(^5p)~win`n$v8G9HI~edJCuz3mM`(G7`b1gkk?HZI*P0o^cgy{O zYOFePHim{-9jESB!>fl27skP0nbqMoPR<;48>d^LHf(+@OShhw6E94(an>{AapB*K z9=1N2e5dmc%XtXwC8|sss{u{cKv#pu$k|0)D;uOe`5p^_^g3F5vdaD@E$7DygbAKS zo0|k1CGzW*_c6!fOdb?!lpT7Pf>w_$Ve3~LUN%JhUJYebTeC!=?~+v=%E>h$?8R1%_{QGP zz2Tm#h_Jvy19BzPg@n{M!&a*{hh2FJh0JxfNlS9KV@i2?v&_g^ofa=82AkN@{h2eR zIP$Zzfm&jYI^KGphc+CV^t6EEvf<^5oq+4f$iU~!k~b#sV1&t5|KQNl79IR~pWLl= z?1qHrI{i_=#l`H3zx4OB&NZgemg_9Vc;9N1)>ahDrjrNm4ZVzCC&wpo@knMca`d;jL6!<9=~z#Ce?MWSEfMW~}qDbKp!ibE4s@6AA- zzv1_32R^YEwp$SWV*xzotlP%N)w)U1t;u%Zz`%a>WHyTH?PyV_;<5Gks(H_RA|fI^ zz{SCVqJZ$qda;X-Pm5zei_;3iHtxNXAI6$$Mu1f3eod#{(GC| zOaimKoqw8%Nq|E;ch4NLkh^D((9hd*N9#RaYiO_g1zt8Yyy`iYRS06h@jd}+EmlPn zswzyT26Y7XeP6cg@_!@svYcMJ?&@CUXfbu+v(Y1SEq5aw2wb2>NZO@aRUW&$w)LHu zFKz;#&Gyz#-9TSx=*aeV5uk{;q`W#bx_tC)Y+&;CwbH2k$-t&YP&U25q%Q5`<76<( zb@UM$bt_G_zwWkvl;KQywV*x(E;;Zh*74$m-&n&mSw@puZm8>g%Y_fT=yxfvA2*Z$ z7(miGem0jd^MPdk=8r&Sw?^QPT)HxiIvJL93}`j|s@IJ>&6?7(Em49+ca{Twnf===JX(>vdaK912!)<&{MRuKAo74uT4pBMRq78R4WN_OP32xIIf@SL%!ut@H8w!q5bRXP8HHH~4oEu!)^lHYv!9AZ85B>Is_%lYAH zC=FsNrrG_I&**gfg+(0`Xgd=CcJ) zQF=Buw&2Rwe*_@K7%?|I4ct?gF(Rq#&kWgrQyD-4>cIO*xC<eOFqbp0$S1Qu?CeqpV_*VK_~@o*{hsh)TM91RP^Y-luxOA=5?0cVIiQ2 zoN{`ewYW$p^1g=&&)@S!7*U%OIEoDoIK>m+kL;o~+MH3n2gf;v-2HphVi)=i{mbhTU=svs4d5y%?fpap3(ZoGjP1&p1}#{s!_Duj4wAS3wm)x1YPta z3g5EvBgR!OsZ$o{knaE@Hj1WR&RZ_`>dc7x4iMs0soH9son)Cbx*kUbwX~Sb2QIBX ziu-b+Z>x~osiL8*t14i%F{C#=4ecjvZ*gAGI-5EfR@U$n)J866a{+KquvMKKT*9FFt>>`u^~+z;B$Jn+7`w6qypI#bI*X z#SMgHV{IpdKMu%_l4YHS(>RbdO+eo$h`>9(I6Nq|o^JHEGoRjVi&Go-zeb9^(Dxa@ zKA7RK+W52sB%+zAydZ_f!+3akP5s?MKPF3wL?kn{XoA*SZ|4>T$L1{8*H`mo)z`Z` zKb`^y!RLKjx(l;ZYWj zp|K4o=G_&(mu%krI!9Dk_3J8m_^)WVn(`WI?I3ia@^(5(oqm5XkLK{>O zy>Ua;btuFDO2wlJ11L2-ibG`VIni=+y*C|gy@u^h&+h^3aHi{(=r;Tq60#8A3U%e= z0}b6AE#8$L$E;%dFE$0D^0?YTsNcR#l?DBOMc$hHuql=JWZ^1 zGT-Zvc`DYlmJ%YDN@RIkj{m76H+R(g@6(?YQH`-+nCIS%PcUFFTl3Obc}=t6z8<<9 zK&D$@E9EHm`SwvJ&hDC-?Qw%Su&0-~~1cOU}PX2dL#2v-v0`_CL|))CmZF>jov#|lg-yDdtRpIrX}L3VI8$Dm8C&s9%lL! zxMqn?pA4J2Lb=CD1*5ZY&GgbcZ<{=ou<+QgmCiM<*UUAmR}i76lOF2h1u|#D8R+TV ze9!ln$bl5vz0gY2KQ(~sy{;zHp?DUZTn%+~GFs7BrW;3H2Dgg3`}!J{at%B_H4MnL z{Dh`|xia(Hp7Y|~$}Ch{;!3X&`#- z#qG8s9apmqGG=y*cuUxv44j25b;rphts9@NvC(NdZ!siU6IHbj2f`w1HNhM1DFcyt zCHFG*PS9?SGT%#tq{JxsKpKaLqoxAsxo8;g_&sm#EwKubg7)A2SeDIByU6jXZZ zy_O#vH?xIknL=%-)@NzNiB8H%u|GwvBtZZ*!LNXHtInIr zDc+Jh*2q-K8b4b)Zfwr_u`vjgfNd3=fIUH^u|{7q8Dj+EH@Ht z9rXmb@KC%u7Zs&N=#h0VFh%Hs`6{CA@z2doE%W z=fuB|@QN`|?}o6}u1zLKL)_*Yvm$ZMq|(QHGqpKrz+r?^lXlr!r@ zvGJF4RbtbD6v`J!l8jtbxMLEhjX2dRDpZmj7`Nn2gjVdMtHAsdJzE{sMs(Uz!P%PC zRx-J}{tsn>=`h~bMu?nw%JbZLOM&u%dTImNRFLQDz1OsNDg1Jy{qpQgkGNR3jIn3k zL5abM3jZr%@Lk6*`9fq1tU*Z-x@?nn%;o$XZeo^sHyNja9Eg;sRAvvqX?b{w{M%c8 zaI^*Q{9Le!El^Q0zYM5;eax}Zsp^^pIFgX40+8>;1L<@2@MW1NF+)K_UAUFWRJmI zUETsGRtw*>%j3Jg_rzY@U)Ul~tsN9VK@Fp_o45^9+stC4-NPAWp&UQcp}cT4yh1(t z*5<$N@GT6z*5{YWqePLZtTFrXE>bW;g7=6Oq5?I8QHWnBG>o=D(=kq?ni+T85I0&-FGqFXU=ZznW5F?4c zxYFw%?%%TxxgQ*FHk@gM(HonkR_K|DGk?57Km9O58c#7ezqpvjW3r(Hb5E*(GLesS z6z>s)OY+N5O?w=KCEP{$V%Ut?DR)25V214alC9C{yXp-M&`TkV9SlAXv?GUSUioIPKh4#%oFFV-|*AE^%Qg6&0l&S2A7y{j^_((d?4of z1vdfSZ^Rk&Z10xd(&~D~K3=70s68*)X;^2k8+PZ{mGW`VKdFe1(bUcruy|*BX#R~c zc1BMR=dtX>Z_hZG+RNfzFNW+38fxjfyup+h@us+9b)&r+_~03H(X&&t*lfCNzx4}d zTT`KUg!G8(es3@H44Zb>o|epw0g%pps(?>Vgw*4n!=88~6JoB?-ZhCwGNO%?-7m(vXFgWaeHt*_v zp0-`QWW4cUDF7ZoDOSPEtkL1P5wJEAysk|L$c*3DhZrc`4j5mL-?x4*e0>o?$$Yfa zu-AMU{B>-|0+7VN1nQ3qKkY~KorF+!jdsr0kvZ{_>n|*M7YS*iDP>fkc(2cM*yof_ zuW-Zpi4yM7#o|x1yOId+ZE!PA@_QuS0pxm3>)U7=VoIa)tHbz>yCh7t+~fs@dnXNn zZOg0CHD3*yZj;e~Ary(`dTW`tVutsbKeYCZNjV+|td~Jjfy8lN&kf0Mo4A>EobOFp z9~|9z-300@ejR?vY;~_Q=`)g@$czhC;|H6p1^^HRm7DD)mc&t5fLNbqKHLnWz2ita z!LuDX*?#p#^G1sg>GWjHHh)bg>)n5j66on0KfY}|kB@LockjxBcM+;QbnV|UUvKBg zsPJVZV^)Np(3MS5h0&3R@8?W!kzDzAhKWDWJ^I(9;sSy`(GOW&#$=oA6xrS?bIdzxrN|6 z&-OT1hrWPWH9|Zt!<~3X7ZN{=z&nO1lN7uPsQzbkjs&;~&;}^zKj!qWo&XffV~)h2 zgg$@TVbrb*ZO?qD9#ZFKpve|=)8`;d9q};vny{kr)0;mV^b>f5n*bX{FVy;AggE4B|my1 zrsqqEo#C34f8jY_@Y~it89S1ve|_sY-;<{39@#vY^I z&Oif3Nbm5smmE&w_>wTAr`jXKs&HqM`g)Sxkj&2JLbX<3DWr-MC^}c+Y*%vsrV#n)-+Q*I{Dx_09JY{>Rk)?314g z9{t0Q!Z3K}#GCP7h?*AzPuUvqsuW(cj`8oK1k&Pb&vU`DcKfvpFvyd$<%Cdp} zuU}h_QSmvuMhQMa{_)9w;a)LefE;6^qhpW%vFksZ@guy0W@19E{O{JyzcR_smeW(w z&`dVr<`$TLb?j$5fBU926ct7&NB{Hb|9Hb#Q|kh+z5yXGr^biOCiJc5IwI+Bv1k_8 zi`_$FzN?Ez4BR8#zcWqxz@1}um317=qhELH-y5dI+<0wL9>nqwV+R7PTRc{qA;hWo zY_ZjyCdkC+vZNMY?u^TY5viod-=vUi24|jrA&v7L47+fuO#FR>eF|bY|9#?q4l`Zh z2a&#uTipE{N&Zf))@|TsllAO}|A%|UQZRsN?e-tT`iG(Y>;oWn(*O_j4ahM3U+xw7 z4ZztLbl5-3=)ZKY0v`bRlo;iIl;?lB*ZC<0y9ynS@_*3?KbtY3qhJRK(&hYLCVBI@ z*tg@Lz+1Q2O=X^otG>!rAj>xS3qSu`1HeqaGI3Fm{U0g+FAt3YngN_5B;EfPBK((o zaT8JiHL{%?`_F3R=hON405A)>!SrvCpK`49B?{|qB5S1{%N@8HZ%;Kzm>FY%q=Rn--SIGQke zu5in>oFNP2tbB)H+8B{m9Clwt&xWv-Uo=!icSWnu8J;;Am^K) z#U`t*L4%gV0NAB9z9t4B%9u+GFsgsS)1P4Z6DpYb{E!*#J;=5z#HiLm+tRZwc&+}P zn9cYS@+!L(%n79jyY9THOQw(O>0HIc3pRRFe^~d7L5n#)g7_epW|nK`B(DM3uQO2D zYA-ruiU=%G{9ln~th5zEHs#&9du=$WVD*|OX8SUt#eLW}8Su;aRWMMnJMx^&C;eUB z{r=6^hd&g*1`QCRYC3xrrYg&gF_te@d?QFmv{8EHS)wE|{s)Y5Qv&(}KSG@5=Wu_b z=nqu(gQCS();#AmaI{BulcEP^K6ikFR|h8!9AFQta%6{{32cRQ)a(rYl3qW;ANux| zL&87wjq6=&{ez_XWfzn7B>zV#Rg=rLQV|luN@H^q3&a`~6^`!5WjdmIj*oVB;Ni`e zlQ_@2pFb?tQF9#s7}%MaU)pk-=D1yxd-qaWi{5AFqafo=s^uQt%PtaPojY~b(M|QR}s=Z!sR!tP!V7I7&(H`dl%B0=P#^U#7=C5Y` zPcmSteoK^l+*I$Qw$=I!hKxQPn~FY~mDnD6p6>I*3|ZH*xc9&C4_B>N$xGSha!a@I z!iJ#9y*)yMMbQL<>0}vf7K6m;vo=cs-+Fub-ek`J{-G30-=&i_$J`P9YV+aL@m`R` z0kS#c*-@7!C21)^h>|g3sYadyzwGW7nU3nbem%YLOtkfV?=E@Krbb7*FQ8H^-NUbb z2bk^7kWhl2ot!sy3^5AfMxgFCwO-B!R)Nx;xHE$?Cn-)Sv!ZWY&FSuxV7b-g!m!LW zq{<|wZmU!i;ZZfVBDyty)gIm^haUK`cLft!XfSh3Fm4!s0$30#zJsbC|AAe^KHoRu zgy%a>mz#YFSyfU|8Kn-2Ut2SqpW%+HE3TrE%~4je)UhGLpfG-#GzTL!&B!WAo9TCZfof*Xj4l(aB5#R;GI_X+yOeBWvjG~&MJ?}_{}Bz;ip zhjQi|VSv0kZHs8!+Z?zQJ@JcE9Lz}+^{l#?3-X=2mQxvIi0?7bYx24{IzSdL&Rz~r zo`vd)R)L)bA#5zj_9j8!mip@qOTikSt81njwVJ|SUYu}OuOswJ8~3AcQ{%4N&28;oo)JKBHTxfk&eumuO#3)KD#*ZufuNDNift$cmCSpWJRC@+p zvrBn1MWD~9P)>|<)?-o;w_qId#b|X7)-(hb$hgasIMT1Q{*@_nDI+7_b{3QSC6(Im zU^B|FzmGo;@xGC=$6;Sx@i5{?imZ0v)SP$=kOuMyAg88h@z0o#s`hp+$&W4^W@qyy zmpcc~uAlMDa`8+qIPP9AEF=XOF9f=gCUaiIydmY1#{UOaz2)~6PdXq263?Qi4eVEv zldE*;-!xkZJc%3XeIA@^#A$&kp6G<}`T42?#^;Aw2Xk(~&ZhRYs8(>D?)|*jEVuJq z?m`X8053|$07LvdH~QSAj3{>9(N`Fw4}ldn`N{FI0<$t`YdpQL`~)J^tx8MeeGr@o zO9}yyB|Yj+^GR2m&Il}bPWOSxUYgQ@-&IYtG31i_ya((@y<1p=J zn%S`=1-h?+fgcOo_tv^sSjkw9v#f(ekr|fpqAmF>{P3A^OR!#e7=eKP48PmwvFmDL z39(nX9H|xRKU~$nz;D0$M;|=>wW_zJU*#58aif_`JYP>~;ONPwi{P=ReyCD(jD=s? z+0+vn7Eb0U6DTG>*8slC{glH`TvbyV937WxM%yAtLP!{l?d&QA9nz5(fqZYrwE7W{;*6Ta%~REH=OW{t z3Z&)c7Sk7Vk7J>V)@OjqZk;Ch;0d|t#=Itnme3MdnRy;h+AQwRZ4&Znx}CXP_9n=q`S_sAu1{3}F+t9rhc#?Eil7MA4@_ z3saQ>YsCD&tj9FFQ0-rutX_x`=^6~Zm-Bm?<(HX`I^;>aQ`1vtYi&_q^qT5E<}EbvCv7)ee=fW+-$ZVn z#u2bA)Vy}?9Ub4<#NwYGK`+ri4D&?>JNn?3)M;p%(3Tsv)ifp|4vlr)1YI`EdX{U*!al)p?N7ah+pAfa*`kjn- z3fWL3IL9ZC#)$s1m;Muf(^1f{n#OM7XyM<(&A0SpV%t9V5zr$`vZNxY$#cLTgp+o$ zCDrQbwOFZKo^B$XdWUYyH*+U#6B4nMB<`Mj?FvrC7;wyAPcYkxu+JldFIlICW*$if z<6<$A6Y3mnv^6hHqnq%gjGi8LSZdpn088Jm?i1Rwzmj*LZ<{I>8QIQKXHE8dRK+=j1FN4-h~ForphTAdVY(cmOCJ;-2WR|2paK}`E?mXIr_>w*bC zl_t5xp<_QNcW;F^NY7JAr@rAaDo8k}GjHdCiJGa&SDp^xA)}GtHr?h%w>O?aQujyo z9p6AV5r;Hz!aWrhODzMyQa?EnstWuCC;^>R$Pz2lEb=-MM5DVBzUHb=-IX)0fNso% zZ81TBfRd7@r%vqPdZx1ZAw71EG6Uno`*HmG5w|(USI5}p8&j|hwK^Q1qS?_ZEUiD) zE_>r%X{O{^tIi=XH%ZX6KLWjWs88?fpnH|_!~!~6 z#X|2`PpuaOjOc_R5*(k0d50RCl+0Fd97-ypRoHp7x)wv4R%-ZBTV)9-pl4sR1!rQ` zZedI;BS;+zmsC-->88Fk!H*9f?B$`uR2i&kvb)o6+SIdtx=(2Hl_D0vL+=3#g<7XHmj%7BvlLAc{Rn@K zQTc;1E1t%W!F#{mSDnaXa=U+tW~_iR2m?gbjKYuR9WWg$KFhM=|@ICX@O_#G%vl41jE)psM+;%7KD1^*V#(_2-3a4>mtO zPGh&iLzllB7$Q|~TByIx%rfc3<0|jErHB54H}IMb>{?X62ZP+@Rh^_`6`UFUAs~cX zCk!ti^u3>W&_DJ}xUY};JKvxE{138o6}5JXoR%H1dBf!ba1*5iD1DS)Z;X#sYeL){OH+-l}iZ*V)8O41{AV+Bv;_5tH@lxF>g~aqvTor4I*yl&AtQb-xcB3rjh1!_8IQ z>|%;g&U~_Z6pSCJr!Z#NH#X6+T5vRGc;KRf0q;0Vo@(fKS~Z#BH4ukFHBt_K~C%O_v~4yU6*7GIN1AAmDo(m!MrmIxd)2hk?p0c>{DXaB^V*7x7H zf_*XiD#OCVS#2yWlMa#!Y>#*psQ$1^0O$Yg&Ctn(iutbZc^Kx6wOlm=Ut z+?-!D&*tjyO;6tSMSk?HvYdWIz&zt}P`BfGMna^Am?5TG7r7p3?f;@&=WcNUYDyV| z=%o1=znXWFkeqi}nQ6IRo-}BAW07xih8%YFURuzTTl5_-C)cfqIoo#N2l$$y^&+CJ%A)6*&%TESV%jP)LnOi_NU3BDBCarmkyvpmX zr`=E~*(;vfI0K93kDx03S0=QSoxjGR^o%>t_})m}suP*BL%wW8C}>WZQiH z{P=Km@Huop&`ffDdy^+E$AR+9DW5JRdP%&5f46GSYr+WO`5>rsVUj%XGT*)>oTm5m zYK!e^l1*Xpz9(R&@cOycW)hd`o$dsY2_DzQ1jx`<3!*886hch(=UmOM4udZhCp}Do z^Vfn)9^tXU_6r5n7$LNG40dbH$3?eA*ssDfqLvHrlau(gE~C4jFRg9G`!aF&@5|HG zrp`Olvx|u6U-Xd$Dtmk6u94oD+vkEveq=uU3I!fe<9Z~xGuZ4-O(nE~!$i*bZsDID zDnmaL*SZy)VuPIdA+%uOntDWbx&Re z)M!z6KyNOU=w?T7@RE&m2PW*u&&E1C7W?F;16}><;<7~dF?x#p14i6-PUyZnJ6SXg zSER6s9pAsT5Bx0r*D-4`3C#Dqb7^5H?VrG?AZ|nqZnHHRAmwUnxK<(eDZB83bP9w! zz`E)ZY~!mPv}$K+5d6WUS~7ARHY3KSzK*l7NyRg2KF4}Vy$FD6Cv5wvuB`JH#5dcG zH;(Y-<6ZnJo+Jfq?_r!&wZq_`vRk4uaFwL`_rhBgLf)p<#$uw+Up z;8n|pX7W{{lqcX5+1Cwz2^DU>(SH|v$H3jS@*g_IuKteCjmF?j#i)0GjGpavfz{0o zf@%F`dO^bQW16S6-DhGzC5NwuSnsgFkj_jMo;J0)fVl)#p^>&*@3$n6SdY;w_1KO- zD{>R*yD6v_PBJF(xwx_>ZUz=QS3VlHCB>!`iz4jFw$}HXrrxuIeMb3<^+xsR$lh!Y zBu!rLhvUTg93S`I-ZJp=-hY=dyc|8O3uD1rAA2}JYHn>qB;;znwd>kq*nLW~1FQKe zlf3&*>TLdKxa#J~ARWW<5GujO-d-uz#uHSKZC@?}5D!;U{s+UENd4Q7(z7?KY`%3h zFX@@Lc6apwU)dp5cWB@vSW{ppO1xe}L?eN@t>woNpq{)-esq4y7su%1!c#)l!96LzntmY~m7nIC?>K2)M4-z7%nI^tII2bBsVtMWb9Rg_?nNu@dsIDg_xdhVUT zpdhBSaft{&oPpfT|4LD9iesL`GY+xyiGo(J#0~YB!pa>}i*>F1KfT0ur+4n&z56)m zHLu2dyCb&^j{T+SG3i$(|I4U+neILzB^74FT3oT+dT>4ZQYVZc8CJZW!Sjb9)H?ke zhOoFz_#UuH{OStGJ~kP@lZJC*x~^5n^edP|!(gWH6WnV&8g~-#rg(3h_?diVP)kfr zWBl%KxXI>`=*7@K8?Hq4_V$iTrH~kmjF8v)fBL%icqp?zUb35#F^ZDQV55**V}wG& zrbSVy;Bi)7s*j7zLZD#MU_nlxk(V_XYk#(T6iquspo zkLTkv^PF>j*Y7#c@B4e)(lhgY-}i6$sMJdA=>_<)ZM0h4gMxyIJAklJcW9)fwM^MY zBlWJ6hHL&`eT%_k+kDidq+oGy()=F|O(PaL?rD~1YGKur#T4RJq6O)~?9&f7hdr9y z4@vEnjlD&*5^tJw?0ow4r1;>0N8TFQ@v}>fj$5g-4*z!J6x7K3zb6j%pa-Bv14=-3 znm1WcBWkF3MbOSPLSfR^icNv%x3_}?jB#BwT{*yJvx^~6QC-A|Se-{U%4UXjb;%B~ zH!IBgtqxn{*&?%K(r#=y#JpH|28BW;fuqn8E4YO>JlTel@SD}RecFD=4n zZ=m?)Bz;0||A4yqsNF)ynAHRGX0zF*n%bFwsd_Ys`SM|nl++(lQm#UEf6c>;obu0H z4OSFH-m#cgN`%hmcg*!*zp(oHStPvwG-Ws<)e&8Jv`jda|kCaTND(ToK%X(A3 z9hB*4LCTwnV`%C~qxtggjF!F6H#$Z9R=n?lr)4RLv_z~Lr^nCsBIJ*^9@d}QoC`NN zEBT$TLC}Shx$V+62n6C93}UL%girG@g+~%ohFrXc6bM6}24=S4rAtM2%Ddx4m}5~8yiVU@RUUz}6>^>-^=FnUFE?hKz-MJ; zp@v%?mX~8aic-7D6sM-7ucMwdgTUP!l-e>mSic%BB0`+`tJYLfb|`7-Y<9k z=M`Oa-dT|5%v<&OD_-v{z^4lf3vrTMmL}i00Zmz&TB{yrGo2J?uAaUE`MRc|J^}a; zR^5ksh=@q2(Mm!XJFwT;uEseNu^j#)?DL)#U%hjT$#gEgI8ZSt602HlUpfTd0DZZ+ z1)Ya~R*AWVS-*mH3NpScjL^~H9ETHR_JUl1>n(C-`2>hE`9`57Cs17X{$(NE=vhTY z1*@J`V&!CwCoBa!Ol?pI6L`QMlo-qTq0y^}Rme5NEL_4G*#^aS$$y?MsKQk&0d(9J z^AjgDEEIR{Q_z}2<(<{R6Kp#$8ebO#qTKjMCO2G*r6#`J(&hSi4v3{uPx$7e7cOww z*bs8RxzfJer$cTqjLi%=J#tk*JMYe@6Mx|KO7-!-2B&qZC$h|Gf#C4yTPP` z@ojBwR9zjNh4{F*tPd-5RO$45I_|qoJP4Dafp^kK4PEHAW8K*DPF-CcC))@Y&5lC} zQ!*gm_wpyY&Js2Ok6#2W!NPvOdv|zlGf_OstJ)V7(U0mP{<~hvrv#`*>`7*%3G$j` z)OalLztHyx=fgYG^qp@NeLRZ#w@knm8{7>yGt`8GJz7fdEA?omf#?|{C1u8sfVHPs^L zJ!lF^FDkLhcqx&|e-{7yp>L{dYI-LJDp@^M3kqpm^i=gkQFhkpZFoXw{uW&0a4qQU{K>2!`#k4O*8}swq%P8SNpe40j zeQZm{yf~haIIDN^pL71C1;&1;iT2T>M~;N41W0GICMIg8`n|L3u3fUT`MZp9O{>Pk z+i%9jRYbj6D&D<4QmNnX{Ro4>I7*F(YM>Bo0#9vVDLxq=amz8APLmC_;@tx00r1ve1NhKeKT zX(_p#Izs}|YoE2-SLArnn z*7uj>CocR6vA##4>~nxlU1!L(lF97_-A;nYpu5cjoY8=$mUi~bsf}R%FAW?(<*)?2 zt{xL&i5kHw?$WEO8R<1bUW`9qVO?d$lGjx@03y+(jD%O6UbO*R!uk37E_Hz&%JOIp zO+b52eJbp0YXFOp7cv0-9|~%!tGQl&ev`CAPhaF%%GIpQw#dQF66HicuN{9~d@Y}f zMY%0ELDT<}c~Wr}`QI0-oT(ZXH^{r75NO--b?E=qwp_GAdRUE!fuZs=PbqlxF?PEL&9e$UJ?Tpqprs1Boc1csI_E_4{QvQ2M)q(4Xh@55+(Q zAI-#~v3_oDZ#2o!QwJNp^lOb5pEd1qU8w`16KTvB!S4GFS%x8pSuY<>>;ZAAVCXPl zmV~u;bX;z}SV`*%su)nz!c=7c%;slPzTpE8z{^i37h^ro9KOvN0u6?|$+~FUXBVz0 z&LigPgf(lYpZQnqSfj-KJz|527J7BWThMirJH zcgsDW6>*(sTwh**(H4qvoR_7grK>Q;LHFOR_G8?-@1KhYt;KvDg*uYtd59Kz?;IY= zbt(5~juV|jsm+bGck4KtEr3LnlWM%Dzkbb)p+=j)$lB zzQ-+2dLLsu6?S3V4F$dS|76e0@?VH@4f#yBn~HjdgoLaH3bn0xmD)S)wNnr99_;P! zU$P#}z_e+Da016917AX zaZHMDb$q7<`gMZ|HdbEU1%Tn8=I9lYD@;xf7t~i25R?tH%VaXCK>-1PqH^(NDc~J) zuicAtG5(ZJ2+DcwAUqiZO-5g=0idGpk z>(-GC(g|KgKtZoeRnUu@2W(4S!5Bk-Os3zii*B@xk+n=w>xW(c{n|c3-=R&Xu7O8k z{FUVHTFa1HYGco}(A!PbFT!MHY2qf@t=$iZdE^<8iOtC+mae3^wxg{05Tid~j-ZLl@KoE<|SdBP^o%I+ULJnTT!2C#>wf$Tk+q`q-;}kj;}hNEb0iGBV`* zMKzM*RKLGF#?%__te`lTSc28VURr4ZkXLOHCUQW>-`%5d1CQS z+Wsg)Z7>903Mxd_RTnjU<|N&yZ5M$ScHNnByYb2oi|_qW_gpOPD* z0bXz!-=nOU_^ebbKFWq-pJqO!AK?Lsp#EA6o@hom8ttmc7BPJK_Vg+fHSz1}F_RRG V-=FuQGqA#vcW<3<9TZ>WjCc3KvI3X@wgwike0%^# zadRstV+UYrrSD`cYHVn0WDH=GF}5*vG6OI(v2k+&1O#9mog9qytzq3jd$bp;$T%Im zPO3{J2K=HM%THqJ$gnNv+b!7r2r(b@`ZcQ~Q?)IPyV_1d`;D~KD{L?aSP=-MxQn@F zMoNbJ;cEJIn@*Z^o4j4;S8bMykKMQ~DSX-7SG~L6Ki@De$GnU`eS5f1UqxJ&Z`(_L zIIhobd|E7gD1309J5bZTT0Syzz1>s3ktecqZ(F@t4#{pLJ*hcq?R-JnSzJ7GJHGe4 zSy#Tf3PB1v9{0X=yy>a;z5V)d3hY#1*D|X9(DWu_*}Sb`h^UFt=cQVABjMpV;;q;B z2j!b}-NxJKkPG9hPfvSCP7sJ^^$(lJ5v#rn+A+3s#}U4dUmv82PMg+0K4PtUKNu%J z_CGfe2{=F*71iucx#>}KL3yJI*dw=gk-gO7Q9_ZOA~6e`++=pVy^FPcaap-3o^sGx z#(KGqDEjiS@z#rH{e|dh?ClsOyN}Ao$K#Otp)DUuzov!F9;eoagIiEiY401~BK6fr zzdn~<-OT5k5Zj!*j|^Oq4_%0?-A1dU(~B2RM?P)3SA8+dNRC%$AuS;r2D{{sue9i4GsUM6EZ5~1JnkJeN6^3}TR*H{*WV<8^DD97ON;g}?f5^vK z)l0CQySQp-%LeJ%@;G{!k`>%2-8A_;(^c5F?793z66V_Gv~wWqdZK;*?q;X2`SXWU zFu%3>p1JO^Ro`PD2(|9j<7r*_9dKs;Yd4B|xPq=LhsI;ZM;A^n?#BlhT{K-pR7$Ts z8&@Ch$BDpz6OKDVrO#(m51Z?5M`N2eyW4}k8q3#J=F8?|L^9F$-_ANV7OeY1 zWZrpQo(3Y3t=BXWfC->xZGh*8ad~%}s`?bwt;z7fw+qKx?FPEB3<^K zeZLa&rAmF)iiwU1)W#54M$aa|Z+CoJSLQ5ATW6ihUUqDQ^J37e!tnYT2QF}{lONpV z%`Gse+#V9Y?#Rt6KrwtqoXsBvBmR`S{*kh--L(EQqrjEvEm<2lS7_Ry4768&B=_x_ zC;h4Y^LIZ$lrI&IfOs^M0NVDgo>SylJRav-%}*k}E6hg`N)}`%V$hC!E|Jyi(Q#gB z@!{q)sR=3AA5Sb{puJGF3c1iEvhXq9Wg;26pb{6V;8IO!ADNk33A7K<@4_U)ae>&2_v%wL?N);Pc;5x_DB(L{>9OiaIcBk{-e-Fh^o{ zr%B`+^`Bb6=DQYI$-zTT2GpeHBYLUyFYFNiY|G&LxvI#5xiNz)K=ac|E^^eYOadc= zfcF`7lw*t$E@7t_BIG0^bMKz$Y+`ntC$=9wUwG8Ssx~Wqt9RBkP!JPO@$399x&tdr zHh@JNwqZ8%dtUJp;#9&+04UBWd-bf|&$CU{*4b{7NZ7mkTIu;a=4+TV zL+ngN5!Og59F~SgOTP)a?D`D4 z_{s1>nmVL7P6k2vKm(pn9Tq`IJag>b^hCPG`%m%4_m`Hb+`dC!Ln&sXYb*~)jEu{* z5=w*OxE6tL%S38Aa#+mO+?E8?Ktf6 z07V?5>O7IIo=n>qm9&WS@dk|z7PM&zq%glfyK1uFJ6=Eg>H=zjVPJ8?BZVWsfrq2C zA)6yO&QOcd5S#cr&e%tjV@P;Xa`w*9cw!OG*v=BF7WW*9iSTNQPyUL*=QbrJ+CLU6vUCCJW|0 z9eS=2d~o@Z@DDHEnQm1lyMPlp8xDWQhqB`k>eh~}>*RVkP3jX5Yzb^^V!n)9vNLSC z@~-hB*S>Mnt}0y2RfOy=)0mgswlZF#F6Fl&G3oF*aBtzkW$1mjbgy_n-yFLju9De9 zH6AW9CK>8 z6NJohR_(T*1&gDxjt|KEF9SdQsS4~xd&zy zATB{rg{5~Hb}Pr49aexAmM2f;$Yg@uT~;h8nimhz5X-Fge{luDa6G&$7sKZj7?IM& z(wO18{#@KRH1X~-G>#%&GMrOfDiUpE{{X4SrTyq{{_@<)z67!;^LOYs$J^y^*DSK21Q`6c^v93=R(8 zndTShgCT`^^Yc77PLq0lI_{)=EJKa6QCE-8r8(r*@yA+FgHx`Aq6z`wvq}uC2A&*M z(a~~|i>o+w!$AAuj=uF4$b_K*LJ6r&eqWT}!Xe3am5_zpyC6wLOJpWg>hMsA(8RO2 zSZPJ&)(BdOL0Vh*oT~sq1f`Gb0|OW&Te8EVgH0)6i5eNPWz4O|$qogQEeI9z@E9{O zb-gkhfNrr~g)))wF-33D*nx6(=YEYSYomVrU`=!LAGplaLE%hae@Oat7qAU5313Yx z2N?CQ?bB*P3)^tbI?w(&1jNheN*DR1~ zI^=VYzb>N2U#0O%b|+U-j=<&%n#5)mI=CX~+SmffP!Lg+q;#>}SzJ0lwSZzZN|FFsu~GbrD)GCx zqEi7EAWEfoYROLpU0~tx2i+RMsl|qwN01PO4GX~8$Sy4JZ5~*7G&&TYh=p4ka*@2_ zzQSZLr$9Y6^$5dc>0}9D39h~i$JK+n zSNu-!#9jtM43JufgK{D18cQSZ3d31a_FiZ@3Eq4W{8&v$n7f`!*v!d8T;d)oeVu-~ zacXfGodF}WwIrEPV$y9tHKXn{ZPEg4auw1ZKzp2l*_xyfyrg|QZS3niVkZhw=-TO| zVyionaT5rH%=f=PA1xOT42XxbIn_aL!__r3>NXGk#7>jk zn8j52(#{?-4)gWLqQiV~lPcy7ubyd6Bt^TNsW>;s$!h6U5ih#!@wN)g=l%@$KlZ(kY`O6stWp98HR4h2*) zC}!~4Vfl_D7S z4XT&!D$*5W`0)eV+MCufleylTJ)LiS#)6^6fr-qGu`CXgepsi8CUc<^&efU+XvlTv zGzG)=fr+UZkj?G)Y)zkL)rNM#It`sPvtO&@a#DR$+cVY9$5e_ka9S?sI7de&j-Zrx z$CS@fYA6LQ6l;MMpF7+j(Unb9XgQ~u3tpS_&Q{CbduYBN2CXg;V3h{c*M_31V3bu( z)qGlW_P|8Orw#?BMs$0ll)F6xR=Us%8l(uePYbTpH43`ykMs3F-hSHq&aZa!hVWeTrd8-6T@Z{M>#3C_^+Ji@^jW_NoSZ3!AM~#3?@yfkj0w^^-u78r%6BVEbIBdTZ)FtAK zvr}W8f&`HT4Ai{;fP~#dDyWVerkZig>ORA8EO&eKrJjL&Y6jtkH-h>y}$}cHp$L=Qn5~#OeKl- z{=aA^6eYhFo^ut&9#Z+)V!m@D=Xv}r$mY%)a~E1kS5`! z&hIz9iPm@_k?vpFdOgNBYdR+1-A1ZtflM<>-ROW44*`kt5inHgkO4#gsN_@Eua$^0$-m4WESBV%lpkkEMGj<5UT%c>lkt0_8q16Q)zgwQt%wiVhvpmp=4uwiLFR@Z_ED&{ z=$t>marEmxe_N=GJ?kaW@(sSXRaccdr?4S`q*THR6mG2mo4^`m&sh+57qGQ(sr>YQ z{+g%0HGsv>qfoYZe&ArXPu00z;7VHUC|~7rpJ*Dl#uB*jlkz-3rTL7tXtDyR!XcIF zK*+Q1A1md-;W*XlVtrH@UncLp?viJBhT3WcZg}ux_nb93bj@5U!|kR^_dkJ^!V7bq zCgKx{&QaS^ulY^hR+%WWs>{G-ZnSVP-S4_~G9b+Z1Qi1q6Va$mokIO5!cHnu9O!ZbPosD=}^vC)+B6XKmXZMRv9Cha)PF0;w?N_ z83V4P@13DdyNY%E-rBo&S9lOxThjd?A;(MT0L^qRe8RnYzdu1x9h0%3;z~-={pSE^ zwVC(EVTymPp3`0HU@)N=j82tN;@Xzlh;%9BClrP#)3zBkN&thK;t<3=zg-24k|lMkRBcth*#G~v$<$ornvVSL<8s&SNx z)zPRWEIJ3pavF)_%Ef%rYkq2?Fd5qARO>%FZ?q*=UGZ>{z7%n55U_s`S__z0>iXShB;!64P_s0W ziix`xh~6=(+m8cn3k)u#`Yw45R@&yJP;{K-3i(I|H@07z>YMN&R>wgs7+aOHk&{Gt z0UhwWo)2l14Q6wAHG8ZuFFW)3MnKcn@^ zHmMJo&bxu+=^9aIZf?uz3d&_;IdD2aLsZ$M&?cA_$y9xu z8H}grT2^tf?~&!^_8qcPcbN@tON^~jqvb*|M8mgR(H*cXCea4 zxmEk)JC5(MqmEE4FyA9A0=Y}XdjM{#8AItosM zAn5y}lUxetSy%Vo@6LvBDHbnuNZj$!WAN8(h>n>w+CPtYmQjv=LREjF63*AVD zLL({?z&oT;l$3>iW;NjMOj~?o!|xsB;oj}Pq(rgB*Fm(E-AeM2l&Jj@G=5hQvbdE- zt4A)LEEux2KN~e`vxX1}Cu2gukUG;9U2DQAj@vN#-AY*>!Y6EN-s)(=A*MVd(=Yl1 zkf-7Me#%0j_9b%WXI6ZAqNbKB3K%@&W=Fzw!6B+NtmVb11k-dORDGr{x zueff5OYD%kKVdQp2}=k9f`bGC;5Oadr{zi~#xY^<>9?*7tU(}pbUllg&Jd1_v5$;1 zvenob;^{&{Y0j*gu82^R90!jR z#-gC1|57rqGzM~*4F=TECG?ps5yEKQf24*kU0X9rl@M#5r-iuWIO$+fd?m}P9$6;M zk=5 zrkX(G7ZM6f#CLprt9;vRH6m5Go~o!2>~2Vbbh=k$j%!A_H)0n7f?U!&EfI191JIVG z8s~+YOH+dsg(=2OZ$Pa5@IDtulvEf^% zfuN}(Dbye~PD0ow!n(L;#{Mx>?E)a(__=dj^CG;qgsFD6a{K%9wd)u}e7LWV4{<+E z5GkROkyI<>^wxveNtU@w#f=R3p;Cbx+cX~yjbJ#(mLdwmh=Y&U$k1p$38JKg7>t~x zM;%K6R}&WvjaWGU*BPEdNL?z_{xE`e9R(Ap&#&59xR|R+m8ZvZgd6`DK7@{eH6A_0 za>%n&&mKhoE0^D-8qLbN#Ty}vEC_@t-3W*}NNKlQW>G&9!9)z|bxcr_76W>RJR|(k zaxhxbQ90ZU{)(v7%54mLH*=FAJ?bh8dBLAUL@vB}>aHg{Rh8CUt4c6a_hB zs^IFP$?%zVq(QcO=7k&Yf+eES^C>05^MK3@J6Qxt&LHdHMIeuE1 z90+e-mfRGCl^2a3i;J(PJuFPWPc-GhAT3-F<9UydMW?z7OF}jI?EpxDbNERUL;=%U zsvLqxHreG@=h!bw(b0Nu3* zPsW*~wJgo@snc%E(!szio(r~e!yH=MDx1S{(&1GB?iaDDz!g%pR@0wxv`}mjTAU08 z)nZj}w`d6R(2zFED3Yrp)zTVWXveG?mmaN}Usfvh%5_xV2TwU;)Xy+@Nw<}2+`{TN zP_eeHOVvcEwF;YY`n!KymtrD9^)_zHyE-`vkla9Ym={r0toIKfzzlhz57DPP^VR7Z zb(CbrSR`Aslnw!q+)6gyqGi+f6+Vj5DzrfJO2k(V zL5Z*onc8`GJVBdJU_w8_Iaiun$-YD>t!A8Aja2O9K=KTLB@qtw-A$nnXz_>FSca*v z=c7TJI+1Ke$`xcxaIawiQ_Zc8S^+Mohc5vA{ZGXQ?S|iN=!8oJuC)LkA% zv+T)P>K6V4pC`(Jl$Ju-4SxdEJ7@KnC{_nfRp5)Gd+;pOIHJvIe!DH{Dz(-g36OTf z#(ul*gc-9WnZ0?14%@BMd6{4Jekm*^dc$iFN;#v0LC_tHFD~Y012r<+nzh`ppOu?S z#rTSP;G^P`y4=5k?|sB3lP3Kg9yr_q{3%4NmV7oS|C2F zLytnUk(LZK8;9$T$ARM!{(XXRQ;HTz-K^l;PqZAyOt6tJcHN6JkvSjIZiImWiBH^u z%B$yD_GWOV6opzc!Po&DtSXD#c9wydWhvM!ya1FnGoZAO5u-w21VK_PqAiW!GN;E= zt}%IwvZwrKFt}U4zc5XR1{@W+U87FiU=W#Je(}`whDufM%$t;wM5+j^JDxE&F&RO4 zX0+~1=B(h1F9cP9oe!Wr2W{}yYPsI(01U^vdvX z?D*p{BAgu573RBO&nmq02u)O(UpY)VCwxPRZ0t=Z^5nzhubG^XE00&58ftD}zFn8( znlCZ$+3nQvH>5{NUec`7>OL&5>g` z)1DOqsE}Gqj*%olP+YfyeLtO@dgxHKEj36|uE@+hNFj~pUH6|?xa_-v&3f{P)1xoH z-*RGT6S+04u|#_YU}PhAT%1xad=5>AAQ=aTFoXBKr6Zt5q#_R}6-nWs>s(-L8}T{; zN!RY~DwNBt%yi78f%csif)fjvE&n*{Bu!9Tr(4Ej2G_K1*-11~g`#v=k&U$g8yjy= z+zG10!VOe(rvkadXvwFGlg4#l1>E`fUi&0}d?@F{)O{cE-b|Ph5vPN)a>fCCW(`TkNXBLDNSZP#*+8{XB*n_~dgRE3Z zYWuzPDut7X3m5b)l&6%zloaBC-Y4(7z{E~|5$W|TV&hD5e1)5R$u~;YJnss!FRWao zVKo-$0B2U8bjCWtrB$1HX4Ix_q<&HxhfjuQ{Gd-|cry|FgI)^v?HWaVgOKO|aVH}x z%!}uo4L^bA{09idynT?g6x9c=a;?xD?Y-$B#R8#js;S>?D-rtQHWVq=b<-iTUdiQW*Vd6AMN)6<_c9M@~IhKUhM!~)1 z*=;otrZ#l7{lsbS{T+7GT79zyE2~L`+9r!A=BiXi@mrO+!#=g;Dj~PXow@V8JN-VS z!VomwA5p%AzibJtOUi!E!IbXu2^!iDl){A1yfj1fBzeuxi6;4sjnmF`NdxsVC&+iR zMX%%f;_|5>NV=4R5HhqP}tr(D%#848!xs#Dco^AtxR89%w5b2 z#?2e|kYU_M*pxo0oAiN4qj!M-EwYZTi_gvTDw^NmB}X#)l?$-Zy5AYRQ%g=F-#g8K zk~BUVEa-B?r&fw56=Mdxp-+{-mKQYEl_XVjj*FH?kXL}@nISD)tY7J}RT}Ji$w+zP|F36XZ8OK3Sa!kxA z>UsVks%~9(Qu=~%4r>@Jxi_V7O?+MjY*x;~c=F9w78!2*x-k#D22J%t7}Y}SC1p>G zINAu5(`*gkym>6UFd0cJNUGGS@nAkWUbuH3ggHf@yWC~2GtXvDYxr3^T)XULp5eP8if||( zptyW>R`1+ILmNK5i#$u~hP3HAs z6p6MX)XX4dM#Wz^=gXIZkq<{f{h(oVGzuLqn1%+g{n^`Vr*AE)y zvFA~Uii?3k78oIi1-LPHh@HqxuMfNhr|OtyCJrdA-@Xdfr>qtah&(goL-1dG=dO+E zt2wqy-vQ5TaQL>&%!53Csb4Vifgn{lx@Zy__ZWoQ44^&-&bVb*v$oUeKv#@D;YfUS zN!43bCadb!A1bkiA9n(c58D}-3=a%rIS%K1mE-w*jvOdPUssosV5?N#^Ocm#6eFWT z69?SX-w_N0?pNh3v(bGl+Ikwf(kP8gg~$?C+HFlp#44#TUmY=REN}Tw7-f8 zzu14$V5UIl;+xc5Q(&??l>pyyOv5Nngf!hAl#u^c*0O|AGFs6PmC~-56y9nvzaN_X zTl<`}i@POaLLVPMXHsWAk4$_XLV;i{PvVbWyWHfU^$ID0v4SwQENE`+nb^)$YYsH^ zU#rFcL_{@Z=!rtGvShg%OE=Ovqed58%8{zCjYE%dx%-_K!7`48pk7~8*9KdWCDM&O zDY4bDq9V*#FLB&h{js7V3nO>HAFIW5+?g~__zcTKjm zAG0n z)6f(ch%%aC$bA(N-w* z@~`(06xWDD7Gh!{5f&Fuj0|wYKy5E8dme^IFx%2gtRc%5nxF22!XBwLyT%u}2dB@Y z<`&obqS8#2Fy-mV($$UUbs{sLrjFNX;*}JEa!4yR9|?~@3KH1MW!ShL zt1~q}tYB4ALcL`%_T63iA>KZq1HMoh0If;;0&!hjKAlGTd^i+~&@l1<-wqhEWns*L z5SDIVk^KR$I;O&1>4d{qu}B{kj^ry6TbF|Ik*$ng@d*No;Iz=__S~WdK;IVg5e+<* zyru`HwGKlf@56SuNvbuP*NZ)R^- z&mtkCzKDQ>CoaiC=9q#ZpqnJFY@9v4fE#y%!owQd82t{01m<2bl+1s^QmVMy0l||B z`li3G4#qZ402W}e5`a<3*wNP6!O++d!1mh%5nCH4V7VjU6^p6#in08?z1Q+Tu$c-D zwuZ{aPJj=ZkO ze{CLCKmfof=H?`!>;wdP{wk9ImNEZE+-spYkepx7RaBS(9KVwUU{sekurN0KtwhS2 z8Nm6gH6R5n0Ipw^fLT@m_a9j{AY$~7JUf8-RWz@x{fWER2L9mgH);LR1*5XFfzvPf z{o)V6sAg^igrIVB02n2W%}vdmfaEd(nFWf?!Or#-tO{^)a5jehGt5pRd!dtS-6%G~rXR{a(}TjqLjNG5o}Q9u(~pJ;M_cPY}Tj_s-3k| z;S=q=^~}T?r>(XM@Z`PVn!TF7qd8t5JhOkUnR5qiIUl5GsAr8a)Rxbz-XQ$%R-b58 z4S&D181CkjwJ;|EnyH~5z;8On`9mH^)#z6e!c5caD@8$5)fBz4g^Q@`DcmpNH5H)} zJE`8@=;Nsv6!VVPUinY8;uRZov>OxdCb~p#oJ@jXt~2;o&D!RO1mxVCpYBWt4PTp= z?^sBa0e*+GY-_WsNLEI4=QLOT$Y^_oDUu zV(>7QvW7bwvUSvDq`4w+eRhHcyQ`l4N&V_XjjbcnTYB#_<~sR55&uPgN^kX}fq)hNkXqYn#0?*IjlivKxP5ci1Wq%i zAxE-zTDe+bs*mpzJk@b2a z>-tL!=J@+;q6afj?W?=jo0JSQ;hfJQ_bHFFmSX9%LQEO6WYo=bT8-852^;`@8_i- z%s3x_W4Bv2_WM9KTV*@DA|6|T!l}1|VSe^|vkm+B##CRO;_vnJtG@c*)I7iI{ePl) zn3;g;=^x_p|3>wsO;|56A&Y9q(Ds{#OS4mx`BIZhLfzSc4}kW9!YSH{hSwC~SK7Cq znIv~*Lis=k>*!p+tk_)F%lpryue%$fJ|u0_jBS--W>bbeo&0^)6}8Rh1?PS#Y7sd@ z!trdPJ6!%O%vEs8tla*DV3r=Va>-w6xWv46rr`$h;FF)~aXXobmjIGY8++li1>eEo zf%S9#>Jx<1PEu<3LbRLJ0gtmU`>5U4701iOi4R!g#h}cFl>}d%b7o^3u4%)$NBYaI z>s!!ecVi0sI#R;LM!99(C{``^mJ#2}rfk3CW|jo5MU%6XvX#n!Q* z?8OGEUHR~Pl7Hvr?Y;aoUbSpO9DT3wLX(*{!8a{`UxIX1J*oX&OQ4)<{Ckxa15uh| z((&NdxkiuwsPmS?-FQa1lNv+v7!9IR6>aiR$oBB8xzD6I;XWn*V#uR}b6U33d9lxA zjd#DVO7Wc(rSE^DV6{-W>U|@yJ5vlOp1GuBogR= z7qs!2#+^Q!%6`XuavM(oYJU3b8wG!mzds7Jq-+S#pUoMq5zD-j)qZUZ>tD1fK zl|-*(-^!fF9gLknUc||4X!KA{n$7hT8>1#|pqbYmY)r9)NuPzW{ct`rPZiy&In90X z?54vx*vJP?*o}OcU5n*EC;Md&A}2@Vj-w@2$~S7}a!;$3cQl#2gZ}pqFuQVTj#+*# zIgZpPzn5m(v`X(pJLEhAr(wc&24xPxuGS!L*}?SF`!wa^^yMmz&TgIy#r$g z#glvx7et`~6U%`{_kT%me~V0#F7@}y?A7A>x0D$(`(Mo{pndayTTy>D9bx~q z74>_K|5}#;?ZCfSQLuk=wCK;O%uvrSfYsEbGlh!ntDB~#rlR6#Fv7B+5CFyqTnqqG zep_q)MlcDv)=V*q27WkOH;4|Ad3`)r%B};AvZ!OajF6kZ`S4#XcQ zxBzqH&EZrHyYoX009G_$!|qr{Mk5qDme17&5oA?|8}T1-|M%|7zcRr8GI6;7$K3yN z9{~T1dk$bHznw_?XWZ-E#vm~H_4t157)oX&;`e|cBi0Ia;8PMz5S_=Y!tCZurP#gZ6+XaxWVF1Ht_HOx*vIXYwcKi&50r#oW+X!a?61 z_8-icf3j}aIR85)Kte|}7&E-c&Wmr_46KZ9!a!}x!c_1(d`=%pO#Hr!i-x<4&epz> z+djNU)RAnwU7u}%jUu1CRZq^-Y~0)O7ypJdS67-UlZze5>4@#e`zMO3?VIjYK-I6j z4g1@t;Be);k!m2`ep&veYqi}O?=ES9AQNV86fuEwuJ&>8FnTo5(llfs-BQ!X*hRv$ z^IJ{V&k=U(W|qxeAs#ivgjC0UJNJWsnYTaPMkHw4L8QPLF~!qxefu4aXBla+zwu!p zb9~QdUVL_0{y0$Xrievl3j_og<=#MPVKBC`K9s-p zj=8b7^|3NqY0CZ{)37` z9{lbl8b;BV9_`hgFttwLLw4`yZw)c6ZBGa> zTrrP8yOBlr**p6|;1O@PT2a}CvJut8w9}}#cqVZ%42W7tWm?!XXWw$kqs?`tka6+Q zl>MH$E4`ZulaQIE`md{}DcPXX$E(34;K?Tq+2o0x)95>FmO3oR)=KJ-%h>R}XJ`t; z94HSQh$>c1PX7;d_+QaSy+tuW;-19x{Y>TiA%!McH7W&X7ESW9XvEL=oj-e(x4x#E z#WhQ(EV-3wxuE?wNgg!j{LeS~@5@2Hg;VZsWp%aqiF(!j;i2~J7O$;u_I0t>&e6%# zbK*(!llPD8^UV4-`3L3Ji|ZS6-$ASsa74_lPyb-KyxJxI*4F-inMMB*70&#ZE$qKu zEQWe|w__lnfcuvm2oSlkWb=Q(%&V#JZ!z;XcFKQ*6;@!A|LA=9pVz%1kdt;m{lN>v z_vM$Dm(>>>$w!EqpDw-&^hh8Wk?8dQfVF==3ixLu*3n9T= zZ>YL3BcO}k?Oy^264Y<{{2#cYh8Miq{8Wm`__T5bzyAX^U&E_^JA!|W!7_>n0hkz= z{}dW!{&R#v4j30^6nZ_GZ}S@I{N12Xk^qK%e?Obg!T}85{>#zH{N$>lynq=p*238` zj!FU`zr^)`0Zj5rVn8T{N%<*&1qV{Nh`td=N1;|VQfUX32q7^PLx=kqU@!-Bj;zqz zp(rZ0wH#>)b5hBEv6A-Gr?I-QQ$M$0dg49T2%_*90MY5B4(i92tB!tr(4mEjHq?Cs zhK&r0s1Dk`ZejulQy z=R<+D1(v8>(}TnE0x@`*M9mV+%z~KEDh*`ohH7f+HbVx42|ZPJc!wg_DvU)89p^ag91H|H zv75Wu0Wso^+bj#jt;qCeI7FC~K@rMaiuu-Vzsdbsjq;rVGi_p*?0Tr1%-Cfud^8i5 zU$yNgF;H?1>r}J2qe}dvBaPW@v%9v1RYD)-C;onqhxca`hhtenky#q6R)-agEtnwJ zWSZ#&HLXeSL29ib1-I5K=sUCSi_Jj^h2xWHiEies3*sHbtsI zX7knq%=_R2GBa#{YqMEpT(x5ii^ucMJrSH%xXrqZ5Q*~kHGGVb19=`DPX213RRdDhsHY-6^>OW14SXW zj3X%kD-}zNMJ&Mk9wixP{}UWCBv(HL8?zT@GejeD-9S};)YqSOy>we*%zMk*Of# zLhnF71)_AaDf*YlJCOtHt@T%tvLMl>;HPLI zP8i@Zl5oJ*5Y&;cBFhHok|V{xRY;_wK$n-s7ZYBPUME$z)Hk-(aKk-RxjnNS{QnRJlsNiLr_m+YFX%aBc}OYAHCT=dugrr06aq3oak zSv6ODy|$A-NJ=a%FUcU(Ak84m;Nu2+k0N<s(h*7-seg3LSkIusY3V3hFaNL zo*%|Pmmp- zS~E?!ZxEVWnmgZgn37VDhgD3-Ya+OFE31x*=_dpVmdp7o4NGd&P4neyM$xq689s0d z)oax&y9&#{V^n9P-cgbpp4+CMrJuiUNLm(G zK0%YOna|tHWmsn)ekQQ_f#P#mrc}IC)PSt={Pc>7UgKEp7`)}S83kK8dsAQD=Oqn| z3h~(-?k>K5uYC9PJ6bsY2=$0e%wejXUpYOYVbf;Nr8+!Kz zTWlKEj210#f2%B|Uv+6a^5#v`{=AIYB-b={jd)G+K=Hr~#UJ7p(t>z1@8m+&6zDyJ zalAC69@}BUe!T(TA&$X_AyU`PE_eF< zV$!DO!Tqr8T-1(Bw{`dOJpaN|!vihSJES;?h`z+)gdN;5IGxbsYPC4E>iYN_gAc=4e#B2+*~2(4a7u6x|<|p z`X~D(BBC(gVY-SBNc4!!NXUwLiY1G^OID}TsQD=#sTm0<;>RSy!|)_?aVWi+Tz(g_ zdAI?)dCb77^{M9Yhr_q6(xI`cSltc{dm2JKB7#-qmoT)>(k+>)OWp6TTM%;zO(75Q zJH50!QD@q>#*c9j$$?)3Cq$FGwZjahs-!TaHo@eDzO6Zb)jjwBS(+G|m`}}GI#D`e z9$0%@J0(o6AO4wEo0Xrkg|Uk*pGl-~(0u0jJzAdNM4SwjqlQUSO7yeLm1HDYI{8Np zxEhPEE2a|0*@KG1DBG7m(|+0v5e!@mJovDiu)8{&wz@nAKX$MmTOw8l%~okGewTCA z?r2=UYCVXt23SijoDPW(MH%C*+Jvu6E9NMEZq+lZIoa+N{w}Pz-ao_s0(U{%9g?QVJaI`F8a@VfB9blIyN2YLIAZKl4uzM@BxXK`Z^K@x?1 zoiWZcywh`>C7g*Iol)_T_Y_$3zu!V|5p$ny^xYDjWWn<-xQBa} z?NL5uU)POmhHK`|F3pD36>yk)%6P8s`AuaKj1thlO@HyajJ!@ws~I$XHx@UFk&(`w z?do$b+#7Tj5r!4Qqvj;gSo?D_v+8rz%8-9@wa(dhhtkEChsXEH<)&5K+PvPG-YrML zPf@dkOiMFx z#J}gg@-zWeilp;)*n{iI?8)$OGmjEho+hj8ZROL9<=H;24=tt!1Kp>V z69=hprB_N?x$kGMt#^!ee@?lYpA;2`uavjGz4m{~eK5T2i`zStzm-qQj(Y3r1@n0B zMqzETy?PUP@$mO{>>pkg5QQKmA|kBsXlw-dWvnUzv|nq#nz(;&>b{~N90827#zyA) z!nSUJ53k4oR%W2l%FM*Y^$V=v4E*687$$yaZsZ7rIQ)hd(EhVU{P$sQ5q&3pD_hgw zKo*XFpet039URSVZ2*kS3~YaT|L;AhUoeNiA`Mi40D|AbDz9AqRw(rPCrm48Y~^C? zWNxTWFKlaN1T?gP00~iJM?(j5J11KQSRfSP7p_Ch2KW!N%x!=K$?6*_*;?z{{Pl^r zxr3vVh?%|vfCK19$>{%n!wif-{|0V&<(iX)1(xac`s>TV@#-4AeponP-2o;52M|TU z#rhhY1m=I0zSjM%&fi^uE&cI%eUF{}RRXV{KU(`OeJ%U5oPRO%w~or+vilqRa{AUl zPvpOnkCnbD5X|!Gk^m(o3>=9Cs<;2$~xT*_c`x z11Vz^ax{DeU~qB*^REJay`g7e1>TA1+x>?8faQEeekeN`TdM+?*?{h~`tOJk_SgPi zpGyM4AwY*t$i?(G)CdPqM!*qq`|XaIoeB8qW*zv^TQ}=3VU!+)oK1>NO5KH$Hccq zi-;pCtU6oky1qiXT5kf`rlYH{>YW3tedrGt@1K31>neF0fv9rl#F@pqkfS@g&}$z9 zx1!i=2**!?rMCvb7p!OGH6bzJ_xCl=|EG{MkB9QxW)CUD*bO1F=RW%V`rTW8zkAQ?d7amJKCjO?&vRbq z{PRBNdk(8O%dYN2bkSDUwo6g@vcXc;2|=5pqb``>7(X_P*p1kXYO!h%sBt%Wy}0!5 zM(mCz!7%C(eW$G9(?_iFH!2KA?^;kLomdlF@H6Zoz%nBk9?}$r}L` zm7f_i(# z1&vmbBqa9ih_v}=RD|DZjYim=1#|t(x2W+huge#03zfM*`K!}Jc6^Ec!ZC%(saM~I zLzP}1w=VCw`%;1{_nWF%@5ffVXUvn+cNRWh(wFV9dYyql7le1M#5$GONtw9RXIy~ur+u@0_A^T;J$4v`&B|P&p1NcrDF*{zYuqz zhjSxU$m4O?m6sQuW(DTos*o09Ospl>=8TEB>1^sOB#tERlI=sz^3~jpveXwa31}QGrY;85N%tWa>%Vd1C><~d#GO$j$Elqy?Aj8N2 zmD|md4ed$T#a>V)Aq*`aRMLQ!7Am6WukqcH>*|rREN@EUoMbE^S257C9!(mrSmtQK zgI-9AUMpo9DRie{yRZCGHMB&JqpfJBoCXUGusm7F!$?z5?LvYq@?4{dpqZk| zI-D0)3=(7~WrD`(@wDfwVV2T`p!Jj27^i73(Fd`l(j};0CDR%!)7?<7mpI@ z)C*zvMkOQ*^76ZRyPr12FxgD^=BiZI;PU7CSoO%|a)q1g1R~4(sug>1N_+bkib8Pc z&B4kwy^CLFJ{gvFtuIfupRky8-96&T2-CtjnCBU>e|5JU?L4biI(`BIc^{yCrMbU2*m6}^l z4p9{HZDsS(JD23PQNH9e^RcXn^YTz%aAKK62DvkwN=c}@#Y}qOn^G3tM`4}tDa;dA z5Vp4FO&#W3V@x&ufg4AZtNV_rv?w=2lu`+(PIdEp4!ayb5Wv;jH4 zG;U|0O&l&F74fO*q5w@lnHclD2`T;YbLFE%?l119bzCALS7wC9Ta5ioOO2+;SCSK) z)!5IgG`8Gq&A8l*&1j82fpg)b!7GnvcUin}ufi+AeC#oeM4R^JgTi{sbq)>aBJ3z{ zB!`7>1ZQvY1}#3m#8KRvuT7SB@*Z_2mil#jT(u_sP{-k*P|NZ9xztBu!mPCS+cWH$ zXT_;oQ^#~4wn$2{SHN)G3U)ZVa4CXt;Rwi-j_g5xyhL}^)-%p?y5D*!dDnjQ$>q0Z zz6m3DHmy_$omd*X;Ta7d0+BZ2lo4k_%Z_HVIYguI;e@u7gZ$O#qZZH>}- z73MhXA|KuNjOyTH>v1%@2BbYbbXV+8F`>CIM{fw9o0+90P~NmG2pvQ_l}l2}YR95e zMsNDdrxPe_{Lb5a%blXT2=(2z771^P@38NEBg=<7OsPsIe6qirNxo?vBnCd58BVo% z-2i=dqi~EG?#FrG)~%_9?6`zwRSOADHjnI_Bnq?eh}O{;wSDOBOo3j#pvpN5H<+p+ zl#waTg8mOawiX|}-P5{Y1&R>jj(bLJ($-C6sK0RXTzUjQIdF6*LsnMo!Rv`F>9|H= z4p5bjw-EHSy^IrMq6;jxu+fBw-!4`Zk|{{@@pb;FaFZ@jBc*PwV{N=+RpwInSBi~P z?qYLz1Dy41t#s$kQPa#q#fYuqBDGuA4jCH4D zSqa+MvF`Xc2qBrjitD+z>a+OW95iu5^FIoZOlsU2p3Dcc+F`>|k`;LV3j76c&f8id z@rT-gb+FUv90RO8^bcwEs0>Iv>8D(n+3FprqXtRU4A;ty&BqdVpZrZR&#O4<36A!F?YgmM`;A_mb&Sz!r0qR;l!Re! ze(!_{X4Ni>lU&4OIIwQr_2L=4QOqVOIWB=~vv%tn#ZSK6cfv?vqQ;^xQ1`US5{F08 zY{?-eH=DW5{(@k###DboRl^P#0u_*sN0D-F>m9fDD{+MzmFuYuQZHL3y)KGqecVGQ ze=+@^vr2S8NJ|ct3S9ryr6^Td&6Uq0T`Wf`!2cujI^)EaLo072oyI>S@3(;b0-kB; z%&rsX5P7y$Gf)VvfBBrZvOHg2$$-lTle}QTk-z+woaX;Q&PS+dHd|K~{E!oVNYZ8I z>g=T@%6lhfEhS;fsHnGcX+yNIlBR^{gio8o(xw^j(8idPe5zLai^>{72AOw?Nala# z8icVfa*;?-s*QpAvt(@c2|g`kMot!Yzs+{i=;r%rkm7opS!VZcQB}|^`vjbF zl1$Fs_GjF9eb$jm!QWGvu!X z;2ILx`ND($(4Y_`2H1z+OJJ9u8U_Nu!LJ(dw%6gk^dBDtCHFUV_X6*K^d zAZQ@?{WlE;0pf#x)8H^UfM0&C3rC{&)rF%G01N)=L&1>y+CxD10O?mBLJqypheYoG zXCxZ3&xeBSZx02-0C4~7H!#pW&ize8VD@Wr$lv<`gQ1ZB<@0p60zl9G$Nknu4%h8~ z@db`xxVi$%4L{E+xZqvEKjx<)K%}Ywh$~{`kSM$@!rB&!1V+OOEr-WI<=|)x5@7{L hB2Y+W=KpW`1FJnefyw)0X2Ae%L@^5stLdvV{~NMHqOJe{ literal 0 HcmV?d00001 diff --git a/docs/images/Training6x9SmallStylised.png b/docs/images/Training6x9SmallStylised.png new file mode 100644 index 0000000000000000000000000000000000000000..113c861ae05af56e110abee5b5b407c19c3d0474 GIT binary patch literal 219242 zcmdSBby!thw>C~UQX(Z?(k0y`AtjA;i*!hbl!TOYZbC{$Qo2JrMI@zLx|`qJ&m+Fi z_dDl0?|J|Fu8(VP_G0ff*P1cr825dTu?bUAlEFkHMuUNY!IYDgQiXv*xq*R!=R-jP z?`UE)QNX}ph*?QWs>n%7Qm8oDTUgnc!@$UfC1@dQtMxp9=*Gr~NuW|t&>qo+%A!)x z20ak-tj39j$?hOTVRSWn)>@fm0!vcIm@{GJ+A)|jacLqhu0w$Cx}u1-URUC6-(#D;#!31-HrdGUiK-J?;CD=RF~rAN@HVv#LL5wtj1OBl zCvF-W>S3{UyvD@ykB4qu=|cutVGw%t`xa!W#jq=2;0JIyWh&cW>-#c&$#i+U-CCTR{Pbm#CgtJ#qu?I}PbZjZ z{i2xOT_&7+QVc{{G8`LPqVdW5L}(@xC82O2n!JTGmO$uu97=jvZrUEtV~WN#i4(|! zYZbxliymrfGDrt2UbC6&c7k7MW-|x~mR_ z$ycmRl!Mqd

e1O&#_s8{Gx`(^SOSc~4bgkQDOsRqMTr3D#u3GAwqWSJY%b3wPf$r+8Q@4r2tvP-JDq*$vlrGW*8Byy+KTIprs}~>Q*8DLW6`0A4=Ilk;7v5 z`f}dDov-1dbu)s?F}l^myP95z)&-jkrx;HFwVEFFEsdx&{{t6^YYKDLZ@&GAc8J{i zj(#&FA0^H82zX)c)u11>xA|mSYIT<N}1;f7X%86K{W_GBk8EDQcF9HC>v?QWiV`yT$B^D;Q1MN(=)p7{zU3JMoN& z?76=3@+a$I-3TFwO(FGFNOLPaQ=b_`?Xz0+D1o$LlTqpxDdt4n*HPNGh<$mfqHTQPf#zqkhUW@GA^L~OI6r_S9 zkQ1Fukr*nx7{QK1PJ@jqV-jOVgWn@26?#7tGdoalr238iAU+_KO&|CuLb zRKnfX%Xs10FAI5I20hQFen!_T(LT)Q0+;HqB#u5};(}U-^!bTf_VK#h5tS$Dwv=Jk zHDQZ7hkpNk=aytblD8eIr7j8q1D*0^Xhp=I+EW*to5Ek2G)~B*hmJZT?Z9X`ppZo$ z?lGL%cylz}oAV!E`h2EwML8SdN3DYT2Ei+U-d|Fd$_zUnR~`jF$TmnaXw-r270xV9 zn3R4F_Xz3ES2L3bj!2Fij>3*u`v_%$XUP-FRPT=DPH5~^X~LfN(kI0Hh;@Az6;~Wn zCrhrbuA!iwt`VTFsxHMS&4|w^s1fx+@0mybK}wN$6^&Ocb&vOf%Yo{FrW4mWZtdHv zlpMLTe0R0QM>_Ey@!atOy;i-#y(AhM8eAG!V=pwaH}%H2KYD2leGpN-QY@XD$e8Q(k=y4F`+crs)%mFMI|6?ebUz0|dK{!;O!?~9vtwe{)7i;bFf znstTI58wGej(&K%@qEL{#n~lkb7#CU`_d|Tg#8D5YpHbm<{N#Y69fAiNG3$AMBP4m z_En?u1;(}5waxX)1?I&mrUSfS;ETWyNF*3aBuDqg$#%HYc^h2LZWi1yo=FO}6Sfn* z(Z!g?xQvX542bM_VjZbObWHq<_>#zysN}JT#f+&0?>+}kYSCkUyMy%Xb|VfaLTkd$ z5h7}mqu6R6HY`S6M~5taSnOL!TX^;Fq^b3%tZu9_4e0h?GAl55KJWihSXpB#S7r9x z;(2kEkoEa&?-b+6OMi_(jp|AWcURqT-H!_h=2+*rMYv++0Jm-q%P7yZYvZ=L;P=5X z*3sf`JF&tqcP-ey7~0Qq1hBW62EC9v4o$oF@*Z;tT*&i~kf^$-g(&GL%qTXQ=Mf4- zwcg8<@8;6=GMv(@1b&j{aWlH8j})#xE6LbS_ZUnV9DkxFgB%Irs&hPIBJd*c??Xz` zEp{0%D@NDT)oZS+t(TkGobjAtnNj=sX~v$jGUZwS*M4U*1T4R3eli&$5zji0Rj-}X ztG>QgrfKT}ty98Z9044)>3lU*U9+y1@Hr2R**Fj_TwQV?&nbl00GeU1}@AoHL{Ih43k zPE=Y_v(e>IskpEBQVB`%^@*+@Td4QX@z&kmW{kY}vfK`r_ z?dWzsTk1^_d*NO+(HX{UL}E%l*V=$L-&=0$LmM;KsjWo*Rp;x3h!F>2rEE>O$3L`)sYv-xG)BX=r%~+*u}SuniySO*FkY%Bg(YlyLBrEuKF? zBi=`+U)OtK{8@AH`70y2i`bJqyQjR&)U0?c^<^_|=TS}r2Wx8Vb$G0KbV zZ=Sino<4?>7Lt~xPNp(yFVvn^XdPy^M}<_Aa|o+?sIRSmGmSU>+JzKFK(;GNDKg^4 zxwO2LD97TaOq)^ngZSr^ufWz(Etj~ZQigeku*oLcizR>Nthk4G2Cz26F(-e^4r>Q`6YD0K5snzenQ5L{E)2KJ@$0@sn)*Q z=4iMj-O$UrjQZ#!gkwElksXh5^!>L(jkJC*Z*-2ot**2-U){X?^>p)>C#?tfp65Qv zHShFXU4!jL^rqh)>*>$6(>(Gc(XF%YEB6z%#RALB0$)lWox`yc>%;0&`%@$*WTXx~ zRIMAP*RA3C1@L>Gf-v9BVAl7%pAZ%B-S(j8@KyBUoGEKNK8XjgXLJ76%Z2T#z81A* z_VeBu;_+A)Jhn6<#UOmvdl{^;){4|CJq9Kz_j?l^=c$QlNH>bl()s;DFV`M)tLwK6 zZ8yWvUwRu)S9}culp@?*Th2mJ5rz>Qqrf1*62l;ZBUtbhfhGBK`~>z94E$g3!@LVsew6WZqQSNPaBFv#F9T<~Nvx|JfMX>VdYe5_Q3VWt<<$$v=tQuOzmx1jm_*$%vs%S z9iZ2N5pov*hqmS}#uVMNMQT%m^i?uM7wxSA!q`i|l1rIAb zD?6148U+P~kdv8(fU4A!-_5~a!c;F@TpR@0*xcOQSlzf-?VT*yIQaSb+1NSRI5}Cs z87$5ob}q*5EOyS+e_!P8I#TA&rcPE4E>`w-6wvD$o7lf{5vHPo-ssQI-~BXqxBAze z?3{lO3k;AA`Ux8cF!(>$22F*a?+U0`xtrVQNLkr}IRp0);p2QP^w;_S<&%Hi@gFU< z|J9O{i=FeIP5<%H|G%k*v$>O`y)C$>i^#v$>v!XSe)zkg5F2#p|Coxu=lQR9!91rPVID^x)5#Pc^YZ&hs^30X8YVCNW4~&L`Ait$xgziKxN5B2$eHI$tFZr z;21EParoNUTW)sDIyom)@%+4ZYc|6rcJ~D0KQ=qqzq57o#IjNFu>bPj3v26XN(CA* zxc~IwC~X82Eb2IB*r0<;0sEglh7lfJEwI6%#=!h%j}A&$G{42tQTM+8w6`>33s`>0 zvab>Tmk%MutnzZzQUHbJTwSZ~-@or-e|X+t+4FO4ALdP1j=jJ?mj zg$MCiC7P^peD6cJPj;%Rs!9cqd@p{XJ*4mEa+iFc+x^c3wG`9t%{UMBOjTM_H-#-P zo2BqM=4_R9kqMW}%=(<_pD)KLM{Gmt}Bb8fdpqcnbfG^)JfcYA5rZC$U?vP{&e$^L`O+QOi~#DY;_@j??lx86%}*srDcwE0?8c&bt%|4dx(!5AI?hPcO> z0x~`o*U6WcmrEGzo=n>(UG6r|Y1>bIvGN@5)t+eYC$VpMM&@%gLI$xF!NhZ}*+C@G zGb(P;`@KASF#cb@d@&FOJ3rK-r0L=%|601Ur+4G=ukl~fDs}`dG-D~2kPXeRWm~#) z7m42cUkN%7s;YQQZvwGI9##KzO)D!ZNLlKNPGTx)P@UFT&|4yCJ`0}lIsLhyzWt0e z6r#{pKw#+ojJ1BNoJ^l-TOOE@Rg?Qo3EFDl{DWgcp1IveYM}^H1Cg$l;JfZO3ancCKWp*~ZUpycA{u!pi1(*U+>ApzrfZ_(F5*J;wUPGN8(8t0dJC%GCzs%? z4=le6Lli=~+0<%|_tZov7sY&?;;u%VP@E z)zy#I?K#_Az0X&YI)9xObdk@Ih&?L85;+r_Y0GOR84^C7S@L?WE_xc$mDxff@~f!g zA^kGkuY=^@9y&)32Hk6ZpI*H!fG#w}D=m>(Cr8q?)i>;k1PZ&m>yz#=2RsB$6oMZU z6NEbq#mCr#co1i!uA022bb85p;JKb;n$wj20#j1)>+-c5h;P`4a6TW*-k@*u&bmvM zst&LmSMDL(w!`Dloh1L66O%ZxFTyuxBD9(+!pGm$nsk0DLO~^vDIYJ+^w;DrmwJl-9DGqIM7q#%?KeI^(<& zOkl7#uxR<)F4mpl3Nl<%8?H~LDW{19TndW}8U@URtI@D8#jbE!MxuXe*0hm)4=KEu z55!8$&!@XQD%?7qFvVhN03pY@fh<@w>V>u#9IE*1w#=Y)3lSPr_e?QEG(Rz{k9IYw zLHir=7m{ajDJcwd-iMJb8eDh1j}hC#S!aqCJ&7|Jy8ZRnk>Flfh0RR`S>(GxGvHzYde1ah?$Wwmcz93#k+4H z7E1n*305Jyy6}WVWH+^K(rHrp>puaLq5lSg83%(Z6$i5z&Z=EXu^#(gCOWbCYK2Yw zy!(Zf6;6j|FdPZIHum?Lyj`DNuY>XCo3XO+xV?hXQk8tAj7@hkG$WWLg9mH|MPs`O z2?CLy+ghyTwP8N{-BmE0k}}g+l3HvV*7c~+w*9#4w7U6K|%BDi(}8y4ST=K zwD_bXWX%ncroA?F#8+^5Hi4Hr^=$0AJog*x$C-K>|y^FjV8X^iaC;^XJc={V)ceAo6<% zLl%x^7ChI8xL?NGOnvFu;1oYkj$veCLhp6!b0DT~+JoPTVDwbU5<*5fLEAZ z&$jA{A}#kgv=G0gz~}*ZMq(DJvsqy~MCjd}Z=sNY(`F9}mR!h(#y&FOgB(6xj@9pX zlB|r#kRSJ<3$--12%5|XF!uMgFwI_{&XK!^&+P+;R`iwEH$?>565^2`MYa%!e}n^P z8nJD()645~Jf;OFc(3Y>?ghXfh;Za;PPjKMZ@hMD*Axn*;*p~zuoV&M+zXLN)KZ1L zs-Q?GpK292DybBJQ=cLA_JQLqVmKwF$Z0X9`3$Pg%V@tVQ=K{&E2C$pWoj5|>JG%i z0VJ{ttHEjP=ZNMPi7dJi1!Btx=G_0>IS%N@mA2D7Jv}|B9;4VK%^(tVdMdd}GY4M; z6k1M{8h2h|^%s5qoUjxnsI)qmL8ATLCj#FcrLkA)H^w%gn5Vfy#rN&?b!EOU00Ji| z0h`Zh{Tx1u?98|R6s{EjyWBqEs=ERw*^S9NgB`z{h+`WqTC=NSJmIsu28K`Ij^!dx z=tsn1BBfH6)t#e3ESXQ)L$Jp|aVX z5lP9mSPgb;a6$oRBnhtozThgGPPr7FGGt2|l9|c2_s08dAw=+pIBW+m00(97GZ5Lt zLotlyOZknw_-Li#+^>(v^`-my2o_gvPj7EBADpBXudIfRSV3$6<`_kvR~&!)1bC>C ztZwTVvA-gd*P8}tWWvC9;Cw-ab2HDuUKRxVm@quwW!7VDW9(HvCq4BKtAIflOu?Z6 z*&t*}`e!jdBBy_le*GJ<$3dcFV`H3-i&7sxe&iJCc!Nn2(B|9uX%86-rM}a4MM(1( zb)xXyYPtK1*$0an?M+%{_Tb1oH16t>gemPZ zQn{(IHu4#!wihkaSq%Ka_aiLm z_>_`!|d@cqP1oF^^zKn33OJ}iA8 zo$3OXQ5!Ag{oec_jSFpIbNP|`H(8P2zNxBl2@LhnRxP12|KVcm1aKqRH}g-*o|7}x zD{g?zh2iv+bf}G7h?5O^Tc4ZN1_}S zl~9d`LNJI-?zdkp(~IJy`*o4J5h{0~xfQu!@qP{1TbnSA#-G5=KRRu`)gJ`tG>eK2 z#cxj~S^emBO5FG8JvpMVAlL5@K=B-5-v3@2L+hvLIG8bpj@k`@?oXdSjZ!j` zkdQ=b036CmD$%FC9^)~hRE=yS1A5Sx@l z4;)oN*~1f5y81T7GK#C(^VKv&Dgrf$8celrt`ZbK_EvX*F;mI(MR_5acfGg>e?1CE zj`IBGe8^9L)ja@H?b)-17kvn`Y!dDnfbRKE{_J{(mgKmY&hPw?fP&Jb1Nh^*l4z{c zIlpEjz0P=}OYuG`F5(9QOiWRU>H7dc}HfZ z=Q1MA`D#;rDJ9eIrj{a?1?(iTIlQdBf?c8tL_m;yDPY=-M{=aaS&U0x?#$GRpJ#JI zse%U}1rcLL9j&j-jGTyvH{}XTqN45zPrUIlngW}^rD(Fnn+!zczjw`+C*0impqm+h zz?~%iGNSHS81%KMegHNr@~k)+gA4g`m&y6Np6I_6|bgLhV{O@`R3x+%6=C) zuK_yK&lw4~6eWSHy>`Sv@qBedL$-~LjlbPJ^d4d}I1acsQpECo+xC`#cwi8+w{))- zRSe>WhQj0BxGvfAeFMP9Y~x#=zs6CoH$9e*c!CC!(24P4fBd}1Bf!@S*8vcY$-Fr# z0GPo6?Xz*wOaf{NY_i^C+JZ1K-q!xB^Fw{DqqHeh-dzwK@4M5o=Zzb9DJlu=OR2px zC4*q$z0OBq8~yd;GuL1AF^*NR&8%hL zPd*hXYhO=V`94BsKbGi~itE z*CNO71&I;6#RYT#!CO@{I1+x7?GE{np>FK$?U8HeXOs zFp$*|^hD4T@m|r^{V>>LGS79&S^$mUmBTI^b#zkHH8hAyrU~AbMPPnV@Vs)QYWK#K z#Q+YJ=85PbgIvoYLJrH7qcfJrjAIv(R+seCGx($#`gDhYzbK0DTeFw}cUX(qJ0m?J z@_&Ki2ap)8_HlH=UcS1H&Ie)mi^Q)W*nC=}_~^S3OyEskaOv!a`Rx^NH`X0R?m|cp z5+xjWIPfKKf6CT;Rj%jML+d|z;eN2(6Yy0k{|Inl0VQAY7wD}l-x@p)m$L%zaq3GH z*KIO(X&n30^F_vGLE;Dz$uNw^lc8vUv=KRZvmQ0su5N=)#xYK}kd3{!x0jQfKywxv zaC(M@cz?V)^oFrVBxDMI%pa`N`f5JX3b>M}qu3(l-=3WkRqRECgT8;yy+iZ0cENqY z-3C+gi|xL?K17K%71Tw_&@{y!hN6NGcFKpu#Kd~LcsLB{ZG|Ua6bla1g>ERkQ{!yc zTv+C=_LsW)fHbq2wsK13%CM#O7uF(M;!OC9q1QnfI;*k)uoWW$90yg-)5pv|y2$;Q z)$$cMLG$d@SXrzI*jrZTqLvf8MCxaE^OH zVR$qb?>J!`!q5706R@4OcyXaCC_iw~6~H>Vzs0Cl2Y}r&#A}BG|5yNPE8T{KcR^WM z!e_xEnr{)szgku;mBaMSLyzTm|L{lAs76un0J3@loUj7I!aA!H90F5}!j zEhbde!}ChU!iwXYR%8$C#R-?InDaWPt9^fjU92M8LoD8>=|{g z-@F|LwS&$dEaP*aQ&BGs80YcK=pfV8RL+(f2QmY4cXQSyD(~#{5C%nM9jW6vgV7VKYMdt&gT2;bM?#p2%oKbEVtu^X&}}G zi8h@Mrwv659faz6tRzGwIUJrXxvM543gh1#eP0>^Kx64qx$jDX)|YvqeqBrM->e$t z?d!Nxf(bG0t@yOGX&=&{E@DozP_Y)gWFwiD=}g_Ja`U$s2_@}&{|v^s z4{$iU26dXZZIJ|_>~U?GDE?&vsO}MfIFS8P5xIv-!wAqvP3Ei})1*jQPZcThBfNZ`AFu&UZIIuy?#h_t=BjP@S_@3;{_(mk}Yz)_s(C7q4r#*Yr zs9}#4q^Np(bV=DB^}ri+0m33Q^N9nhR@{!8|BGn!midSnq zwt)9!xw*J>KD(i7o^ksdIce!&pNYnh*GLd+pgYtZDv+DVodu#YMXW5Zt)@3g!3)& zzY1=lReQ*tC_zH`2-HZ$74T|cCWh}}NYxw;4i4V4NxTzL#o#!kvPyzPk;%PFO;I?6 znm{CUPXvAyT|&8~kLnM3iG)f_#f>M=yGk<;R|n7!)=ixo7v7K{%)VCG0GdG=kTd)^ z@O#kN8YtEvUd3hogAud~SUXnB-R};7aM}m(g2`Nt<1X#O5Rl7%ty>uZT_OOR=FU{H zS>Tepl0|$xhgTGgo3`WuQo#!n;?6_Gqw5CXMp0S-OJe!i`zJ(wIEyX-DRq{=u&DEp z3*vnb@H2htK&?Ta6{P<-mmO_mpB_WDNCJ*S47el0b%a?c&)Y>6YixWml9PqLUAqS1 zblaK9x>6z1w({*HvLyUuY=XH!4;QDqb1(48tZ1s(1iI%d%Kcw+SUa5`z zsE>Efd(^Vp5y<8}-#@5Dj(1JmTiu1u<^=>eFrT4h761f~;9(y{5ikSI3m2mp;1bS9 zqUuOU?(D;+@=z$L2eKgI(n@50^LcBXB=#5|?Jd8LjrFV!q#-VRmH7D!(2C`5>gX@Z zE0DV&P0@)$p*|Ku?mO|RKA;78fp6Y?59}>t?eUArAz!-FpKh6DtaK~v-46y3IKGkJ zp2;lHmelVE0P&J5v9hXa)1McksqyyavQxm-;!)D9tMCp8X;1_}0YQ8>CKfjw$b+yf^YGKvK~i&k{Z3tLXBo0BY@We*di0yzc>#hlea;LCpY5t?yUv)BXD?I@ z1G1ZPq+`b^{bz9i$8x)J=K=dB0vq5aNz{AN?zug-l{df`7=a3?I!v2^^ktrhXeX{+ zP8$_jYf|a>cK7y18LBUYI5Yg9yvub=AR^3wB4hi;v5`Pbq(Ij>A2-s~mjI*V%y|Hf z>UphG-cnZ5Zs2E~iYG?X{i)o2y!Dq{9m+}uaCJ=k@OLUB5o|#ZgL@7K3uiew)(LEz zQ5%ziuWwTZsnhMO*KQ9geL8lzfij~}z-~BGv4arzRCbfaU^>HI)k8*OxO+K|+TJIUK+H!b!o4S^ofLQ^OE9IZG)cXBY$0%mS@PV*zPB||o)N0kgV(^uaN(``}o zPanN%OI8BXZ&?aHlO`WpQY8Hm)o2)^@XZk%qcW{O9~H>eJ$&`RJ^?|tCFONoY%M+B zlf4!-Rn^4_LOspY8JQ=NXMs-^aB+%p6h9PX_bH5R9nCS`@r6%=F$dfoGhj=Y5E%XNZKZy=4PdMH!%j zaoX1d1+=h3xGgY~Inr+fASLt?M7&9Zd!2`E=xhj;ffxm_pJjz^<^>$`B2ii6o`q@n z=vLcFbKfPgDEtYbxougRHgcB*C=*@JcHCe5$_eNx;g;1b?(dmBig+EjXPN19@@}%i z@^mZbo@M2iFWn&5o12~8n*C(Af@35k;VeI>hmn5Qr=~hLH#ch(>RtWmCDokLj=07| zL3Z^4D3q0Hi-tq%U4-GG=<9MT|qwSeVxIKWHtc7X^x&0b>tRp?7a)RSQlfQ=ZP z1}=M)eHRH+kD%H5KGu5Xb#FAkfEe~b1l(}oJyFMgdTZnEu|oA7c6LtBqZd}Tw%l#0 zB0hBrDV$%mAFwQ7u|_3-BH^`rT&1R_My;cx)5-ZS8oBix;3X;6EB@VvLo<-{?TqqT z%WzvRd>_S`$PxG3l`g&$^Fsq_&X8X(ND)mGB=0mFDv9%{sX_Hfg$d)u!T5B#ewA_b zwoIs=4)NS%7=7Od&4JrD?7u0CNV$u7cyK5(XdDiPd2MYHlK{)-=I0}E!|wW0KZ#%u z*MG*Z8ZDSilFDuzV8>R-kzh<*pk4VgL8R+TGRe66<#sjVlJr7;Z?F8ecn^d3#J9Ys z$3mL($B(X1Y1&M2WtpK|TGQ{)O0J~>AqQs@egCJj@UeR9CW~4C1qxDvTDWE*n;toy zUZ(d!SQ$f5hnRxqzszTX+TmzKrcZ|4$E=~O3FG(`xL+RtSoSsqIjM2VB9kD~c$W)* zwzNUGN6cfBT-G1IQ5B9!$<}oFb2nAV^lk%p(1IL@vSHALz~xrO%JI12u9^~xSEW%a zoGFPCdag=siKI_EfvLj`B( zy&oL)275$KxNwW7%JJu`@g5-l&@XA==r4e7CiTS8lk~xJ{iKp;35RroH>b_kS`leHwg&?zml#?JP6HA^nN6mPUg4UdHXqKBeYX9Q94+v0=6L&QTBATFnG zxMU#fqEj}c6K3He@K|QNO57&~$b^EgtwGf=56VX=n0fNR==3NYEJ1E^-3!V|giWhJ zl6M#j%J&12u@aezYuenAL+P6>28g#8=;KEMYt~ihBuRJTa~Q6msBh(`+Z;C!qB+m~ zsXIsvShNhq1L2!3fLK-a^kz)8*E0Q@uI7g#+;RPsvZ! z&zxIek)>#_1SE1P@2A?s89MOfp|jSAbw-hrs0YPRrSaM)LW?|xsm1Z(IA7=@!Yok! zZ6P+QOke|*-B9B;isZ}=Gm&5bXvr_fVVOTN`|c5;eHst0vED}%eH9E!qWKP%Mg!$*sfG3)t@rNATuD_TPIa&qWuli8p_ zTjb`t5}EK5_sG5X{FS3_WePkvXAZ@YNtX?4Lu8-{50JGvJcH+mRm()w z?~d7DBEQvcNdOdy;xRCB2`&>%ARhBk%$p5ME&br);#wJ17EK7lqpveyO>H9&dYA^> zX<4K6+XtvYvj68=1dBH2xd7+gx9fW<*xVpNbJ1@Sr2X_X@QKMg!z zAti@CMqvh6Gk0sids*BzYd((!5`e>uzA>|Jzg)nQVuco^Kvw~9@#aLR#@ors8!g(2 z=NV&#rnN|SwIPltULe-Jz5I@{kn>O80s{_kiK=u^R@9lAEydUwr@k75iWlLExLNP9 zZ3R-Qi-`^r&nboY5#&&&S7@H2H8cZypv;cEoCwmem2PRE7_3HPbdi$zd5GJS8GA*= zNNb<^Jx~FY2u$&(2*%XuicreWWCp!YX?tBFB?XrNTx-vVh5grwTyA40uD$^>KDr+M z8)y>9$o#tb_S)Vw&d~SVTxy3SlEC;bIT0h4i9Ml_-Nf#A+yakv)Wek$B+5JE5GZsq zVcL1Zz8`|x;((_){nK^YwsRAj?0QJQ6+zE_7o1Gee~0uD$eZ_$mni4UUyKT~vcwn} z8L8~@LADxT@P|WOif>3=CyWC)Fw{5|iL)VT`s~iD3jWEDW{!Xzj&kI7LA9ejffw>_ zGmaskmQ6T6r*Or%e+`KH`}GE(Y60XPnW0YD&`apIZs0$Q>wxEN6ahE;oJ!Gdi3B*9 zh6a!Xh)N1#M_RNj*vpNgfAgrt$w<4uOLl%I4E?e;IS0R8*z+naScs^Gfj^HISFkEq|c7)=`Uf14nR7(9nem% zfh8=7dgFu2($e8weviA#%tp%LiHX9-)t1pH25I8ez9ys`HmJ&YVm#~%$RVf0d`p(M zpeZ2gaQ@g5#!sk^&GZ&QU>#ZbkV7jLz)o!3E0NcfOpk6eN^|9i_ZksGzQGYJ`VQCh zT8a)}X{u9^%nSq}DT-8iiw?oBVuYH$wVy4}I|vjPuuMiIu1w`azlJAQ{BwK0Ci9c3 zUykZn8UooR+K`7%p=DggZz+HH(D0<}OZ3zHO-cL-DWUpSephDW$<)(CHbXmalcLxz zGi~K>Kxd-XGVsXTR}YnkYk$7=GFlW=T&iknS~M7%wRI8-y4tAJY6;yaq&}9z{E<97;+#}s24EU8DLKX`c)70@d5+0l3XAhwsY}_*&t&II) zHm6tYywAGV+$98Pqq+12znl*a4iG4Z+pl%W20Sys+I(no$f$_A|w z1a*skWP0syW)?L^inakp2shBs_pa< zEQ1VRb%f%SPuta>XOudv$hV>isBQK*xBXHEY;nUW7Fxa#&~mpDpxEN5I5RDc3H$bl zhq5R*u9_Uh?*srX18#o@GZq$>PHRx^s`@07cuap4DkCVJz-^&V5alCy!MKXkpc?p4 zXmjQr9l5+87Z0M-sdxRn5ns3>9;2?Ox8)}6t{~waD?9oc7T%>0KkhU~w&d%@o$SgA z>pr@%zRs>GgE9(XF0Nu*wHV5ZiX%17B(KQ?6)GoUL9%Fj9f1zU{u`mxQNwVnhl7I3 zxB8gOyW9J~^FM+XyuK!BfJy0?*y}~v717QjdqE#G>I({4+iI)yyE@2Do@TAd)ce|> z0tfIb?q+@4XWtt)R2smc4wI5S$nVwvTU7ZUf9J@>ZS`JOrC#DfG=9{JV2dzsLPQ{E1x;%@6%-TFCg5hxj`q z{O^CirUsVvaHRBq9L0Y-UXvt1z(j|MtKEN_nY?|5fny@B8_8`cP~O zH*1gH_419|li%|8zn`4Dd5Z#b|yr&J`b+qb?jw^#7@2|Lt;9xB>&jtM_Ra{8M!OZU`JYY^&AjaEHgcKA|EkQn>&(AfV*d$R5SX`4 zFn1&FMA!dzLjJoq|6kjrViM?n;F|+@^}_#H);%GAz`dJgh9*WvC=b_$c1LQ3?n+~T z*?2z90zeedN1%p4%ceo?rI!n<@?E(q;2g8K>l@k_Q!DAJ_8W)Zck}HJ!6gLsw(*IH z0uab>TP7$X*G^>B-$V)dhff>ULO{YKrU7yDWi%_+!Vst@oPO`@ai??v!#GN8c_u?T z#4y6d`7mdj#jkn!g>_CVPi<<;}h_%kr*HyEPPmuj8Y z>f*~o?@k1nfW#JnPN&}^&F@i_M57y`5@%!5yV%0vFwEsL;0B+&E>I2HeoBLNPyDV%nZ z{D0iCOagdi?G4mfG^mHnqjx9&&zI)^(xT#mA`{m!-h*?Q zU76R6H=l|P>*b%P5}VHiH7?y6w|NIBYP?d_7EoJ;@y>&vF*A~$e+g1GcLB)@s7^y|r43sBkCmBAC7bPs1cJ@&m+ly1>_8vr*yPi`K@jC@w@QQNWeC}g$o3YL}o;npyDG|ht^=50Uv{It>lV8L?GBTMh z=3e72dhEwAC9=eRIdUQ5cxfXc#I=>Q^7KJ(j4}0by`_YZ%eW-Ob1*nIC4qE=8hk^b zvDzSdJ>9(|y>Pea!=|uBhaKv{u9#OEvth&Q-J<;)b4{qE2HZETRVi zQM}W54u9L=0M10}g{kQoQ9KR>Gr z8H{ozs<`an^NZcH_uIeXSLG~JM@qoNyrseVE_9JQn(IDqv=m2Ew72%2$i9c#?Ya#m zIAxZSeet!axtMCX8`-0N`VZPSd5_0w27?@uwYC2$e=~_{*;q5ABGBJo3f=v+N^U@7 z9Y4%9qPh@@8mlG1Nl8|3{Oiew=Z)-?kOVBbzqT!QCIZBLWgzsc($fcnafW(s9^NAC zyQz1sxdjj59{(m`?FNCH^cC~z_2z?=>xKT%p(WIRZ6c7UZ}wScm^3Zv>FlEgU2Z3` z?8$irY%()?72Ot|3AQqfQ@DXMBEXHrDm)t--$85`4y)v|jBhjLl1NE3 zPo11ht|C8v+=#ELB30#iQa3QVa3}Rz&8hM|9uCp0_x_URp z>Db=T1l2D-DQ3(NSm>^pe882UN>imI5(Z(`?e^QVr#J=lHdEiSZ>lg={FZQ+>Su<( z=x=UT|MIK?K*i| z`hJ@%7t;<n$Kw6prjI{mV$FC$urVd-?LYhvL3jmnwn z)z>|Fx?oX*4Y{?od5Vu*Bqi}6aYy)u0f;m{rVLkM5n4h&>?Y(Fl!Z=ob59i(lyf^4 zOGJ>LEtL#WHJYONfke%*V7uE7m7vAu^tQ|ncgwsz`kUrY*%08;Re~CxDJ>|QU3a_v z=yv~VYTH!NX<8!fIs}qBF`vOj%?zXXJ}E9lKMfgKibj8kh5zhqX*OjeWkXd@Kt&!2 zOZ`$jHam)`uiDiXU)4~5&DY-`e$!QB+Vygp@nf#j(_r;imCceKZYcDQ*xV20_w_?S z7uXd*zqh=gmv2}s;|Uhc_F`$S5tBE1ia|z0YYw0Hq3PKq4t_7B)F$QR$H_3khInMe zmcq|sb5Tt7SxL3JX{k%hN=#$Jgup`M_ur}#=TGO zC0wKm^hN1l=~~2n;if8w&O!_Q6lMdbss>7M|Ap1?hX$%RR=yMx@goZ)?c@_k65JO z-eQc{YS}Oog!)z4sMvn;nwb4CrD5P#H?65P@)_yNfcW=PPpV?3k)@%*U3)z@v(U7H zu1xILaN(eI2qqL6Yh&NivS7+JVz#~NHECRYuF!+T7rJd#^qbbdDlyB}iz-+lp z<1_^nC&84X|D^FtN!@vh_w0y`1D=`0ybSbJSc^{LFNkt&WA@GAyuA@N)7~ zCSU00806=<9(0pc99Fkph3ytuup0D&&A~CN!*I3#t@_6m^ZlFTT6dPDw#;tN_}!v+ z`{7+v-jw0_f1rvl+FuPpIPgq+f5V zXIhU-DUQ9I21fU^JuMUT5u-gG+Q(*)Z-%bF+L`GwF+w}|p3()(Nko-tolV$PEENM8 zf?&Q>@GLP+RS{F=&dx4{ zs$Vd@kg}VG=_DdFKDhq+WPLrgz2#B`rk4040dO{fh5K_MP&f0JiAL>ZaNo~9V=LIB zKN7UIP)g*!`ED-CAY56V$(XPmv|qqTT3*TKi)SR-rpeqkF`nfgtCd_bRbj)kWnHBi z@{>7O^sl`R)C}8kX_qB1prEH`gTzE7?nMJ`Kisz}yLVFBGKZ^Pnn{^fR8?sJA2To* z_D%o)W9+S?qWs$KaY896k&p)I?vM~ErJJEkdSF254(Sr<5|EBz07>bR2I=k+iJ?pK zck>A!-?iTN`To{&&0733;5qy3Ywvws_qhSaLW=zZ+*K#MUl(J{__!W&h!N{dSJ1vj z_dN>7pf!THOaGGl7)Q@1x2s+Y1A(D7xy?07x+H0Nc}23RX{mXzPK=b1k;yD+!7Lw5 zP7{hI5#Wd{rBgl-!9^3KGQWi1p4X$?<*GdizhkcPdeuS%6SNsIQF+98d59K6Dh>t zY>ZV>xB1@QA^2jc_vygRdvVnsL?)u{Foqarw4VSl2}UBQsAaF`g&*;y`dtb`+U znAktVRiTByXsfRy8vFzp(swzgWl*jTgFzLmq7sg+DtimyuO{r8z?G`iQN!fPg*zJA zl%7WecF_Ta{OJ2YFX-PL>G*R;r1-3fw=SvVVfaJ6Lvi!$de7u%a1uph@*wEbXf>jU zBRIBUcEYd1QiPYksJP?UT7%Jdw2)ZFVyHd`lOtczX;tGiJcL^?&oSvu$f{9<*4*FtTH8=629?2Sa1^ccQg)^5iy-7%`G_jLtd7As)r!pi8HPdL7H`8l<-~7;|*qgu%u!-#rX&YiRQGc@QUCk{Au>mvhKF{ z+E)z*yAddadRZo#p}!mmz&<rO_#_sqU-UCY*4AR#jbk)}cG&D3Wf^_P`7{%bjpU6<0aPVY5i7y zeHYb`aK2|>Pv3ByvlXdfP&xMtEy#s!bz1(MQh$%iVxX>58fXrtR?u!XZhG3N`z0Fd ziMe*x(){rH6h5lz&0Du4K>hq9M0VtV3lUJ6xYVa2k%!O3ose;aXH7pJk)d&jG`lDO zvxOCOy44}={N~-QjwS|4BQ7uaXN_x6v#C2${* zg8eWaqLhWY%%O4Riy{=&K!zm0Dw#qgU8gn3Xx`yYtS)b4l1ZI9?CTD_y5I6FpMF4* z{9VT@kJAaVA+)=2&f{rd>QSw@eE|AauM-?SPR^_ubX}~~p84U-g=M^Pb}mZ?-t?kv zBLog1anUK(9S`P&&D!KttT|a}SHburWKxtmZw9y&GyK=CESig%2= z$!CSj#gzW|!hP~o$uljL)CL?6)8j8Z_)I$SrfhbP!pgQVseTp$j1%(Qt17mK131VhKW7;2|IOlH)^KcD|Gx@ zOj1$(vRBHQ$dn#=LZE)AI8lj42ocq~sk|C8EgS+F&6iL$Zq2E#tDc*~Y1}h@_NYh# z7G9xMHU~*Pdx)IySj#x1?`gKIKtw$u1gyRq?@VeqoGjN+`x z#I;NH5eCaSw2$Q`b(N}^5?h(#Kl0rBA9| z9tVll%ZuCuv{Nd(+Mff}Wi0Ah367T~)+2R&t?eP#$xn?)1TK{d;i!p4U*c&8*+gm7 zgQMmRf}>^)Fv3w~Ei7tsp)E^r7#mpiMI-Z6Z!llK+1jlAIMQLdqogxmf*fX*PBBXd z&YZQYMs`!B8lZZQS@3VCm)_pf_`5AVK+rC`6*2vII@89l>A7 z@1VfxZ6YQ`e9_s==n6yM zJsQTXAV5FU55W-FPXiwH&084+ysTTL5prgDd*;h$d);1^&YNpNqqlCBgFPA&#?>1{ zIEfV|4YFIq3-v)_Ohwjeg-4I%P~<@_86-QZ5sey#uTcpLYMIhL>wt^<^6J46y2*Ao z=_u!r1hlz2-5Oiq$arqk99+j)?Uw+yx$7DOumJ6?qkhZKC(u0vw)!{1Fy_?hoODq&aj)`)o*L>j^QJ~FBhv7p4bx!b(XWuS( z44BiHQ-|mx71Azb*w0zv7zPdVgA7)lv+&ztr$FuTUq`QnI#@)HG=3_c6<6d*Ytv$M2b~9WxNttz(7SP_3 z^Z<>of~cw;+u>hJ$0-8}zo0OVps4;G6u5reFUdmd^aV13NnsYk+70HiA(`@YFUdR$ zmXRp?P|&zPEEtPgojP@rT355@T&TJ&Q3713O7S@XLc}aS9+xfQC652$ts&Y_@f=5w z7Ce8eh9{m4-98Fb=+5#@RD>^EG%)$~6`FAL15&&!Q|~8$tmAF`mC{aWe`d3+u6prA z=G#s3e#xF!qslDhehH8S=a%Pcgq)B(_fRJ0vdE^;LKKVKqAk#UlbM@~FWMa>z2{M? zsMu`ssi3JzQ|NJXlVbs6t=AZdPAxdbR=Y`m(15SEiw{T)$ivRQ0|CpMfimC(o{1s8 z(~AK-YUw}9%?lXkJR8fJM}8*mmLqyyC=Og7M!T`WplGk7V;!3o$L#LD<_snqbX2EG&4jA|GzHgijXyyOILf*`0iwKwJ&3!7|-f*;hF}mO5Dusd$T5_vCJz?m_JD<$Wrx;9LaQUC?0g0aM6AB zs>g9oGYtj~N|^9T%V?eZ3QixmbNaAU2dJu!m0n~Z0P5fb9DPL@@D$l!?j1m46Ux5- z6`70Q3%}*A49?M2b3VcKM5k#rNVoRcgI$u6_1lA%J4azy!*1>)<*7_A|G=}78lEcG ztUbA49JJ>sb48jnO1L-6FFsl{oo1wJ&k{#dxRXged}v>A`&255?|m45sN7$2t~mgx zdD4ux{Q%MzbpR5Z9bPEuG&}XnP z2d0E{Q`5SADlIELwn=QK{HH3!Es&f5CE)?}yTN5)C<}!${#m!MP0`VjI={aAB{8>L zm(sj)m%_Bx;1HdSj|aBgji{aG8e^zLmMAmy9Rcgt_V*mN9!jz^INHhP41u@0b!Md{ zy3H@qLv75r-gE3`f4q|NypI&QerN$vI8Cax?!fK(~+3?XoMRa?xOBm`mU6IP8WGnRz*i<2r5Nnc+FL#y3fyc(WWfe3DXbY&pWe#`n4#( zp9x0=6#58?xC1TuH&62c<9K!oEXH3Rs+|%#6&yv(ensgM`a(uEgP6?|sYXG!c6p}A z*QEAh7O&ifv*>uQWywLPljH8B|NBUH%}soMBg2~nL6r1)JQUb2I$dCBIlLgHOfhM; zn50GhdFXaXM;z)6S15_iG9Cms;je7P8}+wsFm0-AU;7^DeLhN_8q8H~OJD(I`39=j zMaXB^kLN@+RTV_$#}iRBeXs0!qq@@UG|fQ}-hWn8>9jXBoAKyWuVU_kODOm0F2C@A zNFye-1}i0{WY^ui)xMCkz`f6vOqs_~4hdZ^c5D>c59Qy={DWth$GuptaauBJYg%TJ)C5$3zS zqgN$ZAPxIzXKg+8hK#`r_r}aDx0p>aBHY1|N5gZvVm!b7^-q473pKU`BbTjJWtKR) zuTz>;t?WieQ4m*$*<&2jnfAeKyxM#sY3iI=D%H^$2}Oy9l{w1Y0NLyaE_ z%oeIzPVwB%gUR8{}w6C zhQ7;L|GZC@k5h%UW-3oLo~ZYJVN|pTSx;hy@8Ik4nb_@`i>~X8tFK$uE#_5kO?qqs zd4Or{y1f|F{ztizsFUw3cL}L-cPd3>QqDA6Q5ApjjvA1R=|)T4`H-a5$U|;ie)W$F z?70mFf%RJNb6Qo7t#IDIwY}IS%_hhQ`8dgHL`@0;SSRYg zvkps;Si_lCP&?)MVTZw-@Q6)%d*;tCkrOQU){p0J>lYJufSOgs&At4LQ`YmGiLs}q zalg>pr3S-4Ph{^UJ|Lu@Z6Ts}9AZ=ljcjB~N+&vW0b*V)nVa&;YaZ-Y2J) zOD^}D|9*jj9)3V1(_wYL`+R+C-+eRhw6t6Bh^{ZV4RGQ#te#wj!Wn?nAw$Art?h0jQ3AOeYH49pCE{4z_?mVcm6gV#w`b-3kdEF<$7Xv@iS~8 zqL|Y#U^r9gy5RCE+SJyAHQ$DeUeVpCE0~K|7&21@nk>_2C7onfUy-Qo)yizNAqtvj zm=y+)62{dcv7+2l<}UOfUWai%X-ED4C;&N==9x`VQeI%r)LKW;@)cOfk3I; z%o>913P*)a)t5aOML`F;r>BFnV>oAXHGDGe?$x%l8hK}PRa2A?OgpR3SXje^!LnU0dFR0009C9vQ42uc7(DKs~tos>G0(C z(^XCctz@TA_XFZ@Vs89Rt4!YliA$lKVz>w7%YsoXNO9 zP>xn&i+ zR~$!rMHv@w3bVPge)Ukw#J90>&%3qOeE%j|#pL;;gpg)#CXF=Fn)hEwui1is5=j^@ z{h|_$&g(K(n^aAeVKH!-gJ}R$z9ltkOiaD%CY{`;iD^)&=gbdlpbRA%J~lQDU>rFAj<+I~ zgs(}!MYnF$I{o$YQ{ES?+ncGD}Uvs0@yZVPRb8}61rpkwC8^}`gtXIE$b z6<05WMQ?v5Toyx&%xfga7T{S1Xr=M^G9We`>de85o_&1m?8}vdbwmL0Cdq0SPiCg> zn@#tfSPb71L(;l#JBOcN61yoe4_bpq&K`&m1>-I|&2eh8P4c;EG{9ahx|!$#WU*=F zSX3I~GMvA_|4qzJc23#AC9dq|WKzCN_7voCU8KI^AO0v|bFk{$#nmL>g6vwN8?sq{ z8=>>a-$uxwos!@UMb?j)*x1+x$>tCOX3ZHJk9K9nOJ}D-*_P77f~kapt{=hqA6sZc zGOx*rMFxNi_bfWrKi4dbZ>q?K1VdgoppAgtCo=A4v?^@(p8X%_0IL6?TXwnFqE3tR zz}6AW^t+mG8g-xMV}a3Pu- zbQS+wd3I&~Z_1PQx_3}f)6i%1Br!8H|C@l0u3sE%)wiIa;5lBal#-((^k{c1{|q8` zq0YhrvW>NGo+&)X-4x9G^a+!g$09V6Tv!9%pZir%*_}*of0p}`(%3H*ksMJQ8_kSh zAmc8U-nX$~`x29Ouo)jQgjdqoE~P0~%!lw^Q#lKyzv3?3Fg{5R#{Gu39O`w_b!XNe z>@el|YGqyWJnoApJZmRQv3EcvQtd4b*d8IoJwuhmDyBR28>Dj&hoiF&lM<* zkGiH356BO=PPstgIxdkInZ>r~(|o5BeB9pKzKRn8N8g*Tkf~m*w2d0 z@rx(6`EMW2=?cPZ3&{;ut;uL^u8NX!nJ`y|;>V)nMjuO_1b$wy5PY;a9LzQzY&B0v z=FMkQ!+PsL9o_(#-vvgIEB|)NAs5(E08sQpXx0${R(ckGg>iHEU1%fz*t}1+{4q+; z8TsUoN7!oum-}z0mVB4Ey(WF<5+EzW*zvr^A@!-V<34Mt8%If=_ShnWhk}0)16j*a z2Gf1LVEJ7ylzn{akG{qgeG?#7(@the0c)yRSfo;s5rH1o<$(zH8(=U=dreK}3fmc! zYSK#bCU%VbIZ276Ms_0|?(Xvc1zQ6f{og_?^u>}Cj;z4+IVi>AMJ!Zi$Pru^f~ zPriNGu5{@BIc8V*bVsnRqrhMu*|zzVWwJ+~?_%}8w{K_5M>@<~I&el&ad8Tm(_}*; zRa_$X{i2Ja9~ZY_6Cvs$nP}|TfFEW&>X0okn5}+eps3sCBC8HD#X*eKYd21)$&BxQ zHO-@4{o&+xXJ5uZGinJClJ|x=;4dNh_Wv-jJQdvQ^DG=SLK^N0<15eW>9-_iAcE*` zC_TplUNYNnXJ#hlU%uK&WfzNnekDcB?Vw#IF`uKT_m+1vV90D`*WHH=pzM490wi)3 zdI1zbB1c7AN(>}&ooiw3fK?DZJBir%%ELaE)i^u;jLEjSm+)rVH^jUDUzT^b`2~kjJ-VqpXAOLzev%z=G)4|I_gWZUe<9Fz@>; zKEv(y_5&V*`4+*vwe>;Z6Cuom4VSkKXFkX)L$P!_W+U0o;18Jlmjygb%EAm`p^}?KwkS12!-5)=jbGgk?z{BYDGpulft965k#u4N{J zXZw^$7=_}AH#`~6>|`YgX&Ug@X)5=jqx$qpuFE(U%wd4uX(HCvRHfHYKErw?J;N=> zxj14j<}Ko#e+`|0 za84z1Ef1JUa@#6eTwu6QXx4;n?ni=L0L_r$c6Qerwshl#NaoEG-*^&H58PX1;z=;)fKEE&9GK1*(8w4t9-%ZBbb9z%FZ(xmi)U zMIU4Pas({7ii( z)-E<_D$wAVP3(Ek7FYC8u$+eT0=Xf3F0s`w21r$#NB9+TDj4F~z8JVKwjDMqH)HN- zt$s9Zf$g-B((W=$b5D1yeQdbNMXAt8V=IOMO|}L+PMX0xTczP}XY6b+Z^rG&U#TkR zZ!z&lswx~LR-o^cxTk{V<0V-Y%rp6e^oaaDBv-)jBB=&?;@=-_pUEqYg2OhkY73{U zV3D$6RF3*e+bER{ci}^soY^RPx3X!}Tj3sYqnq3PUn8v@t^$Kxkz4&@qOn?NOZBu@ z(2KMAB>M~w{prX4HGzgI(bA{jfbPuKzMimsrqawmN#oRJKTP5tB|t=aKLd478iW=- zt^PV5EKI$iCxU3T3EAf!1htQlF&|KwA@ldI4SB#%fqOkkNIr?)QjJdmbTQ7aZ+~!s ztSnYk1SY5hFR|$?`jqg`%?Ue)mK_~=&>?_TR%xJ$0>CZJy=$lI+F-e2vyBZI7rPoW zsOUE_T;VOnu=kwrf@?UJ{Y6Etc_;W}B)tQ|ixvy%%S&n|eJXWHG<^y+gc>z4P!n2L zqU|_X1YSFg2aAG+=HPGatD&=dyTgYQqzNpRwu;4&s)spc`Ng{Yu$kp4*8hr<4T6pS z;f3HGR#MmNQh)x3vv7>}Z~uXM2sBk@1%)14@;NQh_##v9%MU9K$; z+{;#%FK`C5kFjyUNZ#a`dJ_DBXTGBQFaMj|IS{<2c*0f&n&_3Y*Lzv=pWaY3&>PB0 z`rR861||>x9!ZV{v z4#8Ag)0GwCwJj#GlnnKpjwqF#iEf6^W{?mKrf{$jUUG;4Nq0gn0|wwsqy&P!(10nq zXGhRhSI~Vf`ZL;fGDqo!a{ebL|J4FQzX;v0dHljAtf=jjkp;k&E}XvjjJL#8a}yI( z_=`Zc!4b!>-!jo`*5mz0w=wW~yW1N`67}mp{)Wx^W&t|JfLxTLYM!G#*9CSi(wbINh!}^!>O)6}nSrP5OqNuG>d8PH4 zH^PWRX$6DB@$o4|mi(7kG75KN`Jd=ClzQ2+VRo!~*`VzNa%58Enpd5{Y$i8yn8bV< zQm{?1bW7#y~l&4fk$5E1K~Jg0qFQMxNv9+JL~Z z{H0{%0SBxz=Y{Yb61uFK3%R|$Ly{+n*G zYObU&DkRZ){tMiZeV*q^$L;m#HtVf<(S;$;3AKu05EGRCvaG zTJj{|;Z?Jt@2s9?V0a?^X>Eflzr8ja$IZt1SrPU5pUs;iDCIc;&`R98rzoJS-6MMu z7Z>;J*RLXv{(kv5{S8D00Redk^L0Z1XN1Vee0l;|@o;!wpO8+SLnUxYzA9Wm z)-;Czqd$;SSXic)A7nkon`Kg| zk$e|D&nQ%;_F|14eeHB@yBmn#?V9B?vH4re&3} zox}8(Cs3)I%0mt;7wW_JV0AO@oX>#VB^Za2k>(ZVs3)55eNcaeOIGBMhUr&GA21h` zl!aYXRy!-z;Oq=j)0@k$0nzbSph#ZoULd2?$uaU;S*66sV~u9D@R!XoZtMv3{%}r< zEKLHMY`y{64NHyH>bT^kUt1S6U7Yk5oV9WeEH_)E8%l{Tn}L?YGgDwh{-3dPP?*Cly*aMp#k*z@ZO05Kcz<# z$>kh=4}T8ZZA6llmrrVwd|eEQPxb8I@8WS<7bMo^S3~#3Nbq|d=C66-v}6|Hwfoj+ z((4o5b(pF9*eaFB0O%Pl58qb9t{q6Azy`O)7zypf)cN_wUcFT|PWE{{%WZuZQQL}PUPd`9a zaA-4gB(x;6HywIG62&{42ZEj_F5dad71ZZ#mG1E@usJ*5^_3QF3l&^`x=aU8{K~w{ zxXUyGeNzV;=sX@0Mo&%A`Qa=J!}|aoUEQ2fYH&FnJ9tZJf5k``p~6d2HARt;K)WHA zGOAk9ho9}9E{gdD_0v}bF(wqh2O@pU%dbCYYA~0DJ0ba+dsJ}96O_q?>fJ8 zXHBPL(!=sAba=EaloyxR?)~m-IByxZHg^xlz^k*Uk)Fko)K*FZcB99?o`2Fc&e2bK zURIjae|EqeSUWIE#5OMXoqVOZue8}((`qoq-(evx~uP?cn>912y7%+;Bs4F}2D5+O! z-$-&%n31gb3^4BFOg^~%=@W^`nQ&%m*5zSccl1N<+$dL!hT|jmF0tCBa`AN(@eX$Y z7KJaut+uO+=iQujKp_I~gT|F~hXpME1Nw&t`qK8AU~K60$= zEyCW7{xsswhQb?er zfGkG%mUhFzR(-GPC-*~yQS&~KL?}VO*;L1tNZCU}$KBgu4C0jsw;IJ`1l_-W6^LSQ zeMoGyjk`&HDdy<{yv>Kk(^WmTLwn!=vv8}IDTB#(1|#juiifIUqqT}<`l-#2_u}Ig z#d9P}SO*0>Liq^313vb^$BBcfouAkbSpFsFXwu5vdS~2Thm0Vnf1+!YteAyBfEe<; zYf85D9`NE=3m0(kB=L($_FQvlB9?vh55A%sOw8=b(gZz6{x`T2kg3JrIEiZijT3(8 zNhj@iBda*JwSACRv}$}uUNDUbixf04%`#M%kept#eX##&9CiY;$r9aeBI_z*VdA?y z1ZDm)m9H4@S{`Tf^$n|9TWS0ImJFAJLOl^XlW18>FmP;wtP>n~cKv1AyA#|a^CY621r$2;^+Du;1I1PRg{@L?oj)`CJO@B^@ zTQ5lYQ>+x$wailMP~5mPrTD0s?JIf}Ss!;-`u#%M7DI$vE4pHP#9ODXD*`{U3ExL{ zLx>06Sq~9!1w+SP$tJW?9u}L5^NY@>7#GqJsx2$C60Q&Hjke)V`NvLnkViOEL>kw2 z=!Eg4T5^anRkO3v&0JyQ;B#Wqb~`@LFxW$=LmOB-?Q`whrN;?5KXR-Suf{;2MTjL! z;?DB{rGVA zPt2l40Qm5z;}U44JO(}C5IJg>z2w0&Y-jN%-s`w<4+fr$iG-Y(WZ!rLU|7>WIIilM z>#Tb?;j8NXbgy4V>=ofw|1|DLcW1B=azG(p@A|ev)Ke+vGz0|S`N=mwP7OB7z5B^( z_B%2?0#Ski9#O~Tx8Vz#&%{0`__7?`IK8vR#dtU=IIw(%bUsBp&`6N)tmdvY?=gjN zJyR@W`U2VvPM)J_UnCYXM!Xn9j2cKnJLoP+H>pg$XTqp*9e@*D`jM}hya@IU-v}pq22jLJ33uN7dyE}A>gJNSM#LLfGJTbO)gs&X z+%t1{6sth^uAgl6z!1@pZXn5Jgtau*JR!-HMZO>3$jr4TqMyu%RoprwD?x$Q-2$Z6 z2_`BN!S8n_9&W%I~Sbc+y+ep3+{pAf^3HjuyDmN4q)d=0H z9YY!v)rEYlR?6DyWOr}iORZ7TCKNaXexO{aKLEPKMIy&W#$@P+bSApyYAFi5W6-Kf zjF`U43v>FG!1>s>Mofk@-c7t!p2ZAF%-gQm)dfTEWclJpvLHxkZrySk-85sn8wFJEz$??WtNis9J zWkvY-)dssF;s=sz{WpTjuSSH^mVC``Iwd!gci*ABKN7K`Nwqp3AP6erK@2E##`yUj zsHje_y78BzXQBbnrWo1{U>$TfPF#zNo#zO$HwJ2LG|T6fDW94@8!^O~p~CglS#SO@ z5i;ZF{gpCq`L@50yKFv0mbI*~y>dP&=2n4+2s4{sePQxq?-gctO;^IWVT2N?6oDti zqx5ClOQO!p@X}2itpV@n0351`Z9zOu^U0*k(RG8&Yo=uBnrtNVlP0<$uUv~bVM6nb zZkAOYRSrGB$ULpaGl=OhldlVIT}k3?TBCwgjIeOObhaQ-`06(LlW2&#U2{Fbh= z4;IkUCi-J&_PjcGo;G8Rw9yERwdDw?8kPJc=I!2(D5E@*|G*xN_mLx6d-_UdO7FV_j+D?j>j@X3L5qhMri)rI7o6%h zLamt~+k~#k`(m7d;7j(+Iw!YpIr`yBPaZ&|!%=4kGzU>+V;?_6l3JC&uBrRp!k}5Tx`U1Ci_L;_2G5LO_Di^?}OVI=~fH)O?M-Ib-KU!_^gwIV=1uD z_3%q!2rj|0^1GKUiJqt5)VsF~>{~KX-iYEKIKL7j;r)>IThU9s% zGW0w3+zayBD0j#|No^b))EQP!FgV*c%ImSLGd(uSWoSNPG-!4*WH=W5PBu8GG`hN5 zmY_{C8GO0nfH&YaXuNnV!&n?$!dX|bz zofh^nt9uF)M|pHo6Mhl)Q`h!fM~gb$btTfF;3Q5X77y)^h7tGLkkMTf<8o^Sk#4J_ z&^~a8!%jvMLOX99_q{OJja-r7HH8_M!H74 zP=^a(q*bR%>?xyOnA%l_cU$9}51Pe2D|a+7?{$bDZL zEQnrHFb!n!cZ{!nX@46|zgqYDGv!R*vx8{XK4dC;XGRr7zwk$!I-6nV4-tbMPU@7! z%9%cC303!b2MV5}mT-Mqb)Pu@eg1ristvLI#9&kOQKgStDN&{WQAL2TB;gSnemzAd zT&xr;`12^675@jW5!tM~g>i2Lue;9Vpb0O2O4EqWlcCqv8SfJ0-daYW8mWn(#c6Sj zI-Cp$Wbi@*P3$nUNicr2@BThD-ROUunyGmwHtDNuvTbOZWSj$K@D)czUq9_(IpYlx z0Y+r)!@Z8sa|xS+qSl+FC?XkBJtps?cIng~Z}A58IYVa^ob}ckNV!61Nwm~F*@W(o z4Nhw&xCMK`?0ITrJ^2ow1m5!k*E zA)GyHUGxDxMFIanIw(lluIX|`zP65Xv2tNUxJ_ol{9EtX1rCygosBiTxcE$B+I#5u zJjs+Xsay9XyW4QQf!4d+3@Yhkjq@^gd|^S_5FL^JNuAqWK+CD4i1?0EuZ~vBoHaiS zAd#E5CzcF^UFlmh)5m-ockr)F2(xy;&X!j`~gWZCad=<5(hd?w-{R z%|e?3X_Pk=WTpm_WM!)s&(q~bHpOm2$E=+TOmM61(DiN~A^fWVl>bx!h7SAv)>cl* zh?#bKEf*TeY7^0i1Pk&>S0G}yKJ%xBi$6Qoxk1UHn;*m!Ei*-R_Tq!dJE*`NxwbPL z;BQpmenb7BaOZ!PnZO6*44{P?d`JWGLHE1`oOHLq-ymQ+BlP4IBkQ8iw_}{L)pxpG zxhEhIzHMAsg@J*NrirfBbA=s@veizmPU;r+Xl4FW3gnA14R^iOegNT8JiJYK0K#ei zgK#)JVr2h%ON=S{1yhDX!fLN3-mE+a$;F7Cx4u2!K~AlMK7BT-yP5|luFBQ^&Hh~w z{dYc#$ow%|5U~B;n%?&H)&P(dVZc?94}sMs{d08}1>`tx_OuG)XmgV=E8KpWrzD~3 zo8>o-y}|jm?)Ea+QWpk;sSBm#4&9teDiutm2aAu_)7k#wK==)5=|3Rt`DL zjhWZyChK`IB@>DI{CY&MfCQ{%*|1|a#y~S+hqGI znKE9_R3+}~IJ4>35`$$MQ;f}x3YHnxz{^nO1c1^b*H<<44zRLgiAq{X2i>yiEMvx% z_LW_*fA0b?ERt;?x4s8+*H3}lsU@apF&!VV__gZC6uw@DKCe4%1e6ahTkUT#fj;TR zARsjw%RCa)_J3L<__}D^LYT;a@;UH#`J@HbBI>z^G0uYXlIV7>9ffg>QUYd0kFJ~^ zfH8uC1SUICT$&XgHIb*s2f&*~lbTGr0s7=*(mRXz4`Olu(5!~Pf<3Lx_6xTwd-|W9 zj)rDs4sxyb`<5XLfH#QOMw4PT=Ay;KhkSQtju?17xvMb~_FW^`A?`wFVeduoC7U%`v@`s(M@;_=;xkgU}yyRQ0NHp$T* zaIt4jr)U^|mX>#X1JXF#(sW*P!tK)eHJ)Nr=0vYE z+`ch@OtsrhBzEAom>WeB&tnv01S4t`>6 zBbbV14_YaC!v15!C1V(G+V3uKC53ijLz|n7c;Y?<idb@IRA>!pBZ1k3>P9z|4 z11!pY;`RWja$M(Ri?7eUmTV?u7f36iya?Z#_`5mggpeaf>|7ce<#2Ad7|4g90*?a< zPnoALH-W(J7GCZ1Xc8ppCsw_c>xg(g@>j4L?LN176;PI=icL)I%lA-s(mNTvB9}+| zkS%F|KFL)1x_b=}BnY>mgi*S9yL7bQ4JA-aMc2=ryN+SNmq(5IHZ&KEp{OF>B3u^% zFSQ%gRir*IK)&Yk&8MvUB~-s9U0d9hR9+&a01Zqnb;!l;X>UTF9i;yo5D?g`%_O^S#^rri>5d? z+r$@swx_ZExmNFPx!Jxeavh}QzrNkesS*^1#qG@gon$ta=}uwQNHQQ_&j88r7j3{Y zzqyYy3!-9yV#)VoS{(cI)VEPCoi?!AsmiR&3zpUFH19kU4&A`-*7N;on#ukD6lbo! z`%t)w5%a**E6P`w+!KlKPYm1;idD4NCNNG<*R`5^vzjxCT)+lfPo-e;FSI_*1BzHn z-%6J5F9!oivDmqmU3S~aii)ZU@q1CQ<6OJ&vbOOA+nI8jTGt&@j_NZwg2YqphX3qA zE4A*%!_|{-_@ZYAEueeg`-wUCZiWK~@oRR|3|qJljHClT-c_K4r!y@d&@MMDr_*J z&(<$rf#kaEP$$x0Z03+AY+qy+PVsBqa)8++nKYm<_Vgt$gn$@r-a_<@Twnyw;TuzrW45YeojeH^2#FNfQlZ}Nqz({MUrYLV zoO!EXt-d3NdZC`=RVqll9LGIndYTK!RuuXY zZ?f;RmEBIwLhXN-cBR0W63{{NBQ?UnFM&*w?lq?Dxpm5qi)L9~?Wn*e?C{B-NriwD zuGNX?-LAJXi_>+rfCQs{-LbP5!B>Ao!C4CrsHK7BI)lKUxc6!rz?g2s2EfthntBb= z{ZrV>%I*fUTa2xgA(IdOSdgp{lX|%Dz6JyO!5R)q4-uM*UoH2QoG8WfAH}v)r&`DT z+%rSRk1Cy*B~zSI{WCEG#2(&$%ICNpBXatlZwnY9d>y@Q`QI_r5cVG>ea~$p!phi9 zrY`&pmH2YyMdV)p&liziP&Y2X>Rv~0cl0#3_JI{G8T+BE44E&)i3|myXKAN6fSt~> z-SM)&ukD*)FciHt^RKL8Vx5GotYjoa>=X?P6Y;ueGVu@fzjIo+n=X%e}HS`Q7_hN2zkmlfbShN{yX^o z3t~C7hROo}AA4^d7UkNt4-+ENpdf;TL5RdCDLI4+(t?0AB1nUTGz^G?@f-Q79Fz)&*`ziW*B?EU_pw~p^PzVDy!ANStJ-W%@winY#lp66P3{fgk=1XI45 zde)KUfsgC^GMN%Y-CLC|MZFA&1fo9szddDmJ4|;RuB%+3==E(9LSV=S#-S#iz4K4( zzB7=2pWo@U=EBtLrswAKHFS3GNMG&jcD5R1XT8wL@4X~k)sVemDCQD5Eu}p#AksZ( z`v(lwj(J$IcjF%xN3GeZjDlr%`ui1r&M*Cx@Zd|7bSkAk>Kb*>ek8+z!#mQ|1|l35 zzJk5rBI&Dk^|y3G*v((CHs03Ksz_tQzwIW1n&9|vZ*|$hwnRWo)0g`5vvmp;!)vW8 zPeHWA34g-@frPknKZo5rzD2a!`keWhK;akSl%#}=mUx2QH<#Y~Nb0XY$Cy%e{us*U1 zoubnt(H{v<6s83pLD`-8&YEJJ_bC(qUE>$y5af^4z&=$=I$MwkWF#L z>H~(k+wtDIp;H*CH)L>4w;BVejHJY3h;UqIR?j+T=WfXz7a#;ZgFE)MpHb2mk&R>j zp?0%Up0kq#&2-Z)bGW$wXPvrSXDWW6#+gL2of)H(Cj0^yR8pXP{i{wr#*^6eAN#3=;Y< z+HP#tl_OZX`0a|NjYy?1?O(Yn=5M)5=T!A=lgAOwVH%)jLiUhj#ck=#*Kr&}Wul2c zyq}kN>v|(v;_aTHX6&5UJ8CWee6HLqOo*G3B^0PgrB;hW@3AVk9Ja}CYM!R}1*^@e5?Pn1_AQBqQQrtU3Lq^ z93AStqR4~YVs(BfU;EY3{{5Z%8%{sP;+p)i*C^}idi{U}Z>(?poCDA+yQLneH-$IyeES9m4;<{ve;O zma11~f)AXKlb9FOn`M3}$X&$y6TMSt{uaFf-Way7?v`T#b$Zc#zd|MiN@V_M2n=R~ z+vaC;=l=!$bpez67xZs+ChOT=d2NZlm)~#p2)pE`a~>{GDDeXWeJ=6?y7Fb<^xrVDQB%?m{C#Sm+9xkGP=gUc?)Yz zp|S++q@SRmi`q5>ctLd17!$Nf0FqzBmPr7~m%W6}`DWnM{ZGs?#hyCZfdiLaj&=Ga% z7d|CR91O{QrfK$*U*hxEy$;mYA@jgzLwMj;R{f$vKB;YIhVW#srR#AYsw0DSZ{Y*j zQt*&okJSg7)O#C9PWw%`zv?kiEaKl&*BnE}iw9qUz~+VvJ2Rwg zKa&3pI2saui^AB)90rdW|MgeOupP;{p*nwYU*HV6B>~)$7e9`bCB}8W*VO&m6sx~)iVt+NVoSqN zU_!}@sd3e4dMbPp)aU8Is5PvP{ee&0`rh&D^mbw4qWgWb#laADQ?Upi3zBSn2^kfPC zeD}QN6mjp(tBb6PR^VoRDZp|1 zm;SPbnPbD+Pixr3h~?4?Q210nvlRu&)Yj_u?(r|BDbM#0l369i{y-EgdoT}YMn%AX z>^dW$uSLK9g3nKis|p9duaRzN`W=4(x6_+}rxQrhsf{?lj4wE_v8>yidcBz^Tsgm8S%XpsQ1`a!pJcx-;?Pb%eA$glBV<-`Y$)0 z&#s^9u@Xk6N_yO+e6y!^i`sTII5DMiq7OOV-oYsDeEkFpL)*{aL?3#&JQR$&Ezu2U z-y;v{OMdE7b;4`;Z}Lyy3@U?B{8sQrd4 z!55l>3M5?ZKHaVreQZTdM(_cL+6PY+huEnK&#&qFUM38s{glpoDI`DBf0S;1_cYJWQjS@?*5+YjeJ}1Vg8$V#2M>Cpa2jH;D-M0UuTWj|^7zFJqCE^l zb<$*bO5j$jKQKsFz`W?tH=xg7W>)f4r^ak1-2E!kstr7DF2pkYk%&TeymVUK4~^-l z6weL~WNv0raIgXpo0Pn~dfn^32`n!;xFy7wy+6p%xZo37|tO>P2Rxuym zKF&kcb>Hq*X2*=3*yZT>cgC4zg?C4F4_~D))x29}R9>@^zV~AqrK;go_Uy2o*^?}{ znYMj>m9%!@;L6Fn%eEr-r%w{Vw+GU=p@w&0zI8iDyVAu{jjAkHPk+P_9vSJLR!5(6 z>+;eP$0btLI}>zXgL*u&3JRSQfvfZN2S2*Yoh>#72Q}o``uWAgMtI^LAw@FGE}AEa z+AzTiET^2?3N&x@=#+<@rC^vZ^#9??lfUCP8l|1#Qomd~KzsR?<+oNQuUD8A@p>ty z&8C6}{ZtBd1|8C#>GLsY|@v>bv~Dk>EB(rOv)yR59N_@VWZ5mV;~@O?uomL7>H z1;v~pBPY+=2~w^wU(LRFXy?r}B=H4>w9H}r#4mVQ^T<*lP+9` z{Xwa97h_2lb`Sb=y%Alec4rx!>hX3vEsp2<<0fENwd@I|VZ#Di_7|1y{qTbBzQk-? zhe>(WPEwP<_%`l4I6%+cI=qq#MLt^#5y3a`2%V5lBeXML%|I_LpJrH|FnO;~B~;u| zZYuoY`h@MJVi)y_hxY=sKM}S>o4r;nAd?wzT!l)mUMU;5o4iRVA%ne4A0$aQwvX6( z)UL(Yq|ytN-&|j}iq=z#oZezrp)sni6d0|ui}Rfyo<5+jI!LajnRudx2+9oYub%RT zk~O>E=(*b9B35|+yu^}CJ$jc-?JDHgt&{Z4;M^XmhXo}c@AsE@Wbu6)zpMJ{n72CD zWMi#N)HYRX$}4U(sEv$Z?5?k<;YmhpUF7H8?%wqNA5nAjJ9oS`d{tBpqo?!Fh|X`F zSVm6g=j8ONOrBn#4!x^WMfR{VTWbbZdm6@Zelx@T6W^xQt7C@>Eo`>(V;R1Qv)x*% z6!=E;1`fbzVNLMaH6VPI3QdFWALu7@QvzN}Kt>t+_6^D$INN5K(bIJfAQM@Dl&1(; z2*8F7Nnf}+4oGf8Tc|FVo}GM&`%U;sJ1-v&08Ho{cBLFeb4g;aPl=WaRIL5W=5|(Y zVGr1~ZX`V}Td&|3df*sk2JbGy2fw~M;%Gs6 zgV7s(hedvRZdx<1t!Qu#=f^T+TCfjLOoyf$aU?-*is(EnH2RP@d3pblqMiRMhtzt? zway$<@W(%wr+eR`Rvk;xsx$rm{Xe7bcqR^b%-r9P*$@Jr5CCX>ZL?R$RccYrVtCuj z9hwB-iIzL6@nrR~!GAq0$ar(}1Na+&9m@l=qh@w8_m@Cl=UA!q?~UTbZj|naqeXy( z`!D^Eew`0g%d#(27W>8k)cm_Q2D9P*Oq_XDHY*ZT*TB$6{!0O51fUZGM;cj;jIh0^uxfGx|b z1UMM$v#>HjCF_m*Z$kCzkNj zeE!h9n+0(xOSH%>CqvI3jA+%A&fzD9u905O9~=&H>8LRlVuW!FvLtDz`KJrGA254I z=hd&4^|5#`1Dg*s>RCf5il#zm+wA!j$+T*jngELfxz}tEb_+|(m<*o$zB#oFj<1KF z;{1Y^F~Z*#FA`gQnRuXKtL7!h0S;P_&SnR;)(flN92v6${F-wvcB7se&&fHWUHOaT_f3JArdpHX{5oafu&Tu8Gk%*gtGDN~g%T*(b(j0b%9J zxcK0aW)?IBD(OL?r{F=sls?j9OMIruvdfe&TAuYr{Nq$CWi|KpfsVnuQizp*(>Ac! z^gr&MpXzw6BWYM z%Q?pV$aiWSdN;xx>LzAj;Dt6~kECTh@{HBginTCxOP=ct-WT@w3{}e}l4sS*3eShP z1bI2M|FE5M9cMSR$~tU@EDa^`M>`eWRo;OHDbdJLS7)Agg9%S zVAn(f$xN88E;;gN7%r&?i|~Ey&wU5@vTm}K0g?vW3s72}4JSCe;+JX^HdXrtKDT$z zd%8exXQH`}tA+OcRlHZn(pmX$gu**7BoJYocJ6-O+_d$EUP?=i5Vz!h2^EX5+M=Q# zcOgTPk*nXLyasK8;$c4ZKakJM>lrw-NT_m+jiuY~Rq%{xOo>+joZsgfTpHqMTa>t< zN1)Rhybl#Ypk981=C*7lsjgvho>db~e8r2n(oEU>$${yu%m{s{F#(2;v#D3QzGs-b zvwbe>?4(=t%2a)Bl>=ul;upl7VFRg+@lde71zf|Z z7_8Njq%!pz8*Qd(QlaQzq(UZhd-mTlcb_AJ+~06 z{vm!wv9GqRSAoxB7L#j9`bldec!9KDe&1b4k%78FBli8l3rzsSFXb7>veN?9zEug#INc2-rv%~&z&V8vlByV+UM3`f>6A*%n7IBy{lSmbav;(k=CsPhe{@5f3YuO`gx?c6ymEkqN6Jx=2g5OUTk-se>uRc zrqA=U*U2W40Mq70w-yrFP5U9D@TA158<#i`WsQp>l%;Or%sXZY%R%6GnD(PRx%@rf`dVRAtm{|*GXL0lUWJ^7D{ji8Mn`D+af8xPqdrXRq}eF! zys>uNWl|l%5lODBj?JaIT+;(>XYh4hn*xRdmtetnFGHj0zEM)i>|*H#)8FazXB-@n zz{bJ9z*rK11t~+;rmv1IyPPeYa=~pT-567RvrclRIc z$|tFIdl~DwQp9HYS)2}k4wu@wqmU<5Sf0&J$xuXgSw&oi_J^2)g*VJ~bAj)EBmgJj zAmsSnk#oq!(``%Ueo465EpZ2mOz}oI;HZHuCXy#S4S3GUeBXSy$n(b*{^CDc4>M%`Ej4+j=r1b z^Yx^4M%fr^#KotZ@LF|=Es7E2%;kp17{JD#ANX$}vf6nkhKVpH%^@s%p2@=%309@>4ywDidpoXle^$^Lv-l z*po$+x7<-Dq+$RbJu}_@UE_af*avv8RnZa4UtSfWUO5AfM9DKwVDtQQ2HtLrsys!7 z^|YZ`m;jG(vcwA6`Twe`t@|9!zu+am|<7pLt_dKcYXJJ)vI2b;#%Ijh$7 zv<*J-bu(*OU8}p>4Mo22_F>=H%6S2(>RkCsQ7ot&|0mF0aorANH#F(jJXNv2(s^;N zop$4Bpr~=A^X9lQ9(_HHMIlpZc*plz*-U01t=jE%`VHZItd2a@TAqi{C=&1@V2H6joN66b}f*-K(S zOSNCXXXGLF!nafWhS^CU8qgT6I|k3}TzBsV6&Bt`+BWWdifmrYtdomhC1I8Q?n1Rj zl3MqDi{9luE+`_L*Nm^VKgk=3VK3>V0#?et$cj)kf}6fjF$o0&h#&FBYHKT>62V?X z;}XW33B4$AiP|SMa96HMV1;Y8)S>bvz0(kW1Mgv?+^2z!?!m#5vDn!97lvmGWKwYg zRS>&Fc5xg%Vc30is^qoN==feFn+opq*`#zax(x$usu{U+9e8&B$<$c0UD?_s_yqY` z%q@G5ewT@Rjn5%|&~i>op>MObd)J!t3-U+eqA&Y*>N z;H>~#*nhF6TFZVTsT&DHVwf{O@2K;)6)%cD0XB1@+_zQ$Nq^7>*%I_}HSZT$=h&J$ zT3qJ(Mj2}Ar@bFOa?coFdtX&bDS%S6t*9xlr18BpPp3-|Jl*D2VQuqL)PqY!TgWwD zRzdN=Les?Vn?m_DWZH{tmDc;VpEn?!`fT29uOdMiHlG5V+E3emvSa|LSw~@E*ek3* zjn+ZNS6OwYakKV*+);gYcMh6*m#(mN&eWshAM6*of#VsIJ}2b^rAe2tIqC5JiRvKVz{msO=qp*7kpD zYY{0d3ozPo4mp?r?_1FkxPTURf_FK=L(a3BloKBbIM3V=JyXE;q|+ovEfe=V_g&4M zNNZWrCxN(g{2%zmj?v#)6BgPh;vmVXcSNZUJ7}2%BhRGPF)V=TRS|&-XQNGHGhL6UC}qp~`#YUh``+uYvn?@{ zzG=C}FDNMIvEL@r+@tcklYE$*X~yX5wy-W^FJ=C!L->)8+>cKMq%sr*h5I# zE5;qIt*`yjkv%12H#0e7u$fmJiEMSfTwY&1lypDf81GO$du=?0BDKzPn1KmqIZQ14 z6Jmz+GXK;5P(M%U*Cuxk>*H93b8A~-kRVp z4NJcVZnd>|m{-xXAB^K;$ui&&w;}o0w{1{Fhu6#EOao0vNvIC|XLpvrcpKor^5g~I zpTQP#K6v9;DuXZLHv{tF0=TgK3AsRC5g1SJEwyAjbnFZ^c;GUH)FBRV|6C4xH+&VP zU-mX_Vb`s+!$anXE4yZWC|j{0&gC`lR9f7?8gDLF*RF^xa3AbZz>cmaJ?aO_i=QG^ z%9M(cQnoO`IVG^^v3h4JMHI` zBBqPpbZ$R~GD_4Wk~5BVjXxBaVF)_o)*#%#r*}fr9renjkC-)9F~fCRxT#prG-fo< zLSW^ss$nRiRKvKqff?6(YWq4=#vmmn0KhOH+B58DujtD*Fl#Rg8P!tGYX7Kd^V4CP zI&S2qu&Xsu%jOpnZ(P6Ek)ghqPjN!P&3y;o;dr8t8{5@C=E4lHr_Im8`Y-H>gXVY% zy@%@TEDH~VL#z<~)(WU_-EqaUq0MKDtv~YX*VPz}8yN=^U3z2@0k+YDQH1}C_3ilW ztAdYN&Z|Co$C3 zml%b4GltS+53+b$kTp5p^>$Vd6D7=e)iN~lNF}Dip5YR{-t z2+2cFIJXuF?I@03)>4Cs3a#ojki{w571^?)Hh?+csq)@Jc&E`A?;&Ee=#xYCMEM?_ zt17S~_N|Q3&I@Z|QGjoIm$f)mgRS_17w?}HKNXq^r7->k+`j!dQ$Jy)I?63*EVs$1 zjJi5W8JRO;fmL=}_N!;in9-b#2AlqCvWw@aQB{kZQNSX6r1qJsT?Cq9a+xdLy5R0L zaB$w63ozgxs{6^c;r-sykJ$S+`se;J(p0RcciUFwEZ6V%DW%vQ`7%%L3EQI;5+&?T zp0`qve3Kv6}!%;;>@ki0mgSF&=m3dU^WPC+Qktfa$xcV1oH$_o-(+= zyB)(Pw`%tZr#?=RIJ|O-)PRgV^x2=t35&BOIPA@q_>sT%Igk;jT@>{>*lab-8%67+ zrlbqlOF9|gOW#->`Oxpt9Ns?*QwVCxrG>yMfKxaS-LYFHNelVW zM*`9NbbKL%P8Iu+teIeDO!W%(~(xVw_a>LFEC#afD6fJ!pwSBl9=E zEGvD4;~kNy(N^JB7~U9=&##53`vw2M$FCzGNlFu&+{{ImPv6hR|*58_n7WK z;pa~zBX3iH!F(tREFmkaq^PVMG@*QOg`u#%neF`c^3gH`A(B_-P}9fDKOj4-U+(gB zGPCC7^$BZg%ljJA_5-3l+fi=<@_g%2BV$+)V_8uoaOOPNy9vP5BdT-lzfd3{xQzv6w@GmfUG{A0pC1G@W2olTKu--aQV zErh|u*3;HY=wx)?p~bP^>5lrMa;_`sjLJQWnVnWCdX~z^xa`|zCM?p`F0yiEc$hZk zOJSL^Hl>774skrN3a&$`GAmNd_nzYJ&|UZBWRFd}(Xsw8=dnGr-t} zV~{;8?WF(B)f`vI$H$i%$DB{hDxs(>?fP`8ZAqb|%u{gk!WW*uYq@>~eD7f}e*@Ba zRq2VAY?p3IE|Y z(*UfR=mpIDM6WV(s3?A>yLNi_>7KY9TG-p}U8WOTlR>;2@1ypQtv2%HHe)d^{_cZB z68T#9nurch`0Zz@PcZrQ8tj85SQpHNUj@bR%^83xdsHAz&?jeMdmoBvzQ|z?AL;Bd zLwvnWuDE)7YT`n>#nd?u9VHu~LKDv=-Hr=CFc(WLD}KX+v9W_QW0~nJF~zmbJ_eQC z!bHIq!u8I}UzPdnq|@cVGpM!+)9&Oxc1bIblr_jW<2Uz^vjH`^H)V(-s&bljs_bV3 zx{ZBwrpo^*Z&6s>6!^k_Pl+IQf&r6tkRcI`ev#yKs&ZAkcyraq11o_EI9RCfU?hL{PB}Zm-Jo2s$7bOzYAn@LV{}%gGzDs z<4VgyUQ?T@hG1x~MXWe_oi!0zBy@f`{Kq*YXx~5fwkWsZYfP!$La8`SQ(#yE4~<)K z(c}+@`tE&GHp$(?sLGO#?qR}ZJl$Na?+*d;aE0P@xjW9=bPyaH^7DU-UQ|N6O-)zH z4@~`!5vbam-{UG0ToP<6Q3%dj0Tp#nINa8E~MC#KP=(@6En}M6qr-L}3wJ1Y!&0 z?T$RC(RP#ATtoD@Yjv|NvZ#gT#7r@FybH@o-=eTcnug{%t~eV&RNkji)E{UGdYP3{ zfQ(O#2aYv)q+Yhqg~JQMN3I2<`Dfp!J%m1W|*a`JM%{Tcfh@9 zHjW)S_BSwtW}wgmGy(M8k+6b3Db9<}Ufk(y&ZfGyz0I~9ci6%3|LVi>0r|bw`dvXJ z4#cId|MrFLe!6&m#{g8ExtCb?}3bARvkZ7MKz;B5!SUw zMyd|Y_cdzE`)UL6rqPG3E4YfWN;5%JrV!NZYD62?FJu;t<|LG`rU>A$Yz>$y%1%{x zR;a9^nylSpfPQkLv`W#jKW;@Gwl7g#qvH$!Mi%IIlVV`0)by2~3A>AUbHBF6U2w>U z9%mCywyUZ4LyO5N7HWVum6O+z@Dc3!>c%LCh~qOg8vyoZXT3!{U4`<~EJN<%Q3pM4 zczA2YWk5Ja-Yz;mF!0|2x#w8)1VYpoY z3{4u~ipl1~Lw~M4`j7h9z3tYb`kH}pPUm+(B{PN*y`49OM8sUAyHN!8FsP4ZrW;fy zmz_tVuOIT=>QM+J4+5L{#LMgI_}L39Z59#Ru|z&N`N$n1$EpQpcevxgYFn^pw>3YS3dDy9K$wBkuct4Yuq)%27PzbTEs z!YcY@szEhztF0G@zT=6_K@3PY0KO|SdL;X_~)?}{C`!}d5%@I96Ie9JKCZi1$!8tKfZ(Mi;KCB^SMBH3li#x2HFW5s5#howqO&sO^Bd54RW z-8LzFnEdK)Q3sba4R23Qr=Io$b+KfP^$z)1UOFGBa#gvm&h>_rI^V0_&%TMv(&gQ^eWr}({Ld8l z`HSLwKinqXMak?OUMD}ZcWt@jW2>4;_H0V-jLPTvkoKK^r={j7r~QHQ(e-X(aTm1x zNsNFEXaR2fbAV~Qc3OZ)XM#x4d-i&oLIHlhcLz9vrBPtS^88#LXFU6__vdZSB3_8Hn zxP!J2#XP!at8#e>L?Yw!gp*BgP6axUUe9fuS7GfP-*yG>*1eT!%VzICGAU3McJ@dr zH_pKOwV_L=!sj>{Zvq*r#V=s!2WyyE4gmr{k>e-at)_;WzOP=5a3|#^>4fyR6nFQx zICk&W(r4PCTcPUL>R2-AxN%@_7aVM!VRZ$@f9MK8$UTR7EcjSx+=!Q`DeVda37Tl< zcKng)s9_NeO5yljQP(pCn_D}c2}UT+yrH9`ilX?NrH(fi%HjE`sW-+AjDbq@t^o?P zMbE%8l!=wnA<(*tBxO0u8nJP|z&h5Wb9Dz*z2VXcV_sI=C4PDVaP0IPx;7FuOPOwF#L85tEB7w zDa`=`Q!{0YIC6J|Yfd>E7gAD)5k4sMu5Ta_uW>FWjIt#dTG+L!;okoKgMpiy)lF#! zr(p3&wq}RrJTK*Y`vG~MkIx@VdmIhkEQpBa!0)cD_2%hL>xgn7ur6bI^k)_Ty4SRW zREtC5?ECZySFtYqE`r5XSxFYU=*>B38Af%0=M=S7^=+zE*@Hus0)*0b3WE`A(eq4|9-l(FYx>|f^sfaauJ5bl%U&ve$^ z0h8ru>$WqwKoPLohAi)oxNMv0oBFVtxKb~)^)Bc!b>7W=VZfE z5`x%NWgwFCCd0I8Lcm9Se4sjE0bo!i|088j?cf`#MCgdMcAui;Ne;j7g`#yl{TImoy9LHpU|V)dnE2MC8`~(Ex_1Jj zvELn2fKTH`-8OX+^n^b>l1K!(VV|ZD0Ql_6KW0Es5T4t+zWtK)42kvi*}CR+WtO>V z9B(c=DBkUJ10T%1iC3cJOa?bp;`MLhH;ob0(xYFfDs=Q^M?;cfIlNw$hTXO_-OwTC zyF}S0ZuC0(+_4-ewb&fqZ7!e|+&MS>>;cjT8kkode@qxav+cMr!sz5PiCn#xrK_}I zNvySLqqNasCuOd$&FKq`q@AIl62q)icUWTy&)S8p)5TG|Nw02wr*U}skoUD;uA&bO zWkCM~cgFmj>~SRTsFd|BqQHsqb@kbLt}W4v&w0W_a*P9mx>#pJTQEA>?G7m#(%lzj ze1!$RG_1zL%|?y!E+O~NJ{m2`uRW0)En4AiAukrI)^UEf5ex^sHmKs%>Ky1BQaZ-2 z5d6L=n%J9i{I8p$N`pQ+?i4i4`2HNCP0l!?i}-wOtJQwSq#YjQ4N_^`&1BbKWzWqF z1!*y+%rNn7qP1XW^*fK>&uq9xc3uGJss`68$D8vd^vE;RVe~DY$HiqH$GhP)D9fb` zc5`4*0!4F42cPJRUgd04Y&@0!TRb)C870;?aXTeCQ)5{$O1#TD<->JDuFPHV^WvzP zl0Pa^I}?bR#Z<&~;96zgle-U2+J>BU4TVi%Q-Ko0xbl{7F2NoLf;$ z9q>=GFOGBO3V_LEvq@c_ZgiRC8DM=(=(~6`L*48@tgDGMyZ{bYC5R(Y~o= zb_N*PcH5_6AlE#IpJ^_SM7 z!fcPNkDnYcLeeFzdnCFNF`R4dDFslwN!-I?4XSm%h7?3q$#^e_Wbr+xZvZEGu>|Yli|_ds?evE7cSt)Vs2>{OXf-$2FX4a<5|TUdK?~7 zS~^FoGh55=+En?d&eFJDvRtFASt1%5pg97G_qR-^=|{SBYCen8$y?-(+gA%!jssl6 zM`<%IaB+qis$;}48y~&UGj&TN@I+~^c0V04Oxu^IPO-a(*C{sD!k|_g95>=R)r?Ie zE-q(SSfWcNDd&>y*7~%8ODnYR^O+Xf<tG zG9Ga!a1m}++Ntk1tUbdwQ@;k2%*xMw^Ed^FeZ%P!4?(B+qsH4mQ z6KhIB!>-?U+a?$=(Z&bSW71=S)nHq#Av@dZC5>-Rpd7Es)P+xp0zb;l3@sHvp9w}vYgwgyC48JtG1TS77hrn(;>V+kBuoh%E zRzY$pyN3M(7$4VhVKoxa+SN>0i2Eg+HRg$~Eh^bIX3}wu@IoqIZx)@`Oneb{PYiUw zN!J)`WxT#}k(vcEO44z=p}Y>^wm2&bv+%S4BVySso;d3;^NwL#B>ytKD9?mhLB2@0 zfby3zgN$MrA5(L|^! z3AemXGWzj?Tx^ePkoXNL|K(nkJR=!=$O*VFR;GrO>_{?pDXb;>LxUi2ceg5<{QHVd zpW<+y;>_x)`vLx(=}Y$-iL*YQHS2}hACd^#$|kODQ6Mcdv*XoCiFZ&<0tV2No$1US zM|Swa*ZY?W7(C8bC17U`TK)+}e|e;{2+@^Wfv`w&Ll+VPAdI}-dw6BQDvo8!Yax}p zInTL2vyr)-r68;V6{TTxrMAG2Yt1-Jt5PrFM~ZuF9m2nRg~j5v6f)emRB z+!x|ADpxLjG=!Z_f4SyA%#pLJ3!PfKH$}Im!OS_SGv!q!BuX|zIyt${vPEfn)Q0ko z;Ys$II_~j-VMiAMVDyuE0I49)M&!|il&^b3iV$W%bmdsr;A7Tg9uGw>Uu`z6ujd(b zL}2-bo&K49Dq}6Cne-VI+Uix^<(NW!_qoonKC{#PCS8?W6o^k%tWGVo0Gidt|3_K? z2ljqb3cTUk6|!GB2-tppvYK}P+EmclYKJapXk_Pl8>D}-5-{h(62;cy$96v z=2oGiBpmdg1W?;i`nU<3JONsDSCLm3ySxXL>+jH|k<35cUv?1PaeXhb-@u*)8B>}y z+BuP{tbscK>b|<<)9#A_5)67j#Z$BP^7kc#*k`CLfwxu`t()27)ydR_2$z7O*!v_A zs+$s_SLSxz2X-ha{# z?#QXj@a_vicf>ub9L!{l6YBO%A;r6K<8GyPyS=SdyouyB&Wq(jUiY)|l^jNxaWErl zk3eQn*SKc#)4EXd7I5qU2u;2PYds)E|EDmdgT$li<-1cjg67SA?2 zb;h*E%tL~MPyKZA>HRlF$K9aU*5xp%&@*_rPy&41#YZ!K^+$Y8z)6>` zy*38MR$O8JLwNv(rEFEzo)GIWpE};3PSrKolkd2dn>ZWjlEH$G^)}aXf36U-*Hlf7 z-a&a$Z=>3mcOBm42Ld(>gCZ}julfW|A4G3Th9Yd*gx0gfF1&7Qh6=Sya2CcS=J*v; z^FmfSvHd%K=3=S;pi=v3--W5YBUCKRtsiv$CN2~}anOb7Sydp+e!1HQLI%LIbO6tS zcdeoWoAy7xQ~EO$fyc+W&|^Y^{e@kEjz!@Ttp_HQj7!;lC>b}itR}Bj_uZ=qclQx{ zpS$N*KSD8+<-Lci9b0pIZl$0wxO0Qx-`%TIMDS^pZQU+S^iwS_ywlg!b$$1k#$v=U za&@^+>K5Lm_=1^4vIce(UP##K^^?igX-MGZDjc*$)_ZOga7Lmu)@h)T_~1 zi_36&JLJbyW;dU>G-rjNNs!G+*Urlx8{0|Voj#mfUP)EK6k<980Byn#rtDy@VL1h? zPWbo64}c8&#lbIo!1(D`@W-JZF~EApR{wPDz_S|sCOv*eiQ`D6lt98-AV8vK+w*Xg zTxc6#HUzbDh+%Sihri9+ACKUte5f-Q^7lgW%w0GS~B1NlpPxS6do~jRXQ1U}!Ob!hqTAB4mlwPW|L~F0~l!7{K z_vp0t7CAWlKc~h2-vMC?!Rd6u4S6;!zM+%fs-Qo~RL$b1x*k~WZ9X1UbDiS3n|bFK zW-$M|E;g+c&x739Xhql1EgG}hNGPvj)mBGzIKOK;ivO@Ts#m@8l6b_p>F`slto+hE z0r>+H_+GB4LvFA6^iWfx#DN`U=kd&tsv%(=I#A>Ru&8#WHVCrXdb~ez+cJbKbVQ@$S@9<=H`op3GbRx({&QTCQ;f%#z{rV{ec|?I z-Xzf@>U@=cdC!RCqoFvC!W#3;VeBYRDUdYD=dQGaZwH|5ZTeH^$ccegKXDwn^)aIN zgkQ|vXP==W_WnKtSfs#G@8_K|y6V-4nIQ^0Hz20fP^06kTm#r3?@iPW&g+Yv;$fLf zl<4JB5GWtYm+c=k%&WlR;2-`Uo(eqJrKyO6K#b(;P=;#1JYi>1GE#V1Hj6SoZ~R;L zz^v+KHcYj383QY%%Y`bJ~u(aZ;v66XNk2~>Lcj* zr0(oqD|!yO_~FXGLonaS8B(Xrp&^WS6k`+{u&QROF%0%!vd1vP~Vu= z16Q7JFDhfvpDv+ROlr43Zo{!Jo}~*+II=M4Z`Uxmu)+O$8X{;862KKOMDdq!^>-Ba z@dST?6BLsZ2B9qoj)JNcD* znko0yFPK@Z&Cz*(P*HmmG+R-R9?EYLU!5mT3{mJi)={h`dGt*EX0BcHE#cPntZAsle}xUz-3qgc{H5 z(QDFiffX`j|7Iip{&KFUvH9T%_{`V{W5n~`cOwb)6QIfVW~MyyDGuz5)h9>ngBg+h z3=dwv!vmQ5_g_KZe};z;92mBbaG$ zaVZ1v7jGIr5O`(Hj)USLVg-ls(uB=02Rx+D!8PopaxBzGl6SYk1axVC)VvLv+df_# z+{exu`6cW74gr=0i;}Cs)b0NQCI9n|jh@@ioD#8KKH?_Acp|e6)8pgmboBH-rKhv% zXlthm)vhWz?X5ga@7mblBBP)+{4S_)3ykPjyEt7PMQlv~b1vc;*WU;Iv|Y}$L5Q*k zzCd^%e>50C19I#WwX!{0>@9Hn$1U(xCWJZboWgHFTc6k)!ltcOraU$?l9~8aB0!5l zn!oDVKc(ClwizH%S%elW^O-W$4~{=KunbgUKjS;dGmi>56T1W@?zhJ}U=WbgNQskG zmHww;;`b4FF6#&|;ox8|K&VEF5i>&wmD}=|My-U{d0www*Wdbx!C-PnFJs5e%a}CB zmAnHOM<+*0H7NjaFYK0|u8V>B8^DYlp8Qxk|Wo-_E&9}emcYWbm1m}xt!ET9gyTw6sSt+lnC^IG^noPBpZ z6@2@D8Ij7&-bpse$~;2IObA(76+-rQQdt=%nPro`Ldb3$du4BOknE0maL)1j9D45i ze#ZU%KHuLzUT$~kGp_Nz-q-v391hlu!}Yoi@X2&0r{?oI{Ejr6ShrHUG|p^3BWMe+ zzfH;c?0xyUXC<$OeEV)9Go70Wjr1mO5=Oi0SQwa$bw2A$Cq7MGLqNPq-1{h2agY8+fW-5FE87l6>$5w13oC#X5%NFBrI+j{H|1iP)LhW#uehHr-(HqPKQDU zQUcpyWJr}LVK}R#S>6Rawo`a)SKSHNux(|*D2~>1l{eCvaI1J2&yGzL5o&g%hO-Lv z-J*a#`^;wQvy{Ev#sPc8@qqgcH}*aMcDGZ2NcM1Pzz&~{)ce>kcoJ;bi^Sw1Nv}`A zS#@&=BY9teZzPdi%+*bfG~J2SQVoW7b!DIPjgR7TH%6wMthwN<9I-7cepZ-=d6;i< zc6kE{yJ&n0$z|?xy{8NJizKBV88?D& z!=Sgv2Q~%%pN8j|D-b!by%w+zuMBbA89)6rNC9rHmEmNefi7&Pkgdr^!6>Vq`iph? z@as+{642AH}DxGz}-Kxg_*R3YIB7LS{`&}Cw{ZmIZe5JX=9Lz>5ev5B~`p4{N z(%hS(nFHmYiaUM`u`6(x-y4r$vfP&D6*bHz-V-Jn$xmWVEtiSsj9P23>k0b zT4$nMCmgE#Do4zc%65yZr_~Hh-Su1i%Y(?F*5FYSYG6ra3i02Rjlf-zP(f=Cvp!V2 zbMx)0HNmjUiU+!Z4*tmIqSmdbI4-@Vb%}B+U4QxvQ8OZ4|Hz6@(zt6rb=no2s8xxd zhO#{{Qk|d^BRxeRe~KU{3TIuzzmGo_AK3Sgu4`~s*B{xV#m`V#ocHvQizRjInNV~q zbWoXsQ$};dCc#v>(4aJDs~3_+$}AOIB;j08+Tgp_Puuv55ZHIZ>q9Ch8X+@{I*&EA z$21>;QM%-EOpot?4QRZj6D?#r*-7fu?zO3HR!HaD=VPtC@655=zD9dnQ3}#mYq?k_ z&LUfhvytHJT8zFkAKS&#->!WnOAIVGPvOLBf$yEd(^Ub7%VZx4ajCz}NYD`|pRorX z%I}>w;UtG%2`L=wAwjAnMQ6T*ET;Mj${wM&-f}~*NZ1me`Mz4Y-{)t9JcwR61@t2& zj1Hc8`%3#_TQtYu=xCyhL8B8Jc3z#d_01b_P9hJ996A$@?P%Xj`T>L#9Ca^w&cU@F z60sZLIPceKd%ws2bK~6+zqMVaC`%S`5dP}#1Q)(F&5;(dL=8)1H<;_@hD`& zJ_@7JM{RQ6=zvauT|c zWb6>leY#vbLd(D5HV`-p33Ag%PN&?@mXomCUwON2VpOYUPSdB(GjoV6FLe z>7ZY1oFnySKefbEfN=igb$BTgi&C z+nolf?qwN11N!;76>+5N{@%4;I7)62%k0dHZ2wqq&@(q?*_)vAo%zI_QfnvXze04x zlM<9vf>H6YTI|$Yijf;EC;HV;*sM<5R|EBrvVq_;km4;7NJ&YBu|D8#vFE1imyMiR z9GYS!x%tRi^l{puy`6HJVjR2V6X%ys1L zgPl^SX4=7E?wyX!G1M5$DTj-7-(N1|-a#UXF`=pZ4s|^C{3AaP;6vpVpIKHYRY91? z5bRe^hN1?%(0Eg)OjTfgjJ`A7_gcB6Sncnoymbee`azP%L3bEYa>ed;SQ5{MZ=JF2 zb%t9!oga`D(h`)O3LR^e_xmVo_nX$^v7>2{P=T8qH8pB!qw_>v5L-D@RN0osw3;Xx z?v6W>p6VbM;Z#5RB7)E8e)U<3GjOdEq9N63t#f+SX+FvwDT)84NWuDj+WM#$nV6{E z2L6)R4jI`|Q~)-Hby=Th%)bVnW-_yq%h**Gg%8h6iXIV>zQ;oz!H|FNj(dD>+6}G` zgRz4TJ9aJ1pbiqUTidQ<9tn@VP5qhucgwXI5BxR`+m6KboN9S@KCr&S)KzO6c)`Wa z25RnMgKMSgH2dG?+bb(tiD35dVQNGsNBOWj8-1H|LxYp534S@hX6D}o&YdDpyo}d0 z=_RzYv#Z%Z=6p{C7b9~XUu+WK$HvatW5i}MU4_s}Eif}tV1m3SwopzbU$XCZv5gI2#G`U$yEWhERDtk1XehzZ$DdNP?r+v|K@u^ku54xq>l>|n_7s$vQ z2WfKRetq~r=HMgm?bE*|2-^^*5Po(6rd2chxi!D!h9mE;sZbaN6uZy6ARRGa#8Uq( zgMuW;6$T#i+}J5~n9%5ZFRa6g&>n|e%M`lfQ$RaoA%i$$Dp_wxLwAwPa)ANax;nFB z15Ycqgr}X3KKM-Y2dcY#j19Yx(t$4e1tfE@ z?&Oe&LazU>uiHC}^D1oh3w0y?J!`$XR{s0H5WWU{cLlOUdhZzrLlu8TtH+&!p{Pi9 zbrl`FCk-N7Rr0qUC1aN-$p-p$6JqV|A!|BboeGD)tIfjnehM zfu?V=d?<8yeQdP{`Rs#dZIHI_k#mml zGn9xhhZg|g;DYZH;!%q;*7&*eXquuL^=gKfu8s`E6YbVtPE9AiJi&C9^xG5B*OOZH zpF<&MDO-jCT&Ck1#5>06SIJI?OlW)La*d_Odxz&gXmM_LNPTAPHzYi?(ru8+;si_d zaLGFwMeUACN1&clKt*Y9-n(%Hnfv7d?aYe3uH9RVFQ)FOA~Lh=NqIK~k{NovQ&uVh z6$95Rce;vziB|$tt#R8j03?-p7XkoOXziXqH3DEp*#5ycLGo8P@X}eBhv?-@Fi{`_ zpZRsAqq+&>P>VfSiCF+!KhQHDdzYrGEgNU%zcV!NP2!D@%q91^PIK%m_oaxhHw?K= zesOhIOv3PddG2eLvtH#XmUbEeI}_%38qR7eGkW&qloaqTR2b+iM6Bxi?)!^xdUs0? z9;o^yddM6~UoZ-VkpE+AtqY}D@iD!(3g6V)f(UdtGpZ%aOr*;wTSbWE;)s9E`vR4v z2`VE2s@TLd0EB9oZk*u7`**=6yWY}&I7cuKpmpCSZvo16F~ze#&cG}`@{-!!;(Iq& zMoM!Joj9m^HD8;pvfI7gzX1(2f2GhmQ7D0!6+;PVIF=f?^0T)@BZ$eUUpy|l8D zWz!k2qoNWbekYLA^+#8R#Ef4|au1>_(~IvYp-%VOu4r;^Aq#1Ou95PJ%qE|ffD?2vgkMhKx{K-eYd~Ym0 zPvk;*Sn0vNR=8%jZ(W0euJ{|mz9+AgRa9QHN7ZjJ?6^h24iZKK^BXBIM-&SWB?xQf z5uO+h3tpvwdYicf-xyonS54|&B$2P58Kd`P)jW1vf|{Dzy0cNb!AGZxBjfYDw=%+c z9JAOOGZ7|{xO`66pR*+B@rnilHejr%$yIbq5{{l(34!k&^jRkv&y3&lSAkuw=4$+` zXHI1s`@>gbCr-MQ9O@#X5V-`-R7H^Jclq@Da?h;cot6Yv_M!fzAofXUl4I>HqYty2 zd`1mEVht!n_&#&;0(U6H=unKN+FMtUTm!-nj?9k6tGeNL znxYNl5HQU|e-@Z2#VJ`9B4j9R_DS*=in13WkuSOnqb6*)swqdtod15nORt% zPd~S{3nO3@YQx1KiWDVDe~^&|7B@C#J&up$qN0use7b|HN6mV^S(3LYf%}i|02Be! z59yhGQp~Rq$e?m8zfq%~q{DRmn|t@}U4{*ok|I^o$abF{gAc>!PsgBaPf;-#sV^X8 zjJ?|cXS$^H{+L4<24~q?D+$*#cd2=fvWiG(T_fMa`VC$i&~B|n@h~vwRPD0cZ4mRU zgk@czm(D7?7b#MIDV3qRu}9H?`O*X5+i>R0Px6DKw_5CgvwyLv=>U~?!Zp#+Q(zBg z*fjI-kPq?B*At#ThszDK^BGHQR|8MM)6QOPnbvxX`nm}H^_ghtC0|MU^t73SH9iP- zE@!8lTbbv#_RkN-7vh}+#uFG71N_+aN3$v@-sm#k4F{sElBw-*yL_sTJ0Z244{IO< z*<>1kzXj_-$%R7Z_=|plijg`s+r)oASnA;oT@#k+>ToI2b%FYf>x#F)qBs)Fe%G*F zYT$5&Fi-c1MZ3&yVGkt781GtSk(6CFlNmD5w^`nM+m2Zkn{PNJ$ zsz3-h!w!OO!a@%`JGOTqQRUR`TTrT$7FfqZjixo=uXmApJWoDgF9{!Cai@ee4qX4r zLBrY%7d4)mVqcak5G+ef^x+fFjBTCcfODJ*JEA)J%-2BMs`vHotEsygaCNvt-rl$K z6fmn1y}DR0)2SO}qyyjYi_*J#UDwL8{2>(8;!MRsKK9sSVe5%s1S%_maAhNg8Ob%m z1p;U|`K~r0I6jxyzl&0uwWt^tmyr_F!SQi5Q&ZDS{275C8#K%ZEy|!w%cRnliT=@> zmKMbm%lF}#WxF^{eWsuqk>13H}CWiuwq4r+HOGr(R$k@zE zIC762Nnf@8C_C^wO4R6U-~00L1;!+DCqwvb<;V*89-(Q+_=cyBLvfOw{iL43N`?hH zc6;|`UMWPw-;$Z6IhK^8%bzG4b|U0u5XOSkirQ=GjwrT)0cjF+ul*`!cY-PVhT3A3c%}Z21D4Dp7-Y)I}B)OzN4gQP|V!->q#% zP;N$gZFrKjKZrdOk|`>BkxD%l?=+3`mzNPDq8D`tm`X&heaSldxu_Irt7}=Y{neSO z40GvjcIXmqnXF?Sr4WVZldIQg3?iASI7l^Pq1Our)~FCLx*e5zHY_)t+AdirgfnTp z9D5&x@_rn-`=1cdIfV9e)APvA=NA&Ng}D$dK~HLb?54W<(N_f3Nr2? z$e|Gx3Yxe8&P|QO97(r;A=U(<6fd&qvj)y1IN~O`+wq_L>opPyTZVyt~ zJR1_y{8MhZe}tAY3;80iT7r8%jTkv--uLr;`Hzj4V`lt}k}13MUo4{;M)-!!Z06IH zOiB_$Tg!~DF0X0AFNAqn)(Q(R(884A=bB8WdCSL!`dZnr)}0Erv9!kMvGwW9bHOTY zZ&3%Ih>q?N+gDPByUm1ns&F4@X;hPXJj0km$=EWU@L<|m1;v*lLpl69TCN1@t6FPH>?Cp)vItc~5 zy@|GOhHeTd6K`zIqm$T3jpFOQ=YIv$+_l^q9=F7ep^$Kr)-PVlM z<~OsMU1~59%0;0eb?G%RlE z>QV{Z`;f;?Av|QDa}{yvol5subtsUcjLls`{gGd0G+7|K+5xq_h74PuUT-Rq*hn&0+i+6T* zc%7#D)bJJF-m0hMz`p-Aw=>>Klk_S8t?&uSw@DT*8AcIk_{jnpk~@T0s`u(1@FT?I zKX}pH22li0rfZv;`gcgtMO;>}9p>K_6(4??ea~oQNS}8gT;IUAc|2EDIqajj@lCa^ zC)eLbbRsCUC3Cc7#rfZ{xnHI}f#fNl@x~gZ;H=N@>uA0N_QPZHcSS_!d(;4+P_}+_ z52RX;JdMh~VT+wYRTs~97vG`jL zC;{*U_7(v{%^^pd+oU}<553oJzhk9=+S&28FS=5NOUQ`P3h`s}!o7Or>lbaS&k#`| zdfAwz2DAB#i>Er;AYDRvWz;05<#yaiu=l7H# zXk3DGxG!xyWLKq5I__n7tS#^+tQ9Jv+izYx++xSEx(vTr-3Ociz!bf?Q712tW7P$O zLLmU;`|76?9H-)RYvQ(PjTa~p_xuom762FgH@gal$gy_HX$kj7)s_@&o;%%|%Wqby zLi?B-&atipSCys!)HYkgp`&!emR6Esb=cC!a@h2OhqNsgO(TU=a<{cBphiR+@H>`y zy?7k!|E|O^LSdbSQ{@aN@K3@hWIyZ)NKraTAKMhaZJ+KqS)w!$w-ZK+C6MR}S z(a9bvco=CSlFQYq7t zAT_c4m!7#(lfNOq3cSBS0~5D)AJ%${E5)te@BxoK4OE6D!7$mfL%90dxOSBlXln+2 zWzvnD8lcL|QhDCtvTCR*c^-$BACG94Y%(^w8mq<8dgf|5)%=K!XVhzV2My@mtE_sNJ2!@y;O!7&HI zz*SsPHYSe_L}fMb-o`twOm1HzaGBCTe|z0rn>4RXdJEdu6zH}AL}&Q`d8xNEu{BGAhT(eh-I_cbQtsJdMU(VL+y@8~s zdu0*>qrX+Sd7GS;jdM-%yxRj>Ce-)YS&Ns=2uo&KT}Q&DZV57;&gq6?%ZsU}7tn98 z_uC)Nei>^IYG#dU)_sb6o4ePs%Rq4y^(zm;6l29IP4lj&=XBY59nWg14sx8mjFqIB zEYutLryRLLzNT(j;Ke@yC1+5Cr~*Dd`*_3QsRm@rjWRjz7s>L&@4OlW@6~&;2pC;c zDO#iqhj>W0>RUaOy}ham*B!}H*rk70RduSenkX+zk=d3I7#nMYy{eOW2`)a$`ivUz z11vDF`n!AeZ*0Uf6h2bruOS1#Ok(82gF6?Ht=E13>{fO?G=d-*Bx%OJEvcUL93;}tukMnWO{X#RVm#{8}qks&v; z;IHlBX-XPqTcYO7%=iju`t%aPQ1FlohDod!r2lR67j^6PknH8?ugp9XOJq|Qd9j0o zdO9|75c;`6o7`_ia__IzL`txI0_NXl`nT2a`Eba%VzKGw7hU6@xe_z-n&i-;6x9zb z$I($9*H<%@8K?!Y%o+W6*hxeXtEXS{sO7wnn5Y*V!I2sqt}>7GnzURN$Q2xYB&v)9 zNd$O`+r!2oVY&N0E5&!0*Yr7)HMo9xCA)35SNx3CeZXhWf>cSd>|!4loXT%Byru@9 zch}awO%YIA_xvRR9C;NNAv8uzbIP|Hy?s(F!y!wHN1uoc`n(s@-k3$KUBeaq>v*^( zHR$#K8+0H<5+Pu_1@dQ4Dg~RfAnNnG0fW3f;)iiTaGkR;{}Uwyt!Bg*lL7d&{3m?6 zuaBn<*-`O91|?L8H%XJHHx!4&9R9`aErglB?Mm$BTwFYucAe8S$Q_g@ryyVZ%@jw^ z*o(M?F09C-qt!D2L%sHPs+Mc|FhPvxFu~OAR>$u59tUuZjB$J;ITy1>JjIU0ZJ-A& zozk?oFC(;sdOkL|tMFYQ$ZGeW{cN_-FthP){GNeJe$lp$O?C)6p!CwwWwN$kSD2!1 zb6?btZ*08bvzQ`2UcC`&$ePjEm~mbu?t4f~GjvpB98_GAWIz$p{{7cco# zPGIeH<;THkeoX#XlO1*}U6UK`Gzo#r@C_2+%8)`Fj7{kRv_K~{JPk`uvzw9i|go1T5r8bZd$qbN#^dfIngq?iZ$??9>s z!+2T>g1qLlm+fGNhi$X(v*FAXSwZ>Ew<4Zgq!Lza|M>j54pNR+DLt|L36M^J=A)_a z(t^|4RN5W4nT90YD&I3_?o`?H@wMNL_E(ye=T?&8)kiGsd9|%ErCoDs?r8qtg?t8g zEGR6c;4l-rj7y}6U>`Lr|7E2Sj{{dy`G6RbG59k&yypcNWVoFP{2~*;?bAY&dq{+{ z6BPNyUFy~>JA`IN5e7ME%d@SHrMIn_W)Fz-EnXm;-Omzyv4p3*qMHh7PD{VGtnIXs zZb)`p&$_7aA8!F%XX<==G32Y3yL+4^GCW&NRavggE5<%|paeemQ9MaAEFrX0H(WD< z@%-kXxiHBYt4j||!p;KE*B%ZRYK=1S-(w0{oWzVkx7eku)@&tLG8DXz2_ER*5j;2= z;87__>_*16^MBLcZ+toXqTE zr*VgbL;4;iTMD~dEajGJn;N!Q@Nivx$!$;LUf$rf%~m?XY0Z&P4)81SSSW3LRK0?{ z8eET%O>8h_irv326ZlHNcw3AQ3_orln zD~As*;b;g_;7&jr^$$k=-!iG;m$Hc=9e=NGJW^QWSG*`*>?pNx+ky-CfJ2$z*G)1Z z494On+T!%1`)rq#f6y~CUQkUT;WxhPmGdr4*a+`VGb_tZYgU9jyquzSo+3nVOqJR% zIQUZgM}L2dn`hqAC0Hqdpck_?-l;D)=IdK*|#) z|26y7F$pSgNYH%HqUiyV;7`DatH(vo0#X|A4=v4UBo?!Ix^~D9_o(Oh>RyMBtXxDy zN-dL~yS}Vv?rc+AIy?LE+Ck5^+j>Rb@6r-4ok4oSqoXOZ!Pxed7Pq>TOIL&d4$nI8 zEaU!GI4LwCRPvJYb%YKyQ}lY9Gl!b-U?P*j`CFIGaK`)7)Ys>5dc`egwefJ*_)e9L zL{Gza1u2`Q{&|Z4g9txCcePq;_UWhc4B%dEx2zmmHxZn1{fKctEkEs?cU*>d0t_su3 z^Vn?a;HgW6Z1z=HWH*^p#@FJIyfUrHXrUOuyv&8+0PP9QET2!FXJ;b6QxVSv=}{MY zoPBhdwQ;3-%R2@u`vcb&SQO=H{j+@>J;SFFe9a3aKy{Zgc-nB=4o_f+hwK}$2jPuU zK~+h(2r4CZZ96i-Q;>W!T)N@<-0m=Qu_$@OmAxP$-YGhd2O1DuGa9jb&LI76{ynqzFD7?;H7Sf7`qI-D=U{+t_|Fc8 zgS*1Rd6)zHx}`U{X<0w$L27!in(nWyHYAde9aX;5kmkpaLs!X?T;aCMGE2?vYp%U0 z^AA7a1Ft96Fr~VCpC;PZykLnfC48xqEsW&%aWJ6-|L?Lya5Jmic21IGe=56rd+-Iu zyoLtTioIv&1k?Ao76+0gU1P<_CMJf4vIj8>LeN`8q)e)k4}b>3ZL3=%T=FCH;NYNA zIE{eXSlUL*Ujk)2EqfIHF<<*0*DU#=NeQeCaFu_tHekWi8n|})aar$T5Fus@><=B98*ZTUA zG|rRO^Vs7y`?qwfx*EEVr9|Lt$XB;^&V+A3R2^ULy40WT)+@XB?KP+RosI#C^^bF; z2gB8kH=jR$-fn6?T$B;Q;uFfi!0_58s4)Gi=h4Fw)J<9>R-ZA9a_lCq`?t;?6vqdv?R?y@auTX9DQOM1WfDlHdnm z@vYppgE2UADwu}iZ}MkpW;o>N7uk_~6RKJ0&L#hYfwY{ujYjEg4X3$wuC#aOgfto~ zS~q*o3Du{2CJurI)}4>H>5pqQ5V1)baAoN-C9W(5y32ng(|^XM_W>;9YhkDaEk(an z5*%q3{Vm5Vq&_$U&}R03@+E&Sh{JH*%=bv~sBrp8RYiYuJ>)}>(A}{6i@8d+ zj*dCFjy5kJpKgb%gC7v^{=VAef#J2t1l?-Y_`B06aXKfv8Jj%gbU z0w(+yrmYCqR&&2b1o9(5k-y#d0#Zdz8&D3Vd(*|f{rg+moUqn}N>c5ug)o>lVQ_w9 zdFGd>vmi-nJhk{P0B6O2b(@fYA1C1?eEs6ev!|zl6@Pg3k{9cTph`;@Fs^#1WYiQO zI|U@lJT}VW9&6(_bJGHR)Q09@kks9c8EtCnAZ>g+Bp0E8`0oVpLcSy^Y;O&lyv`Mb z$_mfTlLl2xE1=c?V*vwbl0W=Ux1`a>>bWlVosS^#le#V{D2mNI_6F0(x4r}8UqJ6@ zJG^@t9UXfui_zHgL%;mhw?!;0tD22Ml@fwZ)MKm5O`4sHz3Eimvz7wK>~G6C;z{U8jIl9D3oyZc!4MPX^<+kovXR@)9&Y9?uq2++e89b)nM z$zJ>Y**VdLa>8R#w%kc&Pfe_`gQKbU&8cw}2`EkeXaR#WFVp$~>}!DpL8gq3uCpxC zVZ?bsPA#q>E~m2KMF93(^jmuWZv`&zTOjo7y8*-;eM;BoB1C?UJxm*lUF7vzEo=Rm z87|nbSGUz8y#yXYsLwH}0@unfJV2>m0k8w%^!e@!Gv< zU)!l#cdBsr8L|r(`aoVF4B*_dQGE;xPE(`sZgIy)?HmyG&kGEmbTHqOoE}RG3wkLJq-Ran zaKr-9U;VO;6Z;M-m`isyqiQ_bjK2~Se-s^ifKNu2{g`o#m;mpC^H*6tT?dY)>Zj(9 zA_+upF#~H1OXNp#HBbiyltRC(YfIW-bQ$w4tuJSH{cQP-x(Y*Zs7rk(T=5R;G?0OAIfhK2^*2j|M@k7D*_wD+bwSqYzij>yd`F3v6~DTz3=J_in- zQqljo_BhQ){;J`P;ZHIG+-dOs+iCoTW?_x^i;1=Gf}*ZXq=jM=#L+h4=YPYU>FXf@ zKaoNsXrh{NTa_`NnDYn>RVheJds$KRC=noYx#^rwilEam6ks7ZYLt=~T*qIi(SOcC z;9&l6J{AWk+d_J&-?jt}rHJX179?JMmHgH01~v6kWw}$6yPI^wNHw3)=Yfr!4l{FH zljAAyO)0_#l9gm`^*}`a3j!9%ks?C(n}lkAUh+XBTPyoE`QHdz zopG3HO!?81lC((wA>;v#*EhQzNXp^qf@Qazh(uh;0IS--rqdtFijc}y-QsDT4x81>bPyBM}k-P4!Bmq zL+!(m%4lA=Y@?sq@3bHzDl?koXKv?o()EV={zFSXuF{19T$(jJ)7bElIWqz#Xtvsg zQ}9SW`P;Aoq+7(P%PaL;J-XJ+4yrY;J37?H%WcjT8^!o2YOV8+gAxOXCQq6?_x*8* zBNh_~rj5XWIY@)6|HS74raMLw_pXN){R|~KCv^4ZQ*Y?y#fB{9|hYZ&QzhBQ; z-Gn4{Vqqc=@Dn9nW_A=Sg$*0q(LC|I`5%0M|6+as?;iLI^@hLYzJ5 zyz)I1@|-wF4Rt((Me^JT@axUunWz!BD`2v&hK-bu@#ZN!{4}ZAG%63ntXH?=-%Y7{ z(Id06jVpss{m&%sTTvtzML}I%5)Q-Hu9Vc}6&CX86xGRLwg;Bdj`q`*zMK;pEH-G< z3WWrH`&xFg{NFTCU)4F#VZ2pE2|7k?=}gDHVO&wUGLP#G175TZS?=Nx3c1^3wq8Zc zi+!h(Sm$C22rm!K%I79Mt3P^A>UFbx*G!jvP@Z!nxPp_$jSv01^hyw`alYnZFUxoh zc*Sa_32N!QiOIYf9-5sd>H6bx=arQU@U-?z;aevfki;kbH^o$z4a5%J?^}2XIK^ z|6T3CCEH;QEfs2+2~kR_#_5IbiyE(5?>%n z#DA6I1l!+VCf3+Jj_$p<&e>ng_@7lB$M&I!E_2y+sb)rKJ1XZ{$l|IwmotBpg>d4T z6sD+6I@9z&r8AWVc>mx)aPy8^In>}>Ktt;<=K}u2=?X3`p802GYFTqyP7v$Rg=G)v zddycKZZ6>H=iP(S$fouK=Xe3RHvxVFBjxL?B*8!D63kM-fKc0xtm@wP zvXsA0FVbLB+ElxiNtG0Rjr1}P9wSa z7*^k3#RaVPzsv77hUle@jog+Nbzzx8!zAu5O z=lPBQMCxTopj?^B02Egz{AVcs4?!3}NC|)obg3^gVdVJJrudG~_*3Z+)NY)f{!;I~ z#oe?SZKrkVSMEn#J&>dxS$p?k%;;&n|Cuu`eB7KiGgB8WjzJ&2{%Uk%0ck?g_rx!D z(vi%6sWZX8l;C%(Au(9qz<{>la6K^d!=iEZ<8QLPdre5fDNIe&(MTzVwctIqTyZmF z=~~A3Z{(8A(1gwf>hQ?^w`9;4jUJfpe%5{UqZNf^+3{PXD&EL@-s2=vTSX%8yk}1V z#OO-?JzB?OW7L#WwZ*qp#if-2YVZ4g?=FvP_YgKHSL~d;+R3{zS@m6iG){9ITX-T3SED~ zt3w7i_QelBJJmNPRmtrz?fWAd?cShmNN*~)&lP7@Lj42B26P8^heo{e^M{QGT@ltl z*nj=xfDP0g?j{U1B+go75{4)AH(|DE`@>KPr6$T7>r)(!t*WecWB91)A#{w^yK@ka zM%;K{iFSISkdiEh9izrsV7{8|dnaO|XLDzQSL#wj(#D)N)H@`KVh)?~c8M zKn{`J+w&Yvy~NqE=lI~w(4irN*i9Aq!(*DRlS*|`NuY&j>F3#>sSh6bSFU3fU?Z}U z9=sAsj*b3VqG7$sP4Su0q1(=PA@@Ju+FE)BOL@2h3*3D!KSue;K~|&bXzi=b{{A`4 z=O(6%N>AqO=fw_25GmNR_w%v)MtiMu&OY8F>qUp&t#1e7adX8LTaQ>BW_-I;hh7(M z4e<%)j=BE022 zwd0i0a^ht6T)xLAR`b0o)y+>pKr$oa`_^}YAD7=RP0!pmcvvmTC=pg{_vmrXQNVYd zCE^vxjt-Li?mp7a&heeP+!l|m+4WBh1RhPE5xdMw70OoOgB={DO{!lrHe(bzZlS$P z8`7|gQ&~4_EsI>=+NFi}7>P|tT@ZYqeKOPsF;!U%Sj~)JZ2;aun>wxlY1ltw2ejz) zLgz1@q;Cz@K)B{0n3F1>==^&3hs8ivpN^d^Z(3{4)u9ArlU3q!=P@ZGLIox-FSrA0 zvtzEw34`xU61+bfE|erG_Q?1&LGUGEpXCKb!sO4jA!g92NJdUedt z1X^nDa1t`affC@Q*7sgtsk{maRrH{rWCNJj_oxB`4kiKM3s7QKoe7k9f!fk~69$pF zH{0dIT5Kjdz;H`}FZ=AO(@O5hE?^SQJS@WpYZSNm=T@QyZfIoSCA8xiz$O!E$5zPm-~IKDOtfO%N)}ypR55Y6EU*xb_bgH zmaWsjYiBS)ke6_v2YuVyyGr z$kFy+*8Si#(>PypaNvV0nQRw2;@Zm5%pU~I!8bUCw+Hxg=>>-E{sr)ZqG+xUcaCT8LGKK3ELCDS@tkt%H#OYk$h?

s#Z^VFq%3yQ}fvPFm5#Lu@sv%ygfKLcs=x>C;8SywUf|z z8)z%Negs`I8W8|6w1bj%p-x&74KW?D^cT5Yp_k&4ApaPcr#x;ry z8kaUsR;kmP!7(`P^TJ08)=n6oC+q4FP8l=Tn=BGuJ0X>qRjCPwC@>REaI?VFlP@4( z7x}}%=$KU>JY%dDEa)eK=S>PIRP*@@P;)Ldq7+!n%uz~7{id2`P4qL3#fd5h#*~1A zC1ui)Vu15#$$?1 zVg3EwNaf(Ub}wawC7dYttMVhz7GNvQU zo|73oz51_g;jD5$u!kGD;X;kqM`k7WfNnR8nEHfu&jG^CG+R7KGnk*CBf`l7aB88y z1ei7hG}4<{Pj@cotzCj)9ccI~V*-hH_Ec=2aK}5jy1^>O9_ty;y|bgJxC{R=6Lx^F z!+E8ZIjU0=^918Q^3oMR#UMcs(C7Y7{?*^iS~peFA%~ET|GGqHoIJnfYpd)^CD!#= zk0xzKMMb){xj(zeyQ6g0P=o(#hm@c-Q$S?shNZGK6IRQ>LN|z2tVD#qF2nmwcjDOP zcx9WZ0sn|7=hYs^vh_?=CBH?gvN^R?3r?2S`O!DP39sq!6a&p_cAT~!L!5jgLfbuV z98#L^5k403_#!bpPkTGArj#YDEi&6^(E6uOg)w340m zOUm+1{%WCXS=qJC=j?zW%JQHn)r3225Z@8y_2d8&Lc#TUKpWh;=!K6LkIQuZONw(9 zAVI_sYQ%#Nb8lVp;N?J%GI?xz$<8Ca<0?!0wZ$&D>+!`eAKc51uwgaBYdzrpHlsaz zKlD<@`|_yZAyy9~0tWha{_h%(Uz@^p-*M=)Wu5b`zf@vacN2Y&qBN&b3tmdSvBLe0 zHeH^qbv;Y2jFilIl02vd0gEDDQi1=?Ex1zc_4#Mcta8OarS!L1m(U)@mkMZ#iK5y& zLy6fA{G8`v{`BxCbIOhUSisEVrrIpngLbwfziA%ri`@I@cC%%dRugT(H#%Jcnh zw^Lxs+rZES#NgrtAhnBXn|}sfvDU(;&AJ-hUM)oDxN1 z=XlqNyDGHu5|&%|`5=!Yw{qsP?hm`wvk)oGh&PE^3dxKJ?MpldfV(*GdrwPH1fK1I z>0b--=)ehp6~nnr(H3LDO(V!W=>TDpOG0s$_k?Ur*SJ`lXu}5<-em;dU?2&#xv%OC zp`*K9<;~1;2Ra)0vd@bW%mh_N^tKA`XSWnzbRI$lP=H413NHzmk~uAwLuvlO;>lAh zvTx~L5Z-sJQSD_S0C`zw-nnN(4prkzLy7X9E=GN2UKN%yI_R7naNj9z(ujC{Mfnk9cfa|t`llBktIqD7K?j6f!ia_+;t3$Ir_L#(ZFJClM^6eS}J{Z>;Sv;P}#*Cvuv=$IDNQBXKj3H46*lk-@%D!Orcdj9x!WuUcJfg`{*&(V8$cA*z|$weC7YF>!^L zdF?E(Fzn^W*>6{G#(m_t6ksyhl}zrsdL_SmRwEe9#3^MQ0uo1%WtHG`Cl0K=(aXr7 z1W8Ku6Q%w@Gb_|R``SYM0B6p_*=UbtyJljDyPm%8O~KNnNf!4qX{L^!a?wv-WLlmL z)A2lL{wa-Vu-r~dm1F;Yp8LXVpI3Qgy+~QMAS~PjF^L8;tC_mEkCxl4*-;^u6vDER zp>IN`W+qA_q~K*W7djq6`V#59xwk^WkkXacLfw1dzC}tGu=)MXX?@WjFG*cXu!fjM zJY4}KKzYB)Vq@)dhGp5oPyTLYyJ68@AE5!;hN}D)43r95{JaNUliIWx8;>W00kz{! z!&a$3)A-69M2A&qg_|1!JJdq@wVm<|HpmEWHYHoRJZ8C&c=#jC7hguUn$nfuV{>a% zpZ+2+1{z=t9nEP_pc>5H@Za#QotN8s)BA06p|e@{L3hVBwpBz)iJ(Qd`!~?2Nz+Ho z#-xJG{lQ^Ed}@nv3g9Vjp1~6lN1`1-SvhY}+i+Y{Qu=pSwhKoXys3CzwV*D~femGR z`L!(6MT$j-tL&Gkv0R!uzm`q6HrcRV9T*IPvQHqU8@7zSn&(KC-|#=tnlRBiki_R1eC=YpJ4CL`-;MLf>H9W z@au?~qTfj71~DltJBLzu z6%-Wamsj0Wg$!A~V{Jm69?Oo{{+hM*>w`(zBR#ZX7lfgen>PnVvQ6~pJ@n}rxYT9k z(}4e!9mKkLps5a*Jdn4)!F}=L;sg=-p9=S}PYo|c ze`Ijw7jM&lvZjzCXI<5JJN3Qx=pi=TBRkP@g4Tr3)t$lbhtKj^cTa*Va z?l`H+Pf88Suw1%}&ZH*-z;;lgJ-n9pb|V?IcNbqsU;DTDvZwi=o)FfT`j&W!LX{SZ zdSJ<)iS@Fg^}M*Bld36m^Cyiq+1-kH-8l-0?RrFh;|qezN^GSuUuG>9u2^2ZUNot? zN*2oXIf(coxy0cY3uoQvTnloJqhHtLlxubse_gm)7uxFV4OGw(&GSj2i2UYve($0t zsm>$2)!pfd^AI}e5T(GtP#2DvIV72a%mLZVW%t8D);HGydJ{O(0Pk{L)44%UYzV_~ z_6z3)dNcu=b5y3Xe-{qn5-{dgen~#y7)C*1v^z`j)XPT**2~wi3g^JQr_*C^q-qHX zi3?J3XQJ8Ew(KbV{{8lyUCHfrU(s)jiFDWI4u2SRi(2JKRh(&6*Wmd<>!1ps&;Nw- zzqI~7u_t_ni0FBTp4AO@M#AQ8E59brZzC$j8%$kvfS^|9=hP=g=DzC>BNI_(Gey%! z1;d*Hb^5aO58EeIw27}zYM0Pp+}43;+=d-LOs=-mhSq;U2_k!iWjO35w&rK9Ym?FT zP#n~t_28$B&)exvaSx}bR+~`PD=qWB_yK*|doM#?pe@6im1BtlDa-IX@D=PdaEaf| zZ0qQ;2m9M7*>nKIh0YDnNGu@8IO^7_CVr0FmkLNVXa{V@xPh{dX009+RKb*7CvSsr zUjsgL^Nd~p0IvT=_wRxKsWO_LM%y`rC~g2LYAn-FR_Z;U^8aXi@35w~E^Sx^r72AW zQ6$ntx*!4q5~NFuQlx`Yq=QuH5fSM`qzMR-8j48oP3c8?lOjkUbcBQs0lpnQGw;kZ zXXcri_j~_47uPw5AA9fB?sczwjq5{(hxePuRU{80RobN`kLU|L_Z<9+5lG^Ne)ofe zSEgiKT^}pIUJhw)rqw29aobA%#y6i7j2}u1CTq()sgP&JdYhz#RLn^brS`No_(oGO`>5 zgyAJeZdMo+1vb3A*XUyNlPr%OUDtdKLFm@BC`m{zn+4unfOxm)@Uwj1Xs0#}r7zk* zw9GtF(tBm+o(b;0GYE*yk;@q$Xn>mYap|s^57Dfna^=WH-!_7#Wd=AOG|q#{$u!H zPN(Q3Oh;kj(ZSNBCH+Zcn4fnVvWIGDqtrOWPkguQqOa?(f}mzu;Yrv6iCdZ}`K>yg zG)T{pE21rDt9SU2Xj%3?Z&dE?xmD%qq{9Utbz_%rQ^vpYD4aAIX>^kllg_f#TROOG zY$5zOB*CN*uP%JLG#Khbw2bQ#KoMGa@@yWxSy{vLY)3rV_vwBDq>CXHulh+o+6ZOxDq zU6UW22lJ6de&PsHb4}(xM&@Vz3aVk3!^5|Ghy7gO(B1xkG=$X6tQE?YVjV>^Ml7UW ze7D~-h6P!v^HZH*y}Y*0l5R63uC!R0wBDYRH+cl3B@(BG+i4kb%{%f0jDnjZ`#cTk zh>LLs4iD_+>x==-;vTM@O&Ab)YI~!WfbvR0_Q&&o;FkYk55WbvCbT@{jCN{~(3G;?^T9WB9X@8L! z(NKTCMEO>5lJ{5;O~I$F&_hnQp~-J5`Oa)b)#IEB$JpSc#Hph9%dSuG3HyflX}!p6 z=ZiPeTj#z91PB(%#omEp0C%V%j~ET}n|w}3`aS|U=n@O!HLi2u8lC9|TaYd%ztTIc zBqS}u2RSkv1Wf}}!grc9?Uw3v%R@q zI$-dVK85}PEnNv!tSS__Q2CwYQ5?na`hW0v#9=s-w)PCJbD zhAGEL$opE#z%(wW!-KsJ8Rj0zwD1Ffie7a)0^Z{9>M0e8;nDqgJl}q<51QxQ`zdWk zj>a5ti)^Qs>h`>K4BMrez%6bo8Iq2mw^5ua>1^xQMkKVxDz=Wg7Gq*nVs<5~-^vCn zbc(kr$@-jX(>CS7-u-p+n%^$9J$(5r#%Bo*KeXTNan|LPrZ)u+f0@H;kxmJvu@Xzq^t}`D1RT&DA*cY=(f!oE4dmQ z%K*Y)a%huFbWgY0X&Ha333ds03(bpu zE%VeO1L=Dh*;lh(-e2!)A9^_^zixY58>mM)nOdiwrHIr_xxTe88M-yQy$n6eOan`C zQH|_Y0qd?s=pK*A$_fHit_xm~;NI?@dMZk)`+r0{K@XdpArrB;iS0cCh#tO^s#&rV zVMGe$J!1D0>+y1@hd1xmJz~Lc>`EE$Z8xUFH1pm*cdm=e*Lx5)U7?!utRk$^lw2pW z3+qxB;}KG6*4Y2mm&9r!&Qge({9i<~G&Po*gBaP~s^Oi)xaAtR12y;H*b1= z`F?f#baHc@Kgh0G(-6#bzOH3R<_H=5*I;OlPIpW&^r9@Yl+XJqze9Vc*<9K40mJg9 z%`7(7){o+}^BzC)^*w4{j^1HL_hzKFgb=7wr-;efPhnBgFX``s3K}a1(jEed;@>+? zf%ct;``>t&qA&yNLlC$A{15c2|kTURc*sXe3Jtr2ns)b$eNOIQ%pah zdvU*Gwm^;93!r`EMcLd{_H*4y1#1~A1;(&#H+Nj+P*h|%lO+ZOiBBKym0lxm1+R(G z(*H#!K{vyYA9&NS1`jgrNM8oiu^{-Z&zB7r#7l&h)2Ls1B)@+*n{cu#cFpTh`m?Mx zW_-6lCr^iKn^w@?tjOishj1Ql?)8>~F9z#zIizjd60))$Xl$`C!iZYj$={ji0y zl)Wal&D^7{_qtSr$46^v@XtH_dS?c_vB9;R$-*i?N65o8OsJCzcG7+1TIWsVqwXgi zG@HXlDIas}sWbYqI8tHuG8SkHXBv8G@*ii&AzLpHaAHX6pQ~{Ne_wx#rVxR?4EvYK z>wjB!!tmBMxUi`3qoMDM3}>hjxLIo41p)ORoL~19FW2W>W`I52Ol(2k^QP=ia%9#T z0q)YwjE~PyEQy{EZ4rrMPaU*oC|bMgc=HCjWtNxuLKsQ;xn`kkUVrJ!XjlpZSu;}d z>&z|czo>SkiJHIXE^mPhQxc76sMgp|6}vl*_(}t(WjopsHYsL1<~1n^r4L)G2Zdsi z?!ti_khxT7Q@Da&PY!Ahh6(`DM8id8cPI!*YW8~YLJ*I-EOZVQZGhW3z=m)4U+c`S z^+Hpf9NwLRL1@yk%rw4RvdWNTj_Y-!WSrlaR#3g#3iH?2(-)ZYBT%pBmg=h-q{J>i z*y)!LKO=MQ6+y61qd*cg6MZ)GBvXV|mJrchM)Jr8{J>D&DT6{CtS#KzJXj&I+Wke; zDNcX2XM8d%OI*L$JW_W))FK79mG;9pHCUl4D_N;|_Rh4foh?OQ6x&f}b%6jepmw+& zT>}G9`%eT&bqN|AdFJG6F@FXOtg{Xd7Evd0wXuHKdwH*zOt0uLm-&xMuEkA-HQ2Va zP*v4#P*po&>%vUHj@rB8IODvj)Xw_MbrW^eC&RSb7oYJK zsNvhXr`FVu8mdsK1Mf~(6R$R$ z)?^q%Z)T>-0vFpR4lY^=@7X2YGG~+GY*U_u{N$>f7(bAQ-uw8eY3Au~(L@b?FjGkl z*Wf3$`wLTpf|O?v8IWTIh|9A;Lh3(2)P&cWE^rqOGfLL+eD^kJQ$z%f0Ah#-!BD(h zY7`B8=n;JW?ezL%g)8qEu|a0Iw=-Z4-;z^-poy-;M1tkKf5w)&Pz3xQf>b*az;(!P zhQ+a2h&>?h0@hJLr7&#hJ zKR45Lpg6G8J92?8{XwJ4`3A4Y()&3!fZf`RS4j_WAT#?i=Sb3x{4J6e&6Op97JLs#S zQA(=qqJl-tkAZ0I;aU?qr13E(N2b2x{n{Nfg$v291K3VyTt}Oxb}T}YnX~Bm96eNF zsm>G~bUm`IsA=Y=+Y6GmI2OSS4?4Ev#jk^7VA6wx5{Re;g5X~Cu((gF34GvXg-k6~ z2DlU+>(a5Z8%lF^bq*H{LA0QEJe(tIn+EOV8)-rnnpx{zicU2b=$PPdyOl4)7@D@# zjq8I!B@!~tjUB3Fnof^dI-^PC(f4Yddr-dcCfOb^x% z@*g^t3xqgi%ir@J69)i|VAjs+^mH9bH4iz@w6{p^p^ISmRG!r_eztia?NeT!ujvp3 zhI{s`{Jmih^h}OZIuQ5DN=a#^rJX>Nhgcj(luf?UE@u1M^)Oy|7ysQ8qJN zzlmMxPA<@dEj+^1O+j2>DT*cPAM&(j-iLfQ#-_|-w**7J^D;WE zHj6WnYgz@H1;vPjM}`md_wak!F8m5-0kv ze76fasij!p zs_-UvM!&9wPK23#OP-L!%dr&PHAX+X0Xu9dkeh72OIx&m;|2vZ1k5+nEUzyFIyz9w z;vbxfHUyLu0_*t6zZ3qS+sFh6UxBV1KT0KhWA?g9EE2CB+nH#=6zA_z6cT9hfrjO< zdDh8dCa3NdQGl@UExE5>S;#49f=ozu*HF@q*Dcc%ExR*XmjMQsSlS(yf*M2<(nELx8yG7Gf9_e}j2>_mod+tM%7V z6LvD+>5g3dI1ZLVl^_l!JhI2}@q*`QdK~+Z}!ft78#p zW`AxbCy)*-V02g=(C3-r<3XE;_xLr)nbMHVVfXPsviGz& zDaL%4mGxm0R1E>T4uVdyx49jJafN78FF_|F;5g0MkabLGn7hb#qBD;PiHGu-j6noJ zLL(+dU%&`&o#6d}+Ma2SF8z+yu$E?~Zmw-!2aiJJbFr2!J{eN<9r*ecdN%py*BciZ zR@tZS5HmiO#HE`*D=5}0vQRFwa3&5?$URYTN>RCrj=!;f_f()RtTf=&_j}AdT z_QRoq%vy5|RT3P%nC{YWmnkxHYp|xe5;2vxHCWaAc^4h!nUqw8{LF82GRbqWVwxc) z_=BNM4g3#wW-6JYAqP*#ENhX>j}&IX&w_#!1qriT5308QOCy`_QCEENp-0@CFJHH6 za?hWj-1LMSfM2Syl)_m`!~n&qn7ENLRT9>6gZMfpfL&YE+lNh0|j9 zT3Mm(?}IyR{oXx{XcKH1x;Qg4*9fBpYy$S0Q@UwV`4r$mM@oA5alAq32j`BBjh%ZJc9)-xKhi=^F9g!^4nJzN6$Virk zK`xM=4l=nfi0dufV}8Ga*{SZpJW_Q2M1#G-e}b$NJ^eN_cl`)Hcqnr3(EY_v<2Y|U zfzX#XLv889qblbdRU1`g-l5+46pWF29G0`J~mHWd~mmOtU92??gSF2S1g zeCG3{{j*<_W0qbf^GANens@GL)(tJCEPY-+j-%IhTh>a}M7s1O8%rHY=qGexlm$nA);TWl&7}r5IlSJF3<3}A(=QsaKghk6I7s`{9ACZB0=1yqH&7y$$bg<0LG{Aj z;o4CC<1?{fIIpi~g)jJ?|4YTgTSX{E@Pp^QDndWvM_1Y;I}Aq{-MnoATF%aayG zs;E)!Q+ab~%r4a!=MUsDiof%uyxh~Ip;CYK(hTHGPO6%zY2@JT-28(6WZMEsc&5TC z@Um-P5dI7V<%hHxZ5bcCiuNkEk8nN!=IowOfasd=qp{O(F% z?>Y4N<7Pb-+Y&d)!WT#+|@8>B$eM9#;PNZcmex#+dXHKV;6$ZGWG^@3cM8 z$*bOCGl=pdhfG~cTMd4F8;TqtX4=11L_J?ayI7PR4UY ziEIZL@E9~hjsjH2I)j3}Vp0PGhny1vt-s7U3Ibx)To*E6Pc!90&&!w5gp1R#P5t~B zM3^Gw+_#STdu&lJ%%hfSWhCo#Zqklq?bAjnW=%NTn(%sO2c)0D=x#)5%OTGc?EGDz7OBIwwjs*cV5m+_s%!UT^@SlJ zcs5f6{yJ)d`A32L&;GMao(TT+obBBu91+w0ef60m=b5@{yi(SNRNY2O7449WXP}V0 zc&LKT6pG>Grjnl+=oKD#Z^sHn>tAEnFRX>e=i_e@#6ytidP_b0SOXmDi{UZhs((Du z04OnwVAEDfIUKam27s*Z)6NoOG6KgF3?4ZcIqd%Q-R;Y-m{bj*hK~etI{X>yD)a*{ z{}V(-Vwwz%*d~ZPCtq@Y^)i`otS&1b9m;64*^k?Uybrkh!6v^7QfMI64tc{Z*G+*h zm=99J@pc#@sr`qKWJ0mTRl`H(eF^D$yN3&=7h>+JS7G^DIzsP1A8S?|5B zj7d`35T*)|0jeA^isO&{9)MmX?LKY+kbx}&ZeRb&4j$WwA^q$;ZSW7D-f_?i^z}V9*26z=ZboWywk{4mFh(G(o!{0{8oV2QrNXSy#s-rhSnwsDj@7jU9tYXzJ>Tf<7ShPJ9&f?zFxKhqqKEOVu8kL{ zg)w~P?CtlR^YxlJDz=mFi`xB$TQ9QX;d2(KGxUV;TW9&t@XBz}h%7yh@Au?Dx)|`h}D%*rW&1_D&_&&uH<|<1<*) z_QYk_cV_S@rHT?26PhykN48k~7Ko?)jefX!RQ@QZrkm52;*^4SO}1kp^5D%mkF^DX zxItmxW#NKo;X8sIk33B!Ic)XAXk*kY_sR@dWDu&o!@+0KFt(eb73YJY-5A7x=o{&~ zSVd)nBfkWct#BHeK5ct>ecFO&J!E@#nMc#EW60$0<{^e0QM}_@t-8ziD9}81ceTaz zt#MYp8cHXP$g*U<_uOxP^~;MIPK`xXLZ3kX^q$TUj`1VSm4{~m^1uRoR}cB!`(Jtp zKdo|mSfNii+|YF{$X1Az=64&)2zSE|s6vyy%#Xn$iFQrU6@I(!yZIT5kXy>eb#^&z z%NCic4+LV2lni1Kj)0nQiADve0P;SSxj|ic55%)_M^>%EbKubqeH`XKt`uWRSN+Nn z2K|Y8A|~hm@U4kXV0KAfblSZiL&JkDv=2osbblov6EgRg?oRbQvU?hZIzca0nUbYM z^4uR&Rj7f9CybO|JpxbZs#@hf2oQsK5$OvfNNEo6Hc2k20}@aa4>Y$@+rTqhE!H$HN_9SmLm_WUgA_b32|NtBZ>YYBjV)rJ_T zkcthYLofG{UtHpBEQWC^s)f7Ca}6=3K4)*9xE09yw9}%5ZuTZLq(w{xDyW9n`;dfc zsXD9wGe7t~&b}mAIzspQvG6Nrmo8ushQ~L3%}8}G!9}vRTBk1kc5ci;!O-A6%}Ge% zU!(Ly;AHLkNH1gqjf0i9U^vY(@Wb^dmf)wWlj7M1`QTcgIvBSqvv+ zKj}&Un{{F%$X@-=zFHceGxFm4u@>9w&Feo2%<&KX>e-nD{V1DSn@jLBsi!X3U+X6y zWIa|YMOwwM{z&G<6SuEyzKh3PNAxPH?AMAU>m;%rntQ4>qnVa0{Y`1Hmi5$F&R)+* zh=+VW?#?_|PRAxjVssAMc0VF8SDz5Mg}4M z!*x(?MT0f#K%XJJ^3QDEmvbP;E;;?s-a`c#(^U}b{xJzSMYy*A>i9V)FQq0gtPPNF z2EC%2{V(dCGA`&4&Q~fW6s| ztuoqt1T=>+HvKl3TM&gbLsYjdeqB=fGe>VlQ2kSDnJ#PuZ(R{c(1`QQ5ucVI{1*!& z_kY;@nP@4i2l>VomJcMSEx`rNhP%t~4THq!(X4jf-k`*gVtm6AcmAz}kuHXf=-yr) zAopTAgBp0(ZGJOZnPJ2{6j#-#^<;NyKVLaMe*z3|ue=#37k#y7Hj#C|1*aaAt+_|Hb>XNeq#X)$lIHU?_;8XxbR0ar3hYEs zYA3Nz$ri67PnxOyL7WIy=P)D7H;CW$185jnPPQkPH}z4tz=@rRh7 z2vmAtnpn|K&w)Wb`kUxPP*~Q?Uk6{QK)P`M3nMr(mKo!Xu+i$X$U2#P)oc@2804l zRkX@>=575mUHtxTuh0Geh9_HGyt($hsL6*!eQ9|)ZM1g&4p>vCW3@uq<{71wVQyT* zE`u*pQTfRH=atw37ow+pEKlg>zz ziV+6HEK4ui{jcGdNm$Zu;D>F?YPpjx*(SZZk}6!v4VO|g@U2V z%q3f7jnO~(`g%7S_{h{_<4PsE>oop^*%f;OZvDRTl`EcL!BJG{Bl@Ir<)O)+2s0xq>pBBTy@pHh#X#!K~`8uNG9IIuZrf9)%sN3 zcvzB#=hdilSQJgnqjSGMTAB~M-65H7&r#s8r=I-{2tY6mtrZDy)-;Wt)XxbtoE|IA zAq@gC=ZOue0sjN382C8W(0 z-p5BtvHS)VK;vQLLZ-bVF}AH-Eg*E5FiLZ6gUOW(?6fh{qX(L70zNYoTg?|?eQXFx znZup6+tX@-@@c0#xMlimO%C;!S5`D@W!J_))ajwr`7<(PN6@Oc+BM4m0swS%9RHgD z;&sk@saVu7=hFPPX9sZ0(9751*FUQ5b8w26o>-j>XKfA$ZNiHa&S$>kQs&Tzna@`} zlIpKTR`WV}97l4xTbIR)TXhgsj3hgoimD@rqvIl72O#0jjgW5WlH4DhF zlfP}3E|xl2OfQ9>9BaGd#%*)2y>N50?ti(()yQA7DikWmI?P|<%8W*EsJ0XTZW{Cv z6MWREIx=}+|CB%+<$vs>IENUs>}%Yu0?!8ACwEi0NgMI-y^a8EAG1^0u{IyeaeY5U zEo=8(yHu53aqOnw~MU)e3eef1G$G85(N zi!k7C>iIidCT5S#J4^MjpmPEZ30*2Q4^1cwMSU9k+pgWgyZ3*)J zD(}NpfvMHML>#<1C_8-3`gD0URKpU+AYYR{H_O#fYfR-?Yiu8@1Z9l#E@m22%-RAN zq_v-WB)#Q=wVy+ZP);d5jTQd;`$q*4lm2@pQLbxFj=|6!E0Q5oU$Re^;LZtK2lP)$ zYL?aq(^()!B`@;2Z7Q6G0p#$kCKcT~j%Jqw)5Z1K@WXa0mB65QYl&un#dy|e-lH&@3g5yTr2>NGJhBt>+Ea8>rY;Q zV_Th)$u=RlrR+y}+W4qoDgTELc~n`CFBvz$41Af66zgTXG6rLw zeN|*f%17dEL+`1S4Bi%QGe|g4VcNWm5;X<59cgDb9})PNeK6$CQur7@_FiN41&Am+ zw$a<49{nT2Hge$-TzUA3mzRFh6;%ZJ)?74f0ByR&zz4c&G^HsZQU9IoQJ?aEMnDWy z4cECkeqX_@o=*>AX|v;!f+wGtoz}`OXoAiIuV~}&)!6KTkm!}_hO@$!&?($ z=HpIdQC0Xy2d#t(M9^@}9h~Ysy>%tnJ1KY{deDzpj8bG~iSkYH{&lzWS;>52tP)6^ z&V1I3WIF0@PCsH4k;JuIDqT4K}2K&C7g(lq23}%z+*nE zkT&ZCv0%OyJYd?14ni1VheY{CGTk-(9N0n^45hyx+X15C`OEBt<+LxmLDE{J7hv}g zh-2!KfNARgSzZq&)gu_3CH{RPB7g!Vo|!|H8v~r6<0rf2xz(>|?e0EeXmf!}$JmpD zK;zu9ZP>BBV`yJL_x12ss)1uw!nwmqCiSAG52HIr?L0rOB^-L}SN z!BX)??u_!RG%YnlNFYaabl!>N&)E>Z#nP;DeC;zhmkLtwL7(1UBwhva6NVXKoGcEL zte3{9s!2(B?d{(f{o>a`Xwqut>&X0lZE+vekAai_0HAc`fb#pjfkRUZvZO~mj6D<~ zWk34sM@>$8dXr)FIn<=z#PopZ`X0k)l>iC{QzsP_Ip!3hwSbfL3u>RvhW_4HOP#GC z%>b{RiY(uki0$Bw!YA8QG;}f49XKCEH&mAC$KOgmiFi>LIB#h=5jmPd(UUIPfVpy! ziX9Z4VgX-Z@PjkHCpEj#ZG-pu<8Ln;I`WlNada*OOX2#d9@h>3>Y7$hReCY()BSB@ zcHOrsp=F|YH7CLIpq~O|uU8TAc?jaO_rls^@J6O7^WLv}TNQ9W7lXPtG}1Wn(ar`U zUurq9m$Qhqf5y-mTD{f}y5LqLV{1WUp94yKjP4B<-y(}|nXLZrm zwdiKSed7A#vYN%cqKk8zWxet2xc$MnYnJ^eXu)yyPV36FEGL55I&V}EcjehiIp5Z2 zlY_u>+sbr#af?>$csv-k2m3Xt7WS^r`guI3ngA@@sE(Os{dzPWQhe$-b) zyzz1Ep32|J|0zDI+6;+L`a$x-yEd|8A*6VWN6S!bVpSL>W1F> z$n7nylq_5&0U}=MKC33;hODp`w_zzIl(<5%!N^ z`D^T+$}0LtjSvuJ4v&_W^bU^ANXi6Wj%PZmAmg@YAt!8C#%kqrW?Zo(XFk%3v{@Wi z!0Q6CP^gJtWy^OQ>l67aH+`ie*R?e@d0MHK0PY?(PJXAaJ81jKC8ncY8JYUzA2-C3+z88(?Hyb z?|tezFm4&X0#>;Gsf)J{uNcsH^Ecp~?Jq60brf@8G0NiqwmRdqWe@k3+(TDc*@vPO}$=CT6pwljInz+H&gao6#`4r*96b z_ZQ;r3+KeDCvrNN>ac1BC<=k{@P`I+?Wym{?@>| z3w!A#kSr9^D^B6u#H?GA^!xWcgCfWEhvP5sb`NeSfqf#N2m>GOfO3lfN$Jr+mf?#Z zZmw&B5w~&$|Ec|AQF?0nF&_X0I&X z$?qzfO!}h{DAFInUNOD@r`-#Y|Z{GTk* zDyXgH_Mf6nEEs%YS?iy`1US5_ARX#mcE2pSQbrd}S)k4vaq2oilv84{N%Z(LmuNHOBCX<$KPb^G!IS^( zS6(ufl`v8C@A(@0`i}PxBWB$A9?5Pporpgl(Jg!ZZhrCL?9>IL?!EBqNXLVDwu(y|& zEc2)I^nk9bsWbV8JMLU?iU^3reKW&fV(}90t}%6>r2&xrMZe!g|6vd*p`_`7g8e}I z_Q9N|{M`X^r zyRo<>R*;Uea`1{P9^=Ib)jo5H>BW?hbT4eivEr0GKU31%eoyZ2ip_6Z8SY$x#5o69 zJsG4qGTaek za(;BPeFsJq4}q_DUEu9zlqc*b(#%Kw7jFm{v96de6=8Wa^)`wB4!EMW$(^_s$Yy4b zy5~l{_u`0wL)LvlQL8lcXV=w7!rbE{TLnTJo?Z=kdH!J(DAb7EJ@voR}$i7fO3Wv4L>2kr!JB`7bm zHSI$hkm$+Z(syO}-K8)Bpw{2NxtN7Ai(as-fFi!Pa9-8&yPCiB{$iGGaX}Z0U-QE& zY|Y34!?&QMcmgMwx;-&rg+rCj^|QY_j;DyP5Fm>y0MY%of3yEX>Aa6YXFn=gz79l^ z2F8ynF{~>}a5d(&t)#Z>pAGHi8+c{6z`#{rspK@q&#NJn()SCXh7pU<*c8LpY*7T! z7jg@Qs!zc&I=Y$PNY@ps(+q#%ovRGvV!AOcp0W0JEUjse?jHaMEHCu+UJ1FP3iBVo zVD|sOdl4G8tcSrqv)hN+d4y8Anyb@*#u3jHqR6_TB+MK_8Z$>KQLLPiyl47>{+VSN zSHryLxB$&_bDGmu_H)S^#6G7Xo_9m38S-OMv0WA_*T|!S;?hr@4KOrr4CZCG=q;u)vTto6O5=gt){2dR(t< zBm~e|)4j9TsoCT4MD1aCJ~D;0aWPWjCZ^&1Ta=|amrEq&*$^a*BtCSWiBY;iSG!%~Mq-a8GCvDdxm;=@baG5*zL^_i}^QpuC(nQST6Q+(fg zHpLYwTsC8Ld3{18+-%-8eBo0))L!aHuj75dtD;E5FFO7sQ+{`8E?CR3$k6h754=i{;Cuik^JW7#SFO7O^|GT-H2T}vQZnL_`Gm{GzOj}fu&*j7# z+$Sh?NbJXM%sye*rgG#9C+Yzbj(d)8mp=UU8qkr1Ci)Qo<}+{0f>@?yXz4*$x7@!c2< z?WmV~x8YnxE8o(z-T6#R<8@4Ow>73q*d(mhX3*DZY(JAD&hHrj-WGGwfLGGk|BfVr z4lWnD>ES!nZQA=Y%U<|aY4?-mg=|hDHUsM?9#<8+mPgV+*ZV$SHB3DA=S|KqSFhE} zYLSoyE6&YUw?R!aVL>m$XgIt0+hCbtdP4gnUy2HAc?aQzU_VHb+b45!sZi!_~i$V!hg4`)l<*fP#B z*YNK1g?gr-ekSG}^hxyGiw5i?D^1@Ao-D>hhUG)|<=FElF}TcH;E(qWjd=t*EZVB< zLawt6wyTCm4n{FhedcCB_i{7wdG;w`QF#@}%oOH0@AAI5wS~U8+dU%hJBN(@A95DZ z#9haa(N9BGa50Uo23 zGJ(fPtI{3f|0MX}jfjhDWeUdWIi+F+x%F;z2PMVqOH}Ak)~QV^xMFEeJq}+HfGL-g zY&~%hC?us5UA|7+7m_Yu8}-s$zxJZy&eeratH2&)$UK2PtSS-gArHjxS#5%Fk>CaI zHGr_{cz+ZDZfgYOw1OpsMV|k4CWdK7iq>!D&&?{`V4c_tI2}3n6D56Jnpr(szP^2U zm1zveYVxy_RYsOOVPFJoGmTOs7ix;8l9U-lWtB?C9=rUSx>xGEz^G5prP;aTQdK<6 z5F(Tz;8o@P;aP_g4@dy`bVAjcq!a}3dwTb3%8g)H&cc0GoV&fT+X*`0B@)hYRd+x3 zlN=9j(?!pyc!}-`7P;*yYI;-B9pS%1fY(SC6koR zNN00plq{aBJ99*f#;-^0@J!0UXJn7KV(V89UIzv8idiTXBcJ$HMJuxhn$aS9lf2S5Vza}F<-gdw5B>n&XPLQ6{ zWr=g?YP*=-C{@e3&BHS5a_(WD2WiKJtI_jZ4MI!dB4^|0;Bz&*gLB`Pl;dp{_$yaU z$DMxZCF-%GqvE^x?ee;WG#pY_!ylIer3p%=EK6oj8||tUu4Bo8eicp$@<)7+FKNs_ zvGRDmWBB}OLdpDGr0Y8`xJS)Cu?9D-RmNV4XB&UYh!v0hcy+XU$_ujOmc07lpKp~j zZ}Ry@qq3sSXkH301)U{HVD9LhRen4+8b2Fkk`{27l9e80{4sFsf%pdc_CZduK`_*! zKSmP3#O!g27t?>#P3{Uj!&<&|$6o$kBS|8$CD;o4#c4Vo4kCw# z=M_cY1W7lUt0Y0jUT;p+U;{Ab{d&L70Wf;_r*rf~mXP~j0d$uu1!%zj3AO%i3qX## zaiTK7(kLWoDkfYa(Ea9({po&VaR4wUs6*R)0azhc#sU1yoBAf_vEgCfXp!seF!5O` zI1P_UYv4Le-n#AxDl0q3hCuU;pOK?#kQ~G1&$HP?cEq zrgUvJhRu)=ca6z5SZ%RM($wPKjIXe$OQNygahvu884Wy_RX_GL!7sZ(+UZOz*Q;Wx zz0NIubKxJzs-t)vhdMylFeujRU&SgOe^x4m@KqZ`&E37@qygT_F0>5{vu@1=Oh3xy z1Y&P9GS9#@Uj<<)edO|=BL##j)h6RZfD8;XXI}uhfg7P*`1DVa0gziS+&kIU*$>l& zgzOD|HRzrNcMz>r?xDS0qIA*EFhE6M2?Cgt@zXK~g&BbJdKu zczJs5zdQM|dAJzm4KZ}Hz6a)bV`p&WIJL7}Rf04S$q^XlRZTDA~5K z@_66#qfQYzlvNwT`L4C-Ic3nRRwokp*pI`$_;9na4vb`8q|=idAbKsl(y5DdPce9p z24yGZmCI8$T1p3Ls`grvbx|4Cv!RC-?NYRADkG*j4$quFs3YoRG%u#R+N|EZhpVdz z_l}zYS#PB3k}IL*Ig}{gjGKsc^dULhlq`I6_KCR!jHXIRK3$4Z?q#&#b=nV7lo{(` zi&c+b8hk*ec3?q9;vVjtAnZFk?Vz2@+zjC`Y(EbQF}B|p3Yd)lmlld2aBM*ObfFQd z>l`NELNl)8V43pq4s$9-QE(gp>_W5=?$%{VxP@$jg!B)ehx(R;%M=Lf6 zx*>H?g79k>Gp^ZeYC~q&!WVYK=YCiw--r3+D{VPtwu^2+T0O@+7mF@ASq+mhT_(Q5 zb!@mjVv?c4U-QPt|JGSuFVZ(P;a=??)cyAnJ8Yb;SjC-fCyoKxqP=KkUJX_eSkB&= zYqQzV7e|Md9NsN}d@C=R1?1pVm6nyi*Sm<;&H6T|I20h-?lS~7K04m@me&|+^Ork} z%Jg6gKE-}A)g|^ezutV{fjtwunTtMMMZ?h`)e4H&9`w)}U#o&lKm@eb6!QU(f8~)v#imDCU_v#F00^U$s2!I+9Yt@sp~mN^+m|dsF{I^M>)X*e(<9 z6$_Q)CIRM;km-}fTovH{{>qA{jz6y~;Y5v9nA(C4^ONE{h8Y;CgwDyizdb4oBc-aw zvmG7`6tSbPP|()it6aA}sOa_6fNHygK^xx0cbuS3Yu0c(w<(&w99Qo`5+;6YCEM8j@xC6ql2}oE8B|}*&SO?vjC$@X7 z0v-lbkKabQK6edoefBJT)}Dx{)c5%p@FH%P6-o;*OE|DA%Cb~1hF@!Sg25Pm?Deli zn62u9-Q0gQhzl7Akk}5PBlk}eF{Jm2M8_1JM8eaE^IihaT>F;|HLXUiB7jHa|0hTxN7gP=g-J1Kir~27< z-EmOc(UZZVY8_N`^W^i+3$tr{|u>d z^xf^XMn{m6N;M!+g6P-^T3^6M1l6+!VB!qM%-f434;Mpsj!3L+e2ND$$E)rY)W0o9 z`rfdGX2dRGgpPn{LX9quXbXlvzB`q3aqlbJ*5oX#05B;3r2<%BfNngkK3H4te_dfN zB>&*o?m-#_6k|@(!C4$ozdX7RR$e`C*xlU8Twq4Q_rj(}VX;4egiBb~92pSpD;oMx zd59kEy8z$Q(Ra;j6jLf*EHORSx=L|y85pWkvnV6U>0^m6^HfJu1+~Ud3v19}roCYf zrm+fH2zf~asJS9^oB$lwBD`7#Kv&~6|8Wr6f4`}3zt=MTvl92S&%{bZPWhSC>t|(7 z8wVufOO`40kKa8`!!MdFcM2PH$h~{#q<4|`B%_SPcc%#8na&(`K2@k`&*L%SV&AYE zGQ?GqC(c!0pFBKIwyYZ&&g$0J6{AG05kwrH^v?MrS-#I*s5_qhnuq&zE?TP@slq=A zn|b(bc)Nwc*xQDd>ZDGk>i?mz2*VMqHpQsmclzl z#EvGfq=%lzhEoh51Xi*;unFt4M;nP>IvrTn@t{K9A`kRyZ zGJ+Cnzv{)w4I^yUi%@v$01;M8kw;11nR@1aJoDq8Ppj5u*4M9LA|U)>bQG&$-!8EW+pKmBWm*0o#L{CxX#urnC(#3b8o z%O5l(={(Ta?;KQXIHWv^$pkAZRo=fP1Gw@VLPO>bp$lJIzf~}ufh`(rJxiM@(1ok6 zXJWdRUX&0Ev9-H;`Qusu(Sh}#yUkO7V@dZ(CBz#U*M3HcaI^gyW;DcQF-&grux`4+ z7}{KZ23X3apFin9;+9V+bVUL1^Urg9yB}4_EzAs_p1G-jyeMCtJ~TG`!2fyh9FYN) zB5PE>+)tV@MX6goP8k;yrfi=&_;Vl|6NB5zNUlLeY?P#9Q)*viBgCtCLykYjPF%_) zjlVjRAG0(sRM%w?!XGmuWWj@Jt5oHZ5D|q@Y*LIfe4{AY%;Nh0*n7{YCbw>Fm?o&W zK|nwRi6B)#q)3OTfOH~ViXhT^?GV zJnwkkz4v*a^PThl_{MMyBX4d~9;PzI0ip8TtO@YeMjKW&ixT z!0BNtwT?&Q@k`Aio5l3`6_}FpEBp-+xM-Navx8uS3choi=xh|gZzg>6QWX$q_>cYP z|Dvh}V&|N^LPHuVKregjdTN#Ce7u`(8bLH`#_}?UWu^eg(zz!NzbcQ-EuYV}vI)HV zRouXcOYT5F`N6~6)^o#|bEz^H&$wu_H0XNjInjH#RHYczuZBHio1-hQpL5+U777;3 zS+I7jvZvzfZx>b|3@x|@8q)F1&PTBJ!wWg}pQs#m$v%@-=+*5N0M^3sY;eteb+hlE zS|UGs_XU%JnN+E}=6ve$IPSdDlw0hIMH?T*plaI#Ir`>Fnb8WA{Q+Lbd7chgC%`ev zq5Q#_=LsRP7pMk($)1Ns zpr_L}$ErWH0Lat)L21AD*V%{x2u^m;_x zAj+fA=U327%MV{HlBVscK)3H+fSWVD+(B=XSwHmV`R$ec+gr}^SaH9eJZtk6^-#}U z=lUSJgPb>2%u8}4S00c?6|_+l(~g)Hfc7%aSHZ|*_vL2OLbo7f}zg5;1W4C>++j3O=Xs za^R>wIX*z&gHS#F%u?rgm(3->)om>j(kTq^HU=pDOJ9wW05!+2`Zs}I8)!8n=m4j2 zyaS^b5Fs$D@gK(>{zaV@6dfbmB#XU|je%ew0DjxChi`x)^1C$o$w+R51mFWpfr3x) zhc;=!FT+Hz3U;kv4-c<_>3P#j--P9gS}uk|q?OE2K6AMZY!bbr@}Tb|=f;|bl~<0) znx0im=YoW_h3g0_snX?A?pC(`^YJ`$o!TbbMnBS3cU!2Qms(Tv8sfX_t9C&=kENoz zE%)Lf*PW#D^-R989@9uZ+gqzk5KER1FRtnYlWVAZtG2P4$$p(tlmQdiRLt(1Ib!|o zn^*b!-Uh%1`egb0v8x~=_->O?>xyq*jpR=}7%30)CB6G< zHBy~U_~<#;6?D)^z}K9n0o3FASaNfwMZ?I)vz0LzA)n=EA`}#LSl+NyrNj27seunA zhXCJol~_O#KWO4--I59{yN{aMCPFAQ{bS%lD%oONBu(pYnp^entqs&64HR)@<56?N z#qD$N;-XMro;bgTr@iIZz)>|HK{qIl@A)}M2@g6kndL|UUotl`eGYtb9ezhu{4HO? zHS}Rx{Jb57y=2=7`Pk&Az=&spcVrDk2R6#pMq>=oDD{3!h{h$WP0seH^PBq(t?`Yc zk2TwGTHI|$CVmuOP6u|+*5C7CBL3cM3#3p_{mX8tg*(fu1_?7So3u1ikBXRE*!an6 zUv6B@yS@i#=g3-WLdEi^>;1&d$W~7QF!xv6Gg%U!(Rgat|3!;1M-#cL+II z>zQMV8tU+V9mt2ibyy`sw-Yuy6&S#z$lPuy1x`Cz%=`!rm9-YdC*}rt79LP%UgJ1E zaK4Ldi_%K}8x$38?9)7(r%Eg$9+;Tzog#JfcF6tY;w5_) zSIUzuZMM_;t|Q6i6Yl`jj&K$4RjvDmLIde8dz!cg)6H2e`1tuqWYkHe^mXvZTD-5) zzLupX%^7f?^-WtkIi(<9%2bb5FzxZ7K0TbR@Fhj{P8y9|GbE+C739V5N+wG7l_TtF z`&)y+tbgs?IW!0BMw{4c9S9lg-eQ1ZCM1BjZR$6DkgT^p&IDbS&X34QZl!$z)}gUN z@NW`Gb>Q^t_D_1JPkxwHv7A4(!R-H7oq3`MZJw&mG-6UF zEij~~BbO%#03o=k|I_mj=*T&8q?C^NwNC>g+2Pe)su7!F7;c!)K2c82;@kpKuv>ow zD9Y$^u~34Ya~Z!jcC`4qdMKEo~q#1+4wJ?TAX3do7`+3;oL8b5N#u<^HN887`ZxmAT75B_w(?#a=z%1^g+J?%H zw~DGxMeW#gWXK087N0JSBnNH3nRW=ju?N^b)|8Js(aNnhku9l3W@P#}{TtIM%usbR zPKtr%MU4oynDm1LZnRMq=<%9si%|!q<3yFL!Cg9dWEh@|!xmBz-iS(OR_J{Z*(tjZwxG z-C=QXZqAHTH=@Yu*O1O@DzG+|cmjqIC@K9#>2i<<2!$Aba{k^lqgWc9&NPvem-7M) zD;revH);euk@})>rIG5g_q8`{u#o0FdwP9L0&@QJEg{ecKc=djj^wqu(|fo7_on}K zU__r&9_+Lt0}4f2`ZnT(Y+erE6SI}Feja=?;fG4LL=<;~d8~Z8Y5O%Trlq{VNLBaJ z=9A9sXg1d-7;_aOTdX>yzDrH5ceWWHBlJeeU3g!PpM6A=FBf^COp*BV_K2w2hzXm1gu+p0Rp*QGY_m>zzI%*1!J(#qa6 zBAerGnGP=SJ_A{6OD$+;enYwkj3gO74kwY%IdI&!C_;rPPDNlMKt zv!N6>{>mWSGQ>&fx(Ykp(!q;$k%%HQFiZEV=k`{kF@C|vi2u%lzuXxRl}eLV<)Y5+ z{=p@muHnGb58_C6NTD$jcEox=+oe5IKW2KVmO4xTSL~IFAG0tGG*;=ca1-|2nSBmU zV=Q^=?clNvmb=>bXel1Pf@oXYbeVn69AQ2r%Un3+txm;kK9uubNPQ#XGzB%SN+y~9 zZcEG3bv-f<-Z}2bSA9>Xs-ihy1=vfzfu06o{(@F4{fI!=OQ^&dJ5X2oDn`5Dd$R7I zoKd~Ib#c0^Bq_)J)y15wlNGbu)bjn2I;}zZqouDq(V(qY?$D)1{v0da9rVuU0YqbH|wV$Xm*hjzem+x|!+kraCi?<&Y&1oW; zcNC$%)%CRBZ_h$TKJYSq;^e1Ny>IzYd`9SG3ZA7Ov8C06r)L1UebYGV=&I<6w8t*R zy%%Aq+hmBR5wS9SQ3#M@sqj^kaGIG86*r@UP)}Yxb(%U5_g(&eq?__#0d#yz?>O^; z5|0obGBd!bxc?5@15jayVz;k~p1zp|3R_^O3L<)YzVLQL=*;)^VDf>>O3!E#@)eVC z!SZKDfFLE9^GbZlUdsP!pq=d)gF}g20434iTI|A-3Ue?6qNRWWiAM6Peu<^2dYmKh zNa?mosmvD?Pymvi2?-2;MOKlN1D(;q?KePiE5dGvF@e?DAkV*PkN1g6KM=poDqSY)!@L><-0E# z0l_7VZ!x&JflqrRmVF~NKUBJWk{^QylV~v5Vkwx2jkY8NliAza*L^}!Gwvr3>qJ8=DmuB z1@6=dOABNpKL8F;;Wx%#MwNZCt+TuaNjJZi75?;`g3Um@0iayS1DM*hy}bgzYwaxV zpU5^?On}Yrakcq3kbfuQG4YZ&J|31{sYB99&WDp5H|)xmvLD)_F`Ip3i@ja4t(&7M z1x#!1J*l7l8p;-C#a^$!T?@&3rNyB4ih>OOkcq$OR)hC`sJD4?V+8)2prQWR)Q^{b zb_0_pe^pQPZ9zslBrOgN8|6yBLaod#)z!|L5h}7>3E2Y@RyLquEnLU|M$@Q(;6%WF zo|j&A{tqw!x_sCAh!Kzr+fyf7o@!XCCC-*(j|a^VW*%1QmzeE_K$B+VW_2Zyx@ZC- zn&I)Y^-G28nap(Xvmkd^Gv?exHAwNz>NQ~WfC(m@(gSu>dIdxqw&7r+``<{-3Na-% zD9h@Z0zErQ^j@V>x(Fz?xa{N$$I*_3< z{6;BxXj6RkAzMQ@eAo9jtigqeqKfN+J0oiMN2brEcaUn?JjVgA^##4l=tk9t(m>5> zd1W7Z02_x|*j^O`M{zF2W{Xi5DSVyTOoQ#tS&FLnpKPIr&EE672{~E=8qK2?QZpAe z*rF9M=M&1GFrSA5_0;XjyJOPOI*b#fMwXNWq7%@ijO=7Fo zXJ~*pfG$bsH!N$wHn~?^N4=xX@8iAyKwN|qt?Y}8`#vwd*A^cdn)*a8LW`9l*UMh+ zWFUS!u2po5>FK8~9N%0LLShSrB|QwF4+iLfdhJKNw`Z?{x^7$L=2@lPAT?*I)MZmx zIyt=0Jfz4BwamgjEm*S3&bk zcU~e!C?zmrz68}ddE?UBTO3Z!qQ@_eXfSrdHW{UDLV9)%1a}>6&V(SYRRg)EfGEjd zvWvjaja0iy)@{;HW=p^FPv9BwQxcB09q^}3O>NkDK)rtZ2oYB*28Ed%zZet!C)6Dp zP*^_*ht68-%AEDu)v^LJs}>d1yGrT0OLL2!@p2&!7 z9We22iQP|5O4k;=OBt}fQo5di3itb30|G7MCzYT@f+bqL?`^y{)5iTb$rrOgvrlIGW^}9c3K$U4 z;=gSKQ03?7^I#i(;+Gsh%l!qMw3;)*7qkUpc)T(SN|oa8IOA9LV{(P{^U7(ukA^R_P4nAG{neg{;+z1s3h@n$4pmgl$q@qwKX zMyIq%=G4hg!*AUSw1mKvgcBg%H)homZm0G;ntN$UmW-7eIIlb`mJktc*`Ce9OZQl-F zSY(W{Yvu)gEt{y*w>IxT0p;>ZyziixflZutfP)qxpt0bgiErpk0By7aLbrc*v)Ys3 zZBQF;gE~O0$uH+u-(?@ZaL#Ufd=bCKK=bd)Lw@{bTqFED2I)%%&Qeg>v!=XS4C7+T zyv3#X$@%3tGeTq{UJmEt;oeKVU@-Gerg+}x>_w_T>P-@MAGMDg(PltoQMDhj4!8fu zf*$~GMg|^~vtRx4$UUa}n+IWv%UTcF^aEk0D{`NY%{g&BXvT&Chl{R@U z(NXAI)l!sCK8xKzoC?|5>mXz%BTLO#8bAwsV$Xk3Rlm5O?*+~U7*aVu3%ig8ClB3&(69K`OD>ScBWwdK?mI|1f zzcU4y-6asF+SampK@r3i!zxJ?n_|;tRu8{bD!oR%c=c4V^o5iZaIPhpFp}Dxi8VggJ-(f3f39?(Z!@uB-A~2z@3_tsKMp zAmzebLC^XEe~9$#9Dk^EZAW~aU|sxanPN5G)`PG~9|Is6oR{gc@=A^29x{ z^Kg+1g?x_nAk`y#!9#U`0*!lzFDiN)do3euX+=`qtj#6E?ZveufoQRJV$G9xvFfed z?JG;0Xr}hr8)~m0LLZchNn8>wkO0=0mR@8$aA}|y_>hY4GL-(gf)!ZF>*%*PD9Se) z_Ul^Js{Kf9ygM0$i+6eBX77N4&n(hCDx-Q+>`h8y8gMQ?(#-ev6Ewj7tcUd0RSCjuBb8r=q!iR17ejjvwPP84JsQux|Z1$t^z z-;ONlC!%$-10wkY%#!Q>LB0XklmU(WRctdR?}P8SM+TO!iX}Z4Sx>t{P8k^C&l)DT zrNlNP95{12#oq!No+6cQ^11KBBOpoGv2t<$=r9}8?{s!kfz#9h^hR9DDXWshE?s8I z#ywW!_4xoaa|2{Y#l97znNsQa)8*T!HMm-$44)hw?JNmMY<3VzxVS^ZkTz@O)Kn@E zvdxx*GWnqZAkpLatP(_YxA3y08pNQlLZWa?X5T`><qAG|x4fMv8=sO*N4ntn|W9C!_ zUuiQRf#oH<{G)Xp~$PM;j_cz-Gffx_TjpFQtJOFcnD}8J2LQP zZH~Rm!=K!e`?Uh}$o$KZ-hZoU7}%VN?PJQDD)7ma_p-Gw2-s!N8M?pQJ5aW^P96I_OK$pDvZb2i^|n{*0!5@6duA!yVEc z{-~&To}(^bN0cj$x*wQi^qhP#;d=123*?-?|M{3oa!C4o z5mN73vBl@)-P6GOiMF9W4Qll~_SPAnwaCvbdU?_YBMbVk=Gs@*t~3-%kBUVmqs5RlMoNd^cS%?CL&uH z#NFM58h(Fx^vdFZaW^qZ?VQ^6ydQ<# zm+FKMH(?FRhzoTL%iG7+%51Nd$R+Gh`$+W{8kXzpNxyl8&dL(;FX&j8e}dbyo@F`- zS8ki)3N+^5U-P`bv!4RrE;PwdGc4zk-a=F2f@F4~IL676YZ`9GmdD$i2mAGx=?=V% zadrD!RnGfPT{dyAT8y0N*8~V2f zVwFI z>&3NF4X<Tnj%qWK%f=E~mt-xd>(&iL>y&q3y4G=IZ_H!u80 zx8J)82oT}$`~G}Cf9SOMNo^ZZGueQQ2c<5d)%7TktHmZTo|&M{&X?k`iF9&>rmUu)%o zTZyrS^+v4g+bnjJUx3e``Cp6P)GbgcZ9cFGEJ!rJ{m44VB(FTAD-V3O747=l2XqIF z?6+Cc9kvW^CUgw!*r&=IB#bSNZ}VRt0mml*6j{Ak5XJS>I9=j4)+UzjCX`G@?xv7U z*#&%RZ%_bR=>caeb7};DhIQ#cH4v~I-gow;08xt5ZM$6v5JLMc=BXQ?Jnf`vt#p0` z8Kda3Yt0bO2=y`vyo;V21cAq(0q8!v#Lo$YrO?f~c?I2o>qO7Ta2_$A8~dt&?~!zwR!uab{PyO2WeCL|$!^{)2trSqpmLZPCx&VJR5XX?bo&T)oGlWYm`J+G;bk zPhwJ!mQzI-QDFS%*zPZ(lo}~xp1O7vcrI+Hex}qPME(^Q!SP|V&poUuy3TB(9*1Y& zw|EH+ZDdLWij%U8>TiK=*@cHLx_@`)0ntLB{eU@=!AEO-~peR;~veNij?e*@aMURSI;nHCxMsdid!qY zG_L!ZvJ|)j`FiC_rg*T2(TS>6p8-r<0mp|Bs*|UTDsH0YJWEDa-a3~ik-n(7ce0@? z;z-W=d<>oOT}{&3dz)c!Nb(l$*6ZQ(7UI>gYYaK&tm--H%-?ru6O*JJ&Lfct=|E7R zcWINRw7J}oqhRj^P-uKyoCQ>5O5L|V_*TvqK_dOm00`vba|evHuNi2wMeqZqz55Uv z^)J9Ya`|#}o$whzG7#AXqt^I3)X${vFYx$xDNFNsqX9Q@WsZ1j>qa$4Jx06(xx8wJ z2qNqso^{e@TTQpll(uXHocVW?i}-%_(%xJk3veeAbiM9th=xfgm9M1(A+4|3SZaUj z1J3agwYS~OXFu+sJ^k8SqF%R2RTqfwOo?Rpd+* z%s6+n;>z)7Ah`O2BlRMj?sQUuCk4{-mUwyf5j^lSY6JQk^-K#i8N1ka<_>`;{e`g^ zUv2-e2jrf20jQ^jWN6{t48Yi%0AtJIez%bme!qpSt4e``n~U!|x?I$!(htG@Qkm3a{dWB38&kq@o7@QJ5Okm&%+7kDXc0R&y>7O+5O-pb+oY~ z<#(v_U_wW~kYgr3R!3Uv!BE47cV>8(ldV~xPQ$BZj79T$MZ$vym;wQo<9nX5Pq7$Ih=)O`3t zlkyd!)Wa$D5T(yJN}F_wRJ8d+%x=`)Aa{|zM>+lc)lu(pS5HXMgk?$75l|Bc#0cR2k#h$b@$)Dt@WVmk?Ad5jJ+F7y{`7Au_Frv3^%|>=jR}AQGp&RylOEMf z!5laWjI<;7c^BrR?70ZwydUnJ2oPg8_hWU`f}ldIWE)n`-2~m2eV#z`toVDgI)`o6 zy5y_(I#%4*T356$J%QdCXmwf59FAuEFuq7eu@&k1F!;{nbS_ucxzGJ*+#$;S=-6v< zk8ItUBQ2o%0DYni^C=^M*N&STpP5D+ocA>$XoNWFC1{gi&=8Rz=)pHtLJYcrBi|M* z&L#1QNQ2m={m618)L^H=2-9}FP-rgPwk*!-6RksLR zV_pnLOC?I~<>g%{OdrM`0{I?60imRb&Hay7=T%l2`(A)P^1+nMW z&RWNe#_G1RShUXiY1_=5=Gi_c$lgvyfa?l}`4Tfc)7~;)pmg*h6sy@U5Z*q74*W?fwu6 z#_XnRwF~j@TL$uAa6>Y$+k~!+5I2j~r=3Q$4COKkH9}Gv@@>JJ)?O7Y{hkgPEE@=O0%{xa z6dSjU`@GI~k%B70^2J!~l!=y>;u46;jZgCY2dli_|QBQLgO{DAQb#B0>LSh)B ziKk(G7EX!tC%n7F*mg}}L<1c_o}-psB5cC<v zdqXdTHxAs9=GiaOV>TP-cAr(BAfv-0pTZ@8rwabVCM6>PWBY6Z?4}8g?oC46{qSVR@juW+f&S(nTQ3@61)!WxFUPU# z=hgSWZ79%-|8Z4_6i$N46_C4E7+=XJxS?f5wUN`h-dS6rkZ~*>;!(YhG7QRCl>rOn zSEJkGs+F_lw#@ndqua5fQS1cf@AT-F_;w=Dde&IO$wRs2|i4 zV9;pxBMJN-?%h8NFvzzfo4FBkQVH;BD}Q=j#0?-)SFoz|!yu?_d;%7rceJV2ZWBXj63r*SJ!A;DG!95J)IB5x-@ z%|(W%2YoP>R?z#vR$N{_TTFh&n$1<%rEvf%cP)ZV#LSU>;9gRB`6E(9=Is5|K|&k> z*2j0!d`Y?y*=|jQ)sL)xd4M^!`w7ifHa0TENf_I16Pc2_a0cgpVAr5Bb$Gl!a7*)0 zNdg)_VznyC;xu3nMNe+#mCg=pY~-}%9)`AvUvr^LD?sfpIh|lBQIHMqQyZuzC5C&1 znCQn>e2s}3Aut679b(|W#DbvWKDrZ&;#Us4Z2RD@7l)hU-?b7i3SgxIL%_Z>dvsBc z5R*ZE1rJ%g0J7RqCD$4uVR>4jmOt%i2oLZ~zX0Bo8E%y&)SynNx1|^%fYB->GqUAL z*UDiX*3F|Oo6^p+2u*jk^hQ^rm3@^C3Q|%yCaOK+Ww7_mQ&^IZbCG=UvS4&%o0ooR zw2a*SaEDd)E=xQkc~y9-D$~XwY-!i`0%{_1Vr`8so=)SoQ%3mSl1w60qP0b)Kso_1B7LJnCDOu}N&m2vB&Clszwb7DsZsUHVo` z(@SQj@eBxVx<&!&I%5|E<*eqSIZFT+q`F%e28?TtqePhij+G??UOb7r?U$=(MI(mV zmRiAn`h`;N8pQaJNFIbl2j04;FlH_u%dO||SM$Ivx<~%ws@aB>DNA};*|mZXx|Ic+ z#t&{pT{&&}ykcxEQn7H@E3M~fed;ZxOr`94!wL7(TLYgUkNrQGG|2Hc__vqk!1yCq zX`Y)#vcEZlQ=EA`=vXn9^W_V%FXrQt9KjWF0{r8PyM!Z2FhUHsf}jL&4lDWRzNw;^ z?L7!KwWL8;>RmR<5h#>vBe8NP?wI`G)CYp;9{y=z zSqKPhAiRGb;$TvA{vJ^G^lJuprxs9Jq~^I@sC@KFJKWSeQ%n-{=Ml<{-p!B)M;V^! z%yHkI=QQ%o6Zfz^joDt#LuLR~^T%WK3}OcE<(gw!&^AuCrN(bRSRCwymnQ3^<4C1_ zW4`@_Nk`LW8sAhOSAQJ=~ z30)}71L*ceXk>{D>;v|{8R%q8!xFlab4vz|46ntC9jW(OOpBkvF6|!;E~tIZnC^iE zZn=>Wq>$joD>u`L;f~|pk3S)^SnRc(33~fNSztZ$zDA^PX0}6Pz~J5JG;5tx8(@ZcEp%I z;?fkFi|YwQ=2CBMJpq*WAME9um@=H8n>cx`tu!>h853tGGjI#^FK}@WxEq4bD&y zfoNLV(_yamB4Q4e*Bl+njIxn0DEA+X@FyVfYj7WX615J(Y*pX0)8Kn+*N=RgYC?*+ z>fD_>uwWgWWnjqUvoJI~nHk@Ju^X>wByqeIZ2BJhI8bvVr-{8$u3zTB`Ofs-<(;d| z8ycFG0Us5WA^=ECMC4BV(99b$?afiDaGbc@A+h#@S@qbyimA8E zj{bgk&EC$E{CjOy&LP|8D$;DI)vm8}`G{%K14py#++Sz%p?uE+4Y1W~_CI~H$Ckx3 z30R;jKmR4pBRII7KaJ?@XmalESRya=otM&CG22*J?rc!j`v4E2i`RQo&!-jn9l3M9 zS3&d8qvp;*#3y8_iNV2$=|veroMIf3%*%1$>F!u*!&KS{rUI@6&(tH|Ky|9k#}eS) zP`=$i1(wP)R!JT#l_@bs=s4aV&L6l3A{hTYvz`2_B$(}d=F&}{AgIX94sQktfK=eg z|IG3Hah2gIG*&s1j!sz^ifo&C6h1YqZ>wDpXZ73*-tfiGmz8Kjcxi9#2%MG>B7AA! z6O!xfVn=L=`@xp@uyQNXw@xr-i-uMp=bqW{4AQd%yIfr6{Y`SM) zn?q$C7}Z^xe;Ae3DY$Nrahm4&{$TnKeFrLfHl{Wfvng4zC8edlaYDE*w^rn9l>m13 znrDq_@&piZ5~8T9r{J>znh;^<0?GChsrx5y9N`-cyzBRB%3z{!EOLpvkZu}*-*kjn z0mz+oa^Qc~u0@9e|Io81$*%+#!v zbN77ABKCkbPV2)1{RFGtcO4rV31eqDDY(S^g6|Zsi(O=+RmcoSGmqI9eJQuw-+Sa( z!+ZJ1ibN8AYpSOBGl&sxSP8!T`LFKKbGuSJ7`J0`r+eq`x*ufm_bwP8O7Rt@F#6=& z=iOA3Ns~=9Q%YkN*PoVbcxk?)^gZy#C)3GcT_{(RjUN|t|S>oA#A9S}(yrBnkp0T3SVMH(RwW(j`J z<$Eq-&omLmcDB5|a1onZq@AMJ&i~k1nSJC4Eq#0P!s;J1}vc?{?YF(7x86tF_JRF&rxWIGp4#Qsi{AF)Pstn+Q#frfzQYwbXlLh{ovUkD|K?s;UkZJSd0}DCQ2=oqbCV38-Xl9JPQoCzPS`afQZ-4E`CMID zJ>`7OGnY3?O0`YDy;}{+xWW6I1IGreLb9AY4faUS9&)c{V&S->zLSIv9Yu`apEhp% zvk0KK^!S!ihS!upa@3LV-6H`_W`x3Jd(tkNp25lKC~pBgLJ)Ld-k9zX57kuv5vl`^#-`AWwu%cT{vq9U&f#?N!*?f_cAw41CfN*6 z)J8LUYc8vF1jH)e!97>M3yGqCVFWK3emjw(cDONdlI^zwWlWhI8%Ym;v2qTZUqeRL zPLT-$ZtGeLJ^E?H#5WrSiVcMo0mDOpIp|Y=jyZoDtHraOHP0&;D`?noDP{j{g=VXgI+V)MWTLKc;Opfn^$IYe)c`+x^h)@)-K)?MH_0B&$kk ztlR=|F}u_?M1<|)MW`m%g8QfNB*xxqNdP`=!r~wV1ScnEinYFfD1)rlk-ug``Ai+C zHPkE|PmVlz`011)0g2)hjwa5i8;S;T_7Ys&_AeX`6x->&Ou2Yps2Q;O$VkX7s*VUh z4OBEA*Qb`V7R9BA{n=wc9V-WMIN=D*on>meA11 z(1=jyS{GLIp5a7-eoDd#f5}2;zKI3J(8{j#*2eCRUcIUbrb|m$G$mWrvW@&yoTI3= zH0+Cc?Ylzlt=2E3vC6W-0*OEJx=&UVPFS+~Z7wCf$?6MX=%9ZwHZ@=`lG|tf&dkXP zG5dPB_g-(RV8XeIg6Y%z%3063zD$YA`Dt@I^_9ReI@LQJZl=Nx4*9(uTj>&`k4c?8 zuknq3v+o;}3q5-f^C@trZZyPyI1S!DQ$U)9*>?+UMR@Y<-8XXV-#?C}9H2aIn-UP|K_Ri{`HZU+ z2ciQPhpboowv?k=?`jEaZlTx54lScQVx<$&GLcGtGkII0b)s4%a#su-3z!t9c^++dA=#F=j`HGkF~`9cyJA;fO5j%0<-FyURa1 z8OP({Kfrow36QatRk9hA#gHCmWD7{mIl53Dzi-i^LC zBt2J=CB3O*(UqtUWWG8OQ@%^DI`l59_hDW_JSHbiO+3bgOkzZqS1>zVnpzcT4Jz@xNnS*S@+DCBXBgA*ndHzdm8_U3J~0h`zI* z)60dNmrss-Wc}51_XVwJnQpsQv0t2Iv@bJQ-Y*>*y(<1>EckoX@$Tg*x$T&B`l)Me zl0)6F&vUfLlTW3d)@Hbv-mRM9nnIroM0cvta~T?*%Se_fEG{&lHSrKpv#IA_X;L|O zZ>0B~+21FmP~vm_@*B>2+p@3eL%VbVhubEZFPH=JGZJ3lbLpjTf=p+m=rB!iMfeQz zxcIe{a7;(<`4bnWOA?q*=Wy>jrr(Wd@PV`3w<0r+HB)ar-7`?b5l^I>90d3D=kW*f zP*0@O!IA!6Q2rq3@jJ_@n}DV&(QjZU#Q>!{Pb0F894|>i?50^zHOWTKzPhQ6au77` zQuoJiVFd7Vv?cuFjS!(E5DN};whjVd*#IHRw{MbPI?ipNRQV0Hh=DdU6SMB`mnoh? zMMggwCg5|wrLQvf2x1IP1z>LSF*HuJ6v0enSgI#`Ie~l7Yfp5=m;y1T$n4)psKq#? zB$POYUu?jHQy~TesUE?dC6?Gy{#1n=?(Jfz5;dWj&o z2%douV`f&Pc1=hZtEAZ$?i~bmk;JDYziyfydysQ-m^V11oDArQI*ynTYPNH!^eQBp%*eMQ}LMgS%LVDL9$L zb))MEF>Xeu)+(8&5tltBLNvk6o}ne%m&D&JY1+rqa@e* z_?S8fT9-sRjev-Ps?g3Pr@a~om~=Z6VgbefYvIf+0Zsk;YD4M)r^x7+qYq3Mthyao zb;~$n{A!1T)h56e6-{u&GEBpw?l?9RV?)M%I(F|oXRcYXY0C8hdDwARA*t=TtPoBA zUc%<9y|9uI?F|@bI|(N8YD(P7=V9jG)^7UHvWy7YBLX)~u*AN*!A{R3Q8L7MW7hR2 zY=rQ1xw8h7VBA;*DOTfwMNbHW?N$lOIyw6?rgmUgmK${E1CIT7gBHsBi%t2*E+Csm zgs+L@0{%Vfg6mc0Y1nuKKAdkh43=d<5y9`jfE>@lc)WAO1eLNPT{`B^I(UEtLALxP zY6w2dZlyn@KtUKd+h#7fW@iB0T!rYn;yU&4iA%?={Kw?ghodTMnEwkb2f6`0;mQ&c zPlUQbzNE+~Ts6}I&w&}4n9zBVj{{T|e zScMJSjP7=uwvxC?Cy@B)-In}7hB14QqMjZD?CDO+r(+6WPhs^k6w4J-w}?4!#GU zPtv}W#qN1A@BaoU!)w;4T>$tW2b5x{r+bv%iB(P!0>mE!AvPM&XUNH;X9e5<(QU;` zTz>*19^||7N&f)G%_coSC7Z1a^MH~eKvgw7s)Dld??wn9wR|iSK$wcTAH(T?2D9Bk-c$Yw8AgQ5FMJL7)8(#c*#4j2n6S6^)7?2#+{pKPc_CLb zXdFB^a$tSnKse#1?+5;mT@P5ZG;Q7wU(-gLt?j89)Vm}&Zr!%!yOQo(r(8i-*e?i) zP2tsZKZL(@ z@!bp2!<;cG*|u6bS4iSMKhBuj7IZIFj{vw{-I2FvArwuwHBBJEJyWg>AjP1|12W1$%%{^-w-3Uz3hw{f@T?KZ7Ciov48)XI z@oDy^?)Q6bNj1H0Pu_NnyO70Jji#l&B~S6=>Yz@wa#2xC@r%8v)yh^$oqM?mo3iH0Hu1r5M; zEWtz#KWNt5+P<8BW|Cvzm20r{GohAoT3&sM3LD#jLqf#>eUJBr4z=2$*CW~ymJ zt6pe?SgGa~oaWFd>ldRT1QaAfDEvTC9)vbjnVld%NX5Lb-5P#v#RE5`E>l5yv2~!7 zw1c1#!jspB2*BqjQ)pZ)@jTyBbmjN-gK4Gm@_XWqoTpqAq#*DVy4ISH51mK<$e^Rb z=OuK(+A*orKW?}<-P4$~`0W(_PVWRkA2hpgot;L=whn5aihg&=llm>7jTA=ahsP2&59)H+vR$$r8cz1Uzfr4BG07V+vpQ0YM7 z+kqa`Jf?{xfG6KDd&EwFV^cN{AOc3j@oyLr%`}XD4cH?U>Mhs*V$mRheUbS8u=mzc zQFd+nFj5i%DxtJWgMdhi$f&3YNJ;0YNXHP;Z6P3_(jbh0bV*AMq0%9ZQ07%g6BKFwtn(4_Jx!NsaoKxfL;AkRrB>&qTN|4CJ zpET3`U0u^b0h=tWwl5Eq79by#1XZ?p`=3RLU8$iz$aMPP4&}z+2bolMplV=kOG$uD zumz=prGI=+Rp(N0&Z(vpz>5Kn$EGVNhj?S*4Vww%kM#7&90DL}abiWMiTqFV6w8A` zNu2&sht03T3_V5QgBH0bfZ7aHb>*NOHnRZqqTf|rnC?m;csOidXG|{Wls?MFw}W&p z)D3#r2J2mbYzZWsRo9fAYbJ7aP0MD8XHj?8Y|77yF5_iHDM%PASL%qV+6$I3MYnrP zNWDw78;o$PEgHB`d0y_?5l@yr@Kga;cL|fvWMF2YIh}QBmww8 z3+q35Xr~&E>5H9%_0!>ybbUc^+-R*-uv=BcVN1{wHa)f$!){f+Yvr5v&A&ScyStX*`U(& zKtYaXX7nKX!eN-8=jB2{y7zYW6^ogO4eD(q7i?`fCusSy^4139QwWDHyUt5u4I(Q2 z^y{*TnI$m)IvsQQ>_$`XUEBUVJ+Z0y;iq!22hJ}Uo&31IW zJu3|v%+aZGS`!q0Ul~O8?P+4P+?_7z*(CzA(sY{5=@GdsC@{E=PXaZ7ziUyq4_ z^=*$4hrJGC4|dv`*_|iPzbM+1eFfrI{WYW_hACUO(0fN(dHI@n-SsVM%h~*%m4R63 z2OH1kMV3+eI(*yhg!rV)9w&(3akuhvt9aiE@2+vhc)LNjtsQN=k&4QG9m(Eq)WnNh zu~6?9O-+2B-(Fjv$%)htvPr)gI#k#r0IX;Nos(PIfLN~FUsvyMjwT%{G|n&Lcf)y&sF#EQ7f*G-fF`k&n7=n5hEo_T$Z@YFRbUwCrc-TNj zzpH(GQ^#^aZF9ZeN`1**y|lJcsUil}%gW)Awx+XGi31k$O^+QOml~X%o$OkC<>6v3 zR-&+~y!d9~%8Dr_r}s`XU*_U!=cNQeyt$K`(($i+c|r($d7qc) z*gyTk;63fM0ahBOs}TRMY*vjP`0}1A5%E8@!@t*iI13OnS@U0BRygt~f9}FRwD=x{ z;LGcYM5O<`fe|r$(gb{DDMK3bBj@)|KKq90F<1k9XMnNFZ_c%lrG70@MLU0BakX}# zCVOvg$TYpE$SL7{{AOxbnN75VakVnF<7s>&`8~dS5aey&^mpwQ4q776 zkC>7@FRg_k)2i!{^z_=Ok0!D0jE1AbAGYY^r^CWvrgh3YHu3Wo2k2T_nybG&Rb#|8 zIMf$ImJ{RdrM7;vaR^#@SG;6PLRNnMrt9gIPbFoy+gGc{U4F{9JuV4eGP&z$YaMK} zBfFo2$N5&{Mua4!Q^cg!C)i?P-uiJzM4e{Jxd^%4t_}O{1=OxoExs-ETBiH? zlnArI6}>`@@PpVNTGS$OpE6}H3QuM%Rg%H{j+aYITbX+Xt94?;{i@4KKd%$~6P7AW z{a(-4er@%G`rTV8h0A?4%Oh=6_70y4y(ZwEI@Xypky=l7$2@0vy?1(TODhpJK1rQt z&{7up+Bv0PU81NhyHE$wsht@sm8-x7!U_wt5E`k>>OKR!5e9`Oeql|`ce3?c_Qs^_ z)IR@}&+HtyB)H$U%(B*VGO4l~j>#N?dL`ev^U~Vx$=AC=qx5k983k>&pT?G8#|!=E zq?fkfn5eb#+Zb?IHnan$zh3ODU1B%wM`#xI>{!+<-!_TRMRl01r`_wKdj-Qx1MIfk zN1J#7;(rJ|IFC)?#9VFK!J#93;zUe*TG6t`f90eCBkv;@>b4V4h$M_}`%jH__h!nY zq}6gL&>Z}iPEvjJ>~M=|c92IDT-%ql85EM&Wp@SUTmJ~e+}cT1FgFw;RnV~DhoawB zRNud%Xnm1MB!HMCuYRdQ{)Pd$zmIo*4W7+@vTEL_tTqQ$E^#xH(|0UU7K%@sL}e88 zg!uh#gZePWX({Hh(ME1i(zr8Q$6o9mh7AY`j5I2&a3IZBq{=W>v8 z{_QywM_AZL{=}U$6PR6DR?l$Sl9K<(4%Q;nIZb>b0Ma=wYvMic{+W}`$$vLZRIgHW z-4VJ=gUhy`NNp8^*j@Aw0*Tw7*;;&dn4Hx-wQ!91;m3KMsa#s0Q8~3^?t;QOx(5k*HzqJ3xNw-`EAGixj3i=R}dury@H-)n9{K0<~i(#(BSeT^04 zk8KzsS2(YQYwkF6eR|L)(hbx6bGrtz=H;2L^y8F&J>2mdXC-Eh`sc8mLONG9y8x!0$O<;p$h`4p+Xr4C$)HGbUm zDP%64Ix1h7<#6|hM;-4zNc-#Wja=(RJ&%2(4RmqL51~377g6B% z9_ANRPzd3ld;&5$+EkgdK+daSXvSYcs+H#zHX9vAx6f%*+NA=FZdaXk_gicgQ>WgfLWw;{qQ zo#a1F1CR^#k5^;>w?0B^epR120K#tWBzRH?53qudol@MJ$g_58cd2Z~{SPz=l8Dp>(dLAM{dlfA^aV4>rLf4fFUh5-nt2<7nf3ODEh zsy=h%K8uXtdU-F4ydF*(c%g>?6WmI8rUe5}X5JMZ4H~wZOqs-luAq|5_oH>-E;}JT zeGLEkp`wW};Oj*>@d1Tnyzo&gkIUlX}5TcIHLghLqml=i{=Ew9@5aYz&wO4%wszvd5 zr$AseN`Lm>K;7XRRSBB<(aM@BYZd0hmLq19*(tb8LHK&VP^^jzBCs&7Gvk7vQ6vkE zcwK(YlfdG#Z&!8t1F@z75EAq7tGZ_ao57l3t-%#kK{tQnE21*5E^5*DWdi6?yyKs(M1Z|2#_tpPhrBqEWV5gFh5J)?Dk`O|&3$xqbS3RvYivhn zOw>9re!Blw@_|$rtSApxk)Q8{I&-Zk?(R#R{HWb~`MbcoUPLu7>a zA++g9f5DI6h6pHU0{PV4OZA+JC*n0|0w9#cw^NUnN8)(q1n_}v@7aX_(3y5ojf}tn ziR}O2%U(hZ&N?&bpfj@MF@#pIP(>vqD%0pT{Cr?dgde-%bbK}!Y#TFhKmUWz0WG*c z9p8WI#1B|*C?=yPY{`TcS_Tb^lb8-wKxc4bLg+U0M8^3q3q?h2om~tV{!Psf17$4Z zRVJFvSg&Z7nw=pL*wndEnHIOx%ifrWsE>SY-&~@6c+9Std+5L~bsLMMp%Rj4P^vER zF|aKvh4<^Ie8GX7>NPnY=*~t}e^8xxhjesTY2r0qoVCi-ETDy`CG2RV17P_b*CxrI zP^B0Bu5KU~QazX1ZXg{eg?Z`ADY#t082Y0grX{cac6o3kxb1`{0Kcu5B-xu}2c1&c33D zpHEY4G&8C!uS^}eeBVUJ^8lfz*1cB%Uf^+9BLjf$cEk1ydHg~Qd9Th}dfab}1Ra?@ z8qoaqWhMM0k^ODJD7b?HjJUnf($O)qqkp&&s)E%u!}>Q zIj3GvOq4X7%<38OQmrl2cFQ)jfAikmJ#l}7F59KTjn6YgopC;~JM+pU8>3A3eSr!% z=fc%wJ!|ed*DF#vr6pRKR0{^<-B(G&pSrxU<<+yYq}#|8eh~?n3NqG%3em)9m%HI# z0e1WwAYD-%HX;iLTpuqTlQUTR6wY*9Iq|;zY8?%oqHoqyVxOqPub;}%F!RkM^vOEa zay7 zAV>u)wjoKvLE{ilXt-Oezq!A8JeO+h^f5`Y z9$wC%23S-x9}Vbqs0dw!(*Vj=ifX}<0_QIM5XLA&ieJ^tFU~JeDS|Sf?T>vD#`g{D zBP+sQf1>+_k0iQ2tr86T=)=DWAC4F~eHeTowVBqoYE*W>w>e%V<8XQA3^v4>X;Bel ze**WqsGDDYnqoin9W3yn!(2)5dJ~2UO+PPWY2eEL-2dyDSUnNwv<~01&Z zpbJ!zVXt=rscsa41}E1^)%XN>*i;?5q1GeE7&IDa3n#*Vw&>~F0Vl;ERxO3F&QEO4 zh&%_qlZKG-6{3yIYI%{bzdU33uS2k51KF{0BvC52YI!_6|5mQHQ{8^6ZzM93M;@JR zj|hzHwj2qPV>itOfTGqz3M>2k4s{njv~h}aWr6O&+i3q;iVE0bKmM_Kik;NB{On7e zH?X%&&7nDL4-u%xp812b(J$NvJYB?F)KNO4VVmWrieEfub0$$KNARCeFD(IiX-s3Q zG-%mKtfVLd(Ji;)^zi^l@M!}$fP;2;SM!M#>*340RaI*Btea`Xhr$E9KfXqs_7~iS z&tCM?T&xHdWklPV&sC6}q>+z=9+$!I@otZT$I5Gy-exv0OSuHyT<2Cd z7+35fwhKa}zUR4D5C85SdC}#YCRK$sh^S&uxjkE_?|<(`d!*2iDoqgMTU4{XYQHVK^s+HVQKRyHPU=$y zaC?TZA=L>bi+-SX0`PmPd@i8%8WAt{xB0AXJ!fEMPjjX!{q0KXyW0Q?4#KVbOi-xJ@UXG4)z5Xlf9v{Y)etU+`* z@+i2~t%3k;jDF(c$XFh#Z^o#!rshV4gMF4u1z)pDx?E>hcm-Xnio=E0^OxR?Nl?*F zd=B*$^%RtAagV5oc$5FlBasTB#M9iR_6#PtmSfK|d*}2~4prwV)eKuNwQ?CPT#Etg zS*HKj&8oBxmJL3lGCH3MA{?{k#&*lyw+ERIZlBvsBgIi3@LdU=WWZ{ytXy3!u}2jZ z;L2^v3WPuA?b4&|&3uxP1Aceb2IGlSCR4+3+>P(WE+C_X|Tv>ciuW;i20y(T9-Mcj`)%$0> zJtsXq7Zv)gPQc5ct$hsT7kYHujpUE00_|l%^-3omPng3XhSIq*TYDk11!L`sAUE>G zIGNoTiACvPEZRQv5eT1)emY}RP*eM!s}l$E*kLWWWWm{bPP5j$YTyxEc~=VSSvE2? zPQryd@%NF{!F5b*Ts8O2kP1K$Bd@(!Qna5j!HKq)B(qaIkP7areS914y`Gd_Vo5S- zZ()&Fi;c)5=KiF)D8$hvJ#4dj4tlJ@oa{qAY;=U`0T-c}`NY2ag(L8=LO?&@iZ?*5 z15>6;1iAKP*U`c{RojC3tMPdeXeC_HEmgttPu=fw;+ucs_OgIaHX@kY75qStMBblgn9YaIIG(Bi-LvyNZ zyA)kjtek)!9%o&3h$oX5VY$x+VDEuPlU$;Xmx zfWrKV{q-0{&{rJ`xW)vu$PqW!__pjj0I@C4S3l8^1RtdwUk*~UM6`XwZZ?l;Ls2yy z58iN5AwPeu^q^eNw6g2Ent_mxJluZYZf_Dy3dGZ6?Z12>Q%>KmFS0w&G|P+*U0ETx zjnSq%s&kYCdP4c;m6M?1?{EvKXRP1(vif=mkJIkHm6MFzOsUENjMXvevLmQs%wZJ# zQ4Z6;8URT--3r(q0LmF&7OE82PkvP0(GG=S%6!3mfp%7Vx^Kq`^t$YLw5$agnmzy$ z8R?mjAed$5BiF4SdS*95u3K?=VSyBvJ>WKQ{u0QTfW|%F_X&xAM#h~bRhX;_ot$Gr zw`IR-?jy0>a;EkL7n4N0T8>!(&-P_V1uS;JX+<|)k7$~&xoDcMQ*{Ye@5(sTc-Qp{ z$VBz6Dknl;=Oes^ln7W8}z9UAa7GgCCW`vbcZ29xWPpP%QA>el6d%q&Hv%iL=+W+ia}KP~cX#C*au zFV&}FU?8OG1SobmP4=D61NS`!1e?D(zW?BP9<0dZ0|>`}Pg%5qv4e()^sPR6YkQX1 z3j|_|l1=v!&_Rmq?zk5W9}_`|bx6Mn@#*8wrTsB@K56h!t!?)dilM@8iRVpsZ?9nO zMq~8BOioB&S?$u2m=}7=d#T}gcdr87_!lEx28{caFG45Cf_}!eD=-utA2w45Mt}@t{DD}$;U>6 zBz3B|EQdmp!|YZe+$ukO?c15c-mPwE#*XVXwARJ>sJdO`v%Q?H)z#0a(VLUPV+mzi z6T~5k2j8NjFYntMwfWO~7@7XcbFLPEJSW)b^gmlRVxON?TwYDJ#3G%}~4lxFE%25AoIuNp=DN*m9`d*HYGF;0X4P&#`a~9{5*|lu05kcH!|{FLxXNk`nsu7OuVC#7UpW z)74?clic!&t1OW}tL9=_7_jQoD)tV#QIor4F0liiuC6M%9|Ir}a?Wtn=vRMwh#HE9 z_8PYc)B{vc30O86`k)nF)pE0C*@pVlFjzMEzlTV`G;naa;@8*^j&j*gb9JqI+S? zJkrhyfIuqjN`dB5f8Fjh)finMgzxBhz17L(X=99pdo$w%N@0erVYv>XmDW7D_`3)~ z{;YdoU_3T$F9d`Y=sRrX*yX{etKop}4+@xHB?(n8hOkg(gYtQ*`N#Zud8yhg6XBfm zTX#ockpu2k`ThK|Y-s!9X>}rei%l4@ePKSC-$TAyQk*Ax%EhMa&gSUjZ}5{+?_8uc zj6vwtJK<9?T0w?;%jjRuK+26dMkqzI$16_*QMCi$F#Zy3{K0qocNR+8BO{Yq;+lze zklbxk{@H%bi}TeJ*8!E&bW%rXzhzmzcp%p9%x15O9>ynM2+d{W+bRWxd1*Pm4v+E3 z4Y}tXTI3+CUmp5K0hKr4hRNuW;xu>Nva+QHAM69H&Z@F=)iZ!iV_#>g0_sq`^rue{ zVMkyvovoH_J-nxf4YtLIwx6z}jMCztjb7&+Ep_w0eNY}2X21KRAbSR!&nM%{Dn4Jn z!y-$7GRQ%UcfkP}QGaK;5z-3vBJkoq_2w(M|CbAkoSI+yyF-VS(** zX5g5aAv5*OyvxOPyNpU*Dz`3>TisDl&^1SQ#iu|FU2^A)s41BJQQ5Gta$-TEjK;~v zCW^xYDKYMUXzj|uDx{gMjpX|5nO^&b%cks7_qoaFXsY}?ZA{kpT{m_i;53&8ee!?+ z%B+^sJBO$qp8VK`4@C&0nf6{4lUDWcsi{2FC@&w$p}oiFuYN|zI!dDcHJ$uH0?IXt z#g?Oni0jr^3KPy=mq{uRWhh=POnO*neZcH}$rE zYwGVCJUd2%i;p{-vbAt#5wSl%;Q8p0e#4#fjvqUtxJWNkq4GBj9L^1(=XD&T+N+No zu&w-HGaz10Jin#`$T;ek%jl0zVV_j(C`5p^(&Nq(%ReT zL!$j$VeRe+jlAgQAk=F^QFrHxKb~RVKC>URK;90pkjb2$%pV>6RNSV~4~w)n_y(gt zw0DUNwD&t&uCggZ4oYwgkLJ}7bW{I{V~AAGj+&2l>#b>wsD}{aNxw(DpQD6%hFp!l z;Hs4gWQ0GaRl36NM3$Ee`e+YecGpy^oXnz`&gfxkZgyIzjGmmk)6>`}!Kt+z7stOz z^YH?SV)2MZ;1$x6@MnMzLN77~7OdU6B>vbUUfO-5*R$yq?~}Vsj;7;Y?`b`v}Fj z%=hLxE$5&H*_uA3-z}+iEg9hUwkiSV$jdQggzV0SBAhVX&jD0B|2ap8R8M)sTN%u~ za9^oBf`3yiuZtM}__Bbj0;tZnUQG%uc-1zj+vs`Nz6z0Tr};QAAiP70Sza!$T1vAG zesMD#BzZCsr(8KWMK@J*i3sA9Bo3Pep~>?Y$wXq-vVJOs8wj0fSbA}LE|DZ=c8 z9qC5Ud`4V9YBt?xV2U9u0<8)1Pd^De;9J~w|2jo}ToMSfHPVH8OCiBYRiyQ?fVq~( zFV7D0&Rr$*(+>5O+DvNmY00^24uA;Qs3ty`IvLq^=HHF3j7d0g(DaLsI%F6(ke57z5NH;U9f3^v`5q3Cih zQQABGDoej(^R3>(=H}Bj*rU`B4%}p(7tuW3CNCOBsLzX)% z;0BTZr|F1-@ye658_i@wyhF#3>cw|@7>+FTR{r{iHw~KWy82>lFx?rfoihh}=X!H6 zH1-jNkxy1Vh#zRx6gr-hW%bgP+Y=s7XTY&4C<0-HDgb z2N3w3?{ONK@OjAkywbzBUiRQR8P0aw)kV=kWX{?7aeo+Fr6$)W)v~JGWHu-?HnZtj zqT*DW+_(dW(n=KT-Q$>y{xHE-bySQ3KxaC$gzFmR=I-3zi1ON){VZ3>G0O~)icw`E zkV=^CQd{YPPUG+HrkGY&1?bgqmh3*Xx1mAPqcw4EPG8Mv3qrifuE3v%A5_)18!L$UT|! zG%yvTocw=kejhhZT4B=7|Cj|ZY~i7wJlnN(CD&oLWm#_OymU}wwCk+E&$Tt#$?uf3 zl||ghu(1wJBDGixnXm3|z%n)I6vgRalqZ~9VopCyMoAXGQ+Fga@_Iq=n+l#eiMqqf zAfa)!_AXZh0cZSaZ8WHKVX`l>eON)Whf;VQT}h3m$7(opaM1clZH|K>^ZXv^w_s=>QOa{cbusPX}1_R6n)Cibu+b6L8gVs z5k{)t*k@Vm@BE5#-sK?51sk3Gx62FyH$J`43@Au|Soyk4v4+|wDs>UQdi76j5o2H9 zvdUy@Z#=FX%c86YcdeWn&eV;}FK1c46CNO4oPVn#vnk!REpp}^)NvznqNY(@UUc*p zsSq{c#J=4=f3JU9G&ZyOiyX-?KLB2}>2e9353aDL9)Jld(s+L~1iS%?cN;wfOM^cr zpHeO*#|2|GR+Ed?qcr8C&<@0oLGp?3N$M`@5AZE3P`>7R5RO(2JS<&TaP3k10KWv= zqyxnp%EF3gS||i|rtuc>Yid#kto=I}#ldq=6)@HmV9(ZGgJ6e(6aTY|#g%{CVRoZ# znN7ZveGGZ!b1Rp?EF-bh6u5$khAmu2`t8%G@k5%eDD57(Io_BtiFbM)F*UP{vr21; zciy>ggv_=NF}>v5{SusWG-bhXzoZ=oqHTX?MWp~!Pb7e%g0v*$rxlZq;n6*x0MPg2 z$6p^vt{o*1V-J;jId)ST8U2VgjBw@W!en{y0KG>T_PbO2wxnTRI;t-UwOw&sYIJ!6 zAW;BlhjmYbM#6Lx$sel0f8X181K%9qZP0^Wc3KWG~OPAW;hj0U;m2}D@9(M=$<*ZCg4La^r6zgTks zWH6y%=qZ5dbp)#Wz)>Bg`w#jxkd}D0Av4L*;%F&Z2TC9?`mN2j;5)*DI!1!Zzys>r zR+&=M5~|TvHG(lf{%Z{XHHQBh!+$-)e?0@}+5BIIi2?VG%pxOkTYjeWE=I+X^n6o{ zZ2I09kOjGnpQ7V{*gor+CP4T-1%t4%Rk)y`Cn^TFk4MSFLZ4o-3Zn6Ig$k^xq*8%xorXByYi}IUw96vX}E9rJHA`h^M<_si( z4k_+(l%)L++XeKt7K49I4g!K?d4eFBu4lCha1c#)mn%v^U~c=TEgIkp@g-&eUw9Jr zExX>rCBk9C;eOGjTah!?nadfupzV}&IqAz0=CCxzVN4CO?1bBNLtysg!)wMkK=$qc zt&ZPQTaKufK<0iv!GyrYY&SiKVM3}K1KsRDmq3HTx&kh4c_-@O;l(nH6dRUfPRE}ExQ=+PV7?M(>JL}Sdd}+A z0hl!?FrxqQY}Rk=ETDdT01Vp%_@>#n6V}rJ>0%%6sQ?E+(vDE95@_5C%@s%Oq$hT? zU4Y){)b%k)kd&Y~Rs-^HGYe2i|3x~SMvxAl>?{Dox`xdQ{Zyzo_zKtvbqZnb1O?D- zzs}AMIqFGE&@{dU8s_k`ms3~(3y(D97NJz8x29FA1^WXJV-Q8Ek#+Fdi>T2((t`nMK>xxP4Kbp!$I2frFqSW>^7cAV~Ie*cLYRC5Gq>H+k z1l^Y?-yjzi93RW?E3@~CiBat+Il+02$k({ zcN`@)k@FzVDcygz(C^Kl!{p;y+#>yjH#pVqt(bENwAj@8^`aL4J}tjt&&;v=z7Ib~ zUH(a5Zu+UV@4mg^NI6|0(0#df`wTdz-xhWl;rfCvwOwi#;0Q%uCQs9T*=x4I|BbiH zqxeC5>C&CZMNYq;@b9?nzB>@8KYCkl367uTi=z7yTI0D%weYA0%Z{9spT_V(@fn$y zJrjI3YE4TQ(xQ#Mm#l3ha}=K?;qrWR@Z&5Z*e&Vtt6)Ch`yJWtTrUZ7`j#6_BZ$`} zbInTgI-DL`Y1Eq;jmJoX?4O!~kb4Aa zlR-5{ep~=VAUSB${HO6*I_n|M!{dj~!L2Mw&G3n*5GZZ|o^pud^k1`LZLkEQ!;4xE zrdL_5aoYzm(OC31d!33!6PW+ZW?jouSeCBPHftTOohfR2HWb}Zhl|bDW|7B;a@50c zUo1s>H6A`8QgO2km5>&@2SB-_>6zv@_V+O#EI!S4EtHRC!coVV6gB|@Sw5VXK;~a z=(KugM-Sa$X9Vq&GCyn|oNWs1|`_yFMXZ59dsLZnNPN&zm=A5)Hpd!`JtKVLg zJ6IJx%xFe4*xH7X!fDRR=9AKkANf5^9b`1l^HG{vdv6I&9jxE2ai zjR(*`h>u@z%#cuxATfk}wN(X+@`ii7mq!P#OBQT4|7BL@Md9z$d<8LR7}k68Q0Ch1 zZUyto3+fL6)KoQF-8kfRy_q?fPneY6nd+F$e0D!K z&0*%m6xF3~3!jMSNn2-G#cwv(9jM%~KNkE;Gh44lhS;sHQ=G)kS1k3vhFl6o=d63| z?Jt_C$3JU1To=WkzUDsVY17KJmt6BWuxTrkZ&7$5lipSiWJi;8Vk`F_U3>FeCB3SF zw8Z~7OxGReVBP!@kv(WwS~ZcX?vD@M{+<@$Dse<>V`qIS6iDQ6T=Wsa0Tcta;)^8t zdUZSdrR6?lE>owL(}@Yr9svCJL_fYH7U93N=s$W4|HQ!@e0XL0K0m>!DyDTv7BiQOsSxAN$IzhJq*n=U4+pEQ%M zS;Jpu#DnD3Ut7}pB1Sm{ud>#JS*Dr1o@*EFSw>B*MSL32Gvdgce2^=e+3B6-JiNT} zhHZo{b7DgctL49|Q2NPHIbA8eSDP)#E^}cVYVBk9$*-?MT6bhFQo0a|R>+C2pnWk# z@=KXMOwb^Uvx2dW-`fYoR&LO2(*BaVlbaA;f%<=X1rWTus?I%eBq+@-KsIaPbzYL; zlWO3-f?JjxnCnB2y45VJn zdmj|Ha_HSGL&>mVX<_rpAMoyBhw*Jjwj;9Vh<`oEDnZ;GKMvFai_xugeeu$VmCpib z*TI5gAs!uGMGCE_NC|5W1nB%k0LJ0@`;;UAa%;)%ZI?WDL6VBB;hFsRDz{Ao!z$dP zA=z`i_GAD}`~hYm$D7`zA+rPX93t$Q`eM%*`7V_?@c^3O!`#A;UE(Xr(9 zw)SYFVYX|$lC0H1jT^^)?;Al^A5R0&x5c@yajxEBHqlAXl&9E#*_}Q(tMJd2e_sY+ z$^4s|T_ogV7Pu7SS5MkI^jn*`#72M{BcvoN*+oBVv}ED9yNil0Mz^a*Pc~VEl~;Jf zZ?Utmq*exBs9kwWrgP42$5pNxx!j=ffpdc|Nx28@5woxL9iGnH3zw?oRPlF^GIIau z{3ZR8`cix40Zyhnj3rtqEO{#mft%pt$fWZ_dV3>y?@?^)z=L?31F2;KANzOS%{^5qdu>kaW9Tn#)P{eWfw;VKkO`5_`wHb3f%-{8|8EeiyW!s0-wg0PR0hm)P~0R zm3`Jmn4O&Lf9zK$+P>c6cvT(4ZeUcnV|OC}a!uuS{ve0&xJ~aL@o&fP{QaB9f*8(j z*R289E)A#$0|sYeZ|;vT+dh(;+OX}@i zsKe^M+D_FEl9Ql9C^=22G;a$2^3Lck!t&kQ@~h{JuQrs?(MAke?>5BMOu*^zT^%M# zr|U#z-M{%wjwCdUF{*%SCok!ousiEh253z+aQi&0G5 z2NiMyV*ZOx$rUiYXSmNBBRR{Q5N4}v_NTkmKqjKEViL2dlNmfu7wgC zi7olTus3sIPl()c-V^m4nj@NvH%`Gigzjs_%BL56$I*92M&8;=dc2`{;FmS|cFGOH zG;EdB0JlnhYk12q@u?`d>Zb>xaEYMgJ=b!TjQjw075H)0(;)bn7*nlL;Gg^5JL7*- z_Z$r__~Bgl*z{)o>HSNgDvUb;^?{HE#69nTFKz73A`;4Xw!&U3UXD+>o6Mt3jZQ;d z>{I24>|efyR=XTG;(rWB8NKKSrZ|~!e!cnIGZNft{I1vr-!^8}me%;Ag3H~W%3Liu zcb-;6q>Emr$4}+?j>aoiGnGOjqV=3&mrResHvc6n5MhIjNdIml=J?TlHYl5G`_Br_ zC%4c(gkOY=Cv;{A#Hwh;M8q#$P*rENvdGA4bSTuedJ!=yq*FGwBm!A8ToA6Cy$I!_ zUl-oFqxI(CLFHS{c>(qB7A^v9PKY1IFYhqpC{Y)CPAt?`>MY>6Ck68_la^MGYJlU= z`Ldqs{HK#0Kj}7mamD_Zb#Y4$Bwt6e>%r}1fH*&9Py}0>N)CC^%Bl45F z60~4**J^4g@uV0;M;&L5Xr`y_Yq`C-3y0whRta0ta$hD-mYICO z`E8sZC6`KM$2uEgi|StuFXvV>(mnz{)GE*&FL>#)FT!>%bKQ481^wu}>*7|6Blf#x zAKj3MU*rM(!MhDsA?YkrVfO<=aENp`EHC&@E~{{`4cn3nE!;W?X4t0>6g@xl?)8X+ zN)z|AdW)xPevU4-vQsFqU`y(2_3D>bqUYsr^ZDN#eNcr4m(-!v0uL{EW(=RHtYwjV zelGWfj%Rys6-#8P_QYkprosopDQqn-0ec;jx{?U)hZ0Om@BMnsq_DEDZYdUuo+-dg zR^gS0JZ}9mgaB4T!c;N?)@-OU*O68UCmLT(weTU$MQ?hqbxeovP(`mTtI+ewPvvrS zLcGXkjSkY*?S1t-IY~rD!n2r4tF9WojfI~P2JkttTtA+}u;>*zi#6W?-=v=uzmvcQ z4G@LLX#XCz0p4rgA*|xLbq?z?@k9&J-7aE^DCH~i9`B;NXMAS-d@jn6at{yZ^)!)l z`bq@@dVs;cEKyaY8+h9qPLKm@xFYTVnx+>#UD!{XrY&7VR#w7O8r zTckzzN>!3RO{+@_o$I25c`9&*y|>lVdb=`S4djo**Hcjx(mJ{Hy0bpu+k z5te5KUoI0>K@iip{cI;2$ds1KS(|**nLn=JTiLVtJy!e*064)&b(WlAS|eX(ikBZv z1>clj_`bT`(24aU#m)~MlB^AeR~($Ihx>AYPi;7{{Z!6TkR#kOq2laaCPlYupF)&uCae$V ze`FqBlU%NanZ;vXO~9aPRWIZ z9hY931CUIt2N7@N##NiASL0?UrlUQiqy^1nh$LXx5napQ5;t7<5a-Q@z zpF7Kc7OPMhJ;BDegwtou$oVUfoX(wguzN_jjO zYO)DNekIFIP;n^Mxod!llFZueAB>-UtWUy^f?g-Y)-1OWC|;Lx(f~C#aM4-iP-!~)S2ql%j9n(J?@Q~9}=DGt>Su@wFy0*F9cd90`sp0FH zZ(lQ}BKle=_SNL@%w{cCuMgsOWnTH7t=wb?SNsb9@h(wg`HuFB^5F=o<_pIWeJAHs z%+~Ub;Y-sOu>jc8^E^rUE6H&u%rUZ<<|OkkbO7z(7w8Lo_?*&(Bzv^fL5pyCOOw-Nbm(^Ni<~mmd?vdUy7H*x0jp{noQd>sa3(z zGl^C=>Zpn0K8Q+J*8LQ8gxL!(HWSa>)gO*chb@^pf4R8Y{^I8M*e`PFqV4oXH)^&Q z?(J9Hz!u;8Kzd7OS=y`))tic#rq2?d;Z-=!M}p5bAlatX^%6y9I_%ZzQo$83VUs>iLTE z`Kxe62FI^w7+&&?ydwsB@17knQ@j9m?)0y!;AbkKuKx<5!aoQ3Y{f-CC~)>ZqMQ4q z^=b*#Ay{aGTxkb%EeFGGLV2~*r1ib~oArh@ZAQ3Gqt!tBx8;cDzP4As;J$Mj$0Tg0)0v_-&d=G>W@88-l-p+n- z!7OgyRdeUzl6r6K9OJIQ6)>eHuZHWuAe?<5V(y{}%Ze!~@tm*Qqnp@IY`ySw(c?h@ zz71h=tW0Y0G|p)wmzNF&d`35o z1vyO5ui`NKS2Oy^9OMsjOZ9RWQ@yF0b)nZF#eNTZ`{X7S7~r{{3d)p3%W9aI@3!&U+{QBFs_bjzToR5y}!8iPrU#?Ow&N9r2}$gk7!0L6uhK;O1C{Weo(mACM4A|^o@eF-PjVuS zIH+Sm&#shcZIy|hr2s+N%60t{J=J18+)fu;yggGY3nZ%NGQS?=lul+o6&Ailacucx zU5nNd@v;(Uj69F@rCD^uptt$8H$Uh*bhUKGWkH-S zaaeEm4W~d(p~ZxRjBGAmm^r;^IlG?vlgG%rRi9giSElscnOR_W^=dF3BYQ+k z+jXE$%yFk%KaRs3jKQ89_Hq^Z|0MBWyL)#p!l;hJPI2fO zilF2;A(>Q7HUB{`dOa3wJl6Yi(7@qXG&=DzU8hmkPf`~>?^PR%bt1IOLUZ2XO9E`y zVJLb32y0m-F#5;b{>T01;$kKz>0Q1kz49#B&4A~zeuF$J+W8sObjlgy9Fwj+XFasG z79tBp)ZB{|8J{&%x+Gv^ad|S|abm$l!|~~RbL)XibsKE+LdbyG z$`bn7%1Qxs<9=7v`Z%6y9)p6d)`;EtfyHpCs&Jzfzc(u;bc~Ffwbkg ze2+CpJ&uDMH<}E#Q{0c*7B(2*TWwEFoICJi+nxg~^868E@|Oqbu}V*}W0W;~nAY>T zxF>B|jc)F&zz`Z|KY+1%oZIF~fLN`43E+leebm_p+Q=?-z_?z@GzTD*pLWSgf`gIo z?PjjDwDV~3)Fv)s{PV&zzk8pkmKr=}=Q~{`nJAmMr3VDlr^*z&i5Y_rF9e~*r^R3R zb!<2y*2LL6#h0&=fs8PSl#mu?MN@mOK4?Q_dGe$s4F=bs6oroZJPlq$&SKnlb=Pc5yBJ&L*};0 zEQL5Ny#=66y-`?mnoZgv?@6sL?-00VSb~%K$$G6z%}u4ih#J}AAFp=}Jsb5<3>qoM zF?_aOwnosVlj~y7) ztsIX?hfE@BQ);D>)UxHlzV(%Ux^(-?m*c)ylH_Yl)V4ztIxS}R6dYm(v^irCHC2X` zyy{dn>RvDRlootvGPpKwxyND z&(6OP1k%9o`9{uBnCywcV27O-jQ&*#r{22XIL40F5;WI=(C!x#$vlL1EmTj|jEvJ@ zBe#WqfYHLSA4Q>Qu~0aZ99g|XU z-1J63C03Fk2c%w+>a5@zJRzD3a=OTV+(Y;*L=@t4H|%f&>bLNHffK6&aa`Q`LdU6g zSm?jDQGOYMJ5vXL>U!!~ds88vDm~%2k?*)^{DZ?=bdsEHc84TD8~8Eg|6}hv! zv=vaqMzf(pK)@&{C{=1eL_mayG^MIEr7BfwL{yseCN(M`0zw3&Hw#E73PR{8(t8LY zfrRAS4|wK1=Zx>nJM+yqzrH_Q7w6!UXRp21z3z3dwRgO40v+{6E>wx#Ya-wO{;5%A z>j=_PY>mcGyKqhL&l4V4dM? zqroup8LQ%LOu8-LaAt%^KP#EAkhV(SJLc*#_jW-RA#?8p?W-DX^aWPa&Mr_9os=TK zyn4Evnbr3*^QIKGs0&53J8*y>197d&p>#Ow!zV6=k017eU$1W?7~(wVrx%9_x9?av z3N4cACMpB9(P;(*Y++8u)Iitgi&iL}iZ|5S7zWh*@-rfa@Jw>;aP3-)ytM*QD9~r^ zmMM!oo4&T)7;+a1biUxv(>EYp-a4yNRP;$g8m<6wck-hReQ@csn8M z_9vC&IP?2I#1!b%{T0qu<^7qq(d$0~7hVMNR3r>7KGBtbfi~!8&=?xLv2`o>KAT8r zQr^H^Q?tLSxaR=6^gah@#2Efyzf%oj-NsS?H{amm!f(sLKiQI)k8BR*{i-OKOP8lp zt>#+{fljH==_a}r#p6dd>jHt_G8#O3vPGapX5esoC$S@88_&LL$hD#eh>nCzSPS_6 zn;6@so8U4h$@4qu`|eXKMULhyhtlUN_xk$Hn3I9ooe%`?KtxI6b6nmFqj0qvp4W>g|nQvN{eArC3DWxEw7Z z#fCVxY{f*4Do{Unymzn*++r*z_d&5$VYUUpw*MJEh=n4q48@i zIgAE8AA?-5#B2%c&t^sg+f0%d6~Q)Y;U z`wvTh{DL%SgN->?V@-qUg1HJ$XhXY1O{`${`o>Cl!TRA{6PWgtDW~~M)Y{_W$$>5@ ztm&MN6gF4i#MhI)c z0SLH46m>oj_R`gIe{2MGkp9@vNS3yB~!-N4)rczEqp$m%3F(F{^=;) z`cZNJiq-YU@Yx3E_#?L~5|(?_Uxqnn3cmH<9r@;#XY%bpd*`I-n z>kU@&!SSn#<^c6go0f$_G7P6VGYwoKMVfdX;5q3xk#CvVFp*Z-N*kJ)t|}{gdHQ*) zX%p4u9INlj?B!vA_cSKL7crEeiy=5V{gMAy!)^OTD1>!?6T;{DI?V(x7|_6hyb0aR&q1a3PxZ(G? znvk8mNy#TVDtWGdc^!Qe^wqBOXOP>4o-?BMA|xf>7=TfcdkaV{jp3bXNjnWFQTnK8 z@W8c#<+1PC**A(|*XwipwZs#T2IrGGZrNOiMsfrczM z>zZ=aoe^a@ro$E5cvezSn0EB3+GQ%sR@6iNT>)LYyszWo*N78CE6rJAB2y*R&u+7Y z2h@4w;F4|Y(eDGng#vVhw+(}+hCXbXK37AjX{`|x-+gBHu|xHyZ2V7nFLJ$n{VeO& zV*R$ShnCV<*dEpiHW9KtJwIxGDN~lX#U`k8*J?{W%j8>e`%iCLoo(i8hAHi}wiOF` z`NsOB0W(KK91amOU8IN3SwQ7v<*ZdsjCQKAp_X1Wl(0S6!o+*!GD8r;-8G;N=Ib8E zHKwEH^KEKmf-rkYEw}xto&t^?wOk$-PAzgCJ7yEBRz1`?qnK>h)*6JEZzMb=Ti}FW z@7cOTX$!+H)h+l2BoCe$-!LpJbb=Y5_3$Cr-4n_qNK@0k*6e{BJa2Kync7LoZhm7G z76f4&vrkfT9-?n%l9!)<-m4*eFgnrPlQggLqJR1XLbi74bC|HzTwi8;O0!pyp`qc- zWy=0{10YGtczcpRhlcw zgy+y9u?aknO`59$n)T?p!$DefTOk)7n{f?~33W6@erk>T<;2q0^iS>bQ-OQip21Vu zQ4GDz`*~?hQG5suCRw?vVPiPDtigqxnKnH$l^SYwzf7u<)puRX`q9TEd#@7d%S6c| zam_6jIhd*Jnq{+$T+Icw!x`1SYlmOGcp;+q@#|McqU6~ zz-e2TdDUgV3m6+Cqr<5%^s>>#1&xnp78ac(rL|8n-iQ&iPs^m_JDfQ=88vsj^su># z_a471DS70Zftmd}$J(f$U@nx(eJ2~!)x=C99rF%|P|0Zws)kA;^;p5mv0pWdq5#&~IZSI#i*6;!8Y=V@se8;4!DZBtn> zBD}90YD9#!2RNAqRxk5WzJCuAKi~M}%UgNkH$~5-rt0=W52r=8+_l68?ovG|`)fDj z#|piOw+IJI_JrTf=!gnm<6Q0tq^PLyzZ%pj;*lD`1|=T5-FWJgGkGy`b*BGh9h&$4 zs3DU-FFvbAaW3UkBB4Ap^G8qln+XGz$$}YHgLzNMTd%N3#3l;8d1h99Ca)mBmk7}& zWm04k5=0lo9J=ICXE77bJ+A(12}_9!tiCroJH=;H8oWeiWZHrd{yi$Y*ih&3SDvvk zl=D+ZdEB$>(E}?mKl|dNGJQ$Oa~k}#`*-#18Spkn)FzYKGP`w)N5@?dFL7df#)odK zsRa$RKPPRUCq5d2J$)AC5&q0ON}<;~`V2m+s7TOzPdZL4FE4ETu=QF5uXBG=VmIQ# zGf!?8^4)P+YxV8hKeM5%KNjAH0S{Oo&Z3WNzCa%mTpD$>H>M}qiDK7xbZkU= zmG^R>cH%*J=giL>X*FAz5`PoG9Dp{R(#ubrFsLvqc-iTx{R*4sI_jNlW4LzI2#xSg zGi_6$o+K*R#gb!=43-ZRjXSCw-teU6ge@Zx2&o`M5Q1i=qbw7I*uSs!feH%;%GNfg zm62#$l#%LYXBQClAgQ%u_6_l8POJHwcAu%&a%+cqcw%U@n?;k1^EZ3T2@kHPCD@>S z6a8=-f`!&?95zAvhUaYW@+{7Lp(RSWrQSi}*6t2F==O6=&WL2^ z-}5xdO27#Z_plWm#3hRMGk+>4niot7Xe8H4uy|FId{MW{-s~1S4rAaDpndDgESxAv z>vV7k>m^)6Npu>HS2f>gO)*~kq(5FYJ;&Xe*5IY+;X$&>p>#t&@Fv}TYPHJsj@pWv ztU{Js1?F;c`9vX&En$`RrC3q^tIIX4l|mL#|5Fe`aa@Nh2%+=n1d_r3D&oRiB|U%E zBXudyl&TOFOHq(IvQRF6L6w>qW->O$N4p$~7@|ZO@&C9y9Pw!I$mEVQ;uyGVNJk<` z;N}R@+dIZ(&rh?KQ)e06ipt}2JZM|aqnJ7j$7PRqQp9lDYXP0D?t>I|lC1UZ+eh(m z##ZMrvIWc2=}lA@gZBD$KppzIjiJ|ER!3>g!~9w?Z2mHrD>8yIdkI5H^L0l}7726X z;e_2rX!Fy?udw-ID6%bK*0zqzQxW=)H`_ew&{v0xj?sFE=C&$Z7}#D*falVJj{PR2 z^Yg_%Wxr}Om(@#lYnHdf{U8#Rz0-@jYeViP#FpbZ_=hgbIETqCwZC@xl+OR?^@DWQ zzQM8AM`b!-Z{dz5d-G~t?^G5|Qo6gnCNTEJzTCEMKn$aPjq-WtklL)-KQq@Q<;Hck z<7=7cVm|9|LyGL`W#VHp=V7UxEf16;$>|(uD0ow(Cf;81X&aaCX0JL7?X>Y)2SEdC zVamFD^3yI-%JdW$$OvC&SKmxr|DuVoYl_}GA#YZgVG02GUX98-h4tLkKBH9O!S%g5 z8tSMNm;O|-PUolCa|!yRoi5i7l6<Qz0g+Wp>wM#s1)#;%BTY>k=aCTY8c z1!vk!@TWlU2-Ai#oS4@f`|{N$HH^8Q33#oxa?)3C_UiDjuz&%(XR$@y6kd{_s_UER%?@62zyI3?)KuB16r2}%s z(udU`#0UEm8hf`I<0n!LshP~})x&ECk+RbFx+_J9mg~#8IB%Lz96L(>{&Z=_+UgCDl)QM`imV4Y3O~m=oh=4sb2Tj%<9(J z)fZF;W|Fo}4Dt5gXg{()WFYGCve^atqVj_3ay4zox9{ao&L+sG*}=Kz8mb*T5bB4d zaGG9-5fKZEi&A1ocxj_s&?;h=fI{D0vleG8kACM(@oLbv1 zG3;>^l@=<0hhsAj?1r%69>@dE0QMyf)dDHIjRaFBZ4BpO0TsBLoj4!a!xmqJ0S81N z2)i{gq~$&&I$2er^6FPPm%*$;O0Pc4tF>lQ8qvnQ@7bu06BpCb7BAf*Z{Jw@`v5$VtMgu^k{vj3fD_%NlCG*Ga<4rpCo-d zGx^=Weak4~_jnqveQ4GCBfyZ_{4UsRVB4f(KszgmpZ}-WEuP64bsi)5Y$iJQ)5=U+ zoMNk?j#7Wuk^EP$4riU}%%@y2?{|Z}$`y#b<8tFh>mALR+sSJQ2Z9hcK`kK%7=Ri; z>p&u3a6Xam31zWv|Map?ONwsujWrV<55Kpv&X$cb5yYDp$4$E3dd!<=Ccbs)+hDEM zclnY`x&x(z@LiVu-%7-Xxk2h@jBX|PG2Z&)E&$Ir4y=2^xML{GCLVNI7+#H~znTp- zdtawgjHM?8QNFJmRw^sqA_QRiv9K{09v=Gg*6^cXMB>arQyk7v4tHkW?Q#G=!<5_Q z@tO-5__C4mf{b2ClL_+myEmlm(8gOg=S3!jTmXL;R@7~mwX)(hk#=FY5~np!i~*m3 z)1Frdrl_%1Ab$SzV$DDa+;mh2yrhx&plUFp$bn1p){K3B�-(+?Z5X_O1oScv6!d+f%GazB6Wn-#z9dw`odYx~jD zAg;XI6>WOQc)kcfTa?QYgeY%_rz;4LAVqXHD2GrY#11Nc!w!})Jg(nhrmpQ49=Cl{ z-E-Sd5wCAmQ)Q@@ae)<9a)c{h!_~g+s`Z(iog!ze+Dt`Of2c+EIy|_tui~4p)RD9X5W&}B}c#xvJq^7v$I5lgJb%>W>VqgKr z_6nblxaMV+T{ofq!f>GqtB$DXPN1YG3gf5t84FP-&EL!{dwMBOm8T@e3*WW5to%yB zp?|$)RdxdN>Yc;-4{fXyDX=2e^TyAeUR-#!{!Y~iRUB;cU3IHRP)RVE^P=C17-&qF zM<$;FQHFmiJ%W_~XR2d({tQDA?d7CnEL?Ce>q}r71&*cA7%jk z_uZsO6BN}kHc81@3u?={6PiRRm8j@my0=zNv^IKXq={&sml_1DMovpew(`7T$EY|yt$+Thg}Z7z=RI@OtF$=t7>RO6%`d?0iQuhN0 z)JDBnXMVelN$9~Aupa?IWKL{Z@2uN1vN&N#33FuGtgiY=wq!bflK4pOe70Fj*{uj~ zr{~9tyq&(AA#cl)ypG+5xnNSONXnAEGp~B{Q(r9vSewZOcb2%o9+5d8DM-NZ@O`Y0 zH}LiC111pm-vfgY;%L(@TqVxO#n(L(`)0Y%X|1>G=?o^)G{rM7(bIiQQ9L)5P$cYY zE!AIay=Mfu!!u+PIFIfCz?uHwH^7-YqvAwB7j-lD&rcqm|7I@k^z_|OEA!Nvim!E} zel`AU;J`G+vU_Xgh^0+mXK~37n*L)-Le-iQ%DQsRGtr%>Tw5|0E}zlKRvxYM%5?qi zA9?jHS-ihoejSIOk1)<3&Dt$C5u0W%jy5udng&HdSaoD?Yc5VySiAStca@s*Y- z65TS-)>M?*0%5C-sBTFRqNs>ZT88Om9l8N_u7%OU*0m?btjM+;&K;+YllIE_=0;5} zFXO_Gwq2H|f-m-Ve&=Q#zp$QfS~(W)G%REaYS~=LgPJV&`Dl86k)P_&i8xL#*cpPK z0BMNVMd1bRWa3P4t3s1Kj#JsvW+>t|#S1nXt~W{B`mvS5zP{VRu>YIec%O#a*>9BS zR!Lp!JZVFsR%g1G88KM6)x5BjX|DOrYJ?+7DZhexbxLBGk;t;j^DKf)cq}?J`@@6$ z-VfO)Rbbf*5+;oLT{TUD1qv`W_2XOXV3lRep@{wD*_<6CNc_U}T8H`TYK#H03EtiB z!%jj%$`=1Z{s-irtSlp}BN4TRdRO+2Ah&H3|9<5oSV*!Z&jY)(8fiL?&h5QWCCOSc zSghldLUQc$MV~CVZqs+Kzx#B0|2Lyv=9#2t`191ntpTO`)p*sZO$uLHS#KwyWg1=$ z&#a19KTgwJ>(C!xcbys-w(2X8N%&NQU_*_!E9!FM;VKW^qFF8kBPuWEFMa1Cw?)X` z>94zok;4{g48G6Gu(kaXBQ1V5A@Fq&LOfW1p29d7Q)oSBBtzG#H7{qD`Pm5Gm8+`| zclvQf+OyT(n+xgrsjmI4MBQ<(lZ$;#v+^UzLvy_? zn%Lw0^UEQsY3F~jj6)F1cmc5tgy8g=u*+O#zp5E(%}*!1WbS)tUeaTlvC4M4+gmd+=AUvB4@bBN4u_AoLOoRL9trsKGXuL6S|oSJXRG9)gfUUoUZEM~&}4 zT!;3&X~Dm-vnRHKot0~J*^io4ES9%BXptjYhhBOr>Aej=D{nsZPEjgcgPb=$|Jn)2NTckikDmG;73|3DS9fk;`q%}4%dkClusRqD^ZNNwaL3P1 z>-R5!D)k_^@gg;N%Y}PQfLrzOsZk4lsygh@WAGmcd~KmSzwc$DML)vJYCNxK-j~-W z2csu6mW@un=CM+q_9p2AHsrE4k{A~qY!UOUqU+goOMq~)61C~&>mF$+N$aF~i;Q41 zR<%=CjXc(jZH|vAIQfME8mXSyQGPvbndDc~6*IIFlagI?T%mrG`@!yF}$m&{@@=LwPBVb0^d5V`$@_=xs6WTGXL(Cv4kN2J+~+;TXTFDJf*d zg294EO;#imh~TX|^iDyc{^c0$5y5EDyZc?aO)7rkq|J)2Q6HSu(!)!H^rpAD%6~K5 z;ZV4<|2^D6YJNXzIjl?ZMYi`#qk3a#Yb%S)O-|(MQqSp$A%W+2=n435Lj{F^ud(!dOcoclFX-C= zNCQ0Q(=plw_LoOD$i#sVC>T8J(09#t`C&uIoIJdjARSQe5O_nB5<3b+Q}xS-&TJ^5 z6VJ8=J|uH)ISxO9hcoYJf{@eVAO~28*|#T3f^63oA&4SOUj#os9>@)21ON7=?5Ppt zKECEnY4HWfkUddqRF;Y2@1!?i^;=)=18<@97z93^dEeg|O%P%$n8ag8IX{RzB?rYE zpYpHBIp*(_W#GAY=BKDnHoUql_j63UG`{mFK8u*Pa7oL&w5wshuh`gebtSK(D=NOC zPbk;nrxC!F{HSD#A0%yFAjY2$LOiBvqkubM9Z7?--!)=36M`ihAR?jSz$%#N;i7S{ zm~xw9x$_%J%7gdkIl!xSs{IaLQf5y8sj?hyv&*Tlrlu)5Dkdk?6%u>J~LAGef z9fi#0zV)RPa3B9?oRGQLes8YyI`nz^wv!`B?JepT!OI3;7|jM=@j;n^Az0wb(&iS~ z{HI$~@Kb-u+W>^s79EInDz!pYA!E?r$j!bw^p-O+y+H^WCG~i2C<-IN4&%XV3PmY3 z!CoGtwdR6_weB*0BXu6wEO`3uhSV0g5sQzH0S8j!0a}M))53Lj07w@cAQ(Ek5u?1n zk5PV5K=WO9hw$tCLtvWFKDO5v7<5>5Tq0zKzpnd*kYhq!weQrSWp0MJ2?8)izus%I z0}n3^^QrBGIaRa~qI^sD-tYJYgj~>rMnF&Gzi8v>X(8+ZZilY4cMcD;YwUw{V1S{s z{=Z(lS};PUOK`ZBSC!f%cu5w-E(k6s0LBT-CwB<^o5_>^H%ul#ZImykGxrX`DdDvk>e*S0`^?3aZmEhTs=E&)}Y zl%WAp!v{)Y5D?XK{i~kmT#pVe0BU$#RBC>9Oiqr_^^PH4(cvnec4p#ZC$oiY3JFEw z2U$Ulv-Wc^%fuY?nCq$aN+HG9&fH|5jRw9$l# zB0&-pEuH4AEfU@+F@h90CZzOyge8{BCp3bsTH6PHysfuQ@2x{;XE$Br4rptiURNJ? z>w@P;b{o{8cSk&z96_>iE&ak*h+e2}e1MJNtu)iFXEui0zpm|{TY5+|b$@$#4glIQ z+O>wDJy6=L+bHzC{=N@Q`%w%z`g$uqb?96Sb(m*+J#AM9Y5m9JLMPI^ee#0uFs!++ zSa^fQB`D>SV~ov$?4yT87RSp+A_lwhZvZ>NmKES2su`^WQ7fz8QLCn=i`1sA z=L=_{Eq(k5pvZlISaIrG@Nga-kaE>hmU=boV1R#3LuBdEiQi!l@B|U1Z%+U!vAIAQ zc+2lIFknD?*`lqWd@mvZ@VM&r9)AJ0Yu;<|ReN3T8EQ?g)KrjZYPm(@#F6R9V}aYLKjp}t0?M) zr(W=jT z=p`VaF$jQO5ymMcn+XH+`%QD4y9N33C2aBCeuXdgZ7yWVSdD1(>Hf`RVW;vlA| zXcoy&Zb+u{FS{t-{aOzB0=-lk?QoHhyKDXF>yJ@UIx?<{kr|!(*1qvq*!u^J}n{Zx-rnb>BH_5Iz_iIO4 znK)Fw$D=Cs-4#mZbm$^8wN1G=he$dRD{b7cS}{Yr7{0vcy3;|@o<7^pW5uEhXBR%5 zSRn!}Y>01!aU->aJP1K(EXhlpYcG0SOU|g8c5`|rX%Ku6l@rDkvu}^J&wxo+Qo3n@ z^Xi+_7gxn2hRc*eKknNlg{p@E_}oB!{787N@4_wU*Xg*idieb{~m7?T;dVOlaNH|48o$`|)AiXolGgbM4jaMX9Ah z#j>#}j=}nS{EuBv&quFS+zy_!vZ*#}N6l1bA)fCZUaw~!;;JxYWMmXOckWbZ+gbQj zv2)i>&sAZwq#zZ=9$!7brCfNbg;p~oyc+fu2Uo-+ELWTL9dCuxfs*RACuW}+=Xi^g z^G9teS7_vUyPQv|p+2Q(i*woTPv3u;ETb;gqy~@PYuD^nw=mV3EWixh%{gW4ef4(g zV-YF_?PK@C!+(YEEiaSU2r`;>p4$rLHtyGo{ z3HQ%enj4(jg8~=wH$-D;M`zPw^>Yndl830DJyETD%-6LiOA=^bNhA~H8O#9u%0W}7_*UMgsQ_}HP{ItOGofqiN-)m zl1Se#hT$yu0F>oiphQ=Ujb{c~2b45vKn+(tK^ub-7Ji}rsPfef>Ic(8`anAtuLb zFCV6TqKpcgQzdkL66z!vPJpV!<~UFaNQ*hNNNG@(VyY9xuew@>iJ zs*+c)57_)%BqZHzd~O|@Ou7}$n++}E7*v^q@^e!*xV0e+sBQ=YJDoW|;~k#%0`4n;w1zsxfkRF$RFxz54tdEfK9hJ z=(#DbK-Kcr9s6Q{vCtx$^eQq2l4|k00sV($%RrD>Steq@7cqkFj0Dq&2H4&s(seLV z)~k3x5159C@b+|G>$ z^=_`iQnvg?oKs+F5oO*p)rDa8(W<_~7U?Gl2Lh2yMU#RQcxDeomwxRo^1h|f0f)%; zuB^v~Njgc|h8i~BcQ7mv;eD#`-GKv`0#akVA=v{jsz$|wjwt)wwR<7&L=?_S%bhXR z)fJq}qk8k~2x8c=FA@@~F8>Ecxds_!hXOFlNS#?9R7L)Lytzv!453Yk0#)XO${kM- zZ;%=h43Wzb^Ki&ObhB1=0mvUsfwa3(L~M%O(5R)~uXgDFMifEO9`OYf3jEr$nUJO; zXZAk?1e@Nk#QWfVj&6I5tCp`)wt8}6Ky`O(z2dMTiI%#wtU@w!@FR75TlM>rT0$~2 zQ|W`Y@LeWc_^kIzt7U~7Omq9D{gujr$2WfMWtn%WeZszTtiFT9jR~tU=4mpY1zlcTBm)xt2_jS_4s7M*P)CC{WzF zH=yb7DE4nJ4`O%AGY%VmTI`{7ZE=qvkFfsh8wDX&@fI^5f93jl$ErJLs=PfAJE%k+tYmivD>R{Bt=SAf{n-s()07<>gEW@wurylRzW{- z^hJ{Q8^uvlyKklbdW|vjn`vDc&tEVGB5xDMM&UGh{;N{amio=TZ}WCp+DjnPw`o5Dzs{efB23ZD(9FW z>NjnB{UKSGEzL*WisSy6CJcYHhm$o$gwvw9x2lxhYu@}5JKy@Mc6}AwIAccZu&>C; zxzvj4#hI^v+OzZ+GZd>uZ|bLC$)`**7f$I77G3RJo3UTJ?NlvKom_*as6Cvj4NC?+ zYTsNlQZT-!Iu2UEme)8@S3nS({tDtNP(N-dwABSW7k9*o>b|1#q~Y^ut@~`_tIq7$ zs@aT~!aKe(mogQm@*?hIJj|`;Wo&BlE2dS%W2yol01pIxS&Z(eB{Vmc{a#=sB9&Zp zt5^AY>B@}~`0`Q5glc5!Gp%9L7yhMT%&MN0??^?l^Fl36Q!3BtYBm>Z^5qJ`Q=C_G zc#jsN1 zoQ8fhdhrpom|x&l(buqLnp`j+!oU=ygRp1;~>@XZk>60qEqFUr;w>fL`as zpU4r?FRHiPC1QY#>_m+22@7v7;YL~`^y<-u_Ddm%-j5be^Iko|v?{M6Qs8^L{Azg* z&JMOkfP4ks+RKwG%%eFiqH*ov@4n%@P-UbCdC!@C^|#qjr|QvT%$DoB@mZuRfrivx zT4dG2^i1J9q!B8`p3;shn(7^rK+}Yf?sP_Dg{i*GPTVe|x5R{Gg)sr%|A-Bqfja4t zjQ$p2=bMX!d1?lRcQIG!JtJTSl{27WOK=v^8_#N^WMUv#7lA&ZV^UB0Z2qAN+6KFzM_>oY1s zW1JLM@FXwLme^oxZU+D>+MwUaGIEja7@^Ro>8t(o&MA%T{Iu3r2XEzIqSzP2Gt3cp zZ@HuHGx`faTqjg&lh8vN_ZFrQhy)^ksV$8fr~C)l{hXC!!7#f6>h9}{^Yl5EH2XRj z>s}7Fpwp1!ec}5DZ;LRs8NS$dz>=<}`XawnKfiv;;bUZNi?=!jyRmQQ+?0WHt z`ilUK{k>9SuSB45rphr0hEbl4c(a zZY}nlkQ+WZOsA0)>ZOP7r(i=yz_j1$7mNN&f;WU|^5rfGyThJZ-m9W|nBBhk zb-MMH%kZop?ZGVK{<-98SDmakW+f8I+Fc0#E{FcYsR3|kW{g9b+BCaizlZ^q&MvaU*6n#RbTKlk=QwPeTbMy!6stjpFBy_Sp7*d zAJxDZ6}QG5hekR;Sz81`HslJmg&nn1n6{jZN=EkytLvqU9ey^Sge?Q&z1 z7|$@c=&7^k&P{$BuJNtR>cjf2hQUYDRnVqc=uCPsN`$Jr%a+2O@ve zQCvKXEFN=l5!5R;q+;u!gA}H-e+8V`fpRrRYStobX~|A`6gta6;_s>XxLN49-TR}^ zIr)Yh)6Z{fXKUHjOsz8px^hki-*-T%3uairN|`b4Nb?Ik~K%Zyih5;Rln1`%a^jEVQ82j zIxNMAZ-_rci~&6y(2@Zqv0BRi_{(JHGFmrpi=k#D^0f0mF$r2PF#;n zoBy;FM<;mnF8d6b5PNHh_VGNuKf$bllE{Y34=6?@g9eu1RZq~erq-eTquiX|PLK+@ zEu71c7fA|D`%!R)uA{1>{pL#Pta=^doM+&aPL9Sw6tG@q_%fLz5H zapIp9-e5U^$o_2Te$2P2r^NhG(f+4;%C6(INi}In$)=v3XPCGak;==8lwRy5p}3{E zws?I1FwLMHLHFscMb%=+fe7l0s+D34ajf4O5GIavKMp{g>iC=gnQk|F5GaFGa)H}T z@OO}uBJc;ofU+AUI=&-%gL{Bkqu*f)IJ{M2KUmQ<9MUB7^QrjgQj2K9F43yH-Og=# zUnkaPFbP-`jnFYXjIgT?U}n8!Y~26GN-8l3A@%Nc@;-+05iAZZ^VMZi)O_W5ww9zN z!Ec4U-XShQDbI;BcysMeJ8s2(eL%V;`LLC*v7J<T6}hE;9i$A)rZrNx{gLl2KtbBkd7R{dj5u7IaWl+*`o3og z|It~55J|pVA-UZ*MmuZl5vO)@+e(Ic`K>iV@%N};io%%a;kA5to1Qt(Q2~w(S+?i* zmK2($_M+DlYvZ_Nat-pL57sO`E_9u`YGZrGSKa;l`>md|)rqkQ1by{s60H<)g!}=o znLOG<&rx1lE92p0J$Zi<`~*L#`h!a=s&G}cLr-B{X+k&xaU$>uvx1xTaVZBs|5N4# zx2)qQ)~B1!%6QtYHN;fw-|;-7kIw6qSt+aXT3oXS_4I#a&ZC#-;^8@2s50Y$cV=d& zfp@Pvr7Mt*{dmNA#4;%8y>G7uGfsFnLvu1Q^ObJzf0LPCp*hm-8Ydvo8%_#$-gQR8Q;aLC6m{1 zwIT^7&cT<<3=aHyJ&N)40d9Wpm)Z|$ULY93jIxF9dZFE7`xn;1w?DppSyHutdo_a) z4hX0_1qf^v$)<>R$x@+WeQWvjl4fQR|7l{eep=xrE$~oV%dCi9c(@7mj?;`J_Q|WP zV7fk?qrB2d>%fh9J@a+W^g@fba>i+t<8hao?%74OfIy5uA0}kDWUa~0y_k_7t6Vun z024gCC&5R}9s?(G{;s~C4k4$vQ72HeEEBJ(%tBPHp}^w8`TOkvRBx_QYUz~4ON0}k zdCe>#rEzp)gfLvjkUF&?Z+}#H)3zVg5lGUYl=Lf5BB2pm!Sjb1@SuxgaS%x98&i&u z=2;MW862tIP_l1YfiaEBLxKfT8@^ThM}Go+7Y~>I7;&jcKGqg%JrM%#FVOE8GU`EL zUFy;KFq%t=FPOb(g4Xf!H(i9&N#t}bB9YhxN33d;|9Eo5^2Uw#!(Qs`<88@fE@(W1tMjsZgE*^s^e`$drIew0|-RKQz| z@0^`8AfQc*tn$2AdBWvajYW!688M?z#oHrGtHw?{4|z^_Ht$STqMnq{QN%#Heyoii zJE~b+WLDn~S}6H{rtHxGQx`?YyjNzU7@7ArPENi)WnN>_{U$N3-?L=xaXUEjy!rXS zS;O|6)X035wU(2cx0K2fzf6NK~^Xkh0c|DX%43RI*p> z)y}=B)*O1bLIx~0EJP`FuB*J);7%P(*hEkJpDJJMsEUrOrV1#X4-xNtTq?YeXO!ft z4!JZ{HhaK(iweQarmn${#`)bqx7y|;6a+D_Gk+}vHxi`jVc?X&VhVfyDCyWxBg3IG z4>LAG2pSLh8(sZFsqlY)7V_ub{XjnYpZ1=w^=Nq7{qyeg11GVHlsB`(wZpZ{2V|0^ zo#JNfz(~H|`kHR02sc$JtANky;+IQSrP)&aXR~x& zU^=1MHHDMs0b)KHd2!|yv}p`~4ZGgKQeD4dCp-7_xKQob(&~BLUf1~}snfZcy>gQ3 zrgH-rEh3@DZDH6;(S7~BEi#D-JOuE{c5rfbvmeIdr!8UZ%G&*|nO4zm|FwBsTgG>_ z0_>+&v%IyfV-k&ILKnmoQ%PXF-#I(_$ph%@a6TN;+Wywn3q3~Ly?b|~Ih|vpml%Rep>FiPsY zYA$`6q3vpY%wsT6%>E3t951LSKLsw>vyFAr;Wy;o*<*ypPjaK?PBdlr^e{MD2!-#Z z$7o^4+D1ewVQ{SXYKuI~yRA6E``^o55regPEk<%yTDU^{?}1NueCvNFmK^Sterw^S zb|pB&{qptIw13)(J^+*`=h2`Pmv)&YK&!iljnZU*;<0shJ~6Vd;(#YsaG`l2phxyB zuyV8bw|}M@0;TSsxbUn1IMqQWd4@lChQhp_7PwXUi3+Q0|Ru~1Dc4Pb8?`(2dqQoE0@UJj>Q0YYKA{F$HA3sh# zy`L8pCx{?$W)D^?;8=ZU^ zD>KeLsbU=^A~8=*X7tP*^4#fZ!@d0rs{<>PngytQ{HJpd##itYeRi1rrXfTiNk)KU z2!7NaBg?Ln6B-q-{Dw{>9oFsWiQGF-<4xE0Tzhf%E8N3k15)|ekWe9V@2^s|E^a`u zMJg>07kFxZC%wSAKUI2yxN2{*x*4!uGJoiHkLbpS_y2em-c>kRCjK(#aXewc69^A{ zY|un1TiY8@kQCkGnh9;KHbIssWQQ9IHZeb1X<8{e8L5^gynAm3eI!yhUm_X3k%?z zkp3K+Ci%aqN`nz<5j=5%n(<_#E4>qHh_xtJH@DU117eJL8@{PdcM}#pw<@I!A+o== zp-u9)uI}wTLuB&Xr^wyR@e#}1@ia!{_IRuTxL2V_hmVy3zcEfL{%6C9i%GU!?Q%Z< zJy0(*$8t4V)ql+BjJ&hNJL;lt?atF#Y@4ae23*RwDaJA(;7Pi!c5)dxo~AdtKp}Fy z@HPKilU-zzJam9`2jGaDLDt;>wXYXMvTO=>*Py5P%>pH2FSo3Ku)&+f;>B{TueKuj z1MAw-L0Pbx7yL?;djfZVcq2px)w~%!);kUTF0`NXb-A$$hNQv=s0)bQrw|l<(WqK# z4~!Ws>H%5&$o?f>wvA55*}u9d;d?r=S&cSQ1B{QwcSsjo+m3=*dWq^sEj0ig0 zW$0PwUwr~PrX`oQ0j+!fkcj@7~{ue^Kkp$g+? zPB_ItUsw}uxzR|{%-mRnuCM`5+px%bU(#ga=+bS}lYWwa@wma!D~ zu(o<_>5KDDzD4Sbbms+&Ku(sCCDt0GL7thN+brD^jD{jmG;xFeD>7+J-HA4gPHO># zk#a)3cj^Uv*1~&{DZiDQnpcYq5j~xva_-#Es=tg%b$2!2Ibygi-tL0}s{*$XEkw6F zFdkd{HoZuNHaWQP%+76E`S~o%z;c*Vlm7&3{JP%(FXqYvqH&_@Go(DkN=GTJ=BP}p zyqRXaNPKB6A+NitXH)ltE=Cig3EcD)qmuAL2=jo=%eWq@zu)esd?^-4Pr89<;Bc z6}(sC3Y!XV=N6@#YRgYOD(pGv&cN`P_c?1a6yagSCbIPPL?Y|uW=*V5st3gCogD+n}+2Is8G3*pG9W5lj>wFyD2p@iSV%~3hK1h zc)6|q`Lx`UQGz0+vq2j!oxc;$3AGX0%UTIIin^ zn#ry27vQvNKgh1Jp5Ta0ewSVIIgH8W`{JW_B^epzXSonbQxT8Uyy0Vm3%*(Jic6W@ zw*5K{C-0eT0cjB(yg0GwbPu2&tM;n-yBA{QW%}P#u6^ss2NSSF;)mv?R7^s z_VmPpIfm3npWMQ~rlxsm-Ks;^F}Z*CS1enVqdcR|FVONO zBqS^e2dDRpV9!TonQmI|0UD%vw8k~S0uH4%pHQJ@KRxHX56sSwZ4OdRgON{gh$Vs_ zG``PW4D)^R=YPQ6yfMPCg&`B@??1r>*7ylX<_7mtCTk0WyC9Zk%;|l@w|Ha@n%%#- zLLiV2*?@yoJ}iX_8v}+)&+kGoc@2We5DSN29{^6@0O|rY&xgV7wCpeC!@(bo9Fkn& zV(^19-0_=$W8ZW=5wMvXV!;9G2(*WCJz5x3bybm&h$RD36CQrdwNp!a|0p=^xbtBY z>&9VRFw(d&zI74G&G(@5J`Jx6uwbo!nF@?;hypFtqv@X&Ro){(af1~&1zzUqUaMb& zmmMHO;RzS1Qe_3P~SVO~JMLcaEYN)4p(BVk9PwcO=3=1OFUP(Ehz;2j8qp+sZP2T$c{ss2eFj* z{@RN=XfGZ>qbdKFdm;U6VwCY$L-8z>hQC8;c;p&5$-A-ME&c)=&oDsM%1pV_kefS5 zB-h10jhGQT$;la`s(f8vU$ARx%DukT3tZpXmztDRXiLgLn(*c20dM>QH9~ZF)W9O{ zJv~g>y%hnmgP6CVR5~LYtGtQKmmrFWt$aR$Ey5}aL&UT6>hlyp@)}=hCGT9iQ(N}a zeRU)&KVC2h@qgHR>!>K#wGUVk6)ZvoK}xz&N_r5Gp<7Zw=@g`UM5QD~T0lS;8Ubkm z0hN$$kVd+@VHm!9M)%o!pY410Iq!SEf4=ptg=>imocp<+D}UGTy6&?RP5zh;?kVg| z0I?cD(Z!`ExfK9fZ(A;NV^f1MrLAkQq2Wd2gF-3of{#twRDgniw2Rj`=5@yYgU1IZ zru$sod@-C>_CpFF!%MQT&9`tFN8Kf*NO*5avSIf411Mb@0(CY3nsGJXwV(_i2mLom zQ}omu*aP!p%MQL44*JC`0*{5HGN=Y>K6B&7?vIxiju`?2)4~5NMKR^hLn%qao^X$9nhxbR$!x`%5_6cbpuxCnJd-R zjOVf`U$Lk(jhX(Gd{e-DsMi_l_f=%YG_JWcCn;9etQIT|)h0@|#cWHwu7*`l4BM{c zPEeF;;zUsJB4LdvnCjzv#iWbf!n_d)2J>vzVDri{1*Zi_q{?2S}R=6}4o z885Z5-i2#nZ5ab`-xD$<<8{9AFMPa$h1p#vB7`_B4WI=lWr$O z0W6v`t!6zkF@GrL3_1F+JXkJKfw$5TzOp7lnv(BH;h+I8@rGGfiQ&=)|uUJv?aVqo;P=5*{(h2Cdk`fDx z$9WOa(e4&}-AZx}xiax-S?%wvjP2~?509W}9c-)al7~l9sU2*z2tid})?@kCP6I6j z-oeJHh)1cd6#biYl<%6uK_RY8P9VL}8APcG=vRxc%|q>P+MzB^cRWZ~xBQDiz@d`7#_3tBnzeXDY&6E)iI68BPk%MBo? z{$#_ELcsl0;`~IL|H$cUQAN5R$|Q(nKkv@(l}wFq<{22Tc5x3{q*h6 z&`@P<;9q@v(!P=RQ%Z_e7iHu0o$iv*ik`8thf@1bVi;(V^EhIHG;j`t)G8?{&`N!{ z0AXhrLc(-A>2A>_iW@s-Uq}z287CJ!vW$5|(zJAjh8E>3Lli5Ayur?0HeSVH7Tn8t zO0S)Fpj%HCRw&v_2Ya}!hqw_WqzlktM=3I{8@=%skGa6klS_f?fc_8WHm79J(v(mu zDlT=A(S^pmyG48S?m>}Cwv{D?G|O4uGDm4OD}Pdv+@^$0Rz0KYr0q(}R#KbXn!U_R zuL->U{LdFYgum{B?F#RuaudFi6+geX(jGnd5m2X+?~Q&y^9zEAepL5=V4tDmP+#eNNixz{o*>eCX4WJ`V(?P12}wIP=>!{*+)X3}q}BhDytdHs8}-e&97;h9(p zyyp4FSHc^e$34EpYk+R~P=%Kd9n!^r;`)Ca8%ZeIFR~pM)vk2TUAqd_`S??LYk^r% z;&DEb2X;QzV;=`&lA1t6>J`&+MWvaz<&w^{NX*Ba_f)MJKjZ5rx=jUkLi{CAf~}+y z5<|i75Z&Jp@nA-$YgA!s9&(@Ef!-s;{RX6NqV--;ugx7{;3 z%ISY)=XILq70P9&^lV(i(E%;SAmWm^v^F;O^jroDo7_8c&to@pCbT>rS00iIroc62 zbdp%J*qtkM*Sw^pq+0j7GQ)`5a3HfDY8kY>78iRKisu8woSdB6Rc+EQwtRG4%x9$X z5=(SnK23UC?U~328>{q2OZ0ZWpyu1?WPyYUolRn9x&JXUgJv5Z1n+O<+Iq=xRPP%a z-mLWqKE#@O{Jnar zDDRPb(Ah4k94AW-y(PRM!EkuqeXu;*>+oU~s(G<#(YO}W+8fQ|cy%hP-R3iQG z0v*)@kf5~sh3}(NN!#TC2b@;$$^jQ`8*tIgrFt>ybQ&iTz0V~HG=nsUkWL9lpIYD< zh3Mritcm28L1O76i{80Gf`JkLoN*Me`N?Yzk7Jx(Hm!+9Gv%eLgr6 zU5v6jcTTF+U3zy7LZc*7@{R524ZS=PZ0QSgT8eHt+-Yeo1Z3B^IQ7G z{{GxHi{=no`{~B_jxLo;pFiIQ+F(Zjw1MC$Qz$*CV6^-~68z9CZ#bTe`8iBvz-%%& zg+-69V7`X@80h}n)Ciaii11x$MMY!5+fy&^2rh;9c(CQA- zn9QyeQQNpT9-Vp#uA&8Q%SjkTQ>sn*aK`W2hf&f zI`IvjmRCiqYgG4;8=Y|;fE>8`E|ZiBtDX?1kt?L4c$}aqC+88v25zE>&egZA<{Vw) zph%X_*zeA&=IZe@Nk8nisCHDzQjLinFMw9!HKO#Cn!ug&P2a_8kkd!wZ%EA3593Jq z_*l@(2b@of|DeG9k;M4hjNRW99X_`<%(_y}X@C5n0yg9FU@1Qb`(jzf=LHiK&rjOJ zMkwz1{7{Nst;i45S+E)6VR$VE40t=~q_LG4*t%XDKxBIC}O8VMW)zm18 zV)ZlB!?7O=#6R2=_prTT#Qf6 z%9Yd65lUs@J@(2PSF6a}kBH4y&fsWeLn}O?@+>b`FXwU9X!m#z`x!6j%ngGI4mINv zz}@_EZ{{_I&Gw6E^N;r6&pPup&>9NNmW013a!WLCOLrQ3%2Ip#Ip}UyUjITvPa;*C zL-_)HQ1GR1C0H>w_g;9rQ_*ezkzS&c@!Js_t_6Czp_%E3Z=fErGTE9fF?CE9?0LL6 zz;!(3mX2!OY2|YgEU5DEP#E@VugEaFzG23|7PJ=tA%34pb_1;nim-!0d(^P#ch7Nv zDu-+vy|_oaJ1e0XejEAwy8;fx&S(xBFdeD46aW#NR8$Cg2^s5RvM*OS`H%~-R}!3r%R=D7 zlg*Gvw5ZJeA`7s7)GGMY^h-!zA2XEi!VMGYy}R{P$0~Zmr!{v3*C@ zK?!VTNVvv3B>^&xjvxc4Q0oIki%GY%=+xC&A-nEt-dx69&r-aV+IO4f57KB)ndmR5 zN6o7yGfh5XE927Vl=pm7RBDwy%nJn5#E3o7sN z>U-fQozc<7TwUB2MgWL0XU?CKBms{09*GW^t$peLJ{&D`Yed8LiQb`^UAGTS`}uIlpAL4nVHvV z(H8yr|76h_2!zJa@KS5alDmba<*lH~eihg=eZiFa_4Uk4mqP@Xj*lv50vVlKB9vBn zmqbi?5(D2VgqZiTe5Ifukf~?3vonBPkcZA64^Lpp z$43P`_j3|ez)!;RPZ=O&%G=+XlR~PT?(#fOgQ_cqzn?-6ks6UO{-1PHM<+YPRgHbM zqW_~UpIk&yZ~r-XHKNDn@}*1hCUtBQDU%&)N-Gf=Nn^IrUdfY6P-xLRgPI5Zxw@v& zGgADP_EN{7V2NpHIPApFTAnJ+vB7Hzqor-P-<}V+bS3oWKf5Hijhu06-LV}4?!FlE z_(iMk;C1dc=C!Qq(73GXyocHO7AYU)1kuF;C3v`h<`Vrj$@>#Oq6c#ZM(4?YnO~`? z4K?!hnMNiRhuK=D=H?Evg-A76IH!-vre#ZJAi7Ev^|&p(7y5Sw22!{8@M0HA{Mu4z zSGSuJMf_XswRTrU*Z|Dplc}piW?cW0qsTGNB&1JO0z@&lW$%vJ7ipwn2IU!<}Bfy}(B!gLqaOl58yKa416;V1%9RR|EOFsf31vZi4exZy5 z$Q;JX8#R$Kv1?WB_gYdNHn;#d(WW}E{UxSlmZa*7bOd{NUKa1Tc25sYb|$~5=;P_e z+uWB20%+M5{qjEXTU0|B$rqp*s&AN4!IySg#MTH%=a-^UV~EY2lcOeU4?X~z{I^7= zpwJw=96_viheurw>d$&SUy+>LFl`m9n}<^>DYmn-viKmczX|rZFI42_PQ12G-BfzsqwiEQoc;N4H04hI+Yb*=4Mg)7JjYJeBpwHWg?}F^>JWEGnlfH9ugv! zQLuTQ8d8U#jWns=Q(sltFmrU|VMQ|Cr8q0SG;a zpx~-B9}9|HfXh+-3u$CS9di<{1xhL?q-PMdE(t}ss=Og+$i(Pwl)y!d5RPO z5zr=EnTP+ig&LHY%xS2lcmet0F<+)_9A)ejz=9vur9{6U-H7^~{8=miT*Jzl`ZPc) zu(1Vv{K3v%hcB}vGEO$RWt zBy9ih`*-yPjFI{N*_>BD3QW`N|%|NhXNcvCKX0t7er+OQizr)E2QB z3Pq=BQA}4Wf9xUo`dq9$5Hcl&Bu%=mDb(EyNIA0iMtH04`DSbA3YQ=k z8yxb%M*kJyFzF8rWTF>S4xGjLtBDAiZJ4c%RoRa``Fa3Jej%V+Y8@8;xFU<1od!jO zO-uFTt_^%9G?zgr80~WY+ztWRG)w4tuz*h->_Y>{mHI4pvjQ^qI#oo0N<#X3rBLz_ zMz1PpAvTx(7{^}Z_8h}>e}X|)n17Z2b)1;3$)94&YW6tU8I2h$i%s;6edOnJ z`o$x)#HSCR5uCloD}HG*{OpumO2hSt`JlKyx`sOQO+&-pX_F`4GO<;Z3C;uroWZ%& zdg)TpHdob}OU<6wc)4C*S!IQ7Uu=02X5#xIj#v;7-mKm^&)Hkoxo# z4*Kg8+101}i1Xx=a7xar4id=Q1^srd)Tgk#=|J0Z60Te6zdt2~Gps;o2X-pnfK5Fo zx&Y10MfmHuh0+tB2+>T!dF4KEuo`P#9Twp_|NQO+UvbOm4z8#kfykTw67p>wokoyD zQ4x>tw{I*hZ}W#-!RzMl=tnJwkg?*!cpl!&xs6SYn!kUoj!Hzo@iG?pWbm~EKK!c; z$s}GKm06D11(j8f=-sdC- z>Sz5vF=W@2@5qlcQ!6WhbG&i930Cp{C-VIo0D1i4{iq~X1`P^1F@8c4By{si|8_GvOwdj-=;%`2ZzsXRS@D*WzK)G-w*Aklpr8WnFcpq|Hc)uUfX38AVwqY(D6b^qeNw{EE^6CK%O1=Gyq9~O1(aUx6X#yY7 z#&FZ3lv-Wec~ZI;t$-I_jk8vhjd@^L2w*i`D0Hu8yg-*TkKTE%lw zR5G42m^vw9C4}N(G@uh7qZG7grC^u^6M}{y>$-ai8z~LOQ0l14oIy`B1iv(1`a8IJ z2##FZ10x0P#!LzA%A!eS1Oe+tZO(@E&GrjWzO!1$!1@%+=&%>4gZyy`IK&fHUuIc! zVIzZ&ydY-h9$IEz|81Eic}_ZCD0mw&7v^m*OcE4SQ2mYoHX`mN^MTVUd|m%$=nlQZ z(RD4=nLB0y18w1xA7o8yOO*qq-wL@R-I2tS&-*x2+cQQ4WxBJe%1|G~Htm=_@5gc7 z4r-AdCi68l>q?IMpQid}jB_=qC#$#TNSn2TBKQ*J=-l59d*3AhzN?q}$BWctfF5 zr2A2+N=Ld)Z+E{5E&c*yOiPxw6pCO0YD(ZOi6WR_B)^iQE!q;^QT)1RRe!u) zv4YxqEiI}Q4neE;oZrnDHmt4q?Gdmt&)QT5uL&$@u1(H!!45-tsq z14}8j1&d)+B3r$papY7gOKMYyT~VtNtc?r}x`Lz~jQZL9H`Nb>m^z)>+s~gy07GJo zuw=%EWp{n%`%Ou7(cb*yf9lQu=sLGJs=~j7T~BCAu6E|QxO1P0m?>$lT$vho>polb zb)~?#I4rt}YVpsf&Pv?9aQ^w5vDD5q)gqSGm=}j}lLW9URGx3r85=A^uhF7<$(fx$ zT)}8K_J3K!2a9b;e7_J3RUESo6&{l>&M%e_NT%Svq}OFiG*3R&BI5HfAT5c3g?Lk# zg#VJ(%Wj9B$hi24wE!Z?NX3qxjoR~>CyhNk+#+`ojOUm{AQ`27o50>mQUkqmNzjI7 ziS>W+Wo+%|z<7i~;WNc5_iWTICjVvI4&^Z0v7=il+?1xaIW5&OrVmnT1 z`PgFRy>_w7Zg%l%h%Pf>hzbu`1_SX^dp2aiHn#wuIPG(&x$FA%DA{{ERn{B|v}eBZ zuQ0V0;YTWDWPS=)&SeB=XLxy>3m5_5?7xa#|F$RgnE~CT>wsXTF$~80o``c4;efN! znvs2frKeXF-}}b;agkDWr3<&sSap1<^{_$#HXLwA>?6KjEC&%1?LFQm0!PNbAdd9e zpR&FtefnqMnq?}(D<0p5`d&qGaenBZr=HszV%}4 zOb;EM-sEC2|IC`+H=slPX@BqlEp;RKF!^`9xWL<4@DqAo#xq1X{GWLYn?187!_1#~@8sG(uIs54xa}j) zAXi&kTW!?U+bao*;oVWUsTX(OhT(V~1fASFb9@HrQl1s2n6p?e-$Ar6bVFak*6W)` z&aAG*)bRNB7LTxNWcU-DWRLi%8cN3xe7Ztymj7^6JYM{Cb=nD9L@?o&T|5%(Vc9uf>h! z;&oJ6d=i+EnKnoTi#WTyEZlWA=Ht_`y*a9$S>)HXgUjxo;}X4ZI#JmL2Cu4f96H7& zI1kE`Zts%Yeg9#>Au?ihNz$o-hAOZJU#Ba>D8>ZY&FO{7enChPeZ)@X6HWa z)tmL?iOhBRqfiFL9f#2|dNPCZQb!*9&)k6hzt%M4Hw4idMsPh|wg=C9Xe~cNpMTh; zYU-PX+PGA%L(P_zL5kAxih;S;`PeqP_eOKh0T5G-gbYOWR=i7B&k>$$^RRR*)@SPD zSt0M-Fh$(CNe8T?V>oB$dLY5L%KYM>SkgXnWoXW8Sb;MmP~~J}akNt+zh;KPMn44Q zn_J1L^DTp)S5UfVN@D5o`V%{}MC@3wC6OCbj>DB>P|os+hHFZrWs@Cuz?L(mtpLc1 z&FKq;m!|wJ{Kg~kpyAuBWmQdh8A~k=#dS&jv!{G(QRd$WFVIJE5Y$l_@VUKlJVWq` z0Ooh@d{%Bf)yYbD_KkR^&XRB7f2zraOm+yl)+L1{Bj`@5_G;BxqRc%Dk9IdmN8O$1jKY-oX1 zA{;RBy-fs1HnB7dEo6{0}XY%aEJ7toV0TY4NXL zzy6lfdo_pdj#WV>1SP>p!AXcZJE=rl{P4~5BwWv9E0@j#6R{i*kLueT15f&us(Eo6{e7(8;#1J42Mkab5yxWvN`r4l zBC*N_M^wYmHQ1a@Z_}fELt(lyDu$7w!{q2=mr#h;GVbB(=w7GaD)rgtsC{<00x?sV zb;h0Q1CnFS3H~WR`t@TEls~1%c1Cf*Fx7FRYFO>my^3kyZ=IEpw-Bref5<-N(B7xL ztc)JW$J8~;wR)1ypuFnBN3ipZ9)|cZ#c1;C^XR}!|HpN}w5DwXu z5u8I-v5If6pu>@dHJ&uj8xWmpS@Hexutd>uK7SJpll)#|kf5OxT2S6(Y0A}k-+gEU z!mGCK7}I6!MRW~Jsg2Gm$wcndB}wH!dP4lUYCL&zj4$0P8j%^9{`Oa_{BH!*=gXjr7}L3@$#x1 zH8{k)F&+<1ONt2Gxt8u_YWTvG5xQ&?-O)4keAnz98kc&yfKFnp@PA@JFAKo8*L9uv z?eNz{(9fcPKJ_0^1(0x{%OV>9ZehwI&~U_n2A?n`piBu-0Hkl8XSXz6>}mUl^}hQ` zg&C|5oQ@X8`1eP=7a`m83!;}IG3Wy?EqL%i!vFFI^8q@ZUJbtxa9G?x_D}i&$oZox zH3Hw>E?orv^t%T-=ZpiA`y&hn<$@Hnqd|P|4_xy3Rl|l1BRUD2zw>FgRP>iyqVD7Z9s2}<_ykjaWrP2 za2FKXeahwWahVQVXXO;L(#*bvceFKq5ASG!)EW8A$T%Gf3-L-h2|P+GMmYX?52=_Q7ttO+0ga zd-moz_zvraxwmP12^mN&X~fT+0k5Sihv$b~?3D)T=QzC-(R-AO%!n&L=``ZmABo7Q zxZz2K!D85fNZYf0&?-2{^2(46eA}QwafKY)Qwc2sG{pLc{9GTr1vJp0`y0@Z9SBhn zX2IGxF}Z*ZYC;t(ABK{V_a?M910xx;W9(i$T({~rJKmBVK%;%d?>0f|4a1BzuHs>- zv$);B%?xY_lL<|J;OsWo-}l+%ajHIZd`p{2QTE7H;mJ7LUHt)nFr_skNf2*BpOr_LO%%Evwu8@PtQ$Mv9qHJbWh51ZN9XeUT4U> zr#}HySxc$rY>EY1&$<`fWHV+h{QO^&^B-QiOzz>Q>cG9b$I#$!VQ3ssrZiyjm~~## z2^b~y!*giO+W|-TmE*AMSi}I$$_hwa?0ehH{@l1wTB1-%Cc?`(CYrW)oYqH5lH z1a5Ol8Py_qo)|_pNJ;fc3)9Rh+?t~% z4WSUlIOAeJSh$AEq1jAHHLJ|TT@-oWB)WwdS32F3!Nt15{#yP$cGWl z`M(!Ufuj*F|Az6iWr47_{q1~~Hgj%|1h{z>dZyHPM4Grc=jIN43(DPASy?zurW3@g z-%Ebd4v7igrA>j$hnW{Ycw7`ol`-3|VQF%ZQIZgou=24(cuz#3izr5*)}*6wpLQm_ z_s!&{y(J`8U5b4MI4Qmh?|AUh$BV)R$J0d2tq!7|WoNveA1E~Tu#z)xnlm?k3=UZi@_*%}S{p64^fA%j!Bo} zO9w~3bX|8zcg)e2I>Pm(9a*@Vj^*-ZdTVeq6B2>osdo^q90=qZfW7u}C3@R+^~@Wy zP}2;?6|cO~XNtXjjnov_t(ezW{b$??9s1CKDDkTHCOm@=9O)IO?tIRcN=q&;2!Bs|Q| zaN~6|vC)Kdv-(Woa)h8NkL{t&HfCMi6a8MfUcKaC6 ze%-rcg^xyAiR2_#*mt3mcCUMS{uA@^X@j$9F%@CN1g^mREiwQx^|&;|_I3~5jc*^g zH{2nZRB8?J2-8(_)0#_vsloncq45JXwgjl0NSCC12%#RONk#&6cp?^h zppsl>sPa-4Cr>N5*2;t_qCNALuC{)={5p=<)rc8_r>@$@8;tSo-6PxYSQ$tyGSd+E zXRc;t7Fv_;Fn*blJy6llJ}w-gHIWU-*B+}0Mk1PL?Pp%sfxaMVSqq@VAU?^S!5{~* zXyo7kq$5-{WvAXiaGr`)p+!UJ_q$^%A%ZzCizW?Kx-SG%c$j}HC+R8Gn;RUNZp*sl zGG;8mQTZ{JJ}FI&{`#eC{*T?)Fsvr-d4C)*LR(Cm-*M zYU!XMc-S`4`(xOoKmmtQiI!Bxjg z+gJ?V)V^%Ok&xlgl%gPPVj>R<3lOK9k;N$x-qW73ce|hW;y`tVBQjyiW^-)FK}p0Y z<}coA(*pq6-nOUtVdxeMI&RWk`q?n}w=*|1q=H6T)UU$$lFFmp`F68Go0KJ0liPns zR{0Hd0@wH1Ow5;k-&<}1@>NWw>tD1)4Ze38knaZ7TD5gVOFRwS=N1)~g5ps+*VCCS zS3iU|lgzv_P}L7N(|);4YjXZWD|HO}q*9oHDr;t8P^3)S{drDi%`fo47wRn0^IhjD zOMj&WF39E0@zg&4;u5yYJAF-@+Bx&MIbM&n54-&Lx|ZkX3)MZb(bjWFTrHzvqIJl{2wz^q5Sac<+n2R1b z$3dx^bAukE-d| zsT0_QTXgjZR$t(5;dg8f#+`$UNE}12bk29)zOv^OkT|mUw8Wy1UQk8z-t)`U4~fve zzVGbch9ZHz?zyheBA4u*mvt#bFyDyD-*y2^+*<4JUU}d^&LkkB;oC^=h!Yg0)>ykBl`;oV|OI00;RzaxLh z05sA6L%)z$jxrsp-VzS&8I#FPs3PaCYMEsSS4tF8uqQ|8_2=HndUvN%a+d(KmTVu~ ze+lUU(S`fAd=Ynq<9d_-LUkpoWo*t8WG#9jm%8tMVnb);ix+=gZ%iciVLFjOdmKJr z-0hB?ThYi!6Soe0QOgPQb#;#49D&mz)-vhg6}1$yj@zA&-1m?0D!ihGJOtTJTTF|4L_Z@7iH_6*X&bEno9NbAA)#NW%1zXM)!FYzfch?? ztD%Tne~KGhLwAmeSIXDzEho#;u|DJAMUfYl2hP(z88V69nTfy=<64Bb7Cu%;dln#0 z4o#@pYkWSQHVip|_ov^F0gry|GT+muD(Jv>JKIz)CZ9nV+pm3u`Ax(h7~efuas6G7lT zZNeSS}KeqDL3`JTySF+jV^Z6h!o4!)rZmBnCG* z4G3y*?g?WRiZq?gatzl(91q+e)<~_rH&z;6Ubsd7>09pIP$d1ge7x~Jo5RH#FY5zM zilnZh{LC@3w%+; zhv94+Qk#&i>AMvty5lZKEKZN-)0ZB5`!<_JnX$7TC29W>3(wzjU%umQ-2< z_LrzUO!L7+b9r<$Mc&t~ zXV?#|n7N#vGHSN=9%6*)CnM4IKvBb={mLCw!}*0TEqUqWH3am#Zi;>n^+f)S;9vv`R zIpi$gD!3Yxo2htxAs}qRi#Xk!N4&sXz25s|LZqw*w8rQDHx@FN!peylC0SPKl2w20 zf?HOPICjO|p-(QQ8_{%Ys+Dud%&$Gd>%v<@QEAF2+^CqR&yTvrT!*-HR^3gbJjzRn zbwyed(QGf5w#n3=baHfJW(HPA{fCb6@j=O6*A|BtflDwEbImQ??~=%{I|yTai^5bY zt6nNrr2rvQ<13rUPQ2Ar&Xc8at#}Q+l&enndBBDcDNI(R_xD-RMb4T4=R{bkV6HO1 z{DkytW9Fh^_3tWaTjc$cQc>-k!E9dBpNh7pJSgpenr}OH2N~=*uo&DTQlIf9X^!fG zWh>B-$+#W(ZU&Ke=pPnWA9f^_)PTOypX;+t-Bo$hje2u|J;+hSNhS8(TnVpH$E_BN zZ`#8vT;j6Dxv0JN_TXzEF~BzmgO{*3QQ!ja->@tzX4J^G+*-N=r~Azsk89smJf5jr z>U_*3+JIIu*C)^wR(ld|KvSvQwz4;KbJA6Pwn#Dq_YNb@ucqgp%ykD%%^ZaRJ=Y7o z&@J{!7!x4Crqo{nQhy8(C=-&-P?5_0;y9*OS|O-$sHCgNqIX6|$ZsDVT#Oj3BMtnpyMkEf5cRRLiXW!S!kSJEUjlcVS0` z$Grbsrp1igb5c#19UOeqL26f@XpXtE>)Fc)^v-VD>;ux&DXb8{rGXb|{BH7ZDJS8M zkzfc`#R_`ISA3?dh)(buFQZHF5VnZgEV?;2OkAWZZ@U&M26XIGAFt^TkgeB%z_{pU?tK`!&N)-hElORYup_}9nL+5PCq=6IXu14>fS-6`ZAihs!(PnV z`(Nao?C8p)kH4J+TkN|d28F^Dk?6i<17RkZI0|CSEGmOnxF5Q@&b~IJO`wa>bEDX? z5sALFap-)~`%5wF5fwv?~OEk;ynCL(4ab(uegT`uQDM%??XB$m#6 zQeiW36BBQlGT8m_s3vvUzuKkd7dfgez>D^C8=h1HCCZ2gCpNo`fqp@&`F6Azdp%WP z)!4{TI|WZ}Z`q9e;_%1kG;F7vdh1SxKYHCMaoSccEDw41rYumxOB=DLC0BXIUq~*s zR?<`^!h(KduKYQ?XCi|t&Bv(Ym9&^Yo^al##?kqN%)a@DH*j7{pd*Uj-$xYpJGO6k z6BYYhdRw^^Q_^COVsxbI-aKT=lo}5sczZ^e#I4vI_ZlZByopI@i`#)mh`Y8$J1!w{ z#!i<$jxs6rGvZ9%Ys0|n2?ZYh%_Q*p`);b~tu$CNgQ|HiPDh}aao6Z)h|7`No;p!? zzRt8Ob*$s`3q;L*LL9Bw1HvlP`))Bz=JccFZ@Ikj)s{DO3)J}hqu}59tuTlypf&-m9KztFO^@=?jRYBLTkH}vmSc5qzG)10k+Dc+w6PU+G63M_oZvy=(&^Zv?ZJ_*C)_d*>5rUHd}Ym-V+!KBpjDi$%7CU z0?&6%?m=>!Ic4v(t`N#J#!;iR=-y+WKEXt|x&I-;{cLN1WEZ$4JL&y^MOw=x%Z%N2 zXUnB!YHd7d%P`HyNN?H&&1MtVXcWD^qLO^UGZb(Os_I*CqSz*y@j+_cN34;HG$B~n|JOuZ& zq0?eJdho8Ub%kHThedz?bRNwIdT{Cy;CGzDd4o>I6`%h88sDQp4YU)5uok=*+bSG# zWoJuQM<9Ohl>fe<_p}8WrLdF(w;IoF_b+3mLl;1;O8eXq{}vjSsXF|7kBKcoPMF9@ zdQC^%!&jhKQ(9_h!ijW8TmKX*r+~FE9ETmT78x_MuFF*j4<;NP{U&jPZ?CH%H=cp3WBG%^AlQybYN_*x#2F3MGFvdl6k#LazJncAMNK4Rjn;kVvw44mQ-wu@wF zP-MC(U9awvbs5B!Q(sK+Z@oZE>E+)`Y5om-o2n_*m^#`%@@;LnVv4}ss(8NaE^pvZ z)oD?e_L9DfB5snA^52ahzasyoiKMU9OI0|fSpMFs3r+0Ot-HvqGwREm#pr&ACb+Z^ zjp3nNCzaTr<841!Hkpm$Q5|oa&A$`zSk9+#pVp5fr}!UBkhjs39e(V?fK2gtqAZ)R zx2q~}sFH6}S5+lG1P5~9soy3$uRX67R+eK-JOLf+3?Gb(B&y>yxV8m6c|y8J^??9y zA-u%P(=njI?>v9{(d($`3Q5LnRvP0|r)e=|6IcZQt47#AKv11P2p7$(;4a!n6*T1S ztb`#fO^jXP)76mhjXjrqh-Y`d@=}0N89FaN?*!X~N*pHC&Zj3{HTzhRmsgs_w+-k*byUeWPr77EBbNxa%bN3R>7Ri5jsGpw8CYlb__Dvoj%>LcTQ7TyiJCI!pUXGXqzh2oyt3;n7gt3~*`EwWS9HKVlM9ApcT1wRJaTD0I&tHf0vZaicK#YcMxR5_>Ey8f4$ok zyZ!uL&zId?9cfy-6Zo4~Bgmz^S2rFua}4;pvT2kB4O z)XW}e7a_;6>`)*r+5zwStIYM^TgV{DFAKVG9uIkliu)Rd6$-1{bF?6~)i1@ceNi$A z$e-zarvfv!P`FE45j#E^uH6mW)6{PUSAq|Ds>(U0Bogs?;$rtx0FUW@AM0;`Io6Gg z%6#Y!p7x}Yl}9Dn1mj@0{G(_eIz3YYm_19eX0(K-33DyhtKd^~H*1WU5hsm20(_qj z?49m{r{d5m=;@J1Kh;^xQ%W(0s&CKdLY6->WY)cAo^hecW(_3f*Lz zO?7RYc649RRnus7v??c$3lNFwhFjwmtzU)LGc^=|0B1I2B>?^yu|wIr+?feX($d zcJx!z3{&&R?$L9kvT3Rpj82N6%=NTbzcgK4G|ry$s1TNIdVJ zKZRrZ&EVpQUwNe8TJvwgjg@1p9LWy@W+)S<+GJ$uu6~qPjx@L)AxcNOz}U#`!l6>| z=~z{mO*a#fL3EyCx*khA|fq8bDX-?^4ud z{v)^T*>NN-OW2FW3(8r{pIYkQ>z!S&o3JzB%I85Rq}+uM-niPP(X8F8R!f1nf9*)J zUv_d#;pna>!+DKSsg=LUL3?G9Ze5olx|DtX^-1Q!>9f~4O|YrMyn4;Mu>!uW*li(c z;z;0O)q5q5%(|$tyv>7&J*P3T=U-QGY5PSxtwjrS z?QPX#F#rbaFo@St6=V~$bMHWi!1D5~TfU2!2yJya;7W>Bc0?p~B z0dyI&NH zrEU`*(`hfdd|b39({-{2>e0Thm?)zA_|8TK zV6mvt{WQHXGfRWGO=wGp<+3D(L?dqao(i^6W_fHAnAx@txxJ5Aqq*H&nUS1^?pD;=Wz>EgW`50N2AHvgiMa7 z?@6&QVWt^U(1-am)96p_R!e%+h~1LahkWLDQbAWXR1d?%g@=?(47z1KW!X^ETS=YUhXQ{Pae^$7)RR8-OMk#`hM1cnbQ0 zV4Ur`9LN`xbfF{(9vjzshlXU(z0$ck72)!MELK}}ll>JIf~PDJ-ZFE+jSzy>%rI?1 zn|Bt5)&Yq^`~2uq+?$nE0nc-933Q^~wK0}RGA_zIs*MqQIEtwX=>HNqlj=PrIfM5g zrSP$n_TkJTTIqsDLnaH(ab8}$Rc(b_0df~!ED%ujk6Z5Kyw_LoN#D%u;Jzf~aPng7BCj-+8vYxvcDqy;B-D~SjJ!`^;oHn-KPWX_pfv~y zA@hnmLP%m2e(8W6T^xU#Z1lXZ+PQm}8uf-p(DMle#!LnN-c0{9>e5?hFr?T_i)suq zzDO%=6$H7s3AaP`lhoX0487cUmbose;$gj_5;r=KUPBYu08CaH2x)QKc4U{5)v9i5 znVTv&Z^7~5`WROUr@{f^ET_ReN(_XEgHGG~{s1AC5W_&p*_BO;V#C`h=?;Lj07Qcp zB@Cu&$Qjt~9j%^(9&7YBi9;C|o5E}N#2$KKG>Pgjn)K(sGEkZ3AMLXN?1_QnLR;C}TyL?^IDct0KiRDrr7Q=Xs`q<9BBT00qr42NMN z`2eCyf$KAnz$2pjorfxn@DiTP3J?X|8MdY=pu1=IpPA3OL9t?8%b z1F|Wgl_XD^h}0!d(ZBF4kvU922VZ-)61M3UTJn*j@$uxY&-Gn+*((w*H>8kV^nOSN zOu{Pn_X+|$8m+BX{Z)1%B|!2be#&P(q^Lsd)vR{aY3K2z8+VD*?tA3bnfBQ0GM+mR z_}WyJ{1~LpSvY=3O4r^kNRN%hC0m7vOPqnKe{EgveNnQI3J^;X;A?4c-mzjF`aezM zfwzcZ8vX;=1rA-#w%P|w)UE4Jn7=qmNFfT8k~vJxR>(eKj+h{6FBG0rr;3<{ea~C# z8R#@G$xqw{bU?e=7!S;TumUDMou%cpOEJAiCoNO5YTKv5|2F5ieb4 zN+s3EIU4wG1fU$T@G6~w;;Dei5p&o1*CniRNeK+Nd%S$Y(q}0{%ZaZFs#x76I07yd zLr$YWyVifvpaD_2eDN?TS(!iei|4(m+pD@SH_UrBHkrf}Mh@y6BG^n=qIxsC==G!U z4Hnenj^1)vGGo1xKSIsjjUyM?iD~U^k3I(b`bqIUtnk7@D(c0J=-3My-~A_s?7$gg>bp{vY%ZlvL7rgiGQ%L#6-X@i5j;k z>CHw6AYt^{AB3p??3%QNrI?l7B1?Jq%oLp9SP4{Eom`x(?h` z*knwL>ogVVS*A~qW=x1dqIbYdd|{?5U7%Id(mB!KR)kw9^tg#BSP61S*QVIs`)iN% zuguv!0J21JPEh5W$K{EAu}?RExsUN{m;Z#Y(4GpsS1_H7W^@t<{Kn=1>9El5vhioe zPqawMOppwR3zrr1sjTBKIQb4LU*dvFl>h2L)f zdfIIBQpqf+4*DevYIelX)dBn8$7Aqg1^4~WoIgE|_VwS;ivFuK;SU)*XnVOyCNaE} zr0wnQYCy=9ZOwPvSZ*iT(aGOTymtXThCHR>Z9hmpQ>1-mBWhM9J;U5@E~|+-E-6;+ zdPW1o{Y3X$zJD#Z|7Fe&Quu-FTA$}n*U$;L6--qbGKcQV19Hv3 zXa@fmIM|;{XRy6zdCS<9bT$tv4GuEslVV^W4m4pgQVk5zjKfJM%U(5iV+`zlj!7}| z7~k)J9F%O(oBofDE8F>dF(i@V`hJ=PRhDPixri8brlF#@vQCg)u z?4)8BieOja&`g;-$!{Uc6V((-4!c^5)V-pr>GI7|n~bGOi>dSbC;XGpVdlX%)|n2v zE5@LgU}5O_jBnQ_^&!&-ol-wp7_xv-x?QLh7IVzi1gqqznbkR5*@HVD^OCn_(?;QBn(TVpO6 zn-$atjW$y*9xh?oQIS!ikb4TbeE$z`Zygn7|GkYWf)WA(0!oNCk{hG(86}kPmOBQW=W;Va zn5Uh0&yOr<&=n`e%cnk9gGU3JstD(AC%Pe5zDGvRPA4FBwR}px#VzmU!)lv7L{*~k5clg4 zOCLg$edbS7TF2n}H8MiuWB`uH)Vu}K6mubn{N$DobR=)I(=Hx zWZZ%;d^VZaf3S=u=%HKzU+jG}%eOz>ni4 z@ogcHiG2TI%p-6U@xClNO3Wm}k#w(>I|>weQ*GeQ^;`u?9 zS*`=%1`B@x(#R+c0=%}K;~xU>#VugP$ttDK5Uw>GREPWhw!)XQ6^M!!0MLAZ8xtDJ zdH4YdNKG4+y_%7ryz6b}4Qc+;iKEb|u%sLy3r=J2>K*^l^6=T!0~8<5@Q%R)zaJC^ z9&DU$7tg=e=vhe~@6aC=eG8fgzUhb!eHI{)vQhxtuK?td!n997a_3VsFLGc5<}R?5 zZZQI2Qk9vlgJur|&_MHg@d+f$VY&R-uE3`scd-)(s{s=h51?fO=8n-SnXK)O0x_v? zN;hm}6KW-w!r8bfg7X+<(+pSy)BA?j`%@<*d=hc*1=HU>JQ*oSreWXcr+*tgm~(Z*eLIPzjA}(0(c_idjgn(Q3A5kY;Z_*Vv&q{-dO^Q zOzwZn0uilk&IE)m*(^6m0S;Klo}_yulE~RiqnWo!C1xR`z}0pmD_NN`Cp*jn=#;CB z$f0p49xl5tVxmzzS}gZ-zH{f8=W5X{qdoC&GEH{y^x7Lt6grr9K#aM{4f@uETe_Ly zx$gUz%D^gY3|t>hiWi*PJsgxbEN}$e8z+73o*5th=RF}E0i)~gFv&SJ1$-qbrdJqn z`te79{q3}1-dg#Bi49^UkM{S7%}TbBM=!jAq*RN0^fP)h=FM6cO0`~slQ(-)+aHxh4RI>jbQh?mq7jqsLnU%tlM!@iw;eY;*|xmdS3(&gus0l?!70h_Y}MiN z@zml;4AgccRBx4vBFu$F`O`a`^WWu*>9L&$o_elpP%i`!DRhpL*pQs)KM1J zw~9QEe!Iiu+mU+@fr=)d=uZ>QGUa*!l$~T*)mdWO8Q81;w0a(G-^RSE$S0zcQWblH zkpwP?n6#N2+p1y8!i`plvy^fTmlfIn6FWJvug}8R;Y*uXBw^Uv=)}liy63x0Kw;!p zac3IN)?7u%HSaWNyK^57jd=OPA)e2i2PjLEQb}$>53nE0TDsWvq2@bS!f<)qzIDmo z9uIuXa}hQKtit_sFs*88&OhW8IO04UMZbtR*rt6615ND(@4q z@QZd!4pX6-)^k|W(ffgh#`9hz@v;o`@zO#H4)GtjMC0GR1ujSYb)us!kqF{A`w$<+ z3wOx2ung^w;#sf#^)3r38b=iP81;B8&RPrq0w4++zXI;^M}Fh~>?zA()%rky2pRS0 z#X7KP9Fz7A1IV!`?mvU)BhLT{oxT{>*LZ=KuUA{Y;3l;Y>X^k_s{EuSVeon`MwAWg zk&-sO`rb~i6sl72psB?pc)BpR=Y4=imQtP}m*%6b9uMf6L(LasQjyp6Aq4o2Jzr=e z9T>fQ8(29BZE^8vDa~jQH_XrFo^42z@`Q-L?w~o{*gTL2$Lg%E>mdvkw@Rom3^~Oyt6!oxl#|otYkA7TNiPih zcBX&YfZ8e5p|CHp(dnBt6cIvChtE&x`||3NCz?3#}<=MI z-H<%PaQD-fBzJPy@O2?bx(5&7Tw!x+eV~kR@NlrEHS_(fyeO18^<(;VGPiLoo)9~K zV%aav*ibr`yD*21&qKRhT;)^(seY4%YWNzvXB$5wOK>y<4mu+Y8G^92gi}c>BV|mB z{sBbIude}-dvEQBZGx>`_-jYOy_BT&UzybP+$GIS)T~(Nm+5xXwYOv#w30J#J)16= z{T@)`C-Vs7Wm&%0xljOF1bLM1bhHm9CHMu5;O%&O=m8t>*8+q9{(Bifw@^{GPp=s% zdunCN-Y-aZv$rxI7u`9T_qr*IW7R=yStr)Kp2PL?+SrC@xhCC$(#XJS#g=71qAKY1 zX-Or5GIoeJK8=H{GRHwr?9I7ssJt0Nd^=HWB!ahNG9guAv{JNu(+J*WET|e_-i<#6GKOjdG!YFzdkLNY@`kY#vbeEsEHehSch&%uaRdMbSs zb})4?y&QBUcTr93DlDsfU$xjB* za{X~jF~oAjw`UyCiVW3f$(fl#cWcCE$}p+ibLaXMk6?SWh_DL$Z9RR%m_|O#+9TcQ z{%G>>8p6+EuZ@$JKExBF43jf^&5jLv>V6|Rxj{h`CIIAsV+0U`!S6bbRX3}lkc0d# zBFic9vvT-8mY86<6Fp#GZ)DoMu+CpADEKQa0ydWSDwHNIFqh*s3B$KfhC)3I7{iDN z*FHwhE8Y1}opk&UQ^ZBjC7E8D1NDa?!Q4GXhA~Q*hRYWVoH;cv-T4%*fOmi*$<}j4 zfg~dhY!UUiMav$-e;}ahgFmnX$Z(RmDEBWi2>4L(?u$=(sLbSfVj#-&G{?2K*v&Lb zF?K9}VEon{i9=8xr`8sREy6QUiR}V=$c6|T;Gu}aZSG8iia}YMz%e^8qAE1=F1@XM z_jISOQC1)R;qbP@9EW*VlsrEyOl&dA!8|s%%#hSUsc&`rZA92=xl)+;ehCGq=EO)4 z+kFa8#2+{(@NeK;K2LR^7_~TMZf2#CfAm7b4hMmW5drVu7U9BS)xqo#)I8FwxtF_k zzrYy0R)Tnu%vgX#{$g0z8=9_9qM(6Cd?awNJoiHuFev)TLMp41e&O!w#YK{A$}l~2 zf9d}?&ccd{*w9$13^-AZ4z5c*3eaezg@q2wd-p0vZ%iKfKnHH_k^cTmgqO<6wpT2f z0gFL$W~a8>2lk^{DnF0dF6c-fTs7Z?VMKHl3mlL9U7l)tx1)g`pI+yGS;Sjcld!s1 z_YPz~(p(sm-%2?BrURJ}60lIVSR(xwB z|3DkcztDyb_3o$B?PJ@iUV4<_06pqgWpmbi)v~&n*%%FuGb_ZLkBA#+KOaT|cQrAf zIai2$l7Cw4jrQZp0M^rs6a)Bxg7!jDXiE8MayGHnOZ`W-%oHm_oW=qly5r=2@#JQH zo{b{JQ>+CmXP>$DIe47g*3kt-B&A&M0j%bU1Z98+*^9K@CB^c>zadOx*wgWHHEQ4G zdu;|*0&6>8q(%A^ro=A7_@hq=o6e0GNBUJpg7is!q~0CjUNsEe$G>Zh|xMyW1+Z^ES@LYr*Qz^&7WFJFI3-qL^a6l65+_1xP zPIp%Mtb-->`r>Dd1qf9ei^MH-I7((4>*DP(N3nL}FgHEMi&{e*pJceiz5H1!8Y$sl zwGAy$`g`F_}+)(=mFPsr)Z_fRzaUznv8PXh6Wy5V$R`UJ;*!B^5Wp zou$Q0QP|11;RPq55QCGi*w65QU4+^w?(4@6VWd3I+n590=rz$xrd60)T(99io&v`X zzeWh$C8Y)j4ecli6?@DTv9WMO$<}x>y@a{)mGY6bRu2=x17v?J;oaw|?;zBF@r**G zN~j_;`m4%V%#@%8TL+{wv0f%`628vULl3Ck>ZzC?Ljm0&3r8M=`xfziOGVbr$V=20Y(Ygzn{ zIynC!!=Y{R#;hD4vtE{1e>OBQye znDZoI!|1b85&*>fJ4mb@Utig%+4yo|VoJ*5K=CaiO;2pP8LS*s}us?zZ4+F1(L^s zVK{R$utP@sVLjde&V z^lmW$SVsq-hOT8H{#IGFr~Yv{;Rdu#LO^oDAp^gmYI@w&e&;p@aa{_4e7HCY_puPK zc;g21U*FQiOC?$5w2mhb>q;m8rW)gF3|}a;uuaELyj-OS3@BI)$9eZcaRc)qRu7vl z*BkQ;+I+*ObLK{pOgRoy-(oYx%&0}5N(P+L=xfmZ?^2ZUK_um7n_}@=;@Ml2!FZQ1 zhhjcn6jEGEgAFts*Zg9F!-vwUZREyz#H*(alA%snz3;B($sDa_xLkvi@fa)WoJ zgEej0@A;GjXD{?CUp!iqcQ+SK%5-azDHO;hYGD((Sf$X(vIp@KZS;x`Z7CIi0xVqrwop4Ty3+_hcsM-q?mJ zDHImX(FO0*=rjDGnYV}NIJha|vJuEO7d(I^ zx@&Upt&L#;mHYR!Ix9C;LcVhvQi;?aLKT?Tr?fVvk{AwqzW$>q^f^%oOQlQ!U+mZmUA#2P@rUr~;qi zAnORrS;l`7ljz^XG}#DDvijP%b%ZhDFyGH0hEB+>bsYVV765}!+*rGJQ$!U?M{mZA z)kmr@5yj{rncL$t=5|DA+&A+QjUx2XIOm^q4g?hiQqbMH4k8~W4~MdXi#?hY!d3Vp zd>EP<$5X4mkJi4$TSbb-5D16B zk7!e_FU|qpOv>ZEg7%hggfnm7^Z6`ks`?kyX8k)*`{hhi-qgELm&6Br!ateo7b6kW zxvuy0I5Pa0qkjvndG&dgJLTebjma~~PyHJbdf+NxH**XOK1-r`e^&TV_r|)3%T3## zb@Io&edQH_9-%q}FYIF2MI*{~92h_KbH^DFc-u8L_^d9LJMEWdGu#O?Gfo=`0=h)E zSsoV8LsOfEZ=iv`!bgRp_qaga9R44Sxn|J-#B=WW{?hl1H@D+f!WA|?=Xj*37uGR0 z#C{f3PbCoom=k%YvXvi33T(IlYg4%#u5+3EuDp#&?i;MY zkl_GppwuE1P^A&CSj`9Uf^Rx42#~M++dueQF1xo6+GnyCH%8!!DabWFc06Bd^wPD< zGLOA;vDM|_PyLnSak=vfx1Mh+6YdB7z&KUV!4vPbbhL%PeNi&zfrYa`NChxK8eCcH zx%bEe6 zYs>?1IcY)>cq4dG-|o?uf8*iwR^jOv&viCoqaVr{vDAB3nl-igL1Y9?1r0dYQ;uyq zs8xNLsNxX>6f7HdHOKsH&yBGoE%YZ2`P_wbrWuaoR)%ef!G_7Xq-4pN-sqEhu@}&q zv%Y@I(a=IebO$_E3ewLjc8ma5`no5w#X(Z_pa%?>ysA_w%rsoh(tE$@6Rwb~=K_G{ zw+_R9r2+RVPQlnt8uuH$uSv$R=)+FZ=TnL_77&XJ+~g4JwR^bd4rjiX02~wf;FZ%S z*q_K%gQS~wpSR=BQx)QPDhH@z>BibcMYGzgt)F4YPr!xy)T^R>yr6_}UZhDmitqVW z?*xKSn*gD{a>e|cHsBUEx%#q@kzZUNRE9idDtuEACtDM>H!a!PcIztU#KL)?2bMsG zJQe5?+)zi=pN^^0WT+xLfn56EFyYkSbEC`#6{nzv2w_C+!)zlLaw8APL=+j(JuUa< z&TH*!ytSQ|8+Rw?QxLfX6JH(JeMW&T_^Z>`sY=_;q>+2Va|cOxvg}C2UeNH_x@?!Z&{IvXFOb)o zVfL%%^1HXm?Fy;~Vy&kwSQh~xA6@eZ=1PJC3)U)K19}YJQ^a8b;*hUeZ4to}F4oNk zg7)R#Gb~%*^ExI-phz;*@4ji2x1rbCL%Lug>)?#*`F7{dbr+e%=~Q^x55c_5<@csx7eyDIJ%aoB%81jktskWb>9aL43j8Pz z$i-&Z21Ek+$R;Wq$S$hizd=6(5P)sg8D&?u(Fe@FQYBB+xkP7rz-^!An-135-M2C{ z5-h{!6XBCwIPVjrc8_EJeD3P8Vn8P8eb6qJZ&+)5>`=&Io)=#nAB>moKZn{gd7e`& zN74a-hZKI20Hpu(mHu7wcipKC2V@gKEDO-vt4%t|>YS*-=r~>JiwL+vR6|9f@mD<_ zT&{KVr2qwarn~U{nmE+WaY0&dOw!br zJtT6{%E8R*8XeXlr`b2Z&573d^G%uOt-FyfPvV+r%I8t{I*Nfmm`s>^STC$liLZ29 z1A5oXdnz}rhJB(l9$BUtpFBeHT=QVSy_f%i4bKnl8~qu$a>Sye@uHcMJJpO);SQO3 z+m8G9AkV|F6dwp{Eb4gW2EUU+p_Rocf3oqIuU2rgA9 zS;#YZz*>f=t~?9r6=7(B-241*ks~jC9=W?U`ri7;04c_{v8W}39xeh&>7Mx#Us48&-H0)7QaXRR6{0N4aLu8<#m7 z3B~fhxGRZ+71_ z=#F-|7oPjc&UQi+i^vYv#hmZy#81EqR9q#f3ojIbbo0|W3DV?*IRd7cv__O}h z2ZUp>2Z_-|THJwv%6heX0!FYDcYjFT-R5`rjfSGlFyVs+w%;{Aak0K4`QCiIw)iE=F%D_ zgfde*1_6|Kw5fx5;jHI@iQs(scdMAnCjs!w+QcGV1GudSs~|CF+@63!X2T$`nEMgf zzxOvhAFSI$P71Q}ix!_;ueY}ds+Ili;=lTL?hLJ6{>yDn3>miqN%VztUWGX-S`>^#_O#h4`1l>Mh zHcG=67kd-ckhJNCLyzJEGt!Ug#)vfioiX}Ifnd9j=ney0@)ZZZ`?dBO%NuZSR^21^ z1>~Uuaw(sLw5bJP3drw`Gt~-32md`2Po2_0A|G!;1tXbXR^>w5956c)M4F3eSP^AH;=HH|~er;d( zt1k|AVgzi^Or)zW`c%AZVY5knA6Voy#c!SejW44Q9>KF>*yL5!S0Qq) zwLT|#?*7PJuCIUpN3!Z+4@kHYLNo7Ne8EBU7{$VXe?`~ch~r?B|O(C?P;Ih+j8Pbj|IE zeC#fhnl+%p(igq^*p(B9q`el$oVlI@ecW6H{o*IJtD&hIbX zo&E%ctm?++j6I5eG0^f^OsZU2_PaHoXUn!O3{1k+Oh^>*^p-~Pz$duVU8zlGFfh=6 zqblSXn;a$H+C~M8qX0tkE0K@ZxF8aQ5Crm>wt=@pAux)q5Cv2wSSq({BmZXlajZBQ zVR)@pL{?Vt*wmv}hn@#nNO%Fce#A_lEXH z_77-v-;p=G;!Xlk;n1WsLf{x}zo+z(Ua?OUy_pxOrLoy`@gsS+HvaY72d`?lO|e5? zHr#xxH&a?af4n;#S~+RcW=z>Y`Gt|{wO`PFW$79A%9E}x%%9)e+sz*k-tx0}u=ep$ zU3PXhVda>0d2P zYk9s&DaNuHt8~CrJ$vv**zHKB*TreuUi&cXNaBq1a~VN^#Er*ZUBaG^1V7QH;56qm zKHG_jz;oDavwB#6RPfE@QEULs*_#7*E*fl9jT|eIhcC_U==Gi&gsKUZBRmpm6*F=2 zt)i+@LD8VA!Ss9gw6^`)mF5%bJ^EJ%4H%%nlBmS~UP-(8)k^5#P;YLxik;Z2L{)ri zp~jU?*Z6#8YcbpL^`%;&>GN`LuE!oX4&F{$(DxB0ixxwZCh61AucYL5I=wJ)VS(1t z#M@|wk-$D+|4S-CBp=cs$)UW@EG)uKkNSM}rLH=DF|58ol1qNSyZAJx{GvC%;wYqg zjuX90K3{x7uRf=J#gQ;}qpn~s1*%UOC9dniCsMN2>OzEDL_1Di<~4=Qw0Bi(Kjj1k zuo)z`+%9%(27J*yi1YJi0jQsmR48rt!3?dJez5xL``L|r2+CKT*oKzrpG_n|79^*%kIy10sv!GxWMRh(dH!H{0|vs#`DucV!25oDlkg;70vg zCQ8TJ5^QgrzAUPa#TWZrCVpGNxQ&~kSh9)t#673t^>VFNWYa5m)~^)D!@4NFB14LN zt#QNpo#YvKU83saxdI8@PxSs_X-7DldCqb7^qwcNzE4=>W$Gk{{?Jr7Us86qlAdD- zeTjMwyeJthz&eVbw9>fhr81<@NS|Bj{9IogA#vF6!QUE#lG(%NT%S#g)*pM^Kj@~= z{?*5JDaf{X+OsQgMV4A&=LsUodxqN+kqETL8Ge2;E!2M*mD=K3mAOO?4add}LCSi314Qo?WJQsBVO z!O^on6--I^b4{~xD6F`C)5DR}aa_PVdFuFlkm5s*QzkWvv&HLY23qspy$J9CZ+7}R z8Q<1DftD?oc_sQhBKZPzlr%kCD@`-4a?-e}F>NfsBKom!r7I~-nJzl>|z8H#PQJ=Mx=KM-m1vvUUmd~X-W>;^Nj^UH_ zz?)l*Q4j=f8{KPkr`W_kal>Kg60Vm)z*1IoMCt&%R0#d>BDN`;bCiuNL|e zsdndD>(%o%uF{0e%;j#P^#^f5A7yio17qB>+^Zx6AqMv{?k7%IEZ5>uxMR<(#^Crz zISFUm@ug8b?YIER^Q93Z4=7<{dRMEYZ>)Sly(4-!|EpM1*PEYBV+9XzB9hNANSi37 zOlEeSztr!oaJC#HJ9|r)NL8t6^>W8KYeDVFl73?s&~G`&R&xGoxd1aqf!n^ zv}nXvXH`|=2$YT|ddBTMG1%sL-0aaPeA57Okp9p|Ke!ZM3z11X7OeKs{hT1bMl_pNoJCY>8#J6?v_CLb#Q8 zA}xmc;+a+Bng7I_&d0$^tbkD^Dg3ul!DHojGV?QRFEBEcTRJlNY%!R)m^w04M3gIN zoWDOd@>Mob>scSf2Pms(5y7{4q0v&^go^0l6b}_rWUrZ1Wa5YuzWUZ@*gD<2H0Agk z%s5PiZgy6Ex0PNM89$111i8H(b^NdR16-`+_IG0N2Xr#_tmAW?f%NQ(u9I<3E({a=e3{l}sbeMY5zNyQ16 zmg2V3kjwiU0Fl1NdwtZw7bF2&eiLI2^BvOD)wPH9(N}tTTBw%xeQNK9ug#d4{Zidr zBO6yuyxZ`amS)ALwT*c0p!P`YmqrQy6!z#zRv?SG1M1s1LpS~7WybS1X7reG-nm?o zbNR+D_#AH#hvSAXH#@@0oT2dKs|F69U>jK#rpj^DE4^7i9bAywL zuJ&QCJ`j7k3SbG7F^o?NT5dTi`!pqEc(=!qogXYV2YXp+}u&)o2XwOKT*G3XUA6hU_vDs4V^_tCO7>z#Q+s?R2BA223 zEJspF=NZ$?5!U;hk1{g2em}BQzeB>9Sm@WkBr&mz`!#mC`(92*;|RUwbH+#x4*KBq zj!#8rmF|W0?Fw2!fHzsIy+BUsB_WF*W)J?{hbz-Z%Zw>62f}GEeIiYz8P8TwxFpCA zaij{ntY_(8$Zq8OBO55J(y(dV)-oMc`^&Q-;la>k*g~mQ?f!tK&`!hC@woCFU#`Gf z>#Rx@k!it{(v)Vkm@dyxAl&)KTLdl-ek$DjQP=``7;+Mon5gNNqvvtkBh%jUSx6Bz zj|ccAe)@=nghOV5f94|LN7`qgB%`Frar(Pnna~&JH$Y?}7-yz*D3+sJH!Ppi-*`=b zR$h=|mUQIAWNOGf5~#4>-LN%;P956f$a{rHjFb-rasczs$+6reOrIYe_q}L+L0}`f zq{^~}d+4Ib(S-A(Mz zo#rn;-bM$_b#;GH)<5S}70;@(GJbU4756JMAq>+;Ab9t}K=nI`0I8eImEO)w21a!g zU&gUy9p{RorVoXe0bT-_YNnHS-RvoiIqV5@)wwFV^=8y=gUZsQv)x50UfE-KtE&Yp zyge*?rA9w1ytSn8Q}RswXn6Y`rUzBWd#ekK&$oZ*sTw^g+$4GOy7h~r;&v)-I`g|LlY-=~ zJ}V_N+v(2*S?_wb4i5%k(fA+119`R?w!_q$6#G>5re(n=1FRpl>P7*} z$5Fb0nK&p0S-?$5mMOR9-(KT#+*$?zq|B!e>FXJH+q*Zl30o;|`&S^U6&F6ycj7ub z2f|<(&%)lmg_V=-K2h9s>wk*v^GU2a`YBa|NVHbJ7z0&L7g7&d*_&Kfcb`|rR(_yYi1QOhRk~2M#l|q>%m~VWSwjuV3vIDbv{s7kl(Tg#|nYbm5|%!uu9$VmM+2 zH|uS%;L~CDX!AM6cJ`H1yylQ69ftLNxRXcS^3Pvh6^+gg%%B{n-kA;ro7ApuWKZZ3qAS^IuH98BOW+=^P=a>qSi>aoaxHsGurCJFLNa)U=?R`$dQ8&18{J2oe#ncp*P-E9hlfG9J zS1xGYLBk%DM9uyQ)D!!Ov+%@U<=8Y{u47Bn&xAwaR*^v4F|gIqe^D8JF_VSSjo~7_ z+nZxT;EdQZkkY~5RxPgzgxTNRxT8k+i+ZirD1rtbDu{PF>Friy@|3t#f(EkG>_>wi z5x%52`pJxrbVg3} zKM$Mf_mO+65&w?oE>5R?qNw(I(nWczoY_bCBQxm=j*bURPx5|JGjrZ}h0pEcuWjW9 z?@>$N|0F3Mcyt)T?W6x365Q#jqRR5_iA%32EUfV+fJB4bXso0Q3w?-6iJKoj32c2Z zSs)t9nb1WRUVn4$IFQu+$;XHEKWix3n6i~HdmAX0UIiKq$UYOE_c0NIR>CoT3>iu7 zY8fchaZqO=xOn_PdRAXyX{pRwg~i^w|4tpe-~UT6)?B zF_P5m?24T-N}*xa`sd8b*5{d>zFz411vcaS<68hAjGV#WmzC$*91axszH?iN0%2ve z8fw`Q!ENi-KV|}i?bCJ@G~HRbCe9ai$fBw{hzJSf-#_&uls_rA>VXj@brQ$bH{MJa zijHK%1-|+Mk7qB!x!&!+j~2PPNn+SS<4H(Wcz7SmTGeXE!dlts_+eb+m7)R}2(bP8 zFW!NkEURwzKb*0PjbQKZ3c~xz$~x2Y^J~L#w~q#Rs~PEgstdRvGta6ay4D0OerTYj zyO5J_#|QG5yDb-}tt$|4q2pSWUs^(!U&Fdzo+b0(*I5$siV)eL6cR?ty&{#lP^^qq zedf0aC6f$VVx>^G+1C0zTco*hX9PU$G;Wl?YrC@p%VR&Bb_z%(=yq}}*V7Jgs&uSd ze_IMoII27o0}0wwlw{@&|XXO3+jH+8{#BAO5=r!2)fjmFK z0!yz8H>a;V8W_`#(gB~%j_TW~cYT5C+NHE>Zxzg7=Z=MGUp(wo%$6`XtMK^ZVZL!s zBb;VTYxRy3k%serb_Hquj-ul4n$AhO^OUz0f@oqqliJVr((2f_3#5Uej?WlF%I}Ut z)Cu%5lr4u#qKq1in4BdcGMAgw6vF^h=>UhBM~S@lI{JF^)8Z4o2|iY#qC$*^`ZML@ zqbG?Yf!2&f3-qlJ4Q0BfE5PQ8?E!=A&3D~TLBj089a{P$3(RmYGi~~*Dg2KFZCs`c z*6sH6Cjfitf6o?^s5V;s$;_RF%TXS8^Y#bj?-NLV|0OBG?XFB3j19srww1KES*7{hlAM(A6raU% z6=JT;T2D?HlH*A`e&q}WoaG^)`Fi)?1jAylm6^=SN~5Jv5{JbeeAt|zR>8-fGC9^T zEwza?%*&|Cr?hz$%A6+F_L2!%+93$ubSpSS__*fB_?%tsl-0vY%h8qecIz2f;4=?& zlHHh|P1LJgZv5cLfg%CiBO5?Q@W=|+!EC672rmMNK8SodDG3KX+xzZ2^&$6HJ^$y_ zsy{< zG6uHutX9rF=T^Sj;7x9~`0}5-@qd!RX1fM$6tMp_Jlt zde8exIji_RcfOF<+dEsZ9J*9b5?wJDZhnC!m{K-F(CcvvtR$xCCx}o0#kp{_J+^yB zZ*4m_=9;pwT65>;=zK(sz}(HN$1p;~TsBO34cm=+#Osx1=a2ijjc@Ot#3v>{9VP@w zwRSHTuz;?CY2H{GQRUZS}IxbKz#g zBaV4Th1IVwnzxL znmSQBFF;#}0K|wAP%l+iN#~M;bEOVuDO*nBa7a`qzK>!u_mxjli{yEJGevmrIY+V8 z>Sy1glKK7+nX$&`T~v)A5Yit2F}|kR5g=C0{UdMqfUjIpeIG&YA3^@UY~!MLd=f}6 zN*AG*=sPVCqUJB!EXw=YzVF{S1<8y}iyZ#s>>e~Ey{(>c&Xe2ZG&CJ)#UH^&Svswf z`uLeUdtD##W#o`D(KxfhY0Ri)3}tSU%#ab^zN&3uu3@muSRW#&ef)2dG2ANne@!ym z%~xGCEVEBjrdBDP?0IAjZmENBIDN;0otX8Gw1IAXgoU&VMCdInNtHM8J(GB0GJOP0i4xLK zkd3Z9FdzOYYNAidai_BCntKeYR3gdQvD`P}GI#RIH;qV=qIW7bEQj znD^g;?ysBBjZbnn@2M)`_Q6RrE?_0Sj53k1@P322hx<3aJrSB7%(}21b22$wuy0;p z;2n=6RSD@HVegk2y45)HHJ8GkM2r7A*%Zx~IAxpNmkIb(QCq3Kp36C6sJ)(ZW9pzN z;Z!FqU&NwvPw1N!UafmnnG0?8tFCNW(yvDUC~Lbl z{lu3M9#klvcRSQO+f95SD4;dYxg{0MW1TYaHOq__@PM$}J`}J}*`^Bl6~%Tsk8dQ> z7}LR=K_cv}kj4k{mwU;;aa?}>FL&Ubi;PW18EfWdiqq?5s0`mXkxTlBqY#zFC(qjX z?LO}Jz#F@IurZuqe?}>0(>9*qF`PPX`jD?8x+i@>Z<-&5_(@SiuH zUj6v;i+ZM}pmLzC`>(zQCL7X-`Zm1;SV| zfGQ>-x~)lNpZOQc(N?^bMpym?Jf~bIcG6SgXc9Imn8ON>Q_^>rPT-t-WiibqLfAr6 zKO=G`TIgY`KdMXLw!@5GZbvM}q34a~qv)L{raPu#6KqBaeRu}!WFv7Gu=^L!`BI`U zMiXFGB>oRh?qoa;P{of33~u5(&MoBFc5ifn!*-`g6`QqxJ5DD3<0c3WD*BYOl?F&N z>2xv{*r+pSCP~6aBH%GC>yhzM0S;Owt{dQRft&6Y5h6FO8k9oylLqL#+uD%6t9(1; zbjVlBWd-*SBa?kD4ziIHOL_@1TB@J0UjF!}AEfrXAJiMVYwSJ=MJ8n3u!b#E+4Y)< zW|p!l*amG2tcxr{Z18!W)J3lu$-2MmSH!xSOB`EhkWWwX`Gxj=7IPA`vCM^xTZD39 zKmBMh3{*(Wjmv|@C{1(GVp+(fPU9qHnxlzeO)ATB?M^1giBYs)Rg|-iSD_&SNEh@HSc$|&gBh7gp z_O>RO#HP_@Xr`dTAH;^)@xMZBzqIc34Z6<1 zWqS>G%3ou$ouCHD3H4l)Fqa$5cxjZc+TOT?nAYZy7-T55WMIj*JPY1>c|LSSc+THe z&KHuCjDc4o)C$3ebGPg|+VSY90+kI}+`ZrB&D&|w)k^yd!AR@gp2b#E9zduJRb?J5 zsHXv^Y}Qk&*s-VNqH7~`LEg@C>eSZ6(Rul@LCNzda;)=aaOnE?p1`X>fpnJ%v zT+H?yRR9%S9VGz0*1F3&Ql$D5)lW@K{8=|IK0Pml-S%T&%Xw0<>rKj-=f!J?LEf2Dj(}|+DW=`{<1!* z@%HT;^&K_*1f55lO%hkSK932nM%RDA#ev>Qp>l!uoeTI`pp6wej7ju};wrC}PmO>%J769M#?tcAlcwK%*{73n@G8zB9 zVD<_@9)I%-KU3zg99!?)tXP($Zmno|=SWlSBdP2I z^L@%MgIT8-M^O{Q4MNkl9u9^>KfJVtkc;ZKkYeXOw=P6$UUBc8$?aQo)7@RZ&6hSY zp$xhR6;wQZc0^zxG}lm!o2xtX%8WPn>L~D(dPWKuBK+Zw(5lSiH_9feWGcndX8nW6 zq@w=?BD?5BU&+Fzc!BJ%P0oKQ^VO|<-^>U-yTx_GC#Cz%LPeofU5WP3%!yS;pGzY5 zGsxbIlY2Ef1;{hfgpWiSpq@)fSq^!6qaHx#0aa1)sPuhPBsUPfmJ|UBpj+F&EBs=+ z^|DFl8ZrLyk-n6`Uh`{KPyWB~vG99KHRGe{xOjz-9*{Lv62;Q9Fs43L|C!gvd`uj{ zSgZ6QRN4OL4YQ{~J~?VO>v5LSbG3I*FRB0Gcx-{93^)SdKXdVV-cgPW%6G{o<&+|Q z(s4Y6rVVH)6e{=FUj$XlL!UvOX1oryzTx~}qUh#Wb)D$9Vi#FV#dn0@gKdYc(kKJc zg#%$AJ8-ml`}qA0L5lw-1Tlk88et<-^qjUy6t;DmJF2#wxG&$o(d?h;92JtZ?<5eP z$nb-0Jkg0W(q6OxyC{%1H5h(FHdGE-WGz^gdh)VUp5Y<2`{hd`t zd$*X}v7bL*X)ryu#W=B!A_bTE*a|IC+s-(+U)oO{msm6?lH z;097qxT~QX<^rFqj_uJBBz+vP9T200e2riMkry-Yh)>PaJgYKTeFh?RaXaxfAnmLi zc?P)w&UT{&&YxLdukE^Im)S&e9q2eZ(8)ivlHv(hrYYwW+7(^)u2x3C{cUrj6#Qlb zO-xLrA(QQrHN?4<2!)-+?WMWt1;&MXX0E+P6_+o=af|u!lBNC#nfWZ_xkZr62{!1F z-HV~DZO>M4nwwZbnAH!XH2{wZUy6}w?lyR6BcO8*Nh9MT2$V#+c!pcjN(q;WtRJ4v z%}i_F>Te#Ycu|p6Nat((^by)i`Ky@DyDz4_v?_DXDt^r8y1hIGBySx1 zTxr{329DMIk;T`A5FAb@oX^P4<8i_nedEcwVnLr!^UwvyiGf?RZUYUs|E z)U>-Ih4TNKW{Ayu;`mliP1inwB^w5NyOxdU1fWhsJ3y5Fi zV;6VqIn^XN%ZR>T4q87O`nG34fZc3u-)%zHWS~jnfd29&>Jnan*yee^ zBS#c`wrv8sUcYe`W^ob(!5&N@AZ#cAP0#xH5(l^Ja%$`(j%EeD+ucD~@hNEcN6$bW zki{|(i_3&4hf!wIEPG(qEXJ|(7NBE)?yMEGflqVDiHNdZ-o*O~PW1L=9;38}Ry#h# zmhjv$JM#%SPNv$}XWpt+YB=`6yGpOf;b72QYG!uo+vP(0jS=wbx}J_<)H0ROgCYMb zC%+JIpl2mar+5Wt`>YCWTAeC<_%^vv>E@Ttr|Be+AO4vd#I0UU2s-_w2$?--=^gfz;X zt}Sz^pD8G;S~FN~_i9kGBt#i!q91jnft2m~+Ni1eM&+-YFV$ft<*qtIw~DIhiN!8O zl*TszQ<(M#rZBV=*hCL6h{J>5|f;j+3~&R5L82Ia81*)2o(4VKDjQ zmcJP8MShj#TAQ(^x_=>cQjY5dZ3ZLFcSVZ`&|-eHZL-eu7G+{uedHVi`}|3LvUS=C zx;LyXx&LdQ8H-0aY|agpHjwX_ht5*hud7?yG}-p0Ir8loY`%M*qn8oJUVG#`adWi& zoCV6!BW-^~J<1}Cz^(xa-;d}|9E#lHf$!bmsHirf(E^=p@Vq?vO85YEXNFu$MEO8R zr?RKg&Rg-dC}i#UO*I4PjSbGPU5DbHtWk%0wcTapp{Ay01+w`Koc$StwBANle3wxnQA#vwl5^>3v6tc+Z{`9z#3ytP z+xYVu z;yhYaZj%aURyhl)5-ka)hD=ue_C_*jry71U?pq3(^~6}t2drwtC-cN{9wmV7 z+W~7HePh8%unMNTM_e%yN-CGa;k*KA07v-)ZS4;@Y6}0BC*X6!<52^O#r~qw`})NK zRaaOc&B7j3ggzhY*jVihWt)!UotH|8@2x`bBWpOU^67lrc}Le}xPOoJAyTYsX)AwL z6TTbOCUD@`uv;{h{hB~mX+^XD#o}<9QZF(Z5m83TJcMO}p3m_>4+oRb<&`?xsLZi9 zhj->`&#!;jb_M}$bzw9lzq+79JDa5?!R;!>)^dwvL z)lihl>>mim^gxCS`yiDTz$GjoW>sD;Vdx6POx0f`9%b!!=-2*9hrY^nRwOPCq5}=h zzn{{qcg5%4oiQ@Xnpzjr&Dyd`6`pbh_X~hn9axGahKan?DfN_dm_rmDV(VX&1^dib z!*$G)raPd?nPVFAUJH$x>N90En#<2=hB_?7L+=$IGt1Xeir09Gd&0GWO&z%I=r_!W zTRRJUI=d_W>$t&m;hJPp7CJ3td-)ESGwSr;^a_xL*0|Uc11){g#Szp;kb>ND6!C-9 zm@y6sh!lhYPKZ#=eo`?bJacgKMPbteixdl41m8?W^siRq5nti0DoIwYIoXn{8`zB= zlguM@bf$7DoVK4Tiqnb1rh25+#+TVkQ5?r}l~w!}aN^CWP$?>@?7|8W)u@`0X%@hT z;)2y^ioB;%jl~mY$NQYNZjQ+TSEX6~x<1OPvf(;0??EOB=toMwRxJAyZVE$E*nwxVXturl0s(3QZ zI2Eo)ru3eV*&eRn3EunwIb2;N_etw>PT=Qja&8_-MIYjW(a-N1bSCdt6R?B&iG_oF zbjBEHkV~X5j1`;2$2+hxaa&<=6Kzytv|K|%NUoe>N`@x^LbCWCAz5rIEd)>Gt=#)UfIy4o=We|=&z*cUQdQ?h*Z_6FEoE{BB;1sf@Zvy7+@gpMUrFH; z3K77a>{|%Mh4#9hNYi9IqDb+>kSD9&_Jo6Woo_WGd2amth3Stox63$R%fHHxPMH?6 z<)1>6^`|j0OYx$*4ShXh41LZF3l=&!3Y*H_cn*u*jElBhlWq95fX-K@;8F8J(tdnP zX#NO+0xRF~=|Xk=&N_b|W~k5VvGnRC4bkyr7yS*v{G$d)WV*WPW)|F>B_$R#D|u=g z}M5(WZs0t7T@-Ck)Cc(3S)1QJkKp1QtfM% zp#$^OPZDQ3(v}c)ljZ0PLglut2%NAF>MMOnOip|_-=Zxb+vF3!BipJPq0n|uu;`&w zoiww)V0E=0xEY3@fd$K|ABP)kT_o7 zPk9tDsvSMD!@7h4oMu(Br&GR%==B6HsT7-h zXA^`W1{vlXtnuYaD{IEFt2*)yy4C)&|%hl!QIwud`jvFf9YNMmY9r3x3@ z13?25B5`-B_wBtJg@A@@QRJN}kHkP6N7H zZ%V)SnJOwmOT^^6JInt(m}O?O7|rbL!tEb=VB9e1HR;H~$}l92yQ{TFs~(LzSzlL> zVZznGQ;26;n9-M664h_d0i(pBN1tKS*P-BL zG0VJQo_LxZm_TFX%IU6VATKJG7hpIgOe%{R@IoEM41leN6DYgXW86wJGJPCvU;QOT zQmU&BE~DlH@fAo8BqKJTYjoq((9{>ng6@Y=MRHt%da`ONoQ-|9Tz>y@pp&2C!2tCM zZMQ+ymA150Y!L!-fXIDNmgszzf^f&Md8C@qheyQTy~}tfJA+gc_c4Tn?$e3W07Jx| z{0>97_Zs#;SBWVv2hUYVIw%X`xpeMb9k8Mz_TA9Z%HQRdw4o#FJ$GWl_ZY$r1RS&k zq1?CWGEU3fTKQZT%Dos5I%wMGBKY0YR^jySJze85z6w4GS`!LH-9`yuEMwzE3X-; zWuT=2Gr$2fFvsIL$2VP)VrwOR!!%c=N`V?Suf5JsV-ijCVf`{_e8f==u=zbRJ;@`! z!2aI}9{6Kf^;G7hnx(&pA2)S>58A)7IgzTSzEgSm&`mVF%FfyiC&HQutb}&qv@|1U zbUjt?ah_jQddgKmFzq$4@}Ak_nzojfV(NP=H?gG7K|)3NPb`M{-J?^bz0sIBYg->c zIedBa!Ti5_0Wjkv^P+~`LOd?)m+}+9YWMf0oJ4!x@BVUEftbS8EJ5$OXJ1oP{{zt_ zG4s-+=}27B>%#(SguFL=DA^HCw2II8O$w426x#79W$Ig$t6^LDjCit4S&Y75sJe`d-+>q|< zEJYgE0lPd6qt>%oEEXFRGJd5PjfkS7SP&=FS5{hX$+3(FI5;rwUeZ*td)Ow`CfVkP zto>+F(eE$IGS2dzb>R%*EM(y_eQ5s7EZc;~0s*m<>i@WN%W$XHYo{b=q3`bjbu)!;N zxkj~_GljXyv^#AL{JFWyCCry?X3BWha{^z>8vCXh7V*m()6{$zeH}OM(``65$#}VA z=<(b3iNcUI=Emk$_1#7P$yqDHP5;J3sIC)D{W@6gxV|JaOIa-Cd^V0?%*rq!1D`jS zGO0~ZP4(Nv(FR?N=WMp=3{9Qs&QWZrefW?UCBH*=(IPG@-fZxlrq)&ZyOcQxsPW?h zp0~;Pf1#ceq+T3HZ4a=y9|hlZDM&y5=Vr@_Leu19-ELr*XaS`VIh1^-!JfZ`kNz&EL-Kg8qr_pmM^WMdmJPg)@TG@2h zvi3izE2E&`&I=YehB0llB(ahHW=oTTGIUMf>wzfPmkgXZfBXT`6$z35>4uBCw`HDB zG!+HTh%1V6V*yL6@*;?e|3Mvn5gP}GC|EtUH^J9-*?|4kow>rFHSCMVklXH-Fwf91Ab)S Oqp5n~e1)=g*uMemN7MWO literal 0 HcmV?d00001 diff --git a/docs/images/env_maze_overview.png b/docs/images/env_maze_overview.png new file mode 100644 index 0000000000000000000000000000000000000000..9f0e80d81308269dcda75eac20d06a2c7005b8af GIT binary patch literal 135681 zcmeFZcUTkM+CHkFfFcA$0i}eZf>I(X0i=mkMMNxg2oMm30MY|VK&01zND(mzh#);w z>AfqxgkD0gA%u`n!r}eSZ}0cr`&{4o|6JGkV=|MOm8>;u-Os(A^*r}X=(DGqEa$JD zKXvL9%Ofqd=ci7c)jD|isgUhVNF+|`)aydFE{j>ZFr4Vc2t&?C1{ZH)7rsCCu+CH) zee}rXqQQ0jH0~c6xJQrGzuZ#4$QR>c>eBgLc=V}w(kHEl0$LBxsL(Y(%=Nb^ETF%5 z=A6a*goCxu$6t0P<(52rw+?D$oEE>{9U7QiN=C#VEADyJ9qf)wKTjyBWv6T`#8#nRcVP3=Ft7SAeM>uzP{8hnS<_2Wy;pnvX6t^R1#Pv*Y3Llj*Ek>%X;WrfWD;W8cfsRcDNp zbO)O58}8}S#aWB|t7K?BgNnrfL&x2LCx6%ful9a3p#dxLU9tHo%V78N_>H4;kN#N; z^~}nCfghuG2qddfAwAj2WLCm`#ItKMM{J3gckIYw4v7)@QVEO~(JJp5upWE-9~H9bL)e zNS7CTh#UT~l@LSk;p8RDf41qQ0}3VCM0M?-PE|>|i%6J_Y?LEL7V1V&m8W)G5%tuj zjJoNg%r&~K3qpVYJ(-8@_R6LT22 ziJ$E6cLC9q<9lvGUhV&+!C&mX+3U;bajV)@HienLpnwvNWU87feW#9gGk_h|n;icD z%wM=!s59Nv2d&a#UFs;CA6mU^s_G`tQ)8&QgAr{YXxhppOT50eyioaNS#hU_=_p2W zoV;1O35fTpLHy#~D>p;lrSB**Zl{L~Dw{qM)(MU;pnq`P_60EI=$T@! zWL40)oUVOPHqr0Fr<*#q_ubqW1E37`#Xpz&8%QU%KJ!i8g347T6DL0e(h-l6bk~hT zJerJk6x)k}KxlEBpDwdKUGvG~wRa{Br6IKCj`2mlj_B?m#P{`E@!-t7k`DiW+23EZ z!F$HVZ}9YjfK5%s{O+hrc*n@Jk@%6D*N2_oMI$1lu7%e`)rB>ZZQRCT+TMWrjr-Zi z3>x44&sM(3zJ;Ub375Z^R2uAO8f_WAts6XQ`F}FfFQ(+)BhTwpSMn(WtO9g|?vg6- znG1g8v`p=q)H~s_=?yg1sbiv`w(p2>q2|Yyk8Zpo=?(Rk zB7os=oEC8$-zoERM#5dWwz2CUcY=GzgcCV0V@l)^pVjRAGYc8tnd%vi- z(2|n-mP|6(y-uG}X9S1bD*D>xn~5YG$8W~DRBL;5Nlm}5tU1&g*qMtv=;zuPAsbM< z{>icD{Y(WWWhal0E0x1a4%*`QMIlPw-w!go4B~Zr3vNZ5>XiZF9ror#xgBnGW#pu) zc4h9xk^;LBU2SUAS0$f^O2)r*F z;~<3O_SN$E>F18p$Icb0hhhthI}#js5JZib{2=o+kEi!cHi-#qBBwfH|A{@Y0^L#^ zqOV4otJ1V4lfPrCWuWMcG3N^-0kc(~3IetBu#qp=v3zg?P5Gr-%w`XbxGZmF5UUDkd zl^-Tl?YVL7ZHXQ6Eb#9fWi-q)o&f3VckRhK(L;L_v_t0~laK|`;137dpXapsbLB_D zdm!w{+%h1XFjw@A+8)-w6~7m|nK81*AEnfp(cAYd!|I=-#lz_r+$pr^Le0$7IxZ-p zQ01Y$hWoI+ePYXGo#ei+A%=5d7QOdr19ZvcSbfw%v_o{UrRd?XW3L z1a0$p(`+A7S)+WZau78=K@TxO9(ngE7vqlXf9UYv_&4zD86FDMJumXRk4Sl$P^Bzd z&BbzzU-;pIkUP$_kuQd!c}80nWdZ*^$x;Og*iAq4E48jWP zj5*EFQilrv@Tyn>9!N$ze(RnRc{L@ouW`)ZzombtwvvQj(CAZA-#57RRpFoL{EPe- zKRocFE))Bww|wJhOJNO&uChJGhY6irs|l4oF20o{`$|ueJ1*zg7dyYe zwVqHZ7B;10wR_Nppk<1{&Guk&zVq?fsVJFM(YX{-1pxZsiE$@e4_GC`}rp~{M&8)|4;rK#ecc;xh7}g zD|IF7>P$BC&C1zdTl__QzU+pp`CCerBwMJzA#G^$X!3dYP3ZRHn!gA4zm^hm5;Px#LEZ>|BVuqR7TrY0fwns=0Tx;Rz znxl4yypj|x9OA}g>jqmfgS^+oI=W`GLXg`no7-)`tpY?cC@gFP{b`J1YF+pH zN*%KEX67M2_UZkb&mYlL!X8v$?lBjbYQ|sDeDptPeP0joA^2}b%1jx4mSHU|e24dH z7YxoFXD*18Mvt)i<%X9w?OPD%y*i@N(KxsSWoFQ%q|AWlpeVYeH__8 z*h%INZf&4NY&{_5$w3t%%!__GcEM|%-hW=*Ug+8W%Z~j2JC5J634R0Uv!;Ez6L(@D zi5w^9c3(tb;K~0+th};Jg%WV8b5JX5_kUm$unzDv-zCvGc!~qU~brrlBQ!D`7xaxn{&*YCQ!$7aBSp`D}v1uH!FpWDRX@>(c~I#(4^KnW7ng zDI5}-PCO31h`0*$2WxV-9DW z#jD@un2H1V+#DqN(PVcBeu;udZ%s?}NU4njX@p?Z!Jo5Et4a z1Q(yd0axZHGsQ2Zip{l+lq?HReVIO7+*AebKdXCii~j(zS&><_H-#`T6pywXXk`LS zi~Ee79cKnr7eWMHK}m*ev{i$;O5KQN82$$TZAu^7=lkLj9Y?s%;jhRL^;^hsPr0Hu z3}~R}r6@w75#A6QZpOJ!5Uie!xywDKE(6jS`ylN6aOO!0a<6LoD+Q}IFP=DRdP!KJ zMF(M(i?!MQoGdZ5lH4BxfXEwBrJS@4soBwP{(PaRC{ zd9?8(*3YAV8JX722Jz^_m;HoMc3k&50H$j+k@D2av#M6v&}*9e5H&{%*8M{St*G?E z)?7Gf%GNRkaL%!MyET;ir<9$ws0K;ta2Bw&GfQ79zCia=T^Rk}&fLE+YoRJoPZw=A zW2G1BT6NpjtN(I=V}a|+X{+YgClBxglJHPK&)5rjo^&`dqs(2NQeos=YKy_N$6@))k}!pfz+Yk z?@;GA82%#c8)~A$QmQIHjMOC*>NCXq>#DLU9F3K0Ww@{EDz~d6x$J#!Sl3O0d%SkY zw(-k-^dTH;GQ};POS3~axRgqDF0s8P-Mr6!u0V^`w>>C7JkK;}uDOSsk0f+8%=Jlr zjKlWGoVO690#e6RVuw1?_!`3}%JloF;4Qz&VvJG2yU%d1@9}LO8~AEEW;yivfp|?H zMIO^L@RuV>wK?Rmoe+?>LM7km47&+Ges> zX0wgIA{bX67XFk^Opf?V+NI1miiO``(TrI^L&x+WoYRbJ-e=gzc6VzsWIrOwYsQZ; zsWi?V3%>l&cl#B6t>p8)gw;=DHn5T@jz(_0vv=vwblV7rsp<+U<5`H*Y>jwCW>N@; z5}r+Yio414S`)@)!Coo?iVj-jxIKeXt5y~jva(3_%i&t`yOZ(^(4%w!D~GnRmV=i8 z?<^GV9($p5X%?-eH$K+w<~eOlcLXTv2AB0H?wKB!vH^65&Teb$fFSgF6$bLSnIi?;WM_?nr6C6AJ6np!hdYWGcS)QDZw#Xx!o@5>df1Nd z@J{zr#od>nfC>VkW0!O?ktR{HqatD%QBnLydOc#xES6D7Q3Jvm&?cyNFQn4C+G~$- znL~JFez@kM3li?t&M+d@TBTWn7t?Q7yR>+n%p_!p^v)dN z@M%{Q$WU*o>TwDzdp}Ti1rW*6ZQsanq*s@&?Ol=zSvh`F+s3dvuo`)xZ$!G$?;USAySwqFOh;k)hfwr z=l>pxIPH?xF?V^WHBrvbzJEB+#C}I?$&?-=4vC%EUQR~=$IzQ0r-391DVI(5y{pj) zjL3Njp7un8rwZF@gL{H@=&N-{7}bLtd`t2W&mWmK>~l8-BE`xy$RTEnhe>n@W_ZShpM)DD}QiCRKCW zE4o0Dh93;d9kz4xUAAxMpj?&ods;K0Yh31+gne?IqTZU6FHNjk7ApladO3J%lWAyH zeEmnH3BPj_8fj#0 z+ny^bF7)*+@54hipUjzlN>o#{x=qYyq`li^0-8Qaw5FyH3d7zxFQG2*_f;JBJ-RTZ z9s@RI`ej)7YmQy!lqp+is^mS=b^}o6lD{WMII9<%7?(~DgVZ|}{0-qs!KFox>JS^4 z-V6d$qVKNai{la+k$cxrbFf)eEP4L+L6szT=HO9Y`PAw#7}aLlfUM^Zu}e^YxemvF z0uL&MgK>lq*+ld&%N!ARQq0|iUd&$RJM-8@=GJP>W|fUek8y|DYMFZ}w+(KCG?>3P zx9~pTuYNa~PH&@h8pNrUKK2bnX$q|w0nSG~kvW)}cqH$Zm6A36^Jd}v{aI?NVyPdd zJmdk7U3lPjmTs2rz78|myv|u7jhtqG!&lZ>QJ51Qg2%S;ui*)W-&kZz@`VS1(wz_e zJ?Z^f-p!-q-{hv6~w>rc-pJtO||O=y6_vQut`zv zJH7s3;FABZ_Q z!X1XE?a&>5`%(EKP*)pXi?*&;;QBbPCgIjoJ1nH%)b`y@F+59g|FSNO50P?RnZmV*v_TKw_ZnIpFU)B!$mxkI=At ztK;=nB8~uExFQbR zBJIsh;QQ?XGSyd|;4}3Kx)IB&Gold1^uu7q*7xg;nSfOQ$6H2Pgf@?^qLYxnL655G z2nEsEYBrLGEH*fX0u)^e6TK?fgliqX_C{grOB*j9?$af#nWm)sh}S06@y zz~*5*NEGI^?kXPm=(Qg{$I{|g9oX{BEF!Y%vL>_rC@{?RDvuFY)tYkS>DqYV>b`_>XMs)Tb>TLz7_05|N39s->e{KLeicX1 zAx@Rm_rf{Q`q0ZRJH8f!^5P}c`$Wrecw6b#TlY5wd955K&^c}nf3A^|YW?IBXXEm6 zw65C6)R9IyFEv#-O9xi|Mz{V2DC`228w!O<;*76h4c#8C#G*fKm@H6&ir#ufW3D!4 zXORNTzl(h)+ojh~pkp~YHNbEYZ}vY;400vNnW_a%Lp7-|pyhRHHMx0p(%zyjF8B~I z+n3}S5J;Y?{*WSNm~6VgVEb6I`qPZIO;rI*0Y9!RU6a`}pIIoTn}cR2Lb6O*J$=@M`WWsx!XA zEGE7_N-@;Jw?(cVv=8lPjr-kX*7PNc_!bpqwI9|mQmv&rTeT3J=X;co596qj6eH5o zxOTfBY&1|9-abcsAOk7h8bu@_+Gx~Wc50RxSCU7 zz9NP!qR;|CNJ~0R&UHK9&pG7|Wf zdp&sOC0xuLT%z8Zb`w_?V*2Kw1?1zLQSi2FPO7w>F{_cz%!d>jEFc@ytKMBPH_MGN z6IE*+z!{jizYjc;h&(JBfd8my_N{-8SkyDFM#W0B2c$3ZeUU*;N=TA($515|e724eOMerWg zoC23CBNrN_G^qk0`mIu6YsQ+lp}!VY8fz~L`veGwzjc>1=k4FKo}1-K&Qr8*9A9-d z-7}iZA0F|lF@TjA_@)aZ${@YFRxeS8p)O829J! zmX4BUFgJmR^X$g7&HtU_|G%{Xu4)e;2-S26Jjt-#>tUHR2(@`?ipP0G({qDS`H`FTBO_7!; zLTaUaidv!{edhB~bHfIL(Cz7J8Hi=6+9S=pRW)cUm0Sv&9^<`Fo{LLKEz@`0c*4_G zvOm0q!(T1HZhV)yl>P_efJ5(AZE|WEm^=eh{a4^mG^6_DdZjul6W1H0P)ZuM{j2+O z^3l*~h-g#;%>4d=nC4+rhox_;Evg{X&;(mz-Dw*UAhz%6&p~dKiq>J}+bJ zYSo=8y1liKMZS3HZE42?)d%gn zrM5g(ZB6{jPob!H7j$c}4RTv&OO^El`60pa{64eW^s>^kZL5~ChvE&Y=shc9w>GTd zr1)}Rgll`(SrqudeKRC@>;6+>&h&uM!7Oj}23q5Pc)KGIE~}xde5M)u=PY75#7&%U z6t5WpU&>^XQ%(*8u3QxMt}f75AbIKDkTvG_!P))pP2hK_W#~q;ve(@{t-574vOHQF zFJY(YQ`lheC)6ZB`KT}kzTSs7%LrpM59oO&X=k$5AW~j=qq@?}A!7bXM7*_Ai&{ep+q0uCCk^_&}T|IMl1&$4u-Ci0az52Y(EmQfo+$L*K1$ z2Pqn)C8;dI^AWp;G?Z&Qh)q)nY)Zrhxitth3eK6-sgnB~sL=|Ymn)DFp^PXr%mg8B z23PgDmKkZ=)M?|Xhd_EsJ;6JPnlGNoA`AO;jYK<)ws<%|JU(k1dv$7BtGbNZZwT|a zf4Q%ZG*m7%-N=iAhvV>V;VRCLH&@sI#9`j3h%dcaiZ0+cT8tH$>4u6%PQMOClw-P| z@#C%(`6U~pkvKykzR?FG1Adyh;WseZ2acnTVmx(cfB%tCVA|RVQ90rqz50=(>sd;B zQ1JDQSv1z~sWrVkCDd)+A7xEXdzRa-j^P`PKIxa{xt903Emt~b%Wo%COFPZb^E{i^ zB%0r(Y^_)=rEtkB3;jsDtGA>c1a!ch>%ff!_;!T$4W#NcCn$ELD=g3-#Ye46cga8RsW{53U|o0SA3ID9@BAl5!VR#;{o9EW+oB{?!@&7?I!}1}~TGm_JtZ^2DZX zg>y*5#G*alQjj%TnFin6E$~I1)pfDz3vA&&@KJ?}Z@%4TKM+sb#@`ViYX>7ZGjIWX z8@1b!P9)_!>AasIZ)|sa+Z=U@ZX%Wkz3uE#15tsc%0~mrW>h^9_-smW$tY%<)QvQm%WqF!gk) zSN6Y9Xhz^8Wu6|s<4_7|ldSfWaPAKlvYO|bqW0Y)zQ;k{HJqLNY>TOARMkaEC?4*l z>!ln_EZwmC5cp)Syv|BD#mXOe>Ei2bfE%_$!?CD#;4N+)tE3KbcFk=WzwcN$h*n8$6B4C96A@Y`3Y(ae!9IuRNp%!- z4@D_TDee5eJDGAcnvxeGL9t+FB?)d^7KT2jFo}_Z-(;VCp8Sj?OVIL9c;$-L3S+f` z_}aZ!IyaQFKg{ZKo}0(xV$prv58-9D`k&``!i2+O6+-B(K4zce!5P>Nx^!QGH$S>y zo=fR{ASi<7ey0gUxj8j8C3_G)|6c!Y;}FB>&CjXt6pxp3Zvb@CkD<<3r5r9prmZs(K1g9LmlQN8Ub(({23sQ(bFk;#-Dc^ z-xBIXAdi*JCFwIPM!af`;bN3}#uF_P4B(t$n?1)s`KhduDRG8jroml+s1kXRStfx0 z21)El#lg$vuves+ZTAYe{)61TB0uq~niK&8V&5O}>ZJC|)#!%FkeCS$P!^Q{<4%Wel?oSy-D&l;GEgMg1$q|WRe_K z&6V!{%oQ{Fz_SP^@rJe_1B1?B?$A{A8Mxa?=EHJAKP=lBDWxwAA&JdThS!U*zNxV*GtelqJl~M&9a%3n}43E&>{M>D`x8nmY{7a1^xmod&GZndt_P zrJbaH&^00Th9J4ye=1!~V;IrW90TJA`l?{fGnZ~-KWhe#e zJI1ge1==*#+H1;UXogC-K?)pxr*@Uf^E+g8wMlhkY~H*loR3fb#@nu-TBdcu9L+kT zl>R{5-I;OO@4*04uB2C?5sDNtPG)w}4Ov=HO*iG&8;FqRm*?y*AbBXpU& zWP5MvP7$d-3aUjls-nb2k#J4Iy4$h^yM;gRW1aDtn>-|f4K$#w+T!3Wkdq7fV0qZP zB+vd+qQh}tyx}GJB{qdIWAT~U@qIlL1o{CFa@>@l9pT}3Nts~R_+!`4)F+@E-u%V^ zj?2y~?ZEE$jpahO4HoV4T%ImDOF>_aGF*FgYPco(y~6Xh{;7cmhC5`8DLBTmOZ$S- zSJhxOFn6!s1xmf7i8iA`A&~klZwzt2sAnpcgV7n|OQRl%a5u8nhLnw$J8f-NuI2Y? z1jNFAR{~AAM$8coaQX+n)1K3)Ne*3w*+su(8gis+>M|`=o%bZl@3cql@tx8+$75Pu z;cP#fr{o%W>4r27vsay2pr;#r`e4J!SDu8`TWxImvH11^{7A+3Cy4cCTS2_>((t$1 z>zNYLo1^tgvC^eB%CjZT&kQC)5O`+Axo8J+@`&2_NpA4hadUpW^HYb_pl4-vHBvi| zE~UfXN`@leiv&yz=s9S>`#G=paO#9LgmX-1vv}i6FQShI{3yQ|di6bE)$J>LOBh6c zifcglP-+TQn9U3Z4~SZaHs!EzKA)nI%lBw@Jj<@Bf8`{`ASQ~YKbd#61I|jp9rBdN z&fo1C!NwuVqB%eZOz030a0kcn#cFrXy4q>7vh*0wyZ(_g{FJncnKMMIPOXHq)NgA< zTii+3`I#8B^_=%Sekk5Wc5lMfU@=H^nllA)CSQ)nFA?fgOI<7k8}D=|Z0pr|OJ`TY z+vKPErqi=imR2CHihEhQO4RT6#ydG;6HW7#WnMdrKG?3DKK*g2J61D*Wj=Y2fRStK zB}jj%k(w@0U3Fcp<-kYcywYa?V~vuEFZ+=NzE_Gis^Ht)`C+!qVSGZWM>>cXem<%u znAt;doLF$wyKCe<#b1YH>nana3;!zbN!Hc9n4>vQ{knw|Ikbodi_hO0VJhg7tb!4m zK4dvo|Kj{Z&*Ly`&|G{!4?1R&-IE8KYAY?+=k+Dn`84xzq>Ook+e*-Giw+V>JnDj*9-rSuORMzh^*ddfb>**fGc!eJ3PSe-9@%-==tQn%l}yC)^M1kiOnr6_cT zGs`fyFsGtalIv@G5T|lzlQ5DE=qp+ZW7E9WrW_vI9Lk4&qdTh(jur!KS$()rU8q-( z-365Co4u%w|4JOHZ~>A#@0P-~W7AYREqNAaiJ`ecSIV zlbBu3E3QFs$|v}dr7PV|dz(f!b9kud6P@?5GD_#$Q4rqe1Rtk{quJ-GfJ?#!`az_X z4Eyxu?1E*EDEqXYpEnQQCJd#rIQQ?xu&h=ME#VB3bH3_^2-;~^H_KgO>=0 z`||tp?>}S?m2gHcNpGcwl-4`7cK5%NMgd)!t8MXPzrN2OJ=4SKf#qMsihPG3aO?n4 zbmkPn=}Pc5O_4@A;BOI&(Mn9}i*b=lyI06p7%AE3_zp|*)k`v!jMb1h!v0r}d1lU; z1$L~Ie#l+ob>12Ug)cd?vnJ3#JE|z{XL0u}C;_JPL1|89%;b zT&|Q*lhjkuR$R8^zD^BY^=V&v$^4xg+Mf+da`^xgWt*4y8h=6AuBVpap|h6e(yotI z6fL?t8rjaUkud-~bDS#k(zIyd^P3KZqak9@DqWR1|6H+C%dOF9mUXpzFHP$R% zD!0UyvvqSEK#ouuiv`K-#^2M3vkwRz({L6@U|C0a69~3#8IjlRMEL9>3Qk*8GD!$C zCN^d4^n|xaul(ztR~On^HL-kOb({4BFxb1_@8%%+l|F5WBWC$Wxlh)>KNj#J#LL_E zz?jGoNI4LpWIqik6^Z&QFNiZW_E!A`=rAeEj43_TV&Io}Vtl9N;))uw6j~rF?&Qlq zvb}72pbwiL@APVKjjmom7#q!NBNv`8rMs?p`Iy|%oyJF&rot6?2to4_e5DR8A9Am^ z(YX0q1XGd=&TgOP1{RI+8W!fAGd~*R-r@-(lq5#zY^vKI;51sr*wz_C#k)BG2==|? zeX^gGHTq9dNZ&r4_7=U;`p(M&WO|!DD_Jjht!| zgv&Axee4?U5!uqDc6zfp_EZqxHZEOtlbAzZm$<1xn*YKrmk<3O_4CR~hmXU+*b;oJ zJ?FX?<}YDUy^qHGn=zmr0?!9!^Rk@I*4Xd9re!=`@g>gey+unaXX(oJ;v!pidJ-bf z{!_;JKe>Jjrqi6y4B?g>4Lj|tg%p>+R0dnfC3yqRg6N&c9|}+#_8il8*p0$2Ar@x! z!*{Wqr)Kd94k5vk;kq8|PdAKU&kv0S9^Ob}*(#!Sq#T~;AY#~sok!1p7v(4AVaCRP z&3Z}AwY7$P%`++Ya6*ia-ymufl(nUOY572eKj?C^D0Zk zrAW~jY)<%Cs`o(aKgAQ=YVSpQRtso|g6me6SemmS$xzKoFR0}Tr(>^Wv91|W?Ps~@ z_vv?6F7)eVZKOU_@b-SC(_V0c^J=jb+LG%xU8dbge|H>5P|`d zFc=s$HkZr{oW^nJF%Dj=_#LY-t-#)4g}3`z)?p>=o+A>1)rR=xm1oI;>Tsc}`uNAS z-2S`vu9wm|3s*S6`>FgR^QPmU95zNK>3)9DfB!$v6|jUn_*^YOTaw}35*wH4Y&BR1 zf_Rv%o|q#*nhV$7LInyl!G_#6smbbkcKF32#aCdA>>{(~4#LDj%7g70io=cV`=^_K zi-77?0_y#|3(|g#t|h8-YwuQA_m5D8$T8@Ux)APG+%?Ql>%GCJWg9+Eo zg5JJv=A>+o=vfqk=gFGf;nCJExLynniZbyqE)uL7=(OFHwm*1_H~ZxCiE-ytc)GIq zfo|N9unnRXhTINxNNOo)*OVF<=uH-`8TM&sB*u3~^G@5di})Bj3qi!f=AJny=-c7T zwQr;2A_JdKIX2~Gkr&=Y3jyt90l#g;#>%Z841a{W!XhMU1c8(*>Hbq`Z)=*d%L)TN zC-SD>06%)mDOonFWC%JJs9iiaWjFmXQr&_d+`50Y|BH0I)P~94E!Ay4QI{|2x3A2_ z@KoySR%5+19V8&zaIj*H;8ckeE47pygcRe-SsI00>)9vIeGU{6&qcrWTF$8<6Iwqy z;1@pdQS^K8Lqsf%D??_8?j|~$_^S}LuJ0FH+r6Dbk>B%m zPN+xGi#lJm%ElK<;W&RTkRG|P5r{hbT91D@tE^h#O^)V>37C@ofFxVB`oYJC z(c=raoRigcf40O}EVH2$tKIVrza~F(E_cV>iv5Wko?R+WM&0J?e9>S0Mm(ZRB-@wx z8OEurs^%$z>5jcSAz;e0NjR90j@|4S$|QMy5+nZ%f1{)l#OK5>;7D~C;=e;wAK|fW ze!x$$+!1XY8~7TSiCvFmw|2VaTx&`m_a76`C7uR>xNbj_!~o_?GMyB2+jZRiQKUge zu#=m@alan#Zxi!rp{1)Ie&_J5vt=%sDBt<;@UHzy zow@e|9JHT%Ie0b7R9Ku_xOsL60~3D1tB0yjP%h=EO3;~$XgzIMl)BcEU0IAZK5mbR z0U4^fs@{vw%QHfU1}gky*&5S=HTMab;2V2Z#}P8KuNPLuNNIIC(dP_$7|GYl%Qm*7 zk-02|&wmdqwGD#cem0jxP)`v6X4t!e(?5OoV^@{OG^Ji3MVVMAd9p zBpDWF)CvIC7=0pzU2^V@*+stBiZRuW%B8Lt-Y|OQ5N)Nac zDPdAcIekqG6)_x3&^O7${Mu(!PMu7G4)h|heo<(YXP+V;Jb*d~G2d!e3{PnGk9&?>IV@3Uax5w>d(YmM9;wLHh zEE(4ZnFKA@YnqR1f&pnbN?aKLIYXZ+cR1~Pd^8yjA|{X2s!cR^1Y&Fbt~<`J*Bxy* zZSC-%GfJ=Pmc>V)%)t_J8(xepUp=Q*$XC3fs~`35RB{jooc1TTrKVocqe);YQ>}35ok~8ia75Ufd z)OY$ZU+{bqW61UX*^If9(A>5uNnv%-WO_e-od2wgMfhq^5`2Fpvnmtg-B~ju#K_#L zYfZ0o|5=SFlTZ7oY!u;|22!0kMd78rsIwDqzviqT-*+5R*gl>g^G|t z&k9g8;XbdE6=@XwfklSl{+PJb1Io4?^n(5>0tA4Oshksv2b8GQi=RVemreF3i~Vyi z=UAQLn7r3fMqY-?X(^! z7vCV63)&gH-W@F^EmP|mK*>unovhR{)x(C?3}6CIqYr3@J3XEN<->Pb zjKk&AK8RzQIEeEk8q@+n!oT~pJsm6<1cl!k4AP{0^T@J1N%}AAl6R}BtN}bHz3u2b z8NO&VR&Gas#e-9aLfD6(vZ)1^+tSfWV~T`&dPlz4Oj`bS>qVFSfJ$QPIcI@m+FsrC zq}zbbCSwNfo1|pb`knG_Y^oBpduBq|E!u16Jo4ScRzqUzY3JuMo|EngYxKFFM^4nr zn0Jnij3XCeUPlin<}|(F6?s9L7>ZDQlJT~Y87UxV!8IbH&UdqB&zwtl=mxYshXvNV zcgUyZ3y12IsvN;lK`gEEiN#(4ES$F zx4dl%sw`aH+VesRrIcF&HTUxzhSMcRFTJGn3>uliSW8$|ZvC~PIaIm9vp(CVbWadb zjbKNxA@X}nGaaS};!VY+Bn#?j`{xwh2s!YO7zA{`Z!xM{qI{wsQa(Q(9s~Xz2-cXJ zDswRLJVBx^<%Sa>Msc>?JRqX3&#`KwEVQfyig`Y_B9eO>1jFid56Xe(nfZKv zg&s(56OM3D^zL3nn5GXwP!T>KftFduyZFH09@Ko;ldzez64lsMMoEY(@Z*YZavffj zsVsY1sN%R=8shw@&XD--%v}numD6i=|3?`i)ZhKe>SJewv>)Go>y);dRGR%7=k&Y* zhj%j*f0$n4qlB_{LZr^hO__0DJLq1l@Yv&I=>5--g`%$xL!#UtUVkKq%;2KO>B=ez zr@8AD=`P&x8~!8QL9f-t;332_e+spQn09sIkLhD?zB>9KUskcY-j5ZJzUJt?8YRRq zw{*ilJV$dTfMu>Q(AND5+kW~lvgPh;xf=#P4IrUI7Zr=NZ6Pa~{?6bF z-Hr0APU{0)hqDh;x{bLa{Ft;0Pwiw$wxi)s@2^|~gk9z*_vNfS$PT|x-bcPsO4Wr( z#>yQqIT1o=?ddGg03OvFo;?iQ11W=-ee51~Q&e7qDEeq_O2$LGUpK~HAWUbrIfnKU zsR6QSW5Nt(_4Q4{lHcYVO!d(0q)3bSvTbDafBe~sf$HKOs0&+8#L=LSO1yO zZ+twiAnE5L40kxW>XXM#O(zjfFD~PFICB^=GgsMtu)6u~7wqU$#=`CX(iwZYzC@5` zI;u5u`TbtyOvl79V$;@Q_Juo}?C(mTYrjLd8Tp4~EHSKxuoL0^Tu}e2^YMdxRc5rQ zwvc!Ty*w_&?bqufL!Mdnx_(UrC)&Rqxf`9`3O#B&zJ z{l7x?(4oJGKvvIxO7=9AA|bvV%4!JrCL}VZ(b5#_FzAg4F`-E)s{`qRYnX|yo zXb2yn8;!8G@Qa{@ZMO9F8Cv~DG`k0_=?`vM=xn?zU>p6`lf`m~HA(|sruV46s-L`^ z_JZTw>7@35y$isBLI|!^H-|oyw~pXwEm`tcNCYH8KMO0SmEp4TmXp3aYyCEl^!CVH zu;fx6%uHZL1ywJkc+9g$N^hQC?pZbuNW79m2`#i>{c#fTUi~n(O{`~hWjK1ick@nP zuXJ$W-PMsVP^?PIbLGrrm$2|Fa2Mct{TLR?%;h0gpI@8UL}@2Fy!X17vz>V{?Je_9 z^HG24#PJp}yU>bBvNZI3Q7yOT)KP@er4t&UqFeM1jT%L2CD;MZW8OdN?D(AtUCUsu z)v&ds(vDIy*TBQxAg}z^m^C#%Vb>>lw%EoS4%|GyMdGV}HH6DblnwGa8&?h9OvExW zgGpegsp=l@3sj};W)3kd2Vh4W%dAPBKYZLkUOssydnGKqlevy&bVUrn<~bWmc%+Qu zZiuFpNqIY``{xfe<zU!)wcA@E^~QKp zh1=;U6liv+s4ZOyYk6c3Gva+~?>@-D3;Ws^kGpP=yz6t4u(~>+!oA0I#Uwn)?NqtC z-Sp!p_hY^a%54Ol=un#5f-u2ccS4@_58ovpYKr90V;h`x!c@xaiwUd35;-tfi+hVw_zAQbqBm-q1s> zk?{d@lgP9CokR9wQ@K*@-MzJ!Gp$v@t30wf=qFTfLjha+g)T$2lU!fIT7rZZRIWw*a-*0GL08xO^ zqk}4KNFTA{+^xU%sM~xy22^RD-;do`#gBPPq>NO!5Z6ns^&#Ho`R!@6dWIrFg@E$i zx9IC_25KZm>8nE^aPM38`!IGvyw7Z6>*~(daz58vC zM&!>M>D+y>5j76sW@DhsUw1mm23;wix5S&G(H*qw8({@DI~E_TPS)l>u9Lot`Gjkl zUIm5q5exMW-iv$z^OIKVz7#I#IlLg}N>&@Acak)D&|K7f>TZQq*2xEH6@>ujmj3fC z5&w^e&g#`4gzH~)+1$Q1;**OZ1(SA!1@I~b_Tjh<07QyVqL99Xt58;4YRO)FPRJamDwBD zRDk-xia~itvo@kN33*OYYuvL#-pM8@MP_YG(_MjOxKeOos8iPqAL+8?u}s&#CIAn! z3yjyqB#a#sw~`DmNo=D<_hW!jHF><#W>-_5jnzCm{J;2m@317>$M63xE9*8jv(!W( zwX|-{nH!~YRhDK$&8?W4IC7&RmX;$_GE2>Yw^^y-%#{nbxDfZije7&40e=alvuH#c^?Rp6~bT{XTJL5S_)Rr|2c?F!GiI9bQ^d$rj=sQqMsNO%R2W z=XcQNZX1-#SQH37ZK=g9Qeaz0gB(wmEvn0aioP&$8yN{;-b<`*6l5JB^}Pr7C2B-qOq42|JjAUga=88YbH!%IP!c`3}vJnuwjUjg||l7l)yFIG>X!WeoN*` zZG)>`gOt8FFf91}k{3-FIK5kf~TycT$Y@?*MX)4DvRx{^_~Q%I9wKV4YL|y zzv~CKGB*=^P2X+6d=?Nqkmc}=NR77Ub60006sFC0H5u=XS=-uzcVW6T#K!57FzbA; zi!9eM=V}hFH<{6$PD-%_?iGA!-cxxz5h6PV>xB1XdF#6d-6f217iM?ZV34ckLTkmv zP4I@9#m*D|X)$>x*;`UDXT(%ogKtgQ0ajfyNkL;z6}3GZ5^CRqZWsai9E%S+#tG!o z+D6~)73kZJ6fsvLRr$e)8N;D6A4oz}gBbsf235$dm4jD#vEh1>0!qvo;2y>s`G6D9 z@lGWz%QSmFw~Y!|{WEkpu^e?>EnJyvnG2foc9CAv1o8Iy|`xt^<_myH6c zb0#rQ$#vM7%Io_bTAVB%w$K-lH_qeHC=A zca&SMU6$H_O)xQz(u*776Ih!s9Ev|s@EUMeuV5oS@8v!=85jg%KrNf2k1Wg3zRkmv zTmKK8dR-8fF|y|LV#iz~u_6k}w6{(8^LK$wPm&J$2=g4fFZ;!r1L*CFPk`!ZECp&% z=5_Wocv{F*!GY)QB$KGZ)^Y6#}o2akY<`;=hLfF1ZuhM29+0W^oil- zt$PWl&9Ei@VvDZ%@ZT0HJzFdSO{OMv!~$p0^hLt zgI1Qu#B5A-(*DQ=!5v@lj{{e~PgAZ46Fl~1w)soVPg2@F#RqbjwUJt1=jcw7<+{S5 zdl$Ai%W4GFb~EVBg&uSJMgB8;$$1|Qx}l91{t^s}>bboUa^KFqXEOCx^fAC+Xyil2 zV!S*3gO3eAc6(&HpUaqsSN(8(rF88w{uFvmUpEdB7G4l3h#=4Pzhxy{a}&p_`ok~O z$lmns&mt7o&^oly0@AWG>-JaEpYPJ=eTZ6uJDh4CrOR||fY-ZoWL-M-*`bwJUx`d# zLg!}{x@yQT$ZA;O1$iczErNHT2X~GYFzd0W*tb99E?=81$ znDO~q1%1udYw+@@!Ung|UoH94q&JlZz9S{`gWu>M`0(!pZI)Vd%W0YZ{zOERlsCd+ zb*4oKs(p(+k8rc+kohpYb;i4uR}DvL_JIqljei!QYp)g5WW!Yy3BLXx( zr=vUFl--|V2G(-?6E5=1?hfCfYbum`i}4i7fkUw~_Ni*o?~wT+UIj3RPHknKsS?|V z1@uckg2VitGGxFQJ^H|LqV+8|ZJK1WD5kX%wwfNm8^)u;c$yK|P;v~-||>@w8s z9Ikg2xxm4kfks7r&b}L!D+#Bg1fluWBpvxMu~kP3s2@s2!MgQVr9_(QhJE zhg>*-!csPQzix{Fe>6rGo_Y+x#4TKGO*JT0U&+a0fxjYO$a4CyCMDc@ob47yZy9li zpQH|)5TID@)|d9eTOz$H%G)>~s4EUFoY02Oi&I^kHkdXSUf{U1+gLdvpL=ea`_1*m zM73<=sBkF{@Nv-GlD$}3WQ>~8^kx08h86B5|&*Ax-F)x}0&K3g?Z`N=hI zpHSaIIdg~eCU5r1_}b z*M&RNe5&Uj)H5iuS4LhFTTVcL_E}f)kc6AsHCDyBshQWzn71}2KFx&dwXij>fA%Hb zqqzNl|I3EHdZpSVYJ54wt-xTXnRrgF8ryxcpmiRl^JF}i%>ZbrbR~=})$UZ(Y1Lep zh|}EoomG9vTAfI})FKA4>0T+h)*U!-a=Y0~n2pVu3d$i~@!#P#2(<#8_M4f319-oR>`3 zt4;aDQ5K#?%^TOUhMRQ%MCE9?e}UE_v$;dML8k2R+o_+j#nZ(eN_-gd51d$YnJYA49?jG z`_}iDyy0avC*&UF1uvWxkoByb7YN{K(H?%tx@mDXb854C6L*Mx(F=EokKM7#s~_6h=OrSt699xdv?ry+e}`ovJtEmtBH#k-P5q9Ed>z3JfdaqPx{+fgUi z2edn$v2g}^gs)S41=%jo8`8ADFp=GtNP1e3HvBfN#c$)P_~SiOHX19;8Gj&MiJK0B2awYeNO(_ZJ`SO^`{CR>V*iVMzXSzVbGu}rhnTFZ%LlUbyHWa++c zelMPYnSGTzrDeDaS#$nqkM&OMMPCZhW~X^pglq-gd2~TSad~M3)5IUB_O}uO_B9>2`8u;z^XKc4b2(h&k%)G3>y?#@HWZ}vL*4}#0cYNl~`GaF&PmZHKBK~@$ z+GphpCWj6Ddb2r1med}JQhv_?7#Gp?-shL{-i)|!UP}HH>IR4RCdL|V&oSy%{i9m? zZ|fJ>U78O&TdQ|o{Tx4K5H;-VO90a?oy&g9jy_&uWcgiB5{&g~PWaV^SRR%3V2YO%ZKH#&rfk^x#f> z^0?(X)exA^f_6VHFiC@wlb=Oun}q~<w)gC(KrVgJ?eQ*Xd<&og7IsHpjFziBLpzPm0s0%m8Q+Kln;T)SM+JI1^1dk^SSVb zC$rDM)z0x3At*_Q*h7D@;UP}<_uPQhO5z2nJ%ue(!${U|Z>TI}v)@$U~g{DXKB!Liy#8xruouHAZ-$nRJa{l%lK>PPV zWj!Ys#nk}*s9hwFFOz?O2ST-Mjx1-@=q^~+LBc?F}Sy$=3?>wPT zCps|hP_&f6=g%kbz*~>?hCXJgep4#f3Nm3wJ!7`B##atLpfj!JZTCpH?q!BRY;@Kj z(&m>b5b7Y(_i_}J+9U+hK?<EJ>l-8SsRFStn@A@@vx+|iZ4p$BWsXzN@u0N)Z=+6LZj%)#7HU0y z9+fBVwv}I*;w;$Jqlfm)rs-%V(*OD(a66vrF5_LS0w)M?Sta7Xp7;MO#h(3_Pv4$< z^H3$-GB0QWsFlivSbY*ydeRWq)cOWkJiW;_iA)Qz<~NTcZ(!^eWI@L|ZDI5JiLIls16Xp-70&m=SWPZB5 zX;~pbu|)ERtKC|d!5JY?yv;gRyU@KI<3@TX$<_!|)Cxk_3m`e-q>z@cn~1AKu&ERJ zJ07EUV{q|f`wuOH;f4Hi(_tVsf2VIT4H_&fx>Ls8op0-_Rv3tRNF9N4(%|IXzZW3B zS~Ld{tl1dd`;FPEecggr0f7~1hId&dtuzX%P}YkFvFzK_;|Afsc8vVF0jj8xxB_TYGSpZVE*+d6BT zN~?pgl9}E3G z*S)*=&TkVi!+2G$S4JKq#ko2)q4 z9NfIg^Yvao%X8GFJuen)V|l`9J;~I%%Qe_N{;uc$)H4+cpRm|HjxA1dcha@%dXgAC z+~MR~UZK93I>1{O75twng5^@>$44$_6_IPFEPAEu+!-r@^O4<$yTjGrwU{zVn`=Po z8s9`YKyD)uUgpw%EaY$@cqucIrL~KpM9s7B-*iTba!^fVA5N1$gFT<3HCr5CH&&nj z^ry||R?VH7;g7p211xki7EJ4@dwoy zKHeQ`z6>!Eg!Vyzf&O4DDA zmclHBelQGLGiGNH4r?=haiSTrb^El#+8-UM`$Y|!kc+CxUR|0h?D7rPU)5|Ze3Sgv z7`P+)l@gT-Dn8H08}$B?Hs0>w45G1W))twCe>+y85QvGm)i#(J{i)THo4wBm}XM1qM%UeE3IfN4eoi zTNMEmc00*mE4D%!PN; z$hx-c*N@YFo*7$ubRtE~Rn&n_8VV{&V^0c<;mwytiM)!bhWQfe>O6$xP z)6eDaR|xnOlV#NYt-Bc=^bcLy+j`Aal%zC6`{?Wo{yNoOrkCOf6WyN;PqKwV-iP~l z=#;s*x5v$oLgz0Lo-erqbLGN*>7-#i+ea(Nw={VfV2wZ0IWq8~)`G44mTwf)C2tUyt%Z+V=|{a$%hLWVH`NJ!fcjCk>s;JxU=->l-@- z=C9feOqCx9iIUU4tJFK>XnB>oLc8E)f%!Y;UPXoX4!$$<6MeMbNmBQ0f-H5Pe}joETN4LVEnL$t%iU$v>q|$90p@O5cPZjkX0LrOMJ##mlBh2f zzO88%mUf)e-k+!K`i+{~rAc{Zv}mQSV`tG2T=gHRk`*bE$D>f6T3-&nUbUKMJ+WG6 z&ItMUge7u*57$)975d74cOj{xk^E!ApVD49QBqO*qB9X0PauiT0z?HC|8z~jy0zlg z0-1y0vq65H<932xx!AkJg$FW|Q)&Y@P{C_r^qR825Oze8BaVh>a>hmnPm49~{^>>f z+RB(@w}l1ltydTi)T=M1^|QH(K=?Y3!D)RAB6G3f=Bw^=ry)c%BF?uc)7FEZxDjparCaG8 zA9a}AEO7YzyQDsw9JqFeb{U8+LIcZhDwu0AmaYj%_ZOQU_-St*SPYyDxgWRR0YLGk z4WBStdX+V6GSqVAA5;PAp64E>7qp);$@b65HZ?iY4&OGGeb7Y>ayJwkf(4(=A->Y% z0%{~gumjw6kJ}4*gWht-5?RAL--v3CHsh5(*_NpzcG|^)@UlTn`!7*KhP^c$r`jxaMY_1+xhe09JVw*?_0t$Hx3L@9Uy&^!FGYbY z;RRg0u*fey`(vg9W~d?p$dVUKc^#~kx&FxAB@I3-V_)XCI)vnjaS$_Y4f(_cUrpg3 zWjDuoK5Y_)nd}8p+LP3l%;b1z-&!}>?Ov44^V^oI!g1X6ufb1UQTcul4h-G8^Hd0p z-Sbpcb~~1zV>l~EU)6JRT)`}Bb@?&>>)$6>h1koD4jmIBJ=A*KFF-M{PY zmE|R`ci8js-`Xxdy?TP%Y>KaKnG0#6)xjaU@j38(U&(Dq>=)?raE@_&d{Y2!OLTXW z{0li!i<{r}SoB<9^LL+ohN&4Blp7MBVjk6&RL`N3e5-k|b#+DpSSeQFh7Q#g`DylT zz);%h-u>u0t06|>E@L^#$nXf7dRV;0kEdDh{Uk~uHq>uB#{gF_ljfL3OE1eW&c=bf z%f02Tk>$YH!7Y+eQwdZT-~4i`6hBlh82k-3_AuCLj855cXfSRrgbHB|sx>gr1N~%r z)fGr;>uM-cy=qhW_wqfbG_Jp}3-Ml(HnJO&4=)W}7k9h7D{!-?;Z(vk)xUrWw|!<7 zcrTzS4JTt!k2_DP%bNu2@-*W?z(3=?p2vpy1?uUe@n-Q|N+YML#)SvI4dT4Cc_%z7 zT&O71xb~}>c%Ih5%fOX>z2TGQ;d*rZWjKw~d8lOrLfZ+jK0C}NEX~q{+4}=F#RX(1gJAaZM zKDThzmE9$x%964B7!Wek%2{5iPl-esF#eKTSU4@`@W*U0)0Qdm&$%%t^Km9S?sioS zzW_;moEaWrE+G>YZ(hvDw4?sD7th@)UWPYE{fWaoku6zkK;cdly5yN;&`(zJy;&?F zY6BKJwgqI0_~tnhWgPSly3IKPRz^|-qSpW5CxUenpswq78`gWD0b77AGskcR`O;hW zV|r`PAsH+i6on9C*cm|h_SmK{IQ^p3I$}}OLO0P zk*4(&=e^9QA_VKLz3QG^2d71oC{gNY1!ANw-kFdtv?yyIS>Yk>%V`kr{4EmD3iEv3pY;M>GRvM6?G25 zbvF`Tv8`P@)#F)$cTWl>1cvYxDbC%vGubzDr(e9x^I?5o^xE0>m+V#U75piMuZuJ9 z9ssKNdbP#NoIxgr0%gy+Asy_7k0Rl|Oj(OcwjrR=`!28z{{itp7V36RA_$Nv8{ry_ zM0v~&X5RoT6){S#$t(^o(p8ItyyB?#eqg}QTmjQQ`_0}DX(}d`#7P-&3wbcYG2i?f zzkQsQrF6F#C_cn3t{4q0^_95-l`p@eM2iXwF1{K_iK=i5Y=y7G7+H?HyAH zprbM3X|Mgm_JssX$2%|fxdN!;;vPEyM0*Z>8Y~~};&yg~vS7XzSFG@@|GmQd+68Eq z^6q62P3cNuNYJ8AWuMbs=erwBYi=|m3#q4aA#$PD1?&wv&sRN^X|G<*Is*=|4k%L?D zA1TOvX1NqsY_jMQ67v$xbk`vZs8gDPwW0wZTi26Wrpk>KSu`^uEZ1e`kQ#2H0o7 zX${j+xaXXrHDYe*x=MBH?z2RNT*z9q6bkyKN2?L4MT(4>Ed}~jlSIhucFr44)&G5c zJEVRbJd^J!{m)Y^y*wX*SMbJa0oa zu_j}Tt$S>Gmmq#!C9p49>a)62W~5Y3ZHqGIEgljij#(+lc;$3^LiP?c!Ly9E&cUYw z|0@OK~Bp zM$U6;c`t0cBo=YT&0Q4<{E0F1r{;_CDHaX(cT;Bg5)iszGN%sNRCeh+ufltS7zR8C z!LJLZPkY5#eM3qj$*)^mXP0|1Ld%{^HXAJ9*mM7Hl?*OLnOM$6ERP zPLggO!|E`@_2F}NYL0d}K#cm1)$FL~agVCyAJJwvi{M>f0fimsij!OJd~S(bxcC^u-Q2!-1_RH>3B5Z zolx-^V}`wu8>4iMSv2Z?M@#Tia^aPK@^a z!H_uJ+zlVe0$r+M4NBFNuY`m4?a)^>O230#uqo44i1ps>Yl3&Ro_AYZXdLdU^zBfGFTY^tRxj2t`uQ4U{S5x(X z`$1sFP(+zrUEAso2FNM{;>1<~!7?A811M@0L30fbAVi$rjyagrKScVZ1$)A@&=NQ| zXtOg?!ahr*1gKL&fL;X4y(odxZNANJw(QOa!DT*J02dj>;L$R$Z<|+&(2K~B3liFS zptWIYJyv11rZv^ap>&7hUQ8DG8B*Vb-uK~&c6PT&3xKU3f^eU`)|ln)Y94iWe++eR z>DocWGBc;it2^9-gKk+1l3X%6tO3 z-SLmSbd53}SaBVuzW6Z{Z_4j&O89Qpsh%yWu~v&JB;9vzi;$Y~y&FA4Q7o?ces}j_ zndh(4+_ObG{)=m)kK#;3+L(@aut)6mzk!O&)JG`4dNSNRP}{6iB13QJ9fh259sPkz zN^Y3#g=yzyXz(E1H1ZG&ub#%A@azA~S3&>rb@$_uB65E8Cl@Q7Tc)j2l98|0Ym2SQ zVT*LpKJk7!m;oJnbUcDT(g3;|v6D|4FyG#fq23NA{W7SNbVLY`{RH^dhZ9gb3i{p% z@uoV})|F%@b-cla$Rd~df707dF=sgdI|+I%50Gw4aPv$l@Th@CqoIGTmPRwyqEv$h zYVUm(yh20bpR~DMk5g;~KS> za)EEZuw?$=1?gfkE|EOcpeuonw(b@xMX~I2xDkGVR`?yv*UMi2Zk^6mSe<|4#tw5v zxaHu$Aj(^Ii`SK1N6LU%K2kJ+v)n=@hgx=drX43qo+X{(wr*-5^vAP+sBJEqYf^SVUL4tRV`~A@5(y%5meu^RyxD9s4%v*$Cjl1+(!Sp%!cd-ak}zX@ z8C!K+G*BtcQ1KvExRvenH!rnVqnT}!GE{=APZ`c&Dt-DjN>~mVEBn62^4J|Yahr3f zaokw_ii1u9+k{EO_WR1I{;`T+U;B<5%{g*_@L-0kmoU9-u+-h)AhV1%L8%=TH|kRH04Rp0g>Ki{^Ip ziyzQ7^TC<=HFQS&=|iTV0p6vXH90h-b0SCdMwP5<=r+@0h+9qo z?V6z;Rv8sY06?HC5emQmh#oLzAfP&Armp8Lsl58uLFPi9;A5jbCCDCO&Cl6homL z8?rKR742?E3yA|pOb&v!qYzq~ zOjuxNgj>G9aT@(n$ngSYu}GpCeCC~?RZV>Y3%wxf_CbpIYK>3*F`#$*t;&dyxP$xX zE&s{D#mcQLw=HDZHKSIOn*n_o@JRAzU}(~C!1T2Jjia??{#N0+QM!E?}&yf|_mcWKj#H|nzw zT1bKseR`DLm@^7BCC-n0o9eZ%G0^iNc^jmd>Y?TRb9OGR>XrbX2II2r4luI`jwXNOJ^=Eqx4&>Xd^|Bw z_RiYvETMSrJx>+T-b^JYBP8dhrdz-LL|@%CD7B4ts5A8-pn1!HUU#m}c<+)>^VtaJ zbibkI5eJ^UtxU+5Co*0<$a14(GqqvmG-!bw{oU!Y>Bgyw@yIj5uNrM0c2%14T8@juTUu5Y23$OX1P2Vc zjXj$O^wuaowrYXI>?oGTE#6e)J+O5N`!b-&k33B>+~t%k;NE1QDNtvc*Z~ z&9W3bE(~HIm=nAs?Y+lBN;!?s^Sui2grP6>* zJY+db8SjT0r|naMA#U$R7PHl=5SPZmRzq?H`F53~@LLc0o@xW+N5uM-71IxlcEF_o z6G1m2n60e4TeX*2KnAnRIS!y}Et14OoJ_}^wd@|-sp8lU)fVw|sg$Z?OwMqHAEE0Y zf%`}511-dY&|4#q1wZ_uCT4ZqtOwYFE=_N8+c2kT*w)YbspSMSHV5E8ae=Tk8YDjD zLBZwJV6iv+pgh|29vvxmk}Uw$z$@1kV#08n@Oe`F8(lYPVAixXy)#sS+rHjNfk0YZ z`2EWZv;ad1Z(nHf8&qsuD#_z0DeGtFPTVj1`CL2&XwA|tZ9%q9Y%}H;>zGX>670ZQ896|$>IV$g@h?Uy}E zB+U$w1=MBaoCHX{J7F{#{^Uh|Zry0g?cwm{$FtcAU+i=0*lKUU8FC}IQr@Z6aYsLQ z@ky%UHeN=*$>y@0R>se|Q3OvSD`h?`!;sCkDahF8U@L0oZ#_qt@V*UBq83GNS3tPHa!vy-8KtV z7Ukc4JzRIVE#(>1pZdd|Du%!I)+4>*xl-_!)4*~;68Y)TLOcjzBIsHMTW|=8pJ=!G zv~ajWi16ySlP)j^hnwq&k_!lB45wY|U-Hc_C%lYThB%>Q&b#AK=h*ySq@#V6fqAww zG1-yO(ya|-k%U*D!&hs4R5m^L_Qs5CD0Owyb5toFX4=|^$-TIygSiS`BkQ9}`~{fm zS|-60=dqC!uTbdy#rgZ(5F(HmJXzwJx*xT?J)$|q>|R~_9S~PO_|Z==;uMk1zdGZl znI_-ZU0ZY#oEDhZ#E?yl(Nb6Lltr$swik(NXnKb3jTMf?SUzkFt4{{_yC33?4mNFj zs)Pk>Ydp5Z017Z#zZSrNq-d^3&oRoJ~4|zPKn>rw*C3Ock8h+D6Wa1lHb5d#Y zv8s}RY%?K zh!qq9IO_blnJUrej21Lpz5UhPQZSLR;^XPuSx>SupUohU2JfeCDupJkEV$NI1@=0x z&OQfEejHZ*qO2Y2q^J}W`=(0|`mVUev`iC2qz|ses5*@2tZuOq%)7CniHwooOTRHK zlS-1S$`zKW$ep0&_Z}gZo4KtG`mfwoT-<8DN&e&6$uu2PQ6KeospKq5KV?`O1Nhn- zorXy>|LG=Wkoe$prxRm{TS{p0r9&PyopTnfL37E2b-k(zOisyx9b-FdyD~!zbEeDW z>s+MS;o5j|o3X2cvV_^C!hs){^tQ&c?#)vD^Qmb0oBqpaKh2esfLhWIDH|a}FWX*4 z{)!zWu^#hAizE2d&e^p~BT7ZicK_vos^_U4;}4TH(30Tt|N5dor?Ij>h4|@~n@=^E zDCsae_-58sN$ucGR1w_Hg!{44;?p|0YY=T)8jqd*BXcc%d?qG zIRx?iFEW~@)IdsPv|L}2;}MTTca1tO$pk)dQ&||Pm;t23-b{5O>q0r!+1t-- zQt@0+sTziq_Dx8{YUBqUG@5na#~NU*&X7BruS;XyR>8FyAihPK9_aHMsvHY9jL6$> z#!RAxYiGDUf9p%2fk$6A4TO$nL-*0yG#9J6XGrFQ#fydTmUi~$AGj>u;)MM+v(}(y zv?#p;<-u6i1YswynFV^U8e}OR7R~9)bo)F_pY}WWqI&n_{F24_o3oL)h~nnLn(1e@ z|KwxLzN8s5&k%PT7Qpw7cZuw7V8QG+el=&ywHKes`f?0W+w~inWww2onu*8 zUzs!st%*>D!7aU>v=@%kwQYXqcL9hxfN z5h--uaiYH#l2%l|Q+OcUDlj@4^u%v>YgI|U^B@MP)kwJg{HGj{JQd|P`?JJLDYjE@ zah#NVV_p$j`8M9_`zD1FHb^BJmb5=rrB1_Tv4C$bDpqcd1I{X`_bd?o!TKTWzhFUaZ2e{}Y77xNd+Pj(w`=aI3J`G5ms)h4y!I{4*Ihw0l<;0tKa8#&|a& zExZ$9TDX*dGZMM`RwQJ!p9$B+KKxz`gZ?HrywEIL((rqX=S`nXHmLG8IjDGPfsZbL zRTg;f=c-@E`!B@6ZXAOBwmXmTh}nM!&3i`98J49hyN6L#@ev&Ps6#QS~^2pGDkoA3XU?h4K;+2EK^ZG0eV-7Di+W%ct{8VrsKa5sA zdl)6boWX8+>=#za7IYGBO(!YWS3jQEzuX;4U8;_d;;CF3;#Z+8V`F7`7kF%}!Pxwf z3e!?3^4yQjyOUUC_o!);=qUjG4+aimO2omSMxnXU^E+dn87wgaZ!Jq>N=v(q$3I!M zNi}m<4QvfqFKhuNW;ojC;lWc=3^_FLe3imH*QHAN!&x0;`H^$ZwGEOyQ1Qi-fWqaN zhdAc3DUMU=#h0God|8gL5I;pmiP} z$>ub7Y2E_0SVL7w#sVKCDaC6RmZY@BVQhV)<+fS3h9%vJC@@%vw>cPT4?QZ=!Rb7vx*F?FDzh!=xlx<>)LfS35oJCoSA~q15hOna7!rIMLqq%7{HsCCYSO3dr?$_ z5=UNl;z+5?H_ox0!WXB$qT}iYx?b+qeBnlFI2A`g`kjNXo6(n33SGnNT%b>aMMFMS zx0MOaHy%{O)&Oz_i(DbTI14i?G1=wOCm&nk%(vwgb5S3oFmOHCy87o{`x1KfMrk*`r+f*m=&NZVPaMlQ z>GR3Y9Z_Ct6rhj&%^s~aRo|Ps%ckQqQg}=A-9K^L3__%15!=}gH_|g&y3cnL8u^Xg zXZ_~$)8!x&XGv$D7dff6$~Y-S-jNa8WjI>Qo53bwcWBXX0@$Pa4jKtmF|cK{&1CHzt+F2=Olq4pt|z1cq6v06=s0$Cd90R_ zKa$Q%rpxb+5|KNM`T?zTferDIbI!kH3k+|G3GsS*?_)!^3rTuf#Y9(?{X*n6Rtmys z;3X+Y%-XM7Zw3vQAub3OEFFPpZ`>a<>}0lI!C?K9gyW=B?aUg)G+|6R#R{Cbwd@d$bWdO2UNwf~m=5tE%Df4FUDV9-lxLuf0AVq~< zAF{(#EUvBCk{!Q<0_aMzY>67+*D)MA2Hvse{#^e8R_9@sDPmXnwPai z^sZ6{c`GzU#VXtWv~|`>;z~_oyFi?AKT9;`nqmwTw7VTC&Rg;E7t6nEiGc`0xBzqYalyLC<_Mz=pfc6?Rp5b}wttXw=E1&vB3hfZ)IJ4jU0aM)S5m>%vIYyaD_0Aue*l>KSc8@}a9-&K zm_?gZE)0zstzdWI>^wALO^#EAolPw<(3ubEWmkcs`vp5;k#%nyDw1&Y?^2ZnR$+gQPe3#l44gJe!1TcU0j_s4=+T8~da8S1B5#vxR->i#bAO zx3LUob+jCd)VkU+=B>SgHyTXu{n_=L=$xDmac;r6GQ06{DlCPLn0+hH$;--tT0-Wp zt2s}+7J#6#-{r&P$Nxg)1@0Inr>>mWpLAa^7#?Y>&QpTesW~BbL?Ot1ywnlBq~CT# zB(;v#$iB5l3qdrh?q($SKjHQINpcJiQ%CS4KSKC9h*p%ZkXzKCLNMn5D#ECuBmhW? zEkKR_#!3fLrj>(lsl_d4?l{fjDrI?uS_@#UbHj(!VI$yC?;Y!L|82iae@uJTlDAUd zK)2rHnQjq9%tUw)WiSHat0>=ve*SQJ@ri7=v%7o43+T&<1kS_mea8TTK6wxwyE$}P z&b%v=2j%XUE_f^bfdRnXnyzu^oDkaFv`Z#P-F2RAHJ;^$mIGscPbjA`d~A@}g2JE{ z^2vRfihV)+T1)=}ITiX)`FFLSYk|ZHhL??7(0&pdk}z$e-Ly=F0zj>9N^eCp*9ZW1+fdS^~;0>8a-MpYSmZG7)oz(4KM5G zPDFIRi|xD=L83l@%r=<|C(MO$d;T5pP-G#`%6Ij>@=&HqjOVajPnovAkcz!Me7!&R zOxG6==gpd%sbA&54EWy6B^rolamr!$mB`dD4XbKT68w`{W&dzQq8gzh9v!LD$)9J} zw&+$H0Zq64X|L@1z`2}}iA-8)GN)w2mWl=^-PS2*)d57i#xME(aQa}&8j#oddc=sj zv1bi4QDkZG=lBb^-kM{Vr*hpxCND>e>^Tku&QYvN;zj;#PaW5flT~LyR}pI%?q0iZ z4Vd=lM|lq&X3-2rQrV>&fp)eiEANY%L5gt4|JfLEDP-$>53Bzi7i zlxoN`UC9$`)B0~8*%L9)pM-8(##4=v11}tIcM6$-S@H>%;8l$biimITv+OOW34 zDDZEiEXzU(SRSl%bu7hzN|jR*1 zp8W~UgC%G|1!dSVZ2Q5EA+&74%r}XA*3(q5!Ti@Vg~Yibc*FCLsm#i6O0=rve`FyV zU*12Nmp;(P$T5F6W3_GcMPTJajKPeJ{V#c_?u^v*@=t|VZDL{fJfh7M61!FH_i4|| zBQ0h@r9*3Z)ZnJto+k?=qDg*nT(%%zubssPMMpBoy_4 zQt-=`#^D)FEM#jw2l>S}X7ZTlAj59?qvlP^HY28~hKvTA|6HVpaP8771f}Sn4{_$* z7M^!811y3D#!2xg+h_S=Gt$Q`Fg%oD3wnR#3<<@G2jYO9(RI&$ICOU>t;taTnd}or zRb0$=uxYJ)m{c{&(a$@z2aWcFauMTo~!fK!Mjfa?6x$Jq+up0L6~ zmARQeVq!ZUa!7?$)^E0+jPq(A`H-L!x}3~C>56TgXbD@ux9nea++TIenh0D>mpa;* zI?$vH3xh`MhkwTQ^vwEvkI#q3CjDOe77ulv^J9t`%7T;Cd4ZuJ=MUKU!L;A(Ehjo( z{sNzkS;d^I=GLi~&nAhx{})xNv(Q}Ik~RQSN8 z$GfwP6Vr@k^VI7q%B!b3W!OwaM4WNcka3O1?wNA8h|zPjXahRyj6-S?v)Qq1k>4i^ zz1<%bU$$&P?ZJZLOD%ckZ z`hGuaxtgLU*!L?v0+9h^u265QalFMSR$0 zLiq9yF|`b=SkjmfFA0-x965acM#Y7no=7Hm7WR&3 z>rR}FP>5=Yf2Y4@i%TSW32xM2=bOx#%h0SSo(FxeTKND%58af4cq4%WG(*>kHT@1V z8pJGe7w6ZStd@W0OZ=5|YPT$v4OPm}2A_oX&t#Qk^iK@M{#Hl`ei~&5M<>}caaWdE zJEA#yw9?(_``!CN#^Ih=qEGuq8P~IbhWDh7Vis+r(L4=Q0QKAAF) zGp~deA!k+GHu0#{fR4$S9;nyCo3F21e}TCL%7!X};%W3d7hUGfY(Kh5`}b*?O%yHb z$}r`(1y$gg&d6AT3lja=Erb;2+nDhbQBBx8HJfv=t-qNgTI@BQReAZtf57oO=lms- zN5zvv?mu+jT|-WVpqeYzi`rb+7hUxH8&m2ZJ(%59R`k@kub*cMDVXCZ7K!I4qg|&y zoKe?D$&SR`(FaRgFI$UD;VN_k)b!l?AICh?)&AVJIP1TbO|cb6h?=$h*TeI9U;EA| z!~{Dvx^ASqGv~xg*d?uM`jr=l1o*+J_TCrW`AS>!yW_d8P5NN=tqoJ?+7}1gJ{zow zVF8r$Yvq20o2Uxn>>gwd&z9DGA=RZ(?5gZ{aSL{FV_02MUmh8qtsvj9N_}X zU2G6cBZG9{0{@c6Q(_JB;80Dhzg7ueMYc%WJ@IF&S%>gG-UYNIL6`NACrUDH=apD0 zW_jb2#-YmGp|7%&__ox%W_gRS&Z(P<0v(cvm?ao`L35TDDxtDIyF4{Jr}H9B70U2m z8|wCy+KrAM zD;onRO$0iP_3;_+E*8rllb7?XLQac8Yic3p9pNNY)lKwPD_X%SPRs% zX?m8|lwsV^`zccRTXM@KH9h25)LyZ5M|ld!m?GzA&!(emZk1xU^ij zgBV*vF1k;c3IWr4qQsAO#fjQ}_dsae>_<~|&R57!Noka@XX>CSbV)=oXvT*~@ zKU%|TwIS2W64{^$VX?~qHz9Fg~mkGuKDv--B@vxi6*HMY^? zv#y$~@a%{{Z&ABa3rKAoXTM#YPG3IE&88wd4~94+?c0bDn3hLZ{Zr9OOiMydn&mo< zq($M$*PvhX*1Hnev6*(BnTuSXwQlqw1XV5WdE%YP0PF9|6mtUH7+gSVjmfGqs<>e7 z5R5h|zHo#(Y?9Uy`zbH(Z#mG!U<`MslWq{S!oS2>KV|nQS^0f)FX8LxO@s914r_&QX$AnQ;|!(@#3txO zN37&~D9A!O6P=;tm&HXZ#d6Tj4}%>)_FHg6_qm~_vKhiB61p*H?7}qA1Zp-sCwhA^ zrB^I>`lb42>*~oi^HUf19WQ+nb;|C=L=Sv$G4(>~$#8e8QU;9+4bl%`w|;|G3+M5` zk9|lgX+-zU9iJ)lYl0R9TG1v*v~t{coj1G1v_5?vL^R3pmOVd=$QMwD#FTG zQ6UKriEi@o_?WsJw+zfgp*01oA3A((X*~>|aFH##5hrns9QevL@v}o>6z@%WLFv}R zr1)0ak%RdMSw_usc4E{7S`u%l#TcJS8c44kp39Hp6zd~Ik+EJe-iv+@bjkk$^ElU= zT0z1EXZxC3`TF$hiMmP!$Xo2g{}!`&LWs|ls!>=KaC$D&`5X$ep-!tIXJ{kW_j69o z3hD<-ve!0=X}9M;;!#NC_+7Ue1B5$D{b6_mIfAwfYr z;s!Yo_9sO?S!D;U;|9wg`c zmV9rwZo%bn&*mwl4U-S{cWWxmP|Rj2{u8fKUS$`|ny?Z@TK8Pc!{8PbUw2W5+jV#D zC_7uM-?Ozyu)AHLfa*bbMO+A+=I56f8d9?3E6hn-CGA;ok?_*@@wF5qk8N>Z5hdRz zusd&e^mp|Mn4O$+!+W+iR|~Utf%`ruw%vW^8$$Cc@D@M~ATtf>DNH-Bq}yOLc+6Tk zjx#XyWVJdpaWM>u7^9|c@Ey&5b=Vd^&J}|SE`v_#8vRV6T$le{PN4kjPYeS&4&~3Z zs-yW}C{WPua5_-X@cR5u5>CN5+9aqm6Rec>e7frBuZpIz$9{HVu4$SJl#Y>p$T?xM ztbpWQ*bke2L5(TVy^$00S?%jhDZtYDoiqI=>Ws7UNw0$)W_?)g8D}7icI}dL`H7Zu z%#I5a-QekSnP&cH8|deQP-C zlhpbvkNV=+PdaMrx{#M0?@ZcdsT$Z!i%?0X<$ou52ED49Ln(j7)| zN-DLJZRV2>a|-BotRum@;c;h^Hdg5l_9%fYGzcr%DOxCMiC^cmm=9bzK=Gw4#RP(*WEZMTvrfEmGZ7yb zAkm%H@wKZV?o!uL=$K^oFdvKDrxkgBiXfT94_Lh1&Z4)Vn%{p~F}hcRVBpMF&4*m> zAlUzg1$*?UERJY^*Uh^&A0h(%Xbrl$-a-JGp#QuuYHD1-FY#(lc^v|$zZWpnwpHj- zMG{d{9r~ach40C3rhB5 zm74I9<8vj8^3^Mz&vliF``<{0JEuk*4WkhmosS{IR^YpivWD_A7QFcG}4k zyuLRztNF$%`s-}xph!7hW;!4DnhAS(T+%Az2G`1yn$ue>s2}JMSJ~8I6P~JzAv9qO z^sbjIg4HM#lqayEbpt)uz%{;+JDfVd^xUz{7fQsK!D?D&>19J;}gY{hd;n`Ra!i^Sv6sS>#sP_S7QNk9E_m_5G(! zmS-BIYoas;DOK>g$zn%1*P3$9#02AT zx};w+Npn_UV?swfRm~$dRk5xWlZ+W3usz5)`4NWVxiXcMMS@>C^ z#S!%~!s2`H^sK&W0t=-sHqV?MzEGbMaattE-oF40!!`Sk!tQl@fad>SRU77(@@pxP zW0ORld5Q2Z%dzLmGmn08;{>h$6D2X0*%N}g0$qRYXtEB7Q?;83BF*N$t z645C&>ep2y=@=y;DKqHD`C##YlIP|MuHN#$nnX?JpVkNm87vD%wC$oN1n82zwdt5WKGGce_e?@Tzt4`=9q*`Hd%V-9 zN-!!CDS|mw+1?I6TLc?}kG>*SJj#$s4Ex^a)0Uo;naW; zV*t}9Ke3J-${6}%qu`tYR2JYB7ct4SME>zv~I(+8DmC1S)pgZ{B6cM6y|z zHb;K4(Tonh$%ud3brPjuJArMU+2xiH20*ecKTK_3ZqK7&6hI`z#zKh)Fi*{mL{`V0 zvK-qGAI~Puun*_>{}N;7#N*~WomRk=8j1wQTY=-hDM{b{AtyG|J!>kqzOu;?>p{Fh zPrk96;hk02PF(~zt`=bDvH{6rEBdFL_R;3v>j6(?!P*xhoc0Wa87<7KR*UO9TJ+1_ z)4{J;52dOK8YX_F%H)-9UJ>ckO^!bC&=lg^(?S7oQ#5G~+R?;o&>rQW(&ZB=pn#g4 zcp664YxDo&kXrHNj$X#>qdaZ*prFjWsZ^A_Tr25X3y1ovwjtA>A!v17)HcN-1nvR^ zzKT~r^jn0hvHNllCtKt)-Lq!8BOYi4)~)-Bj?YOl?Aa>)eVs8;Az4cbXY6RoKC`W#|(mu*AUGynUzm`tpyC^;ubMU0-h3f2hmo%Jyho z;7o5jxqB|nz8?EM;b|z1TG_e%qWSB{wD+r)CBEhsEs_|)A{X9G9{7h9z{F^pN9uX1 zu-P9nv_AQ~UDa6+b4z3f;ig%>(I+bnnE)RtDrB3GWW=N4s1K{3!HIv%LqG)NYqoo? zsW{U3nX2`ywp$rA%i^Mvp|>e$sYAnF9fS;6-Syr*!497J`)04ng)e8{+A`^u_K8P6YReXNrR;!O#mxr3ln>T!*XiY_u0$&df z&uz3f+y*^`otM>Nz8OU{*soW&ZE2%&NrN$zgDvu=*jDE zwlQ(l!Y3`4aIrZC-c01wfksAq-BFd=z-8Yw16}7t6wOGU`U0yjzMbkLryo8WEKeXJ zHG~cC8as?OFJ|Lay+wr<*59rNYYKspBu!PWL~?iY|B3~*E*{y_f+dH%uKm4v?yB}Z zZ~v*?5Q(($?{=T5gS7Ms^k;vaHxruw$k@w`{V0)qlDsPSH`Dj?w#9MC*D>DIH)gii zzv8~FhZNWG{$3(*cpr(jDPdLA)_n|wQ-`jHXEP^ck4{Pz1Pc_O|&GoD5Tm4LmmcZ;1&y_uafJtqoH># zo=OG@Bc5nEA>I)sLbLmIp@GdQ7t0vcA&|hm?2C{XiJ}@;Su1&6e_N|!1h4P0Zl#2w zk6b_NMB>IFF3;SPzY$am^ANexf|dr2lNuFrF799~wuZ`~-p9$sl1rbUF|ZUhE5WpU zfJc{QxWzgwZN-uKSXU|)nxL?EUsHBkDUNqT`=s2{vh_LIHW1K(Dm?NkCjvI*P{w)G zj6a%OLy+;FX$Hk0>5~6l2u+snPnrov4CHJK?5VMugT@nr%OQ-L?u586)^ToMa(zQf z(_Se-A|s@{3lB5*Fo(1muT5bDebJWa+q;|9 zrB#NEfxbu!A%A%;iD#@zU181@q!mkv=I^FnPFV(AEYtbDJkI>V+F%A$r4Ze0Ed0F~ z*XoR2A8%gy9B8R*j)}1=Jh{!)a=x?h1-rc-#~78ZE>SwBq$@`NILB7TTq;7?9shz? zzW+S)gn>pVtc`k0dZiEe1ku+PF7&gsM+a})UeTtfmPj^mQ+1{KNi&akp;IMau$0q^zY=c*qyahf=7aeuJmBQMi8R2FRzfiT$Ok5P4%Iu$pMCu|O&* zy#JA|pg3H)1zMYQSKI%C@avIYf0F`P!3OPCG0x>XhZ6HS(v+u=m zJ2WYf-wPwKyB;8QP{RD=Bk+`IZU*K!8X*>+ ziW%~?w@gE;0tqJ9=n6g-xwP2HEC=tGChtbwlu-xTgmEfbI`d+J95xlS0mD~ zBiQ|$d#w9;LSL<#LCWwaw*!VwtrcDL%Y;UX))xQU3qA`;+C0u?5F}-z=Q8zDC-aPTuXHLbmWSmWu1kCM^CHW|hr0&{#yX*dFc2W1=ojFvxk> zQ#~qrMOg0NQPEqZNBd)dS7d#pWp9LP*Aj*cFz+S;fQi4GB`TJ42+fAPN6Q6ySS2HZ ze_b3}8%RO+ECH|lv1Vxp&qH*WiZofxx8pOLSEBlvM)xP_#esq;;yctlv>hR^P_qde6zuf{#TNB5{iXrzBDMC^oVW_VP4Mvod* zUB1hyIykpIRUKD%)-k2m zm?40X`KzqDW2#_;2x@H6?<_ zXJ>)nJ}KLy$E*FHw9N}60Ru;&79L-zL!1j8)eurF=d$}()um>w;;$>{_i1GsJqIo6 zB;9esCLsDX)*RXuQmovnPG={x+e&A$Z9zxyGy`y55RtVJeH2459Q6e&lpPVh)dqXoR8gwh zUfAf_AbuASWuWl5`GG)@A#S}-t0&D|O zxs^yjjS)BXV+m-xy@3lF%ae>0`gqY9KbaR4*dXbagjrR?_6xmv;Dvy$T#dnF>8{0s ze*$`Pg%8Lnyi?yh_UXBi{L5tij}h4s@kJ|huVS z-|*gBaIQ(LFf(QZZhG$#UWZCQKS>I6o;aK=94rvulxDpRqlX-_Y$DJ_mvT3P<=)|E zvldl)xP5bM_wn^?w&d>}p|OU43tbkUq4A7$1#|kxC6E|&^fprk@yo~{IC_6la(_~A zxy&>=s53^Pg%ah2ge2}|Vvw8x4~~lW1!eG{(&!}<&Ml%!eIKS!a8br|WX@4`IkOF9 z#-MUCtI$^A#Fb{+-6IXt@$d-G#J7tQ;^bquv>{>z`J?k+xw@c0WauYxC z)jaqAJX4GMrOSJkqD7J$u-}xLD2RFWE@Y;b@zb6{4Pw6^#2=ywFJ6av7T^^k($v7^ z?)!s6=|-ALBI;ANej={2Iv(OZUvb-n{>SO>#9K|Okk&(4#f@jf;H=*5)VXJpkoOG;qdSaWoi!%E!x_@1@;=*W>z&W76?zK_h zP699-L)K^a^?;sZ>D7Avfx$E}Q}>|m584G}&f>4ysH%?Iqbe>0)ZLyp$=={xt3pi( zPS}vr%9Tm)tB-H)uzJ<b*AD6a9n&0-1jC5Tb-(eN$$pGg25 zNUiD{Y+s3AZGK zAwz&SS}ehmv)rdMz1g9etcQKVX#MZ);jef>tt}e6<^5W8n_gb82!_MMsQGZ1^$IRL(Y2;h_nX5zC~7aW3aE7&q&i+7&J*=YDzcv#fbJ(M4w zbDx7bXi+;y?iDER8emkMVk{;y7v9DU|NpTTfeIglM64cb{7}&NSpPm3@ECn*7_uDs zU-7D96?r*mIoA|xYV&W8-Tp~lR(OdA$J%z|Mog1hKWpwzq;(=wR9+Xjk9VB$Kux{AEG|a4WM3TS%1ugfC@M&xkW2a90MY7 z*qH*9(uBa}?uIAEF|@&O_Ll_7U-_g z*s=NHe`Oqj9E+^_Axg}_nG1%*P-SWz5Cg^(6peyMW;7C z;kqRg|G}@~|33#-_Jc%;!T|2Q`GoNr!xr+bzL+cUe(JktU`KvYK1=6MzGo(RSLYIY zy*8(LY0iUkEtLN)MwN@W^$j`UkNW+5Ejh=fYx}%u+2Z>N;m@hdJ4G1LRQ;;AxU$K_ znfYC9OLpaIMU@0S=Grp{$rC-8vwU4%-#qed8)L2nEL9tK+B}?Xb-@<@!Ok8g85~4| z8MX`N#LAGNftykptLds>TlT!zw-;hVyfRdvs?tQCD{oHjAm8W{@>E2}xm(n~mBPvD z1iv>8xC!D{YoSDOF%;1IPVgE%q)+6-y6a;H>^IE7eS+=!q}_TCGsE?J>hVu{2bV?w z8sZk{Ij?4?M{Crr{?2rH?){mv41?x@Q>c8=ZIejiohJSLI~sGF8a1<0uK99z&rYiL3<=q*vWv%WmcQd+K@7f=AHIs@`lp9mf6-j zUqPGML7F_P?lg}YRvyoiIoq@&>Mf1<2m17JO zI$Hx3ts6?~$Tk+@QwCX*TW*T9)_ip5(fTc$Efv|sS`$eJ45P2I8*M)EYyS(V4gQf) z*VVbqML-y4D$FY`mD`t!fp1Q3k5gtns|S~q@(^Trt6IlVV8&G z_topdN-9Fad+sApf@@Sv+kk`}X)Q?3Q7F%io4dCR>=UE3S?GcA)uS)3HLuq=j#8GG>vEb9>V4SAxg05&TV=vF}+Jo?q zD>7ow&zSZ3FN7orb~^q4xS)+}{G#(6qxZ~Wb$#H<^9ZF~&OU1@cE7a-pXUa{@5>A2w2IpWc71RmT9fdH`e_~RA)(Sd&l^YPyb+Nq@4dZst4Ldwt>rbWE`Dn zFYoN=)BH%QFK}gYCJ!$#tUG7n&tA>9p2-htP&Kt$?VVzVa9BRC2?((0@-3|%M0wAvl0mXb6lBy%{dzF@ceU3O=Qur*Q3QT5W zHYyyBO_-Gv3{(x9Zea_8JN}Yhx9in4G;CtuH7C@e_52?9bzGK7@i6IZ!1WmH@KCX9 zz8tZ(PZd2ozep*XqNK94r{{D!qqma7ToV^OBw|px>Nf1JlDfI8;fB2<3X9@mW1r?5 z8-7+1m40H)TE{?>eV3;fgS2WLQcyW3E^Rgrbav)hhER_s-0fBYWaJLkv#<^A61qno zYo5|LT&nIAK9>8>lqQiA5e6?L8Nn25TxBUqe4B;$o8 zDmt!>Zh=U75Mm3L6U2IsO?kqzZY{;jjt%2~`4Ihh_%L~rm>$_bH^j<&O{W|G`&m&s^+ zekx&T1$hNDl8?#s&T4}q%rL#K!QC9~e#5G2A(tL7oMJRwhl?__YM6Vm%;7QJoF@>b zg07hF4?pA00iSZ5c`LW|ANQd*p$|5vQ;@-@kB~da2$?{^NmF7&FWU2J~A{}yX@J`qkEmL)bhzyA;`vgooeQxL(L)nF~Yy6zL`Quy1XCnop%bCz01e~ z`a;-(vmtPG#6q`2#B9NYh#S`YBnbJiTq~QBo^IUSe~@C-bf%&#z7B9jgn|#!X>sAQ zXPAa2hkVH`zAkbvC=nwr9+tbP0+C0>Urtk`UpCONJI*5daGR)~u&K?!PLb z!!_U^82GL`w+R`w82g8|8_3VILIP<`F^KL@^HvntNTd|b$lrh2A>HL@NY3Z4v=G89 z50(OsV3SX`E}t{ybVh2XT}0J?CS{IA$M*3mQ+pncNES#y%{}<<$S?8N;SY%N zm&u4Zq5ogh+0>5ll(8Yf4E(1fEseZqQmcnX=Sxm;@Kld&9d-(OC1pt1s$FmvSTOl< zHKG2V?<=X`q^2lGa3SPp`|CMiqqjL;=IHoM zg=3(Fnh|sLizfv(1;MBJ6}>aUv;4>&?S{X#B?&}&8X!eCfVhF1rJ@IegU9gP(I<8@ z0F*#y=@%&WEa13wi?9^mWj?pZ(U5p%qPPsc@3zaJwCm>(W3%-aQxyDa+?~G-{9~{M z4@DMr&|*a0L@gP*DT7t2Hes{bLHUmK`~o?FsUetzY6$n^DIVk=B(S$C_y8cRCz^iH8qbs1-ey@87g3cHmU8K=LXVv* zCM~|59fwfh-6FTHW}4QxYZnrWE;7+e)6LDZ)^BhfyLzCv+bhK#v% z)yUJ*;rTm(yuiDf*Kx_tHAQ?C>)0Zw`&|Q^ls3~Qap|^v?z)RR$4_!XD&C*E_>Bq( zGtOWA=f${$F&T6=5RSAD{07&X%S<+@tAL(((LE;SeXutbyK#S?cchbfGPT-3y9mXL zyeUn85LuzxpbznFk_THP>t&VeYR&wpO+BwJDf_w%EX1)|7P94peZ;P+w3990hw>r0 zN|e2*RHW_V^k*Bbz_>PtrooV29vrzrB>ipKYZ>B|e$>pIrzy;o`|i9~NZ{GlbDm;7 zE$Okn{-(`@?*iTGyd!cWJYw7U!brY^v3}JWT#g~#)Y&7lt^Mub&W#1#nr`b+VnkZ_HxqU zkhdJWJMC!8S^;X!SGvTGci+Guj= zyw=P7l#}Vh_mtg6= zSf4BEhMliXfNd_4xAyvbd!WY!J7wf&b;FJILNNYeX-`lVcyIpB2Ey*MzjMYk1_DFZ zyyofZ-QnFpwdXR>&_=gc?$1hrB@L%G;Pt9N_@3O;dM{z)hQ|7|#P4yDO6DlWC!u~_ zzoSJswv?$%J-W5i7IrWdOPyMC>;POcNr}d&`^H;aqg|h?wDuRCKXU+QP>U1JvsA|a z2M8d-@Q7gz#i1+%M=-6V6ypF(vI%*vb}eBdTLId@2c@ ze;FRp%zJ>-YLcCkrX+ZdzOW>V5&s7<$fBN4M8N1rT8i^&Wus?nLAO~SD+^-psK94{ zNCeX;OTFktIL7W0=&x7stg;fkX7=QVK~~+3D>%wBaw9dkcF+)X2L^(v_0xFO!;f$6 zNoaY74<5uTn^-hFn{Og$GTqlTS4ii(QzSkI0m3s*z*|RA+1ujbc2#$v--K0uX?@*Y z7YN#L%uwFzqn4UeWcO?{(Dw?IfHOB{{D(d9yL!d|EU15x@;F4P#QbEmQ7whAd)^OEi^aI{&jQfum90z zG-)f}sg_Qzr~6Y6LbU$K$09}Wcf zk(>8JA}&qHOoOBsobSfO62D~&VwDbjriZ?)`codDdxyHHH`CCnd7#!szw|LxFMq}k zk>t`3t@2W*pvtvSGdjh?!$aNS_86=_WNGbt$)FRW<~uze(gwZQ>)6GQV0;1cQuFo$ z`a&P=8GvXVH*MYhvp2Aarmjxr@F&kpW$rv}mF%>g_>zsBm8Uez9lqeySm|h%>ok_S zqFbb^`ut#zuXlufod$ z96(*{jBfb3W~+p}Xdkm-5ndFO>Ke>#%355b5DRV9#X{TDtBK@(%f+S0kJXS2QXsE^ zf5Ss1k-I&|es9#(2L!StevCbqQPyK)RA-CXq3x4Hw zb0OuvdiO~wTG@G$MZIx%8xEp(^bgKi7M|e zE~EqcnpQ!`=*(B$N+}e4G<%8eCt>K$9SyR0*D}BmaSnb(eVFRz!lrG(MfjYooxrqL zU{7*-^xC(qO4r`0Or_1rqAp_$nAhO9UR~lWKTF`HAH=&5TvV7dCs4?17(+vIm|x<; zuv>87gI21mx&8r+?t*W_$WvsXdTIpx>})1FlK+=I#h$aS4y}< z0`^9x0a(|nrBA>I(i}L(b+i^_hqqonA{vSDk6lPvGKLqk7ciPUVwX{(&@P0AuJg@m zJX~lKQ%jpNqPl9rs%V`?OM<5q>`oa!az;$E;dH&r+*N0Xkf06eh;>VQ9*H=ySB;W2~#g8?YH)(0egSz{okL3J-m7nn}qCqB<_I0 z{*c;dfX@%oPGEJpiJOvtX4X&#uj2i9nM&5PVqm3z|KUZVKzE#3lhFRCGZShY6^yx_ zW_;HC)l}X2g+Ec8Z*TU;R(H;(kmVCjSQ{x{^69kpRJ`yjP*-u>yzQ@%!{x$$%;#Y% zO0DH6u|VC(#5>#aFj&U4eSBttQuQPl1-HCg0%306&B z1-5>6V$?_W@GxKoHnH;7bMmb6XA1l|Y5Q|WiW!4lw+g-YS6>Hje0y zZ+Ef7e4aJ(oPMRWuu5CUIsrihUd0CDhJ(T&4?;=?c@Poyl_lO5OJnMfaS${I2yXC0 zSW(XrLF|g^A=5ar^YX~{ub|%vGKH>?j(cCAfkjh^SSK67I8bPbFrizt8jSwej0OpHcKtk6Z>4s{G{{*aY^Vgt;(*+jAhdDQ zqGmAra~ON!P)}0re-#mcGcBgViKQHoXa2p<1Ecr4%dVK!$%*ogq?H$Jlee3h9p5da zzils_DvskbfXqw|P!4-1@oy*sDHvvwqu0@)NVXdakl|W4n=&|~Vt@}iMtRp1vO(g# zj}6*=pk1T#;)LmaPcgcQThT}?Ti+tTVH+5(TZm}z=soS(!Fs^vM@xLgE(trX+@OVd zsv41!&U{CiaG^Lv?NXCORO{{mAHiTq%=pBZDeVaQGEGTZAzAJ2Hs!JMDDaTA8{=Za+E1)vIMkp(xv^u8VEf(eBT6OY1 z=7HBV{}8|)$eqxHXw2C3veJ5Q@P0jLlwbIAC5sM3!4QHAr|Z|6XDwFq7>{2LDH8LZ z>*DVINnmXivhie6yDo!#F^I6Z2Op9NUXG33PiqmRm1C6!LwOxx9{maj zuv9E9a8|uoBJU9GqxO8lg#ZoEb7my?NejVbGF40}*ST*pKNWl+ppNJgxN*OHhim_* zisz-#&A3neO^hJN>3=tK$T7^P*%^jwFNF8D=+f`p4U!8rz#R+R%rojE4No52R17Mo zABx?mC4|eA=6{ooDTSX5le&K}xU@cxSom^cu`&o)Q)?2&c)o!fmO&KDrtytAztSwy z+&#q!^GCk=C5Tx60k6R13=dkl+p+>lw*9}cpdBBgX)&|vwsH(;K^07Et5( zzujnK46`CqhIMg_&%MWnhZ7}JqF{AD&>c2sdZ;)d;aZREej?5lUb{b zH^Zh4y6|;R@_em#9NW0Bs6%f!+79fc(>GU8`MP!Vd-|>L+rd$g>6o~e1b(_f;{rb8 z1h!EQ9(AH*XF0xFt&S65_ii{{qlN0i2?J6c@6|4>o5gkWPwbh>Y|LSDTdyaP9$FS{ zyjX<)CE<9Z*BEZ()f;OZ$>;hoYkz+}pa1#v+ET}fU9TZGjuH^D^RLNn0VgNqjv)MX zRM<6hlz+y z7b|yGKZIm&UV#YICy)vSavulioiV|QISz7wTg%xjzT;tNB6biguE!%Ta0~Zu2Fpns zETFheh?T20#5RhpQ7phrQHpHWbt}WRZvfF`v5a+jZ>?d1egTu>k=VVi)0X?$pHklB4KE((zxS)Q3UXlwzJU+_C$!+Nj8e&Q+6fwPNU0@)|63lTlB-8!7` z^C<>P)ks)iakuDsiR&^Y#k@6__OwHn@F36@PA^;f+&e@KSspAzFXl;zhFlbwRiM*m zvrm1UKtaX>)NaWLA2>9+{Xa=5tr!Eqe;rU`m|fg4cEA*!S`1R|R22_)0x?QiBqx|r z4qWJOZO_Uby#E2dZLrA5$L-84c6&qoml}!ZBW@@Xn%rVc#{^0876306F`io`JY%m% z2=hz>Vz%<5>+IFAHoAef17XbcK{v=ekRbvne$L1OVi!wy9f6N6ybit}6*h6rO=OoO zU+3$o<~7^zIO%;9KD&FZ@xA9xV25#~Xf6UrhnbOb#=M_fa46H09s0_>?Yc%QXxCux#(opiCYpitkFbLu zheE{I7Z%Uf;Yi+TF(;C)!=J_s;vGe*(6y#N-0vaufK(CBtf{uclNxT=;Hy~gjb%-| z*MGpdWKX+Jd$!lu$IxWP(NAX1LNw+Xu{qP9ZbV6WZ;tIeRnA*Tu<&L(vnl~6gGyMp z{{`UeJqLL!`Cc<^pY8-#`A*gdNt9WBem9``NzCM{n%Okni@1I=^aR${2s5Pn&e;Oz z&X9lr#&wxWm+bHdHPy5#9`?_ModfMQV5R`>)nJcv-zVv!pAXI&*3D7KKi?Mcm16^+PrQ*r5Wnt9A?8qyQjaiYSZ11b(R1RM7MNMfUiyJNd?io`B4-H%^tz* z(=)jxEBWX(>pdQ6+N$jPf>5F5tEqRbV*v9$&n!<>WKSkEJi0Ms-2m|P52RSnjLd5A zA+0#$z*jXScRKx7o74u!q?iAPu{VKcbB)`6_ilIT-l}%16-s5dTH2Z^YD(JLXscy+ zGQ}K2D@e^#BvnN#NL39rrK+VUX^@zsikRn`OAH~#lEjdZa31?T?>XUtWm{r}@RnH+ijE1te#QS9&AlJ}x{PP7)%gm-O- zEH>A&N5us28a=)gO1*WL#DO8UEBzG@lhL)V{Q}#Zm*b}KGziyz%p4r1mTI=`(0&c@ zCpa5njKH?KRJ{PMEJN8Hb<0{5!?Im@zjMN`(~Kw7Nvdg!FxzOL`YJ-xtOw%}J zZp%>0*QX=g8|EGBIVm-RM5@rgPH2Y`dr2B%Ty}8#;_~n*wy-j*{)e?_ChoMb*K%_d zrM;2~hM&>ez98tN8tA|d?$A1|7gYau&Jg#0m@k^&0Y1OE^xrZ={%^9cVrsazjBAZ) zOOjo8LwWd2XzqUW6EQ9fHuva4+Q7ezfN7K_z)1bVTsp?l#2xhg{ z@YKkW958s~Ca>PaJ~;D;6bGQlVgh(oSWc3#VLC3GC`o&k1le4t^NU+Sb(|(d)g#T} zMORr{?fz$pyEig;Ac^!Uc&0O36P;+d^RHg^FQ!owtDBBm4;7t3mN@$Ycw^Zu@4I%q z`5)1c^=yPjcwSZu>m=mJ?vNUSwXBb5iKRU5d9GF=%r$MMpE^RA>46-jJ`l1BE z?uA1L@rTL`z`gSr6IWPMb*z$dOfz1arIEt7 zu!SUjJ3sMImTMIgHZT(~z{%^QJdo6KwC(T_4cWf0Dy?*Vh=F?3x^W8XW6?|CmJgHz z&ajDxjehbDO0!3hFO_$(tO(-s(#AsgmviNfh2lHh5P{e2>*{eDySJ77woSxMY8*UP z${jotM%-6Nl*->=y%U{c{ci@o87Fk(e@%a*`P98VPt4eQ{FiU-0&bcy>wcb^4?8^c zjIxS5WmqQ@i*vj#jntGJV?KwP~$X5` zLost&dBm#JQMw;LaxHMpWsV_a7i6i<9@5Ced}3d;5UFAC+Z^LCJ`~yU7aPyjom>hL z^GDepV)7w$9f>k}Y5PTM)ciU(@~0&$LgX|$4x2gopo+cN;9>hR?(JzmwbCo`0d_$_ zd3#R^^=fLQZG#rK*LKb_OCcTRjy%cU3h-siR*4tl=M+uTWnl>;M=nAO|QX-O$UXzP#+CR zd-Bk>|F#GJbiYsxFKyt8Y z#A!%s#X#d>iyue9F5e^<)W_dSg>+7N@Lac<22dve5J)+Rd?V@I<*V;h_45&os5|a3 z_#@s2#+1=oY&PO--^~TI*?Ig>bFcYYK_<5E(qL!M1nrEZ+4&JVODKBgj_N@bbYsQ0 zR|&UQ#jk(^M?w9Ky_i(6q*%82Y zoa`qF;fH(r9ft2j*V9zZb%I8&NbtX!#d@^1zo<_ZXX=2b+YLi1x<6L^x#V$s_HJ=p z9tFYr7*Jc?7uQ;J4`jQX6p*t>mD=5(w|wqZqETCcajnX3n!#LJU_q5OHL7ZUEuuc2 zR7x2;nKdB=*fvlVsINP~QFLI`k*4eXWYjWC?}v?jSWr198>17A34^L8nAgl5#8n% zea4>JRMo>|O073<-_8Kc1YYiClA@@Y4Zykh`E=m$;;r$0)uagT421nwCJXhiY>cDV zN9m!-CO+$ceE8C)<#9^8JL4?l%g9s&6D&!oe4)iQh~4*sF}weCW5y(AZs%vY3qioi z)N_a^V7|oz0g1F`;tGe=8D$phHxmQwqyf^1^1@&7sxMB~6;8$G>wPz~d!VwhGYzT> zN;4~AWyiab>YKf43Ni~J79uap*`oIRV(vxou1(J{P8dSb_}H!`32c$?$;@=iS9|BB z>=niU^-`ndj@aiOV44SCw{Gr5lFEO z_?(l2=TJR5MZ|pu#~U1-Pe(&X|4gw-XYYr%Mk@sx^vus?tVVd1(kWb4Zd@0T`8;apY)}8bC-;6y3TmK=5*%9_9c}_n2Cc2X&nf+N52U=;)C%SAV05@mp+}dlmY(2E zucEzK^u4g47kj)!Ip|JagYj*Gy$){Jga`x>0Jw@t$2d^_tB3Aj9Xi-uo10$f_pf

8LZzR59*VlB~BZ!XhUHNNf-swDE0}aGkUYuH;{(v<2hmxy#)Qdlpesqhz z_e7j+pXVL6(*zfU?v*!0OgCOgC#T6NfF;+?r)m=1Mi^s@hoI zH)T;X3G9OIsL*OqS!qUK$jO6X2wy^gyI(RJqFisr*9NCcS&F)HU@Svf$u=2uaq9;o zgF=0SoJ8h6{-I5bG}2smUuJHfw(3c||fm8#mE60KU7LWs{H++@%l zgF>2sAk+$YMaiu=MR8379Wilv<=%b|p^mKjGN--#huQ0LV2Y(&BvJoY`SW8n#nIVar!y6~*X-NQOE9}shJ}q7 z4Tg_?rX1b<=wdo+jRR;+oZDuF8drw^B7I}V9gi7lZjI_a&w9z#TEF>rGt>uzx_F5+ z(pwv9xIHoWGH)H@+;qHdqqrAwd%48Xrmv+5*R#G3Xyn?A?s=KpijYM(y9KBz;fU)& z0(?*jr8=6GxuW(9ZC0$*)N0Lza z8$Z@mLJADh2xd7~r4^{rU$5SBi%^nA>8UoC_q2Q`o?)a>Wa3af;XBYd-LgfYQ$S%@ zGOH8KIvfpVf(Ac(PW*M~>#vf3lUj@&OhfAjHI7RBr93_U%Jt4OSMlHdsBW~}8}VMh zID9e7ong2C)oo`Jtt_nuH@Ws@CUdmv-QUT@dRJWWM(^Z6rJx1WW!abKhe;r~TV0#Y zI-sK@zH7ZfE zeK;b#8#huVi2uuye?~r;`8b2&KC!PZ%EHKI?5Amhd zFHRWp${YU#q6wsvfE-a}-l_DX2DUBD9n{`jM)Ht!{O&UORn-9uH{-k7N$>Nj2LDo&=ow@hC_AkVfK;~AKHu%xX zvjX{6u-i|x;WggfsnblOH|)SekD$31n(dKp?JZJ=()T7UyWKX*&9&C`3>2C!MUjH1 z(IsU91R(UMLw6!;0v~|h4~&j;=o0w&yvHo*Y^7C>pn*bWMg{<|e6j&)qobY1(*^`; zyXqDbw@);Gjv5AgT7)tD0JUGEr`+KMe2!q_S#KF)7w&zs)AExi*R+t4l)KyHh-$i3 zu6bE|Z)k7Z9^Yo@{%-DFPQ7rz6WvqN{50ecn)!i zpC&lsBg+A;2PEm^AB!4494qN)0F<}h+zr_g+k5SgG`CLO!dOuP+|z z95U>dy+S0OFKvQip$`WSc<($d+Hz==sqLFSA$tQA#L&5q z4hrER7yBFHGpc5>x+4D+WCS(dR*` z4D5rNJJfffE5y~|e_WH5N4_6QXM9Sj4m3{}iy3`w_`x9`R=HWx)Z$HlCb4a`=-{zX z<`WOsB6>J3PYL?sJmmYgUw~^ksa1=c>+fN+Av^jtSvHUKVMVwEeY5d>gUCdxHEf01 z@{_ZXl0f~?eWc_6VA&)b>Cn1CxM9$x;wtdpuT8hFc=*nJx^)OLJ$nFo`gLB4*62Np zj4E2gkG}kE*v=Ot2)T;Rv70I0c}VHR&Dbdbzn;``Q3b&M3=0VjiKB7@cKZ+g?%(zFqzT`+&wRjs7J~3KQLd-4> zNy_4{g$BoVScjbl-8f(B<$2&WuceIFbH5zbJe42%krfK~*>durkDI%{nx}uzKRXoi zXUJmms<^To+_dXbv{;xwtY_M|J-ykj`zlMnuW^;se}iIdrkiUEdwZ-lY0OuT<^Olx zQTDddR>q6qsfEfZ-x~Rf@?`ZIjoqT;cmTfKs8%S`y4 zJy&}(o=O~2d+)sO1Ap5H%14FJXLo)Cq5BqmCtN6o^Sk;egz(2u$ZIh7MX@q( z3cNc-LYr7nc#j^C4yI%_PNz~<>DyDEhK>78{qC<>gIl@T2*zzs3oD^Fv=4oAK78&mH9aq?#5fEdqj8}){i)SG`^UhZ0ZbURej{dGac zZ{HS}qHgPzn5c2ezyu8+S2iC)4&XDgUwios&k5-J$G*%5Fee6DXsjdsv zkkAo?Rc*nR@P^5?^#H)=~Y!hV-}#V1=31z!WB3F6Cxt_c|y+DOB$mR_fFi{k_(1 zZYRA)a_QL_&^oiV{7|Q%xDn)@olD=(mA{MH*1U`Ew6>*aE^P>*0$M)Xb`QT2+#Ee% zkx?nTx!ePiJF?;La)uuXR)C-%S9CllLh~)yS4S%S@r1fEhd@k_bdRi!7T+vZ7EU$T?+%xHEaw z(*#US?no9Ehq-h{Ns0Ro7H5uP!6~*Ggi-VZPCky6XR+2;uo_#eJ^jK)Bm&(PyyID4 z`qm(RySpE{vEGWOY@3;7p>OV;+MNd1a_DY&-bf;Td#~93`%8UB&XS?Bc|iHki<2!w z){vp?Ltz2*cI{TYU4n^Pj?KDX*Y$|@?)C18)OY%vSDKH*>JtSio5fS~rW;;Glep36 z(nt8FdrK#`ovaYm8g?p)Lf++u3ez27p6!7dL_6Bztf+u}lj@?(`fzmboUSHUb=q)F zkEC5=*k-->4L(203#IMY9Tg$@ep7C?sH02O#!CbO2+}D~vuK3g3+}%2sTE*XWFk^o zAM~wvejXeJk1q7C8wVHfmO;@CntP4+y4|2u0|Ge#?xO+q3tj1m-}!NZ_yFZMjN9)D zaf9xRyg2Q?GO_9F>z_K1EY;ZjOJj2yFbGf0rRAupc;UC#z2P+F!YoCW`^6Rd;~yT8e{y%4Hb#*zQmvd z{z*HA)29bQx9@nerH{rIG5-#&xSFtXavME@^i=4L&KsNK;Q6+B%v zeYgCOEbkb;hDmK~>+iIiiingkJ8s)UVEb0$w_if7J$S3i5bji+wX6r!2Ay2<#^td` z@XaT|eVZF@)H(ci>#RUwVo3&LO&P8?b$q#O$=_Q6rB``xi}kw<_W`@KDO(;ddrT1r zp)3J~)Ou=qDI-VG$iCxJ8j?AvkK>_m8GTJ7?><#+&0}hmO^+T8u4e1}$ePdwl-=WV zaFeDeiUsqZEc~8F5QU<9eeSbo&JW@E^j0%UP^GFwO?KlD>B{WXdqc361aC_1cbziC z!3*Y))We%T*Yt+61HMULV4RBWz5aRgS&{6J{Je}bcdZ{|9m|=3oFVwA_`pwa+lDDxOz3KY*_zW|OVq3E?kVT}Mmbir}+ks_Km<*P3z)o#*hvF(rT?OPai zU+4{4iFci`@C1zWinaRjBESQ{{FGF$X&j#WOd60Tj7%gNtLb+n&t2gbG0ujq-$Km5 z*s3+wTOJnZ&Cmer@fPyNjpC2x&GnBx@fx`Lo8!~jTOK*YhwK&0$KCY@iM*i(`DH|P zHR#6UCymbww7#fbP$B*L#bQnE>87l0{nG|hLdDY?2hjRV8KHxx_o``pua-P_@qH0^ zY5^d~l?pDyUZ_YEyo^_n6!6wMDkCX7BYRtHckv}q1|0M(yW-ut(={163w`M1Yt`Sw zd*%HQf{Q@Ti9D$6&|V(-VtX-!bdkmza;>$|r)@_{a_-jkAb*riZ#mW8;=d<*BQZs^ z51j)b3A2oxz=>eDtpx~vU0>^b4l^o@k;)D}2iIFa_xxSf0oN@n zIKstGU&65;YLs9mlmoTSGT&Q1-Y#mAUir;FR4zbUv+hrVWhlCh*mShR3Vkx_MUcgEC2I=+Lh^qd)NAxY5&_E9Eq_WyJ#bWx$Ya;Au`RxxV}c z)dKrV?RBbuy7Js{Pd$#$uLcUW@AX4ALrXx}kA)zyT^1{IC_C(c-k~}z*K_7qa+*wt zZ7EGzOieuJY+VYFyG+}XS#oe)a$>nmS?{9); zV{RVqVJ;1ovPeW2{QDl9@m`}Yqh0KVF)_wqN)9u2R!rpHM%)CqB`P>d%WZi}rDqrn zd|1^+lQK2&g7{=zGPpJ%`2B`SQFOOd-7+yH^9M6hH~H64Z3-kd;uT6a#hCGuQa2zg z{R?WBjfYN_WxcotX=*b~7iyk!DMt9Hx+hL>evDSltJ*gfV%DAJa@gOmC?u16y2W+A zsmOkn9rNrzblLL&zzgu)&q<0foRsaK#7T^nfq=5Wy24F*hDa^Mw9C@r$O;poGqUH&4_`dXV#%^Kuot)NV^@ID1(fReQ*bQNGf!y9nXNQ0UFjF|nnXKaEcH&Jq z$OyUSd{7zmQ5xtc$PxP8rbgrKZC4}ld4xEzzHXvPKtEV&7Ary&wF)6kZm}M?C{>AI z=rSI9l^uhn&1OD}b~mMRt}J|QUTq4M*)c>dG}V{WvqAFc`R*-rFSf3)@e=iJ`g>%Y zoU@VmV>TT&F2A6qFMVXCzN@9XC3dapPq2}=)!*Vw?w_#LQ7J@0Y$eS%dzuJ_8liJnjKdDvbv{r=aL&mLl)QfG$b|R=Lgd8Jj(_S8<<1ZJz8Lx(B`?-Hl3!syy)(X!Y?5ohs^aW zu|1aPfxg|niRlQ~O-#ndT8Tc{1H)P#zUbyO-xN;WD>IQ7wHq}bPmb|Uu<03)&8cZ> z2Jm=6WFQkC#0%S7DTMpd3^-1GUJFFL#mQ-Gxj| z$8m_qG7`Y+#@;>kl8)z$HV(`1MW`(nVFxhZwR()JYAd=-OfqUM8(Tz3yZ!PK(ZG)lC0R$7zvFQ;Kcx$J?(S+=eO2Jz! zPWE^x+8z7DZgPg4T_QnV5S=AsCa4*J?P==540oy=#><~1Q*A|x}V5|>7X-;=h9DW{<5 zQF|b6RN%FVI9=0o+$8>fGub-7IXk;QP&zkbz}WJna=>KnXYTN#0O6xWZ0WW*)Lj@! zz!j}5!c=-k^@>|b(UB7GyF7ig*DTM7lX?YcYGM}bru{gjEDqZ^MLk8B z_GW7+2?)&61 z<-s(rNTLAc;L)V>zIS(}B{g!f%LHyrITeQo!WT8AoZ6Uv`$BCY-nD=M>f7F8i{Wq0 zG#Oug*6cqzExhp;0QoiExewZoT<2L#j6BId|O3OfGWlL*d+*=RIL2J*@ zF-z%EmEw3@{k?`zpl04T6Bi`*BT;>Lx9`3*ch>SK4BWf#dvCCJGX$9{(9JHPuSkTr){pOcJ!gkyo5jFQeeGJ(Rf^~l z0ycrTqK=*LF_RwPJ>Kz5hf%nN8H+Lz_aDP{{3ahL zT*k&N50;~6;LB@fdSaE=v9(o)C9OMUY_V=)7sjp^Q)5?470`De+V7V{@9tZg35u^? z7eaJok={iN88nZqB5RKCe0of0e9$-*l6eoaeApbunco5f%>ZT@Z3K>cxLZR1x4`9G zFZQQQjz!pa(9(6}m$2QOPPo|7MB{*-(bb&(IFYqwcBGBiv?JYDo0xBe4Eq4vv83Cx z(}tuVkHs+8X$9Q=O_u z!tJ@g!gf|_VG*0-$t;)?78sob(ht+66DitMAg;g*YT&7|ymW7*Y zFjSpKI@(t#8m)oIv>$KCzytc zSnqXV>-RO_Y04XKn#?ZF+r$fNPwWD& z7;r2e^?8K$?JMoIRRdHO*@@EtQgkscB1T^Z7awLYEhGp3LS8{?L^Y)s*-Si&Hs1Zo z^fhsP9yTbBi}W?Geu&12{Ktb|x zPD6e4V^PKqbU|CSCr0Q=<6O50yXAEiC%8}q4mdL7A8up`qYYi1<#^v~Er-i(UBjX7 zA_0lf8sKNDJ5B2R%}eW8(k9g;-F`6W0LIVjA#ISp=Cd33WliS3cd3~O##v5t)8RMY z2S~@p6g!1S{!1g1g)1fP=2FxXa6{#zlx%`&~zjJQRlci*@l484;#9v9EA8(f@G3nEJx+r)w&y5Mre|srcYP7UNg-5 zZ&`SIunN6j0RCMTrY%CffS>;HdL-r~9=BlN`8f#!_+tU_Y<`69_`E(^^6qgn!kx7~ zI~i!x4OvwZ1n2Y5?C@P9&7P?v#7~m*HeCX=W5G<36yKGFjU{zxlxC_W>II?Qv3wHe zbM1_3L8xW#!>!SCMxj+xrwiki^0&Ui=y)ut z@`iJWWe_<5??;CV1BY+iwh^HyWH%#8KnvzKgCMmMLT-x{rIgdpsl|sLvYQ>?U&2k4 zhDlqGo=sJ0oEjz%3cSn%sb_f59X9@W+qN{T5kg-}Qq-;>JqXr=G+=R@cu%L|kBd>J z;9A8ZTqx6FgLMN6DVeRXXn9~aeM~6p8NSHrxjoRahi)>#zjFiU@+TiaFUg|YBP45E zerBJK|IQ>f{UNJevj&5n9U825lgrp-4(ub8#-p5vwWQu`D`MSW#o}^yCk3IFO@A?} z4|`rK=SCfjwX5%r%;oPMz#q=Ia0HeNw9ecW8&P)DLdmtiX zC^8h?(;cYZvfkEizsoY{BGFvl)bii+|9rB9#ThTbYR}T9A9SOc9naWB6UrgLfiZMP zI1?Kyx1SoZkVCA6nar;sF*~GsQAI<1^+~EIzmESk1ofaj&zm_Bx~h*VeL?dKhQ!LD z=62-mm@)ThX9|e(J5I;gkCrcMp7Mk$q6B-bpmasWe*)|8A3 z^H^gGBZCaKWauwPNVlYFqm5OZWLTbe=JuTsoZj|B!C``wI25e0)q*j781ARs_IS_#Q2KtybV|d~dPOpu97AuGg8~5KNOF{Vj^pztpgv zX|hOKBB16$wJ72lQ&078lR4UpE@GRcLaxiI-Z2?yQUcy-YH1W@mp)tdo!U?wuqAU6 z;VX<@a~n2ZwRyGoWTeIj!&eDo^_%h6Te~q!jA}gyvmteDzpA-s^ATwh+n{;LxoiXm zg!~3n!!sz^URb>m8?Df==$fUQ+h&EBSU8$lul>$51pR(2Lneob!)-@fhA>|H{@q1d z0{rf)fB-lp-X<6@$Vwto7ze8t@G{^=0_IYVW-F(~NZFnR{KfC$L_)i54#xX>KKI^P zDHjR$$S7ay3@+9`>AAVjnOPHVNVD*~yQ2vW+xy49z);CXu z378A4+ZjiQ`*H6#D_z$94Egq$FU!z*;CY@w$$$II$f7m)&$QJKi!-E_oVY!DpK?KL zSnA6WqEw=%2@uh}c@z!fYz_bi)K@~pi+(G*f#0(m9CST9f9?-2IOZ;+X$m^yCzutK zYUBL#kmYFGv`PVpPAUrg1*y zro%&GEL&Rdpvw04Sz!|4ZhLtt6*3mS){*=RV)&Me?qo!r;ZO>_di(F|12ysh#qmRvGcu9$ES06nOH^055>2NyTCr(gL~7< z#I+F@hjp+CsGA(KJ~n1vr)?Cvyb$X3_YkClGTSU>FTL3oJDNf9eN~^z*ENp3H8R+2r4U6j}Dkf=Km)!K$1V+>ak$bpoc+Z?5jwnod=|*%WW8&ja2W zl_4<9{-I{^Pa;~KZdNDuGbhZ_D(CV*Q3`@>T%xAHoVd9%|@cN}@bLDP; z#m6v(>{x+8A-P`|8D!3zXSrr1h--Lte7SVM7pz#F;#V}M0SqAp&7u|2)Oha=BrmSc z+p-a>l%+QqNw^zoweAi#LYS~D`NbT1(~%{hn!Lp^L0#PH!8-i<#jb;tyrP9uLJAiK z!Mc6B^h?52z|lx)(7)x02e#{9dHel^0Dc@bJNjPDHfZ{V=VkWS&G3JGY-j$43kE29 zN^?C`OE-UK{mfr)=DaWh*?lCQu_pqbwnQ^&9~eIw7F_5#>_iODrJ(`UB{1j$%1@dE z0dm-JQW$s2h6`n=X$jC~nUR>JW^zyihMKdqN2|Sh>*OcTlz!97?BZ^iyMP9+ZCZ}sI1s(%Wk-6SBOn7yY3>?8X zDqNA0tiO4uxXJH_7)M`w8`u{gi5(SCbnOn1p9vz^6o%#=>N5|P{_lqRHckSXVo>E? z*7K}?cm5T}C0gE(B^{m%KGn$17M@#r*WKTxG5#Ol{C^Jz|Ml4wjKF9)Jp}ne#+z$t-fwRHo{d}N3M^buPlWutH{RbL|9{sL7xhWW&ti6q2V#)OZ7;YYC2%D@ z%IfJohG5!tnbmIcg1ptAJ~m8|j#+5xRQ;15I~@S>H;GgtaTT6Y8|H#hao{2b>hY7-jZ!NYkei_v1tIv#Kbc8>gR>%&Zt;*rnM zNWu%;VlSucgWZglG6d@dHDIUc$#fv(KUn~`e3E3dHpiHcCN?Jk9Be-!peb9|iD4vzR>jq(vw-^!4UC;o2S#Y zAxPoo_XBmjls(p;W=|9^Y@$wJ58V{9o+ zLxfWG!Pc=wGmS@en<(IM-hQ#tt11m!=+h0rKELa8;AQaHwpZEZrJv%vj|L#)zm9Zi4u-p9vWU!x& z3k>EPUe?*7_doqt2VHY^@T~tJ`>t((Z%%!0C$_n^h)!B*Et)bP@7=wDIag;Cozzs- z66ZwZd}oDF*6E{%u9CZ)syk)y3tP)_@ulBEO_etkoT>qjM`M+2y9g|OH38=dgJmp} zHqtuTtPsf0dL7c$KmK-0b8&s=>adATi>s{DI=J$*!UEZ*U#I>m&oAeK4P4LVa&ZUi zTie$`-nZ0*mB^&Gpnx(MJZjllZTDN(_gG3wxc1Pzi%Xx6;#=Ri5zv2{ak^fI-xvDO z{@eO$^}ByxUt-A&C%tg1vuy7gnDyMio=(_{xcyIHi)n)V>+o4KW%K65S=ZV6kM_KX zqNWo0*YOfYtzY}JD7HrlW42;#ySR@?ly|)3(8sv~QA+j6$v(N=F_-0_rZ^Pap*^p) zj(0?X-ECHEWwmSbbF6)~@utO_aujH_KCTC%!Z;-v28>4n+=!CGKnWYH^W1enCzJ<-M1kJPh-Tpc8oAr94lbv&qh zCSf^uW^X0~Z&x2x>u>|$g2*Wr?{yhNSP4RB?`#w%htFC&d58Ij^9!2vX0$N^08?N! zQ*Yz@{XDU%lB;hGPYS7uMJqHu&)sZLhVHSUn9eJwY|D6-N7GhyzkOr2e?-=h9CxvU+$SJF(NttIw@IHF^|woU{i#XdhhuLiXpahz+3Ng0UZ?uZR+21F?90uGZchtuYBGknI7B;p z-}X`4vo7`H-F%wTjmZ0+rIKS)FgmfqVz%e3McCg6(2X1zs72vXOy2hC&?MKoX=4HN zq|z@BLFGn1^ZJ!^;&z1=jvfLbdn$@}4C%F{*qEOJ7lPxH%t1vd$RK9~dXswKzQO=l z^UW>aAM9gAN&SP59%kWccUx9RySd-cafZ^N+$j6m*MgM6k~x{3i_5H!bHz%s#-`)c zZ@4Vn!{*(Et$5ha|M5zlDpB!F;>)A^*qLKR#)PWo;nk5VLPc$(-R5DsLPd-EG@yUE zu_QU#Jm5sAyw*3Vh1G1FN7=f*r88hX43ep=4d{{wRckfifjvYp%yY5xe{etQA=P^* z59cb3*|IR)gb<2g6Q+AKeg@lPP4h+F?KlO;o}8FSg;ma%Yg7)Jhe~CNmFi2k)+1C; z_k$gOllPZrR78*D9LgCCWwww?PP%#SDL^)PULYmd+hH$00w*&xNbj147p_(S!}Tq{ zp1A}6C)*lbEX6a`5KIDxU2A{;M;f$xD?%?jf8kQnTz5(s^zI!I!nE-OUWf*yt%m45 zBWsMdjq123C~%(`c;TC~9JzO%Dy!GD^#sA{rIlOGLv*q8Fn0k za8EQ(zdRE1NP{+y`Yu<1^T_2#v?HJ$7*T)H%NS}zYwb>E(Y&WygVY6_;|L(`UERSXXV))F~61%%h^okzGrBv*{ek! zRk=bQGu@rG=V9}fW7p;%V4n5&gal9)RMD{EJJ9z#MBpYzX97m6L8RkY0dAA~hSH)Wfh|S*6>Q{5H#bWwUU84i3@;djZ#0@a?cBO>dbCc}3 zA~w&x%vHrh7yY{RxBT#S6AE=90sLFj4eiR8LUe{g3kLfuA3g@?A@!Ku?me!I%-(J< zblZ4uKuv<9ELNFkq?U-@Q2SE0C7;sTiuw22#)pRV@=GzB-DsbMy}hlofz2fwf6K1K zxH>#uO;HM)u}A@2+jcMZ*73Xj)gTF|r%rRM-Lqm9&&ONFDTrw#f2EFJzE=a9gYu&H z)}VG1rrcG~L|_On+10~a)ISL;d0i{nqDEYOH@a?XYj*;p9>WIj&0u(S3R+F8R%CFT zvn+qn1^8*zDZ6{^7{)3uQ#s?j4)!iE7zG7~9vwd`WIgdA_VbSLA!SC^XgXr!PRI79 zz25>|xU)y;9{X;7z-gG&aTA_F^`iyHRzpn=6-FMrp1Ywcgn4ML$MZ5lgd!Ueb$Ink`IMmjIQGy=$mCS~>|-%N__I^|(! zx{#g_{?T5K#FH36Dw6l|viQKBacAGtU{B;(%^+fvN^72|t&su2IIg|3vpwYnc-630 zA!|2pfA58~myVjC_!kWY0_|rTh4$kSRcd9cn7y57t(e6-Q%S%79J#7cwXYa$n+Idl=I#;t9|FSyjAzYh4&@wY~JN4;jf?-YCRP?VI@LgMWP-@vCSFkkudR(9=aFO9vhEw_t>xTh&i8(=;Y zyvH=_KzkRGCT6!w%eLI>9Ks~j(aSpy)K)9Ie|{!_sYsWP=cO_A5T0BXnwlF zzW(t(VH>#}@k>m$+L~LPV#T)Wd>N$Vv;HiFWXhyexrK@tnE^3_33MY#+wi0#oKgfV>@edzmOMEG&D_kZqnyeA z9trl5Zn+?aXz4|Oyyk;i?)!_}V(@UbnnnE zZUZI51vMI%>Odq{jam$f1f9HKQ)xZURoqQvv)DlNk9M{Np7l(`&=W_WV05V2GYDs zyYlmX$L0e=_s#VE?erJR+ciNHf4*^aJ~PF zs`m~{djJ3b-zQI5sd>tiBNa+daw@kuQ)y}^Q%}Q*d!&g-?g2t&W#%9=wVaiil{*E( zMNZsXQ5+~P6mg57h`*Qb&-ML&zQ2n<{ln`5UQZs+$Nhf4-Tis1x%=iOVdDt5Z#+qn zoo_Aix9~FWA=&7)>0RJ~aGlh-7+(JR0xoD`=l7T3z!OPRZ(Y78%SC4!Y&6FV5?#F*o)d*1(7 zVV`)T<#H!a^|gkVM!t%-kZ-p-{Isu4Xm52S297gcgK|}X^bP{B((Smq*^ewL>^vGF9L~y^Bpy^j zjI5bZf+pfyn+Iw z%6L<3DU{DG4`Wjei5rRR<~-n#;2CWmE}L_YHEqH_w49%0e`MDth*lZBE4;m-c9#;; zL9tsb|52N7?D)*uA5gKWlyyee!QS-;e*c}29rT3Q{|UAJRVa%Woq&fd9Q5$OnrC<3 z&|p9c5(^7$=bb`^CFV-e(lf4V0*$rcJ8>$zDmAu54+0mp9(+e1<6zPwp5m+y<&*x6 zO!MpMoQ(-_{8hd(^UsgGZuZq|U6t;UgGxb`#C$_J%jyx`|2(y0iQjmN#c8vp;QsH?^i8pIk)@Qv^?+iQOo)p!M*2w@-ss3CM^K zWe4*clN9~&1_2+j)c{eEyksewlxNo^>hR#u#^#-eI~Yqpb25b9J2X`H^D7v&!Nbo| zK+usP)u}L~a;huJZQMZS=fF;F;S#;D?+x(_#MJq;_K}Lk-v*!ktp4rWyk|Bz=ZAJP za&^+L1+W{{$ftLTlkU>n*RC!|!FVAOu9fXSf)gCRu&amAi(|?+=;6MKy64xvB5~Wg z#n*>MsB3&9`fzq&Wib~}+}3p2>ro(rCS zd%Q-jbcC^T&5w5DauCT+>gF(WMhv4SzD8#>r#~-oPbtSE5xgW4Aq+N<-GdLOTI_q| z#jz%X3t?^eR4;@5O~f(m(P74XwA&7Vt?a9lmvrkhPHx!(3bQ<;L;ThetDJoWSNSrP zkdxD448)zlk_|9Jk-=?vtk?0kI0XH;1USMVY=x#v9R5?>2pK|ss)|MXQjqRKPQ{r{ zo8N6RfglCIaOm(2gssjA+J9+ieJI9@2b`~2H_sjnWp5o7-L1~bvr>(K@6t#wpl_Z` z%2^8+(@Kzr$S%@(_X4L_z>a}b2MqN%>}G^RWCT zB4ASAF+Vk$8hB6Eo5lEs`MjeU@5W&m`Q_Yu>k^Izw}PU;l|H0DE6pcGc3+t)ZM=%f znu=Om17~pmOcPGIZAN|GH=ChG;Bk{tnrmEzuh1GaQBIqT+H%9q3xSX71MaA zJ>NA6caGEN7k;N0_JnF~`Z=rw9#@xHcKs0EyJ-IGcL&6f?(q5(+VK~^r+s~|%<=v8D-ZhP zqpPn6P2rf>!nAGu9~bx$SFM$$bw~l4_?lPOqJI(bXj`S9K(}e=miaRu){iaczjv>a z!>45FVcw|S4=S_~q7hoIg8F4ztygWRUu~Z_P_O?9H~Q*^b8r@hA(73@a%Vgeyk=#( z$F!KA!ypaM1dbqx^6HqX>9N#jxBXccr37%N^|Yu_rb8%Z`_a#XvE%vxQtGdo4Vs)- z(>?*guYmsy(Y|J$oFz8(xMl|NH7h$P;Ec5S8{Uyh=36LO{U#G^eYpDh} zA<{Mxr6x-X+798;GjPj-?OmqI)tcHzS@XJUJ+p>r5@@DuR>Y)uhKG}REB)=zJS^=Sf1qlURsQ!1txM*0u$sho2eij``}Uz9EI%6Wq2c;}7689Tz?Q3s z=(;A=V!fY}argykiKaa+2ZR7Y(#9^D<7ry&uMyleWEayatAUBjs9m_5c3=t)v>8v1 zf3B{1{d`jGWczOs>j-|K{-yT?zr912*qNzMk2w$l5yEKDsLdfq3hvQ)4-R2l72gYn zzT;~ZGt|H{ZKEVf!wOQiY)emVj;Q{zXil&u+^YF|*4VLL^Z@1+o{W*#m7X&B7XO?%g3Lq`QDrmg2KtU(ZB zvX%lYFBQvsl+|DvfI#z)n>8vf|3-J-V`1fY?16`9hR++K7>9I+ib8leg;RTgTDp~4 z0X>7PaSk+tluqPT@j8&kh8~3&&US*pF4nd{c9(75g`i+HS9!{I(e*!CC(cZjj*m2p-vg@b0mn|Jw5B zPd*9rS&k`h;2h1<4p}~@72c9`KhyudvQAjehsX-~aOrX7{UV)f23BTSzdaSN7WvH; zA^}C)l5B#whOWpYzJ zOVzy*o_a8WB6(xEvR_S7&=7#phnA0gmClblsZ>YHQ|SzzI*vs>2Y22fOAf<PmNh@zmFJKdYUa;p5v=Rs+uX3D79R23!1UBAK=wJp`nj3>GDrP48)rzIWUrLtOTGZW}Hu-<1| zMWkC7+{rL?Jneb?5BNbcvH1`A%nfPJ4si~ZyNKk4OU8}2NbbN%6O=T0>EUM1t;LNn z-8Y08VS&ABmfEhHr3ICy9B$usP!_i9h659y-e$0QKL#L7-tdEG?z01PN}?XCwrYaP}Br7y^5 z%bw$w71Ap3cwuK$f%(4#^}$YHfAIW@U%vCNfBUOrgd#kSJne%=C)AL0Z^|&Z)^h2%Yhl zmx^Nnno{g3MxKMnP2F!Kf-_$2UL8V?u%7?1UU({p@JNC{G~XC$d&ADWlvA}@OCnLE z=Egy!AqBEd@ym>~F7`WH9$R2nGMRN2QdqbSqWXV#i(^_f8I$fnhoeeQu7lzR^EhGR zO(~+r&eT>(a8p{Qitz}18(uRe>coPh=J5HWsg?m?5$@g*Q?N-7KGYtOcqDXvn$S@$B z{i8a3c)ZNl*+EAnH`|tgR31^@|2r|{&}dM2tKXB;5_YjQ-KNbJy#{ z%scc>If!sj0#>QcJZHeG2>TEH9(;|dQbX)vHK?O4NAj~d)xj+G`u3l4#tz8B5z8m@ zEokvL`wzpj6?Nj>bIdZ;)O zXMz46VsF2!x0Y%pE5Xlg%2t_Mrm^JQ`kgbQ!Y}!DKoE=Q6&-UUu!+IAUVz_s;;X=D z@$nU{CEt@%N5Bmwd13i2&OiJ8y+6;1T}CSURjZtgByk2Ovog3B->y#oXo}%pWc}}5 z5--|*uou60dwhR)k=%T_<}~-s*)&v2hg2w^w5yxS-bn40toxt$_5LB+!S|!w9ZoUS zr~rHP#TKs%E2yyKeW)zO_N9*p*>2o^EO{A(Ua76mv!tbmWt*+4gRp=zx_H3!A% z;rY5;reLGz!1W8rD-(c&67?L)+`fHh=g4V2$*{=O0?8mcp0oS&6|%O;?ZxtyfjaCE zLD`8hLj14|Cv)I=lFuf00!>9e<6jeRUaeugE zO_*8TcrJ*a0uc{0GQ5c8{DX5iisc{qn1)S;fF(bxm=HyHOxjcNl4PcR!;hEGK=z9t z?JnmB?W`p^T(|Cs&N`N5%Nz(4k#`DP6?1g0A0!BY?Z(eI@OsuD zg;+uZeMd)sMiqMFL`L|x>`1i!*Ea>JTq9d^MX~*o zR@8T{$}7WpMk2Laqk&x?i`P+wD|VgesX+FT`-xkl$>us8M0Icj5>gej;LV5NI<$R;e)x90^HajLK$7XQ2N&_&4wk=puwG< zCnPq;XaE}BqqYaXRMxXq($$VZfB0g5`exhfAaTakzEn#rkVE+IV{`kEvfj_|V!FYP zn-k_{coh+8B@4f8Sa|Jy-tS&_J@3R+J^ya6Ag;xoo`iXSL)!OF<)!abJid}ubnsXh zUazS*$*y9hk)~Syx~oB5uXed5Hh=qSUMmrqqu*3~wx-3Bh2u`zWKJI~CP+o37yHZ^ z?yMap5?2m&ead()$18YlA-T8++<3l(^dw{u5K>(3uNTilE9N1!q6vH}SilQNo_29< z*$VPX6-UNkJMkc#R2a6%E0v)i@f*o-MlY*uD~T5#^EEU;-1ew9%UEc*PTW)pu?I6( znj(kg#3A3 z{ve-cO-0Kq*rmX1k$H8H43rhD?Owz(J9;X;Xl|1>S8TPULid}&tu{D+DGz*t!}6!{ zf_T?zT0inuOSVg%#R)5bXkS+5YIc?`Z93l4K!pKi06!&AUq00O?xx5FMxTl`{OLEy z%6T_jOubhFSfi1bg4t#@crp5)i0zfN|FlNANZ(rfinw+u7AH7Gsd{X2wB1K4lfK`} zXW3W>{t;~PYBsvikXuCZ-cdmbt~S?Ka61wV7a++>I7AKpNtNGB3U=GlNNVCE{;b-` zXBMfMG$gMh^_8g;dcdoEsBky21|09RtK>Y>i=4@6JewZSrt&&~5b{ND`)VQel$!pq zqQvkV+-Rq9J_QC}*B*cPHXU0&%Q~f&m1LKMikRGM`4Qk{#o9 zAIYl_hH8F;!C*S}JlSwL`uAo1ya>$onl-|7i#7lI$P$r@+W3BZ3|SiJ;JJX|erwcb zEXU~U?)NN5?IM|CVsIRpCNm+HT0~x9<0gIu2h{l zCi;;U47oV53M(i)mL+@IIM1j@wX6#L+g^IxEN&puZ|*;1@UK3l!mOK_|||D{pAyh8fcAj5t~QO78)G21#74616ZSsi@W?zefo zBTr-DY7$Gh`aHx~xk>W1S6odw_b=7xiH)8p}*hp1akxd703ruqfcYrH`LqBBCU;Vrp&9<;N6k_AKKnxNFngu4x{fVWax@H2~W5h^`b z`%tAClb=b$xSesx{(eIl0qT&NEP3y?>MGJH6Juw%rVhq^5td}3~hn@2F@d)M~&@sJzo^iCfl)t2t}U=x5ycRkGQ6R97c`dC%r6#s z+?F`f=k=n^>kr4r8?l84Vja4U#eu3D|GGv{sg+#*+gb|4cX(|tT5~oPEgy@2b0Et6XHJ4D*!t&?&oc$hhOilCD8qqWW0*){i=QSjo+ zoH=1qgZin^H@M>_YhM9@N~G9AVJgQ!rysacm2Rw{ekAE=JsQybu)0m1YvAlIaJJVp}JO4rj5C7tUS+h8{*iFbg*|n9Y5fDv$)X3=+ zuM*C=!{A9Pa!U`=%*{7YX`HY-i4QOuDHNk1ROPn#4`-MJo$j|OOVSTmio-M{Dx{@6 z7kY|#w+p02L#4_2n~3>1?XgcjfyrAFEmK=WI$s6I29B|4?ViXHPN2S50V>bvG>x~yRx5cxW#p?{Y6Kp zN|X^Q;Ngr}^Sim34_X!Dv}HY}lSIihhQo4CzFZuB^{Ohc?a-C0AzN+!z#)CvjkjLx zW?ZW-O^#cS0~O<6_cIVgV<-24++MDQ*C+d zRmzybmI&B2uG4Cp@4eTa`$>k1u6zxAN8pxIF<}vnp^Lur>xFR`#bfM|*knU64Lb5o z&+U0_)8YE?NAGW_HoSI&hO-(IoM+~57)yVPa-4B3YB7aKFKCAk4RopS%ea6~Se)E+ z?Mq4)A0Y5}C0hE~@38$hy_UK}N8?1L!;#pKo!nf)M#-vcB))<5e9_kXQe zZ@pY2MK_!R*62&KN1cn>!Nc=2Nw;TD<%It(N{zV#)G_aLX#UsD^ys||lYTH%`BaH= z!!8TETU;rkd+56Et5)xcw#V~_sv$x`mSa=(A+>9#cJhojzb!SsD7rG<==N;#facm0 zF)59av^XGfH2gt!H|zGwfRQnjJ1?wYBK22=llzvCgrcZEB-G9mN#?d;d1~TF2L3&G zV5K5b?HD>q2>ggjs1D;!!c@pRmE;kzP3heOCB}k$@^*fG%nRYe#v#jH5)n7j(;mMp zlgj64Uil;c#)k|oqk`iqy9|J1IuTqvWJpalDiZG@IOF za|k`!phy;9gOeodx)>w$f^@Gs;l6XDTh^~RC;w_5&?BYZ4w0u`+0uorZme8DSQ=Vk zx@t7a5ji2E;K{{g*?HndEW`1yltC%Viha9|O%JQ(BR)4da5M?NX+o*Z#|-Qyj%zRh zxU%XA5xE+hJ+||oW!s^OTO<59w}&&_u%36Q@KFgAHpI9)sPs{p?x}YxVi|1RV+SBN zg%)Y%5WCtqb*hp>Oa%@#jY#v4PNw)6Hv#Ei03uGVV*t7){BsmHRL_*!9Wx^ z4mYRmZzF{L9IkDH$-@s#S@sJGQGel^&3JGrP zJjcw5S=$nvuB5Ifg6E}ZzWr)d^XCV(FBy$^2X*$Unds?0Eh=oJwF&L4nLkhwZ@Je+ zXL0~uW~pnhci$cWS3u^=iAI`^Y}}I6Y%n+vFHH#I^uo!LbIkebmM!Z76r!kQUSb=_ zY0)mjK1YsZFx-a5UTH6IO% z>mjA2e+QX($K+6z7UN1*RqYu<84;i%o0T%`Y2`irqVlZU9;<`gL^Ju(erD~<+l5hM zt$FPHg2;fLBg+Nc+89jXaCiUH-mb;SW78sA2BQT*Uk;og6rE3JU*7Y!o3c)Do?#rk@x_ zj0}1YpHJC6W!6?%;BM5oIR2FB-Zp#bbIl#Wu;(?cA5|X14@ea<-CkSm63y|iFvk?v zIo6%L%Mbb~nNeIgl{?Psr};a+m^2u>5C-?}J}#`k{@m$EW>@u7iw75%KLJap7ukxk zlleHuQLl*ZuZ4G~Yb0yJqc-;wZQ7d7g);AiJVzDhFkSw$x4l6NHn%TQmeF&U#3TRq z_Zl^?k*<&W7Ox;>j|df1W@Ey5J!iN_q0GUjvue@f>seb4>1`bTm-+;$zx#CpoGyPO z9VHmj8eVe&KL&JAed!#sGAiro> zDQbKcHsVbkn$0ID**v5*;C!Ad5d*qY3{U@I$#_tY$<^-^0imN{f%g8(j!p;GLPA&i z3vtu)RLoN)9G?EEX9gOf&TEeDUZfx|9*i&29cEMOLJ)9`vA`}PVf$pQgu!KM-ACB! zVyq20*sqZ?hz$B6t~U3i?#s_!$obhTA9%|f@9FBOXwhj8BrnXrd}c-(YyKc!SaEyd z&LbG+cSYmP1t8Id*$jRLeh-3{Hw-t~<67ccPa!`Cg0`*6k6Y#zLKmMAX5nEl>Dt8s z%7WA?(Mw3JhQsyd!Zj!3-Q*`f+0dW%B{@%RPOHTdW=+Fj;a@d~n4(qtVZ3`yL$)?a zdmOZ@f+cd{9BSCf(2g$c@l%5EFgT%ZDsJIVL&P}m_~^R6tTA%qaF_!GRhQ2A&DTK2cb7fWK0kR21P@OUiUEeSQ=MRoQzQW zsmNny;w@Bfhb5)Inrvz{2Le+)p>@D|g)|QjZU?VSm$L&)ZZi=0Uyp?9K=RV~iE{%f zkzv&#r$wUW5l*Y!GauunEYM#=Tzc}N#yK|(F@WnnOI9(2CP=Dcj1(Ufa~gs0Qo}~7 ztasg)Dl^wJdx}*@ASoSnU&%aP?T=MjT!dl1)4MWa`W8r>h0Dh5XngMA5t%nqbwpc1BHb-ptVqh4|Hmhlc;@d@1zwoc+J63>@(->SvJ2 zbCuXSM$x;XODd9f*%vFl>^@vFNkod2ZK|c4@CB+Ym5YGS>Ou^BXe{k7-#PmV4;mRqEPv`?f0 zKt9k?d!6uDDf0jNWc~-{Ix*P%_{hZ9f>jd(l}NE13aiJyy-coN%Z?B>f-)Nsla5dt zto`D3r^xX#g%WsgV6A&>^aC+$V+g3mxYzAFnRCf{R4Zourr*wh7xKMu5uD^5%AUS6BqV;T*-OYUPK!A71r1QAI^?EeGf5QLm1jf zeu$;I&KsCj__2FU+LnOeBiTMYBIm?#zoLGS=0M1pb6@4{al1&M{K|9NxYX;AiOfx? zd9@PN_}^14OSxbP++NW+gdAIg-c9h8jcP7`XH@0F@hP`w5~Y1NKq1!E;~2EB28_=n zP)EDF?v*8u-bd{(%sN6hMk?Hfp%0W*IHd`Vul1g`AW4TEWtDJ=ccPsia)@g;iy6X@ z;8-dMu$ky6{{MI$6Dh5ik$3xY#_n2xhu^m)goopZ^i*)e4BuT>%7n*H?Q`zKRgE?F>2*ZlRW3YniX zk~bZ%&;5fAzBC%=`(T2X@nrljvZ-uinB?&Bee7-q1c-;#cv(!CRF3}|>AHGZjbQE6 zIXEj>XxN^JmOS?z3P+sS%Wppn4H`l?#1I)CaV1ex2oGc@20*`vJ;c9g zL3*+b1B(2HaT)=PjXZ)Rk$Th)OZ%x~QNGQ1hPV=$zv5m@huKzgc^A6-3wvLVw{}e3 zlWFD!zTz6{b#KuGa}Is<3r2G&v3Z4ydQv@GF5f$N_R3Zv9>mDzCW>U)?C;Q-DgZn8 z&YS&?x^1q+zOV5qmUxmV9E^y;W+F#`3~Ef|NH?!JUkp}j%TnMd))}!T3=S5)_BT@zQt980>{sQ&4@DZ1=ISuPoi#D zk%jSJCGD>FqtHg;OQpese1|D3=-~8V*hzOdR?I7&_#DQA=>caxyvk?p{Ng40h=?TR zZTq|`qer&OpZBzGCAgMo|A;1AHJ3{?eAlWg(p*)XyFD{ZpEAua+Zqxh%d-D8e(6>I z$K$03l@|TX4_im=U>!odD)XTxlOl+7WI+fwZQ0|6p8|Cqg7DP%tXkfX&CIh@+67@&0a?EWQ zY-K-)TOE!&KqYMtx#KbodDBTM?ZK2z$ds^=@s{6#{9bzpE7cv;Op9mi(mF}78k_;p;~jqbo5Ez^q=Rw%f1hud26BK%S)Kf z!5K$5qVZzQEwLO4XY@1$01(6{AtPBZstbyALqZG7&2TdW1a`hJUF+a7X9Nl5zrOG7 zvpN8VD0t;VNB`6huFh(%cLchh&xZ(Nb7y<_G0?R{oxdm{Bamr-@s9cnyLpmS4gLJ^ zo0ll6JW#wEfB>AU-D2kRoJP_xBZD8U^eeX6`%R>9jU;zrHNj9Gg^=<2FPwgH%djbt zY;CdCpxyEuB4h9$uaOgw{*1-sm5XT;w5P>dC4FUAoJ=+Ee3dU{G>FIdV1&*Es1M2T zT8u$Z6sF`jH0ZOFYj*a=h{Y{qorNRgjP6v3{OQaOt=`era!l{a)#;p+W1pc53gnl?G(lFAk(0OA8 zhJsT9{L6M4)39Os!V$YlYH{e8L(Wun!uB!Q96he~Lh8WW^ruuyo~!%Yu8F^-j6^lnBS;B? z8$NTT0%yum0{7D_3O|y{#fevLmu|x`d@?twBTndH* zb_bVDseebd6Lg9-aHqA-Z{M9gQti;4vrGvBRPO-o-aJJcg1b~;qfEY*; z_4Y>@sd5zhdY>Q&4(&3yT0PfZtvURszRiMe=8OSCQXkPC;#lIov)}Dt31CiaGScWP zIgG@eSzU7Y%e-yxC+$Y}b9p`4|Lsq1|JY~Y6L#_*>^T@bz`*AguU;a34=BWydOpO#9rjj*$^*=Pam-AP@#LxdksCHgvFzeqJZo)iHgO!xuLTtEvvbvgcZAnk zJpaPO@pQ=^c)K@7uKn6Cz|zd)A@|s1BfPbOvs(d%Mf4p&Jqo~X77GThVWR#qasC=9 zY;K4E63+U&)|c*w&s|}=KT_6&Ut`OT_!y74ob%b}7j~N?YpJ|Gz!H0{G3}w0<0m2$ zLw$EhCjh@#d%9R+gxhnW)bh)mu)J{`m;Z}+HY-b>gD{d+50Z_IdD0R{jvDuK*RH^xogyb`!fu)E*oj1hyfya$7s0BrF$7~ z=k}0IpujQrp(q{uf-;b-nt8^?*jqedV7Ab(`i}Es@k&OLsimxe#XK)cx6U-wsW(5@ z=orq>d0`q{4Cx{#qVt_s&22}*TDBMiUyQH;LVu3-^B=B@hfZ5=bi!mud=V`UrR=26 z=PUN|dR!H!frqQjb$yTvmhCb&tM}fxmlMqMEvDh}7;&6RO*8f=k4ud`SJj;NqR}Ka z+BrN?JhSSZ4`{{B)X|J_hyL}G3IeX9SPv;JUyVb@kDt{a%9Nioe-~jZ!Vm<0y6)k{D$*N zN6SBQm*tH8Do?yYPK@Pzv!7;4n{D{VewC4}vmr(yXFRh-ckdW>dRj_SUpd6!sl{B+ z(Rx`G&cev_^yj%~Y1e{y2?0ha{($%f6O@{+qDm!0<0&Uyg`fE44EkR*bJ61YNh`iXT5|#1AJ?t)t^M} zIW(obidI#EZHgz9Mye>Sv4f>`l&MjY;t#@NOxY2wp~07Q)}GJ{GhM#rXTG`w{4-TW zA?WtJ=WmQ9&odJpnFBqIA8z#L39|DnvbY=x2$r)cz>iBa9KIp{B-9w>Z6YG8Y%?-q~`|tZ`(O;*IM;u)&47?cl<@k%zT8B^aV7F_=@m4wd z`ub|1v4T%xWoM=I{YG>*7FaK>Jr{vEv5x&1eku!N2J0i+RSlX_jO=Njw}+EhJ>NS^ z0i<+9c{L}Z!kf^LHFYa$ISl#GGs5FDYbRcsOGCNls7W&@0`VXx1CTQEH}IuA$f2?|A3mf z9ZkarF8GyF$QVu)77<~O*4KSL_{QnBhbx|XP;k!3kayfG9U7#!)}#8^TO<}-cuW>O z6)jEUPj+eU0|&!;aH$8QSZWu#-rHy8^PhtY$qx&<9{7e?tdgoD5{ylas)=Wl0$0li zhZ9`h?QPEDUa#O>z1l96o`RSh#DS+x+-u`kv;=r#Jsl^ihE92iTxeCQBFiDS(D?T=))(vH8m5S!Qsm7s|B)^C92x=V z^w5_;8fKYJM+l_vao!H<(Czn3^kVYXU{$S`1_d?E37!bX|G&;ALqmw<|0*rU?UJ3S z#P_gb!~L4i5)|K{nnv?i{+ z(Vb!i4?61(lFtXpE6)A)ujIpS!|Bq;_oY6x;4dPFat4I&1|@I*+ftX*Axk>PUt5WZ z=Qf8_-kdRD!)lS}DcPKecRMT5V-?*qY0K-5%#wx%3Spu}49qS|B<8{F0G5mb#RTbh z#!q_XPr`_8_x!dxoj+QYmDkWJux{j8lj$Am3U_&+~vJt?-^>@*m|O`%0}e+ z`o3ue&xGK#>5S=E^7WZ0ca1i9z#k51?CxP*L);`MF8g{(rk-$Y#y@tM4s{5&Rd~|ud0grd`1IX+QX1-*g+F74p^o-?xi^Hn=Pa(*ZP-t2){atW#>A%vJ zDO@4nuedBo7s(`Ieztf?1=hTQv4~d3 zFmB%LviF)&HbeJ(7%}pu zut`@aryCX?#0?8(#1@sWO`8Z_BX=QW$>JjqX0lI0bp`aMavDdZzcoN6_ZgX39FrMF?j6|Dx9edufStedp`}uF zTuh)-!TLgOu?ck3bE_IyHW!({`_XHZwsKuJSncNu2k8ECR%*(!ZMj}F2R}uc^9ss= zzBytP+|{}N+h%=g-?`-9G?<3*=x;I2eSS;`%CUdX)&HUW$tDAU`dul4NYJ0&D=i}{ zNq~kcz+?TjmhusW)G({1ADm%ubwbyJFuzt*>5JK}o2b&?ogMeyUR=GO3Cfm}VcW8p zd7arfTi=(Kli%i*UVzB0MeOx!77@?I*Dg0Pu6J1o{`%=SeHi46J<#S1E1oFpsb6l% zP9;cQ82?&56pQp|=#M=ZDJFER+C|S#M6{+LSw)oV5n+5tm*i4xW3q8xHttqi4H_}? zRQVDiu~_}hj!V}zklyaMuCkwEZ$|ugug;;_C-tetBM}--ABnqfz+p1W-?RMhJNtzP zx#cm2A?fQmYZJ3_l;X2tdr;IC&9}`e^GJl*)t}y?p6aLnI3WM}@d=lbE8GJKK6NL5 zE{e-9F`3vN`nKr?ZXP$*lgfPgz3$7SiMrWi2VDO;Q}R>fkBRe9kDa;6DNZNA8lx#V==@BsbzP%F!2gNSI>t&`V_U2y58+rn;RIggM3C*h7$aaoc z`vE5V+7?x2k456!)Iywd*NQ{B0@P&btle+L@P;b*#&41&mfJV%+$knC=xLqvk33Tf z^+Ws842knVk6UzS4BJl|BGlfO zUI1*5M>BFpFqO0GW0W4c21Tg$r z4ta5W-ou%3D=$UyS@?X1hVM&?3PjJikm5&9|EPGeWsJBnIU{{N&^}$azAg2J^Qj3R zXBT3Yf3`&>8wNOS1xQR4NcNbE=CFRp;ok4LbSJLI^(&CtgAzKj9WPKg>uwE~knGNU ztd>s-ONr`(JMK(gMDJeIG3p%pS#-YiiFRFmtRq&szH4yxgeeYZJp7WGu)<`-R!+r; z(`)YX;{smcWI*W&QdzqN$HtU`3;i8uN~_{1-qurGUE^3HRcoy#S~$JqgwFfY{_6XY zsX@Hcx`J6#-DC5`GF|Y+C2_Y>X9Y~hod{Xg;jhu87T-=7Vqp4TKdU`W3b`h5>zW!p za;62fN8ZY+nnKe3C4EA9c9*NCWGkVrn60Jo_1d{{B}Ce{=K|ve$acEd&$FKtw}uU$ z#C67THx>WhhktPbC+~>i3rX%|)u@)Ht?h7&Q%3ZH_mE#4qbR_#gLUaz+zgE<)0=)c z?!0IJIZCdgrb0jq+}!fL&&rdWZO0=Tho&eweVgaIN<*5ruQHy_RGx#6HBMAY4dma& zC4$-)q`GW-vag{=JNJ0md*N92-1f66yWdSH8>2B&?@Rw<0^!@68W@S|G*Q0`!c}zmC?Ms`v!CPROcrWsb+4)!#c6gS%6t9(cSf zz;mwc5ftd@64l^#d%NupcVTpn2GU85-;)NPXtjnJ0RD5Qh1-ze7=&MsYynp9o~1DiUoFqqtj4n|+rZar}9<5y=5< zw-VP6Bs{qdElHNf`uBU5MGc{XRb2F@0pJFlkUAL|UZy4tk?_x*@y~pG__U9(?kGVEWywEdgpnO&Nt}2RoAkBca3R%0obA1i=Nor7!-TGIh z;R+>K=R1Tan!fns^Ik;);tYkWsHx*U_1zMAQIdgMsV6Q6IYnYn%Ic|%Jo-+P#^ylY zR{B}=cIJcajRxmH>!`E43DvW5Urx=Z=7hiX$6mDg)6_!|Ivwd3?S+=(wFHls_F@=8 zg;f4M+-R(O+lvCC`821f@wijbx*sbIA0Ke5Xm`7jI{W`Ib>4AF?*IEg)hX*_nptk8 zj-_R}^0*aF$I{BwtXw&8r6rEcEvvFJ6GgK!bDbhbE*v>g$cdICCvFlPI1vRzmOuP_ z*Y6+he|Ye?A8>Qu@Avh(uIJT}ootmKWa=^QYU}-FR(K8@Y@r@DQ-pLNe!)2_Yuk?} zdW=81mMgwSySo?}K8a1AKM_cAoq~ONdykZ^V$TcgX7(l7R-gN;q!3UbG_3pEqCticHP%(9Fza}Tgf%T3e6f6gOh&qUPz zI(;DQl~jHHV413MM*Yf_YvM4_MyD#z$PdqTj<82g*_z5ZK>)Co3ZgA=XSFKQ=v=hn^AIPQ?#SR$@t zW%P%Ojzu)3zwel8j{n{IXGEv*3^RNyI%J`$;$LE+A4arHTyHz17kWCF_tjd57ISV6 zBq-PkFTw`wyZHVT>6u9bursAeZun1Qh4?stYo&^62Edd{ktIZ5C;wUUy!q8k+*6QX z5Oib`(O1h=qqofxYnNCdWgqb@#5qgXIeA6qiOtuZp|fId&2GO?Pao>n{gbyl*dt&+ zulcS&l8my|hI@x_CP6bKx7i)ZopnhbpW=#wul-IEGa;3^7pJt!RXb2#R=UNaa;kRb zLxg^P=p45I#aGvmU+M2Fx`Znz!`ICo6xKgW8ud~my)F^ItDcgJ_*Q80$2pGti_7UU zhTG){5r1XZ+Xh-sYL=Ab&QtlMHG>tG5hoq-b zZ6C>_mf?Eb_Rf&G(nZ51-DSe~ot&z>uM3npcLOHQxOf9Uw~|q#-+e|v{`^-&k22LM zrlJYMAxN1BLfLC|c#JZi4(bMv81KYCQl7g>O*<^^tonh0$u!BA6x|?%ALTmRL*>QU z)^fh_SIfNLw^tlcW$R5@u*{BAUr(C1w3TyvVLstg^(r)+n3K2OfFxo#LfrpyGksOk z|1#6&C{&QX-ab9oYsH`@G1DoF)l#(|dhS09>6Vis_zqinD=mUggxo3C$G=hp_vrw5mCPtBhIAE#e&3hBkb;0am6f*s*${zKHtHtPOH5{Lb1ZViuG22 z>~`c-@x;gIVqqRSD*YFBO|DPU)(d*U8S$4<9wIeu5^Pn&{q?aS1u!7*6<=VNp{?0o zkv~PEiL8bNeWzcz0Mi-$@R6|XiGL`eHbeWrbgsw3xFG=T*;~3Z5vB`e(ymvNF=jJP zX1d8tx4{$Ib>^0&Y;ijCbis0iuea}~~q9yT=pgFJNiUA+(vY&zC zE4m#9kz-qOkle!h%w;C<6yCNgujZq{QM({*#Bk}x=zENVifcm?+L7P(uR*r9CA@OP z5oJ!lyYOpl%OoE(A~z@$k)FBX$qnRncFP7@t~#v8N^1qe`{8oKp)WKDw->tf&(dn=w8C*_($%Q=J|o=&=co6EJ9T~TrQCNmR&N_6 zxpmZFqNw(`75`_HptL=C>Vx@Cxukn;nU}E7LOxx>tF`fdPoH#OzA5$_4S$h@H|bhE z7B3FH&|5Xv(6fD`*0E^$^2oNpEEcjgH62s)s-IP~5(&(i8`xZ2Aa0a0UHGc%27}Y3u_*HcN z*QtpDJS%Mmrs1ysJi+I)oq?;{`nm*;G8nJ-NnUsZj4H=9<`UD57PqcE_gPFVAf?ge z5lpBV>-KhAu(mKcWW!_-TQkA@2`WL*Q-)GS&VlPgv=!48X!hJTd8E}HZ`Xz|cFEUW7lAC++%g@ML6MG+EJ0tN9p2gq(=R_Jk&;L|^ z7k?~c5s(Pl1hwWal7snq9IHwxx}5R0UPC`=-!roMfbgPvlRsG?HQk zkoFI->Ib@tGDF?w+ESR8sOC|$+o#!u}bL>W)^Rc z$*6a=Z0&nX-I4L~6F%yUA;{7E$Zqp);PRd&4>5neot_Xt}G+h1vFRB#D;=HiB70Puz@=^CDO=~*d2O1NG4Q)LwZK7JE}0|b=mnyj_qiALXL{m@b)nIeWP9CWRO$v^uJ zaQ6>wU{Q!!Vyp<^6~9%t@(-i>g)AU&5y>6Q1v2(s`PkX@C~)V6VdKm!d54DtR0}+m zp{9;pYn{v%{?kRPDtobJN~L2Lp_}_E^mD5Rfkk~jdl1W=(#xfiYC;3Owsunt z_?=2?vx*90TB310^^D2V-g5=%dB^7}jkJH~*%Qs|Cbi$(RmOm`3beGtr+%;#XOT1t zdb(5;TnhI&cAGP{WyC^WoG3)6iroLUWev6r?RWNB8+WDsSFGf|SYYkV?aVFN`UhB@ zj;qU|kBS<7R1*772VJg~pwKE#+vz{)pVMK?pjNs}4|o)Kh9k@+g|~W&g|xbP)7R*Y z<=A)~KZZk=XLnh;e(0UXO{D_!fGk$@v$xS!mA#5HgBf$r0GY4!2Mbhl?wS(y1fE+; z{@i9Pm)Mruep@hXf}$J}r3JcUmLAM#ha9}qxQ>y{aBjSpVtAyb)-d>eV3bSsV)WAA zxk2goyuN}8oAqAsTU^y~@pbW$_}p<;0)3SewOVds>)jSV#^`MS6_gV2L$YOA|N7*L zr{n7%2hp~^%63zo>tl`-P~~?!cZL#aw0JtQ(czbd#=|PEYgwy{L|M{U^mhYxdV--` zl^mzfV8Gb7&(8)H|L|C=_;QNiEL-0!QH{tgw}i$*hoaqx*NB)#RJw!R5mZ4oZMecg z({gL?);pN_%mkrSue2Ht)Fl^VoK~ z-J7M=p_7b;$!*S|@Xz(sYn{Km$cOB_`xhTA`0mXuojhP6_Tk8_mF*0#a@L{hUXzxW zXipTy_Xj%Ya#!e~xDZ8gZZ^e26;1gJfM#Fy$o3rU|A|Sq1fx_Zk|Q!UhkV|s=g581 z+rY@oYCUApP7^#yRN9pT#PcJnpQsLyuXjQ=fIokolCzdQQP0`VX#d#L2_JY{hsA(* zqHG9{oBINP$zlu)ANO4qhfQf5!#r7W7vWpV=|wi<4~A489FEXATh#S(@(#xIHB~UH z4l>>r;qr6>B{!I0mP&QT-Nb6)x!-ZugB`uE**eT~Ze}gL6`D7UwCCCQ*XP^GDY9%9 zvfI>%A&AXP_i(CP=JsZ0zw6(IZ?s}giXm;OgM9n4rByo}*!M#8mfPqL>TXZH@FU^J z^MZG~fuCtKd)4I2f@m(Sll>blOxXBr4b?dm$h2;Uy-v9I;!->yCb`V%(JJCTic~WW zzhfsi>;2H|&x;68RL6Y-9Y3~e_vZR+S)T)}(Jk|>q--vtPZoK(pB|w!oMkR7T&oIw ziLWV3bJRssnq|fKMF2t*_NOSo0zk^xjaMxcAF(x(3c5qE51O+W32L)Io+B%4?q2WCCw?yE z#>{63lpYT4xarADa`jW>I#VGb&t^WXAIM@TO}})rn|~}&_mzpywh^gG3K#hJGru7B*sP@{IsuF0z z_wUP-6QWYHf~$8D+svw5(tapp%L`++VeMcxM7Ot?AQP1|T>%Fu+DW80`(_5&n7<3} zusgi(u*8v)U6@#WoO+sBN%8^9M=+D%1DI>KbXNRl{eEx#+f>8*0BJ04kDFZZUQB{k zJ%{r*MsYuW&v=jb@qNZS!O^oKr`;rTHdXNg?)v&-^P|nxXLjlp;q_s<-N7$a1)Cem z!`hR4g2=ca4r!0S`{ntemO^Cee+Wn0yV)|mv>4;&^0V9R+OfaOr$ur^l}e!-dD`#? zg(*VFslXu>6!Ux{LPQ%BHPWoyILQ%t4nzSS5!{vmu)}kQu@Yf#<4Hpz1xS((G5F*6 z7YNMG)E@&X+<8%L^4b^PSG_6QRC6JRs=mu=jgR?h@54^EGFmD%->ps;=bbQ(cvR|g zHHoYsDW3SE>Wbqd!9N!;2Jvqr-?z&P#p7UyFcatiuPm;JSrjFz5JEyhz^X-G`;BbT z5-O>p_=h%BOA9`@HRXnjT(z`4;pDa?5Scg-pwzYclppP!VDel!ZPw11vurZnZT_nO zBv`}?tg2tyCIy9;xW60E4w>i_Q{kOKjiO=kR{kSVMdrv}({{u;iOLK(5!Y$VZ}ZC? z|F(qaDh6hc4wOD4e7188ddJ*WviJ5<{KJ-Lr{;E{t4(zQ$$cueJfowq6~=J z-vx{=x8(HW9WWiU9dSaC6tWJbXg#;^!D|#BcED6L|;k> zd8WUqOGWF9tS2Ycb}ZzJ{`h;~z5HKY>bYhyB6nb~+hAGfCU6sUm{TJab3iY?+~`4L z@$ao>Tf2t2t+0HJVxo7(RPG`pDCdV*lhk(kNNzY3osaOAZ|@S(d)qFyoPvHt$(#2fa`+~MTq`V{v=o6$%!&dFU@z(&L;R2` zh^^s1e`g5GkIf-|b8bQEb|u>v`*>a$4rlFs5UeKp$)gJPecgX{-{izBI>`7qmg@z}|ok@fH!UfauMMQ$# zw@xcVMc@VQ15ac0Rh3}Bn4feL@5f#hG3+j{;I%2X^TLBcQOizVY^EF?e27Y zOCKzpiS!#cB zI|GkqFs$mq`|Ze5%U7*dh|_dnDm^k@7lPB5>k^eob28zy zF`xR*`ntJatzS|iZ{zIORN;{vcksm0<~me@)3ll`mTQwY#_B0xFIOwhw5FxIZGRbB zRn1(Cf^sPZ1Ou>vNRVU13d2*4x|kV933>t=Uu~_aiuYuSZHikJz1B>zJA)@^?cgWC z8q~%7L1IbjbkA)sd5_GS-r|5%`JZr8Cy-iwH*8Pg@2 z-pKwYu2NC|-9RIXp?{5&qS@+C-jgsGyhj(lkP+vYJ3h8~)t4DGzT#ZrgfzBY(YLH; zOkIPClDgiO+=tu(PS4C!t6nignQWsxV`5<~dg)z^!_ufvxX1EUyT^g^KxWU|U=eK6Po zL`+rw?ogta{XMr`3RxSgjZeH42*E2uuE|VUn9N!WUcioYoXT#i`U-Awe!V{}S+n{u zh?j@B;-7@}bdLmWI&2e{$W^g!2Jseb{q&4Ah3t&gyFvv(e7HmxspFA&(`LuTL68jc z-Jm!!ABkh6Qvh9-jm6!IpM)Z9$dbBW*r~-|Nuvv4QGl(U00OQG4y)Y?9#xE-z~DBe z_G8vCvL3ypwRF*&|Drm%n_x^Fl!=h?hMs%9(t`u$%0OdxkrP3MGWzpv->JX`=C9uQ z#_)d<;%sUxs`vn2h{l4hfjNgY7UY-$=&1-fDhrEVXc zh=1FqZkAYAw$+vi^uR7$NErscfm9LW0p|EeW&>f_pARtG_%sZ9`lXbenR&+rmHF+r z0PR>8IjmWtaN}Xmi`jqFNwkl0?Vc8FA;lhko7ibK5id3|L7ZN)4U+ch+203FG3T$h z8ZM<6QJvs4W4qE%Y41OYBKh#OF!fySP(g5Ef|ZiKrL(G8BW<7D;>`AMGrPL~7p!m9 z!?Q91RB|2TmqQMlCkptM@4R|7q+UTX^ZHNMh6o|{00LYa?ywvi|e|))6oz zxazc~&y@B7W)Z_|M33Yh289&89=f*wmgRr%CjVDe2)?&}>Di)VN5wY2`b1~z19Z#m z=ri1wblL+0weM==^Gfv zEvmgPJoa_RArO536BdnITb5h?@W_K4t?W*hH{~|vd7n{pcIb0HzovkJTs4c(4V1xP zcP`2}f;XrI$$$>%5FFicC8GQ}>U`&W+3!tzJ#^ZFr0t@a)lX^*&i6Tg4321&8&378 zjc2>pUR3ee@n8Rhok|MD+!t2#s-TaMqyQ`B+BQZLVUM3>|m$EEf zJ1gQTHZPH|0`Di(ZXb$zFqF{)s+sKF*4}-XI3<#k-1-lV0IP* z5#cz*dR#VKr=gMR2yck^XO)v8RN#9Kl!{PmAS`~eh%oyBjur{PE4@{A-Cr)TMNm*p zGhUGsI^Fj~!9?`Kb!wzaY;0!iPy54Z+Zthf@~XlNSjd=7P$C( z@dil+%h+>1)M-*4yQD;o4nn1~syq7bg(bs-s@9xTRHt788b9z*P7&{tt?;A`zq%=W zZj~a#GJ0a~Vxfj&$o~m+fhX!TH9Swvh~S|{tgiC*Zmxx^{!OygYT5?3%V2v~JrR6* zw*N^=`+2U6%2Gl9tc>oB{H4>1b``CAT)Rd*!($9&)j+kzr z`umCY$VEikn;I8sKw!vf=g!eDr3O!+2Q%GxFY5^>{@O^3)X7(Fu_fP31A0UTu39Y| zRWog1GWU&s@gbHhJaUV352~U5pNRW#<&VI_pNVC(7g1+U=sdVM6MO0Bnhk%-XH#cL z9$DD<&F#ZwtuC39AqljLLEqUI*$daAzvTN`Z)|ev9+iH7M>{GUwhPd z<_V%*eQUw0KM@Qya|IhIELs90+x@Kx9kNrN=ye=WaflHY>K@AlU*w(uaaH-90sNV4 zON3B7vR!>~p$|e>M_Wduoe;*{b07e;XQoOt0(i0)hd*e|g{I!cwtH+qx5RwQ|_!^5)Nd_u@RH(=%+ z>s0lK&$4caPqHfTMyVO64C?3=vN3)a_gj+UgXWbj;1xYV$Et;jg)@LiZm#Y<{KCPDSp8yv~2{a`yzeQ^BH~8(dR?^ zRl6IEg~?E~ZYU$wCwAD1PZHG2UOl7zuWeSI&bg5r@vzVPEKWTgUd^tu_Oj|~H!z9U zvD`4FkGHQ6OzkCw&_Da%p30mIT?ih4l2}<*a$TV^VlfI)cMiYrig?kS6Q~w{DPIk9 zHx&@~tH3AHtg`Xm#UFY>mcW68EcBS-*^0D<3r_sJyUj+YL7&4Grf}e2Jg2@KKEE~+ z`GQ%@K3cJSi)Ys#1$t+=xo_tt;iR4Bukc%{MrDI}v4zW}rx`Jsz9Sx8L4QrZbma#& z!7mOn%+2CAhCp83Oc(%d7}nUMHy~B;{5UTotMuAy!y?Pmy|W@M=S*+IRVjCbsC~g+ z9d4N=!Yk7=KBTqAQ-dT0{%;Nge4VcG#~ZHoZ35tS0yb~o?#VG<+4-u{4VwUhh z?D#O%lWCC98yJ|(zX+|ie6XX0^ZWY5T>g`5-&Xg6@C$c_I@W8k^d6*K8p&fcUCc3c z)Jv>x5(=^!M7hWE)o3Q3>VC+Pu zn-XOH5aqGn_Y)l-V)lY`Xi+Ux3r))nnmQ>%0#*EPKaAci1~7}+tw{{f3ecKYaiY4m zVz4jxcU)QD(xU?OAle{Dsud&v`C)qve(!TZ$UTqL4+GMeSz;`1v2dM`n0;X z&-2Nt3)cF@g1y4s{A5$^?4Ju?Z60pNkZ{x0bm^IX!0-gP{+oD`cbkv4tXx7r`1ZDL z0f(60>JQ?L!Fzkh*s;X)$s&}{Lel)}PNG)i!M>>9kc-yfyCtfET{D-usHxT}m{vg- z{GJn2D5tDiQw4!wZwg%y4d-9|eTK7}sWOVb)$$S5rY)-I| zo_E@wLWAJp_)`+k6}RiCYwi{V2nD>?ek*keT$LYeo#jX>PuY z_P%;SR5ijmm^2&B{ZS1eu?a$>fhRN2UKJ#tVLo)#zst&(II5nTCzp?iY@Nh<)bK7c z$UECzSdS)NID^zjntzcLLzc#^D&khRjeo7p#tf8wv{%yaP_hC+Bga}4SJPKi729=M z6c`sJP&yh|6(=R#k^~<=a7A)VH@S1+C6SIocE2|$`Jbm%UDNp#=p{0#u*>WvVbLaJ zyl=CpyTbVCl`i$1Xz!g9n#~G}sdFlK=hk}WnKH)QFc0R&%3n~SRD8SSL}1PzkXaBg z09Ix8$#kMBF^KJ8*tk-Bfmr*fY{sQE?An=4Kf?PQ<2!HNnRN^FNpQs-GPk8rvuV~7 z=U%w*XIbV6pRLtN(yJkphRE27MG;i;$IY$MCQjL3v5_YdIRPl{`-M=5>%;bi)_bnQ zRWF+D2+5?O3Aq_a4A<%F`X23VqY*mS$<}3o;oy|>LmI=6MQtpz9Mj6;7ske`9pF&g zwTKnn0Vh4=k($eo96>(eJ*K)NjaEj*vRC&!P})~6czMD5P-m2S(!m$sPHWFdxjg=4 z*n0E$VX3P212qZ*Ei*LUzno{MB{7oUdP^mpaMTHi!G|wz|5cM5|HEVwA|L z`pT|}GMhm+58Fue@AI&NaQDydc3APQP24|uD|ffXQhk?-Spff$`vEB0=?%svV?ZiS zP`P~$Pucd%(9*!nm8Qg0_B@xpNDSzR8&in!mX7b)uwH@`IyzfYyTjM3E@jmA-I&3^ZfK-k|^?q?(6j$N~>xQ`Mg+^xLv|2&}x_C{NaVEz;D|I3& z283;p?WeN|38EXHvPc5(8S1INzwBG`jgDZbE*mhdDo017K8*c5?1=8G_D*c{?yy2< z+OAkeub8L^Twwil1ODX>8OuD4Vm?f+FP~_bcxpQwa3acdT0^vcqlJ`dZvnSD`r$pB zvgU{P&EFTqhAT(Q+*_`%`O)5uh$Nr!_mS{QuD{ImBfT^-BX!TWJnZr5(f9k^v|7%* zs`7`K*P+%%8*_hrexGgMlkbCWK8|)GV`(?V$N+q!Ci7ZV4Sp=t?ev1HJhdd)99{e} zm7Ykc{Gm1?)q9~fb?x4!^7eHYBOSg~J#SY1$Gix&8rHcZ4t?WFjKs%23HRlHnwLKu zRIu<|d+CjK+T@Dg!OffM9?OQ=@GjD^ch}n94!(lY9yZTsJ^Aw71Qn1Pt9@jzb!FI# zN9S01MxF0w(;vZXsYBsifl0lu1wF5mC~b<25XT!=1ppUM+qtHb@{#64#7#sa ziT^B7owU3DuC?CJTn*IEN4Vb^i$Rk-d&2$rn;qqgA^#aaiv$DKA6Yq5Ar52zWeNBn zQ^KU>iSqZpPx7=5&KLvCl_-OBz{v`EDFOcRHLUhVn+%AfuhuA)wBByeD!XttXLb^f zVh-Osmm{1HWBRi`gJt_^389y}YJd1MP$GewBNbmaR7DKF^mAm}uNQy$BO8&lcC;t&^&NEyj#U59n5ZA zaDzBQoY1&Fececqj-_!crTF`?+x+<~k0gkUm5xuJn%wYwqwaw|VD!u!|Dx`J;qR-;_iAw|Flq76U;lhOa@Q2CGYlIgZ5Q9}m8{q*zxKW~0k*4GYj;E9$sM^Smv4M5x|GmEpzP0$r`@yltgUq*-h^1%^Hj2Kq& zY7DyRfid@wKgf&vqF3WsNNV6^nMXKOQhS$F&huZXM*{V$jf`9L?1@gr*B|utJB7_c zjJaEx33f`Vjcy?G148L2^poySMw!!gb9&zoq04G7^C()NuG;w}XRVdlpYVvORoH(m z{+#UhZ2i6^PQ6)(Ou0G_IRSG*Di_mO#?7d7hB*+G*bxjqFSGFi&-gkhjl&jizI8vv zsM8+i?NK3j{8CUg_tD!qW#&T|wTSLrHopWb*Fd9d34&D~LG7hxwtA#)k^R~2db{|R zVqLX06(8ZF##>seE z93&|jsNul-9)GWLY))Wiz;7v&RP|Mbv##Um0xJrKDr;3;AVJ>QQSvsfBC4|cOx66=X zd@uwV9M>c~%SgfttTwMd+TZ2Pxi!f>Vs233IzaYy-7nJqV4VisC+`)~mVdN-AV(Ma z_MZVTpM}_Wnq#MH7g8ZNx`m0hG;1YSSUz9532oJ!}(H3c++0%q}pMcLCLp|Wzee##r*KMr!-k1m1pYfBo>t_ z3EwA8*t*^>B9)!xoWGD0*9SZ-lEqF#J(kZ}Rl)rDse|s43KiH*tr*Q5^<mD^glHxu=QRX0ZY`-vUTmgWMf}HdUYv(~z&2^qn2B&yW>jbw0b>MFwAk`N=*mb3jb#rWY)7%Lt7ec{c+ zsAIy9h*TI34vIHat*kcEL;YB1fzwvS1C-C_@_)*wauzQGzrW=lO;E)*Xmwji9+}F| zEoQuyXcmse=iKvEXYA2b_e-`HAyU*VhydFDF0Lvff~Pb5y_Fd`E2J z*lO8O8ZM`4d3J3la;wXh_onE|plR+@Xw(EkDHr@F6hxu14%BHRFKk=|n2XpBzBahc zKE$2F@I?O6{XXHPB03YXJ)l_n^-|O`_qlZQGM~59SYXTA24Xs>p)uDpMHZ_>OwSwVT3U9}!0;jwE z@{#s=FkHT$R&xgpnsrTCmHqrGemRo4gJu`^iluTKH9+~UwlyY@^@PuaH_ z_2Px4v?qPbqZl#$0FO=m-_Rm+!$tOiYx@}Y%U^gsWS_eN_-RH69NZR#Nn_3*|`OaAg(lU{=NGV<=t)apzF+(HgiLzGac zH=5Z+I1CT8p?V9EU0`!Rpb;*RaMMcAvG+uY-q+>xQIqw{rr?Q)OeAn33IQhux3Idc zZ9Rn4DShU~GDxLg_Sg2!!}Xk@riCRp&Eb_@eINtz1fW=9qEfxnLzDGix-CaJRt)q2 zt)A21-6e?fU6ada3N-w&d#AH?4z7Y3 zgSKzvUNAP2$aW$ufJ47J1?i2l%o@_CTJ>v^<`0yW_D9PqR@yF7xlr}S)FhLaAGH-2 zt$ugZE8Tbz$<9HRt9^Ze2ysaVvxTML?v-fnZ;GLtt_i4~LWD1cwiO&xlnZ@?%?+yz zZq+VXBXm3}@d0wS)St}TK?+Hax?Ys+Zx*V5QdJE42YjQ+-I7arF}1^g_<1?UOL)JG z*6?16JWJm0jbiIBB41Ce4xrlwqJD_N#J((?;i=HAND!vchq>(kIqZT*KpD88dPkR>dB$&JxMdA{(7kji*uokJiI(K5WZ&2Y^;pYtNWhcKQr5 z$|(HjN2%kDw_AsFdBM3^gXDMP`4iJ6n>jyvP!(L@3-YT11W_cMPcQAi^&}tDeSqjz z$h15TY@$Urbxt1r1ddqWttd}}kjk>_Gg7WiZ~e^~7?V1ntOznL<0RAh!`bRs&g^+x z1Q#S^q9iwzTe_5^R(G24Cr1XJE0Xm9Z|QZnXTQXMaB9+6_+fyd74=jcs_B~FB^rqg zNx9EyqK1@cj%R8X4EyTZW$)5ZN`@RieP;vAPD72}QQqE}rVFOKz`{9jPJPTE$dYh1 zaLj2ZSu=TB(6)B!7gA7~mdj_VyElRh>zB{q5P|`B^4MaIOI=LwKTU|8EhKX1=g&(_ zG3k$106242^yb-zuRS+BKRpe0Akae5z10a~U;2G-Bz3{$*>rv{J?p{k=x~@UI>-sd~De99ShESyrUj;*KazlAT2QfMm*pJ^r=|89REWwrm@ z`Z~6Yc}m-L9rhA_xkn<~}!Wy|) z{jm=V&?|K2VDQhi5!|MHnH4LJ%2g{aUcFcvCU}hiKv~ras;*p>vR&ylE5VAe!YFUv zOv&WkeCT$Yc)EhYDvN4@APKZlnQ2SFbftY2KIN_3=o`~btDT$=GYk@4FPb~5iZ*+s7Kvci_6yRL!RF-k_Mf~mg|S$1$_U}B z10CA*j%eZpnmE;P)hw`Vuj%_S?e?6yMXklOD^WI+&8{s$zZk>w*TD+Jr0on z(^}IKNOR6Y?ndllRWEFvAXOPzZg6fo#p+#G(%Zg3%n;s1bev*|%Zr`wYKDuzW}Lt8 z(TNtJd4MiE6}uID+(oJ=HD-|u`N?WE-PW=KPUbR`Y|SkzVr`cIj}>#F$nS!WHAg&> zGH)P@8{5`r)9j`>o(VZI*GnCkZO+tmV&*h6|F-UT7k7#F_t8TaL($Q&R#oermmb;i zKdCrkp~TWl0O(Pb*37=o>nn@%N;y7T*OryuTC!v?h!jMNV=2w?mY%ldq$ta47;r(w zwF#IFy<~;@=z^M$!XR8BhgEfpFn1FiT(^W*)gpb;j9w>FuX99Mu8A$U!N>lvBokky zkHngqYtbFuB8&u_S0mv^u?<>$Est!UmGj`1ij3RvIqmy<9WN)Mt3!(ty#MfKA5s_J zZ*Y)#?bgH7HP4!JcHF*I6hqmtZpzKe6E}JhXa+Smi%%IF1hpImX*com)cb_&GL=5Q zQORv6;~WrxiEZvX0%2dPNmjR&=1Uy_EZxWGeX1r4^eWw32lF>{UZO-Ce!pvWz-sCN zQ1#8?)GQC~bD`m%f`63VuF5OnrtvoDe(gv(_)WBS(8dNbX1fcG{3|((tnE=TsVy$y ztvq}f?7FOXpuK;8uh@j^nD#C7m4iIXc-K%XUFoUqm z1-;xCN)&AUM#zguIt>FD& zWj>>438;wqlw?zR)uj6Qdl+Z|BQbd!16_IfD5#|~bT}icigWjcq* zZr{XWbNan8Q~aCB?7Vn8QTET=lItBKO18%ot;V@;63(I}SJ#U515jOGxDPK_8lRDc zx5QTqEG82>T8S}%!>NeQP;`HF@ROl`g3C^k=E%ZTa!#ynVkt0n@>pVI^=~eVpL8n+ zkwb~O*Ok-wJ^X` z5yR!AenHt~6;qgQOek%)b@WI#ZQWh>ylt|Rc1p*;CX(9F$}`4I7AEm4RO{F09HS!$ z&%(|gFoG&0wah;X6d#vTBZtp`60~{2nVb3>7ADON-vzhCQ+viQtjaanXIWPkfi|VG z&nSsL1!Z9Z-TsH*l6WcPjrAPk?RY;n{A_O#PFdUOZ_RXCO4h1FE%AfgyGM1Wpa_Z3 zxQKbrq+WRpOMLxZ`E+G5Pb5#nhD0=tDMP0$n3v%iG$F$2=8j7K+kp5d21%|CdjvCi zxX7K{sKfC)v0D%G05C+Yc|+#k!dSyD`P76|$sZ;?9oAeGXN~&&8!J}NKxs)#8xf3s zv5|}6q{G~+@g~rhrIP7O8Di+X(crursH>Bh_CL}Bsuj(DoBUpCER^mPUf}5osMAFt<_M|_9TzvC)PQf@hSGVU}(13(Zi4dLJ6AC}Nox=S$ z&In_cjEedWnubI=X2?OcdIxukfTpsmd)5LwXGjrenA&+HZFcrdw|C(SaA zA`g#L;;893v7{;V)#b9~B;r5KtkWU-k4A0 zT79Z6oK|n!hehNab&qC3dwW^AUv1VFD8#t?Yq1m9RtPHk94ts*{k(O z(~reLTob6kFpm;}^moo!{;=#{p~BakOkBF`RbcXY)E{(9gj5*S8pA%|SyYrY8{H|k z*sx!N#1O`9WC*pH&&(mW>h(5OKRvpbp|Gtl#qUQW!k0`x(zgGEgWQ~#Kh~(xndjg6 zq!rl)<|4R@0_=7GZrnhPg3$T6q14J=CRCT&To-)M%v{>)1y9Fp3{B*i#HB+(&{2`vSomuD~x!M&>aQ6uUmvgUA0qq0&9wuVE+=k`AKzV@s zg`opmKDTybIh)NZ$+VYWga`NUH$GId@J^NYTS#Q_&#IxWSHYzBfwTF)J%!n~$6c9^|8B{O zJNM4EO71ni(=vy9Y(gZ+Os@RF6e{oNA!f(WWiy?gO0RKh?_d=GcSA<#Rojq5Afx#3 z*Vas)hf@6gq}~rk65I7J%|H@wLw`=UbQgev)MB&O`6u5&gC8|l8TmApBsxWhZwPuf zJ`HY$txaE&@!tAk2M^exn$E}r1V%RN>mu{UK6b^a5aWC|<- zB;F|OIGdAmSh>FZgxFJo$fYZ~P_|H0pg=FSNl|HZfbH55mQ^q3FFL0R#H&&aGsbt5 zW&;t@-aJ_(LzWi<_O@w)p!g2C6|&Ht3;>+6nY+VJVi!QxeC>7Md+oFWAwku9Kqm$T zB*7ice`FMf3pXa!gWjf_e9xOF#yhvh(LIE_+~Uv*G6WK_tZQP&YpoM0L$*SG-wup$ zRpkcMq&hjeOm_9eMZ0vLAlV8yrxh^0^YHdMtzde#Q|KxJ=^{d#YS7~w#n@AFyY`!3I{`MVp5GzqpK!HR>VU#XIhh!85l{OC2OQa}D?==Z33L+&U(xgO1 zX%QkKy#%C$P^E+pp&B585K4Nv;djox_uTsrUAq3+-pqwt#^LM8z4 zppP&byLDkSHP;drRh=r9pNzDZ+YT%iPukK4Si(%JPNAeS{*wH4JKlHG)AzkY5=GI! zq%Xtkcq~A#n(n3JC8o6$lHfrycas*rBU^MzV_gR=i7hRWz&!nzdnhMnX|Gz z^K~`PzIIAtyGY#liHVkycZM&Vk(Ju|Rb5_mvh~{sTLmR+K%95q!hF($tj9wiAs6Ay z(k`iwqjmiq4}8>&UzfcN)~xx%PT`ghFkBmTv=^YR|F0Kz?I?!ZNdxl zk|~rul-*3*2**Z4#9k@ig!0UbId&$$#i@70xlzK}SvDxofc|X8Q866VEC$`8m|D!T zOKS3e9d7+qEyooF!8UvMUxAAg_AKE=TQ3>H;1q31n7}jrz=&p-f7xTsdQQ%&4htI< z9W2>iMNQ)`H$BJx3+Zo&FDVp;bz)+?wq#{^RjRPGir~ICp+mE38nhIrl-ZTYExbZo zeMR{Cs(|gR5W#MHtaL49H~u=k^9zKP2ANOO&_ntC9M@1cb59K%{Fb%$Ae~T?v&GBM^hH_2aX9YR($O!tg+Fi4b-St9*ofm;GEHR^wIXa$>yRo*Bm9-}7-=#ZdB0Il! zix)*iIflqa&4|uMx_u^Iq0f^h%lQRXkvraHZb%czeX{eqNftqcP9Hq(Unm(#UsNc) z-Q)60kLh#W=+r@F&V-Rp_OpTG9kHfG4_5wwC3pSd|CIkzvN@oz*3Ez<6|tf=3`Av z_81wQdm;{qw7tcxry*91K<7ID$(QmXmD7Gju6&HQnB~)JQ1xo1DN&K89)H&rk_tCm zeM=S{U)ilm!Xh?h4MTA8@%EZ1f}_3{hyU$f@Sx{PvfG(hJ{sEl%0J5Otn0KG&rKql zi6`!=m!7GU**%LZh+3?8sa|?Ozv^?%i}}elKuUtuF*f7ET^Tt;c$y%*iR>&#%Qq>X zF^>H2{J@mIoNN%U`vCW`nOlKGLExq9GrNgG<$HxJ#N@PZBv<3ZWuqF_FAKg$dsAN% zFA_FFuPdoP!tSVjl-RnTxR+OLdp|@w{6loV@3i;b{&>rI4(iI~gL<7S6HTx4UlyAH zBRvFoU&>mu%CY;16xO%fG+~3>!x-XG>;|=g+MKpWEzhA_u~@_MNA)Y|*y`w(JpfY; z$V4$Lg@7}R{*fgvySo;eM3IUf-_^6t$%gPu4B$Q`^n}A=@cJfje^sD@j@KYbeFjRI zYfxzrAP`1^erE$!P6u9n2O{dHZ7(TYgT(vr=Q&49-puk+ydQU8#NLpb;c(R|M z*PwX~yxy@$Cd@wTN7oE~oSZ+)NwD+r$m^XlW(16-(t#6F4a-_@vjyK|lY4x23TFM2 z2$c@=$SrdSaK>uaRh`aCtDU0}-EaF8uq%d(I5G}R%V_9Mvukd&TH3^(~4GFK4`zcIT+ZsKs54Re^n74 z99qC07IKITkJ?-&)!}m*!*!h-&0LJNLxZ%BXm?KZbjEAzWca~MwSPQt4#6BZ^O_x+)jRC6H2WcyTs#}HC(Y*zv!QfLPS^O7>wT4` z-%DD}FhSR=u8!;u6;gUGx&oo=4e(tnOY?5fX@GY)=>2Y!J_}2ldG+vN71 zyCw;=Gm)w>!WPr|Q>w;RE*P?dK5NMN&P@u>mxO1zloN5_%PA*Uz2zw7rk-&h&GA(| zx9+F}_{iNY@9eaf@fdcXsKo0dsyK6=4EhRwVxYXg-T=6T}nUssNDCi$zVzp`-Rja z_yqI46vgCoet@xXNO3GEYXYgJ`ZI@6;(kk&qXNpaof&ceti5Q>$?kv1-0d5D9effi zmvd4TVf_81D!XReW!kjDn=^}Sb02#s1Q)T+(VkacO6*r>;LF_Nw7bKvG+(ywz@EVc zT@&TDLAY(JIyQ^fI-})v|8DN=oTec8D@dEw2LD{oia}8>HAcWGa*uHP#-O-X1Pdq8 zJ)$r3BOkd(cl0;E=5s4rdY%{H!EOjS*76jN4God8)?|1D|1 zhznjCs$4pk#MWBhyDW1*U>1U zaxFzjlG%u#eHE3WVZbfTVe0u2&axWrx-jyj^s%IigT2xVu6q*xrPBM zpt_GfA`!M_hRc>3k-HJ3grbGZ%^O8y*hv|3gfP=8bHqy`>(9Su&B`BpoXTIFGf@ST z!Sr-(J&D9J?bFWMPfx=y;S!s8ua?f{saTr4LY|B>*aQ%syHsx|z@#SX`}_1HdtSXI zeq_GwDYglSO?w{-ojiT8>1=96P4c86_)*=rJGpY4Y+^SCY>ZeanA~xfI#bjm?!H5T z_J&Mw#U?;+F=fiRiS(Q*aK?_kgqDDF#3{>@3(2;R%+paYoAs`tYNjbxT9A?B$)5j^vy4>iA zlI9D3x!3YXB!MffUqO>?568S0vh65K7eL=fus=K&+fE-HTDUoXIdgN#Bm;0vCh_5q zzx;DQWg4tcpxqA*WOas@xnq>!btBZXMF32i({h?8x82UWrBgRL=iIRPQAqkGkMKjg zJ=Rs#c>P8SB^gzPnnTBwDFaapIDkAc#dT5vh-2GFtcpi`9H~gjqip|l+o3F%g44MV zKXF_qQD#X^#36B$%gwuvb^5<+aIcEWo3y^Q593WA7eq10$q?1|UsAkEo%Lw3{-#M@ zZx+j;1D1@qs}m{zf1gk7{_L!YDYJU9dm-)(qw=-!JHm2P4zsjT2Du|Az22leNtBui zG%k&vJtSzP$yh=EJ%a&SrqzCUFls|u4jy>qA<2L1bDDn}%RY-2?w5vXi%1U*M6!@? z6!A!VQB;-g%_ztko6NVRUggYNyF$2SInKR+&F@`Ob=!zO`=u?-V98x^3ME>FDUlD$%fcQp z6e3PEoFkfp`p29Jw;H3Ie91pm5SV@W+zIr(GDpLdeEwX+H1d{r5Ez3w`TLauRVe@$ z|MEUD+=bKLC4t#mCD(wMEu#B1XDYWB?fF~!_bYzJuEuAu0g1->P2hRaBK~3w7g;nJ zJ~HaFSryJsI=89$`U03zBQoTWydLrY2~U+>olAR!>h#?29TZDS3&*ABa@zG!QaOKK zfQi?i9303IO%=s~r94_`CAxx-o53!OP4k=NKP^7txPx8ivfPuuTXef9H{KE&G2!$<#;6?_uFz)d&asgHyra?jmF--!P-(hO;Y=N! zp?@MxeyqL~@RO>WU<%Ee`0Ou&%RkQhVV~YHa7fi(GsJ##=KwC9xzgB0GpRtU>cc-w zE^L~t(AQ}3L|TRq({&1- za3PFoRAFrTv`3hSdk)x2iA~X z@-@fqDT{RgIpdFr>zkh6Q~gL~hL#52yCy;ut0y;wc7rO3_q;;9Y)@Gq8~Ma&8^=89 zxy+rDnZ)!7{jOC~rWGw&1UJ0OclY)aUyV4~szR;HO=Y#rclbfl}lSbi8ZYUP!!`VIt z*m_UK!IzpXq%kc0%li%@?@RfP$u5Ou%d4VQo}KUUmaybHh4=?HAi|vjl;$Set?nqz zW>h_$!uXPc)~~_*V@GchA9VLLR!wbsOXKJ4Uz~=PMZ5KZ!QXK;zoy}-j*lr3ObOPm z`S-hSC#JgMl4R|XmlKO*RfEIV2PKJwLyu}&(<~YH7N>>#q|-x@M{xg6YQ;*Mv~%Y`aN#vNaE0;;H3j#h{nV%F7mel?rTAW3EVBih{V`l8s9_1 zW#UDCA7Whd&8zKiu#|0>`3tO(|7*N%?p*M%e@fW1P_+8~(|to-H(Z0C-~ zuPzyeWeM$sthmlp@zd5fjS1P?79Ltsa@VfQ6)e`e>i{5}vtq1Q&r_X@^)&l|2Y6kq zEP^7c*3)f*_oi)dd_tHhHc`JBHo^PPQ*L;at3XPHd_UXaQs(2VJ#M;wU7?!Z1neiA zrGGUX!kijVnoXJ6T__-32MAjwEQ!WTlW*Go!=A_H-NBwHP(7Y$q7W4G^R8z}Ovy_s z;q2-htI~#9sh#;Foi2g=anIP4DLSSc}hAPQn~wwUUvto zYlt@2J@#rm+3l4I%bHQPRQjmEu8Y7qQHOXG8b7aBi8^`qSg7>sd|&752c_wVugI{s zIdlgvE4?6ssLA5KI+;v0=9k5u0h4uL(!G>~Y%|#hzd(cUwa!lDW!9V~9U>dBt-~`& zUAV!>NxV>U=-7$_iwvodsDL)Sh)1}U5{^uE2rh5(YqG@87Gzz8L}bkV+?;<-dq6afR;t^oe$LvE^z4gF5*7YMof`6Rh1Q zp&8WtPU!RbeH4PwSlfKNUs1$yars6`5H5uI?+JRr#n1aPU%$ICzL~cUs3k6Ra1yz2h=^(EyPY(dCv(r1ysRc+7K~A4QXP74D+$28TTo$LX=TTo!IvP2+7EoOIqQvFokNn> zk*T?5<1una{B?Vo#~xV8>4lj8pqij$k)^e?>T}D_Jaek_kNKYwYFsGk=G}rnC-)0^ z!Qdab;yf?bDe_-{sL7*YLFLuDS`iTTNQeug$} z2DLpYh*a~e>t~W$XG3a^$e{PHA~zg=o5Ay2kQ`yQg2OvG;KZfjKtw?|{V%QHz~Fy^ zpAI&O$9dk{nwajJ?v?rVv-Mb;M%!K&o-~baxAj#wC-GHmN~BxRhE)E53AFE&p?Y_E zf7172^&Y~u8*C>^R6IsLu|JZrB@nW8Yf&J5u}I+kvAaRv4%xZ4XdtysUN>p}LGns5 zA*#m32t}XCnxbIXmVk>AY~zP5A@H_QC#QyO@oS_d69d|Hdc4rG&JmVV!ViQczib9o zv-XPh3gD|f9qhg8ZnptXx@y9oi)pH%Ac zm*INV6jnws`+PRyim)nBQa9&r$Xmt*!}@@ghhS*MCUB3eu@{HZ4!UO&NiLwgX(C%9 z(n(|E!YCeK_Nj*{sX)lOo(20Bi@{pkV^8KgyvPK9Y`Q@S>U&`VO)6i$>!nCK=e~sm z8t^tT0yf59>FV2rcxUIwW~i>`^18EobZ}m!o39-1iDU$RCh!n-4`##?;?N@Rsz`#b z0wGOhdB|e6#G)Bys@&zg`GF_XPJf?K!cQ_kX9Ek=OQ$P3 zdyEj7y5ScNLXt;?bn6J?pbl}9rOF#?#fOX)q+sJw1;&|x*iUxk4=3xI99p_QI6h1v z{O1`Z_`AQFFM=GgkH2-u8XT`D<%#;q{fQDy>GIA3cZ!o zl$+5&Crve*-i0}+is@p4x(P9RS*f41(cq85S<@BA7o?K0PZ;*fXI?ZRE|1+K{wkNF zILy6P-x6tycmt3t^-Td(-Pr)Zl3^{W@mhIRXqgGsdu$TLq0RD`hEm0AR~v2cl3FG7 zHT8{BRiN^P(2n7Qa<7vNs(mtc+{GJ5Uqz zo2_E}3%rz=zkuMyw!pp_mht~S%cY-DlYiL%_c>%ApwB3yQ-xtMpqk%d93>IgEB+|h zdC}|lhwK{0`z&{l);4(9O^MmY;%g1UGC+0P{AKsAh1RoQY?l){k%d#Fp7duT+DS3tlw2^az zXDrvJG^@vv3y*9oK24F0guKHjLhVH5`ntd0+GO!(u{`Ee+bQd$)$KuCti5_JSvxX6NP4DChS{Ys04k}*R*cwpdq?WYJ5!0Ve#zoIBP_bd*oVUfz)!gh<@Ej z;y1R9s3fm=mc38+tHF6&N1Vqi7j{=ZZ5l$*va{Z>^9i!^0>Ev{)r>u^{k>LWZJQEc zTvuqgZZh^q^SLDl8A4kBp@zJ49$xg}9$AjTxV3cx?%{L1 zVmE-?mH;jn4qADfJ1|T|T?!sPh6APA!MpA)hBqY$H`G)^I8#U|d@+Fdj4AWVrx*=l zA_8Fl3Q4TkBcc#U8|Ah+a((|KG3POQw3J-`z*&-k>zMvMorM)dRTR zGPi^;IICrEWTx6hY3Bp!CRCeIH{bG_|5U4TI|6QlD;p-b?j4azxjA`)-&7%2U^b~F z=8EfdWmhJ~uLRV*x;T5T2nwIUq^GhqhDpUeQd8Yxmbigv5KiUj8&|HH|Zu z`f=8O?%UZxrLg7&?AXMOIw=#~KA(+RZ|$+jMZc6lEa8^~euG<+KS7{M7GbwdXGX3k zr6|@J22=b~p&b$mbr;n(nhyh1qlF~f!~sP0+i zH1M`nA5(eq)hU^|dc;VQ@-o6c8?ibd=LxS_kamZCa0|4D>fJ0yO@z0FJ09))#<>wR z7TjMGG1lpOefkENq8TV{o1AVs7PD|9Y*K^UjD&)&%|uh1M9Jncbe=#6GbUaQICEz6 zSvnx_8gPR1FQLjIlBK2ucT$SjXgzD&HCI=D4by^V;`fEAwe^I?fj%i@M1jOq^(HE} zqooq0K^WZQW7lj(YV#PZ7ZI?h4w&euGmz$G-^B93t}0)dWU2?}cO_annoIGjMoVB|&I@%{%2jkX_D>8@X$+YxF4ev?rmU zz<2AqC#9p{(F`)Itlgh*Ar=qjNet)`2`)+f-f$Kr)MLOhIrVs>MTD~@XuYn*in`+% z%CNqyDG`})8S;T51>nDrUYuBo`muMO{y$g3oP^B>3)m^+h>1lZA$ON|F3+zrkJx-> zv*nj$+b!?s)O`hzKOV=3^XupGDP3my4abO|h2`j%;2sCk{L}a;CY=Xe;By@&yVuEj zBsO;QL8t*O^1<2p1b|Bot>t`mY9D^1jyuNKkPs0RN5!bbQ7IBjRDK6OhLcHxd^?|t z)EQ|4JEbN(!vc+VR-H6y%bY*|CtPACGo$ScUlP(;1$=$(_W9A>wL0xl0o`9fH`}mQ zzyT-ua{&GW!KR@%d1aFG{v^?mh((pHEiujgqlWQ&91p173LK(0vAs?<+;dIv1s8ae=pY$x+x#Sotfp&!sWEC6I`6 zZTFAVU>*|(X&wtT43W0`D{N`!MkjF0UX3OU2n}omqFr_>;}>i8bLqFg0Ocmm0iosY zSk1R~r-C~M0zIr5dj?%_p#Az+c}t$R~geer-6VIG$s{PaR(7)>zx55Xt2>CpQ(k zOcoR#G*!RPL+e#b@%DTjy-1%VZlv57@GoS@RZBmKYhy|~CB(>Y0s(#F^F#)v9>5Vl zlyLgWVZsOHH~)O}Pk8z`^!{Iebsx}_J8n=RoJ&)(AF&Ssnc@*k*=66_#C4_qwL5j* z>cbya=MOzU|N5^J=O_0czWMF&>q{%!4aEX<=$NCvjE8N%Q330X!%`*JMxa}HhlNe>iK=j)9wQY6dMjqUJg1t!BVaGI z(%zxQt&#q!OYHquWUL1?b)B^(zAK5%989j77n(8xK4$bSl3Vb{{ny zh$+~1@z>1ViE?bybQSXKr!K7&pQMs0Z zn`>I3-B(cywpd}Dw9{nS9OoFNNF-QpVnS@WYjU_WmDzTrIbDd@g+=r9;f-6M8u#&; zcH19OM&r9h=WcqM8kH^#(|^)1?dcUD$8bIM&P}9Arsw5*?X0)@N+ zs8;0bVxyN@p~3%VmSwd%@-RDAi)$!Z{!Q+mR_Ul=Onr)N!~%5c{u=Y5#av?86;DUPam5$tsu;w(h?5MfVi15)&r{($uXa^(h>lE(!RC$LphnUE4T~xEk z7MKIb!-gK+?r49{r0MQN_>K*q(Vpeg<{r~8dJOH*g0XP~ulV3yFAz&bj*(!OisGx{ z!;fyiSk%*+v4lBkhH;(7ROq2GR#5G}FK&@pxachlKke2tpy&l>1q&_|r=>+ZnKWU7 z$mFy)TO^Se3GiO-ORIU5-3YY?|Xy%`jdfg$U68KKSTy^iWAQZ z*u(UC@p21TEp2cWys|KO(mhJUsDE}Un;?CTuC=w=*(2I4Wu+2OTmW0T?4r;-aZ%4o zC1ev%eO=9dS*up8*>uLYN)$xdmBv}A;Af--XIf7}2Ql^R9PQ(~3}azixWD?&VPE}J zDaXj~sc7=!HAf$o-AV2@bUc61dp?(UgxF1O``vc9K)cdPwyUbcMtPOF(uDc=^|tZ| z8k_pMxq7qEM^nCnVB`2_Jo#T@7CT+W#@&rObkfAI9>B-{V7*>>HSZltjlk&}Yu3Fk zm>nL!TPU%omZ{KXMT?q)A~NBoX5+BS1E9%O_acMUmercRxmNVeaznW9uhHfHg5^ z{#K7R6B2XQXz~DXyn0fu7AtHZab|ag_ld-LqTuvr#FAGjQhv2Cv-NzoLCpIH%Z7PacGMFN5LXwqyQ3AxyQv-QB-&c}o=zPm@)W z*hNuh?W!Qh&qRZmHLO*06>tqa|9M;JiCSJbXTQ8(j&r3`q8i=BWa#TcnLi~v>0|N| z@`i6BzOC9V74|uiu{3F{y2WRW*@)tQYw+>6CMBUT+N`jUl5nz_+`H>HoxO&C1+}w* zI$v_E66O48sJ8?!U>7|d@5sQt>+?&I$`G~=Xw%|m^TU&RtB(IS3ji`fS#B*8ppCqI zlT4wMKm~Y(&3J)o{0S6V_YX)2M{{C8NRk_0H-e39Z?4m->|w{ zj_~tUxrA_v4d0ga+T4b>T{UExOWx?nSt*5q$IZg#{qg0(5pjcTb71+v$sfo0FI)k1 zpl8qT8{xDU0j0m5_`$2*TUN9=E{n4&AXfvX&IBm0hId|K*tk0>Z2###UK+EIbAZ`| z+rx`UMh!Ew?RILG3=xJzviq3=_WP7%Ma1GL-0NJCRjd#k*CZLdjTg-;qm@j3{g&Tj zomA3PFF@yey4X#1RoF!@v|1;Wk^Zj6<6#>7x7c8qaSP zzW(zANC$Or!0z_b{+m|{MF+rv}8^O*AtpkQZO0f zVtTChg2jgOkv_XegHTJnN-Hs{Ue%0g(IivkatvBI^A|LCt=MJF(7E`DC)V8~XELb? zF+Yq-1Jn)UZ?r8Pu%TC5cuY=F|6a{Od;gR3s{k}TlSv=iu*&(V+0<7K9-{8#Tsi0i z%y1q5c@GYL?H<`uXWbKAJ7U^U{@3ok_=NGaR`rvt5esKqlwNL)~grXkl40hL z2D+MxdHZiC;!|I=Ga(x^RXQ^l>rT*SQknWL;C6?3^W&~|EI0q0wJXB&6&wI@7B_>^MP`PQ_XF{(f*d9I=0fiug}u)vmKyGtY&`1`Oh z6?9-vER?7fFnt)b0UX<1AD-H%q9`tl$hoBJ7GeqwGa}E$biVUy6OEaFhy)$$q|93Q za5KJmH{*hiM^(rbtsyP94G|5H?Njb9W8ux;>%ybJQ9MlEhQ5G&g$QyfUq|vqekM)srfe7Nud(09Glw|OjIpJ(I>L^d2bZoC zS~9#FO*ca;Xof78QIadFoS<`f3e#!oXC}QiEtH{ zgW6~H@UF|atD2>gkov3xL2lMTF2TC-sfeIE=8FS$fVS2=gY}}xpO3*%^19I}wayGE zZc{`qpSg!%Q#BlJ`GQ1;Qyp?%T zIe3NQ;4`WQiq+c@f9pT}D4#0ZznV9_eBgs1@=+uh+tkOsP+* z(Go}fwQO@ZynB>e^S6`I4XPD&5qdp$6{_Zr&L(Vm4nh~ep-$`-^iz2{@?Dt=Mf&WHir@l7}`d+59+E91T4D9sgac^jB zjO%^AetcyyI3P>LX7_GBK&sjJ>{>vIjb6!e2CqA;=l8wt{`tJ6=(RVw@|-TF3C51UscNCB#NT4ij{9W-!i)&7V362LkMR%aQK6hT2lRe6Pg zQ(ek+o77wvLCYVxdF{n@OU=J_mHJs{(ZWHN#V+}BKET8}zcE&wzO-qbw4uzMxUoSi zv5ctGwsutifgVp+G8mnFQ*xzgQeotycv4e}eLK2-reMu#+HdIXI&p_lPq8p5S!K@ zalfitRmv+BWZ#I7cKn8+{ZA^QzXHmfO(`=)>jg#Y{-|oz{AvO92Ylm)eU|>pyiLmq ze#S7JmRTcd!GRk3qI7++!INToR|D?P6}Q}bROn|r#-vf zhmIDGq&kn~z-A6&aECM2^bI)V2T5>-{Qzi}c0>;GCEbM(IK5tPK*HcfhAw&DErP}H ze!%B`utIMq7F2JeboH;uhM>=Mot}!W4$sw!hQ!SA1_yyf7iB{J?pIo7Nzk~p$T&U7z-@ARd@k#2* z8202Tgu+oR8dUNuZFOo)N)e7w5HPvyH9MZ(n~rhrqyKg-q4~`>?(Z(*nok*|30rHU z2Jd1KLvK(ek9_9I5i1<8v74JAXF++|6TrrT{gyXGoFEwEsHz2o<|w;`8=yGXybOMb zl*#cNk&K_OD&$a3{G)6UH3U|5`j3#o#78k*oapi;d<6BbjxzFeQMZcyu* zXr3&~8GoyrAJWlTgPiTS&QaA_Y;$q;K34nIw70D%a;DN8#vQkD3As8D7_z7uMK2k+ zsztFKEJ$Sb1TIV#Atn|Fdw$C-PlKgs%R^V1DMl64>Jm|BxZ4&^6v36xT!RFzB%Bf6 z;S(+EL!PTw(`llPhX5Vmh|N>_1_I5m9CGdEI0-&6!~h;U-Ah&rk_yEnWl?X&4!T?PoQY zuP;$mA{}|r{>grCy$_C*$<(O%O;}IlJ5`5VoQkdKEW^=K;pv1$Wmd*lcI|3~j$=i& zhbG--YMoZ`#i;r+M#lyhP4PGNe7R4cfeeI_NJ|1Cm0Bi<|}a11S=fbE1IQze&;IS@vI0w^pi5*`r%IHuAL&dDP0N1 zNvLMpre7Yt+V9J8?8s(DpqYxioXr`d?fnmXkO)(1DPy{EM~5`k^D4^MXEp6CRS9UO zYFMv4ZA()&LDXzyx@}&;qM2g94MX28TEiHPm|(=lqz*|Vq7qUh_-mOo$(RnkT`pTQ z8DyDM<{;9R(eFI+?=%3uVk8+eUdSGcAt&CgixDzEB0=j3ww*V5Hg>F_XjMHCs!zyIqZ4$NJl= z7;p804@UJDh5dI__5SidmPt!?ytAe&)*vc^(Lntld#-ST5c7t)mrdlcKhF|WA`tVE zi247NAAGPCj-N`;&Xkg-K`Y;1wr3zC(wS+dOnQ^$FG7I_DEKNlRx z?fg*|!YrzJyJ(W%nHa-i@;mW3|GWimMV10W;#Ta1dSNkE49tA1%SH!ZpwaQlpdw1B zIicg@tFMT;%*cbdQ-u!Z?6KuCcE>HQl`_BQ35RGg1!|G2$*gHV+JD7TZ>j20MNhu- zR&UEmd%D*N2tAK!qR3e(LjG>tIKH2B+eh3Eq4#B=W$tJG+#0Z~Zr*XPx#}y4nCT&05`L2R z?c?oP?3>(6{iWtil6&Ham+&GHtO#iC_$5CaalPC>*E?#Shj3;fr_)k-hbZS;JxrE~$TZ9Uap%HwU z!RsGYf@dm&4R_W50AaRS>S8TpQ%At#y^?6>pyS5i8$0sSLEp!BU{N( zIS7Vv8sux|Hf`TG`AodNQ@H*hXDj>L5PUm_*qEDkU)p`u8itmh-cWF?bzOFJq+Dob z3M;jUI?dloz$@=7*J*k0TlX{8nokOLX56cT5;uI;)AW;gQKyVJwP2g9&E@6yjx&y{ z<4odKT2xt(KB;Z@NYCk5YH;1FoX8vSdv>&ySeGaQ?Q)R%=0M+*=l<#b>DaA_Z=hT1 zH^#>;*3T%7%|LcXtQbCymrPU-Do_uOF4SDy7&y6c=-c+QZwGtBS_H#e*fPW-R> zjaJs=b>+35CyN~+Jn2|81hG53Cjl^UPP^6rOhA9V-e<)=>KHgH93<;Co;k${N07pS zOy$k`+qwUg$4Gh43@Lna$G8NQ?`Cgg@2uQ69Q#7EMcp-A7EEDzQ-BAbbDFc5osmiya@r+5nyY1zcyppjF5%N{wdRe#D^+&Or31#&GHcgP!YI_YOyZiP z2L@t(&EmW&0uc7o?bP$a<}Z*h{gXF&`tz*6`}b(q71Uqh7fM#UMP+>#Za8#&*nS3p zJveJr9%S!)fD7L4)sXbpEfiWMzA?sl+%Ihw6piyz$K*j20!>I=eWZGf+28*6O@BqnYwzFzL^@cfP{-@XEg_k)vM)jpzu@Tj*3$Pm zN|}u|>vW(baHlz{$Jbb78oq$o8=1vBj|DTdLcsU=#5+JKtHWic0MPRx**9oT%>exA zz3F=O!-8i1)!^Tc9qTuS_p(mmq~p|@=G`(=K$N0@ef(IqHfCp&FrAx8HPjqT0k?*w z4CVi>SexO@dieV-0JqhyFn-K3a>HG4mrVe3SXK@;GSz)7ynLWRe#N|w=Bk2$d}Sw- zV*#jc>ev-CSVZ~1e z7e$t9N3CMc+hVM4FkE8RsQ#*ANu?Q=czs8zm=19`EFsSVu8ohLVwB) zXyIvd)7@=-KXcwBw5z4wP?X{3B)wtexCH^?%A!i@A}X>sr(cAZLyP6OYG3_k{a!yw zOL-~SNwh8-S!E!o3>En4X7H!kkVcT_Z zS}V0y;z5&rX|phqsW3~@?3@yW2Q&T4eeRs|C7qnla+rmopR)#e?Xje<~Y z)6F+I_Z?1K&Za)}U&wwRXUkYyNe+ZJ++UvT`Me`j$6vaZ8jpW6_jy*`Y48Wz=3o*=>RE?-RY^RY0-G_?49xepWWlU*0%hxyqzBnT2mJz zw<=0SSanfDfyc|)nIhc7&<>FzaP3;WSH+rub|BWEFly)mge1ocCh$pXXO(dL;uc@q zmMV2;_s*N#*_vd0=g8`0nBh54eQ#|ngw?a37?ND?V#6P4l^Vv#O7)&plG-Krapxcc zl7QP1@c=f^RLE`%pUE-WJ%_)t_@wHYBCFI0cU}xTgEAmYMhx*1EJuBr>SAwoH+)5U zeAA@R72i7R#Uy-daE!V@>_Wh?9+zntdfEE6G^(vnfoNNDbjGkbZP8kdcwbVBcOR8I zX%qtv_kWiY`9Cna3YE_DaN@WcVJtL`dYSJ9cKy2GNd}htxsr^a zw3`pe>#l-M(Vl^ltU!m6`MJTsXp*FRU&)5QUKKtj=2YW_pTm5Co-Y>%bPX>cTr0>-J6G$Pz6h{NcTjepouk;vkFAo}uC>ZgHTgFH+%v z8bnbSuHL13ki0P$k}uu&F8+GHfnF?2Br#uVP5orF_G6Zrw#_*6>9hWuOj3YZnC(0b zyCym5gxJarI#KjnjgVs?zGa4;xXR@mLC&u=_H7h36iRdbARXE9PHMu| zekVHj|0?S{!cA}A#) z3PKMkpWHWr!oNgLsw5Zs7*RVLC4Y0+HU?r71!?u^l#PQ#{3M-K+p@+LbvLrL)& z42-iI5)jl5mBl3bp-*sSWgY8+I{?FZXaF?C$ZRU`A?&4w4~|NU-@);sNov*7G!X7EmQOwyc@i3VqRLvkrk}T(pFUZI$A%c-X-m0M zl>dyIqKjh5stefMpp&{1V870wPaZ4d2VbP);&0(ShG(NvS5vUOVq~dPDY=k1bf z8+O33&=%V@HKI$if|PVuzx=N0rI}BBT9S}QMF%748)g4PPe71!C>&E=Al8NmBfb;ZgdHJRf zq53vLG-EfBlQ46&ZaQ8iASsAZ&`q8<~wvLAv z_H9Dzo9NjlKJza)&B6N1aXRBLCzoyEdv<`R08uf2hNhEondQ(&+ljk>gQ;`pPu*X! z4&@2Qt1WZp0>%BWAHDgxZ#U%kaM`JrNRaW`k(dsi#eTkeRI-fcl{G8QcO~zVCYai% zsY^fyj2da8UV%V8f0x>SE_v&*^^W2FniTZRXAI}F@SN!e|8u1gtb*t7Ji4_KCXC0y zC^ju7#Li=<8`5GcxX=8_lg-j@$@b2p)|tw8GW?i@>!YVftxb$!kuxXUGMQflK)drv zCvCpMHNPh(Td{DV8+OKRQ5?}_Jhlm|f2`bzxWkszUwSjlR&6l;*mv0PEV8X~$2r{f zf8c83j2Qew*#~u{SP!&HCSx!^KBJ}o+hf|Mg+O^hEzbR7Zii0G=TgHy-M`X52h2Cl z$6|_VLhhnTDw;p7Sg&R%nGAfgEwmfFGJkWhL85&{%X>P}?#qI)S3A|^B|MXCSxPr8 zd7jZfqJ>5kQs@XIsYbC$9w(?z6=~M;q6hm*Z34TmIp4Muc})dE43)0u=BomD)PF4# z<0}{qnR^pfX)bem^G~E(cVjjQCh5Nco~ z(dyEv^K;CX<&ohl7P!dF?^Cr_p6b=#s^SAgQ<*sjuex>Pzhl4tDsv52LnCBQGA(mY zLNgT~6sP<$aB=hE#-WNuzlv&m_;WpcJ@}9^97q+H^t7n`m1*V6tzpV7RgEgW^yZ=` zP+$kh3MhP9TVhxign%xG74)m;n2h$RIHGbVK|^2fvMhoS#%b9dDrc$Bx6~UqCVs z)P{TKTD|}xEtL6=1Q908Yd9jBN42d&U*TQPY03#Aw4C>?Oe7y##x4Q*2+3iH=JIKm zX^3RLyeK1iwQ4nS9BErqn1nX3fVlEK_jr*dEC8s#qZ+=|2hkt9+()bl_Pf#5oz0ht_#u5ulW?t-N4K=b%}3}9ptGO8 z)E7LW&WG_)=hG)|DzVX+g4KcFJvfYEXVaOgh2nm5nLVs2z2zCQZ2nZVQ$2Ump4V^f zpM|c5on;tc!8D=XzMw?;(oI$fT6`^{dL(;8jL$Om8Y?8$SFR))mo0YA+sYKq>DYu#4B5Oi zrpM#+K&3g!#x8j#V5aeo@VU=Y!9z3D*^UBV`&4bm_F7OgCi)O-ImJMffsd0SkTAn@C4_s@secK7WKUj>xL{z%<^Wf+hgp0 z98~SFX_bbmh;9WpLCuw4cJ?7FVNIRSNBg8$y!c%ch~5q9LRr53Jybr z>FMWfYe3soA<#flv&Hr`LD(m2P)tt%0T!Z_n z8J^$lS6!u`LZzZ$oIqESr69Fdo8{}W1^E9V*W1r0hP!Za9c#A!VGf|vm6pSYsuKd% zZD@~Y2Q&U)y!8k}wX>>L;wJ7s%{>6dLUdM61?-Iyf@>AKLN{$|8z81jkyWf_#uR`< zi?y6B{2KnU3N>7Ox+@!Y={8~QADQbd8+;4(lRf8GnPcw1JDZZuAy8l=J*zPOnb!T2$&RL>CBfz*|Q04wa?MddRCx-_~(& z7Puq_5iu_wXic(a8c;!Rskk6o*gQ?~#9%r5{*JhFP``LS;K&QOpWtwEbH1a%{USrAb=^(8X`=k-p}a-nZ?TYn)_!Nt zCaB|5INMr}pC2TKL$D}Go@HR;UvPz=3>6Xt*qPUERbnyAe<`BR+2Z9ze~%+h#d6vU z+m9J5)ZP^}J~c58ZS6$vx1Y}QAqKb~7@9B5qddjune-2wKF!}^1f3VYD}Nv09|Zb& zJnee;oH7DCw!V$20aicbeP)wI$o*B>n2)W&OxL5b9W`#6#3R;r6%wiGMf~^SsV~DU z870PXCyCT8>FrNiHl@|<4gR1@B#H_d7{tvo(JHr|o}(d9wTKICmJ>PrBJRN?r(^}y^R-u^2d z)0CD^)*MbZvX;KlrizO&5b1BSLEL*LY7&V#JJ&o}ug^d8u=Ofhl5%<_VKpBhzwcLn zrF0{mjFdeTAwk3t*1R)vT>}}YKF~mvEv{4~{9@99@@sVCkN9+Swq)xq_&6I0<+y#) zgEwW8&+R?aPceYpkgFK$0dAqI%o%*gcvS`e9%Yg`E62-)I+_0oztes`E)VdsnrbU4 z$351#8JD7p`?zgrQ)w8RI!QFh3b(0+@$}Kaza&gHr@NMcVNU}hoA*4*jV#@N`EPZ= zj`TLn6=*X4ML?4dTKEn@RBxb&8T+FJv=8D z9(Y*<1g{!h<0~n`c<S-mKiw8-oz|G6o@K9}_y z&Om0^V7tb;(=O{{iV*3G4d+cUD(_jdAW;HJqcdqvyC@CLDSut!c-9?aqUJq-B(bQ{0OjAIr3P~aX@g34U)WWzjfBlMJ3UP(H@{ZY1 z&d(4jF?nUP8ECnB(8l?oC7Y~CMOfIGton;oh%&`y`M|N|0~=a==R)7HWcY8Mly)lH zd!WZGv+PX41Vf)fICHx;pn2XF${!m;ZXku%CB0fK6nL)&*w=Hw7HMk|Bp;}aqV4^i z8=pMEKd2IA@x1@`S#0o=*SKVkRx=#=JTpKhR44a#9t(h{$@bKQjL_^qD;i53k1{Ri zOIto5If~W!%vDt9Tu6W2c3g9IQ|1H`!#3H=JT3-Kr1<=;B1&cDF1vu*JhH8)lJhM0 z#OhvZu3xrz>4(o=j*Bg$t+J9BwPYOrl({Z>4d}?GNa+FEj;FW(jx2M()U=-UM)DDy zmE|WJ1T)3SI2mLeD?nNX828jHpH|r$E3A>vCQhX4B2P`qU{npq+&83qloORjP16XOZrtbK~WjG-Z7IoeAbdz%QMY6*$x}I?Fb36b=Hj&N*3^JikB9QENb#g zlA7JNslE3_u0mIz$AO)fTs&)E zmOrwCrKs64#oIa77_I}l^hKB=uyJ`g+Gz`R9`?98Vs1zVFwK?h$v|YBfUO@<1b=vU zSx*)xZ?HH<1icR6ORZ)oT=Akd^s92U7~vFW!GsWiKp3km1@cf~$9jK`BR{Xb6*+CH zCq5N`PR^(%(<>!8I_J|wRB`3*ro#T{YU_0X}-|DEqMAh;Xke#@)Dj$XV z@{%3-cUE2@=0w>UbzSCQNpinnZkO+L)hx7nfwJi~1GdF~tqdNsqr<`%B;F1>@}-(I z;vlv)Oh)<{UkQ6Y=X&wSwtLOa#okX$20|V!{n~xsa|wd8lg#TbOvE;W z{npOg0&s`@yr00P&6;v|eH8urR36yJJN+dIkG0b?*!BA|*gp~O-u|jQbUC&#De6Kf z>X6~m3FfE`z4<>{w(9&x6e2#Nso-W)+olC@T{c1!2d7{tcUO^ zTFhPqkJtU-Gl`zId+G19r?Py^?pHID*^)xS-;mX<7IDK-OJg?Nm=u`dd9HQsKw@K- z1Nm5->`vEauerE#yG!FQ_^VA&y>D9`oxho=&@;5l;4mq#76@qDSiU-BXvzn~jXbLe zxsN@#5Mlkbgv46x`v=QWVw7GcERA2H+N#$W)U|&6^EJUFw80ZbMPZ%El1XADTrp@veKPYs0ZqgygsoT(60n#@Z%uJWLwZd|mwkLLp_Pww z--J6qWdu4VG~?u#B3A^Ybp?y=VB#dM<~+?;d(C1a)OP6HVz+C!j8FFsJYO_G``v zofSM1XmA|tDJOHl2;0N^ClKLF>EtrT=S+UhT&(#4V|>!XxT-LN$dcb@By;I zwk9Fb5g)I2i(yrA@SLuLJONJ8*x@owKloI8=TMACcQ!n?)j8QjO_pKnC<>ZxM6&4A z3%eW?UDhhu3a}MP>jo#c*fvlr;Zc{q`taJjW9~_7MfcLDc(YRr&3`^>imFX<5?gAW zmDg0CvFZ{i()7I7>Pg6gL`Y{pL_68~Tl2 zr%O*#U51VH@_MA+Lg^0hAgS#4 z@DB;2f6>b^Bl4;UMrBxi@lncH(FL8gSRMjN|Cz*&wcmhiotF<=hEuOP*XMzf&bRO=!vYXK+?B6-r+lC}{E& z74`b5uRmTy4hzU4K{Af-I)YoOw>}970C7Be`J|JahBP6!kJ$88GgJjL!bew(mt{VMFskJS%Rsg>VuMrDPy z2CO886c6;bcotskY^a*o?mbijka`7O&Ha1@y>gOu+Eve+@LP@4o?7w@<*VQW97`Y zSntvi!$nGYAkmXNvIuC7|M|%w@!|poKy(@-hTY$mUmtr=-?_beEV|P+)d)gHG+$~y zzOhN8h5R!Qon2k1>y21A-n@Ed?p*-t^4SaF_dfRAhG(&c0w;4H$X6E;eJ|2dwq&RC+=K3O4O~@3+#QE^I~~)dmBNaq>^EgEm7`v{ypT zugd{~pOOOF!-|(0O=N*C(_Mp+$Z2V<8)_ zEpw+_IAn(w>21s~Py4aqx|QbJS?&`zk(%x%>R|UV;!)of`f69OL#~pJP35OxXA2$g z*xW34o#_IlK@LpTyirl}YVJ#y*wXBPcd8K$?&kQn>a;0 zmJFe3V9r(s8A~%jNj_xwiYT#ZdE?&)LXg7_|4zV>yK3%2^qUXHtp{@fSPOZGOPg9G zqb(-0=f86VIp-3^`C_gcki)ecniL=UTHi6p(&bO4(m(AA>IPufjd7SHD%S7!MN9OZ zKXeIc=%Ua~_~lH#-TD#JPUIO&?SMuX7t7B;XuENu_~W4YcEX#$y~&I96LsC3nVQuc z>`oU(x+f&7!b;6hTflFq+|x9AO|ss%nSF8TjK;FdhZssfzV?@?C&E7 zaow9gh&+-8#;;0d;Y^+|mw?>dH4LqW%xf0m--k#GFPp{7d2aPFoGQAq7rZkJrf-}m@YV%=&E55r&M|zVTi&f#S%T zSs<@=6{zf3bbkCNp)a4>+q!*bvnc#3nBczb&Q>P%%pFWF;rHi%yxWq3&)c^=e(Ns1YVc6wQ>Td)_yd#K z&!vAjliYDj|1+B1Gj2t@RguTL{P1NKQ!;lg${TWpPZM4|d1Y$mEHU)c4%2m8!`H#L zVUoLd0&!V0=7BB!L-2;E&}3F}o|5OvySLVw->tu&>xV;ollFy&^2Cn=(%%%$82P3V z0hu;j&1tViXW`(`D((b0zgTvJ{P`TRMO<8D3p?XQ4l)odEsK?hIQz-Ze?=yI5P0*MEuk zIlEu6+U?gG9$UH#r=`(rYF@TO) zj$zR0Xu;!Kql4h_XpUjNi#*aR4_>Km4U!Zcti&6hS`)i5EV5>2E(wmx$AbEn_w2%!;`p^QmWCmN*6HvS!SjyWv9_0KX};tTVo);!*KupX#UU7PlCHUT#0>_IOTm+1 zna#RlQprTH3Cp%CbKWObZ#*pov8;}%iKeZTbX!yx&VO^r))1#S2$pG)dWICcg>`{` zR6twvsl__LDa%ki!wg_@7+XPhT*f$DcbR@4oopdHTH>aBKu>oB*|8AtJ;A~RPo8i2 z3m>VBbhe41b>v2vj^1zC&Gb7_wGGF*h zVJPr`4PP$2ejeL@J7E8zEaPo%3k`Q9!>53sqgYIxrc%XNQ3|~UdR+2AE zc;k9$?Jcj%Z>;$BRHSOUAgp5PnPEk?spHx}VpnOTpa_MLO~9xFV-$FXAo}#mlJH|J zQmU^`ywm>;a%W!SDuMPZh)LOvn_R6ZoMIb?{=B?W4{=%ct;bMiFtg1VIW}g>(z0J z^u>3y^3~DndkD}1*BY*?eHzc7sUI9~?qA0E*g`lsTGnYI>%G?Qj`4?^V@T;N)IyGk zSN1IQAeUnjpa>HRr#wzTJ=#C-b|}qT^BdMNZ&GB@nJqxh%$f4g2*H=r&~ z2fLmV)(6M=jJIsp6uGKnw5fuLkC8`3)NYsM9QXrFO1sbZmBihO7{bbQkBp}hQu7pUgf9DZeX{(k(N~LUDKVE>{FvKiWE3*hMhn~S5jbm`;VGDt z&DFE5b}#SWSmKF#Y$u0~`w8imJ)NIs_1s?%A3z-X3LW^CT(ChkPXCNqPqnX(4;hSu zVcs4AdX1 zB~!FCf9Ux;Y)d(#Ohk)!3B7-jpF!hA281QHoj_Xb^(keg%e2GXUjOvdez_Z0F5Y(h zJAg`G&&UMWknp?&GD3Hv>RD$a9uF3;D$IZ-^l;dklu#Ayz1(w@jTBDO{JGlFM%kO= z9e{CvNgq(9+461c0@cz3WIjVI5H8uWNv`+?Rtq0>M7{~+AwN31lsGAP-4R}Z1r2RR zJtGzxEhY1|RhdkewbfZb&fZY#{#`I!&E(*|RzR947pih^Ms@rLuKSmT?)s^FPK`(L zny$ARokvrtypLvb`s$aJ4xAS8BB8k9jwiX1c{hbpX1>qGugy-#0dgv_`NpM}~Rty8>qd_nP6;pL@f;PU`i* zge`E#)4tKlt25Qr?juFP!!>sM_?br6%E~n~i zq_=?0%ld6pLlhnu(n*8In_kfrsV&dAOVtkX);hDz2!lpssb5Bh;YL|!DVtaU6q9ERfhY;bYnQG!fB$lBR)P6;A}%VhA;a48`JWY#&C<%fEX-(#Po@%HokUXzp4j9VyD!%MUu6*=+NfThQ!(B~R&S(a7R}2(f;a)LkqC z@DwYMl+LU#8)|hi3sTG|pJtuXfsAOXB8u4_W_V9CN2zh^q-GqVI3^T$$!WToRVXTG zM#zYxeal%Y<@?>_whdIOuza&tap^0X1>wF~Jp&w^8FzTw^eeAH7qci&Gi*AvY|vXi zK~tRG+B@*|g2R{ZQ_V;19sVr(c*Gy)$>bY6E?cR7UfA1~pF6 zChfRU+`&rSsPg@WZ$ujFzT*hwW$Y+gq3FZ4Xb*Wf8HSZGL}MNiBzasXVG$65fgbGO zVQ@SvoUtjZY1G@t{Eh6dD~vXwyj>6*$jX}Y$LoaMR{hWbW zfv#&YD#TggVb2r7vwY6J?bTtpceHqajg=bUk{v_%ta+a&i9V&*)iMtSEE}#>$_sx( zvQS9J~S_YpqdN673y(J^&Q zr$cxh7$QS-GhLGEf6KJa_*TLV)sLd{D-`0`=hx(lt>Z~=7&e8yq1GOL{YgIK0}-nk zQUmaOL_pQfBp1F_yr9VB<^kpxt<>n@y&X=jZKhyXp$idy)h*_tw}MALurJnf`+Uu> z%*=Qort4p!rz7GBnm4H$V1NRr-(Ee4%g^n1FKu;_8?wHiC;k)kA#2bh#lzn(J8Q`X zY(?@Es3X?m8{Qg?H{=HFgH-BUZN$poQHB4OamF0_`FGWoH2>*yDEskm=g(@&cJ%b- z9<3%yUveE9!@8x8rG3e=hVX7q*wn^9r+iZ}%k){Xr4H#N&TLazNpy|L4@i=2yd8X~hT1^rQB^YHa5 zPNIH%z4wkP^KaoMz05q}ICygPWX-0WU|qIGlQGska+{Q5Pv_j7e;>iR`ZVCsSF3;( zQM}wj;VV1+Y&m6u5@FWXQORxj+pO@F+2Fy>ONj=4gQ=&PS$Q?Pfp+)#%V*fO)Y=2a zXNG!r;^QZKPnG#EFQ_6s7sg)&@CMd>#`zMLb@TcvPW}=iO+-vx ziMTadn^0S}^vTYDx|8Ddes+g^4K-RJZb}5ypX%sp5W39f1&hmb)@(p*-MMG^A#-73 z8;r5n^UOvoS<)({#-&nBu}1{G=19^9m<_0Q6f!$GZFrMZD{btEvAUUYlTf8R`r)U*5s6v!aUas141j_N>0>< z)kn8`@#Lsu->)8IqO?0UbG*5msTFB*mFbVE=ci+G+oh40{M5tF^W-8@$W_u39KW;4 zc={uk$m;tdk*szi(Fa}gFQY#_^p+)OVN0v9vqU|x#*zaI|UudJ%wJoZAMrb_Hbf?j zL<%COjh*E81(eWBKBTc`-gXUCHK%JN!%JgoL%%5HnJNvb!(Mw7WIYy8=Ss%*5E^NB zFQdwr`f-T6;SahZwZnYZSr|j9Fk{2B?uhMeWpJr2(&g7EONR^24CR!ihB9Bf8$-8; zU6<^dKTYbQCwf^&D<6_$ZW+x6qtuQ|YuSaOz16bx-lcH{wa(G#HYhI81BWtci#WZV zaJP?zWo4zFx)vm1yvZwnV-2ENdDU}KdlTlZ>JJ)~l_vNSedrOnB7Em(+l~c~p-8e# zk!@0X1)ZER*VkW~LkH2?x^v&BdQcgO4%{|RBOkUx$P%5eN75&UOugc$hwu}?ws#L( zf5v0xT~ZAnZXMKkl3;7(m*VkL>}92EHplmWUh%W+;AeCg2tnX3gQ(HTHSOB;VYgwymH<@_XIedc)nz^Rvm#v;43lEAEiB!0ZyWz}trvXD9D>l#s(RZSNb9!bKzM zHPJ|~099!QLuxSap*m(r=xHkhn_}D7a++7pIac5+lmo+n405fh+6Rj`0BG=QzB7ot z5aWW*oT+dQADbvwfO(2Sn!tw;xU}8R7R^BDmNscJ{)LPD;^JjG80w%BZn+f0^b#Cj z9W!aadhed8&2Hpgc`e0lnGIasrH69MVMf->=H*+Dhe(g0tTz;sQ{5~F1=8a%6WA$Z zyqUdA%jAVxi;Q4UMu^DD;U9l*;^zr#wPualTA4H0xDH4mi9fe|N<9dJwu~d`(R0~T z6&&k0)VPKpe`&%<+&?o6A~!d2Yx*Z3>oJtRf}xf2)czo`VpA=2oqd8Z)OvOJq_Xh# zO)rg3KZYDsVzNL8ieWBlOH9OVCF$psh)+Ax91g6vbe$OYTWPQvbK=Dg&C>L|4l4s~ zN*s~%#u+xs0WIHSsrm1ejZ(xGg>pTbuw9wJ%Zt?ly*>Mooop2Q=jc(!?vy=tA#Vgq z&ubEOD_gg>^-n0MhQ>My?Y8Ip3CsWKgg{u7uI1B4L9kI|Yf2}oUN+x@_s28be zrCCb_g1P~zxiCMy@l2HiOzxHQh|=6H*%?b8IoTRMT0YzBgbRYhQ5}hd`GuYK8=ef8 z-ur8H)77p@DM^=jO;;BWv<906so|F9!fW?UYzAN4`1HtQ?5!!{F;r&m@-Sp=RA<@s zQZ9SEYoP`ULR6*9Iz-w^NoOoW_$#EC+?k0AR#JsxFb~L>W;Ifq-t~|*Ye3<#wkS2Y zpG{57iDo)@bkF2dBPg3HK32U4F1hJhje+5@EFZd#EL)>`9KkO}Ii5#I*ov+k9&!mqF``UD<_)0i zoZ*rVBjxpc8Tc7?MS+v^0i?vp`mOjU;qhtlhtHfu*5*inFWc$}rTaKlmJ0&?YFUOk ztP}T??k?51t{tUN1HVI7Ikwn=?v_~F-^0gSO{E9*H_Aj#D%#tG1tG)o_rAxTl&SU% zMRb*KABLaumfa(%OwUj=l|N4uMUb5X&C8virhC_Njh;~$2uyD_Rh3r~Ah7|(72O`5 za8Hg&Cu2Krr}o-|o^MR89_zqkUp)cY6}1t|VHlN`ZphNT`439|Gv|Jb^NqZBW8@Us zc{e!gt|=NCF;M9Ot=_z0sfE+-1J)JY=^g`W(DT!nvYDj8R>({2q3R{8=8WVvdH$d* zMD$N>*;r1`5P>}CGe_r~ponBCO6;UfG4=nNid1gDwd9}57@AlORYcl$jjYt3NOuOp zfWU}iqNqv}llI?XG5E_nhKD4Wm+Lhwlhm<0*Yjc>pDKBYYJD5Ck*JcxfodS6LRNJ7 zSF1^qWMs#+*uQ~$W(^l$hSvedu|k~q&*|cccV^?h#Y&fx_`gN{w0^uLq=|3|{YK<_wxYG7Yg`X1hnJjlCI z6F0d>QmYKxyrA{wAnRrlC;5FD@o%L+s$mjz@#i+u!AU0$w=Vqj3+TL!!Nh;LZ~n7I z|LaotnyukV79fLfj9(u?N=M5x0k3WNTA1=s@v)(#U?e$bwGC&(razT@3Hwjd%>Vgx z|KFP)vKi}!T$ffnl)mdbB!YfN9#hm8HF|tF0`@;BvXE|>j#8EVyq;z=#w(4ZNj7s3 zvKHP#a<^jNN_+Y8GIz|$x*IHQ+H$;4O&{Y?O#0-Kwaw!m+YViEWS}MTm++BIRrvBD zPjWGDUxnVMFLN5vFefd_`<`4|?03daytpO*64`=E&d*webb8(oPTEM>Ohvqgd^oa# z`v!nDo5_#m74lWGSCg0-zkivb6l@3cR1=~Ex>G7V+bF`70!dYbm zJ#?Y9V=Xq|HDR8Kvt+%XscS5a6c$)&kINc6rNf<4`P^L@8^s@kzd0fli(dG(<<0 zqRnm&Y0uXFjX=@3Og>CtGTV~(<>9E1l}o!I#uc1}YCe(@Hl+->)f~OrxfEDL)!$G7 zId4yfk||BlZKf@ioyzN}SkZ>@DR^naocwjk_)XRrjepA5eH}0-#&6u$DCt3J$&@#h zpc|YX=~NUVENcF|?PI+R8GCJ`CWQ``)PaY^wi|4uSDtF|=2~UF-6I(XJ7!&yY^?Si WlKSfux&H+G-MDIbg>>oOlm7>bA?knt literal 0 HcmV?d00001 diff --git a/docs/images/minimax_logo.png b/docs/images/minimax_logo.png new file mode 100644 index 0000000000000000000000000000000000000000..ff8f2787f283ccc07a17eeaf9260a24ef7e85925 GIT binary patch literal 121469 zcma%jWmp_rvo#O`1PBt`0|bW<+}+*XJ-E9C5AN>n?ry;?xa$Ce4z7cI7E5D<`>a4>IYVk9@#-*%9W zieiEgRg?IKZy$tB)Fn)1WFV;Dj^Q96L(Czd|6KC6;l6DU5btszA>O_1A^)7qf%@lL zSjn7s{~SYV{<+XeZwvtff*(RcSU}kg@}%=a&KEIfGF*+DrVTYZx>7c0OY>84K{!7@ zNLmA<9vB2{Zp2M}X4W_e;@=ymip{r$jkOXf7o3|BJuoM!XM3Jj0EXL$SFh*`#FSKI z7}~%X#7~<17S65%d_d7PkBho;U6sycl|&Wie6Hh-7W-!Q)as+o4vg5;dR9ibIg73x z)t1wg-!5v-tKRF@V~noQkLBL`sVm?e%O0olP7@R}G$ zYtU}UFA_^n{86+B9e2@&C&w|~GSL;P^_E!r>P*mxkU~8u-~RtQzaJ151`EL$km#aj zN9F5d3T<9gvb*65kKC~DCowwHkPW#JUSAAAt zdi2~3<}=-Pma806dYmi$#~MOlXyOE`UU{1wL4!#&<(qP^c_0#gquZu z{1=Y}DU@>!r=HtzUHsj~U(I{2I@NunbJFt;CZhvEzD1i=Ex(&*w+CuYPk-+)o4A;- zlJ~UBxLg+Oi8M+mqZ&@g?Zz3FTP^H&srPv^IhS&!-rq;e&3G9(wx5W&c&Z(k1%5tB z(#kp*o=o^RR)Q4LH%6%uJp7g+c~-CExlU8FeL%0DKh2iQ&Z)D|U@ySqzeecfd{?@% zx9m%0nU$n=hRW}Etu5?M6UCNK*_wNYiiJ%)c1zs7*?U1_a|hxZVu>JPunhh|tPh6` z^?&TX9sM!IFSGl)PM?voFi)&e)-$m^Q>{=X>onZ2Rkp-uzIxT|uZ~lD%gJffsdfb@ z=UlE?Q7ZYi{gM;SIp;0z2jo~;mLAQXmm_kfuIo1^zxQlE;Qe32{`F-1Lx09LDtRnc zZ(ppgAacS2>~;_-AfM^&Ziw`0cd1XU@gx$pOc`2kyz2}%&H_h5s7A?dwPy1LKpleR z@x3Lw0e6%eMk!2QB|7z?W2-G*5L-bCCRLwsrdl$C1#I7Wb@{Y3F{&$lrN9>Q0#d=(hnwl1E zT`}%{+=29*ESPXT2%QymeO{=zr!HmL4sRE}PG;dmwnpyL*7!SdnBGF7i_ydCaHQ2+ zF_qJTnZ9cWp1O4}h|UK8%O$q^Hmnmr#X%pe_swny0fH<`1qng4ihD{$f+#_FL9M}M zF5q=}eqF|rf`-WR%(LE`)5_@_?q2FF)8&}(l`NyolJN}b>hJ;2%j0?K^H`S1?=n%$TxCa>FgO1%Klq=Q-zKKt7~wo~N9#+q|F8j+2Yqk$YBEWRVCz zf}*^0(e;OqX72LUAA3lM*&P3G$NcAk4w#@KHC3_!Q93Qz%=@)qTO4UTP91pNRiL=Xjvg@mVxp6be3@()} z(-&DOK*s>O;VB*u7Yk*|OOz+5)zz$d?NT^!=;yz2Mfz|F_>c>{6*t(B6(l_Ghj<8nzxj z6xhOZqviF->nH9m?S&{-S^v4ZehCA1j*tss$@`wG>qze_AEvNzz3Xx6lm*@H>}*J@o8*F9q29(rZ^rj8$}3{tCOaujXhjwtHmA##?(nnGslk9 zO9N7rWkp~KaqAx6!6wP*se;*-qks8z(!xV>hdUkmu-Ri}Xcv@wy4&{#k*?`Ka8aPtq-I`DPH%5DVjVOMt&QYJp*(0cQnilCb5?eP#%@02Y z40mff+nnM8RlmC0a73AmWu&M5V%S^zC7gp%L3T^IUGt>EhkXj^zU{IPPw6(yaZZFCHfs?tMPmqy)Pyly8srF1Kx@Ahp}{0F`7ofFz#A zvg?3u&@Hw~9rf!PnN&u477|v)%%G$i%#gy6NQZB?BP`PRj9IobD1^@mlxo&D;E95D zufGCZQ{<2Dm6^f(Os*$8 z;6jBFPuIXefH`}w7hW)G87r$~ znoV~-Vw9Q34(EX5L^x14rQpjo zxmI6~;dU{BV`n&ZCcu19FUJ&t^>E4uPQ`1OGUVLBphK4rZC9r9GFUQ%I zMNVr#XH~9uy#MY)h>}gNNcCgz!i`ZqWdV&87=R_tPEe~(FclJ`{`uo%pBuODP6#() zM7s4#AWv@XZYqrgmo@itoYHV!FU6uzan;JvG=*A< %`H~QOTkjWUMmFB5#q+uzK zR&z4@-L&iJ*H_4=h-(vAKVI1}!H;-Q@l~qr%>8(&EPsXJK?F!JK4JBh&dTx%(9T>g zlFalEChsvsId6A)=I%Op-5I#Yt~oWcdp(SX%?Jq6gN9N+XAJPnYXS-WfA=gsr}<*5u1_?* z!E6OQcFEXUtJSJiqQV5!$_@mWd?_RlDPq(rPz*O@yS=)hnm)ZWfnS9r^7YzpvYlFM zCb|V#>fM1#DIS~@KYrRdW>PkB&VvbtSIa(1s*+Y#%eSZ^Ccr#&y8VbRJJ#`9-MoI& zpWU)kXX3kRvO-$<(!XMD=HYsd{7vG-70(V$n|RB8gi%-7Er}xJTKfmq_;Rsohj;;v zMMAzn6;PU$FPOgDE4AEC?tnD<#Omfd96A(Cwc`V}SV&uu`)eeJA<}O2V83_k%8k~h z1BmE#+8}+cO32B0>)GJvUGe=EJCW_MOQZ|5W^3Yk(v9ZRT1VZQhNeq?6u)I7>*lom0$DlPY*V%=c$}woBZVGR4^)=x5l+zV4Nw*@bHWY=vM z^{nd>+4-*DOnoaO5+|$xy=F<&-|Aeih=LhiW*JLEaDJ>JBZCjW{pDqu&Xo_`R=&}_ zOaB=gd~0=@lsvEVeCM|%`Ie$8UjMX2-+~UbB*x_36Bd3dC7}@Cjxe$1ux$X0Kn;j! zE_bb)h~*HL({+j;109A zob2~D<~jOyrAR>) zx-P&bernX$d&ffy2C-6Ja{*!Gk`X!>!Ou6SyU;Y4X78bXU{fQzXH(-Or~5GddX(hWcU7yr z@U#iE0^126+50oSF5B0O`JOKb2frsU! zoeePa<$Vj{{X|9Lz z@uL@H+t%Zd>O!!+4PVDg(&B>7ZHW3sYaC@QD!l;94}66M21OCB;p;E(nh!6j@tk^c+(rVtI9Pn}@_ly|&k!$p@_zG7qhqm7exr9Sw^vuDFfl7ic#SL1-EF zFszh=XTR31u+})+O-a3`?v0-M!uS{d#;KwZev&MXYSBI1R`CC@3_W(F^3~lYEfn7j zm({3UctuYVq9&Z`PHsrQE$&t6zFwfF45`Zzw;xxT*}q01Y1p15E4Z~k+sg&%5_0&3~OJA z&3{t-v8B*#=Nlb zCs?80p+ zo%J1qZdN6;iVDsCVQ?Z?p;S|7CLRV@y(j=@_gD$!#8SUd4q*Hw_dc$dB&)hA?<;8j z0LQC0dEWU77ffAKtmNfYxH2&<(vJqHz)8OA|~c}Jd;$^_}X-7zqu$P zPMN`T=^47JqZeqlUr3gW9h9h+e8o9dmx^;{N%{tOmi$(l2!;ca#tHGsz5GGQhLY(` z9eg+C38H1Q;{v#(d+CQoU5c-i-`WYd%0MRc*>>fsL46rZQu>&JW(Y0@{+g3UzvLs2S<@hhPnnB)=JUhM-x>+0EL!KE^K4`xKD;pty_DDqxi_wY$FHa!3 zA?;iujvA18{j06!CDpvAm#ZA`K_Ar0l{MTDqGmti(%xIKWkqK!n(^_JRCBtoHcZVI znq}_EFWc(JAvwqpPU!12Y5)=5OG6YcsI~aH(n_-Mrqp-?<}*wJ}Y;;l0D=} zAEGrvBM9_>^#;LYP06(GS1<8P4tEHrbq#D|E^uUk#kIZG!FZ9Gd^CIFlj{OUdiPkz zp?Y#OoAIiVxg56W*_Heh+P$B)_IRN>lD77ryeFpu z?mo;D+~~RI-C04Usrz5k9*}NzTHk3#)jusb6c}h=L+OxwW3U%U#@tguzRL(UkQ%=< zRF504`2<=OH&K(ANu`hcz+HU_dO)|bVn*pn{&r6blnT~o-E;=uI$vAa(y$Es!R%*$^-`rvf7UY>JISuI2-P!8*|74pUkv-|2 z1U*kv(GWISS)twSuH)4v`N8}WsOLV+P3tOdV%EHz)fWXFK(MROw;MHp<5r|lOmSfg zHgrcf?7jip*uec^BVE=-`ntmyT4rFapgAn%yoHb9NCFs9lDgP6XCrYmHiYZ0E}6aA zX(aask8>tXmZ_LiH0`o1eN<6*2(R(m($qE{JV&sHRpt@w>7Zqr&?voDR_m26=N_x= z2Pr9|L#dMjeF}eUe3hx+CVJt<8GO5Cu##Ep_Z_npJN0rAmrPRG73(jo71+;#Kl(xu zASivPfBS!c-AVa9D8>DpCmb}0dOoGSmy*A>%VE|Qfo0K-DJx70+saO5^ffJ7@$pDi z&w5@j%Yt@h&b7Q_^&y|YIY}Qx@0II#e1Wo^mws2=|7vy~F8si!AC}er@Rxw0pMY`> z!m4_y8EUqy!1*t2-17s4opAMKxt;+Qx-0PJMJ52zw)Uf65XlcuMz4(x8Y~Mv#r7+y zwB*8hGAMNw4~QuZFJzGOVEV2O=Rx%&!M%GQ(?mri36#&`SJ1{q*vu5&m=Dp^{j*cC zyJl{~yMiZbSH3y;SN-L>=~hPt*VOM9KgGNkNoCw+IP^AiwN1)qKdNZWB%BQn2|Cy2 zmvcSLeYo>yqF95@{z^@UF59&zsSqC%9$y1U{<57=#G0` zqrXWHeR0&D;+?R=XOOp0T34anakW3J z6(zgkDI&Kk2C_#zD?Hp5N4PH!`W;f|CVU4UJABG2{yxbMsCx%=JWvV^ok8@}^?LW~ zjt%1*fWvFBaP=!E3P6}dH~dQicm(1VEaDWj9uJL8dcVo++ih}Ww1gSW4IspN$+>Ty#zH+ zSBVg&s0KD-Kp90jHF5uI=r~wz-3J_BZJmO|akYEX+})IAk_|OUN&gNu)sr&n$Vk#1 z2@YQ3UB+O?yKngM_Oan;$#h!#MS9mzY8)LC@RA?+Vb5bOBe6Ni=OrF8G^g6k#}m$& zKhsyl%Q}FTjJO&PM9(tsdvJoykerw(W?BA_D*9_EZpcK--)!QW9{x*rlL_{`B?)sQ zORZN{fuIqhv4FI=0RmEJ>O{7PT`LN5QAb@;_d3oL@-shx_FR%yEn7ANbzGHR?N3<4!DN@M zyJxx1Pko-3p!zY24vg7l7@@F9>!T>LJMf*E=2dKxurhGTCZ!r@qFZ(WjQ2e1|qse-bV2WthKmBge2qC~% zqeji~Z(#MC0{Wb)#pW;pT*w<#|Lvcs^@BJ@41z|(&ha**W9iYGzen}z7Sb4D8UgxA z1WRV;WpFUf9)_gW=cX${U(B8d^|18}z6}8538`orwA+_DZA3IBt+7k>t|( zBo@Ti?Rpl!B?(XVvxzB8}4asa(Xmf{}B7 z;R>xx`!q+b=J7O3su1QqGnTNu%Nr68HH~_4b6KiQX;ZU0 z7`3nSLEs*nq)A7i4OKu%dg%6tDfWFHn1u*B2-7(Uv4m(kXK?OQWpnSVy~fT66mZ7g zUGFO5RqRsYE0q~_wa8wS@-FB&Dy&Obyx``6Y5$nvmbaBmq-nPaCaY3>$asTVq?jlD zh<|fzFu@SKhS4rG;l57&J>sSWN!h50uI-yt^C5GMCIQA!8?kT6#V=lb3o3eQ=Uw z*nh61jC+|1+BFQC!MC`-eMM(weAfAi@Vp0_Q>wYQGOFO|vJQb@Z@7(pG?3#k%4KQ3W&l|W!iT&q3Hba%e zQnNhuurJK6r3j*0+p#F&C2!>XpfC|VqgVusmGI}mRhp_7&;EuT0AAyYbSLA5=diA7 zKfI4Ya1ZTxHWNbp$0=$PwQC4|AwGZ9T>;;t1OS0ryvw>9LrrXvR}AKB+M64YB`sv7 z+AQ_fjpmKwU1>#=9ge1L@{LGOd!+SArMzQv#^jQ!XZy?T3ZtHCw!ssC;pA zzS|D7^gHyENitn;ql?(I55irt$?F);%l$`$8_PGtI`|}lu}2BlY;tQ{;9d5)U6|H- zYv+pjjRq`|Si_6n^Hd@!ikUbG@csEEgpX-QF;hfG$MR#8(@S)hBdg{Hamv8ThSdk6 zT7$k!pM{3<9T%e|PF5-}l5c9#lA2zzq2y=@zYI|my`cRuHTLy790rhm5O~csjt247 z%*uVsLEas_4}fy>v6J`TJ39g$s-SuL1bDZwzM+97=O;2;{L|SHxgvzEa2s$)N@YjO zZK$Q(3C@y=c!$TlNXVv&n6LMJRdf^yOQ)DsY3Sdsa5OozA8M@AOvwPQfTQWgmn!~A zy&`&CnpZlldpOkH{t+qly-;hc^}#Nni}!jy94BUA%->KrQ13~8h!J%{fSRxl>LRLi z59vFd+9E-+#8(i>Tt7_-zUO%4{Zuou+?KN(*x}x!HUfVm9CNs+RG?=DXLMd80iYr+! zP0&cmsz`WUs{ACDiq6w3r4%On24%%2szx43*lQ@ za-Lh2O!9~|?%ig2TwijoH+vp>L7p~ZgSPPXrnkht8WPlwCa4J!2|q;o1ocr2Zs8$J zP%ekbscrVJ(X^eP({iR3uZ;_zDnwni-*TjnopDR;f zla2MMrvd5NuUnYgiSi%=Gu6MM6(T0&8Sl@j&(F|=R$npK|H%um`JjC;*-THn)c7Vn zJ6{~0CXJ$_4q+EC^WC89PM&r$lO_Qc&88DHQpt&A4V2I|W!GXYci=7&j>C}JyAWdO zN}^m^(OS6sz7Cr4?{c|7obou(;RF6&tGCjFRlKeuS?sG%I&S%_NIj6awL&Wot)vh<c&+leM1^9?2(KbhR-VVgxfz{33w(N^7(m2F97 za%nkAQ-LJ%FU+AfDfyZCI{}ETMNKNFRd^(3&g6YG>Dq8r+$Eu;8)q4($E(WqI%lTD zwI_nTN1Z>}WWChV{yVdsqZ%22{KUjupzwE~+vbPJhA&G`79L?Os9`Hm0>d%AR4oqNB7xb-!lnx;`a} zP;AeOHQ(SQgnq#^AGLfE{W=#ekT3|s&oeR%LA}5+r+6XzddhU&vASn2ab_%|(xge=msrMN|qyn&_f zb0?)py#0j_<3O4n-GmY%*xF?}TS+(&z9*gISy)Y{)eFg#%10um)k-r$+I?PfQb6ow zt_ouUY$QGEBs&^k)K>YlxAzwCfHw`44gxkU;zNEs zqsR8sSVB)Dw%h7~wadOfG!-I?liBjQUMCwJQZx^M*dWq-mw4&hWwu;+-`9h@B3*o1 zgCPkyJ>CFjNnou(tVn5sX-~-tZPZKrS+(G(j+)R0|9w`HY|pB4u>U)-x>lFmnd$Pn zoOzF`pO__2JSZeeafMoLG{8||r9wAh)M-N!i|ZAmyd2a}p9}$Lxu_r?D*a6VBqkL1 zeYZ3*IaK>ZI5J*iN9Vbo=;d7LIL~-%w=|u%11WjhOyT?&8yn;Dqq=Fq*63QFbib9* z=tth8LuL*38#nU9IHhQzSSO9EhY4!5_=B2l`>Kj%H^=G$iP;n?J>|eREK~lfoOH=4 zE4e1eyWFbseu1TPbKlg|hwh_M$A3^*56b7Q_14BtAN-~(T>p3js=u>}2x2JPUy;1L ze6yO>DC*516}!$RHPA#g@oyvO$)V0_yRn)fKglg0gbn1WuL6Wgg1vCYY#a9|+92vB zYx=d!=%Mh)A$X8(&Z1bhTXjmGCqTy=vGuoncL<3&F-_S}mwecpxbUO?d=`NR&Ar`3 zDw7L1F8{^kBNc+n`zWUdBL`M#wXi}x@g;P^7TOJTGVO$7OM3sgSuNEvVucWmqw}N#)9RGzgh~ zzE{Qp1b!>eNp2;hAUGyps7AU@v|Y7P4&+C~V;93iq5;yl<`ol^CXqohrROAm5f1pwO4Qt(lJl}uGKmy2?olk3 zw zj?l1^&XR+tpF@^WON2=s&c*)rQ}eSyVX0W{yzfEQYWJ&YBZ$r0>MjIz)%WLzPYC88 z2}AeYRB~;4$R|I*{Y1PZqp)&`4>8a?K<`MfWe-K=Sg_K*Nhsp9h*v!N1LTn8pxhM+ ztt5qPLFJE%o>661Ri;;2M@@CMY*}|{i)!7H$vbDFHvL?i8JE1kGsokA&0C@;{qW9s zAY0QvArO*S-gOa?vM%T^!s1U!@E`#sXk2NyGus}vZsbWXpj%FjIW~P5g^~}7iK)8h zM(co8zcAY=DKm^@cz*_H{^2VeBzoivd@wTYa7(tRhRa9Z$;R+-geBZCA_cVdEY%!Poh=XM4zT1 z3!>$yYo7rrfBW}TZth**Q<9{G!g@)>Tngfhjf(2M$1P2-VAvWVu?_^&4BseIgp2uc z@|)B*ejB? z&*&;^W6TW71(dpGJ@(gpxBZMXwY2oN1^uSFxp%5Q&joiBRHBPI z8skUQ^{}`{a&o@X4Nb9EeKjeSM_x8f&ZGqVa5BIf@Ds3GAQKVAjl#GIShn5t?H(!` zeKvo20dx3mib7pRN_%gV37aWeeT4Txl9kqE#zotHDvnYInl_Q^&5UHjrIpa`?FEj1 zK*@BKm~$yj&mWv?zbU4(S-oM|zOe+Q@!XS()tKFM3dXZBa^Yc>@~5?_R&< zbfv`Ka!@VcmXdPAP8V09lj9*V=-QhRx$S$hDHIZ&)`}sm1$Im#J&=-o8emdNrBX`l z+7-u{1;SDE4|0SedKI9RC$$$+*u-sr!@emMZx6PN8)0=T=~OIm>`Za zXI++w@$G7{5GXy{JU7;xX%7(ZaZRwD>0K$O?R)lpXnk(gl4&z~cz2HyPSIk7r}UT6 zdr+`Zpza$yXPxiv0Wjzreg9=M0TDgN7tOQd?C<37*N$%%`+zfj&uKzJIYe+7UY7{= z^hZB^0;zQn&-8YxI~-IFFY5x~g+8x#g4woDrVt#L5X7|CPo8=1tq9v6mFc%=hk^7v z842E471eELzORG+`&>g4ZaP2jA+Me9%3CByDEi5S z7IMTd8Aj%T9f<@`FoA2m{7Xz9xEKwiS&hooQPQ~!&sV^GpO+AkB38yqRrnQL(3hX@ zB!5+{%;Y+=JhmQZk|=r_J|H}TN?))|h7zk399S45nMrBv7e^*JOy7MLB}Sj4Nxz!E zxakT>80FDQ-g9sDt1i|lMS`1+czTcnm1llJ+j*ox%ja_8Tkck1z|l8(4ux;h=4GTX z;a<~AxTI%%d|8;7BS7S%@;BeDOD*Mn%&ds>@Dw{GcZw-F>o$Iopp=omh2;eQeWP0N zrCU-M{@nEI-?Q5z`c3o{a5XvWSBL&5UiP326W7elAM?e=Yq&DN882&L8QC-i_wc{z zMj6f7nf9}@d>rymbZ%#;!*$`PyAK<%n#zIo7t2pq;g|Y@L7#v4-)!48w4<1HI>ec1 zKi}HcfkCT_;V4EblfkHmVf&I`!f=i|T#E6l|lAfcL6&B#F) zBAZTWLA)@FFk@CtF9{b?wP9rYdgKIJJnI5aXL-+_Oylv{2TqlB_kL%XUp|JI79mIf zz3|5!Wwnt$ZJ)=$ab$%#-J#v z6PbQl7C$=dx;p1d<9j-|I-upx7D>R>hrfF0!ulfFY>BnG`j0VfiMI&Hb(6hP;8lEkdaamEwK-|ib#Zf?xvIPs)vDnV-VNeh9ku5j zw;umPVEDy-{xNl9?)v*^{gFv4_!@-uFQR;rZszIqqRDF)_7EW&(avgxvjdw9o7)pPL=86N@pD za|Pmt3myg`KbBRu=?#s1-No8zEYIXJ0Oux0ss1i-88BG;fQCb(^CruaTZJ1aWS{ND zxXG((Fts-f3hC}pM>Zvxzl?_KI*E6w%r05*9XwR?c+C*h({cNMK_`mkA^$ocVgMK? z`4-2fhr4B0_%aqd{}9$;jNupt&5;i`YIRQ%G%b&AJ?yr? z&hV7I?ADU58NFU;LO92-Bu2Gqp^b=nkyoqv@CMM3*Jbr&DI<(uAbvdVhKFTO73*~C zba8f>Ah1Hx8)Mm-HlTa{cn$787qo)5`CF#TIf2`XKhkOB;_hPkN4onH`yPXT+>O-? zY#u*i+BUpfZfei>_7U@TX~JD;y#WoA>aiYCc&q_K=3V)qV9w}xkThJV!kj6Rc&AXM zQCigOke>Uu1G!D8)u=Dof1gY74Cmd;ml+i(hduhmr=`_AQEdqj6i!?#;kdV1y-_vl z^`!HV^p0W7djq!h7jJjSU`3#56G$@HjoVADA33l2Bx&Z3cwGoe+LzP^Rl4AnvoI7m zv(+l{oY~&r6GWrmR@RGI5EWPBVF%zydYNLzL8a3D9L4O5Yao~N#um|8k*(6$PO|Am z>c#*4f$}Y#moYz>@Sd;LJ)DYk`A$!ieU(!&@5L-vczTVaV1$qB?IQh*QCZE93Y1`f zjt}zoJJYs0^bBfa`{+Xo~1DG0wQcRJS^H;dYi&@V+k^LYfR z?}YU2;Fg-|e|0J10D%oiprd0)WAx;gcc9W2+3bE7tVz!e{Poq$SaU3YemBt*j-dOa z#CTe;SCgmfRP}Z&eS)rQ@}{4IaAnDr(+btn0!B18qDoUBE5eVS8FBX-(L&~h^0>#$ zl1cZGRY!S~zp4&uP3hju+4zUOuq}*aH|O4yQ0mg49aSW_jHqUw2x^b@xrwvr6=ZR9Dz%nRU>U-W> zR(-E0bhJduCT!sEbxB!F8nT;FK^~+_S;VNZ6es~#IVs(aL1B}T=$w`^o(zc^3+qiS z^Tf7BrKhxphBT!H~LSv zFaEkMEZJnj`mZlzG4o7h<5sL>KD=iuChffABr}dklIuujm#oI z70h}RO)M5g7NiW+yYIB+X*)>tjDD&f+9vfRHVUi|6hLQupr}PnHmWa|ST?9t7Tptp zFOI2XH;*49m+zhzm`TWHNzInBR_sbtc?0K_*Pe88j&B+Wpy*8l8RdPGpj98;IkMF} z<`moy1;I)-hCA_f)_sCJDVhIaiuk)x)Q^!`?yhN42L4xpzkvQAosLC$38o{1ajJED zJNDDKOAVZ~%riSEkGr&+odYFQV< zDf>ZA2uG=GyN$gfB9z5pw65I#nRo5rE(Yh+_D7{la*oZ#4#~Mz*LW+zkSmZ%FC6hb zJzZGsJ%-!7QKX1*Eu70R`!&--mm%)MMbmPrLBW^s%SG~>NvYCb0g*+qQ{Bl|dToY% z)|~dK;Qi!>y;S-81nC$=iwHM42+|Ixoz_qfM%C#0kM$#rdf@X7F`u(IWUG4zJnS?i zzB;g9{PD&jZtG_l`wv>DaR_t?W*T!MM7hh!Q1tPl2wnqpn@znkw8&FNiGJ5Xg9dQ| zWnR~@WvqKxmf5apcgaL&tlb^nUD?r#m9GOU&0`^eA;RM|#dcom)<C~|WAJ|*>J>36^qf-5a5bF7& zZa5h<$Rs2x9^)1G>ArHH-nr4>| zNpwe$AYi(=OnMYXXr<&G22J58KJQW3N7$=d1ng`kKW*;dlxq78hJuoht53+A?0~Gt zjVn;-87Q0rDrQikmY<3OHvW@B!FC|3N7V_USqR{}(tO*ibA={vv+7t|8&x!0Ug#P7 z-SIUle9D%gS1+?Mf9|i0`ZRP!9GDOHW;Pz{@ND%4GDfn>90dxkDhN@grx*y3#fb_e zQ>zq0eN9)3R@{-9ciD?m(Lg}GT&C(_+wN@gdx7d02p|m?#i}wn#QHx_b)L>l*WgI!|ble@t*qR_`tCM_8I0_3u$A zrv~YW@AsQX3}1eO4u1bu_y2V>wQg$rG81jAgbWG@IJ=y8v%%}Cw5X(kE}Lf)^eu2L zK88nDBBX{sckD*L3(;j#mbl|wruF0zJUMj6zT$I(K2LRp;Cpx0%X85EYf7M_ljptz zX(UIn6;sRjRn%oxCK{j7)L#&#N5`S+R~zi4olslDsrwpfH|1)5@ejPc%n5sA-`3QM zHZWxG<)nU!o+X$SsJin3Q_-?FQJ9*0LovbPqD&f3moZV|$|>2I_)*Jb6&6~J8jCROpM zzVWiyK2b5bgLTijL+8om;y}&2$cyGR!E!ud@!@yK;FoM$&Z5U7Bb=m4+l`Dx-S57w z=^4A8Isa+Pcte<^KcfA{zP4@3C2`A0P5nWge|1H&K)o&2{1<@Uw{Ygmw>l=iU=B=&YPV^1lVv^n6)^^+N#To%8lV71e4P6s z1{7tIL(p6E@}=HSy)(6~tbMwmdq#>0)ROrZG8o+juNvkVE%`4xzh$!!GJC}_-_SkS zH&Su%J;HmkMy<}#R%@;NFAJ^(4w#l=yBfgnF5{)$3L!QmGVVAi0@%^hc>)P6IC3H~ z&k8AoP8Fhm_|piluE?IJhm08`Ila)UAVZ4%*_60*gY^>3iH5fpTIFcbx;`eTP*4^4 z&Iju_lFJhLw2P=MQKv(9AAQ!)E3AL<3*}bWjNPc9sas^I{N#yRmRrME*~O4Ly&#eV z1mk<=oB2`SX26jcm69?Mf`poN_+|1`Zsx1a5Q|*l;S-x7R(>MQ6%K91+4K8*lRG{7 z|IvRRfA-XhJr6%?69F_R|4Bo0BH*5E^x}_Pd#kJCk7d63t~;-uH-p1H0iB_Xv|!Ex z-&x~d!ofX#C?(3HwWl^o_J!IS@B$sB!zzUfgRp*hdoR3_371G4>{TIl>V%$NqusZqJ2<+`m0>nQ&?NZeV(pR zk8`y*UE(=l>p1{fwZ7uA{3pwR6AJ|iX^Ny3rjU_e!&^yj1L}HDOJVv#`$(dcAw4l8 z{q+D<#x9kL3I)S4PI)bJwAYghV4Xba?rjkj|!NxQ~^^wQLeI(tAwds8OQwl+inlj3?tS~amk5PWAC+H%!2BfnnI-Jdw3fKqYM7?M&BB#*H$s)@W& zIv8Ki#46qoZ7uXH;bIa&)Hm;Aos1&_Ki3$KI3FatVz?){FFV|Ifo~^9+L1G3+ld2+ zNO@11WAt3)*pa{tv>)HoJxn0dOX8_~MRjs1x?bd(Nl>$N78xa_BQNTRnw7NWJRb5mnb!m=7wMp&umU7PfO0wn1w|U8U3DalH z`hGu00@PSAC&n+*nueFtmhHT1$ax*dhR+MpJf6Nj?+%;4$9>w$h$$#c27W}Wm5-hW5V0*x8igHyw4Dv%_CH-GFKXgIHA z2Tm4g*Nx#j(~A;x;Ar}y`x|<^#1h)&?1gUW$$in~nT365-s0@Qc6d4}IFs0AYXfXT zmc94xtwfyARQtqHGoKW2}r}xAPmxtbPbGvh_pyacgN5TgLH@}-9sZo z4;=&VjnC(KzQ1=Z7O~dNKj)nLI{Vt!-g{K-I)l%ju71C%`eS0MGNQjNVe@&7IV|)M z=>1{A&tmJ!!oy}9NjgT5OGyM*+GOmzD+3jS)fYt!EGi)}rd&J{*|*eFN}efu7H;;v z?B+(S$JAYE3^cVBo!$)U zqcIiDE(QT7EL+qUz_p^iT$HbAuaz#@TW2Gr9qTs_5yC+OJbvBCdOgs(^e534s-Ti9 zl4>-W0OdD09>t7!7wA%4zUx8EJ}@5y@f>1nk30Lf^wVOUx}K};HYZ#DTbqoTiORP< z6up!7_*dmmXO4hV!8ve6{y-i!Q|C!$bx6F|N;9*A4l%$XxU9I9W)9PQw2-hA#uy~h zUiR#uf@k_kn{Zb>PQQ%!P@OvMR{074@`2^5aSTP~pE1Pxyt`4 z2yCarx45}u3i1b${ZJTd&`=4jAr;D&YYi;tBP|@~js=$z3!;XiE2Mq+z%;`Vny-NY z8k8r~09ELN`V2yuGOxV3r|d!k`3-0({DB+u@uK|3EpLS&Rp@8?(flHwbMvq(>gJc8 z#Fo(k2@W|c)D zWZ##pVG3Vy&FyI97UGUpvtWR87u z25HGZcUYiHFQOYS#jmbL6)s?N{=ya*d?5P+%6czb%@|Z4|L`Pp^YssaHc;b3ZFqN!YhW z3-&|8uGe+=83n1l(;I#;+Ut674dFKet~1%ze3U?76*0{6$IhY3H&Stq4JDyRg4kQG z6Lsyy{~3II*>8*b31N0vRJ&}yeIZGfT@e18s}1x=w{EdCn?H!%BY|$pA~#bYsPbN~ zMxz!jdXBF~3sJ43*I!39)T1$@a&&Nf=65r3NLu?=x1E!Il72GbAVQxVclGu8Py4^9 z>Y4{C{AD-#-%*u9_$@kycf|^dP(RSWNK5RA8vMkiNo$_uVoL0X&8hTF<0yP->11d5 zfiu#vXQkoDfpVIj@9SjM`tEmF-EVr3ODn)zfoLoFv;kT=qJIA};_J{^%PD2I%_Cbp znOmpqU0(A|Y8PTtz64zt$Zd_1ymzu$K!cL%$LP-3(z@}mi4+ix&Bp5{;|8&geptw! z`i>@Bt@!c0=4rGfGcRRiOq91-Z?VK#cRflyU!KIbS9m$kbDCOXrcJ~>?_tLkmkhkoN;D^fOG2)1SY!Sqy8aGl%;>m9_5c*Um6*KBt_SZ(pa zc4iA9N1^4ETn$uvFw_x4bh4{wj3LZ;w> zY+n}QA2FSg);>d+{W^awoGBmqWTNT4#huw!e->)p^v5P1&^2H!V-7eS|*lR7xy! z5Lg7;336oBj*OxCmQ}d?Q~>#7^jo}Qp^hT3R{U57C}Q|qaQV9+mciem?p>0w(ng`v zA0rcj>9oZaITWn2zlbi%E34I0yS}Y3?<^M$>#7+xc(oL(ba8qVO5@mjhZu>on@dVU-<%x9%j>}Zp%ez<2D z>TY$_P3J$6faMSJ8TMTlhNQ$Owip4`W`q?-#pPC|3Y@qXxUhX^6F?{K>+=Y*^fB=% zTafH;1Y)PrzMfvBeBICeW8sO8IiR|EFV`rlNp&u@0r+B9+tvqVj5`fEbQYrHO6+-x z9cdoj@g9C--J$Y#B0$Sq8utY)kTWlZbfqB-kxUGYW$L<3j<1mRm*7+eM&B+qU+L~t zMZdCGF->TBLQ;kqc(RW0Go1`bpd?`w{Gt|0uK4XMBi*Ng+hwN@y;GPiA|gq_Ra?^O z^+(CVtqtq^?h+e;0bwfid|=Syj=g0$@c@sU%CDr&0|F{Cb`4E7AqKU#4R*jk^J1ai zmEbhfo959zF7Jl@ody&M`oDSdo_+Djq%~l>Xcigt2U<|H7pxGpI_5Y}zY2&=`!&ln zeSgS}d|Nq)sI@ydd(#uYth`Z>9Wb~AL54ftK|uE%egyuHwjw0<9Q-LPs^jo~8D-rh z{-90daT6Uia$=J_lIa)iP$ObyN9e@3CowUL;n%&}u%off40I@;4Tkdg3UBHT;Z{1y zR-3=LOuFg}74+aL;5*Yh%7@tgmk`oIA*i9W6M=n)~V>!V22EyzFVRj8nXKB-gbW?YmfJ5fxDby zjL+c}Z@l(otc_hi1P0gg^u&s5a=#%c9FhPXQGjWzRZ>s!jMly4`d+UubXn$DgNi~} zjbaH*uBcV@wml{Lgj8;a-REK?d&BtjkD|Zk2Ru^-9rh%tp^L&2@L7Qej`%%ZuR94Q z%tS;UdtJ_Ux`tHJpJ9{EO6{K}!xvZ|*ICTGlT@d9`J8gm>O|RII>Q+L>NwVsH!Z4Q z@=IPb%5771JCb676 zKzHM7{TbZhhF8DU4Z*CxIfi5NOQxbxfDFHg4P48UpSNGg2pF*8s)hZJYVH-AE(r^P z^=yilzcLwp43!F}zxzyxb4~^*_q4=yzzW7d9(tt=NU@(epgJ#Vi-|Ttyt5Xgm;IZaaL$7eKQ4WiEF_(D4bk60acBvd-7(`B+Ttx#Ft61wsh3zP(0Vqf;Cu~Fj z!%OMmOrHjQf41&Ut)rvm)W)f`Lr)}ROLOY*e|vB*ppE)we2DXCD&W8Q`yJU^L6we6 zj<&8pnM3E^0KB!!tzEn=Oc9`GHo<@~^aRE)|i1{;Je1|2u~CzZdC+1fE-OqHFCkbkV>;t^_@c1A7pz)CkDFfT z1s5gr8iR{P`T-b_t0@Ens8^&%$m;C$E0)U#cEC!R$$xD-pFTOXyg=mpCp4T-{eX{yiQm@lYnLAwaVVWTq!Nc>@b;3sb3q&5{(-#Tl=?%#vi>`#$;!^v3u)mt4%e0j&iRAWr0y&h4%TG6VMbPibg=P zO_u^HK7;9Eqj`ylW+k^VBVV_q^tE{faq%*5g~TY-ii;^i%dc2R*n>iwjQlrioT$@9 z;?w5;+%f^c6yxpV>oV5K(uV(X6+hwl`#B&KHBm=(4+#Y@eWYEc^W2~nbsbtu=z_#` zt>KOUiw2E0c8=P@TfH>fQ!SCevrXlSl8Ni`LMT!DtH<#;T-RKYApv8zm99nW0n@+o zqW_)hVP%#1S9_pO4F0^f^HDql7pt3~3!D`Xjl>L^X{n7vKMzJfn+c%%kOz$lFV-)} z*pllH{fQqA1G=fB4m7FPux^o!8p>k^Oke02(c{6pDDPB$mBmxqW%U+S$-DJ?_mhr+ z(<4klxZg0%$gCE^AnE*#FTZo7;Cp=$p+s@F$mxm)O>;#jU+PLgsgDP-I&z+8oS!(1 z_y^BEsw3nI^zVGZnV~^eHxXAtwT3TvL=k1rTQ@t%-)(BEM2b1{9DAHiV1&MEs@Z+g znWc$78{|LHtQ*Pr>!U#`wv-yACcn`p=j4e$v7H?q#mD32|EJx4=r0B+UuVi%MjW@K zpW*#CPmU!7in@4M&iTn~t3ua<+_U0^h419cM$*@QpIhl2r|Vnn|H+00Qv0jc((LZu(wQ`skFIfg4Y-yN?H5-}3Zf?~rIDT%X2f6=Jt!ZjK&InAo9*)y+*zV5#`f-$vy6@xV=Xlar%h=ND+|NXp(>#(1{1nmYGQ95} zJY63$VQ%5|ObXKtEBMLYp)g#St2@`!&(D`i+7SO+x6UrpX;woH{+rkV`Uy{?>5kKjLp3Z_!2H+i4q!zH26p{IqTb5l{j-aq+&OgsA%-QjWhH;JQ1r-KG|d zOE`d}vtMpM^O+VF;3Wk#H$p_Ie}KW>Koozh0dslTQ@#uq`WzSY!EP;8YuAZh1L&qP zcM|s(4i|riD#l&aLz216(+iO#uN# zD^T-yN5$X?=Qq|ysS9epj8LOqOT)VpFOiBsTl*K1#*+Jcj60khN9>@}=m-9aDuCG2 z1c0!{%zbh*Y1fYHJ_z9)Q(5@b6VxKo;sSMX3jw+^oJX4=g4?rE%Y${5BI$>Mr0!RC z^ORnYv~7=pz?tSRQ^t-&61O*^0rif_RUQOIMAoj#hSJdg|6k< zgl1S|5onc{12Z7tTLj7Fv6|52pDu=ff=z~j&eZ|mm}|-!q)>Zyu?J3ShAsPnnCt=i z*&HPwj~RGljLC-WUzmUs{WCmx>Siu*Lp-ig`6lC}2`MXOjFiOXsA#>5y^j@FOrdRfJO2ggMUtT;wgu@6_H zKN;<4^>qIEb$~$i<~mI2?+nn?O$VhZ8LrU?5kFu_ap{j)xIv%;i&wKH1>z%Z22LesH2!u?u#LM&tdT&Wm!7+p`wOI(_O9Sv3m(n1izcdE zWN~mXN>jHIPb$L*VJzw1@-oSC#t=W4^8u9hQm01Txba3gC$>NXy1Yf)hU}aNQk%GT zG~%}%mr3KYe>HGC4i|;MArBv_u&@*89Z%Vv>8A~FH8Rj0ut`(vePg|UjU+}LvM z0h1UHkPtBndO2>5dD>_zFGczA>JJHyDXX%J^{RyJ^HLO3@M*BdK4H>y(KXMDv&(k} z9y`4;J(|c)GUP-8SqNyEby!g$nf?)Vl9ctf5#muFnR8{-+}7-&=EcZYxayut zT(3m#BUylwvkN{AOzjtP<$@=jKUF^Siddsm^2LSw^;lhLD886&9Ax|OwT$(y4CiPg z7UcWxp#dTE&a3d(;nlhdGVRYxzlQ}gGjUo->Cn`)EB~lw@mp05@020^pSa$3J}vv7 zHrBN)|JMlD*wDz0SI3AIk8T|{*h4*Wrm7<3p@jajdZwNXBc;zdmB9B=9)eqE|9gZ_ z4}HjDKxQ~GS#9XY9vkL9n6FuviS+-`)K)~3b-M?r7q#hZ`tu?@lZPC?bvllw=NpE+ zj<4WVfC7V=siusf+!*_^X2PNSq8pEVD?_)#F?-=Km%z|nTW$N-RMu$a7S_D=+gL(w zIzPZ7D0>O=&8{Zdr!$ajY{?yjBSPjrUYwfGhoD&PB(GAhq=pTvGg)FA19zyJB{{v} zeYy2r)%#W-uq}sU!K7!hn0vm_>OTYN2^GE%=;jpV$9nh2JV>AXu`iSU7sI6J>`OjA zB0m#UIB*-~2d>*>=}TLhG5V1+_)If^Q+4j_5etYV$G$JO7uYLLcE5OUbc;099g7_D zS{g4m<4XtyqyMlp#d>^mTwjqvkd*br_7{?#TU3R%j&syjphV;F;>w*CH^NP*tBLQHABoZ&zWXPP{^<96 zp#dwFFinDH9_F*zXox_MQ(OoK=2u=2!j*sp?(~iFOm<|p2YNO2dW}HXp~hI*?kUZ1 zy2M@BvVZVXhubgp^!iBxufZaZvwx-6M6PoLC{IqrCUId#<@c-(%~(pI%^vk$|nVb&jA8?BSizCcZz2E#jFNf zYe#@E&KmQ9l5G=#U1TufctCss#|z@Bf08ZsaA)1kk9p>YcPXJSswfw>uPeiLP9{~_ z{>zwsX@Ps_b1~;f{y@=HXrF%(oO3YZt5g_SiX}eR<4I;CLc}~&+hk~$AC@)BUEgz< zuIL)tA7~mPL0PA{r__h+NrBak9vokBz<$e^p4Gh)tJ6a^k)Nb5bbSx?^Kla}C;aEJ+Se!9! zbIy&1O4EgLRgyY4buwS5u=L#YEqZ!-BVP<`So2R)Zl2#@-8$E;YFZ-ToS1CHMY&F` z9aD+xedzBfOs1Ht`$@Q{I#Yz^2S}`nM|2$u`P|5G@!+$8Of-upM>i*`6=dp z5}Lu2A~{Oc9&BU}ru4dLbx-2l?QGKXyMr}i+@FV?wvE)&?BgXJHe zE4A6vA!2^>^gUYf)9xsUy}VaRSLB;wow~{$S+P4?WxL&Nu786AD9o{smFuGx4Uab- z4LS{|i#t1Vs*A;k{)5*)F#O`gzPf{TVN>!$8VMB9B+0${RxTpWos2Pd6)S> zq7CA;$~rsasdc-IZ1?HF`;K22r-aDsn{h@RZfO}u#q zOwU5^yK<1JYd-f;eXc`QS2*8@Z}r4<#?YZOq8M-A{HTBoF456HOc6u=gt{k_9(_wD z3VEViMQQLEPuJbxb-zYDsy)YA0Jjd0O)u7Z%NY@kree5B@YsCo>Nm=uT6DQKcrc{rTa&GKOaLX>$Z@yfbGW%J z+XSAxKX^WM!Zj2XZ(Zg&gJ{<7Ao&l>Q(&2Xy!F6ZybUP<`_;ojzv;098aTke5U~H) zUs-p|YmCsrJ8^R+vwMQ8q>6atMBXx)_hq)_de`$Bwc|vGLwM|1yzd_KdjIap=Scf} zi{0z_Vo_V1ZZB<`xZJQG!35e5XARLNT9h;Ex$|X;(>caeS>a7%YpV4H9O9n zGBeKms@c2z@hXwi3v=kz0=8#Hx~9e>-ht?+;m&b|Dhh5&M>FaX%75V@bmz}FOiawT zN=Ue-}#B)N~ysqdu*baCQioz4hcfUPV0n3hDf86u`c z2MY`5>OF=`tk3l0y+*k>q@por=;sq6*_WHr*R9wHvQ`jjhbVm9BWU5;Mgs0Oj&^5ipg9 zYyXH)cE-*n0JNT?sqj?{e#hm6BP1|*cB$CpqhD(VvgTd4Pas3hYSCeL^2RQ>- z*0E-fML79gY@@y-p90A}{W0ha`rM@$hYe8eO}dUoYM0uP%=A`o&&cc>4I`J{i-DQ- zo{hlDp!1P>yxcSBq-qrO3b+W2me(<>+Xu_3Kd&i7jH(+8c4I$2Y=PZr_3sM)VDOjd zGTtICFii5rLCnJ{H6_e;0(1L&zWygRJ@;>O8Nxtt53>C}ex;a)^dh7#$lwqC<2iiU zhpqONGSZ%%l20k~i%ydat+|4T+gif!D>bKgUAYT*v_x?1eptNH@zRcqFWj3Yu&3T- z!D9BLZL500*?kle)$^5a7e*&673)DgjrA_<;Ia$LkZc>Ze1j)J)7G4rxJEV9=FWJK zYMOWz$V#Bj%XBSoPXIDbe`-)RF7U^|_R-1=ALel)X|OBfW3kWF4x@G50~5mEGJ7TZ z!qUJGp3)06=ShNFGhZeEo}&@OFi%~B_Q9buL2elNu!C>g&KEng<>)FdG!lv2*RO@e9bAsgeWvi{-|Ei<06ughNbw^2i}WV#5mt| zX^fRvxc>mTXYv|ptwPpkp4IA@#Hptgqh&8<2=o6yPTsuI@- zSM%)tBX6$DFGX|IY9A(nI?DW%SJVlZ}S(;9NqDhj3LpIr)+RmM85(DVpu2?{X9MTbBa zEPQ#l3e#!4d~5!e3^x(g5y5<@k?Mtz&R~dhT<+yNTa<$@Z4^aYhm5o!&s+QN1SBTv zc)@RjGd>0ci&Vu%3ItqVRI%tlnzWy}81WRm$dal2qw^JCvH+%vj{Nk48rH&?to4|oy3}jevcc14g!{p7uNa5Ph1o+_Zunfk5z1qG$*25( zDbjTQ)bq){wDdFAWl9nYEV=Zo8e;MPu*&-Nb=AoW9_@yqiv4HNH3#J8}0 z(Eb7c+q!*oB}EnE{lXgf94eP}_u`tw$sx5d&enWwu%L)!xMh8o#4_w!*22B3ufE}6 z;5wYhJI0iO$4DdPLH3XP35W0|`CQ!rzx(>6bw9;wPxFQM>u}*yJd~?4-&RAHYorV&Y8vi0u6WS}zbM^Y=kaTu5 zkIhGkRl4MT#m$nfF6y)2jZUOoT`m;Biu}O1fGdC%BbPCDPI8IqOU+fTNyz;0ZM|`- zp4h4m@{u>B0=ntmHY!G3*045z+s$y0n1a+lw5@fR=x;v-8+U!Jv9&FRx!|9w-gPGs2=%#9y~76OX%~fZd?rn_bUm%tWWOV zAsgzUC*F14J!)msdbsI&Jl_;{=0ng24QrAg5h0!c1=xhgdC3unpc({j}~m`eoPcvXhYy(a%NryvL@wEoZ#USbo~QCkQ=-2^2@icCG5IF8BSno+Prj z@Eqm#>UMWk1b&2GRCY4@gqH4ieE1HL0@d{7J>BJyKTliM1VN$F`bsyHd`MxRkzVR z`JxJYm3Vqyg!~2l<6`-Beu8UZ#1A(xU#siV&hPp#N|)eAn$oLj{t2FiLaeX2!ugO% z9c2g}9dRFWx+D05RAy*t(l>pl=QScnLcH?Ww5Vhu*3BG9LZJiyg&&4b4n3+NRC|B}4eUW1@i!0_1CN0*+klReyTr(e zk&)loM(V&yU=xhIrLB3!?S8%nI;V5 zSXSK@)@%13{UbaxR7ghJc9K|4YC9QfBMjKkVeQbBI8KVoV zvx_6!Uij*gyf0d1PXAd%ll&P~gs`Ht6Ix!T!jaynmc;Jp{7uXF_g_U-X*{Fam~qU} zCPekeJ3G`qAYeX%2JhS3H=Pyt)bVw6Q@w`mv^Iq`T^CCen{M4z+G|L z{iugXtwzDTgsAp^R@eAF_7#&RJ;ht!*Mmexp+S99#4Rg8to_BKXOk-w*Ov9i(-#N# zx~gX0<9^(WnOdP~@9Ft6aNVx2D+Z5A*`vKvzhmNx-Tpl=Jvd=g?u3VdaNh~NoBHL% zx}k;m$iKn*?k*+&M^$nhg4t0Grh}g)4|2(&87b1DE3_?KVKTJw+Ma*1-MeJ9!+Ybp z>U<4YTi?iuOeICEYIXJZP7tCB3B1M~V)exoCep^VF<@!)_zA!BXR1P$uqA7&jZY-kB88P^*gljkQ){`KPa7KKKDN*gleN%-ZF}joA zX`a^p38CRV^TTxMTS3g%^Q}}-_c$t1CjMJi>^Ac zl3fQDySV)*N1YAMIP$k&)`Q>I`D{7=J-jbIN(k2(=4N{KEgh|XfV@m0V%{15_w-s3 z-QLm^2o*07AJ*-y6mc8b&9SLTn&aJ!nzOe%cZicXebu2yu}NsjJs9P-cS13r;~<+MbSk;>2Eyw{Y)~XU3WpN zJmjvI(Uq*RX>K{EUJomV;{Tjx7-D?ijLYo7M0zt)0|vdDJY0K0gKirbZ zBPdLRmeq#qxcMB7L<}ncf@lY`dmA9IjoVmy-18Bw*S>HAfU`Qf8k&Qta(SfhyjWHF zt@W!1Pf0e^R43(iT-ssK(&Bj9QYPirv8h~=!L!TM;ih0rDOg!D@Qc3D1(UM9(tzz- zHb#heLB5a;_d_j%BjF;%JChmRxuuB%4bj~3p7M%P`#B^>^ZlKfI=U#gWF}I})T0Bp zOo;H!%~k%GSYWSv&k5=Pdy!DFtl@U=x`jOfDnj^o>45@~L5!WmQO#@k#tyra0vitw zbQ{Y7l7}M{JAMMgC0#8C5doF^mw|aCP zw$_}UB53K_sjKMJarsSl_|8T@R&=NvIrphF-$Qb{_bj`3@~*X$_pCbzr8%Dwv2b|9 z8t}tJFS(}a>{p4KKpgThPd}1D_6k2o_K1rSC^lwj!s(w4)L=X0L@yZ=YqNNrtK0a> zWxJ)=?so?E3)NR~u_GmU8x>j$gg^Tg?e%?gn^ge5aL^O%TydMa^1!j?wK3Ii&C@p2 zrybCF;MTo%E6sFlfA>`3+#dWhFt((qYRTGb&f3Pjec6>7mdx{?~~KO9G`Qmk(?=gH$=3(_oeV@IVw(4$q2TL!9CVju(^B-O4&6)hNb%@wu5 z6R>D*?q%-vTW01HzboF~1?FSG+FnAX$Ofrq!F;-oR+pXo`&^v5gh2b`xOP#Ig|TXhz8JGo5w2l&Us$KJXAdmev>wMfJC{r^;{FV) z{~~L1_KYskiX(}Mw#ZKOCs0SmHp;KIzp*3IW`(9k4`}%ijjBidKJ@M}^FI7*Nt*JR zWK0G$cf9n`&%$SC>UR^f;+~=~RTrThT?DK>(^NcNtV!>5HX@gH81aS?uPT;Ti>hAa z@qbKZHOT$OQ4LS_Yk?faj$<~%A~j#$exy*!IhdQ?I{j7Oum-L{-GO!!ZK)NC-yK(_ z1#ar?-WeME9UJbF)}wd6BdYm%4X$C9_O;7{dz#HGk@IvKTgagkE62YW&dv*)4Egqw zSmLxN{KL2OWS04_>SAf3f6*QAz)N-Pc4hw5Hl#U%`}dRL)Lpip0)fKge5O9;*?|uh z10%W*jFahZPg!e`o;8d0>`fZBSTCmTUcNXT+Hku!ikC|{`w^spyJ)g<^d`Q6tB?q| zEjg$Mu&hT+LCCA)-Pc!MS1t7A_@Ivy8M=PW)^A@BbB0N=c*JB`jP6&w14~OuN-!&` z=|3}$W_cj<0<<%@Oc%_sFLe(%dx0fSZZH@X!J7=BS0(36z&}+ zLPz42t(*0I^VpM5d7-gON|tsx@#OES8D|DtchVVNCy@4N(_Xjhh(-N|8)r6S4I*hD z>%Er$8DQ8*Ox&TpUrGjh#V}}&dqlQ>qS)t>k6Pd9H1(RY%25W!RPN|#OARWxcY5%V z&P2XPoLgv)Z%+Jfj;*iTNS#>k#i`Dk|GYPxYUt;IDtCVKT_KBU$(EzK@)(EZsz z-nyb_6BmT{S$CegUY-p&f>eiCn1;Ikt?=KNv93<%q8GceannB_reIcPa)Cm)i|_Y) z84=(~JD6+!KWfes16gW~u)@>3pgm(=SZ>8- zb{Avm?KkqcOjFrC-txoIexzzYg}9A>N`Drk14HDiHGI4JUnf{pD=RPgPY7g1+1}VN zVj5fHq9G$ZWQFHSnx?w_PUxgjbdjmAuF*wl#A;3YS@eW9J>dm2wPw2CXz=-48D26L zf{@gL?()$!JFl?{F%TRx-i@9fn(ehU-MO?MKJ(e1$>I6({Y#!CZ~{}m}W_t}Cd`orw^Z^WMBmzaZZzL7(QdRJ0jt8uEd z=^cGO?c_^t!o+RtW5w~B5)Di}n>Yss_83HwoOgFU{$k52#=XphU~{Mg zBw<8gzwmk*K<8}&-lTl^HKBF@u{fCQ9TZX2D4w*LS4a#4cpq0 zZFWp^36k)-I^)&_M6l1l@oB^_Un~FC-sGlO&K{xDe=MHg?iv@%E%@X~`oQ+_=1YiA z98+GR4PMaIi88nUI}=lk+wINsUCQn}b;HXYww<%Q^TByq$f1p91`b`kq_BFX)49AsS9u9xX@y}J7R~U&b8883Q+D< z@RMRA{{D)}p&o=YQm*B?s%m*kiwaqMmbN{3Q(yeqquN@yyRHLOOANTNmY!XjE+75> zn|hgy4P+DOZ*Mf{)#I3!8fJx@(Cs9mw=6ib=W6sJ@2BRmtpO@!&Q%``M@)d z{&+v&{z671c4#&JZI76u(H+)Ov+@jHgBeP>J$4WR0Z5Ouqi|V{^Z0CX7~?Z(`>> zWVV`p>u|DSpuEh-qZYSXe`57YFA|C6M>8G^2ht~TLYI3u`+4f3hm&kFt0xUEabit> zzH{m&`{K!t^8hEOqRZqFy%%d24pAXn#z9Gyeisd&lDu;JXuMUvTPVm4a(D zSmQfT6YZr9C1=Z?6p-$_{gXm((WekIsSQw{iQE?@ZL<0N4M8I61lG_#jmJWWiw+cbEKP>O1c)Fz@BWsFz2sX`ZANH zENT>{w43nf<5DDW1rX9HstTCX@P$~OG8;DZJuKr_VsVQF+B*jLRV)CG=#!+GU_kMo zAFVYIi71XeUa1+|X3?Sa8r!yj?B=tZ&!G!X2&tpaYpu)XmaOL$tcwL>;4=n)AAn~`LX*|4Ghy70OEdYbrE(~9`UbIT_^t3Gf2*4F0nb-etz*gn>M zDu%b3zLg&A8yfxF6n6$X4-7C;Fh6-0cmH~TfsiY%%NKO=jP))!KHF1S228`vJ*n*N zp72gBid~RQXCa2=TW?Iiq#-)6HVMY1LCNvVD2uOljj^I2eM!+OWjME{g}`Zp7N>1i_RxU=JdTBqpDlua`@39X z-R74f0a4r3+;k&() zZ)R{peReU~4EAs5i2sD#qNB&W{;|BfQ8;S0j!oB8ys@mvklvBB%zxyKss^P8pAbW- zk;#m|?)u-s@xv2MM-3+jo<3?0h8hrrjJmXC`vZ^E9&DnO%EYxm99MNp&Xhh>hOFVH zQB~cGQ^8@dDD=>B4s)z`jYL0KT}$9Oqg$3Hs6FU-Wa|gF^!EsGsy-$E&f^gws{*{K_=pSG0Imzwdv0d60m?OdybABO1 zG47{}@!P*RItiPABNDEiFxu97Vr2r$;V>jy?8!g76GU_hr`VKk2q(XOU+uX7G z=l*}98}0u2-`WNFpx`nvS4nEM>Jg=aCm1zM-g|7&`mo_mmbj8o%_4TVubRqT^(0v_ z@Nw0wmt|O#V1(bBGkOgj%E4`3PLp?J-Y>J$VJ-X^ZI(FR`>S1&hiYBq*EJ;DlZ{gv zWn!fc)%v`(3h;#1qf9RLvA3fzLx}I@=_LHsfUP*@%kKE6BUN!5X(L}-k>o%WOHFO8 zn@~yQs#bziuSb~9Za!guBk#G1#`Ub$^!Chib(eCN;v;Qn;Ct3|hQQ~4^#8tbJb`UX z$sN9)`x7~P)6Q!^_rorSyI1t}uJ%__;2kbTr*P9-4zH}S&6r%( z&2++fmanY0=An@dX<|OzlyJF6NM6iqA+ui&2$GY-Fj{ZE2rHqmS)b5^#q>(mXQ6NB zqk{6|x}g1s>P;(C)fPnX_uX07l(WkH&!2F8cDsevC(I}t$L+Oc|9%dpyolDY%Ms}0 zQ$Lyt8=maeFtX?2AJhrhxV4%!R2ZLrn)#!LlLB|1asK-KrEbOh8UEYW=Q6)tVP<0e zX8yNgU9{IqBbc1$JJe`q+#aPpYc+IWHv9!~eqL$gyw9pkdOX6^@#dzFXie>^$ekYG z*$AUO^9v4ex)DU_OnjSp-}OpLEV8R`Sm&NmGk;EJ7e_g|3M7C z`&i22esyCufWE(bM$^b^^`v9YBmB3yqYlHY0gcdMeoI{Q1Th7HKe}>2C8&!#xGE{b z9T!F>c9^+(UzQnw57#tT2`{KjDb^X9jVatF<|=XD21UadXMUTWSa3u|%hFk9J}Lk9 zT$=&fYE@G<@T!jMg;sG}_-_|gomF!%2@x&Xr}FC9&=1k;GN4-_bPveDu?j+4$*K62 z$<>w=+G&$-CrlAE ztHWuT;?sWL)6!ux)tKnyn^b`eh3B_ltMwbZ`Lb0tpI5y?A8DWDyU#IikLB&yicX*u zm+iau)BU{t<&;WrTE_Soa*&Be?8pBM(Q2u%bRaZ4ZQo#`#ZJLXUrW&Y|A*=BGY}Eb zUE*BgU##A5fNTOy<%heJJECJh1?1wx^XbtM+Fa~ezJsp)M>}#ZnKVnO^@i= z^y}<1=um0_peud0B)MdNr~+l&r}!xP!BEPl&$Zp++YxtG0X~wU5;cYp$$`9b97?|95*yGn~$=OK{>ezFbO_G~gqH?kte!~7p z$>8M=2O~ik6s9RG_ZXK`OW1QDe^lFlq7ZU)?=O6)p|49D{EnvT=?BP#4)A6Ny5pQO zp>@RT0U!d>iq|qtb`8yl(j|guYwsX28wp68u+bPb{3Ij%-NINqnQICJQVPj+L z{>r|drk{-Pgoh~xFG5deN`vkOTX=Mwn|g3JTqJWuKyVA0g1n33AxM2F*~s23wo+9ThZUU$%`ikN~tX;M*XOIukb>OUDb!DLNQjP zgSqC$y4rumD~JBBCh{!(wdr@Z{wH5$A$D_@RZ(2kfcDaw`IVc5jGKayJud?rcae<> zWMvEiY^WAI%SKV*`lV(gWw$tNhCj4CXEQnU!M)IKB>8ji$|}ug&U)|qjOehxHOZ40 zY)wWWq*qwU&T{wY$DaCOl`$`L1{8X#`~M;V=5CuOrama|bwk8oUNRZqznJ3zq$l|5I(4LeFaA0{1-<`&1VIP`QTwja4-cjuH#C2+9S_8@H{SNjBpBaeJ<2W8Aq-JjQ#bl>~# znd$nzq_%M_`BodFUVvIW6Zydb^!1*m{td6-i9%lNwSMaQ|LA(lu&Da({aXG#f}Lf(D3alO1$>mbmCUcGi@VXD%jug4KRt&<{BsplkGJ2QIs zHX_+xv@jtldPs#TPl7!5g&LO3Pl*pZc!@nmLAZWC(=lCzDx2)URskGIqe^ssHzTVb zws}q(>Fd!tDS5VdtY_wAyyl9keNyWpknaqvUx?km}LBpAiT-p z&<4{y-yhpoOtNXoz`CO~BG&+R-k-auHw-0)>Ul5d-PhR&tQ`e*)e|=_Ir9nZ6b_sy za5n+owA_hghhTRi|A&uF*Jnivgf+$gT%wmg>n_K{q1h>;pwwEB@bzBk$+7akee93- z--J2^+KzE3R~yR5H)TB#BsVMEC6m1N4#e#BPOfD-Q%69X0fN64yk0pBZXU;&%naBl zF8&(!eju@H3bdl!N06`KCy&7T|*V7r`=tR@?tsJD1`_Sw--1uyo*O|8L z_T}4^wxbePzIi$8@-t3NYHJ*mJ3b!zhwcs0>mOi&UhWzzPl2sMAli7+jU1h&O)NJ3Y!a1qb zBfa&cD_ROiV}g%*cytqV-iXvJuQm+2t=#ql`%Lj3=hNqc?e?g}{A%$+G6-{&>7V-@ zarKTMy*rY}W5R52_mInsot6-o_SaGF!b0(hj0vq!{yFK@^Px#s3E?+6m1ZXc>Ryl+ zwbA1*8Ruv@Bw5{pGP@GbYOgL036Qn&zLNL-*_8VhgMqHq&$JUV!e!-Zv(y;#CSd7? zyMVC#pJMa0o6OX{x|uiR&`mk5*te@<&OBj<1FXoN?XGA{jn0}3kHuq)h;C5e@>!w7 zqcnZ;Bu>K}Jovf|DQ*_K?CxRzZ-wvz7@c2VEzAIVa=Z=ZwERFq=ThV*=TF=6%goVq zfH5UQ6Vk26CGaG)xf%(KD_cA(qseIIcG4KqhxLZ2BH#C+Z+-tm0<+Jn&kV#(B7k}hUgkA|~U9o?Fy(Bh5y zz@~fC1YXM*NxqYGuKfyCW}f)hYGfvQRux9%m7Xyre3p#w1e^<{8KaxkiKMp2IJvE- zG=~*3nkEDs$(k4MaFzD9$piAJnSh~9n(|bHyO*+jBK6$+<#iT?xymm4cX>>T#GfGE zD{)ZL?gaaR@^-@L-k#PDbfhlPDz;`FmzchzQM^X1>BiwtQmlf4Yh!BL6f$Tt=90C+ zpM}S6Uwaig(Z79Q>sE;tO}X3!mUXo66fRVJZT7r9pE@O0ZMIoDs~n84mz3jkCdA_< z@dm-FbN^{yml5d$R~J5|f0Mi4WN50s^GS9%`9Dv$_oWTt z)Atv>KjzI`>rBvFnk{QJ+t-~{9 zw7(oxtOS-;f0$~xaU|T6MjrH1(ZM}8hN}2u)B%V}<)SB5sXpA>fJ)EeDB@e5E&@`F zy71&RiHeAx#z-J>3f+ObF##9bTYRW$S|tRSL6`tj{s-<#?l?6*pnkiU%AYLr)RBW0 zhp$&Q%K~3jhM$ffVCYW|byqcY+lML@8eP|=>S_@IiggMwEiFuqjoh@2wJ_0j{v*bu zla{FUPcvJK#&wl*RDA_{%q$X}r1exYSLN)8{?9Zs0UmB;xplamiKKN-8hJ%yeLWtA zTJUz{C*w}sx|s3h+^cV0O7ou)=^0N(6E=-wj`e~d9mIh^wWp1-=3~Yr+AQ?d%>c)W>ZP>eWEIQrnhQ(U@_b5!Kpd%bj*4Q z63ENUct;@JQ)sO8Gs*QA4S&fn2aRe3B`k&3G_O0La7wRsMI8&4BK$>P<`iE~S8aBoue@?TT`nYFi zapTlO;CS;)tNy|z*_VB; zTC5nYp6Rp=ZkuYrO>_>X?m3ReDp{CM&jc_ z%@fG-^aaNGb`7I4kAEIR^NNFvu``(}Xmi}n|Qf#~ZtL5fB zkZkF>AgBY(bAUk2k)q%uE}9RSy!q?v-@lW>LLh6F@4W}yd?C%wSCNJXi#)K;AJI)D z;abFD&pt8z=zh&luS=PTC~R*$MI$e+uiU5KLv?(IqTb85qAfO6aE;0yI~oMe{~;4a zOuy(A2dqqJIUc`x()tdhb%~n#BKRvOSiOS>(s;Qvi`CfhZTVVf_cxahr^`HfTz(yq z&cEh6zz68JPy(NZ*V!V>#of!YbT-wuiF4rXxZHl$&+oMz2OF=X>nMDcxpUFlwEE~u zK-!x_%n~>_-YBW59F(>&M~t>IL$_*Lztg>j&WS9_GKC220ZLPkf#PH_5^ z+W8#|1P^9Bk}?vc!HTsT{65xnUu;O=vi^2_LPv!YP48INk|O)8aFJc?B9aP=FF8ha zTm)WTKIMyw9L;$w71pABsgW)J1EM+fwt4>#RG4P*Iw*b-EUxiIC~Xz@Jn9V}OOpMJ z%^;?j>z>RdCw~j);*z>_7*Tq{9KTY^RjjpkZzF{=rbAB%kOTU~KiuzlU zCIAE=VDST^yj2%k?ZRteAIo5a@rn2SK}0BRMY-K#!r? z?SJ)wZBlt`H900wcMo(;NtgSgX5_yE>1hf?{p!j1qrlT84zA=}Bf;~%SFpDvw0>$R zwro+If%YV`xUGOfd+--@iof#&SQ+uyYx9b0i9TFoJrB90M^>p9b&E6>atoup|K)L+F;)jQaZSZc z&1>P7&uGg<`Sa>k0csW(bE5GJ4YgZ?^F%GE>S?Z+$?y- zV`bw|2ARItAGQ)cXQMh)Vm#uo0oet~P+zyi?kK+Y9P%4u<#2z@4IjUg;2}E8;r0co>YS@o19@<7VrcXyrtCGe&Zp$k zINB@R`SHbQ!1+V(y;Q1`z>UpIpn%E4I2ej&n#mz?3V1{kqcq8j(+v?S=N;-gZdd!a z2hw`sgLWnVVCjHQ&oAR`bF8zD_i25t_FthJxLXz;K0cUsbn6Ni)1QLx4i+2f8TZW* zksgX4&Q}2u+=4z8VU`=wFJca+^9a&hHlo^9=t$K~uNrUBDxN=(%ca+eLx)S~PHT7H z@nDfOccwMDZaTR;ohoy*tt9a92fFZ8ci@RjD-E&5*}r)Cq$DwDbABKAf}9;Fe5;ah zG;Pz(grv`VXI5}z{+zg>=vb+1Wfk-M3o{l;-$A`ZGy!uJFStwt4;Zk14b}307JU}0 zsb&9Xq z8%k4Vk}>8=x?uSdmVO|pZc~gPHj<<=A8G>|ZHQ5XcyfX_+P$p%3W3N_>>)1U9NJ9* zKL^!v(k+_Kg9RwQ(2@$#6CpfZf#~_Lw@MC*AKD&e=Lyz$k6VWVvMZ^fj8Zj7+q@x-obZ| zvr&i~d)Pd2Gl7r|k5{g{2k+H4L_&T~EVgFmUoPVoOIpdUiiaJw+sK-pXqcsUCiwt^O z;zc}QZ12%_on-hOy*_qM27xR|PaLx3@5fLV;;6SO#@r5DPGrDVBG}VPD(>Z-0mO5sTo0Y&{(4@ zgB=>59fQ9Y^oe{BM5FG1WDc6eb=Kt<)ibhJ(RJUpDSF=Um44`z^D1P42$N=t;FXWR z|K?|~7j87t?t@ItK^+1WVP^HMs%4%3th!uE0B3X`X@7-Z2tl*y#1UX5DI9*gV`Ydu!|D-Vw%&B-f^BMSO~Qtq z{w)R*yJjP;?{j`Qdi=%@3iuz%E*sLCzaeBhU(4B%PWF^&-GSPU{N_iLSxhixF3s*3 zXmK&Ha5~sIy(xvBzC7JmIZZGX5vQi(J<*wb7=w#oE#H{e*eXU)0~#KH^D!b z^tJE+-AnZ5m~5`Iyv_mGoGZ?LqT3Hrdp4Wj#m06*srBkFu2rc`H-(?SBS_x0Bn_s( zXIPWRlY;h*A6Z`RQ)5+6+vYo+z*<{2Ep*2^)aDSxX-Op~<;tud~|NOnp~I$M^n(@#zK? zLgZDyYsTGYs13zEJodl|$H7^Tf^4q*w^Edb_fg#?KbnITo*0Gyguz^KuyunJ4scJ! z>+6wM`llmLsu!>ty3y4+;cAPSr1#6}gWnTt-H06Njqgf-wYj9hQ0EQPMf3yax;V(N zyMZBS?7CM`F$p0%Gp3fSLFg2)29WGx|H6Mo=g=KQ*2I}G{sT)=LH0&O!y4yv>2!Fd z3uE&vTxAf?TGgwB^%2|VU{5X|jRhEK8QUPoDN1hlpCs4SDY`A2Qb9sC@MMstH{ zVWhl+6M-vqw=r5%SEYCb_PkbEU9MQeq3QUXr&2d;Sh~2$Y_5M_8p^(nB*%JgaMicC zrrv47xNB{Z)J0yV^k!ucckjxKxUMfE;cpc$aY=BGtm4COTzr$Vh#!`+(mN}6 zTPRjt;>;b4+uy~@Sg=U!#|hR;D*Dj&lWxj`apt_ncQ5>y^eTu_jqysZOxqE<-gRt)#Yce z%k|;&B*7MHHqzgx#1#H|Q3iNicx2x68(A^HAJzTrp8X?d%)Ya}F3?MnmXBj)$puQU zcon3F655hfdI16fzl9|XvIgL~C00Z6gK5u!ApR~};C=(ml`?Sw4Y z`SfXQo9ZrETzs=DWEHNQ}p- zGTs{pdi4N1GiKIPSEFP0f@m}cn&}reYp;9`|uq_%**G!rfUBCkrY&0rtnm8CYZ&q z(lXvbP_YcrR?W1Su63T-`J|0}LDlX#r@8k zJ}7u1Et~8*2=Mefe>)TJadt}xRY0_D^_WTTKb$g z%xV#l59|k1B;CKD(Fp9b-Eq()uKDsQG;99O&5P<4X^y^;T)Bu+w)KPARrlNh2riFNWjx_9G7@ zKCJ=H+_!yJk*N|ScP91nc*$@TgKERE{{_e)wM?bHMA;dbg1-7keY0(x=&E>{E&l__ z(nDST|Gpn%kmSf^B0!N~d`0rk+}@1FgzJ?hQ@a4g5T5j8o5W< z{`#le_>i=IY7)7B33FUJIP>R67)H*s{Tdoxs4n5jb5hx5O*tV|57P@>Hv4hNm~A2D z_>T1{gN#FV3(ly9I)|(pD;PpH{`*DC$HsN!Xf|~}L>{LM1{LB->*?e&bLp3KbUlwR z420qjLEUmI?6Z60M6V=1`pKF(a&nzVkESlo-wJ(jlCGVc~VDT zrW6+uUcbGXn4wxoFZ&7HRRnP6@j(re?L;PZXu5r&Qezknl z)~-%N^aJ|Ee~PO&GggD&nF^to`dXnm!`2u`?j6-z=VDP6yBg&8noqWN{Uslq)PLM} zS17ChQGzDkBZ0DLnXIoYuTR4rXhaks#J{A$iGz8X!3^iUfT$XbPAz>c~Oxg z2i;X?{2Pj*SFZs}JYjwzx3R68omp_bq2+BptFqK_XW)AMr!qiUej)?@M_K+0sx>8% z6dJirRzF?UCn0(2{A_jysC@#ndl8VcO$tAZe{&okkn90{U@c2?m9_H@6&8s3%WG7@ z!%a0Y&O>}2Bu#z6r`Mb?1);@%M`(k=p8-RtXdnj<&$&+a)$bkitIU13FyLeCCgkMi zLgfHXNJoCTPVI*a^G-dC_?OAq*#bJ`X*ut!yAK17l20;1e$EZ@%k1%8Ory8pBtO7G zueFgu5T=U*aQ`pgxC{%#Db#|cUyBMIb?;Rf@I&jK3x!q6yDvDe+e~=r65&i5jf%eh z{EUpTs5|?jvPPa5btlbXmXP?e1o?CpL#0DlQH@_GSSu0<=bVm|d*?=bH&;6EW29Vu z2lumeb2LqYytC-yjxD=)_gO~P2s=P-aV+J}J=l z5pu348Q+y(&TII=x<66~lBM=56RyVwDZDmO5NG zTXp43z|j`_(*VpXQtmVo&qcI2!m^KVx5Ww%HtPfkd>aN^yT-2W z&x%p|$JL(5(#6>b=;oZ99^n~RezOKI{?B?z`(gus93}9=*-`n}`Oe9hlq*!5`>&kY zf-So2hbakJ`~HkyB^9)2-8q5dliRD1JozW1tl zkJ5&QI5wzrf0!M;&&Y}y-qch#VkYSElQ#IANMpUCA$br*tu>qe68rX2fQl}ZH!;k= zN4|2L+6wEfluW=pby=4tH6Oq0stR>b zMzG)a@IN8&Kh&2M@Cu$Qt{IpISxB&-TsaIqu&&xhD-QkswnHsvaa}+nRQ7z*ADsPB zV?3`v=b|qJjjHlPWy1xkR59ww&%&-i>OS1LqZgM1X&kE6yxZ|I_dteyauI63N@n^crqs#G+ONF-xp7JLZ0}FXjD8zUa;cg+@7kZ$>G4h~&B`?| zLfw3-Y6>ySrFCDB9x!56_b3@B`}{2LDpc~Kq^EBvV7&Opb6iCR7ZM*J4BEW2)d*`; z(f&ScoVEU!y8nAYJrVVpBSMO04xV|ub7|#s>l1JKy>ivOV8@GE?Tps5%UoiTsHPBg zs&BfHR?6sTW~xL6y^X`;N|m+!hXDS>cQiD(xY)H=|0El_3P3?`9;;U(^W)FQEgRCB zpL=YL)Z;_y1>wSr9yUXfp)> z!myFFZriZKg%@(kkLLWyB%6%F9T9ZU%y0v(;*g`O+%oW@Q|OA&-`0Zz1z2nYtFKLA z0Nb_RUMb-$sWs$5e!Bkkj6N}n1UNo9B+Wdmtd+n6w|fg|jkoO8-ScpSo_JYr$JXKnZ2DPbz9N-tz zQfFBE1_sWr=(}Jf)$h>*>$@7T zo1JI#S$bD@;bvVja%MCJ#rF$&!kTGL+UlmE|A=wHi1fOIO+Mbw?o?%sy0yGZmZk@h zDF5r1)6XQcySpM;K1?uroAY?`%BUD+(&iBp-bia#Yrg?Dxs)k?+8NHr=manNM3g-L zlkFs7g((O74Mq%qFw`FE+R0rQvd-Owh0sCoe9FQ{Q>U z0k2o^@tBqVSfa zoNAiTKOB|Yn5Yq-yBt_x@z{OU8m)?(t=FWiv5(g=M4ULDgu%`*K^b`)ctJ3 ziy@w|`A}T@hcaUt7{&;aDt&TSile!W;YrsL8pB#yJr+tuwmtM#Zas@U^du`T5R&MB zc1r5>lM?-O*-}hTz+FbGk3w$eyIKAHN=m09kIt9Tn;~ZXH}77Ux5N<4ONQ&rTax6j z*XKdewFvs53ds>I-6TFAP`KJI{K@vFBW*tDk~dIAM0co-^d(1sn%S?4StqD>uATn} zI}K=2q+ax8J|#%0t;l6slz;fj*b5^5kCGbR!iA1FrRyQ^@a;O&{$_6*Sr4;QXxk|m zo|Nr&dI8NVmkN7v8?gunpnp9)cXQ>9xtbPNq$h05w=vXnTWMuPizoZ`A57rGb6F7K zK#gFOw1vC+Pu&+Ws|}~RN^)<~?oWHJ1L{0?rby>C2|qb>PMr4e`otEjA^y<4e=u57 z!QS7LBt`IYnn_j$F}P&yPLnJ(U-Or6=cb~&M!QIfEcC#E-EY-2s0j9zz&{b?e!huu zso>-;w6Yfz7|_Z;s!^Sl@Q$UQQDtF~z^<({>oSaH#(l~c*u7DqzY*V&oYI8;(J%=? zCaP8>sveWQDKsAN@rsF(@Lc#6D37SFZwmllikLNR>M__<_V!QhoqPq1@W(YNO6 z)Njp1-d_i7Zfj4w4=#jzZUaSsJz2*R`iBW-?V?`c@w=!65|lLEpd5KarxHgt&7uNx z%OcUaJg;aQa6DQ2Nn_SqPpoZuYUAbz?C!IE(g*fW`MWmmt`NTK{S;V+DcQe%^0v&% zFWn)GH!j+4EziP=GU471q&KO2A&C589~kZ8!n|BF<9UbvqEqN|t9j$4_Z(5R%B9H1DiJDB z@$0Nft59?`8zpMug({E9#$4fAogc4JS+jqh(iWs-#mPw;=(@({cu~;l@h`WGuc?EH zQl^=x=TDY#iJOxQol}BnmP)eH8dv3Jw}0`iXQiOL`h%+jEaMUeTkZigUis2D7;617 zXJ3`|H@-yzA4);OK!Rq(_m^)ShNPQH#4W=f_5YD*zL7tJkJIX`cZ36k4UeH!kN!u# z|LNeZ5D7n>1RgmXTRq(HxPN+d-+^+dhNX3pSPyo-Svf3)ZPFPL`S3Wx2epI~34E_j zo;ti`U|7EZ%}qLH)1US+`H^(Qa}qamOEf#c2ME6hX!Vthg`smK=Uhx=t6ArS!uG-2R*!XgO8^xBd0}1NItC zZn}6y2CiFyf@rXC5zzZxK!M053oz`r5k~G7X=MW~FjbxMvNUqrKNKsLJ2^|oY^af* zI8kZBoD~K_Bui<+)Cz}JG0Dzbbz>vd=4>96tCRS^XBN(nXTrd#cVI6W-DCum|nkUtqBr4KYuYAwJ;@}A1b#4+ZA)39Bdae3%(^@=n^dW>OaN%^~)lD^i&SO z=baZne9dJ!9}OuenVs6sd)T=+2NB|!EOc$cAg;r!?F&P>dkDrLKI+t2xkU9+h;*_P z-Ql6BP3-!x(^Zv9=%(4<+N25>4q+*%vom8k!ZCX;_*$<4{yJfzjJ{=VDjh=m{L zSc;%_q5C*&n2$Z+Gnd`6EBan$*h@l6d5wqMlRmm(jOg0h-O`W2DXMstX!5v7wb(yd z;b?PM9MUUzT?mJb7+vjXwEqlt+G=w^=-uM7#|%HIj%Bx$E6UJn_N-9q}zkElBf1c9xw!U;<-*@O9sd!bSJ|F!Ss4o5DE&NqPdzJkuWj^ipAc&_hW0%3|qsHjU2PIwoso4;MH7>$?xfM=PT00^iIAlEAGy3`L-^N%&Bbwj zv8s{3)ZV=NyFbsGn#+=~y-`Pzv%{V@Il~z+IvVVWXoPY3LCj+1a*a(*AMjTLxt(Pc zHvGqEzBlI!F)=hE?M4Q&5{}cq*5X_R@6LdhZd_;9r)-*E)=Qw8#H-~Xg(*`vUT-+O zie47dpx9jH!d`IJNQTA!j>ZfCb+V|ui~4OJN!-o9M`=Cy6}|9XJJ~T{#x*62arW8{ z4LQ+|XK8IH3u+ILhT3C)89>_Vdt=~pGVQ|a!nQKJL_Ybw zQ2tm@pbS(YG|ngIGU}QPv%o=?Wv)6Ywk|g=e6t$=IwSf@F@yak8c&RF>X&^~We5+6 z26Z^os024|asx%Wdq7*-S)_}_)3%g#k(%&mO6c{de_zifU}l6YbP3O=ub-I?}$^7H2r9Y5hJaY7=(4B3lO>hgD58E>n{^oeMFP}*?% z0QL`-IgBn@iq>pOik(C!Fcr}nGtR0=f<&(EXutSFlTtYJ%x4FNeoB1d3`T-Pk6=Or zub<(aD=2K@m&%$v1A8VM+<-c?jg}1$WbfjxS%{TF$Rhn#kuc<4EKkgM)Oo*umiEgH z%t8)7$xf^_GHatqzpbSu!gZEg(DQ!wiCp8J?24*yRaeG0~#Ip>ae7E~w z^^%EoCv9X%t2Y;c<0Xoq?YRVuq>cOrM4sfE?Wx;I=vdujomaCLN-_s)q8k3*5U4K=%c`3jl?#3{dDEF}s(zodpyjry?$&`a57?{fuL!_k%fH#);iyy)*RO zTq#P)9wpv|5fQ-03#c^T5Mw$g6Q3WS06_&*#&OZP|F7l={dV_91JTn@PiemhAj+yqKolhaC=d6jY@{%!;8tHc-=IaqGcEvUFhS@R<=DiPY;UQ;L z2ny7(gt%y7cL2qXP)VZNtVgW5Mp4-?I|(yS1O(vRaB}`G zv@I8124LBcQ`vjO_wF={%mKNAn`u5P?Z5DD%k`iCI#`QMX{VTKjS`xpd)tF$o!fA0g z!Q@gNX!3MBur=@c4=ij8TKTxf^~iP1mGT)@v}&w#Esn0S1G`&yp77Pw;6?Br5crV~ z`cJkB8oa}t%1wFvl6>m+W!Jytp%b8jlC&=YOdChZZ_n)xZ+CcRd;-)L{zviZ-x`ZY z6Ns&(-pd0{MK^USZqCA1n{V9*w7M$=g-vhxp{wGE!*onF-}E}p04_B2DUo6v1i3mS z=b0TqsoA}_m|JR04CKl$D=xA?#Y@*=Y!0VX$B)l>smSQB)Ma8%vz#U3PP0d05-Dsp zCq(UXuri!1BvAZ|E?a3GVL!g+NZ7)y(hTuw(bV7KbC$|w>s=aP?7gL1LQ`gbD|h%W z7mKi8ZB*4TE0sDBXpj5sZ#2;#{>i8BT}Tn$`289e{NLB6@;KGce>@IEGD8kImd#O5 z4jrJ8k=)tU=C8BH3C=El&BSBcKd${G*fqIKGi#(maeI~ST38)hhxlk_bHWpJJ!P0( z*n*(!CuEs~3>$jb)qHHrFxSFm8Ut=1i`NSXKjk5+P)m_*dUwHM(_dIJTX0y{>~+7B zOF&fJxRGCxYRiWT{EtH2=DVBo!ZaHv@!L{MGX;99c81P-j%!I~W#eO)xod*+vH1;8 zF-GgVCgcJ)AXxO?UQ5Pf9DtWq~G9{6z-T$BkivV zT87O#-8S!gp1Eh{T^YHG2@YGI3IZzV%WH~gmfn5)s{7}Cz_al-qCP~hf{|t8ZO}bWt&b@AOq+vABi}MF}{-0W$ z9_N`rjzxWCoxo0hQBqiJA&_RWbAKkIQGK_2f9^T5MB1Am^WQ9h*=9M_PXcZ=boi8V3RO0Txh)PI@O1aJ}K8 z$LiOkgpc{$b7h;jNTA6?{+=P{l5X1hc1p`?z=+_W?sfKDIMJ;Jz3btM14#QFB!(x* z7%Hz#3IVqnAvG%cp)i`71{)4Hp}Q{j3-Lz1kM+O7vZ#NrqgUt_hN+V}+Zo%Q|C#dQz@@d2&|HT1 z&ph+;a{mrVwQ}J!*YLs*nFxG^TVBWoCrm2;6n^>n+&P$NuOvajeOi*2mMU( zYGt!%Ckd8PCJD#+(_#u1j$|NAUM_!nRfvWshN`lGyOd=`1>zQgpg`-{kfr={`Wtzy zu$v+RhW9)+n1M0o_tOtU6mq>LUrgH5We0^h9hC5_UVUkM zV`w0{Dj)dfUP?fYPDEB#65Ba^aOF5-B)Xv^f|Zw2gtpz+m)IkmP?DX6K%X}Yq0otT)C4d89UsvvS;!^-G^0o$MvtH{|O!| zEj;7$YX4Nd^v;bajRNNSZ(i-RG~%xjS3=meov+bspqL!L?fh|RQjE8!j8(Now`bB~ z*c>wwj*TR|saYE2dVuJ~bw$|a^a5GKt`6#pmgXLWFUE>-OFx#o%e7Mo@8_eeO#1{NcW^x=Idns-5(Ft3%zm5g?Wp);8b&SeqC5tnn@VR3Z<=n<( z1J4D4eZCF&2SrtYErbb}l6I#ZklPIXy7bb=kX&*0_xw$ym{>hI4Q24*OfbQwp7)vzA~P`&2`>cwym9sVKmNlVMG1ddVM%@}>Vc+T@Xk5>gY z{x4@3ivFCxH@%i4@%zU3$8U~53BHsm6Y^Z}yzM%b*Ft&{t$FF<5VT0u$+sE%FSKa8G}AxpN+ubM;OM=*m#5*XkwGxnLwa5V!7ou=u&$I zR*dCXa|^zRG#ia&8S9~_E8jb=GnOb`%mBb`~#M+JbG-q2cY! zQdUpwU98SY>+5%3xfPS{#}oF)@Ejy?5MBqgwDqNV&ld!SUr6Wm2n8OxNa?jJk>P}~ zB%$>OM;b?`?`X2K{OvSe$%Vi)wMb)M^l5*@0UynJkC{8#A8TU|Ov$7}@U&7o~cokM@s=OYQp#+l?;{Im5v{XJ_B_}4{I~YVyMOpX_ znaFPZ;m-$mUVE~VwF=Rt4)xKUA!eB>TAeOdxThdAx);C;sR9k$Y7-jc9|H`iAZvQl zAR;OQ+MUm0t5DTY$b!jvjD$E=Y`FSex*?_(zvQ3>k7b{stiTy5m@Wp+Ejxl1GQIza zH-o#jDyBhqJnC`VGMq?7AbH`s=fbdd?^tol^Ft}6e%@N}?s-++qu0aE4I!=x%jb62 z{_2upP>i$XImo%WLMg?D-@O-zNxe8#n}R|D?`f6N;w(sL{(^VswZMn{8(dhtR#16p ze9TZTY}k?T+XHiIbbh1dVJ+z9W5)m(>4AFHEaX2Gh)+<93@ETcfcztIIS^WT{XgFF zNc7Lojge%%Jf1Q$srwr{d#NDBXv(SH{3HrDCxYz|kzta09&FqN2{HIoBy(MXOEe*` z&aUtgXEb8ot54##ha;?f3L5-RoJ4tU{5P@1P(;67s6eQFqd9B1br7@8yOO55S|+cz zk<+3|`wDy-JTVR>Aa-mU)9foG$`Xfep|6!Fw%9W&ayRcc&#U}Z%N%hY__|}AZ{%O= zYExR&=e7CD06ua(AteMj;@>D~Uk6D0I*`JqWANk1u;{W2!Xqbc{(#uX^scLYfy^s) zJg{VhxOIF%c~S!h@M|ZZb2p3}W9HNIDjc$8zs3_53g{MuHUPoFh=-WQqqH z(v-(BXRX!l&ggc#N#Jc76GrUab^MCcbnG0Hrd(iFLgm2|c%8WVl3$Xh=ug*h z-f9}k8@a?j3AfhUv=&$+6_XA_gDNOU)w5Ux<`j^+H6Fly7J2tI)CDhC7!?*$IBRCB z%TBF{I^7}QG)Xe3Yzj=BC(2~3>$fj=tDAbJ&L^RYKpaaDS19OFi>C_5z-wJ}g zzo^R0d71G%aM>cfr3G<0M2^It1yVdEbj2F_$XT`6LdCl3gPhH|q&>78rixu&Sm~UE zfzw=elJciB+k(6yD4LMhJ4snN(|wIJqVtsE#r8B1gTzBC{MET97qjqc0P>r5v!k$? z{enjB_@(Y1MrG5M6401gTr=K>7BcrvC0&?%UKAFUSl%i(_xZbUSq-ht#6+nr#qxFQ zqm*^C=O6W^Hm(1%IU#rHbKMTC2?olqL zPSyOsM_@q8iv;~*u!mZX9Q*T6*Q?zjHGig3uQrQf5;YVC7S!T5mk68DKU$%bJjD`* zx0rs>&%eTIP&%0_(_p)!Tj2?b4puXb45j)Pwhu-2NlYAGhHTk}_3P*fm|SN(dD1Q? z3BHi}nF_=UUvXg@9THHmPuar<2tJcbSX~vg@gGA7p0j=IH|UB?v8hkdN{_LW2U2hM zDElZSzM5~#T=5QX9Bu~4^J&sEG*+dnYmp7K_>V!C+}EhS^%&KXF-pz)rN{bBkaeWWm5!UtDv<7m##trC;uDhEsSEEHId_8Z6!Gz<^ptIJmHd%J6`-~z}!eVA2?NqPk z5yu$N14y#MEc77EMa>51Vqu&%w+i>{ybp=dm%lV;C82>_s4&cq#5W=X5?g)j_0xuG zpuH0}y=Ak||8`yPORwVWa27dj`(B6su7S6m1#w>fMY5k(aaf_i%U@Z-%J+rehMTID zxgPd-U;Zj(gZzI~y=7cfYui4o2!f)3bR*p$NT+l&bW0-)DLu4^fOLn{&`1v5rF1LZ z-ObQ3@L%q|y`T5}I={siX4Z9`=aEvMl6aDNeG|Vg>>tiaqi+4k{{tPXp9l=@#gtHb zbI!Yrg%qxGS|4lejVxyi7o~E{iQ*g?c?g?&3_fvj9~1irK$cwZ+L45+jaLd`l{M}T zsE4*Pc3>DVbeRtM(IBZs{pjZ@B;_X3ZYdUdk%Nf|fg2LW zR~QeDwlym|nBi9qYmnDEaYY8x0&T`e5J@p?0(YV6a_oMm6dFcMVrGDooMQA*<*`y< zXoT8N@u0&03^52@08S~jH;U@pgxxu1|})6uGngDG=HDD?*@F?vLl)&jAUM#Xbe zYZXQo#2sq5a{BaHI?u_zp)m5|A8o`;Vncm6{GJ7iAjs51rL?No-{u=NF8=%WC5pYq z3EA;F?l(T~J*V&PM;szgi+ilJjwe`dFo9&}*E64rm zN$(O9KU)Jc4P?F2dBDsrNdj0XJOz)Xu)I$=?@A3V#od<`2-tgR2ta04l>ea3X%G7! zNVJkp?wlRBf1SU-F=$pse##VY=-iPL)uYF=*sIg5%Xl@7;#n54=2w;wrd*R6p|E9& z6WRjVFSB!W6k?0jU)()gB??nAeQUq^hn2+>dH+VMfO^yZP7GwR*9XGhcB~X(UFtB` zOo|fy`AR%QoY5DmL66kgJmIOL5E{a#PUrcWMXa)Lzw=J{8-dzrjE7>API>%oK-{P% zj%=Ew(W(l_GXPSG6C#ujnBUY~I@@;R@+ULq@T-D9HzrIGR)IKMN!Mv9F&rs}t0~&l zL&d@8jkxL4+8g;-6`)f*id|;fw=941rhQ%B5ks94V(NI-_kt(>C;<0eONgO2OjWh6 zZ2^V=;jHaNR@@CxKI^3~uWw9Td#Ch8|M{e~zr=0<){wDLHp+K<6&k`=NpYd~IwC2p z?PMsW?8J{-NY?0+1HMRx=U>t8vFkK%kp3|!ZE5fsdb4{QZGrdyM^|tDr?Z9FFW1}I zT=Y-aH+^?`PSH-!p)ZSp`$9kf#_w}d|12yt$~N{@fQ-MKp3Qov)Nk=^iBG(@;!P{L z%L3h7WOITmI6Ux5am#7#gS}n&2{(!Flvj|uRi-$&xW)T%V%5pA!977zhg_axO)7!G z*=9B$hOR>}Ls^r|P#{^UsjrDNei!(k@dWh%$SeW!H=EAYdw6y$T*rW1;S9AJJo>(< zevKZrgT4GE*wPKv5}=NxVAUE1p499B`Zb~&A5_m#&!;?>{i)LUGT@Y0V7JemNQ_Ps zVrWVT@QZOh`yqUKJ%)8^Fuq&S`~<;%?sDAsZ58$%Egw3S4eG9n#)Q^$sI6vHVUeNN zQ&>8_?@!fkjR=*e-BaJzhvtd6UXHZhPy00stIxZeXV`1kd1mix&R7o(YZob4Uk}e- z%h~u#5@F6vvbC#>{j5XeO`N;;U+atv167wWTHU9%1 zWATa(=~g2zaR{yjHs1S;D^Y^ql%hN19>>PNWLYWlQXg!4Y_Eqz;e6u2(H13e-_Udv zj-x`_l^v$YJzUlJqADg&w{QHMdy5x;*^xpiea`$YDWA-kk9(6X^rw;Nz0~&Z?w2FWw%IV_j`sK6T?zz z+o7`#q7m_Zs-yd;U^~7jt`e;Luio!7n!$PgF^uqH=-W#~>>&Ka1;CA$A1sfQmMt2JJXdBX ze2#`>KcSh9(^DN2qC7_B3N_I{mgIUY^)BEuCq1t}4NcOilbrZ#8qTkuZ!&z3e3mSe zQGGBI}#za3?ncB!VnK-h(nW&6aDXdWB+UOQ7fi7-y(^B85@LX5AC` z?cRr*JOkIJIxllRW{X=^9nSmC_l%rCPX8b}-J2NSqf=+IpBmHot#)Mb39*84+~^C6 zyd3Br5Emv|)KIKg_*56~(Gdcb^BHQ(kv#LDwE52)&%`CCs(ELml4$3EY3!Wx4G#+Ori+G#681;GWI#3 z|L(5j94Pjkb&0xYa}AC=sWgscCGvk|@q_9;y}_|#+#FpUCTA9DvFiOn`rFoP=do5P z@_%D0`6V({kH~*S?f0-Y*WShOcl;XoCmrEWAK4AnC;GSdkNerrxs$7@zVsxpd(B+n z3rZv()HXr1%^>H+QOd+fQCmO-RxQHid>o4L}-+4=x2M5$&Ut0`GN{Cb!i;zb&*U+=r`L{U3a>c{avo}R3- zj)~w6yc58G-JsJSq}oihEucbr%t3_XcHKz!Htc=4qSntG!WBm*T|)L=b?Eow6VA;? zQ7?O|l$%(2Cblp|;O5}n3YKVj;%9V_b}?_+Yn3bBDAU{bbiX<-e!z=hz~SgoBXK`X z#3@XCFg@;zrK#^Y+YMF`nL{18EFPHEF+4z>5d2Oak>jG$EEfHu-}2VjW5}og5LAE9 zah(N&X?C%`8a-P+isGX(L6+`j6m<##QHysATrClf(@yhS2SJf)t(mijw?c9se%1y>RAph!o zA^`Z(muW82a7{6Embe;}XkD5Dvlq2#ndD&9JBm7JngEM1SKTq(Oe!tKq7bq*fBgC; z^0+xuu{*+chp3}}d0mb$i}?U2JJOQ%ix6rH&wxHj2eZQLo-q({*gUKEZKENX$(OIR z?wC!MX-5AJX3_Uk{kbHOsCza}kG;drT|`bzfn}VMrppVg0CybO$t>k_h+5hF*Q|M7 zqKUPMy&OY}NjLRO5u`mcGmiXA(a-sOXZ|W(rGgn$EJtp4iut#O*D{KXc+-?HTPfMg zkCX7c;3KKwzusHPO!W^iFd;suet#D3|FXDm;J>6Q;8GJ?PG94T+E3jAk;_fL}yws)BF>+Yc`f1_e*9^UA zt@`rA`G6Ifk2}LvV5Db#eU@$Knss|1%w5iicrJpKK>qZ65@uqYW%5-2Llu7m>&Uzq zKDT3Ss%OI9ta#eF$PUIqNsDMcAh?tc+k8Q~5V!|Nob}nxnXljJS?5+L zh|DQA{7^EB(KC!FQH0dkSUsF>qkyK4CDscd;Hu zzSlq$gs3;JD39noF5j`UV70iLqx$ZN!2gk^uc8m!C9dx39i@-`xH7O>PSGx=m90s_ zim7R*&F@702Uh6{(MkQ~USo-$)*3BH%^mKtK8Ja4Xt(<*!$>4fn>{YuIF=!laqnJ} zirj%a8{Duf+=Qfc4L<>$T(c(vZXS<)2;`UUHk~LYzxhhtjLcvSx}pwR4aD<;raRQl zjXesRS^8)M`wu^E)uYim(s{2U3#KYolZeREe@q;vigbC?1?0x>UDsgbovEa*3#(>V z6eNOfXMf$-+27F}OFD91QEi~%8K@0$vOqpZM7zADc7yPO2ZWUL+62RYB${eAzgh4q zzM5hjGR;_jL;17x^9DNlKBrl1{|^+6GfvvP7X(gZ*H(%rQ&HJVdgaNl((QN-*2_%rdWBlqw3&(O^akod+zeRlQ+j^<|jYyKx!C#6)uof;{b zy`ouA4;s=oborfbe0ybbKyS)BaPXC0tsM0m0dhY<>+vJ&^aBX_(=?$a?j!nzUkh~h zZK+$LO>bQ>GTbH;1xUgwg29tL@Ld~=^*z*s@ou=ar47xW zw62?rssaPxDb57unhHXa-&H(|AY%T_P*A?3@$O{PZ~CXjcaD9IMZPw6Yy#bSdm6Uc z;A&9wEZ9zuJ)+m9*eU06{9ujkiCdybQ2f3?lT;-GjG003IHeSYpr*0R|pmVd~J4)kjJgfJ2Y_DE}|B{dl%0kx+TKIjvETd@jfgG;0cPIyl=Du=b}Y4vusGj zNmD80lC4@zUtO~v&0gQs-BAo7|Ni$heD=7x`0UBm&h^RD$D{TzOLiL-VZ;9`z9L8R zxQ{$R3g1UmX9+9wt%O|VgoG(nfhkGFvLE@(o+Osm!R+)?iKR zZS>e|8YZT|X(`S&6TOm{#UZJ7L0apH|Lb%#`WkPixz}-oXy3<&yD;}g>QiK#ZKY}y zMnQ2+My;i$=c8a&#OlO(?MNcUB$V4~(C~cR;&lAsu6C}{*0n}(Tg98SSm49xXT`>- zQSHYuq+gb&PTeMdP`-Z5a3bqc9N7Py7bU5BN=%}LGF61q)EW|@y_1LSbI7Kq52DXJ z)G7`gvR`>1lD6okYZ-M6?rb1D2d9G$ z)1oG+!cIU-mKE}G)BeKM!hJ>-S&9XOfA>MHN`AIw;>>`dWw%lP$S{7xtWg2a(ON&_ zCI*GzQ7qjpm>OsCv#NVx@Yq7Ze8ugdOcT!=mbA%WZ-g;v4H3_#mK(9c>5EX&Uf*cn zVm({?(gNkup-i2QZJAQ{XvAwoX3Fk;^JI<>P6k~<7Z(d<_sX;9PdN>xiDWFB@#P=l z<8Nx*X5Eq;J7Ee<)ZaKfd2OhhLnu#@!#OvVLIQVDI_!T;t&PQVj@I0LViD3bGLPnL zBA*@8K}a|Bw>s{3E|L{1)JQY7yXD$XZ>3yS{%M11BDC_JjXtu3`O5c{g4WJ**1GF_ zYMVEUdzMfT4ofj}N7*M@bU|DqLw4N(-Xet|YmM0_7os1)D@9FVP0;^tASxu8%FES) zZA+3TGT+k6*+(#V^2ymd|7x`e3H)d5+rk{)S+>3TZmz4?MY{01>Dk_rzzT$X_!SoN z)t1PncS>D=22Nc;`mf}ef?e^efoFlK2FY}h0e$Dg)ZEYfYibD0GOi(U^29m{cdL`! zPY&jdP0@gFV?nzX8iA60_DGMikIx;JolF+S&beeqV3HlQ8dtGlZ{s&|{2OiS=H?NM zm$WZvY+J}2@mh7|dJeB|9JPokeO!@Wa*x&ROAaWFU^zT@#)pM&j=Xhx(*U4dIeSsFji)vA=1g2P*aP zFD=uR+YuuhQO8AQ=0{@R)cn?AR zI}e;P!YpsV4||Xl7oSM$j9WF{QuEgHQVaAjwyj+nVQq!;LhcgcJPhX(6r7pLOe_LZ zzS49pEh%4k`lUO8TV}izxnc@)UNIaf#86|rt`~^6(i4<>yN_jWe|{d}=^cbcw2igG zLmNG-_JSnUW5d`0&e*z1bzlKFcUm4g79K82c`f+~c&?#~y~xJR1gdNTz5Kp8LZ*XK zU9PRETn=5AF5;x9PFUh3C_Bo7K5@-Fem#FAF1XWeU?g|mc@oOq#w*2^%M!_to~Le~rP#Nfjdt$=mJU zW9VS8Ae>!`5B_Kmv}MEfC;viN*hsu-$?v1IO}z6MKP~s17otv967c4c!~V&6{NqI? zMD;)J*S>#y_b|f8A$=x{0}9Wm$OEBOExKK{B&T&XMK<#zsRL0iJ<6235Xu2K4xp&C zCuf)KXbH>qF!E&>P_I%4`M7k%oWkAEkz}G;U!6ivPc#@aK4J!A`gzyA6Jcj<=fgJSH*)^E!Ga&_sVfKCRB^dJ30V!)inRXRV z;*o{Ix0+Xmr{#*|gKOaaw)%<5U zbro!><@sqtKG`l|L%Z=KAJ=vPI(0E!c)!KFM6;n#p~kLUqO^<3&imzyJjUqo)ToLMWn{1w-JM0Nv$yA4D;?AIav8~eW3{YhuUx~=HtP)-131rrbSFv?zc7w5S%x0Dj%(|^VYvBv{v-&(5MASR7Z)6`{7%LzcYO~KUUkV33V1ldcClVg z>uB7BTl;t_k=?i1S@_jf8s2iW8`!3(d%VJMB%T1=sv&&Q1#N1jKSn);1K=-5r<%Bm z5cT&>^$&JJnoMRX0WTwdIhDYfu%HS=u& z-cgy09r}Y2-zvs)2-*KOjbyZD9KW7zD2rJT@lWBATZgtLhouop@q?%fN>lr#_56L3 zrK|Rfg@v&{3DNlXZK!(oEpsK4J9oNO&r{U6EoM4>Mes#!PYNP{Nyif{KFRQ=<*KFi zP0I_MsQF>|UBvxNGGCeg)A{45(!Daldomr_o1H5fLxW;0htm;>`}I)J_b#YubEN1{ zV^}`@646`=9Yr3tjNiU%`Z#}4T*lthM&)dcHHN$Zcy0^E`2^&Q@=wJ>g&9>&)D7do zEAu!}TMfF+-8uS<$^7%tB%#7?)qY5SajWu#_S)}Y?>4Qdsm}YYx_Op0ko{K|y3dR? zILvO!8-;6BW6Bb>O!|{WnF_&&ZN#G!$NjN-t^JwPJ+V2--kgaW)W>Q(ULkzza&dA} zJBl;J8B7US<@A9X3?6KSacSIDcx^#RAmkfD$f;PNnc*H)~RNuMT zRB^DhEkMRDs<`!9Wfm{Uj-^<`)gz_J{2iZMatZLG{k{?m`>6(cNK4eQ@4aTv^7}? zrXLfFT17hpjO+JgR^{gl1~t%L3^LD615Kf=E_)q@gZOky6Z;e|+H8G3^Us2Cj~nj% zu-w_B?-Pv2ina502Xk0jN;YH5Xqc8kv}A9V47}6bR8&@!;T{;2um1lZw1iNU zpWFL#c7OL4;oo&(=|>AUGg{rr_Vt{N@4$iM7ELTtWN%bR#GC+tEwJ3mLgwZKB06nGrAkYEkWb$#?eE3WEocHTGXoL?^NQW%^NFLUur}8SOd8>y zR-Cd`KmC$9MW^~&b6C+?PKuatgWp7HksQp_WD`($5O)~An2M(V)`(3AcJPDL8USMf z^AYe){5krD0TPYmVESfe>fE|O!-p17yK&Z3Xt*NwQFrWE-~s!4H`~93=YTkL^gk8o zyC*$0upV>;1>Hz$w#LpFLIUJ923q8aTdc94;VjXWT04@xp}%4!*lOD2W!Ym#4qPLnQx>|n+o$Fj zlN)Ap5H2%e>!f7P_;Lb34649gMAAFHzr2mdrir64n5vq0_W%{4X9AcsuO(EK3=Hd- zH1?>OFvtb;65Yk6Bjah7Hl*}n!Iged^$%m7c5QR^4?_3Jv2B?j)FjcC21OpOF^6c9=x3bK;sO2u zoJ4nSuaW~7KbTV6)t&Bl3c3); zgUwRE1iafCqoq<_=uP?IdA?=F-EkXby(D^i!L|7Cb=@eDo&5afZvcMRXi&P@^sntnLY5MTLo^_y~bK0QUUhSsF18-xd8jo3_4( z$ZG4R742Q@ezeq$lb6*M@!V8YuD6d$q$B1i(zqcqQB{Vu?5L&sxc{xm?%x=k*X5)A zz9dOK;Z_ARULV{CeZd7`$9Mm`^Xk#QFqd5s-&UJ{vunGd4;DEqoL{ z2&?p-%J8VPpVjfa8~PI=F01&0bh>2PL@wj#TcDa)W60dmhZzYT#NFJA$;yuVOQfmg zt>O2Zjl7gXbbgMDZ{H|(-@Ka)$I^LM6&hd&63P!>e zxnRD|B#c5vh5nA)r7p3fCbceG=GmXOa_5nKV{xCY8)a6%q9h=D8XWhecR#%Hl5K}= zLJH`^A?1v;P=mnOBri5NA+MpsIyxbIdFoj_i~$?}3T_#*a{47;=?Sa&d5oESp&Vg` znv#>!;%Bq#^-mv1&6!s^-#*=NTq$glW#V*w9r)gTrTQX6tu$a9o|5zDNZE7iFth=N zsHKsmO&TN7?E)TzsZPVJb(+|H&g`5)3j`RZE-Pzlz%8!Ub~jz+aMjzlz2rVt&>;5& zC-eOKWwx$4M9;#Ya?eyQz_g_-|8~o7zs<+Hjwf>c;-qJh+L;pU~UG$Wn zD_0R0`~MtVgwOnS9VF)Lwb64R&o@QBpPNum2$ngbO+}IXM3DKnqJ8<*e4r_DCA27a zH_^&{BrT;*5CkCAYi7KQ55nUG@LyR~$L>3@VV|Ud%D{tbKavNw>H6<)F3D^RQIVH{ z&eIbOMSbXdf!aNhHfc-*!s6^c? z7FM?~{_t_%3SUc`8e{)E3tu+}6DV9y0fpVY@aG(Xeb> z>Whp;c-Pa=nPo{qUt6MN7YkwYI+&z)XLkmT?pas}j)PtI%HD~G9cH&y>cP6IQ3Str zvm*j^!XOIf%0yPP=-KLdr@9ug7v+60wt71{)*F8k?9ZqqTGd=GWOt5^iG<=Slb~%B zv84HTVPrwFN#A$9)3mn`>|5Kqd_v&MeA|#-^27Om;YT=P#y*IJD{2e3qthx-G=A3z zC>OKi>9ZA4;~VXcl%{m*H_wqaoHII}0YB)~w>Wc^^ZKAT?C}xb3!JNlZhF;}33zkJ z5zG!>+HmL>nz8Duxw*h4ve~SiAdBk`hLfoj!_DR?dh|Th+1m`QFBg5@pEc+TEKpG1 zKS_9IZAh<+dIZ@wrc5aHqS3~3tS%>Ek6J1?73q7M0Am6QBD>Bk zw_SV=gFDY=$I~gq-ib`<$ZebBi$^2P3;l4MYboN$)-$ib4!oo<3IwuirOR9XOd=6v zB&Ibu*Y9SZLzVg?56{jih6AR59SGYR}b%ptzjl&I`p`w ziY!4hXf`P1Kyx3$4;mcj76MTaIJRCK^8>H8gWHKLKgK@H-4wbjE)LpR6+92Ng}qPo zTP&jHpQo$o&~SvkI1 z^kF&nDNrw2{IJ=V1%wlaxs0dfwGbDMnP_@EChiLRGceH+V?v9M3B#|i5oeW+n_JxW z-^lFN!NRk7Yxz>mZJN)cdlL+t!l*Q}yy0;a$m`!ep{9;PY5)m))Fji~>|1N@q)iIR z8ZEVTu zG;NYLAa%*}RUtG+18c`JIyrG`^5;$7zq9J2R}7UlC+bdapyn6iErY)1E3@#5fI)+Q z6oP*Qpj_m@AuFSg3+ig>B}XKh$;T!Fj?H?{^zx?W? z2VaY{^^B0e7^$DbYCrmQYi4+1(Nt=bEGbOp<2g~bryU>=-?rJ92klhPUh3$xQa*Hc z(xZdz#ZAc3n${@{rIUJSo0C{yXQ*94=WdQl`1ZygyI&Y zVX_%P4E;Ibn!BIn*Ea<48Zw;cthfX&z%O3ae5F3|B(X+T8@Q}JoVAi2?d=I1w5s5b z6(SmFmG{fZ@Sdn<&orOXoA3J~wE9!gS%nU?{zH0QrT25;Z1n2(@>`$1(eBF^3T$x& zKS35;DnRV(>bwNZP5w{&P)1kG5k7^-1ZFl5-L)#oS{0c5A6Y2sfya_mvJ8yo&-idGWA z$uA-skjz7Zh8NKpr;md-h?I3Ck`qxS=}sT0z6vOt5*V-q0uw1Y7Df6re!qv^EHetb z`a(!3-^qmM<1ewn@*Hn)Yp;xya}$%$Uw!`^#Hf{xJ=N>E1CYA4(gVed-LEMGldXUS z-?TJWcukDKRqLrWAp!ua+vaYUS~u{QW2L_v<2uwDqtJNZdAzAFt33%%qUul|Gu4>k zFnmEB-^?NoUnn4asFY=NXY95#=r5vrtN@-pt40Ddr+&fVA70`0k)S)oP!gzTIk%a}2piKj%iXGygAW4j+MWzHkL9a+-J@Nx5guoye)Nt;MA ziz-9yNm={CvXeZ(~iWHuAs|tZCO&to{~IC&WC3x$98CUzCl*<8|k7 zjoejfYrNGQ=nBIM1vw}zcB0Rp9!tR3r&Cy4d_#DAQ})9?%f~oL#r6f`-%ihpdR!Zf~lrp*>wym6Wx|#MZ}60eemYn-%IL zR!c~cU?F#u%I~*zxRf3m>`Y(HT3MQ0D9BEzi+5^7MY9T%c^gq2;gGm7h5ERyZ zm^9K|e&@D!eyV<*={a6TQ3F6YP(-FTfbZ0iz@X56)QKFoOaq1lI*ITcJSFROw0+-v zhAJrX?&?+E`+J?nzfP0sukAlQR^p?<~A#5VgS_67BKL4BkgCsk_g*U`p-b@7%+ zN3ywyZZRSWv+vMrY3ran^1uape5_mVkW$v&$<=U(6s4@u<8e$AMdv8?w3w$9XI7Ua zIGuS)*>$bEhOA#RpGRMfo34wJANwF7N`2ebvA<XV<6x68{Z@zN3LHv23Mscm#OsRem;hnejiZS|1PbcvLXbYG zSv&yR{6rIY_O#h0xRPe7Kwgu)X`tXn+zowE)A|-B!edW6CeEV7e=|FEu9u|Ntp?D+ z8C|jT5rOVtb}v9WmAlH3Z>QJPU7&379}ZExKziBcisaTh9Tl)fgjilJ5E@E5%rQ`a zdWf%;My9>%9pP&}OpsLmkAt*ETc#j`6^6Loz)zA|YS+Cc0jA==_3<(oQf)>uDWZ2NI<*F# z54C1fBlW6yxGt@ud*s-I_UMS}`~35~PWtEB0yDp2*4$Z-5xEA^;coNyT34j#xjL@# zfyA=1tn5EJV`H|nl&(K%&L5XW2NU?;+s15Qj)YzaYlRLoZjpMogDok%m(vfu_cw7W zn|+>Ug!#RR>@l^t_@;B!FccNAdz4m@J7{wgNVYl+v*yP9dqR*PfWai6f~4wb3Gdp;t2QoaU35;i)# zcHDF>vdD(pPIscn5hS=3^{0X3Obi1bMEnhy$JkQ?-+65R8ZljToIJW*l*2kqKA3YK zQpPVQajxU=Khb2NT+kN`ro}0DY)ttAx-t$GkRNq5XbwSFD8AbDw8(hT;7p%-lc&I; z{*73cjvFhBUlmMB!)8tlOkg>7x8+47=*e)0r;yMt&?SzN?h|epHSFB z&bOUNgRjNJ<*&zndeRR>Eku%n#ZZdrQed9gFE7^fsA(!IKY!O$@(4mvT;J&3)M2ao zd|^OSsu#*WQCQq>mD`MagJn=;)85nl2nHS<`AT4yMMkf*#TbZ_5zt z?qzrzITF1iKQ=(rjWp3X$ouGZa}hqKqB-I5rL^dUkY%2UArV**qP{zBkybRB!}!ds z?dv0A4k~IwDUH#<^+vnG^il!{l%`Gkck#3CGu|6RZ}u6t)pPiae7;A*MdgD0)WBl@ zxHh9}%atJ~iHt{$x?WU)TZbBe6ecZcH5y)M<+sWg?sZ+p{&16478Gzqp{y+Qi4(zusc9*rs&Y}uvh54v4wcit^^?- z0_RKX$*0YL(NuGn>3msyZ#o|8I>s@(ylR4P2Cm9%As@AQNQ9YH_8vC|uw*4*p zM&ZW18*f)=wtbdqK6B9MW0oh@j|(M8+xB$5Q#2BFmNh<-EWQlO{aWHE>u6=GwnB{% zR20Y{YrXZo6xdoultE@uFt4sMrABp}A5L*b{Eq#EHa=5a72 zu3Bab$(43g(Pej-h|z0JI^Z(rDcGn>9RNDQ+aQ1>8L>~#TF1}YwjlI}-|1wY470V% zYv00#N|0rehJYD*>+>jj3bzZ_uFTh7W89gxzdV^8`R~GL!K%NbT^TvXsgl#*u9-{U zw`j3;{q}dTd&7&iehpncipgszbgvh0`}%AW^ulsiE3%yjXp*AyZTN)AGfsLv*3M{mVDJ^hCM{ z@3h`cNp7fo+%ub9KL3c2y{;UHCo8cV4E^GGQo-*YZSVTepJp>nt~;DO*5Kf zC9Q7jD1FwXM8hVY2+?TClvvgml{{DZ)DO4Dl`n+p6(r!oF|Bi2x0h(|+L`CsOp(v=XRvvgnLp)xalW zos5eDt@U*IA6cA_A)xrsmD)sDKnq1R{H0$V@BPFC+9>?T*~dy=uZHE&4;8++H>E6MhYc)BzF<7%C!8p*gAg~FHp9l?95 ziD`M0*Zhu~w=z)Y;pfQ^Jig=&*UFy{~d_@JPuWcw+TNj@I9tuFeRe^eX>^YrZ%hRpp-C|g( z!00p4-9719GB|s2I;Dd+1B4=*ze!W7FyHHT9zCDNhh$gxUI)^LGZX0M{A)!n0eRkt6uRh~Tmi$3~^_T?ku z3-zIGeKuro7=L-jof}E&1nul7_vwY>h(lBHB{F(9v}1ih`>c_J>kDcEyWh1tFQAgk zjX~>;i}$$4$L#^1w#;KZMt@L)BTK*Y{VlbEmY@5;sjSoU%vxEohhC;BZoIOc$|>!Q z|BwaV>@T++2tD)jkB)|dYRBV!G<;|tMG4QL21WetyOcS9=J=B$x3UeK(thjw=J5va zp`K-`u`j95zr^)dW9132dOhGUbiN3Fm%!i=$XSTu6(gp*;0~|us%_$Ih5X1jQ;t8& z2B|&w4@YON?%?(joL|JzRbYw5uasr7$2Phn3JC9v3|VQX>aEIW?`*W0h~u6g;y)r2duipJ{hsO0JiTWGVW&N$P$WsF2b9e-@^yWyCY*n`{cbav0f z7{VWak9Y*u^4{a)k-~7ck`Y$9+O_uHHx`nn`8U zXDe0HZkOtU?}N6My3p1=aQ=WY%3C9D9J@L0r7WqzGP+NjEeD}YWC;61)~Ia0uNZz0 zi@PpSi-{YB(Kiyb!kI*xG|Y!rC#Qx2PM-zXMDi?fu$tbh^>I(#2(&T?T zAPjzNrkH+Gw%3)$^P6k-tr)EeN+6_-W+UWzlucej)XnF~)8E?W=F_(B?O|-H02}AL z@h76}8t72gOHGLBd03^^z?S?rt4`DW#;f?j&bDcwyP*FoXu@19&uh)RIh4gj&NyyJ z@7r1-H%nr_pxlZ?!wvJs>u6;0lkSE69grYo$$8jg2hpawF;5v3mj-HGMqP}D5nv)e zfB5>`qSp=V{X8B`Dqh|@Fu&atOy5!vzmUi6NgKfKCTaY}CmY3X^=I`b%8(#mP+@nX zSo)pj2e@MR#Pr+{Z<${UbzG!&lu;Ww?^6B!T!|&h+@TZ0?v(|R&Q<*7Q|IyH6sV<7 ztOg5*P9gJ~N^SU)G|M0Vr z*H~o@n|w%pcQcYmhvw0hXKkf!*bnbBnuKm`ZTGZ9DjXMHtVpx8`!qYc9+-v&`TuqF;JI9;s&)cLV-J(jLa zYBWJgxS*`C{Va^7bNLyMZ5s)Jfz>Najupn5by}RIL~X?P@jtT&-(*m;;vjS2Cs*}J z*uAf#<{=_V_N)UIDO#$TPZ%3iQ^O)i-j+*r4!)yZv*r+u5PenP*n1)?EMxY2w#TT% z2}GoQ$Qur3x+KzoYz|hDl&2qo?pA@b_Q7f{_o6e5{9@u`iBJ^_db21pS$krW69ZI4 z)e-8C?Hp|}Ijq1Uen7zDVlbNI-5C{Dw9w5irb1vslWpPVl9yiU*0`9!{liTCUBW-f zW3xrR8rEjfu%&n1VS$(gx^{AQCBBG4nbIkV7NCdGw%POUrlxxmQu|>psnMv-2Cy4} zy>Gz7h8!AGBdt-s8^8`oMEr2Vf}Y*mMtrf>l5@MJzEq!BZt8y-vpRrLeC-=+bdDD2x_UizDr2o9SQMTH48OH?LfLqEm0=H23KaQ#?Chl+Gi6>^<~Iqj}9 zu1l_|_isEH4)g~jfG_o0^mmrW7CoDg?TH+L&>3lb%cl{#Nf%d>x2lZ%Y+sx@9Z43= z4%Cs+i;R`mbESw+sDCzoWO8~zI!cH)cSH^nHFABIN2rfYXBL+WB!p%FBCLqSoOD1| z^)frWOr*nHO+P}gsO?fT+o0)zSz!-syK}G%lIB_-ynN>T!-WVv+uO+nPhSQW^khcB z+#AX$u`+g7DER*7vtO9bGpvv@!-{~ALPn&0Q;Vl1+Ns;ZiSbR?F z?8m76-fb8(UqF3L&k7zz1-l?)BY~`Gu%&(v7SOw6vD~rLzubBFfU2;aF2ME~q2Z9H zaLI`Tae@;iE1VV3?*(MCJiJ|u@GyD5Kh;4Utrfei(i*&nWbkR_tcILZV#o%KVGTQU zG_;4t-T!Efg?07Ivx8Pvft2NP4I+T`V>@mhZ_bvl{0`Ko$6P&@RFd0hFU&S)0C5?< zY1T&2!%Ttu{z=kQ*J>kZJ8dvE(a_F&?nejky(HJ0}%tLKL= zxs6E$Zc9{H*1kTecc&(SgtNZ@hUCpJNfntv{UnUe>D3fC1E{}yUjC0udt@@`E>ptV zRP}oVD$wk4IfmIUBRczHZzPR)3_gnqD0>`si!TDUGjG0IlGNUNwu+tXopUAu0!YWw zaG}{dowmaK1x`wr}%svkhXdZA@nNEs*WuDSW*>1b1fqKdQb0DysJTS`ih4 z5~V}ByBk43x*57dy1S*MJ0&EBlrHJ+?(SyDA&2}fzP|7G`>(|sSZgk8);!O>=h^3+ zz4uWa$UL@iGJbJ^FQo|RCip4SDSGU*DGpdke$?GrqIJvB-)gWi!}o>I^Wnw#-}4Q5 zeh;*Ml}6R*#$VN6Bl1&?f_5xEASL@yTBExY%mE(6GQ%awZ+)LYg#cB|m$bE`OA#09 zl~Cw=vnfVPA3*s+X+*4{gZ%u(tgLI;IC*;4fkeIL2V4?z12yeb*N1TrNFe*Bxd?$W zsG_tI0oA`;gIuwT#A4Rq+^>Z?ouYUeW}wDxIWES`xJeP`);%7uUQ~2+J=}a&_-48O zZRbKIIj;N6{78C*%`Y)4YrO|5d|pBs@2TU@OGIfjVbhCC)H`u0PYnc~eJ;O>NF-1< zF)V~kM!X72Zt=r=3z%1s(%ZbN;>aVSC}SCI2|H_@J~FV58o^9&fFZ3n1rI;-{)& z?3w3NPvbJ&cBYR<&a6mL;v<0i zFBSXM^E!Pd7}Qvl#Pg>=3LG#l48`*5AtJizvySBtFx364kN$R1iis);1xtLYeUmYJ ze67LrPITMkMtDdnEOWuU`>`Sf6oSsvicY zPxEj`7pQgnD(F9R=_nKxXSv^a{2JQ)(c`gUCxt5w+YPlH{Tu%(mpI_;z6+#zp(rgg z1;$lu$mo^rgxIOoC-5R8yzgAcRy9(+9?!nB8X#wka@O}cM>6Sy(#^C%yPYWNb1g98 zW^6Cz5((Uu+xtxQIAyvp-lmOov?XgPRLN5L2OQ%|gZg>cjLT|Kgr7^7lM)%Ju8~v^ zIY2&;*9c9u7N&!gU!Nqvj^9h@F(D;)gqU}NTp2QD;5NrM$sXS45a~q}n#`J60lN|* z-y5VLOq*{0N$;WO=V-i?`#a)t5?P?ipq_kTs)yWJxoCreU3M!7W^Gtq_E5W~4F zHCQ;Qsw&hZShW0Z9cF2w-W|q~?b~{n%$3Fl+KZ+n^7iNFdHUYdDVWj`v{UY`>9XWP zepoCm)y$5AAzlGpOeu;NmP;y?elC!6x;`O98fH(dZ6qN&?k3LpMj9+{MzQoGrM~)1 zjvsuSfC6|+zb9f~>?KPYnjM!mh()$*p&=t3Cr*nBdTM3}QE1+gm%?!_621W-rG|~X z-7fMorEeV)$N1-CAdA2Z?0RjU9DQ61{R{qIy7P;1V2*nf=B{uZa~gKCxQdk0FUd#U zG1;|O-)W@AT>khX6+A&(TvZW=gf2nqe1?(M#6tlpV2dv(PWR9p z&W<%vXiDclNR&}u)VhnEf`By)<7+%h8vw2RKRhLdpqC!Mx^_w-C zXA0XMvczp7|L;<7uZDknfY$I^s6H@FYj+4w+X3tFOE_n*6-|bxBMszFggBBH-A_fg zpLOCDlv5w3&{X^9mA+ zoj1V$!8~RDa}w8SjW87Y6@`2Z&Ffa>AypD9P-+c}Pdxt%E+umOPe2Zl%L$u!(FGC$ z=Z8cCBEpdc%Jl3uN=@#f?R7k`CDCU1)V0nBa`_CcNum zPG9oUx!dn<-qE5KE2Zv{A-K(W@Cn!GdQOT2Cy#$?O--u3vZn%>pksk!u^`fb!xHLY z%?=eNg|DHjZaFitCH@KUc1NUuhq78yFir7Y+ZW0Mh?KrmU-}7a4j_aFHi!LO>E>^l z&e7!ogfVRx+6(92CLQqMS_T^#&&=j+=X|<>s^k3bEG>ap@xRTz>Yyi0gcn@5BNk(B zvjmZ;@w|pV7H$5uKoix;MZH3JeKjSG!6<)^!WT~7-NX|e0a&nPeY9R_(bq|8mz`jUETE?-$gz1QB=9c5d=&p+)dmw%n|kw2?KPIxcwDK12d)2+a$DL@kDF*#6+R zhN&Y0qjYDTu7vmmE!PqsWlLGYQr(QR+F`^*p;unt@=cSbr%lYY?6`na9*wWaZ0CIq zrnMMZzcQ%L95L|M)0Mi+P;V@oxk$A%rdPTYDkn;6oj$t!ZgZAjQ+$j=m`@l*3qC=? zD%g>nJrc#)h0q)Z7O~ZiaCbv14>wru!qUUiCI3RWzr43l=%yCRu05h0U7?#<`<&mMw9b`xbo>M>L1b2eN0hoh z8>@QRqSsA>GEDuzR#t={!=6lw=PR+XyPMuCOq|+1VctW^%{j;0N7~@>*%e}ONrl$R z<BowfXVML*jSax)Jd*wLxfq&dm z;9lm?0MWw^LzuZ+Y5AKbBPD$4mo#IbL?-!u?NxqPHM{P&(_N0_DmOz_{GPdkX;$>A z7wFZx8L{WOi@S>&0#R7zC`ldY&*M_6t68+58nlHiMkmhXw51(EmQ~m+>HJ>+g;iGy zW6dVD$2%sv2##IAt652gzsSdZuW008W*&2c6%dpA0`1p#_;%k|%Q?M-f~`KP7|$#{ z&Ci;qsfk6#*M47dIqMjNH3}m5;qq=ds+Tnn%&%=2aYJ3v#Ax|b zE{&)GP3jL?H;G4y)7+!PNiIK37bdr!2MF6A$w-evePAF2=nlbtlY?)%G~pTf*v#0P zyV^`i*Yj9Y32rQ})|$nzMA+;fF!}1EtkE=)cubQ!ROrogJU^>XnhV0e>9x)U!oc`N z{fQW(Hbv?k#Vj+{_|0)L?5OVi+{lu8zH(G9ez1emkDL{es>E0XfGJi);}HWC^E)Pj zS6nG{ofwEsPp)%DpGnSU(RkJI*zauTrWj89EmuNqbc)VBC8g=@zNu~Xl$jE}$*t@1 zG28o{EC@&Kt(((q1F=@LS9^M4IFMjT;9>14KbeK+mkwZ zG%m)kii}`}QU1)Av7cJn$ewTMQZdiQ2Ia#CY5+XXqX0lLv>eXY&Dbd|o<1VVwip1a zA7TuYODb9s8zZu6M^9#@|}ptxDMbRB7|@R&0cU5^I<=B-LrJAFh=EX0&!Gar#X zZd3IPP(o5WGE-U&Z|O=>D=$2?{ymj234Hud+Y^qnt{w>Z$2N#!Thpmhag*`?3jK6) z5Zo^+2!#0f{8^C?ju@hDCS52d%u*EKrFqXK6+HuAC_jK71)g@#Af{SkKqd}LK*0|t ze{(d&)6$S>LL}RCySW_H+eV$6q`yBSY=iGvqbDacx5=>_eYF*dQcO$#du3zm{ltPe z8*QwXvo1$CB?Q@eN{y^Su71F+d)89JCg;a+P8lh_-?Uu3A$9+ZrxPORR{KLhg0q$M+%k~Rk3=>esVH;Cu6+(#pqd%`{=-*~#_L_4>g zaWG>1*aB(!{g8={*rFuAs|+Zx>=MU|Zvxt%q!B$V+NttD<&4eX4r+3!b&WfJDDn`C zM-tAj$z%BNnZh1VCcJoa1__;(_T_6*0t{v1PUf8EhtSg_m}(@TpO_PFJQwo3tXVax8PImcQWjN_PrlL47e$ z9b2&4cW?p3SbKUNHy-o=s1P^0EkcKLT>d!uhqBdYKM3aza23utf^H)C`evd%$t9!T zigbl*`+j`O5xD=%ss32-RvP-(v6F!jUeFQIYh0RU>B}0>zHP|UDnWAL4?=kS!-g7w5eWSr&cSEhQTzi8i&uLvifz^i=28!V|GY3rf(WRv3 zAU|cz6~#nRzxoyBCl}ODw4$J#6&ORlW|>(a^_yi&otLrMN&b3(K#b-W;dR>tOuy&{ zJM3PmTcnP3232^l#7@T1AOeG+yfhXvQP(udWN{3_)I`wFPP4xJHS;t zW8e^xd=<^>d7XC_HlE)Nnx9qkA;2P%aqSl!9vu!H7NM$*)Sz6SMxHa(`hyAys!aAh zPhRYb4%SHA3|lyD@||43F~bH0_BWD0%H`DA<2}#JLcaH6|IhA^EU=|bqK$v`rFDg8 z6Z$M^3?xJJkM;Xw{lw_IK(2yR^bi9iFaLGJx&eeQNt=Cb)@^(tV%}EAg*?pkFnE-ZwGo_NW8AL`feZE#HV%}eu{cDhj1PYUq&AN zqVPLCR=l`#tp2`bd{ir2SF-3zZ)v)~EJuDaiy^JIu7+N?!Nuz!j#Z+IvfX-;CU4$M zO-n_@zd~cTQ5wba)1FiY*o8=2C90C@0wTPeGDkM~e$m}6JErL~`M@6M^1i`hUQ)NJ z2cWS}=S{Z2xpl@M`}%8SNdTKxXZ}~Y0v;3SU>ml`3jM=g+a+DdGCud+!B~W5n9O?j zRj*{;=PyCB-@ez8@+kG0t;ujBxsQfux|h>?VrJe~cb*-Z-+7N1FXWVcq2;$+5QEJp z)9}-ubQg^Sh0$kg#dMvZmEPa->RKOPh`H-2R>k_ra3$;GI$j+0FH0=_b)#xAKy49v zC`mH!D9gH=Ff5_<(%GlL|5LSkRH!Gi!UZCqzWP-%#C7lmCU}LJW}lQm`%QE&8HXL- z$KU_u>JjY7@(XKqsFJ%_OFBi>_qS{PTZ)uv5$S1tMa} zgVM~{jrJbvn$rOnzE)i+-SP>fd?7G>RI1y)TX(Nk*0ndJElh<2W~SWjVp9y;RWM|d z3CA**HY=1iNZF94rm57{qAPb8XzN*EUtiHts9XIeh?^-f0nJ4DMC(0N7Ne&1&5Ey+ zD=Lr^b6B>w6*Jb1XEUbgGs$wvEOtwXN##)NIJpE9 z(}(uaD^K~SY5LzWcUqyXcDFVIt7eh2u8=3s7k$|pq-p7{{e_0PoJ+hEXSK54yvd*_ zA7XyeKT4OV!iBCQwl2@ygcwEZkG2*m6ZwwcMOov;+n4x}817|vhqh`Ii?3x*vefH> zDwUD|S}dSLyO!?roN%s=*S-35@^Tl#NoK)s+5iZdi&2{?O7E_jsuwo{F{UTG6`Vbl z$7dqdu6jivo_<+;)cme{r1W)BRR7BHVuV+YT~@75^e^eja2#$%kJSk)q5EqGqSiA1 zfevGUW3F(E-M>`FJR3N$H-CuqD)xs3j!)*-z3O@|t0jWp2Zslg-I-UiLN{)&_gu|| z%udV6LR#VyC2og$15$JlxSo5S@NzbstnMG@yY%ma{xYcWtq$hFPA;HwHKrR}69+K-R zZXvyj73fn5u3M*)8~dc(8JJoHY8tD+rwk3yd_2f4{EYF=LbBJC1ZpQGc!Eu9Cz_;9 z$nOCi(I-7Kfrn?+i-Q;#;ItX~_l1R2dGvIVU3{l}MLJRM%^1oG^R+*sqR9unzrU8K zSze!PJ6s)Kr*rQBJsFS~o5m($dPS&3F#x#5l@nF!^bMP7RIZKu5`L zJg>^>6=-*8@R7@4(r05SY{HK4k4*UX7U25e84L^VjF;p~ua8L^Ho7aa|MDooR922) z&*M)9?S6FI8pCBPffkHA&zEWgjb7J!67xEbWTN}~72XwRiFGlkV>f}~(Xai~zj~2G z4!A3arOI6K^z!35=n~pd-b?#`fqYJwfZXj#r9vpG7q7au*A3?V;U2%(BZ+mCf!`6V}dtFjZeaog2p%jaS8JDrN9?u0_F3g6ew=0j!!i zh*cF@ve`z^Vrmc1OU@g**!ZGb&BU&*I>0}lab0sT=}kI176%$#%)-wAnYle)u&FSo zi+q*b>(Lw$b+&T=Byi*(m7eTfjFETweETE8IN`}l3!5sl?5p(5`6!b-ru05j%0ia! z5nIEN`cx4v``sIhp0s351TMXnT_lgAjK67ReCbKM9$$}Hj%)TlQa-_XS0;#@BIJS@ z_L=nV($#H1!t1sfj;0}yhE9NzbmCf(@~MV-s!G`ScPxRphe;XS&khDL6?9RK-72`e$G$>x)l0%%|AtBXKO80cW4lYMTr`ifysO>iHZP2@ z!`=TsL3@Y@1z@_u-?zY~Q;RaKE}CD1v|xixY9J0ceIjqMOSg|5LS1i}@h)>LiMdmk zq?t_7(gXA#@Iy|foM`o{UnBl z0?p);l*!8@kInHq-2BhVd`DYIGZ7CK{lAf`cR$A9B#G4Z{aXCxJlMGCDsoOR={d&{ zsy>`|&+7Ke+9=h~-K5|;BaqSm>HF?DM||;fU&4n%k4%2g>$Z?%?w)_QK@k;UHclao z-}IR22At4b9d%TLT}5vBSAkPH4KTtWIJ1V5du-)&^rn{|W#31beKl^1!sEL|Ml33t zHS|9=Mx;Fs{ROy9KE}5$;gwM4E6d@Rp*Tjt_cg|ah1+2$UQerQ!gTm?#>?IL(iS{M z%%cZE@qi=TSQwEGq@BY94?~EulXdI&dK+cDDhF1$@UH@;5O9)tRGs?*&kRT^rF{KO_XM^vHEIB zI*V6-@c$}XLH7zf*N%?K`)1Kd!Q;vgdzt3&ylcrFl8v>JjT4yvR5llC$i*4+kRMLU7rXHCC#oIWn_uuae zCAc`&;PPXAo!0si*Gt>4#t6vUK+VjG(!}?NC;uX93wU&`l#@Hk2bKe#_O5O#?`JpA zP_K{wQoevP-y7h-Sz2Zw{PnbBYm!ylX{r#<4`78ma8Dq2Vs(DPv-UuSzkgjmi4RT1 zs8W-{v*L~z#07bi`yU0>uJmIs`jr)Y zY@FE8n?LY+dyDDR@m^Y$h0s#ttrXgvN=a}(&1Xe}ww}2FK2<6^$Os;{GKouQiQS>? z^O|5atxho!f^!9Nw9UXNGu$qzGU{Yo%a2n5Jb@ANdXZOD{Kdi|wX4*;of9Sse6t}c zhQDR!J@1JjsOs+58{_3v1yy->emCj*z?1Xpl|xxfe(biheq(Bb|Fii<+g~T&MWViK zlS)HW3!U**t$v_#K-!oexUKKZ;5z4LAU~}+`9Il*wirKI-{zkaRplX2ofRdkz82W1 z;{6I{>ot6~%4;}&QwUHG<+a1aYX4fWYyjoHe1dbT!ug9jW%cut@lPj?sz90V#L7hH z&f2V#4!KsS$W)Mpkbz?cTypq&jFF_qub)~<%8<{d3r&*ReimM z^NHQMBZodxnGZnCxrVoU)|#9Z-4{JSkv1|DEOhnx0yf-ivT+T)tyw+ojJnG?vNm$u z87YzEA9OdIvCTLu#|ZRlt-LUvA=P9EI$cVc>hgJTN!@DG6=HO#jL6+2bzb|X*B2YC zb?`OB$S^))j&O*vY9g)=Elm$UnqRITt~Q7m8gXy(^W(wmB*7!HRzX6wkHJ&xlLzE5{S#L5%=Ut^|VR zTMi`YcWWkE;8139kIHG>_@jgfwycg^T746bK41CD>XbB!IQ0M&c%SitSWaJN~Z!i zr6s(d(u7Q%tiw?4N5x{wd5&-jY6Mr(E9COKL!#>CNf&3Z^+>*5#=v7$Jj`)wCfk+) z-vw>D3)GbEDSJ`p7565%;6*z#{^ndPX;s6XC>XoGZ}2z50zF{#3e!DRBk1)}5bm8u z>YBu^fO|t0(p^}F*8Q$qr97yRV%GJmp>Ue0`x--`bD2MH*CHaMu<(seqlHfkF`a8i z?rCJ>Q?r~F8rpAE{@Str=J_AeslsYLW~QXQ^b1?a-`}B1gT{55bL>e?;tXLc8o5kP z=T{N(n~rN>C)`uvuqxBLg-z1m=eYtF%?qwCH;*QBbIXa3@L(K@!rCh>^ zu_X4Md0eaX&jfFQtq@uJ?rhZmfby=-I=Y0v){Rdl^#hVBZ>$>jl{QUckoHdg(^2FT zjqEn)7o3Q`!fz9=Zn!O$(^!~7#l-R{H{WB+d+?)WYUZ@{TF3|y$Q_@!j9t_85?1Ke z#Ngi!5PY<4uGhhtnMN{fP!J((z^WHB$I%zH zTg}hnlol^E=t?K6HaVoizZ)bx*(rIM+G_@Ml+u{*%}utnp{V3DZ;<2U%WzO+EGD(b zn9$c51lH=|lw@mbErLb8?7r~4nDQkAo(y2a{q6(TyY*~4!dK6dm}#9x((QB*K^3gg>YH^34Pejy)z# z!!syn_vWafc_5EuAu`W(-T;&fvncd7PfFjcfyt8Nayu>g9iFf2Nfpt1T8E#W7-EmKw)~QqKZ?t7 zsr6}^0$}}xjKnsNgZ>Od966nUFRVA1-j?bI^GZ~ou!dQrDDYsnVOj3!{;X~j=<=6` zPK-*LcThk>$!L2!BC7RpDP2xvVtF2kFCoL$=GnF@ilZd4DM1nE?TNM4U-~}#%TPt8 zh3U=|SX&b%yKS|K8k&HGT;PgjO+nLmxybFA+9#? z1Xf?C(`@5Z=Xa`LJeGVS6PWS2zovC90IqqLy!M`w$kIjch0;OMz}#33XnbZ*1_u}y z!f3wld9hb`%~jOBU1ibROw%WsP9f#Apw?@Q?H?TKO7}aazye!an`d(bCY7WzBipR~ zhiy~(;}0KaXPH%MPd~|+p66&T`r+fhAO<-4(BQllzUzgwZHr;s5IfqSrVh1_)^oF$ z2M#W>RYzH$Q@yrhO+hf4dA05*jeG!h`2}X zDSSVG)iXH@{f-3b*#?7Vm6w+=RW^*UVq4(CgJ;}*kr zJQwlTOzIjN!_yp6(xxtTox7)WV5?*=3{x1U(^Fd9cZ}$n{C$_guS4Cf z3|yVE$E15I0G`%v?N4Ee1%LYr=aTTEN@sF*@?zw-YI)`r4xrp;`IaS31qnWWf?pmP zJp4Uru0TgccYPLj15)&8Q8Z~$6wYRvTbZC`Mo_&}$E#mB0B!H8U`@L35FQI#C)C=d z2)Y}W;TZ3fm7{-?Yj~V1hL|(-hYG+T0gn!<;1_86VU%RouVLs|GY;LKuBo*7tJL=< z9^rv^yjLP3q|seFb(k z4-|2j1@MzR;=2*D^7BH`7cZkgIZU`PtxFy))5m ztz+Kw!tE0XAqquE$P>!@)jW2)ieYh2!D=hKqm79;$X z=dbL%Uu2v2^*4qSHTu}H7BqIrB?~HN<43$M0y8m%cFBpyZHDd2n)CAO=Sx1DxhfdP zemPrMA|IAti63DbR>T%>iVL1lU7E9O1@cHN>V3=dkaRPwG8gIrqM<8SEnouI1-?3nBU7!UuMJmWtm@YEHtqsc^1UKOYnyAM3Ifb1^HQ6eg9)wygnrRcL|&@Pk&jHek=LJU4rUtJ1F(OfZ}~#Adf_?!>Rp==P%Wp{m56`acMhL)chVMQ!L1SM;dP{IOQ{o%UK`cdOxHm19YU>W;+wE0=V( zuS!aLe7#L8U0ViEKGfM|7iXB)dAOwWiRTg$T*iPl=8*udtOgP3TJ}lHH;0)BYAsNW zm?fy8OOSF|0o%$bJN^h1`$&ztsav>ylu-SZ(;Q6503T{v$oK*lVX8gM+vBC!I8|gX zd&j${+CH9L?oF3I^0h^q4wt;{m`*={iis=?4U(gwGr@}KV)Y9a=Fu-NZo*rHJgm>bU)|kWP>)kW!1tc$WQa zij!SZubVV(#tK^Q`VMQz=hQ2SBZ1RRGSu!}>KkzNZrn_82*)1`fKKIa(adnkF~FYaVWX+x}GY9^EMb6lJ#Df5T&=+t3qbc>xV z4B93K!zJR<($zsNNf^A)iXy!`=If^)=QfDln?ZOwh%f4W6Pr(M2%SXT}(w<)@) z4-0-T5gHpI4<9cHjJP;=;G)o4^qeqLf9A5KU}}eqGYxFs9RvnUkDXHiI?qdUq3vdt z^7sQ8_C4+z6h$>xSmEnp-A@f~%Y9jT+lN^UB!5`l;weOIPygj^GYkGqZN%p*!IXFcx=CXeF5kEdQ)4jclqE8t0Anpv~jr%ulGRg5($ow#bJK2 z(GIyGH>DnG!{N2Q@|+^+sKeZnx*x+2Qrr&LWskk&c1;($*qkDL#+Vte5=Ys;OgGHO z-NbDdDto%8roOtf9M(?%ia`5!@h!V#hL_AVm6Ws4M;SETY8=-Evp!B3Udg1VOYK0Z z5OLE&PW0&45>B@`#&>oVln33HlM*NUA%34k-z^^ihJKkw;t&`crVJo;m8XJ9+8V}9 zLF8)3q+F>ghZU*#d+!PO>GXxanQ3fsm3Z){4&2^F7S!Kf+jvLZ^Le>k!-Ve4 z)rfT7L79+`HjlN)q6Gvo&Gic3n!Z6M?L&F-JgMm;g`OG7?hCBgPXPh}mxqnpX%� z2hikP&0*8_0&}{;<&skArJCEMTA}h?;ZUthmGq>R1$tDP;wV%9^WKsNz*lK+A(%42cx9?)BTIdaW6HI zNn>2onCso{LOM!`MXejKqQhs-z1%-<7cPI9HOJ}hMN)h6uNS~!z>c1tB41IgwNZw5 z(LHt#*8Z0SQ!kaHyKU)OXj?>7l^y~nT!1%WsS3QBaXtN7GtBGd@(R6%If)pg`1cbT zD{+!8?8ggqfa_d{M-J{2^h|bb?2yaHVH*qcl`ot=z}-Xp3BtTa9!3Hj5;Oso3>yx7 z3bznwraY(@Za2i+-RXU&lE(+zJnO@V#Sm@x8(hKJ2)5ryTt@NcMyQ_JCmrNWNq5ks zs>rhS3^efmvnKa&Z|jUqDjO^wn%fYTePT6j@@{7I4%SxNxAo!#_O>eW+S5UY_L{uP zpfJgx{B4Xqm^amF)ERC=o}@F@Z7c8_*#QUp!E< zPvbo5j|t~DX~*3MTa~Y(IFEscZ^t7VH}FKK7t&&`-sy^IF-G}kHgwbDaB-mO$~e)w z8?0{&IKeFie%{SjZ|n_4ix4(RP3WVwG|jbFys0xg&#F9>*%XJIJ9zn~O-78#-Q}l_ z$sIsW@(I8WUQ{5pk`kD@x|WcqCR+9_x6et}177qY(^ewAE{2IS(yOtb3Ld!u&9tn; z7hnJG2L(LW;98DR4ab{br#msiWxS}z_uL`RUZ!c_@|p8DEF%&Bt~?WWcQUW~)-a>R z;pA0T?Rd!Zt)&;4pqsI#`6W*pcf^H=Y1XK7S@V85d#zb9Z0w68w}4CbFIMT<1!%&8 z#>bMWDZ?5RP|xUR4YX}rSlP=p-@Yk=PV@*LdA=2*dnLD5%fob1q2kV^a>>{7ZgguQ zUiUB==OTi**GOZZUQj=jYcU)FUpa!x<&l&g#+uF;O^cL#Vq3WKm-PNGs7&EgJZj) zSD_)oCeGLg#npGWhy?@_bJ-UT52qLGCT=lv)7Q&<)qt1xjsDDfO6zT8?YCjJQ}C|@ z%B)incRyHLgjdCC_3b>-o+C5-+owV;D2X_5pwxA~u##>5M0H;w7C(ZC&1<>@!c6uL zz`G3pT*LtH_!iD_VK*GB4{s#O=yaC*CkA)ZoBNg4bW!rt`$LL;>cvTrm#(Gns3n1b zI&72U>VBEm%`ycC35%6QJ~OuaNN@Cp`EEUeTQm!9y1Ul8IEVpC-4o|fr@@9SY~qd^ zG%*XhsWXq|G3mQ3 z%td|`JAfkWa0AN?VQH$9C>nR(yFPcFRN(r;e_+>F=s`t1oXNst;Q`W=*E%bVw?07VY2G0^d~uVnIcWkhKSbamzI@{3=6=5x zBGc-IsU5|Mlup41>s8;v%G#efyH9qh<`h(K_S?7^7a3VbEg&Wp2X_nYQz25Xmy--l z9wWS`xO&%5+uvyT4RyW>;_ykFrgBh7om-y!}TG$)Q&Y*fIotZ};$O};s4xgRIe+y7l}r7cZId!c#mrpG`I z&wl&}VJ!B@m^>zmP-pF}-Dt$z*=bUr638RMH2oSoK;|X|dS?zfF-f`G$vc_8v(jx2 zaEcLM)8L1dBA${CyyNR(z6NWe`E>P(H=m0=ztdr;{S)B>TFg}E8WPBNJr<=IYV%M!Ol>|(us<^*S>%4%SM#>}Go!>1iodsY!OmCXtP?osn z(`*J7ff{v9=0y4~EB<`-Gxi>C4qGQfC2KTw{NB)kAZRKXzmxk|Eh->5qqLM*Je7~{pe+quHZRy zLAJxcEYo*F?nwe|e}T2cP#l}V6wbW2$vBW6$p9XhFRT9+v-l_6^dS3KX8k6t$V(fo z>wOR~VpQe$d-DGIwT}efP@^yH)cwUOT?^__YiM%HG>&@u66uzl20bm##F>!h$P4G_ z?OhL%-sgJimhCm+O5RebBN=*pSkLK;y@QYbQE1?z(}nJ`nY-Ojm_2Ho=qrV1c{D5i z@?58e!o9eS6vD~SQ5W8HfwDe%ExXqm&I#8elAUHAsX2T-$W$gsM1!;wcNcCsUQv-3K-7D^8>5Edkzz~zcfnwU z-qV)1ADU_^I0uF9`(}#AZpng$4gNV?AF&KK{?q-rb?$_;+I4PIyai7Lj)cp4`Wk{v zg&@pjEVmG&Q&yFSHxfR;NNq#%A&(Cs@d?tikS~XkG@CD?k;4pF=napMulnh|&Ki={ z?YW7`#r0c=_n)j8`-JRd^h6cvB(aB3Z~5$rikM8KlaA&H`VrO$YMqk$QeZsf3t_)| z3W>GJEsrg^?&wnaqM+qh(F~S~PH6hfm3IWh^2Y0$Pv{YgU{6-8pDo5sH}ajllztMM zYV~gJ>ld?G<=pBVO%?9-r-naXned{k%4=hO9J5}1i8Ld(r&*Cak1HS+>9f?F+>mSk z9+UGG@Bw^Zw!LKZzra+5qpPE!6{c2^%Pax0C9^i{Cam`<#B!h}kii#Dt2PkMcC0%A z3D*g?v0pQ3)sy|ve?qJd9YSpF0hL3im*wJ4Izz&cI`RMBnwW4;eJn3uw(=~jFu<1! zwbVZNWiR+vg!IGJAwU9hjS+=ZwZ4DerrcH} z1)>Y%;p%;k7V|2Uhx);X2xOMSeE#W-^@$w*s~*bSV8pzq{Eh)D$Qp{LLH-!16`8!Wg1 zE*B=3tD}8yQ8w(XA=P+b=wrr&L!vU8Lv)6y#wqDg<0wIsP zPgryUBk)xT-;6d?Eq40)Q>>$D4AeBCuV*@B?qa-Tx^St~{>O!vMN)mkIH&hZPx(=+ zxYHz%0}1`P;djoZJX8B+mB9Hl#79uU01p*h8L2&_KaRUw)yQnqjwHKx5~Dj&!TsMA z-G2?AEXBQK4|;pXwx4~;bAM=Dqr1V)WgU48CK9eo#udpomFVNZx0QPUuB}9bo|`G( z(Z`p0%;W~yu!Nfke66=EVSr<{`W+h)(TPuT6lq#3b)#!9f^fA~Z6t>-!Lx-KBNVME z&Zu>}A+OAK%r_f(@a?mRM6~`u6uw0_7h?`;k!);E!=5!!^z7Y>Onn9R z^J0C0_I|(O6i+71PLNAEGuA?H@g_pVO8p|Yv&sR4%iXmosvDMW2JMTR>(heqedayAA!=~yAOYQ~b z;^~vD3WaJeuGR>VYrlRyuqfbg%&fFuClUWZDpvE}-GX#_$e~diIIgB$)gahS$D?@D ztA@0E8D*vtnN1U3sbeY*uAF0ab*X&Q2eahyo_7+D@aY!pAq^YpTe$%M6fcr-L{B(f z2&Min^OgjuLq3BKFL~q{GU$ESBpR42DDL+XhHuj7<=Xr9QLfVZz3wHP8!xn3gQY($ zaFUc)#KwZ+-k%ERmcgrDXI zk1ML%^qtnP$Qe)EB-X;1U@FXI*8+q-W@%|9~zpIOgz;QW_lXlRG(u4T)sJY6ygE%c&U8SBDAGh|#ng(tu(WRL` z1QXd9OF!#GzT;vzsmn3Hc`dlx5Nkg68#cF`>tx8nqRQ?4yx7{;lG@Bgt;-g?0Gez{ zg6yp2nDZON+JI4Bsct=4o=3>DF!i#Ey?8llKOPjtv6}H~Aa*4>^i}jv%yrfV9jdT0 zC^Z*H8(Mb@*DxN~ao^H^Fcd3vzyg=D2_t?T{hbCTKh#{EGA)tD-&;PmRH5{9PTUa{ zVnBvb`^QV%a)(%I;oB(T*69^)nek(BH#ecX^N78U--&Fn^ z0;*raIeChnbbmkdGJJm!XIC$@Z_Ybw z&t{3>j)sBf1K%r2Gss3WT2tq3&kb6z(dibA{ttOseR6T|U0L|@WKVl|YnEu)2!05O zUQ0|@)n#yw*=0dG0K6cg;QWZ#qiym@F_`DI?*@1IgEK$nWvWfA&DTN!fnR-OjwEc@ zU()=zz?p$KCJN!+q*R)2{P`c*Z`5nvitUo_Q^fftOPjOciBQQx@d7_n?wy}vXP$OG zqy7>*CS4NA=u60=pan7c(xg4gH7Vnf-7xjvO8!aK8pgwg474NwpKfQ64B@}sy4pS}Vf7jBS9llg(AN5M~!E4UaM!lgP<>~g&jDO=$ zPXAT$%FwBHAKA|(pM@=8JdW8>fn;{k)Pn#CK`%+NE9L`vmrM7(wNd70N;dRH)Hh@* zS(weZyaBNi8uPgr(7WKNutLBEdN%xee==t(-xHWo^6D2zRnsYPTW%dosTsdL`(h3> z7ArGv!{O-U@r4u&w+Rz4M)>n9MO&(0!U-`xbuQg)DF2{-W{a#Zxmc zjFV+wO|azmy(`;D1Lw6Tb`d5)1~su9J(b*b0iR9O46F&iJIaHZ_$Zc3_=bY0Q1;|< zu7UVpHzpe~|MS8%QIZ6EV(I5~rHqL-#>7L_ho++x{m7qfClT#}HeBx|ll$yq6~^Nn z_vx%l{}kz3s)be{gKNk@18!VS0J&>9Yk9#@K&|EftRw+^n$LN{oZ#@qt0kgEVFYI{ z>}ZbVnh@6%DUi=i5W?2vEc^sD%67P(4|G9GiL>|k$Rffi?ojlXhQ(mn!u~M#kst1} zA1paR@4z7K6}~{$V{WonnZ9UzAZsq_{yZ4m+*c%BOt-ykLs_?cOL|ZFo4Vmo_mw3k z?*t9cSbIDtE`aQR+vlLD=tGllYEZQ0o2dmQwK|bi9Ld&`7=pFpueYyPxS3`N_eBP8 zK=>Kz+>paJI)@=(TPa&zXqr6Mo6v0ev&2`7vr2t(8$u~4W~m~hU8KWuIS%?ym7qID z@5f@1#|jn;c&n4NCdN4?jHnCh_OdMbP41K}E(;z*cY%6@==!4(^uhILU2{j%d>4je zf+*w5(9N4bo^EZv!BNW3PwmLWVB~3V@5GXqYy=_E!yzq}e;0E8wGSv|UHK$6eVW_Y z{p3f{wDW!YLhwH^_n$&&K+75GOj&(k4sx8!z>}vxN?;;U2J#)4-Hu3adckiQ2*v6RIK$VK6Fg)#Xk>dH<9l-N}tGKE^*{x3oN$40Q z9@=i})hE;cknK6fmwTJ-Ir#{0&F*^aT~s9}K&9RFw0K_My9UI6lSIL##U^xjE`~-w zzeJRXeX+9TWaZl0;6qT%!i%W#PlwF~>8=P3)4%wYOK&hS>v;)zBvQvb4C*NS8#>YbTGG~>PIwOMtaU^~a$mD?YuZ4lkpKg(^faDiZ2cUIVZ8rlMVJDg+pcV76C z@R-(jel1y>MN^qTAZCubYNv@_)8_ZAO|C6#zAKo$ za4qU|Qm^LS68*kV{WzI^rC=yqFk{N&=-Dfph_y8hyG5gv$_EApTwD*WE>URB;F)CP z^n+EFYJ4vQVlfVpjE*+ke;#oyHFODfBkZa3>h{l0Z-t_*>n~U_{)Yd~wgK%|K)$>; z0iny-)v=#wTM^3X)BWRCJinP5#MLJiVw?sfYn`hWzu0!?apvf}dNucvOX10EnQ`b? z?Zi64xkzP3X5cVBD$tvrSi;vMO^8@wJ+5IM)F)e z5Y$+7ly?19v%Izrdb3P7$iZ6s0NUbVP(7l&NweD9sGHPwP@C80! zcOt&ArI^O~{*X?P=qTki5V(|+@7mYuiG-}W9rSgft}Nut4R*m|mAB%fWaNIhKrR#0 z>(o-s=+gwo$)y0=)Gy-@oDSPyS+mr32r`MM;t!)pzJqD;!lEn3r2QNZZMk%%RA5q5 zFx~fU?v*WkRFzusy;K4fShdwFruv-}ueMR1hG3C^l5{~QC}JL8Jf{lKa>kK2PdFPK z*VOW@|3Hg+Kwj($%A;q)C6xF3wknR?n^>*&3XcDtfUWlcQova%pPzhBX_HC&YUyUVhrUa5Rc7vs5m84X+U%Pr_Q9k06lu9ygMK-X)e6Lkfs5#-r|-BvIx4o7 z=8YD}4*tf0xTtk!CwG9MFi)k-3;Y9^$Y*9v3uDed@B)G1Ckzx_wtebDqXZ(-y+j1Y zLR2Y~g3i9h2@lr(wR%&31_E1PAZT(weS9kH5iM^pdf+>FU|9j5F(k4FTA=;6ajg`E z>Z3WoDK|F4&v^9(?ozlN?cp&pv5;T|HKK!LZLNSSwU%Y4Ri}CZ`{Tk<^Fsr^HBa!# zn)J@}a{m#>HSS*HU)co zCrUfLx94*UgEaxhUc0o$3+WxeQf8AiH!FJDcc6VndO6r)JbzSP1-?hGC=MtNzKJ`0 zo4!W<4l9big?@tweY~ zETI1IjbaXaKyewAm0I3}wZexypqa6xas+sj_33lOTE}6kcxzaVpyK`PW>5fh8 zWwTRPk_0ciWO4`lNAnXlK)S!0ANLs)#(FfqE;9q*)*2nv)7!6lY^SlEVib_5^FmQ} zI^jn6LCwYAFw93qjX`A1w#-q})BOyw-(Q(l#0D;tM`|Zr=8IuvYGyK-xO2j4qnCXE zM)FN!;6PNA=wi^UpU-B7e8rJLofckTwTb?hGsEjO-+9&*d`(1U0!Cbo)0`z}!w`~< zVYmM9l7Nc%;7baKvgB)T+O%3T;WQ2QTwoeWJ~1n@lovKiu3?u}t^-yF+%@(!w%#3y zo0hR{PV(jP9JpcDs60<8XfQlBA~f0^(yx4^O_wDY2qlL57&uy=pvUn+5bwd>s@#v% zwd0|n>9jMOq%!%>yj3f{nsl)kKNj?yrXsFoE2w&=&iH@2FEyB?VPosrvG>Q?5$!<& zt9j16Z&_dj!nGhzJ74CyS+%_LIUIJK^VUqF_{wYdLn;^B`f+M478>jyeO~5S7gT+u z$&02k(yj1&f{nRK3{w1|UcLLts}1s~0AyDw)@Jin4};CmD-H&>pw_$kVrJnnPTH zwvX?@P93@GrAV-ISL?=e#iI+u_}Ln9LjgmY2QJaD3nnoX^$Sd2B#YYX!~sQ*)c*QG z#;hL-bafkQp}PpU{P_A&y*HV{_?wczueJ~TgMq>uSr*^Od!V8^OSr9J`5|24(P-}Z zco45Q$q*ERAew;OP2)$wgP9l{P&~~AV37?5Eq<$+7eOq>w@Y= z`Z8x`_$kp<#8BZchWa;X3q0)d64df*9;<`31xUxjL+Ut2yVVk`A@OM(ZbN$90#bAc zLwlRbnOygg?&4b4;?K(;bnfeqX<6egMsXEkI8#$-xnb_`MANr5s0%9@%e}Zdgce!{ znloyvvucMbGqWZkZ)?bV$Zx|W{E-iy)<|PdRh~#I3Yg(8bAJQge_be~TjG%Hy8`4> zr2_!j4@PotrjvU6onRgpT{n|(rjIRCq*a4i0dLi*)t4+aR+MDyGRzF&=6LRPj1Epd zM<{cyzj!3U-o)CCHlzMj*D8@FBKJr3OVZEN84_C=+6pv-FhQ(a(~>l#av&-9TP5yX zNB2RMxN~rU)*d5d>Q`Y7(~stP6V0w#{*5Ph3)qv=^$-oc*Ax%d2uOq*vFhk4@6Z|^ zoZ+_LI_7|Nxkt+86z1OsD*RPgo!|gM^N7!v;eKVbfcs|BL0|Svo$mjfVRo1YOtq8% z?-TD6r;vs94=`vjX=|fARef2Xv-A&L;9Q>3aQF;(HX1aRLO0h}MoVg3S*MP4zI}1= zGY;P>kjUVt>}wYbGBpdDP#tnCXL}OQX4RS9LTf&zAs+?Ulb8OVGFerQ)fi->ljo6& z(i7flR>g}i(l|%yVz*0yvq-nHeR$TfK5X~ig`L`+u15<|PH`Zw5xuyom=Z8Q8464a5xyxx86oY&S zZ>^m!dvt5rr%q*EZhV$}oMzmmGi0ac&oiO)(t0@xi$Pr9BmUDb1KW&gJ8SZWbk@ug z!jh0-<^1<0=3^NsP)DZ|oZ-snPQOK6lKh}#|E72Sf3z{sMCdILkD1I>qK6eP^fVAM%IWzu1_(PmT+V_+B0Y%H1L1+TBOQ z-}EM~BDZu+0lCyKZaQ$A{)EoHyHvyO2*f+rlwM##!iZXTT0B4+Vz;KMa1IG^D~y>F z1NOEQrJjE(3xZY*(=;8u!9fVVQ(Ce;nvUnlUFnY;^w$I&5DfbTvIh7X1*Bz8gXX*& z^js2!4o$Hjk~Qt@-Su^*re>2eFpu<0HaebC^ye}s=HM|ac;Sq-kh%cZy)&$d=eq6m=z>F<$Pvh@8x{Imc9AXTqlI z>)26$!x<3cny{Lhc-)I0F_K--P-lBMS7UN9WN+{rhGA#WR_K3v*L-7 z4sN2{GlRXYoK`9B>3TD}5c@U)Hn);~)J-AdoJ%gTpICLCmO^`!GM2VBNL=!Dfw`g% zomAIkZ_^B9a#p?YCOU!Kt1%nFFaGj_;u^6sP3!)Z0gUEY_4|6j8WZBLOur>*E9$Jm zgYrghzDPd3NwNnC>4O-BBz5t6;l5B8nB8;k`#Zj12&t}njbY_2@hUz+NF>z8c+>C= zkwz@_md9E}b8e`kM3UM0>fz|Uk6mrCkpbmFT?LD{Y%rNAorELmERX5-=GC14 zj(~#w_3{iR$UpdxD8h#@7|ILVQCL>HOE){OPG>tkMM(xL}i{ zL_bI|w5RF0;3Lh^7JrmaYW>uP+&#VHchLe037X)UgDb_l%1`=K1MeKl9ftK(jg;CW z;uc4G&!Jw#%h8PXRORVzrHBako=8&I957otRU;+!)fo^oBsaF86(mXR-@tlg8 zmQN%#r-B!gj*1>`U9;ez9m66RsvdE@Ap5{hETZ>MMbSd#cp$lE^4M8PQ}GDvbc+4* z;T=}!zfaZz9wwaoc<(0gDa{hK=#?p)h5y&^qB1;bp#+NRc;Ql1r_)LPDB5F1mQff9utx9Qy6w;&ZVCTby zoleeXUIWH)4^*4`D@P{PlT{NtgocYcA13(GHBGrdO(U7mgIVG}XCp-GbfYVb2N}TI zs51Dq-(1fx)T2#E8>^4IvuBlU)V)kA{I(Wi&e z&(BjI-==I{Jn;5@OO*uwr*i*9ZTEh?T~joLJ!-YI-bQIZN5}Ku$;3|h-Ry>xS~-gC z+tQ80K_p+bEw-Q{m$RSkp*%>Lj*K zPg(PztR)aN643*L@UBrF+Gux*j@ECuZMJd18;Z!=L#vFO&u2&?F3m-#-`Ygyo%S>K z%1>(>(KImGxzJg$N~4KfDtO2KQw|~3rMaAC>`coAP)A|#K^n&23}`us3jKATQVHILFlw%$KuU>}Li zuE#as1OBS-4ehC6qC@oafZZE|+gO?P3da8q*!_4fysv)A_&r#}336QvlEEDhG1qDn zVL0KpzC>weRKiKPX``Q+5_MI>*d}2t7fA{gWF^w;e%PdBb4ifmm zQ~tb}y5t-RMVU7Jp!8*=QO)0 zNr@ejV&dm3v(7V;&3|8F)}fcCr*KHnqsEsWNA~u}dfeb+$%k(FGVmDz*(UdA8KJkw zy>@X$G%7UVD3gYv2*J*4XCy-?PBW>2Kv$G6FfqBzlqN7sM!a5XlpU(oQk5m60+Ns?jvUH)Hy0cd^IDKbx0@ zZkZp!Te@_ZvGToel4KS}GF*gSX0v#q{g67F@;S^blAXz-_NUl#A&~06k`AVFJnL3X ztJuLUylRomO}=i!OH!+B0SD?3wI#kJH4%$5jA7AIWLNhKW&*S|2|jFGz!8E#Q# zy7xV81B5UnAx{U7*8po0GCxq!{anC3W-Lp0&c$>9t_6QA zn%YRL_6+;D@g)UdpRDEXOB^Pye)Ur25+Tp`jT~n!$T~yW6Zp8!BYPR-rZQs7~aGHgNW8|`oMg&lXK_D<|oVXh8Cb`*NigfK;#{+=P$1b`JB@`@h*mp zXWBQzpJ}~vPC*pTEd0olR|aljwIjANPPX?aB3pBB?cScglRm!ecDw^*C4p^DiDa4*Yk(&I5CSzV*a!F z&G_q>v}o1PaXw~Nu`W-`Xj7#w-Lr)+jCgCBh}o6BpmT(60-9H~$1^)q>@zs4 zw9?+&pl>~=9oPEzAlXhY(7x7&6HHR18@|r4UshXs9WnQ6R}=e>MB}gegMHyJJ7wCN;-dBPd0jPA!@=WT|-i&7{bxuh+~mI|Hf$* zR!N(Ap6FJ(P%TpME_$Uzqn?oshmN9Oi^mIF6L}+$ilSdVVxYf>Zl+C#gFk+uKCnY4 zugl>>jpVJqh5KF(a5pDtf8kH|p$@ZmA$M13$3!u(J-mj{zf` zA^lR5cBYKhufiW$%|RDGEc8!veMkax?4f*y^`KGDolT#vPKSBV65J=m^p7|ec#`k< zVLp|BMC2oe85o0hmlAu8^r@kZ6791tO`n=CWXI&gki^<`Z8{{!+#q_xugp1|3!1Lx z%jh|R{Xd-OoJYG?$h+(5zd*3M8IZpX>o9XQDK=*QK{(pdZ|zwVFx;A)tcJr$-*e!6;C=>Hk35lJczF@>*#ii4YIn-q*0^{ z*Fb#~r?DglBrxlNTpc_&bAu}g_qxTU0UTsD&B+*&ia(@qk`Hmhl%we+sw3Gv*u!C8 z(mw4qVDr=DiGCn4B_}zxMtV430#fo~JhW^!`q>y~JapPFIYT`LY_aYch2v+-vmFvh z?BRlQsLN`9<7ch1N`$~9c2}rW_Yp{AjfG64EB~mtG`E)bYp-OBu4lyhHQow zvoc~IS#+#Ns2m=nW)4d5505styYCm@58JaEb6OrIUo1>87q-ruH9JbR0SI+rd@mTt z0NoQdB;(J|r#D4@PK|YXn1>_9;m?HAV0vRX2;zoOvkl#FH~+$AY!2@(E;6biB%c&EX$qxD16J67HQ;wOiIyo<+44T;XY**FmJgEqySlul?x z#E#fd+yEMUzgKPcEC`moWPzX^u4|9dcV+O-lsn-Bdj=hk4;JpAWm|X8+B0&WVLF9D zHbxEI`05W>Az{JM;RR@3`qQm>4BCCO9?-GMGn0B>s<~)d|N#Y^Vm@PzXL)8O>;7Ygp zt#S2{_O#_R=v7UccGdA1@l(R)XeGP_Z-#EiN+L+x`)ABSR84|TM#R7%P;EXB`&spLXYpSm^wo_E9iudx+~p<~yvzAa|vTb^Fbt2voq zp|o!A-J6-*hvT|`+&&?wE_x+5ybjN_xPqsru|s;vMd`aX$H{vCVoq+ zNq&n1ip6PE=oKQ#!rWJ$9wXvlQgG(qOT!Sgc&{)A9C+dpIHR(a7b#P6x1IIJyp3=( z`5N(twO@_f&;vA>gV@>y6VB5wZxjg-2AO!0V7m);s zDh1F<(K_>>34giBHE5gPm$IdPEYCbzP_fW{Xyms2!Mabt;Bb+%B z;ZFB&Ct~+yTAvX#YmXkU*T1h8>jtu1i0r*gT@O>2YV(%Pc&+TzC|Ut%Li`S;GpPC7 zb&9&8`Qss7qTVFg4Cfx-!5)uUmAmo``>J3*_21&DOMD}<5%H)fL4{Ub&n@md^z3S0 z)XO%@^Yrc6PRKZFqH+V}Yn# zTtiv6a)n@mK9)f#rh+3Hf4(e_`~hSwN!#Nw>1Me4B33}cC@opGYJ32GwSCAi2Y3@& zUOeS|B6!tP4Zv?!InN$JbmLK<5h9ilm1u0M-sHWGHqT6Rn)|BC7UfY@Nr+;C5$4=) zTa4o|@@C73ZP2g1e=tx#)vkClHIUZrp>)4u!$A7A=&k;NN*U;*r_@W*<^8iub7L)6 z-c~nv-+kB`&AO9GPfn-!^q+HleQIr=XY5M|Y^O{@<@Hk~BwO6Gf0JLHNsJYL*e`nF z)kf@k>x`j>$3n;FMC-waRPH8=GJ`B zqQEuV!=?7*yF5QW0SnU2nZkCKXrk5HzMuG{4{%BCBoN*W7aMBX*c>+=wEAc#tI>0W zmLxr6{j7?&$}R72Or(_h#i@qX#$rlo+h{&My$GMebW37>7xmglRUXHYIsju0PN*D5MVDe&&2fP_ECXW!7DU9ANXUvlv*^ z9iqA%`sm&wQ2f+B@VUzx(9XqCl1!lP*^JcA70MiD>U-uij$3GskT)43;X~t-fL%Gb zLI5+r;aHpYZb^GDKT~FsdY7Fxc=s!mq^wU)GZt|}-0>cQY?n2!Z5mRC!UZ@S=!I%> zDO}r8QQo+5vE?Q|H0jJ&{X<2L2xIk-P zR9r4$Q`6}l(cvTnFx^2ZmXR(z0l~600Qq6<hrUeU3h3-;a`Hq@M|y&9doE0`m%;4!}WaMsVcIGjQ@5yd^|X+C<8~CwZ#U!RSt%Z z%^H<(zg-y<-?iN3PS*f`E9n#V>37*sF)%iP?Q;zNXwz@#ED9^6l5H1BT(F%9!bf7_ zTKM=@q*kdzzj00_SfK|hK%tCWl6uEqN|Bi;{-^ewV8?0|WC3 zyMbZDy2sksUp%`W%DewkPF*!qP zizV91kDxx)Pw2b|xv3j32r=`@cS>!-cbi4dZ5oP=lsVSpVd%#T5Zl0<>uF}M!`T7& z-@G%k&LHPldQyc!$H#IoD1(gOLBYt8k;X`P-`8T3{Ku~J=Rxtu8!FsaOyh^ppnkQt zQF(+?OKtgUGvd5BmX3MEAzV1Ig(Vyv3Csr%t&x+>E64mQzuloqe?)`CNjQoo+4RaBpG5Ft3_(FtLgK022h_K|Y>z#0V-nU1^`_khW7)*1NpXRY%YnNWa%I7N@`#SQ z@U*y|teI7i7^jKe8D zN)q4d&GdJPWhFdQ4`XU0fKy*Kz;oz)d7q)TbhV5Tg z$}LE^POE4_qzTV+Hg!>EI=S$iA_L43AT8EP>icw05fSNfYZtKrvPPqv26;+vp1SStnOt08-G{1k&-tG@f;_!H{%2U>z{zQn;1o4h3@) zweE&vEPbE)g(n)6CdB^Oo!#|5Ec3@vWQoA>!62u$uO7s{IrHcV<@!!r$|%QmbxDEpI+y zNnSc2-vcIh+?Phc$9VsIY-XKWjr&}ic(}llfL|5Hq*EOpIdzpJ-23Y(+k-fK+eXR% zI7nbsc=}U(mO)hD$?V46_{_&A%6*}KUX#B_VZGURt6Hrm=GE6RRtu)PrA^29aC|Fj zES&Kt$S~WS`^cWVM(Vl`$F6LgVwx*-Gc%P7t{h_g+g9Hk=e@h3kFAGW*7Wf&pUCpR zK&%!U$fbgeF+WfKlLde}GkMGWE9kAA{kfO$M^dVzEwK3$9i+y48)`FJYT4io{1aOK zXQX^_jVWwrfY@e2(_md$jd4?`(7iX1ZkdBw`Y!W0oj-$E0z$m2VK|a5!^wP4f1&alN&?SF}PMCD}g)fmGFO=)WD)q{WCmEa5w#lksq2*N;VR#zx zk{6lVKi+AGYw#r?K(nxHsgVBW!Xl&q#}|KLy6IjX(<*NldEH(JLiV@OJP(m^Eb2Zf zrU_d_{oT3VGl#A2m7|Wbr9Z8Q6Ad71*x?ImBNwmSPiLSXq9KC+Lo@s>hvFpuO3L1b z{k+U?DLWiC6KY@NqB46g&cb~36Q%$9MtjGLx$l~E#QtII`9#n?{%6&2#k}3Io%8UA zXRf#sT+E*R{g8_GArSqhM)PyqLx(SajI@pYuyVDIax*gF^AuU?j@#4Vpbe4GH>86H zd2FI07AeD8u`0F-Axz)rM&5-VGA6~~2O)WoOWA!kOVpVdU*^8rp1=+-Mz^yK;e4XO zb3?Ln3Q2Njs*e%r$7p68R#nW_TSkhZxy*zh+prps(ZNy!_8-{L!(tasiRH{cWy)=z z&P}!il+u_LvPnbR5cqecjQh(#BZ(%omNdVWP}%#lyma$k6E0%k+d?*~x)N-lelc3* zoGKjMAo|iMDD-|d%cZ4wQj#vAU7>vYkL8^ardA+hfy!zw%k5a@CUr%wR%iRUYZ1Tp z^}lUiJBm6h^zb%r2vVl?{jxVoQ)W1%q%QaYb=T=a`vUSr40TF*4r|MXYVa!})2BgoAZ2Y# zbqU4g60_~@71__H@C&84y{tKd!X(N+UVoP2Bp$ek{)`{0nn;Ccub(=hK|=}(g~kjV zQ{W2cv#h1rPX?>uy#&OAx?n_2WwpmK+25jRQv_3s`7}VuddS0)d&lG^&eI5 zh0wc|4e_JLoQtHFRpvXVYY|+=y(F=#OYtbm4sN9a;~VdYpU0yb5xTq z2g(PmX^nisSu)a_uA8MN>l_w5A=JQ{iDGQW<^HO)qYI|f2Wr2Y%ho0Jg^N#9>i+|x zfdI<`mIakh7^7UL%-(nM9m&!ErCGS4UU=VbMRd09JsZ|)?>Vi+T3TSu$-{N4u~sk1 z`%6@YaXE;3&rUwcokQte)QM)9TShGC-1A|$@R=NN@fYhm=O0oSk&t8cb}9C}T0T=( zoqpjMHi4G=ZQ`os1AsOekXT7%=}#q!Oo}8aK9(~!jF_T)fsa_Rw0bST$AL0L7EvV$ zNF;rUwhBjI_e%@E0n{D~Lu#L9<qqa2pp{yk+Ua(t0Xz5x3lpNj@yTekBFQoL%>B@(@+KPslx%C=0*QPRB!YlDl zMY;{oQxQU+jqpiN^GC4xf>Fhz5weFe#n%>09EGF|5YfYw>35mVC_*{5^D!2d z4Cki271(P1#Ln%XrmlPl<+SHRBE&_v3lskNmHC$h_5WlBlq;dy(C-+EUo838EA4b0 z-t|=cyxyG>Bi$M8m=b?=;^RRvb2*aIxUt8*7(wcMlH9tWeO$4?3kqpqN{%DP8to={ z91RJzAP$-k-f6nlcYG@SeT>m~`i1z_w_*BY26K6+=_y`-rG(xM3gncB1Dd5pnxu2j z{hqkj6*-*2tZZTNW}AKQ0U7lK+m{({3Uw~RRd|}+cM5p!#Ojj9Gcmb`na32^#Ns`W9a~;%YEfr6Vw1SyHP@tQB37*bt2s0gbQYt0sDZz5p(x z-}1^9lJ;$@0${lOXI##xS!u!dHm&58dgDEPrKha(ne;!EH=tcs914CMSqRjS%{poT za4Gg$z2Dy+1nBzy(v8sPVu;a>p^6=Cxh_k>E;p?6)s9GD_hzka!MzfFcRbZrQznFj zxxC`n;I^J6q#=<7PVHImxBfW4r$Q!GH(1L=+$A&^a{Cgzy}-;j=v-M=vtD;P(v(7nA>tWT?)!Q zQU!oo>LHY)TPIkv@9;}eOp|hX^MfqtKOAuGKhEmi67Vg=cn3}<|04?14iQjok~*V2 zR#?y{Tc=&E6VAuV{c2DK}p&U9jB!3rp16%}!UkU8IZ6dn7tbG!GSqi|lgNDT5& zMCg4X@V^cm7h3jow5r*4(BUv7bE>qfY-eex^rC+%Pp)Ut8tUcx&3OG!za#Nhq;bIOp2uC7{1d|m^sX#>U)${cGOrSDT z<-_?~gDNhMBIP$8I_K=U^rEL3WLsHtM+gT}n2P398Z+}*6m`~F6caVaRHcNsB;B%$!5l&ptn0A2{;{DDG+KX5?`m}& zuhj2HU;dfcLVep?dMt0M&IfE|hX32~0@6|onDZskoyC=U^RmagIbN2o2Vj$d2YN2i z&L*a4YkDAc7{zkWov)4K2zYULV%QPTl|Eh5;m;klo;}`8XVE3Hj@G$&w*ofc6CIJS z@Y3McO|;Kto@aqEPDyAAqAWC*WNBKa)KaeHYK`SC+bg$>AX)&l)XFXg| z@{8d7%kN@`85CoYi%;!6*!+i2j-XykV;CZ*Gy9W1vO9qU3F5PXr`?nz@q%`)k@o4D z_JQp0pH)lcS^@DW7pD^LtswcxfSH|=F2FI2w(VaPmDMrmVxXv|NA%A^E?py^1tI%+ z$Pn72@4Oh*$m=Ytr5jlZei6W((29}ZYxeHRbl8Wz6_AHL1x}+HBkCst8teqXc)qA_D z3Gd}W^S_k<2l_&2^^KehbCO$ka}s_-s@1hN=|Y0w3>*L_H>&Ho$<+$FJsV0;cuCiu zeZ*7ONQ1>*VJ}GLoO`BN{uq2@qxvNb%BB|y&v~tQA+f*hTJ9Pa`KeQcua{A_65UW; z&SZjX)Dt@7*jZ-xhq?wLk;D6RF!tnoDZ_oy&w}oTNCCI6_LW&H2xrpoN=)XdT<-Kq zh3|`gNh8592QkeA(XmlIbs!(;T26zs--0@e%ay#BrX!<}Ock0zaSZGtY@$Lz$}Wo% zgem@?b`=C%O@kEQRb)#A4xDh<3QdF!us%d=Ap6MA7(z#4s;rxP75u}xa7;}EP!%G-2j!w<3oXUi5!q(y|E;;QdXOL z^cANM`qk%0aijrmX~UX*`6wrjgiWLeg-5^se=CK|S1700+Ou>RcA3>qng7mmj4%PB z8RPxV*90rcvldU&<=RXwmi_`G-2M@L3OW2$C1V84>IAxX9L2b`0m`~5N%q#=T-hx@ ztiws6nytGQT~-K^30exoHgcb`LS%`SY&=5zDWq_n_JZRhKwjk*hDV^}oKvfxr!qie z@YwedT>O;h!8*qC;$L2Mo)_{)$AMIZJNLuTsd`m>FLlhjEYS(C`5o}dUUFySv+`>l zZFBpBzT=5VYIzxuGFP4KfKs}$Qw&KE=@p~k#9_>imcxGDuh7_3-*!W!lpdzX9GQ=@ zCOYa6a5J%3+aO4Yp0g!i=}61Z)*aD4U(rth<6ck1J#IH|-G9#(KS#GrtxalA(b28G zY+t!e_80bGwfA6=PD`5Q&#dP;uajA>HAB$r5JGtU3d!1{E=sxlY?-&NOf*^=vJa^x zI${3{#kr*m-l@@(fS6oOg1Mw*ss-@E2%$q^INS z>1l2jiZMwUx`u^`X|Jii)MVthMZ>l)Zz(gWYdDx5d_bLxpy=YB*E-G^b^KW!0Ara0~%y708p% zXx!u7f)=Tg6XlqQX%2aduY;k+XeBc%nWzDY9}Na~=z5^O^#(c8i~h&2Kd69nxWuTAUt3)>VRfrl-9Ln)4-H^0DvIqzo_D7buLN6Y$a z5O;Kp+D?@j|4F@(LG{r*yWYlEx->5RS0eQLd;)p2ap$s0#U&b9tb}E$z1+_3wc6)T z`rnVJfAz8(=t)HLVNkF~nAE~SBCws8Ic;T4P<>7^*pLnMpPoYRtNakT%Tm-l;N`%* z;dZv4U87}LBpU~vV|xkUJ@=sbsTowMRX*hAf2zwxiJuac(gAAc$9{8EPoBuD3AsN( z9$}~T)QB<`99$D~tUCMAl$cF^-=y*78KYF~4er{uqz^Re|*8=X?v0t-t^J|!7 zQj?cGKRjO27!v7eGky4e8x`3g$XXopQ)pE*md^mQKkg_D<=<=MCWCu{8Qg1Hn1g_Q_G0-9K+ot7htr9Gv~5Z&gN+RY3b zR!M;*BnTS}qyNl-Ky*J~cAFPik;8^h(`aWG5qe+GzLTenx{qr?6`ZPjDhygcUBF5C=cQ4>fCi)gGCs)@nYt7%)yRf^ zdr#OcUV)ibNi-|)T(i9&aXCz48Hro7RCXOy^t*S5TgW1eS zqZoh$-grVy^}Nm@+&*h!8M5wPO_*FGzYUcb_zli#uI%}WK28HG?zuCfXWC1$K2t@o z{Zjzj}5;V(4CG#?LxTd>H=uo zg#j&RWp&Pe^L7gj$=xX4P?3IuvCK8%Rm}G{&C_>hO`qB@OXLLh3+c9~H*zHx3lK#k@F_uicV;WiEc@)4*KNg0QIYU& zs|QMmY`9X175-2;P@bzjxlyj6vv^@uCr0Sw;;I=u|9hpxngl9l zXWK4dwYh$@-s=}Hme1(ky7*ZoC`z;l_l};57>$XfS9@ix>ha`1Qr}%aFdv@JJ+FeZ z(KIjx6k{M}EB8Td$?gpw68||^ktAbWv!2Vh>y8CKUH|ulj;nO^~&pu2Qo|D z0}9q&#BejFx!xIIMF-M7#;Nq!5BF|Tbb(mbNr!tKk7aM~B&6ZlDU`Vg&YU8Ml0DZ4 zaURT$do|*AMCk>k3uJK(aw+7Nd>2YLH{iT!Z3XCq6)sSHFWILJKa2EIgHHiRY{8VY zeAb}&cmY)+6+8imgi=a^gIw7~nYP3NO>Dl`9#d`;Jr>b|;kVAkX$oxGk6+r~RH;c9 zYu=M))nu}6$6?0~C=sRvq^lm2%VCP?h zi@iujJav=WxqCUv4~n?Ace!EFKYm9|?8SSuWjS{3UteU~^>$~S{Z`Aru2CzStq1?` zjH~JoQm~Lsj;&1~AtaI9UtH?R6n1^?D>g9xClO(R^y@Q%3-yzt>|IrXWrVNeUnTJG zus{T#f_FB9)cgUC0Wo>&=2)_W=h{^^=6n~z?b@CYe#7^NBM&4YsGQei-oK1i(kwkZ zp*NyI;fpPp=@-17%66!vwyQb`(u`PrVla{IPv$=}XDz{Bt}nZKI((lURTYe4sd@ta z53bPLVCC2pAnH1}3A#{&Q~G$MKbUFe?V%LOPtS3HoKs;_5N6V7hFq8_&T|%NxNq$| z(m~m8#qVAcPXSw(2sJuaVb@zuxA;&pk4~bFb{2ZR3-2-b*_LY}&wh7xL!%v1n}}z& z(3-Y7JWm;dM2d{H$J+L<%qSERT|6ONBwc4scWH|LAu$nvqL`3|97yz%{#BlPobE(( z@c;Vy%DAZ3?rjAW5D7s*r9`Alx)~&-LApDK6zP_h7U^z=ZWv%FX-R2@p^=86n}K&c z=Xjp;{NLwy_yqd{d+)VZTVG^4!lWdVZ~t^F;#P zyj~5*BE^bh@9aX$$gE4J!@-jM%XNzH!gd>WBJA+;-PO6F`~0qaO?M(wc)YHWZ6{;) z=@Jp{)G~bddVF@)D~`n)RuCD?jCF*b3EpyXpd5uRXlLMc6)b zcs{OkJ^SnutZu->j-7-dp^PI)v38D^rqN9I*-^USu{d><8{Daglr@JY-t@gCNELyk z?07V>Tc;7Y$bMYk_h_@znAiInhhoe?MrrX?zP>z_fLBHco^W(>xA`|=1%DHrNO(vB z+8ogRFS?%b$>$OlK3bW!&t2c|=HdftbOiJgz!nRqX#m#LtWE(Eicks?T}aU@z&8Nl z`GTjjkCB!*J<5gf?W?Kz3R$z~M~lG}oP)1>7&2pILH5l*$RwD+3NJ=pEPM{tu(qQ+ z^?D}+7B!oM8uEhBZ6V}|!ip~z3vUM|xn8S@9;rkl;-9&NOOyKX(ztA9VzTw0FzAjW z>&p6&!k2#JJ){IVvo)KYRZW4EgRx21c{uA4ulG%FKfkM*?baY5bPuINE}L%I2WBek zd&d9TtQL}e(SC)~ie{p&>KYTQxc&x*Azf!alIKVO`uB#XH#}(C8;#MciZePJQ(0X9 zSQ?8Yg2k8G{q{SX`g3dcQXF~X2m8xKmq->b@Q6?x^({P#+m`%e>qnF3{6{Y6p}jkE zPlSWJyDA!MWob{kZtV3pR=qbJ8(>@jV_eIV4%=^y!j9nE|wLDvz*n87R zh=QAR=tsSk<^MbYiwy{xN1V^QR+D8v5rJ`?oB57G6h2Ds_r~$^c`fmtJ1eNxpsO{F zv%MD{xVf+FRkt4MX_6u=SA0wHQ)?RcMWe^Hv-h6B7H>Yh4PGJHVqdZ1{mn)$cG5-% z&?fe9d5+=X$~tA8YrghQ{_Wuf|4oA4)MZE{Z|wS`kpSI8)Uw3O$wuy`%^o=8?R{FF zE=NuxvWPl+3_0BUH%o`X!Ho5bqZ;D@?Sj}M?YB6%_bl&Ep1Hc+&(ewg!*G!!tB&sb zQ8maW2rdtS>fx3Z!V={C-eSnG5XS(Hhp)23O6Z7)js17_*4K2t2TL_oU#^Zr=@h3L z0X+)cu0NlrNvKJF!nA!&$=Wk_MRD3>lOYt9>jaO#9pv!sBjX!eeUP>)kz$r}@@tt# zd9$=W`E~PQ6Jcp3Wi>C{jntLmm_Xh<7sT=(;ljHIoBGKpqP)Uq@$ zJwVP2D3@D{Y)y@F;A-xm*}Yj+YF2w}xXHn+O>1n>JWph=;^cyU*DzI4=9Pci4M# zkA)~^lHy*h+Z*EAsn5NTMHJfSpN#h;E(YV9%)cZ#_(bdm`L&+^UYVZW9!aZf6+p$&`dVQOi#`?3E&Nz$sn-b!$UV7}7la3GM$=5jK5#PWeJfaP+wcSsCy21NaTje-alaocjj4H4QywX8(ji^{gnr zx?8bE??mc_KEUlO2X?KYJr;gLIDQJW=i}&^eq_$jbvaW*1;Lp8>zgPj_Sl3V_<-*& zvB;f6((&yl#zP4$c<=1Nv+#aLOg7FHgb^~TgA_`-Z2%{~MiidKg%a4#sUcJA?Lr2^ zd#8S!YgA1smM_;N2=j#|a@9nY^Wzj%-x($28jUu8K6-EO*9u$4y9gdW&yJ9H=6G)J)BF6T z;=-AB)f2=S%%@-A=f5SjNKs&)wLPGWx>{6q)%2vHLbeP1+AmE(?Q5{mE4;v?yqor8Uz0WUOH2W&&`9nl_?kby{#5eTpnqC~ zWv2hbdz^1$i7Z6jg&GBv?l{bF7SJYL@Mt4Ij$}dWOM7f%PUz^%wX|4hFy>QE?@{>h zG9v?`auZFVD6llsAJ4knwKX-4LPWyDpO^Lyo}T5v!S#D)4j;`%1Ben_^&80i+B*0+ zRAgbF)_E8|UkvAGlArwn#JJJ-c%)pZBQ#utDV9CawN{ zCh*I`Mk5u3X+*N^4k33fOtx`HCV;=66vcR4O@}r&sd2|~o1oiPcot$c?OA}QPY-zY ztUs$wc`8&nI5I;Y2Dn+aN;1#{hO>SL;0fu+F(TL6#0l9IHq zGFjxw1@9r38QXcyRt4-wj}?LaDg{l{!}ROFl z)LmFna3MzzDBj_paHJ7lC4c4+7l|2pxE|T}CYwQM8yz^>jY}%iTBJ!!#~e?)=RC@` zHcMxl$Hp$y|I(LJDV=;BRl-U(<;beA3fMeBxu^&@&F0Uu_4*D>zakIe=Q(tR^A4p% z{zk$-u>8~q4hsncwz7zMY5rN($|6QFyNt!#-&2bZ-EFBKXNrx{RDE{7c$aADVWu>M zI|^^>_4H};#AshlaCiDcZET57=NJ6B86XD@GrngzF4nee>IX

mL9~gXEYC^D z3D;f%b<)833}*(}(}M02 z`%t!r1=aetmb=K=$);N7=_cjW?M=4kc(@nSAq8<(Z*)eRe1EYB zb$iJ68~q6vO+=GWX0rTx;#^*LqrgFQJg(e!&+c#K@&mp)=q3o`S!rVeaL8w_5Bzgj zo-F*6qZT=d_Of0Aa;@=$9=Pe8JP)%8GaR9H-a6X1<5-L2^)-D3DXj;e=_2>x*L8bV zBatELqx-I_k&=g6ElzZ65)8|dXul3r;*!RIB1aw?#}q9zwz9A_QW&B1m$6F6ta7T( zu(?*rC|BjzfHiulJe+%Kz|yRyjFS#Wu+nT5gwPz4dMI&>ogy9weY$~VcJ$cIzc589 zRB)p25HcR=-OBdf-G$(EwpUACO%vx&ZINcU*%os29?sLBw8GgWtgk!8G%%@w8T{RM< z6m-aiy$GUH^*S;nG`PKI-$2UA#pAj}A|Wh3>V^O&B?*g?*Cv|3w61{Y6ygW)MmMq>ME*wf6M|#_! zPH$aVhP0__Bt?2^na)RA^FwuJLPiD@BX`VnUdtR(9`hp8$#jLAICibsiii{nf14cX>>wQrg8g-EsN-E4e?|A{*#kP@4 z^XuW^IrsErPxJL@%YthsUxf3yeUk<224LReTfgmI4_|h2ZgTwEY+dP=@~&1Cu`pMkf5!mb@qJ*cmsI8_#EVYz*0e6(kcqd)SpgADCuov zIT1?2iG*>je#+p#ymuc%aUH;XY%35^#Nz9N3SO&=2 za%S0G^i8ZgE8D8$b&^dj?FkPlSMz=#MD?-r0qlP8Dfji0E)Dd`PSb~&fop59QW!;{ z_w?~izgu+&23b0GwkoP;pYUfv63;3!a5}Kw;bUYyI||sZ*lC-x_@_H6U*Sw2%teT1 zPjTR>f~{OQN`y4^;Z_VA3%`Tz@L272 zxOFefQxK3+;Jt7=D5kLk`(`kbz@qh_Tkvpa(@;su{^uC#t~5^9L>z2ZS7N0Ig=;b3CYsyzo^# zO)aR|WOUT3@gwI#I-l?Cpto)}Kgl_sNb~{ha z8XK8@k^`S*fSh_6Ph$$^S%zfC^%tIlxMf@)LK)qJ6nh7wzzIUBwUsq8N~JNDxu%p+ zO%<9$-J}q@lsBD8!+=(kUah(2Fitl|6bmlbn@F<8>qFxNO*?^Xs0s$CTdvwV)slgw zAn7}(iH>$LG~PvGK+a!Lp&Q80rp1#yB|W~QF7s%dANgmpX7!60)wvX8>xXg4eiwhZ z$)B=lcSFP3dxSe{fT|I-5j6XUaDJ(jp)lr8&ojZ^cV`o`Q7nNbP^xIJf_g-*$v7>Z zXiN0sQ1u23Udud=ZUWw(u*1->7l(;~lQ(qsT34vVrgJ;wo?sy1sX=ON;$-W%yN{92 zau7hs24L}yZnFKXg%a}JF3QVc=B>+Y1@bf+LfB_&c`EPk!_HB|{D-s7FS^gd63s}* zC0roIJE38xU+tq?$f9BKU&SrF3EeZs^feq()V080X5Gl5HPi<~{kiat<5&;{*LQ>M z{jxB*af_k8#at^i3@3aHtsS9RKT-=>BtZ>-b$Xe%MT;db=>FcuInKgCEyWNyoEjKC zM7Y;|R2>bPX-e<4+o;;`+`S}cY~yfCUg4YvPP>=Q=Xpeuh+gCfrJfWx%)h7jONX#g zdK0Q$wFLkf?{sSK|6zx};H%mC>v;*`Y{fgoax&OJXq{(^`;G z_Qbx7<*}=L%19Arr&->cr~38h2G(&_wVE4$<)SDh(U26(7@_ zW}la=iskIrR;pn@W6~q8KzYlS^eTP+c!D<5;W#ij81|g@Aa=y-Xo%Yp4#cHWsQt(m z7puTAAaUX3V^y`q51#HX#+Pp4+#4Qm=GKn(bh& zTNTbZZnK3w`WSh`ogZaJd&!MvaoC<8hnu*YnPh&iv&bZ4m|2*z3+^I)f?<)_Vs-pe zJC&BGUoU6v!mGFNh=g5=*2>6Ei+g9+xx*y@mrU6=KXWD+#FZGYKDW1K*0L)wV3Ha8 zU>s@$g+yNKvbw->)DO#9D=&UbaXJP%VYto!K2SW*rcwLLIg`9w>asd>xz>R6c5-r; z>To{(TZOnIK9O-;JcWP`=dgF$Ut(93`&i*|*bqgCWK3W4IJ8S1IXVLK~5<8#70kDphnX=*6@HBr8;XX*Bw3&Mk=P!8I-ZB)x38%08jz|-~xiipOZ+Gn+L;&=gBc}`^^ z+f`MyIu5^3NC30 zfB_}5+Qp$;IR&6qQ)b$Nv4DqQvN|6Nf!H&UL-h69nzvmY!zGKeYuBHy#}A0dTgkB* z7j~|Y%RKwbRsR{_INll_uM^L<;Lr0HS^Q9n92hVtw%FL548|{nyieHxq;sXVadN#Y zo3bt|Gf39<%;h;DByb85UT`{DFVrGNp@{u?D!8{ixs+J8;_n2N8MmJsEvO9Qv*j~PiP?UGo06{NP%>^qr;Xtm28XZ|zlf;^_MUk212ckIZ z3S5H*#9&jVJlXXCtmlc_O%QCqPWr;=RHR0d6}gVwJwEITbFVsefBNZ*l9=;^8b~Ur^m@V{s8H z=OySSv*SGQIa=~@1=ly5xSG3d?iW`aDED>1zE+cVFS565OPWVr;&Dz&7n7gL$fgjh zy;UQrd3~EdE2-EP6gIBzZcK(?G?>~a9;g7v7PRQ6fSj&J&=JpDM#mo-uUJ6Jdy1=yq83z%YD_J(%re6EAcuNkn0T9tFPmjm1|E!^MPo8m zOf0PaJi6H&q0wPdv3XuH#;Vtq_%)Bx*TcP-4S;*QGl9q9W!zMM`_$Vjzk~*a}zlWgrh!{@NrU z$u4V%NJqtK=Hww9DI}+mWP<`4;OxM4H;^oU za@s|gs3~e}QUj=>$^h9FF74i^Im;@k>1=ebPekK`- z;1geL9uo5FWB4yKic&QyWs!tDTLw81l_MJ2FaOIB+=! zBQJIq+9Jg%O;D!yHdWo}LzPUKXzR=?DQ{P6`);4YHEw-jZLbo<+vDWXVr8s9UG{qt zl#o*kQCkl&$={&zQ~NtS%Q~N-l=^0+hR11{g^*|V615s#qpH&&pJ0p#T`2!gvcq18cWOQB*J0q$}XajP*gZNmKLYWLqtu_*u}HO96Gx zlFCKI$m@8(fHinjfw;aOTwksZL&JD1(_S#jDE%&b+u(wJQ4%}{kP zOxt6UTr~~1iBRyWRwfM6GI1X)J#Lq}PCx&=xg6%&Q|+3{x#!}aRcnmC;xL;K1&e>L zErF5!b$^(l$shNxC|8KK8xm_!*+lWYPWbdc^3lb^Fk>`FhH<Jxm*-?-y`L7-#dpy8J5u!iQL-!h=SBVW3)T{T?U%L7B~?oYMex%;^LEVx^g zPmd%M$?Wx&jil6m==4A(&6D%bj9Q9p3d+}O+lHHI(m$APDIukAc%1b1 zC4Qfxz@dKXTgjCZ`y6)%wx=CqS}|~WNzaA_UCv> zME40axuM>Hiz(`!RCNR>z3?bBI|l}+cQLw6?P;sVq-C>{e`a6mUdCc9&1Tz+Y09~T zJ=i%#u|sg8`%6RXb4rq4363`jnf>I5O`)Ar$AX}-kA@IAGrp#Um+1728PmXo^8UiF z6C@9i+B0YZKAs6vs8T#;+IxWvddwU_oUHQ1`Jh%1H*l7Dyg8}9)zj-$m>i8y|4i4j z=x|vMrUxl!F`yS|J^IhwoI2stK@>#E&0>acP@?Bsu0yz&?S42N(dUOkTk8W|zV^^R zJI1=twO*Inw^}~N6<(k>Mf-%<4~o=Zedb@dU)ZKJJ?jzD_G3J^J!cC*b+HpW8QLaE zq~03v7X>Kwe}zAO0?h?-J1qAzz|NfH)x*CL0gcpB;@(2fD($AP*K|nWeQ1IHqou-4A<+U3?%8?hR+_hzzU!A9~*-&)w> zZEoW$CRX3Z{=-#kVT?c$d)+#0VF3$6PLNqD_o?8D2N98xL62mVnLT&z{wde4$x9(o z67cKvP3p$ZRpl$ooy>XN^c?%X5H;2Li=)~VPQ8Qy(C$MVyHZ~HM4YBm$!KRmBNL`U zTc;OU3rtk~xuHn}4;UC70YN%ymCawgk2-8YzD0ejWGJ0a+?1J|n6v3jil$}A#JL~Fq^FDK^$U*dc%)543db#8%Gk!iiRZxAhxH~Fb@lvL zt1c}o!&kF#7T96uc{P>fiJtUd2x^ef&l}bO+BKMiZCJg3{P1r${#f+o2(3Hm=1ZYX z3&#?KAq*~#|MEwONX;d&3t%7OUCLeGuzh;gMVn!d>M_-WWA$lDtf5ypRrXT2q+g8{8w%w_bMm!l-Z)F`Mm{JMdYN11t zk^sC=S-+g(^su6Ta+8V{QpFH60gHsrGFv@=Sm|Y**ju$V6qu?>+tXdA#^*G~NRRPY z@nvlopDIS4%S3tO=kfQ)qcpm6dJJGT70F|w%lsI^nYy6(_*YS}oC>Oh@$T#g$Zt_r zZAV%6heyxfdOI*bY=W8l2$@PMMpJ)%$JOxW`HoPa(DIn9FJ}TSgV>)~XkyZ<5}y+BD)I$OP-@afeZqAA@yH&u^^zk);<29+#*O&#&B#po=m;?7GH|k) z+2w1(_>ZvNMic9R=S4D)*uTdmbw-{#I4jA%%ssbZAE%?vmS6G{U)eJN7HvC6~QFm8e8WIpW%Nyi#%y z^jFLm!N%lF%{Bv+^J{jNk5c~`;(k#|&ru@ZwICN1E!#mHZZ%?`3+iut70Go7k1WYa zH6nQRv}0!EL47X3z^KLZHqb*eABId^ciYOwC|lPv+I>itam@|mD!_XAfXp*TJEVs6 zXDB*_1L=4`sH9xp$k9x)Z#(?KH$T^=YgaB9jt5JbU-dX?DLRj9oKOa^H|*C(uJzWg z*s)$ZrjA8lSMF-AJHX0oqgU)R(_JOVT2k@Miq+y2^j=0b_K)ljQ)sSeyhf3KmfD_s zEGcYCuxB^4;5Jdt%03X$Q3N&|XCIc-V|*f(R#ja~!L8{n|B)F(?Sxmpw>zJoXe>zO zXPVbJJ9s}rV8?Y9WXn*7gtm3J5HQGqTb01B-oR%IS03e0CvDh@DH!Md<8}?P?uGkx z^zW?XKBR2aM)(`tR=+! z%K6x<&6{R1Mcb2;^BB#hRnYK`doSJ3&!$>fUrh$53gSo|XC-b#4Ff7sTTu3rCD|&y z8=mxkfF#4rGp^QWKO48?`MRZ9F@mN@nP`!G9At7TN&vj2zA#*aBO_zgwqiqn^qMrJ z)Iqat#vT^?sr?z+?tlsefYZx`kFYd~uVGb~SxM4cBlJvDEB!dsEjp-?Y*kj@A4G%| z&tZ5FlXkXNOWPmwQlQlseIFv6KYaUN5klDaKEw#rTJP5YaybyRG>eK(o>Ce4fe zJd+bO@UJAd9pk2;W!u{zaU+&pNs{XQ{wgwO+d2~qvWVv9EODL%+SgqBN0T((o-p`u zLH1ia$E~9h)`O8MmL8dZ8{Pf=*!L?}YQ+cm=`H%CC!hG7AgkB>>qs#%)h*z*4>qn=A-X?|N>u|l|l zohcC8xYlhxJKYWtHL^yM4oZUDwuK7m{=af9N3o@EyWZRh`-4Mn&oKT*gqsS6yQ=^Q zq>Sm@?QE6ePTFkYLoF}TeFE4}1IU1gZ@1b=Ej3g)%Ykb ztRCI)?o5r)8fk&=$6}Y|dVnJHbp^V~EVw~GPEyuK@YrfF;k91V&P#9a#s){_Q3o?x zb!O^RI>Cai`bAEattbtqdY%#WmLv`S&B+@~M7=N<)82HrLcNk3Ggs~a*bcjQszmZR zS}-+Uy`7M`=*C(~o(GZLV~18E6krTx{@u^lE*upm5M>j0>(e}WB}J&2K3}Lw>qcGV zlyt*h;pwzPO9ATQ2;idU(ZFS%C{3gzv$QqLMw{Own) zB-*F>iWtW2zTTi<1s@t#F}5_$+%sgD8(>y}|x-3mYxm=A9+bsyzx5E(C67mG4xD=Ev; zFL!AIw_cD#f~#p*Y*jUoX;yei-M;?%*=XXnX0ZMva?sXke_gE8$q>-zaYZD1pl1!o z3yp4tgNJ@$m}$vmlXFcP1kIWjv?g56M@jRsJC&7@j6VuM&$Z^&U`ihS<;r|g#8Ktr zOJ+!N64t;_!Yl9TuHEaMSG7CWvy)eKt-4^9iH-?6y#g5!lsA8+;6B+1eDH^M43hZe z?=HgHW=~ri+3g3P|D8pQQ~j_)s6a+Ote?Cf&CXZfQK6d1d4$F;R4}l9&deowXmkCo zBQ|7vRm(!p=1h^Sc=+A3kOMKuo%`)OPWe z-ox*wYH}6&k_R%2Uv5Wb0aYu{p7z$-#G5$iu&K=kfLe5WWWZ!`K=3}nhI&Qu5U>Eu z!ZU&$eFFSyj1eK;ev9GJwZ8L_atOro=$oqKPR|!&0om2BpLcTmSY=n0>0RQrvsnj+ zXNb$flRu_CO3om!ZC)gPJ9@mZzmOq5HtIWBIpvEnO){=CZT|7WpILzBW0aw?WyLMdMvEvW7;F|p19h#g7#3xg~ z*;?o@gL+4ibh^^4+6!3)PB$IY`ZnVxSwfPD(nfoqjfmak=?nqXwtGpN=%0?I@%t(o z&5uVt_wQ1ei$O){yz}znaX4Gq>2VF3`nAVil-kg-f_GB~nSObq1*F|g>aq@cqBmT@ ztPw6GRjNi zZth3ci3PQfr#S@SQ(B3F$l?$fnYB2LeB=mo@sJ%`Gb<$;2sai8o)C&8PE8g1oVUuB z=i-a3$2%(iq}-(s95GhUd@9YgJdj*I6Op5rVgX#+;scctrjIF2ZADDRYgYJX78on# zOBLgQwFRC?gO0wB{>Ov;hpznn@PoX5qOnPbZ%=l7j>B%$$0dBqGUs2e;oEWBGKi-P zM_Q$yQtx0?t~}#s=ihOC3f{R2nb3mU!5oaUs9fb=m%s8_{%Q+lO}SEy1T%O9MmJI$ zu9)+>eE&8AyO&w^_Uej0>L+uB4V&!ac)a4YhA0^r-6!pWqtk-)eD45p6@!SX47HQy z*7%`zTS7)8jS>(@aP{Dw!~kO;fGy_aGJrE2`)~jI|3AHm;V91 zUZ#w3XuF6Oh$j`=##N1_S2XOXTuEMrHHGFAg0#b?3AQ#>FtLgYmRNGyxSpG3AAQZj zwjvD8vpPcADRrd=2E9frrZq&dw2M9W=gNSEcNApjVH2eh*S6wWdOF-=HqsB8>?*uQ98)>Zs`bx+jXtAG(?!vvW6?8Wp{IJ1|Krhr zv!6&SF}#O@MHvV86>ES+4bHROi}sVtYpKmdB+Gop2R?lY+DZa?AZMGm*db+!wWov} zVK2fWPQzZke~v6ICMKWCkIU`Z?B*ePq^)PIpuGvDBLU%O{fj0N)&NLv= z+#4qDS_MHX+-wO;z`MFy&AQLJ_iP2E@Eu~Hbx2f}+$67DW-$f}^R84VEFXiz5Zwlx zns>#I+*_%THCb0R_yOwQ$^LJg{g2m5a{cuV+hTKIW?d8IZpo4>5#)g!uC^(Ul|N^V zX-$X-bYjH;DTxvsQlf&agD7^aInQSh9;Kp+j8VXF7>eDNV?^QPBd9x)i()`~NAtsy zbC|=Zin^-i2U)mY4x6pNNwz?OfMlJNjKJP&=Bc2Tf#KU}ZRRl$Mtts>f#_}5O*7T&-g9+#j=%vc-5Tr)G+ku|0i$=zqliG?7RhmC{m}qRqWSML?;={|+LF{^#4oNSe!u~vm)B1(LjIqv# zb!G3*veTnNS?uwpN(4Xh1n!&nMzs{n%9{QMi~bI_{}g;Bj$aVI46YsaB2VZ_{LaC% z^1hQ8HP5cBuJ6hWz+O~78OHA+BClF^B@cE^5(}e8GM(FXB;a6U zlkJihd&&mx@yrv|Vr5f|o!7}o5@USLn$t=jJr~oMV5QWU;cy{AB)@FB!gOL>ME25pN=}ISS0)% zvJ{)K;W<|}w(T6($dOQjI9BUft<-{{FMa!!x;bT*G_y*-fY!R9b(W+nhY;z zP0bxBD7EW#|K=J>u2WjCmLFyaw_{_w zM5tYKpriFrf4J%<2C^*RM~_DCh>8}T#U+GOvyd}Ks&cB8n7QjdXQ|XK4^(c+35E6P z4i7ACZ9)dm1;Nj0TmV~%U2jf3VLp?FX^W5~{Qsp4{|P=*RFz*ufi=&gW|it#d6W+N zJU#@|$EM{B86Q4dQ1(^QpO8X&l24kpr#AvPUXq_gE=Hxv~n z>_3BTCD9$PRy>yx7@?rWS^gZkx|R(Un0@5*`xVobPAW+g2?8@u+stFDSFr@SIPv9Jj6RC5pH;T z#IT~iqh{oO8LrlV{6zmWuK&pMkE@1Cm`tVbi*cI;jnBpy0AA8;C9``);za{pyzq7+RD^@u7oY5oIG(?N8pVjIEAb2=XM`=EC2nFr$bmaet>#xW3#$c*_&%nL> z%?6Cgl)2dn+%yKhDIvRxFkny^htHip?Pg2wD#qJMf{!iKZ4>rJ*xmrxH#TNsx_Wz^ zB%G1*`$Wd(*&I_h9O^hpW9?-Qgeu2OZTE2ysF>(7B{Uezl(57QB?k^VX%vk;B4_mO z?~prciZ3Z|y?6830IEtT`onedfm&nd-(dHDJ*Qy|W?XUmE2_R~o)&6=^}rT%BcfwO zU0(E4!1vK2IQs{ZG54QY6P`~mYoyW7*gj=nB2Zy^*O};Vy3tTbWDM87%HFD3n%xy` zihGonyO!u=B%Xe*<3&t%nl9Pg#Rr6!plXoz9gsjrbK6Lf4QwA?T1WiLcSNe8W@3BC zJ{@7Y7^EYhgwiwgJ9Ru39zM^i$s?1T8>XC<M^dqH%Gs26hH zC-%;~;f9`z8X5-vQv9UcrT^+7cbmcI62aNIj@DcntH~>s25Jc=a8@G(MA6Td zn1-d|AlnejQOzQ8nvqRIBpz)S6X;3YdX5 zravM;RwB07f2H8zKGOz>YD||!`L}p@&#wyOWgN{#ia_U3z5OmOvo%U}p8I4E*J87J zkCG?-10_u1G2N$etT0V|Q?)BQTQZK24N_r%pl*>#+FQd8w2tLtih9;^M>)zmKAZ47 zL~+(~!_wF8Ef$2Oes`;s9;sUT_n-uC%$sbX+|%{%Tz(e+=FZxU@kQwNF6Nvc+mcsL zCy}zVol?mNoUI!dhJ#A-4H?MHl~Iatr&HX_96c}m4G6>x6EKoKTm=2FYEcHTBuPm< zP+4SSFq-IAB9^74{~ByaVsSi;UP4#b5`)BeRRwdW8Z@63*RW{WkQ?AF*KJF066 zEocar$Md<A(%es2lGi0a7A={gOeVK;@Ni)Hv*yI}2Lp+NK z28Iw>T-B9Q8iifdyXgAZlf{WV0Vh-x6N0|fp>yc)fUi(}EDU|Lt%T1W;@q(LHNZ{_l2}gmyo0?0yk;YX*`j> z&h7YDJ9$ri-z5%5`nY4@2xMbrN>O740GLh|rNTly3mchXwjD9Hfz%>~Y#1}9xLP&t zKU@-9fe+u^B?VK6-!QP`(F7H_-y+7ktVujhO!6mA#> z!`fx(MvJXF1qSP zQ(`zQ!MojMAo-mvQWK}B48vcWdY)&P^d6GG`F3J4RZ~{D%W7+@Z__Gl2$9kuyzKhd z^!6SKA)W|b**#-W_~M;m>&f6f)-zMgg15eWPOy8mKxv0f;1giMSR5&XUEuB6qbY-dk&2?R5((QY&Lc)h>oXPO=+$4J?EHi|VYjxAQ*crxAk>n+BR3AlLj ox*c4P2J`%zHX~an{oC)*8&3(heKAcH?;(GrB^1QVMGgJ`AKD^xt^fc4 literal 0 HcmV?d00001 diff --git a/docs/images/minimax_speedups.png b/docs/images/minimax_speedups.png new file mode 100644 index 0000000000000000000000000000000000000000..262139a1dd9d58154de7f4c330f09b9aea8bef18 GIT binary patch literal 43148 zcmdpfcOcdM`+h`2yNpssLpWA4GfxX4<8bUz2^lFB*`uCPvMP?Ui(`-M6)K?!IaXH6 zNLKc-`Q0y`p3n1CeE>%Ok*etqt%D9LW2VWe5JX3dr}a;I=> z)~t(KvxefuhIR0p>LXO*@E?lnIN1|xlB@Rh!v8pBtaZlZ?AbLK_<6$`3JqF{F z+C&$snc6(vaI~kfiEihH+IFAfKhB~%Fkye_{_D>cJ4-%0HcAjK30}I#xHsz)C;7w{ zPg+$^(onV2Vo_Vq)*n8)R0S=L$_9?J^Hm>i+I}m@*KVz@Qasaii*_dkCDn%QtXQ`- zzyIKFd`KnFtGS;0_dosf8+S!4wtSQOAKv}-gLK@n$Mo?yN5rBYCEru=hFzyIKFjA3QXRJ0dT`qO9v2iquq5 z1kRMwjVoU%Rn*IMD=j8`3L(zTt?p=dz3A^3`13bzZvKrcuch0oqjcN-bfRA}vaful z+xz5k)AW)N} zn>}ZbU#}Lecz-afh4=8;AbXWPq8ZjgVqp@uO;pindjHCOZD(9e zSbXB8hcCMcy1dCRN6Sx-8gGx~qD(ib^c-i7;|=ilKh;_49cfk@U6XaW<{%vjkf?!k{#G@QoJS%I`r<%tPgrg(W@vHJbx~xi*!PG;iOSu3j$V|@uxv^zpP6`m?zJ?#sViB#V7EE( zwxZ$DlyoKHX&W{H=Um&@)C_M!j$Nm3=^m$UneGW^i3xXc$h6re`o+!ZOIEYzCYo9X z+jf`P!H)R)2ctU#4wpl{sbezvxGY+AjRL7zsBh^>7Tm6X+taZJg$((ZDG zm#^I7^OL!=9}>z8@o*KV{duP>;}YUNNvlJyM`?aR?H*!ra@Q%N}|I_qs7} zd56(HaNC0`8C7IlkFp5up(E2{S327@xAA-rx)9%LRa<|dqWxw{?MX%{Cp)R#0hA{m zvn%_oXSQQqw|Vz=uH>AK3uO&4%Hg8qPez>$x>qXunC-hBK5W*zoLL|LlKfa%>V5JV zIuf6u+ST~a>mzrT&Q0@Ae0Ho=El_0JvR9^pnj$DKh&1qaoLZYVsc@z*b@H>LY~Gdn zc)Zj6L?gd>U98>QNZG+kUPZ%~F?^l!3JRTdYRZ-RS?`>K%-HC7hrb7&>`3YCOKFSK zw|!7JD9pPrnV_EvkByfUIj;LL_j*t1WVD{>o1g9LrKD2|XUylmi4?teX?=)cwy9ICiInx#g7r%%P};wI zqcRkW-I`w2@%%ebGIBq_@I=zlmflXjxYZ>1avIw>?i_{~e5LSMe#-msafR`sxo$pf z6|Q>a**5jeuJ(=WA=`(F7F`+!e&i2@IrAGnE>rP%rOf7MFl)vabYnL0t^!9iwq~33 zoSzN%QHP zmda^i5BOfHd44)-)}O@iHKl;8W2^UATGEmHVaPC`FUDpbGD;^+;Afjgee21#o2K?= z2WQ6FT$)1+4yXiEkhiq`#e$LLmOfs^;Rm+MrH5QI>#K{)EjAyj#RU{imGE6yoS*d7 zw{0Sl^13H|clyQq%N=%CWSx(=@PH@lodvg5T#?i1^;G81+^YTi!u<^j9J;yeQhGS| zA!LM=+_l{=-c&i~>&~Ts%Hus^)tX}mg)8lR6!CqYO+SHYD6`R^=B54Ez|=!o=J&!6 zlY6yl7lh0j)MhRmAwuM`FHDt5n;eg${a(&vW?OZv=dF+Uk0X!XU5_9>b}k^TtF^m(|ssn6Smc|Tf0HniT@=3&Xn zO|8XS#kr-EGg!(f*6;HRtqv06A{}+S6IHKiC}~wN(J;q`uAuXKq_0nQyNh^53smZZ z)KO@&`h7lG86;G3*Pk%dUl*qq^{VdO!-Ii#PkQH%IZX`@%m|8Oc%D5a)ahB5&_$l~ z*jf{hPtdwvDP#l%`vG6!)P?&jWp}7NkF!=P&cPjyp5$D-<04f{;VyRIivV$kJ$~+} z^XLA0T~0Q%c{>0dVm|jYyjudH*qV(|d+2kqYC-ktE>8;uKOOWDJs|x}*D99{H-7>L z3)K1UpzjCXGslx?=j7vT`Wr@Gt=)FYLGeD8e`C}a6$j;R@~L2yw``luYEq5LzLSUw z zc8)y#+qUx_Oqi~if(VjHf7!n4icxI**EtK;lLqhZwR%UXp`UN-Z8+bjY5m20eQ92Q za&Q7vu<7|oNL2I{d__)CWr~J{#7Wh@*+^fhph_S49vOqjivH#bX$uNf=$ zxbr%M@4EZU@Y6Kz_WrS+>gpkJorU-Ei!8$sfv0CEWM)1nqz_-0Zus7mZo05ur8~u1 zaT!V8hIn?iBX~btj_3QYpe{URdH>aeB}|&v)o-+Rv#-BZUT^fRwK!Dj2=rX-CNl?4 z9#GMxB3VXec&OQMR+`6iU(RYW(Y|`=YntK0XrT6RFTCH!ai~BfxJl5FJm8cm_}B=L zWYgfAO?+PCQu*T~1*3YrW@Y1hCjZ4-lGNU{VOREAzBryNnlC-{a;)R+E@iJ4t9%~u z#kL_A1^|W@#6f({$Bphp4nGsB&(>v3^;x2T8|3?=zL?uk#Eom?;4sS8Y->=lq(3)G>^LmsDyM>XsqnX|{81PoF!8ZMnOA<0;r?(}O%C z0k|1s2fSe`k#FMK;zQ}HUtZA&NSQysUd|cwHh-v4_nG?j77jkdK=Y&9gTW$JJI-V~4?nUI`k{xv#e7l(s;Rj9b>P$umL!Uo#MkUTxBO=hxtN6J{j=cf)1 zrf@8m;2F%hyXwx)-@JpDF20W6-GkYIqo`V@x&0ed+2m_fgDu&<;|b)9E?khi^}=+I zrgpZ|>rP*3X0!K&{v@dl0DcX~cO07AJ0osy^%VJRcaTxUN^fCj8I{X5_n0t;c+z|g zO0hCQ_WrmaqI%QaUZJ-2JUr=*`8S6K2+axfd#${fjIa`u22swj)i&ZLi0B#cu# z=JD_n`Fco>>B`|!+V!IX3G_FUu2CwSJO{NQ_C{P%oh{sRTD70y;!NX1ldbeY@iR52 zDhNC%yQMbhSQt;nZIjZZsmHsT(bRcV`x{RiC@*R#X5UeBFpP2a{r0)PSHOM{3PeuO z-^gr;iYltFP0i}nQcD_PMdNR%!8xcp)(%iih*55aa(*Da+UM?;ghkWpC;Ud3#GY`n z`X`6}q=SROkIA`b3<=@ZrF-gmYt$pR#+~{iOQTMwUS7Fbu`;I7ae9np+E3d}xH3#W z8$n_p(T;mh1Wc8LgOBjAJR!{KKJqs)>?RF=ExzvYW8>UKN_>Ixwc`!rE(!5UHlcGZ z?F$r=-ILy4T4ZC^CNTNA>C!;}m;6rUbt6JM-PaiLq!;&hd)(7=`Q@~G=& z9US+d$H5&O>)Zv?IS1XZ_r=AOLh{k;+)tb={p>gtnj&pC{AJGi*?Nn_R2diB`mXnn zHx=kPBcu?^Mk%Fc{(jf9y~_(p*t9{eew1Bqa=BXQ(<9`E@_Vr4!-mQf3=}Rd!;B73 zu5Z5A*8>pM{BcA{tT&4Ls{bw7`ZgueuYb)6iL^UiU&u+jon9rzg>ZQyhx73;6MQ!* zwTwAiD|_J2r;SAG=Ia<%n+2#in38=H$U3R zIe0u^lUxdXSa6KLXjnEi(%Bgs>7z4{n2>nfbwk|+_btMo zSljbm=Q>%MNd9I+u|ZML=zmKo(FOC4<{4(=++N#};=@(vfH5Qs8wQ+s}L`N8B+HRxThw8S2i@ru1>QUiAs z%yOf7puovt^gX-Lo4+<{YgC6xM$R!XclYx+wE`3H=|=6@^Z~)Ok!jjELzijeeGcEB zP48huo#_DNr%fVsC<)PPRnTqTCwj!{ZU>~>ayomMxAzZi#)1Q zo zT;Qk~TMr%Y-1HF8By*oFr#~#-yb7IGi2`4Nf9S;bnuCZFxRW5tKAgUBi8yfNoL4!1 zK!t_6ufo)G(!truX0HIc%efFrwoNYldh@&;(xyWa!b^9qB|~XgD6oIQdE%pSox|Xj zthVTc5<=0?!s8n^zJGl7&_uX0Tw2=kB>-n*!*emjDQMqv5QN6ux%Yi|BEnsJ->5i==oZID=tV$E}s61%3uXNqg=?Qmrky;aAVzYVA z(L-Xk16r3Ko0XOw4z}F6gpK$r-#9=xMxJ?mxZXf>{LZFZj*qLvQ@(p>*RN5ar?S=I zwUewb@n2A5t=#!?abb3>P1!vj;$Ir)$5C)oEkcwhDUk97YK9i({RxwfK)8$v?6|%q zYVvxJ*%SaO6K%7&4=6zaP>e}Tf^Jm6gGln!9XMb<8>J$@M$Y8iI-OV{~O7?Z6x#{$yd+PjaQw8UtYLHT<=S zr(ivof?0d4Izg8{-6@pzBkV_yv^8}Ij}EpI2%QSP^|7jv0|k#0!YQkKCCOUE!LGg> z#Ok`|p-p8aq3rPYrc!bzg^#_KTn;T(6=~=)ofc<{7WXFUWa?ae#g&wSqkl0oS+p2R z?N_cj#a}Ve-OO2J+1{U0#B5gD@OnKv+gw zbQ|h4+-VQVSM)$5quXQ|SFfR)RGyp+IoKK+uZ!ta(2wHcJSwWU{(h?$2rc^$tTR zg!&$Y_?VdQCqoL4FsD!;&)-lW2e{Ic1|FzJ$`~7(0J4xI1vZ$Sc>%m*vXfAm&DmFv zoNa917nU2Cb6y<>z=Tq>5h?17_I6I5fLwqTr)lKqf=IJ2OGA8^h1$DqzmNFU-rmXlPuWUF~14-t~7i=3-^VyN1tVWTi|#JQ+~9l`%4>Z5oO< zmZU~Svx0K$+XvrcwLNv+`{!Z{MK0+vdK0dUM>b#OcORu3i#f~Xsx#!Hz%X#~@D43T z%G#bP&!=O3Lp`-LTY^v3u9FHUlv>2`X7k%O3~=ZU7tI9e@O}zTiz?juIcHf(aDPt! z!b4qEx?r6w3T6C8!At6lTQtmY_*5rKDjM4Dew^DCZ7A^@qTCeo&QSHGV=E4O_m2&G zhb|S5?W`&plw?0lb4Y=ljrW6$`Nh8gu5xkH=(W z2yQx>l>05H2HLA9QqB&FL4ut5&ePq2su~*fC+}{4Zxv|ZV7Bl22YzdW-Vwe2jEQ}z ze*XFEN;qG9a;tNmY-3;#v%BB~`S2;0=Fr4oi-@jgidu}4_PIESzJNZ(Fo|Qp8fDr6 zMcu)&I3y)ir~lK_rO5lshZ;vVD# zsjO!61h%a*#Ih!!bJC0#WxN^b=4U6XO%y}zzdgm<9GCj@4K4NMJ2t5A@oR>segV>) zHY;XuOQxV2fGt3Y$@hS67o8dJO&1JqpkL)Pcy`5JTN?vRfa}wnna~wj*Qv_#jFpY7 zS>@?ThhqZ-qFf4j?Jq8oUVjS9HSQO7YGEki_=@I^B5oFyp9_$8o>n7z21FUI!(2xD z|MHiFg&k&Q0hwnjp-O?0t0K>y?)z|RYcHDIk2W)Uz^ zoCqinh-O)z33RZY&&gkJ7dyC%LzGqM=8smy_C-BP}%)jnx zZXYG}`Y2S*fu$C?aEDIgUZbI_q;T`4$i#NjP#cXq3oIZP)Bv3NWE`k}0q_kHWXgbrK@{4wDze}Jc4M1-5VnVFeo z&eyw5!ycRW;11qJs0kr$EnvrxK9*b~gjuRkKbPa@S1(@!tuOXCMV#P(x5#JFz4K78 z=pF=*_?Oo*<2Y>wn}zZxJ^uUpe`oLrmyNXh{AP+-7Pce)^!4raWsvgdAjA{YE$d=c zwWmRRU;$)fO*+Gl*HJ$F=S%pTse_M#Xz=cUZFW6+6ttZs)M`9;%{@+_T0+2qJg`SU z@5X)gFcK7Qmi&P<1=|4x7FJjw?EKD4IfsfRL9a8)?It{c$|4TxCN1is3ZJ^KUH{Fu zRxw^s08{^Xg-<{?-WT&~Cv_BQrxWB*X4}G}p<_n8 z^fBUirf+zk+$!HSp&aqWP2H-g0L2s{uNtKJ4<7#ChWzzwIpiWT{F;F+^@Xy ztUG$~%L359_VNGThLxn_xO8`ER{|6snH#Gv1DUsv^Xd7Y&J{xFD~5KbU3}jAw7*hB zR|W-0yLQb_MnX-W9;@QfEhRIFbyYL{>xP%F&OCByr@HCiY!lifCkX;lYLiKDeJOwM zuD^FdL>0|T0V}-_ax<2q>UdyudyDYBdIqOGj|k%l7zD+DL^D;#bBF)TE9cy?)wY9adpG~Ol|ep$ zoFIQ2G>jG_C`7%pVMbUdit?V38`3uE}6>R(|Acq5EE@hol8-$+Z za%u&cxJP|OyaU41{8C|rSW+~D2lxOg0Xwn+%*2?y{q?^8wWdP!u@1E{@04S^v`c1cDmo9#P!ZF z-E8HzI4q)u5g%D4Kf7V+&?+pE8_)|DPJeqMVSkOTvV4_zXB}SL1gqW$s*9fUL_K0! z-bM#oF#;&FQ+#q{|F`P!>LrnDh1+^{fN~4Ndu_#1WNCn&&NeZp!xO3ZQ$Run=sz?$ zD`ZAG6fB9?-Ua!>`X#y9B7pAwGTr{i<>k2G4zdC;=O&who8R#$EiLT{$ZrRxbG@|F zO}aq={o^)VTDrP}=)H*fv`j6br{0wbgvS>Py_I)6tI8xbfrAx4>2aCrk5{w&UD5&A zUgZfa^D1i};R1fO`s$vx*p-c5(oeAg3gj(&g}G-Y`U7n)hC7q*FU-%HmI1eJjz6Q8^%a@>1Rr)$YcjF6Io33gLhmU}Sf;8~HzlG%gV)GsqT*KuTuh;Q| zB69k!11Ji7kJxXo{0=uP9=Z!U?#emf>X6NeVDh;6C${`8|3MRd5QXzr>A7LZ1}CEm zqRbyLJCE0B!7Uqsik@=DW>vJL2sx_muZ_DdKy%M;(U8cfiiAc8Xcj}~3A}e-!fG9+ z9wq+A{(!sFDX#Mt4~b;3Ry^J)0Wv?@AnI|bn?!aPpd`2wYN7UF`Y|hq<#rAOck?Bw z%oEyzWKzvyud-{rU{U-B$ee}o2Uf1M6qKjARR$k9ix0dwi3}Ojs>N4;;Y2*Gb(3-6 ze5|eBvSjzi-Kio%9UlreaRU~s?5s$eKT8|jPQM+w$+QzPkPpIu;YGv80Z|HGZWbgJ zthOMo)bxA(%%S2YzIa1p|17gXhb!+!@6Lq8H2FJoPSopHi8ZS7joiYl_+%?vyOzt@ zJR#jH=W)o#0hA5336Eqb{*`Qh`RrGu?O3J1V{9DkbqK>~+0^4B(KbxbOvwS>ir#TPQ9Hc@RA^rxCh=1EFH2>x zh;Oofekp>%(2Z*iOkh^LCn`_Xi1YkgF(T2uwK2$dupRom2cT551?qRt7jKi^2XfW9 z572)k)Pl?~MQ*ub1kbc-5dP$ad`0r|^7Lk%vqMF`p|;JOpMmg5-N6=^7^*LT!y~g*ElAY+r< zYa*C_-hwxW0TCL_OFGXU&F;O-VmB^DHQ_n8>^{Pp&;kwQC7=!mfL%c*0a>xdZ^2hO z!1>1GYIdg++Kgv!?9VC@x`Ki-61J`wyywg|0u|x_P4V-y!NLjvfI-ntlG1Fd%CYWD zn-3!?#?a20feX$uyUiBgP)&#WeP-9Wk%4hAa`@0`>hWv(7ZrWEwFx!V?{2Abe!)ut z%PP;g#-{6`EP=1wrxBX6Iqn1?$i#2{IADWR0?VPeE`wS@Xi-0zCnR+s!(%U7-^y(w z=)IV=IEe5283S#kJmQ*~G^K4)Q|Gc8)f_@H!qtLuGb)8!xXI@Y!HNJ7sG^VQ+PnP$ z4a%=$5g6pk2Ze#F?uVj4Ak0aD+WCqhJ zc$;>sxp5B$vaNhofQXCZ#RABZA+sq;*vFRL<9HaPjCzM`o+e&zV2Hlx$c{UX@k0}i~RkF&M z9o1ulDRCeVInNDVajm$8tCkp3L}-GpuS=gh}h9WiHgs;~RCyePr zPC$Cy^cs!nGJsO565}6Fro#Ty-0KepK9IkT-QBX6o^$sEI43U78U$Ra;9oJ|B7`@p z19^S9@8a3nM`SOI;O>CtFvwcu?Y%R7OVmS2uEn*Ba@I1>k-( z-_Wb=zup*aL9fKgF#(3(vNzz>@{)IyfwTZO)A4}s?zy@*4pl25DEDR#mF`E*sG|=7 z&QT$$8cfGtARt|f%gh$sTV5G&-jJxBSxF!)7|!tyQvdrqF$%d6bx3zmmKZse6{Aig z1xP{`?uVzAeK-DTh{Uhm$j?j`Axr}sb_PNv?c6(3-wlvZ12xnX3PNoMn@hxpUhWbw ztxC)q9PLJmiiU`OutImmb^Gq2>)p8)6@ffXA-8b85~FhCz!-yF@Cnb?K=K4U#EhE( zD_2|aO5@i>;XYnOjtuioE!ZxBI=Tw^&-7HxfGMpP)-QBIR_MNO$!FQSE4VB;G{f?< z(!{#QNY~sVWxUHx&jK9{92q-n0E6|4@VAZ{?Q#hJ2bzY?nY>$8$u2+hfB2$;Yt~psg|1M#;IV4l_Kpg(=gL7@7z{Jt~a!0A4oaRTvV7G zL}|Ko@0G58@s;X7)rL9~cZ{hV&!CbrbUGx<=gm*Rd7C}Q9g`tjD+SJQIT5uZoYZ%5 zKz%2r&9;UkIu;~o`fABZJzk54vnZWu8=N_FCNn)4vq`3#48bfv-{?B8DFN-ef^l>m z)YuDvIv6esz7^*-0?uYYc$&|I^sW z44Q2+WHU-YN^p^*HCDKl4gK4P~>c21ap{3G`)79W%mBjeFf{SxWx3K5@ zR1Y-!0WRB^}H#5ZVUjuxAg=PNcYoI!>MbQ9C9A6&(5d(JH&=9a13|Z?lQZ zKi%F4-CBv>Vj;&&Pt%(J?C1zlY7)H+*hF-nzvfZJnu9A7xsl4YQ9He^g?h}^_eI&J z20R}jz{C}5&kklAO2CAIeI0G=O!(q-cp?|4Ry{~gG;Q*;;)CF`(stw34>mc1iimb7 z^q|6Iu(Zf2kiqbaJn%G)-<0SXyvpl!gO0}u`dB3A9&Ws`umJAl`a zDqQA~RT!wh@oAadwACN5dq_SuU!0lhYDn9gZlU#`#{<4^&Z^X?IOs6obDmP;X|xv( zQ(xQ)7s@u+seG*sspv7GwmN#CC^Wg8HKVTuUo^d;)5vMBy&}B*Cj)u)4^;ZcbAw;j zcm%}Ow_xxzYSh2!4X95hSom^BQugO^6Az*VZkRn&TU%`4MKRJ&kkQ-H>{aVg=YaZ3 zY9ea&6H{O%1Sf(9Mvd`U3 zRd=LHbE~_>`s~*nRLAFj>!dcCA<0VlxWsi*$t#Q!iLQy7UV!JibU0G z0Svagkt$)U7rdx4<}0RU!r?BmI8r;A+Xyg=ryof(t3ph5rAHt6vz8*1M;;_7o! z!=GIU+8;o!&jSG2nV4QRv&Z4f{-qFqMcM`H%h}ASX^=b4SV!{RsdfIf@68|$exo^A zp@9p(6qyK(O|&mB8a8j@BubJ$9h80{D9+9!GzmDj6!!a0)YWScZZVq4;C0eqn^!%6 zdL!U^_OkRPkY1x&9z|t*(5GoV^PkB4@7nC3=4lEfdj|fq0eLgzxpe+f z4ajq^+53DX%LI0>yt3E`=8eEw$-5IXsPyIAxwRIeo#HX9ndO*-w}97~%uMPb`&1Zq zJo#BM|6vIJ`5}UN1NsPNRL)9Ap&h_~c@J>`*ctyW9Gt>l`;Hy`RvFA-mcqH3cCsWt zAQkcGGZ-&9hNzF18o&*vb%LnNiPTr81?2USQ%JS$I1K8~-3d=M1iqCQgKecyQfj~BW86`*(n zeajYe&{(kb!sAx-~MEoUfBUg9A9KRGpt^vlg7^P zxReuhk_RYBNw}Pc6>4kxN*D~ifS-}k*47rZXs~=JwwyZt{B-+#v9AT9V7nyS{mAV} zD|jAnWZs++cTU{~TfTDVJ#h+AGf;(gB2PymNXypm_Itbq12m(6$%j~Wz136;m00X+ zilr>$%Wqunk8+U_xb?g>l5~t6&$NLW&TrlE_QY(&s)95kJ@XLJ3@V>>?OBo2L!S%M zji*U2s%UNZ0XRe!f=>`^OBywxi|6-xH+%ni6_)DtVMJEpB9Vb|AM|U86hrhYy_b9s zGTu&)^^^fr-H8;F8$Vi)fh&t1lxC~VOkx>_=T%pu7T-}ZE_@n+hrx|nX)xN#yT=IX zV%ku5z8j{qY!!N7*5`+>vuNXL_AI=y8R-^=ZQ6kIVc0tf^ck&x^F>wlu(&|j;pi^~RBYUI_3dZos@VFGw! zz6148#y&yAo!$aNivE>PbfO-uOeL&wcR>5J4~+SPs!xe@LM}=Fj77wH$z7jSR3!<UJCP; zw9t84-Wd4ApU(s$ytV;ml_m>BO0)y^sllLH$BW{v50C>Hp=KD(G}sJWMT=}afG;S; zKqN~B3k8@cBfrW`ux_qcDcl_ny?9FzH!1_>)C|jZ&X|y$fUzx|0DO&r@8m7O#&A4X zA=14J?T?hL_Awu?nt9uh2bxBpMJNR_XBV3AKn6n`D<)7w>-?S8XRBYwa3m!z+-1g$ z-bwy=EQ5jFeuzF*cPsE&dKXN77Ki?47jz`}J4aV;DJUo>PNEEh$lyJ8JeF5KPo3lDDx#yxB1*XWP-6H2Jin|%L>$0sNBX~zRFYv{k1_oAPksBt z59kSK$x$DN?D=RodhVpZ zzyDi6%S_;Dv$X{5FzvJ&NnXOGav6&e82~#aA2V2~cMF-T3yb%G+~9=_;z1M>K-rsQ zqXF&Qz6Y(@5P=SpKSFt`8zG^cE?;iI{)|UxQ?TA(FZdcGNhDo!Mk6OtYnJu~u(J+` zhC2+NJ~R>jkEw6nUd!#MakPXK)npm3C;sC<9o&E89WVe`O73J_vx34+7Qd%@K+#Z#D|bZfZt?blm$9uaD&$qa}3e# zH6Uo%P1I|4BB#%XV38E&u{~ldKmAu7R5YJYB8XxQ(_kgYqGTFoZ=wezSn9$|zidxc zXe8*m4^33@1Y8sN4|KI3uE^e@(PVmXgW1gV#*_gx9zBG#zbIX+D{)S*PppRtU0Oe# zcW*gYyqpLr1YgX7fy%mQ_)HXhPn_dBCsqkMPb{iDz+D+o6$?|XvO?kV5#uoqtiG$o zxHL6E2Wt)`Zv=+eN?<4_GBGjH#hg)S#p`tgL0%feIsx?xAjQBA1U->xfV4VretTwA z*8$|B1#uzxOWlF4Rj~a8cswMdb}9#LsZTL1LasK-q<%>SKC{t0=NR!VAR?Yyxnae;?RQ=4G_(qQzGrz61oMWIDXC(e`Xo0b3Pzx-_% zZ;y3FkP{es0%xex=zAejdA)+;8Bp<3WEx|CC0zcM4pb!2CPVr(QYVCGtlIMG<<5h3 zx7Z=Im9L&frLL?GXfw44>WPuXmO^enQ$&{#n(9pN3XVi~e%?>lN z)T$SwsjT!Gklu@haPR6{&fVVf9I&@go($V6A>FY9wpu0}VqqM)SR1}Ti;7i30NVSJ zu8RxWp=i*yIge(w-J=$9{#miuwR`UH#o(#YEH&t!?6Jg^-uFM!adP%+GOb0|u6BKz5ntG5ca}iI3(-F!69VE|DoLITA72alohbNbJAj(F55Odm zB(_>@@-&S2vIlqU5rj!8HG`gCTOrT_Q*bw^rKkbfN()@F&(UZ%=tX)qs}sg?E@a-rZIkbdg0IMgEkfWT7k?2BP%t~o$dzp z(_0H2aR3tK<&djyHVVb3tQI-e$i9WSF=b|=E)Owq`B`B6ueQX0M>`c|J#W)Z=vhiZ zndRDi9Sy)tpJvdfg)|kfqOlMKbh&Qy5b(PdY0&BD6nH}wO^^`EZm6RDYEfZ7N@gHJU zCGORQwYW`&nYBvdr-LT^tyWp}HSQ&7*^y&=^l~(DmbB9a-1;t3d`wQ@ryc?7wFHQ< zr8&O;qhnHtxN+t#D8{YGCXJfhDiHcl2iZXjh^84mp_ehKn1?iW<^Q0BJP;*RUS56y z)~Sm8Q2rDP&>;tK2w*;X^hnM<7RKcSr69AQ(bYmZ$vvw8s}eqy;mvVyD26DDS2GN_ zbt1|3R4*D4fkR3#N|FZdfx8|7 zUA+_lfDwxED4(AS5_S2vfYaJcdtQCqUwokQTC$mZc&A$7bXIh|E7@UF)(fvkBx=_~ zzol$<9X15dDy`%xh;R_iTd>S+2OVSj!urSLN1#s9`tS78;SQuhPex@u^}ch@@D$+ewAq>7O3n{mfr;l-^Wa{UC>iV?BwTjoy^7pTjM~-)5zetN^x@BB= zhIAd!d^%?y)1@baIg~pRoEUVQLjJDu(z&mKDjR(SZ1ZAEeOofl*D^NZmk%NMSLOhZ z5FS-nMCWj`PJnDxf@a|dx79<|I^7&Dd|5{i6mv9^tzZF^nr`-$# zZOw(@30#FGP?@7t0L*&+4qXR0JMQmN65Wr8@TGIPz*0Q|y4fj^E}ny0Uqe&~{N9*i zXqoo@a7oIgzgF32cp7e1ET*v;wik0RK>_)nfh2dAH@WEgR^nT=pWJ7G7R(7J9nQj0Oo_C4nSS^(dI|>3vS38%%Bo@niIs9 zfu?jLIDJN28`1xg#L=_kgMQ ztkmu$wi1Rojpn9CMu34WK{Ia<5ZX@Qq#FTIwA7zd?I;n6Gh3ymzI1FCpjmm@+;5@w zrVsFZCG$Ce;T5-(V1&a}4~#V0pW#awkT(X-Oyg?8xv+;^JwaFmP#^rH)H{OVzX zX1F`V%@58imi~iTfBS(00Kpy{jF!kFf?4WaU8Y5&7CJKoa)=MoN6~R$XiNi4S;(x8 zOibR$I+KcyjA||l&scz#?E+FGfwkr%TdRHK@wWczPdF+8wbW?%%WMA2%^4riXtXD& z?Qel{&%SEYvtk&yKWE_JU=;%200R4W9}#*gh_p|%;YK?_!1n@>R|3LdIF}~~(88PK z+JM&1lwzF#RrF$d{%B~*w}`|<%ZF}tKkaK$5%o}#xv?TpQsuHUyuJblRPfHGUc&3Z zitCOIMb}UdEudv-HRREQ)Vk#{dRTLCmeNt;5U-04m(q@icbadun*-X$xJ|xOuhla8 zj;ZW_Tx+dhBB=qzbvKM$q2qU&EK`Bu6DFnIMdM-?ox>9=OIR-ymvSLnBmq6}Sw!KogKh4ogDn*@ zhDqbi?;rQ0v+^wMi!&NE%qM|6_z=i2&Wc%?SL=(wnmuz?qKNaS-}a^OhduvH^|}-|W(S zg^TBb)}AJWKEe0D@GD zI45-^I?FK8$y=yF>*CX*a7%w7yNC>SbseiCMDVp{tkaw`4A3;NXCqB;A|u+OrpgZv z>dtG97HyDbyX6-ai-E>O)7t6Ai!>YMVSwav4QvWf(oZu6tlSR9Qlq4e<}F2gqaLRv zqjm_Dm4O_@PQcKWX&FjqS%QW^X5)|;jP1)mG{GEy{Xw&DXwVK~k$EK*Jn$p{c3k);#yKi;=-oVx?adGgLIa6S!w z8`nS66TcOh;dCT0$QKlRfBv8Y6njr(AOIWA=-Qltw2BUfqN&ri1}=nZAQJZ>dt+mQ z_xjxK8B&-g=c`(v4ljX0R5-yb)TZG+Ea5S9J_i8b(gSU%dn5$jx>iqw(O%OM)KG(Z zk$j>a*@hQJd|lrH$6y#4Q3p^Zh&qz=A(56=(R~#G7e-JRHR|I)@sObjNrd9xw4mJG zFJ#p!ss!{jn%iIiPjwXo@mx7NMDjj+*gYVN`AK1NW}HRsV1Ir{DVce!n-Yi4GUCf` zgvLnD=R#OQ1a%kMk|E|F(HODT(jvonyzcgwHV#AX)EcGi?ZZ| zIy%NtTLlvU{t(j6``v0ZGQ>TmoV#s$iGih{eBf5=G$J&>zscqRKpr(`sTI8W>feWD zS)6InIS2GnXO2hXHT5%3*wb_%7tTY%U_O|7)4F@i9gkqPurm0`m+4+9u%WjLI%+IW ze{W{IR=Klip}z>HicWe$u2(tt1icxkeQCudm45QbJV!&u6JTo5gPS80A`vg9X8!eD z9Wd)U+5)STQ8-%$Oo&!zas^MuI?2z<619~9hg(k%h`GLmEp|G^prFHT?bdt1P7CWg z9I2c_b14}Y+n2;BIL1o2iqNYs+HDV?rfHX0vuUKjhg&zH}K1c?}=8vty=7{ZPJFL zWe0W*1=aRH|I6KP=kPI)I^fvOW8#Zz&?y`y-wXfxiwZK1Z<7?rj@p^#0iC7a=j3&i zRF%3b0s}!`TS6_B)n{`G4O{T45(N4$ko9S2zaNMv}1wNz0zTsXy> zC^*HnOs`bWSPeDWXJOQ1{cN%uCqSM$BQNi1(GIczrnk2j_tMU~QJ8*(`(9cQcNO~K zV;94#m?l7KQvjwW%3-3NtqU!wRgcU`Z}>+m_4QB0fUE5i~dB-jK!DB zfR#r9VdP|)>VJGy4@>>w8(2{CZ4=JixRG)3;>CA!(_{KAZ44QjwYR7@eTas%nm06> zH^A)W0LKY*gA_UTM9nk8UsG84z)BYefw_TZYF`?kg?^^2Z=vAu^SF!eAK@U*5}Q1` z9Oh>xP8)))DiDUPw8HEw`R8#0#?teh}Kt!EZX8fy6p`^$WHs`%3sm z-m@X1s^<0a#?8?pN`i7i=igAJyMTpK4*DpG=6UGlgWc*Lu_;8skaZ|tO{!%Xg`)T#iw2NkYf8D~qqKJc0*E=T)ZX0+|tX=mGPATjL zfJ{Gl406Lkv!*m+FCF2@lkX1A4uE)%BEwkEm6og|u#xK-fQax^_;QeXqLylNrUg6N zbH{llu|1(s&1q|?;lPG#;5(_Y&&nvn ztE>BeKjOTJfmhHSbES+|H%k#pj#SP0@EcVej!&en*xLj%`D5_${(B0qz;YC(r7bP) zN-2m#-QCSmj?Vn@rY~LKmfGozb~mV_hainSF9*+J*$4}1~_a4C*?u|(-OO! zyM^&72KwW4a-=X-_h@k1;Kom%Sm}CEDnEgD+dq3^g8gK{su}LcjtCF+Cv&W3>we2X zb__5_>oy0y%jcU8O=2)z;U+6Ng4-PYzYlDs+8~#E{>8oDGS^r3Bd!Z`Zj?AUPlc70 z&Si1Sr5%5N0d3kEintRC6jvR7-F2QpuhRR4-zBzhrGIM9BIPO-k@q>T%k+MWnqadg zXVzaIzB_4S2M-GTyi%3jl=d+6D`Ah*NHMrL?gb*p>{(!dSHD@jMx4950yg2 z-hZ!&JbSg=;_#nR;xzZibDtKw#`bV9|5Hp^+1+D9j@q>pU3|Z@c2l|JZaO+THdG=n zlz2WpiBnTEhG{oa^x3_rj=ZD;LG6D_`}_G^h`U)^FYC$0%g}4&dx5 zk3Iz@B?6oU{T_^*7lFhbg%6bPE)QVi;W-NjTC?^yrueQ=3i|y){qMFnu=?Ryzqt9Y z3(A1D4i$-qUiPTvrz1@{){$Vkenk}rn=c@-XE`;k z7>%ryRFqrgcyuz}y>J5eAud{RPthqljJh9W`Hx*F(X_uDIJM&2(H}M_yF)PT*|X;w zbb@E(nan2Xn$-hki*HsP)pBIG405Hgw zk93w9!%-tgK_{zm^N*IE?^QO@&o({_BC-Di-aL+elZI7*AS)F$%myC1*=p_48MK8_ zP!8m}@1oLsiq#-y>Od4SyQLsp2JSOssEy?K#z~-79u&SLLWIKq^}@Q

  • aYHm2wP z{re-a+B!NxPk+dY$jHhnK;6@uwF&(T`)YC1xU}k#o?F)zmEZfW{bzIi9xo?!p?Rfv z&Nr!EZIcw3o8zsXpPOPKS;tVNQ@ribEj}z3GZpQ> z@>a!uq6v|{rlrJf*q81c({RXP-~q2%sZnUaJ6e<+;?lNAU#q z`7R)-CYs3p{838b5kZUl5jSlUrPv~j3>gjFE&n^hQ`R|XEVKrQfi~p3Q zD{?#6ZlvShapJR|5%%AU{rL^98@e2^@i?OVH%r}YE1~NO^Va5sUzh5?-5y-`mt#?s zr=Rj0-92__|Ni6U0bC5pR3Ae$7~^;sXcia?RaKd*%Lh?_H=~)j_Mmg$_2g|NP;D;xRz~ z`y?IImBJ)?%O}CwnM2V2@2&p25cl@E!%l{immY$fU6kI_$AsYw?;k4B6sEBsrThLmcM326j*Uzs9~h9_#&oKcb9`k|Y@+GgL?jw~$R{wq)v~?V2bQ`N zY#UZw_{|osR))CWLk#971?;76SP|*K`|49rM{#j-D!|)j2DB<1;__B=tHKYjd++jm zGoGh+4~|a{iRX4JibS9BHs##tyOtuqA6uD*KI?m@ z_GD92^yC?cNz1JcYxmzeWhA~2<3@p}h587Z=WqaDl%jad=@DP#4PWpm{_k`C`z>IW z$m;9s0|67bXwwH60`=5s*q}$_P^x(fRY++A<)-{qHVtEC;vbgJ4=){>P9uD?fK)h6 z!nG2NCZ;amhE7UoJzKA8pf~^*N(Eg^!df7~J0@lB-&_P;0j__n2s>qDq{P8fZwfPJ zY;@U`A3b7poofwDIyxnkaY>f*b8+AMbIJk&C8LUODG#$4K`H)VXaDHxPgnAN|Do;> z4wbxT;J+ixTB8_jJ7FtW`C4r~E!%G5UOa^RM+28zEhrj%QhLdYEmZVzmd`+tZ@qu? zpDp`8pGtZL;}@dJzcR}i0RAwsj@j{px2`H=4}8=L@ck?3{-lG>>=}3xW(1^x1kLe; z#lLF>=#@uz#)1nz$8k-x1Te2Y*IWsH+& zX74?+r%Jnojc0@`8$@*0FKbHpyVtz;a8a)a;hV@mJ>K{8GTo%R32O5G8jqR}kJ;6; zSfEF;i;4fa4@QaN0cr*agAB9>)I|b}jEqt+dUjWx2KN*^_%XVKkz(i>ZVrXO9N>AL z3`35;w>v2f_SfBw8eUAEVCK+vy$eF-y?;L% zx*s5~E9A%#$A9SS zi=H1beCH;kmG``u<|EzfNyz6a7_e3pn+CB^SdzRM{c`*sN$ z=l5C6U)Id4WI4;ky$(^*Qw!J!g0Ig2#$S|jF}CIVz*`;P)Z7v%!)D{mCh1n_>X7S+Zy{Rxfx6BXXppxN*(7J z%hj{#wLREiD}BGn796(&Np93&oO&D($H#zl(P^J$n>~kgeh9DyhYJx|uX>LaQV#_0X&fQ<+bzZFFSVYv>4u8wdXe_lAVh?|5b1#$wtr^7yl6j`{V#M6*b4Lo=1B{ z)z#bBNP`}%9-xz&e&@@U2@iehX2ZAY8Kra-Z<4! zU^nTfns)U?gTeR@5OG31f!jvk4QWM(^$oNE-+BJ|7%Rc@{=^f4WKLy0HFs77`!AV4 zvlQY?N#aRst4Fo~QF(Wm`u^zqF1%lQY#;Mxy$>F7pW;@mcs8=$w{gw#^J+M?FlzeS z3fYG(N_HE}iN`@Ta!Y3zNG=X&lkESEFh?bjc2r4832EZ`ORS@(xf%he4;(Sftd_X}3PS$#E~g3UGRWgZi?50-zme z2#0mdTF-1^r&k&GdQi*|A@sW)olxLPB79ao(9cXTaPkBqR{vB#JYjaIXL+}g0qgk9 zKFp6R_yGecEj3qjW(nGDIm+0AA9{jOyhC|5W*`54nf5sL#Eu(99Z?laM=}p)rU32| zna(k^QK++o4acR0Lm40-oZs30^dyB8Qg+Z`;{pVLNE(;hC21RACOb(soU;Rp+A=uO zDk+Q?-x@-A^EwzcH30c7zF#>?tJvZ+5EKC@y(qu$*M>49?8?KIYjvO|9|%oy+}m`A zxo)CcDfz3LH)G_U99SQm6w6_Y6^UgNq}UNY?PY0rSUb&K!r`H>|8kncV+Y!^4jZ(! zJ1%pFUWDp#XgBtc8(nKIp}9!Mo?TYM!<(TlKhOoWcq$>QN@`=nK9@7DCv~2)E2#%m zbf~9dA3-9p1vb{fqzp~{dQc=j2Gp|&ya9}1C}?sb(*Q;@?!hqy5=V|4aWgg${9#Xi z4-mq*#5w%*vne*8%0#-HtOVCx6U1gC?Z;!XTs+*M+N0`u&G@d=KW(pJYNWm)NKt!os1~hPo~UU2NWY zrNpYA9#mu*ZcraRHB9@cQR>L1TVB+VD4Gd4&cha=UjvAAM{ug&?G(=m>`LC8RBfcK zsvwRRVq844aKFDf|~grHFmDe*5mA2enE#WO+OCS+XD=44;zSQ(HaQfMANUu&p8s1<%B zaGA8>Yp8bisqJrh370)^)-%MFm-lm5k~G;0dZgw2FFtM`b`Vp7P#jPFI8S%7Q-})N zGLrG}jv8Ptfw1Z%KRp$n4Np}fFgC29PN`AN)>1n{MwSd#$8V|C?titr>hO!w5%V0E zGpC(%>E1=`Z!!8TMg7{i0*nmszDNZEKpJT2alo0wK$fCFOe<_vuLtw+>1vr_h~C9O zL2BiPi2IACWCwpd!@J9uoIMHJHuTQtTk*!0w&}nbH-f(IG}55}^)qOy_dgZ;&sF68 zT-0-+B#5;=8=6y^+pvLo#Wo!l zhjou}>ae&B0bM*1?N2vU zmH>6teZit{>ig(XFa(Y$&n`F^V~(8)cuTr?oTpEpCfUz)&?E!fDGLNpe@c9NM8xsl zUEnDHTS$EmBk&WBEYdwf2lloMWTA6u!~q+GS3tTh0y8>eK5@eszyTkQ_FB4N=bW zVx_ce!CQMP@8{}u?`98kAxPp>BGR&ocE*t_JU)=fL(Z-)cf-&y_T*(69H`K`hPly` zC}pYM_MD-HakAx^fZ(}qtHl#B9pSN!Z$g9Pig&GYy>?vPBuq!dPK}VV{SGy{UxotK znTV~}to?Err!E^n#KDNJN0Np*L-C%*+H7w)00t8g@rNM1UBZ-3-=@6%NuQeHIF-C- z-MM2w7r)NY=z=CvxPR#D4hBM9Il^0vaLNob`ZTipu!_*(fEA6Ek5$VU@qQ(ivE$XXRSM8w! zjzI)>==ZPiNviz(4i)Z#bfF1>_dqtsvk18LT7BKG+kk(*0cC`T#7syuj8ItT;a>?o zqp1u&h~BHw@tJ&2r&Fwz?RV% zA@m>&rR!Cdd-uT<0yET0{^F97O>kg}d;lAiZa?1MPZ4JC55xh%#!3f8oGd6#_c&lU zi3Dj`uBD}=u~1&Rod=BT(A-zBKdYQH{1L4D=O({@CS*wfR<){JiVr)(>7mpOL!c#O z8UZR+p024~)aZ98z_&ZaqxzdeDKPJ3>ii6o6c;apJnAp#izBWhFZLw07}KAZ{L*1O z7P{oxrG7gJn7nEYdI^Qk1o&epXQ;9w_*SCfe-YIGtOQbFY%JWQqJDr3E@a0GIAkz9 zfS}PgUIE98Bx~f~@b^DO_UH8jEDl&3fF*<6EHW3n2?=*c=53siHnMMnQz3QUSt%g< z{!V}K6Wt};r~9pYKq@;9F7$yQ-IWGnBpLX5L{R}P+j$A=zC2NITsW=o@uj#Sl`5LY z{5^pju8Z*Ue{tNO@9(HJ0wxN~4ZnIwCgCarcc}pLigqgnlz>#3az$-Gna}`sJdDD^ z!-j%LFZ%uE{?AWAr%0Cq-5ZrN)j2fbm4j2BI-7ddv>F=|(-klZRq8J?4~9_KVoCEW zKJtgpzF&h>1XhpVzkbvNOL_uxkR}j-=LUcvjG+xshA?G5RGQzFtFO<)^6R4TJ04_( z(K(o8b%FY&qO07@w{GC~H-S<|3a+B(5LhEl?G$;dLcyG*mZ^S(f+BoPC+p7-Q#=8c z1G;8bR)lig1Pz8UATc!d?d|Po&~i3{T%7^YWS}P+o##w{fLwp|#?d%v^1y)#r(C2P z3ggmWP=f(y`JNnIOFbyC!hk`P0cfal()j;p1A-(X97#$r0?s%RgmQbJr1F*yw>kIBV-W-$|Hpd$p7TdX28uY zgDzMKRE$Cd-Oob~WECCkh9Pgz0df|2e7%F53HP>%f&2ehN(Nk{)@a1V0G`b*0 zz=2ICO6-3Ko*&aOMh73~=#6f4<-UE)!_wlSs(~Hc$VlYls31F#l34RsOe?_uq0{VaiEJUMG1oaRC6M;;Lt5H62;fG!GKYJ6EgV5Mt!0Uor zD3<4W0g(DN%-v&OAQo}9$Ekty-u;ovu{>^8R%tjDT&A-JtO2AT1(}iLLoRCo{X7GV z$UM*__Aco-@TWKn(0x&-pJO$sjT%9F$q30~FNJ~l>@-5a0^oWLB#W$d0kq-Z*3bkb z-8dX97;Y9g3A==^cHJF@njFvNY=GE2Io-L?n_^?W$$DyR4PPyv(@Esic zQUGk>%urb&z6{7qs#y3!%m271{gEyn+%|e~qUX55E=zZHf~z@(UuAWI15rUC=E~OB zf(A-Vnon%P0TptwUlHH|CSpEjcs#RtX9R#$M~mvJyhnNvHB0U_2bgQ!XW@^iAMQDT|hYS38rk07Ejk#I0{Szl&E<(Ib%~9?PVy^*VxBKj&?(1$arhm>2y8DFkYMG%CI@9I@eu+MjBkY#Kf824Hmk67iY$p+P3DW1L;wbQ{K(sGnDNi;v z4-?V!1Dh(d{7aAHGK!``7S#zhV&QTXh6G#d7R4CQntn8+|09QyX z?1OoU+(jj&8Yh`f=cJv}f-K3H>aROfB1|mFFH(V|Y~yaEJ-PkA)$6}`k8T`ihqynr zwsGxacN4i^$FK-Y_;8r+b2&lu`jIf znR~YA7i@%s+*toYq2fJdsI$sGb;dt9kO;h`sn<*p+k8mx2} z*^q@(JXua0Z{ODmz5Dp?_M%hwHfT?mfqaN`p!QYpj_jlnLi$hmVFF7Xun+bW?c>K=VYwOGS&P8>L}u|o>0IN}Lff4vNzxvNIF1HG#3@_O z|9uRDJb%9ck@dFVgVhuhFfZXCwW_zI;-NE6h5Xe2h-ML3SKisNlYt#5U8o!61&cvW zTx8sItUVzl1!fku8%;_~iLP*H5?a}9{{GcNG+3G< z6^k)AMGqH90lRAq4Xkpy@@eiEebC+%Ol*UMEC~*sGpT4{R;zZ~v~qrFPz$rnh&DJF zsU2W3R9%3=@zyf*F1`ZP!fqP&3Y@kL+GiW%xY>Al)p$Phxo_FGJEtM-F%U6J>T>>r z7oe}*T4{&^buSB`#9TC{g>Gg$8xEe?nvuJA9+%`~uWLJT2uN}$si8bySv3j>Hqbr-LW7F#!2KcX%`q=jHQ8ArM6m5H8!RpzAhVKoh0n4mx%{g;>7e03c3jSzFUvPRYr46W zth88Aa7-B%jMOxT1rcp>UfsUxrBk`K;ZL~U(DLR@|`7J2Koc{8oFyr;0HEn59a#W<%7i%(aGlJJRS&!&(=^lmOcqKF(dh!j0dw}p( zS|qdVetVcz&qh+yJqA|R2Hp{MaDuagpn&Zl%oUs2L@YD2G#H%xHT zwvgru-iEz&Gw~6XO7T*@LaCy~HqF_uL+#Bp4jkrR7GgT{o_u?SfAP9~S|DR4$i>FB zpCqQx;Z%B!Cq+*BY@~Beq)O3|W@I5K)z2Ycq&8|J?C1$>_H2H%N1+#!lVW~1yHxrc z-gkWYIzHI!VKyN^kq4Xzu^3Xl@l{aSr;b>iK!>iD0DC9T%^VZ|EJ0BtQH~>BB}Y#b zT&FilHH0UarCXQaV2NB!l?+W1#b5&cQNOjSvmO;wBO+RvXzFNbRs8s8_Xum=A6C7m zPjWJx>%#n+*)3I_m=MeG3jc#NxcNtpVtBPL$Tng|Qb+J*Fb1kVZApw-yQ(*qH18rnXg!cBl>U+zh~BQZpH>0kkb-$Fd6E=xr61tg zxbM(80VwTJYB1B@SRuz*?&WTL$Dtvb4j`R+iRi%NY{();{G%dh0JP| zao*eiH={jD0*TGE5$Oy;#}@=MKXU=pXOK$XnN&(js(!6q7<$^(m=yP(P779bRiPdi z-!>sA9-VU3=953L1H{)j*Ab_FLJEJ1$D)g+T~89gG*Y6D?q_l3U5?KRW)7Z z5qYOcv_C}}hQe0*9{UN6_HiyVn&ICeD3WyDQ(+#fcX1P@{XnQgLYbCjr5KGA3r{*iye(27sp~G)drY z*8pd7$>rXIxr5v-yr#*%wPf6SOZh}yvXi@jSr zkm#=t;iihcG$)Bcrg$=Purk1n?|E?!p@32B`atA)V_z#cba&1bSkGosM;gjnHwjlM zgq(k?E&WHj0~3S0b0DecSu3!8NF9b=9+wWdt_n@K+$gJMF6}&iDC`-5LSkmV8?W&G zk0cLt&*@Pi@?EhjfV@g=*v?5&YAt#KKud^g+3IPX_$q z`|#*Suh9)G>N);LEO$Q^1)69^X69|HYoFe~P6l%22zlrVw;vqH0`%04M&Ez{Cwsr` zhs88nv>wv4z`%XrBK)barF(^oC-vYniOA$GyC` zC`%YfBRp$bO`vXhEB^vzbErK08Z`Klp$#(!O!Gt*6&hF6KV^-m?b(6<9C!f)^6=zWw;j+0yWEEG-!?%R$?#XnuF z0!LRTf!jLP2yK3o2=G6!iR3{>a~kRB1spy~A>%HKs)7N}oIHAYhnFjCo3=b<6nY&O z>I#4hAL9sI`Cv2l?hXPZJvfuVcLvDj%IejWY~k2UJ{q`cdZc7;9SwbVsOrgB<|Wq( zVFT>~Bk6SQXE{UQ5V;KLr_Xx@XoRoiOB?-NkLb6h9jeZ{fVZD9l$yfx{xm{tE@GX# zhdA{r`%LvVr~$yUr{)Gx(}u1Ie_y=Qm$>}doPJ(u?dc&H+CJtV?yPLnxgIcOmh(cL5(o-MR)HYNsF59 zo!w2-G8hd+JDfTAZJ)tN!n0n%g^8q;6kols2#yb~$4uby7wLRcoVFS!sUyqcTwqjr z0%mF#yf<}+VKf2p$q$C=v;3Zo(H;bdddXBLn`%LS9gK=ej(y65g%Y#tZi<4Ky6^c z5)NHmQ~cK*)54)jXS;x%1$GxP_(IBsXq_fFRG`3h!=^pe9ULY0w=)N=dc8RvuJd@7 z*;@BOp(iQ=|Ms?(1sU&ghsn2;o8TA@M+%PcDRx(Z;B0cH3Q0m@q8pY7s4@Y=^?ZDn zpz7iXWxuvwA3hj*-D7yoT(rv5E@~D3ioUe%a*6H8aItp0@FmyvEJ`-9_!C{xHRU;c zs%sU{_W@eBcT>2h+wu)1XSy@PTRY!DhA^qG23ku7kx`H_$?d7gOm~MC+J>L)?p6a+ z7i*TQIzM)&nYTB>ITaZ2(^?|bM?$|T$WaD(;}b5D-W{D!%mPmG-^dsSsSD)=tBC2v z%JasJ5lS~4=lTq2j5-f)uNqWD+muW5up|q`RwlzJqxhJ>eMMu zlu1fwYl@a8g;-?IRr|?8(`Jsx=6~@n&=+}0oF81Xvk_X0GjAp|?LiC&uhHle_7Exd z=0hTH4&&&H-vDSlVntI3t00y=kf`UXa*hjhlJ6Fu^t7o&_O#v0+3Ar;7}?ny`nh|W zwB9K+PNk-HtwKk?=dZH?78)evW$T)DQqFUB1xKbg%EHWnpA?t^&zhK+H=%Vd$Di*A zqph!h10XOu1n{WqTrOVdtUk~_lG`(lf5*~5==ftwa&qH4Uo}aHO|(exO`5)f!*RtO z5iJsDBQ3EV<0eDfJQ|;-BCiAX6VI?M*Ev^&VKSz-8lj^M^%K~gzi6LoTM43tNxEw= zNtb2o2G90>wWsh}hGI=MxTK~O3>5Fc1a^bn>oxp%q*4nhbeL5KO{zy+Q_?;hdiexc zBqo>8*3Z^0y2irhlB!5~XCfxHx?I844XlZ%7`58hAd)_HkA03&*!;IjjsvY82*7m=D}RORe7Lxb9{T4KM`TsV_F62*K1Jt)E?M0r=Ex6hKI3fv#u4NwuFe6O|+Q zQ&)Eu8a}o&f@*{(nkT;IdWd_>r9o^^J;QBvK z=`xe`5Gu*n>~2|#7Ty#FxRFj0>x)f+*BkCl&hqJ}Cs(FS4_Hbu^g1QCR2UFZMaQQm zT73g0E0MJ*IaSvaq|Y-`>9bz!*-KVtT?u{RTizZRw3ef5unq@#*z4j9J6l2d0To%= z<3(N*gaemq6k6rst9&kDrt2-wb6nYndCEB|9F($qI6OR>eyEM*<5#LQk2Ji}4aWkJ z;PoZ)9yisFYIj!^wb;B}IP8Esc)10DRZyWZxG1TSvWAE(3dni!-l?47O&PI76oGUh zI2Ww?u2)yg3(UNl+6+=wRyKl!*zeopO|!c3VEBx7>CGpAM)EN!f^=Y7p672S|G}xi z@L6ih7Kv-24!2iOk9h+^dgs=q$hWrJqpl5F(%4eDhfZ;D4ML5JsRTiKV=T8dAXRmocC~fWa4{RQh`u5+98&PO31Wjbsya^Rj?ZfI}3+$bZ1>~eztp>GUIcI<2nLfK4a1B>iMW0_!0ncw$ zV45f-CE#(ZAY_lEa{~naAZDnRm+wp#9mmjfjS3SN2| zH2+*BRiObgsxfc~le*l~!43jngQ{ocFUOKT0}a)URw`o(H0&Q)5a}30%X}Ut$`PRp z=hjhu8x<&LnHA%X#wNB%cNjdqOCDHIGq}o>cOdz$gTconY7og{>W_Vfp4$$mZE*$? zCF+X*$wH#TcM=o$5xD)G({#N`ivBTX`3*Js*HiyFvM(p_eok4vj zt)Os0^PZAkj~%#{0>i_omahE3GlYy~nU3@cc3I6f*^}!$R1holQMShhS)U2vl|C^b zT=w_$z=}P(0j1Yce8WIy?XtOII?_j8lK2YB&q%{^$9Xnhi>`Xd$`;%Wu*mV%^N|N4 z37hi6C2a$KE4ZWcwPF39d^loy;Y8)94-6e%Px_|$^$OR4A6P_KX z)MmTV835!UkDXPhnO)(Yx&Ki%?98p$%gmO&23v?Uy#3)oYYacbKy(qDf^p>SqJA&_ z-{U~fF`nxuT4vY${fn+MZ_Ms<)lOJ^!O1ndS!{|1?N@=#dU+1ko|a`)*3&Zt=`lP} z^#n+_tM&}Y{ED!56tJvwUYy4+qi7?fs-leK4dv1wb_gy2J%NeM;fnB1fPk8HXkGk> zlgIDawR1{}BszBJsX60I;clP~Gv8wL|74k+_-DJR$9WQ<^t7WgVK>St-33#j~CfpV1;`$g$+zV~d; z)<;zwXX^Jk-`p6?5oJCrS0C|K;EkRjuDkBUjbgA!NMSCUQp8m2AxZImIr4broJR3{FVCIS2D!0F0TAiUnHIG;q6z~gY3MEcIvWH}HO zUc>xF#aG_A(Jj^(daCKNo{gCitFIak=@LCM9|ETAyN2u2H)VjZ)@MIRayD*w2np;_7^?E6iQ>H>v+PJFfs!T&(Zl%SMd07hOHfQ z8Zg4953XKMF?UiQI;Y3T1d#w%;3J2*uDiGcN1mQvnNLYgJvfvoJ*Cn$%feCXF02=8 z*iLMN+cEnQoRdPpV^;!c&Y;-|gK=_+Ns@4NBv9g!2C|WwC;)=+;z*mw?kc{fyuio{ zm_j&qv%Re9$>F7k7x!WCjv|o_4*!b?v%D@d6}TL_itpg=y9j;HBvazB&Y|@fI*ExD zVFZACq$KZ5S=|=Si&(p0Zj!d~vm&RZAw05q3$zKQHgTJ-*p+ae3No#^7x7+@;`OX4 zK-Ky7iyz3z8z2QsEIM3_gqs=db|zmN6PAa0(3nh3ft>}g_%ZH5nugoQpCc^}G<)n@ zId)$jBN<@ls83_-OI8RBqQ%WwA?q~;Ib55M6Vz?83&m6LvuL}MsyBcjUa9+zpErr^VGG& zIc?7SEF3K&PiBN3S+v&lR_H~Aq(J%JfJjw2sP@uW0zS-Doj471LZ~|gfM7&F(X`j# zlHVG?+9OhpEhNIngH6sLg1w82h&tfx-E)zMs_>z2<7TTLNGb-^rs2df#a?tC#pN}K zXJW`vCWt#l=Qt#zi_=xF?pjMb!*ll(!N6I@^}t1<-DXZIl*t3*%t!WY_LG3$2(+eX z?W1+y`ruIwy1rh){Ra*OLhBf{Vo5R@GdYaV{JIQprricW&9jzPA-u!~BPWIO-Z+ar z+0oNOKGlzjF_~2vNK;mT9NG4iK@OSvfOmAgzIvxC8H$#hXA-IyRwp00%E-%;yDLk@ z_q9h;t?#f2l+@$^Ei)kGniIC>^#<{Xp}p=RhMcIjA^1uoN8zO?p199*aMm2Pn1MB@ z+q8RZ6~7vYFdZ1CrAPFouU?6BGyBvSqAOZwrgXXL3+~WP*ekee)d0-WUjRoLjJO5# z3))v_f!^3xxI1^gd+dUJbP609-2{#Hd{*UNJW(En8vB;&B9QJEPej#g8vytA7!E{| zhEoJY3tbVP8~9Qwz`3i8P5(~cqKEO;*ZdE7KufI4y_4nNhcFutiHkR$)qH7w`yq+s zJ&nj69Yp4)yScKQN}8RKzqYO#07$%JYyidY zQ3;VHe<3l68HNS%iYUrfnxmUJl3bL!HpO4h9hA zC4+=7b}PX&?}bA(sydu!5u8KkEQ}%M@Ft*d+_-TbbP>C9mI@r#D8IGy9HYCsyzX18 zmtq&7OCp(eRjlF5rn1eko1EMpigMyw0dUJ9w$~+NP>tqqx)n#8P9(=^PQYfU^js&w zXz88Q{$Bc?o$~$?wQMb3yRnxK)}IKEWFLH2K&@SQ`;%PNdS27Bk?c4vyRmoY= zZ$z{G9T+sSs7|EFxhvPt1Xqb><2*QZ^%XAApp!1;E#`|X6%7Ji;-Dwej0J>RGJsxC zF>hFGY$Zo19lRlJyxQ8%jTUdD64w~$(jCZ*47rD76_z&8w04ShmPhf{u8rUJj6Z}> zXqjg?S{s_&->*XgiZCp?R@s?U`*Ifm`5Ze?)-=vM85e`f1d8kHn4QYub|r?Q)J2Go z6H)tR*v&|0vzy@Gg+R%w2f*QE;Z)FMVKDJmQ=zx)0%sE5HPyRBh}koqkl%7G0zua@ z*y=~MpDI0;Ke5-%9AiltE7}pppZKjN=QBlHSu9fWyKXG-*4_p4DUGZ+rCpNcLcY_d zUxy@5%q)%f)Sor^ENm;byUFUM3o!?E6---j9|T?gGbZhZf7U2|uk85~^!r24_=k^s z9=;EYRsZ~-B{!sazL3`w&MNOA?!Wy~Pc9h6tv2fWKQz!s;ctl=B$0k=%YJul`T+2` zHRS!tX9Hd>3;eCSFQrcHCCcCaQVoxF^YL#ce*XDa|KATEOfyc`I{xwMF$qvO5D+jt z_=8po9^+?diYH{f=Kr~bUq4yV!%z+$y+ZKAjNRV4{`r@s(?A-8ANr$X92U-9dT4`K r{Ei}&$nUT1dtLpfzlI|*`V~w2O1P8^eSXY7_)l6wK|JU3^}GKMcBG!f literal 0 HcmV?d00001 diff --git a/docs/images/minimax_speedups_darkmode.png b/docs/images/minimax_speedups_darkmode.png new file mode 100644 index 0000000000000000000000000000000000000000..08c49cb8260f7e07352d028f97a41ff4c9ddb3fd GIT binary patch literal 37935 zcmeFZc|4TsA2(j4MT^t7Bq>yukcg07Ic2BD7P1sZVK`yPGA23YB%ztCW2k_Mi)Z`vGj zbJLd9e*u0D0Y95Iac6Aa#0~u3y!u@R*Vn&p&CKBb`np6?t znn*4=SqaSbNT8P^7` z@Ni3SZP_6%eT)n7_^{YS>9rMt8`>cloa2UbGdG)`Uk(X1xAd;Dw4a;wxFHwgoPcr6nkhar-!(A?PKrJSU%~&#N#j%-*S=GZB*vuNMqT+jKb6a z#F?)Vn^P%kriPkSZi`G8vs2=ZZt}-8r2C;*t)3jFjn<3t#fwkk!uVi z2fO4c$H!Gl>_j#r93|389-IPWB`$7p`9^}dkC%5Va<1uST5*;Q;Y5`dnwsQj8ckZP z8RQ&V+y6n^%{=I*Fna1);NyL}cQ4@VLF~g(*=lC7bX2tLVlxu|F&DWwyfnkN7%*~W zk4N;V{XzrV4yTIsoM%K;!zpiLz?1~6Y(NSL8`IY0OxdcTObIQ61a|a#&YRhM$^yIE zpDuO7RtI&w((OnxyP{X5F;UDNLG!a#BbB%|$9pYK1;E|1_{(yS7ctnsO+tuEr}k89 zIjL(DEzTP&DKBcMG3@S9(D%tWMj3&WEp%oVgUJb>@AqwOOi8`}{;H&UgSb(`@>FHa z+1Wpd%u@ zSjw_JF%HrW&!H`?D4Df9RlMy}*sMIqsS-(fg;hSfW&^p8%9nG(ReotpL$0x1aOcUi z>e(Q7~4jPct-V1LxFFo$j;lDvn z$K=HKK<$rwF2oKI#AY0g+=DDjA*H_4zq`z2YTf4acvhYqUBR~mxI?qz$j@t2RDB%} zP!i@|^w#c5SA?6UF2eb0cPxVuzPZ}lG4OEj#EIDQmFiD-T6&n70CAwcBN6yVk4OuK zzbFe^{h>WLb7Xe08M_DS^lPGw>avOz+oBINcZ z5z!UL4)&c1%!2&;9GTr+=bR4gph0V(d*=VZ>J}0{7gE4vNL_tJl&07bPI7sqIMbAB zt-$KP#a$M=@<^e|sFPD;oZj0KkbII+eqKAQ_eAXc1qPplgyzyqy}W~Scv>9kj`TY+ zv(QbYSz-$~6;Q9&pVfzl0tp0Io}|nvz9zPun7Yc4Rvg~EbU-B8b>EuDlRv{XCm{4i zHAHostai{uJ~4*xP-oykZXqk*;tdvswWDOC>o}NW!jsytZ4rU|?UOmD82boPxOdpinqj1bVn9+o6))wdQDR+jpGM z-#uVH_4|(1;G$LDG|6FJ`V} ze6WTLm=Ds6ASpIn%O~CiGODe88;6pXkcx#XrIq>du&()@E=)5${FVE>$zcEfth{#13wiF$mbW6O(r3bJ zowhF(PM#Tz?Fq4edfK%)*y-|gE)7l84mjA4J5P^{s>-c*p*|;?8uTzpdltziPVHMp zDpM~p<}cYL*l?9ytYtnyr%N;E(69pWRabQM7r+(WmF$&TUF@c?&HHlYLcEuGra0xT zvLZopElY1m(|3&-%E9)Po^XS$gj`6tit7rJxSWg^abdTD2Y;{4*xnU97C+)bXTe6Z zohWGE4z_g75^S~5Ip?mEekJF^S~R)!vZzS^qmh?sloUJr?%9PM zqYFvcVF8sP3NAa%*5FFHpqy{_7=6x1amcQ*FZwg{Oe_3${!-D;<{$zb&nS>Rqwq&;+cBlOJpEV z;So;W_rl2ce^k;o5vXNnWc;w>sK^przd;jmX#Nk{@0kg#yj`LCr5==vZ7q0C2i7yb zR&8IS$5O1uZCJ5VNre~EdYRey!b5UT_eva64vo#(arwmB$f;dSPq9(2OYkWSB)2Ej6;l@>LmXAXr`NW}IG zq)t0=I^J()7!us)n_u8;{jXPR74`O*OnAz=p-egC!IRE(7@7SWZW*@NG|?%dRsM@f zPo$1RJ~sz28Tg(VG1Bpwrt0qeN zUx9E>Nc*m&azQV=o5|^Y-+H9p;#P(8IcWwxz(5WY zS=YcS4J+AVg{+ldGPOZ6=PE}6bNwjvmROF+-m_NiFJJ#-wvhk~5?9HZ~ zwAXZY+9O<@bp-=yD$?H#S+eCcYM-D~IDWcJG3^;l-yCdmaQ5j$P9d&i`>p~Vt;5yg z3+c4?szg~tI70eRYJ9oZctY=EX;V!VRVn5}b~E$Ar1N<0Je`5u<^~^%ysUHGQmkHx~NPFiQ%BTF3Um5V* zy{?ZMJ*`jH*5BeE7VgW1VfLPtn`t9PyiaW%e^&5TH3`+le=tI_6Fb4&A!t!Kk?(Km zV+FE~>>3J{V$2gGwpW53ju)Te5k_t+de;6IYsBwm&3M0JC)MO|?650x7Sk2i6F2~7 z3YpMDVV3fF4X|?{<^y+i@dog~vO;Jb>h#Rq=3v*R_*+J0lkWxnt|}Hs+Z-<@Z<#){ zWa$u|x?=M3#5NE?cf}E8y@%BoJz4y!dt6()DkEt{y5ar2p?o=1LjEJcocuAkX2r2Y zp`?2ew*oJpbKuxV54G*vW+Sntfv+Xn$Km)0(yN6vi?*3-S?=Q!R`;dZ2&WnnnAa;n z<8BBT*1aQczPl;$NZCp!#CZ>KsST-!c!XdVFF7L7UT>as9|}{FiS3!rypZ+szMb_} z^5&Do7p!4RwcLq(-g1sPaY#e6vrTLUiAyf_68VgAk-!w<-75y{6tlOG<8^Z84ID-> z#~7ui&e>}9o-=&rn967!!ggAP+J3-t)WlfN)ufGn8k?w|YY<(YE?3*L7B{q4a(VUk zRGK~F{*pcGip#5Y_l~^28w_67GJlkG-F~2ydCR<@C|{c%;}>nJ>|GC25;+xe8clG4 zK{~CGJuVZQAw{-I`yIU5K}9)j zRUFv5fJ<1pEqs_~&0viM0HJ&+{QhB)wU3E)TnoQluc0;4>&JVdALnU*O0oW~`t5YFupI}yC~`+~w>&~gpr?SR+? zk>H|eW{~?pUMa@i?ch%gNnE)&6Wksnv ztWl?U&lY-7n9|=hh1x@V^-qtILr&P&WwQ5V6Q_!(LH(2putH5X1YhWoN4cw6hGBO0 zGNF)WWGlE@TCC=!&BDA5-Uw?q9t`PltQ5-Uqvdx)JkpYG4MiV?S!U$f%|lS02_e!+ z(|P#?J=+id(PnjpW4eHgV1G2YrN0(IWl*zRHe(-PAD&IziozH zycXY3F>9X`EnQI2cdnP^is3QmDzkW}r#iJW*}3T1r)vW#dJc;W*M4(KJvjg)>m69s z#UD*@KcLcB&Wks%P$|cF+CO+pc3pF{;0Jf8Qf0IUK8arAwJ*^rm{m#nv-n;0A_?}rrL5Mfyj_yqI ziZRHV%e1d_%FE`{!_2w9qWon1>1!WvL7F*f5L+sVOiq=HJ|qT9H)B!<-+qVI44JuK_&ou0&vs>@0e?rsTCNAtbVPV|iR zOwv%j$zL&uSb!njp_UDu$$Vo80e+u9v1gg-{m0R;P^pF&0a&{yfsbm{_!}9;UIsSf zS+FzVB8eK$d&g9j(tGAa6fno$5DjW18RQ_#-}6dbZ5X+}sN?jh{;_XaUv0m+GXS6S(@w?v^a6)`dM zZq|CUdrxL$J5(JneO1G6lgky3krg2Q|(O}KYtTWY$Swko2pwI6s9RfeU*If)fR5eG&@ekCN zzcgcn(A6`g^ZS$Mm($ChM=C$6MbsH_T$fAU9M2Y1F3uM?h0F31LGP}reT#T(--9B& zfQrww-&|tv-Db>=P%GQ}C*TP_7pQ{2T7TR9jgBQ#i59^#wjo#ezGMeqwz1~6R&$v( zHOQ@PU#WX@M=Ra*;o28JD@V5na4F(l9ShTXrQFKk(|0nb)nK#fp9QBXY*=@Gq zVn;8WbMF?M3|o&hiKe0#9vndo0cOs%^D&Vd@n#%5S*}HyLWXEm9YwQHG>6oBhNB8s zpy~ch^6nbFFGx0amj3?6STgq6U$2M*IYO1vj51CtaH9k5 zGnR_I-nm3eM|vPw{zZNANAx7&#(UCy^sHE^I%$g3qul{*k=)1DS$dhZ)1dJlO`eD9 zZq-A`YSWTsQxaR+cv4WB^lLAl{>Xu_6@kVG zu^FJU+wEnrqPWmNyPIjt;6;zRuKYQyS-Jc+pVQX5H(n?0l(l}K)EB})#bT+i+dLOJ zIbLAx%IH1aum29H{}~*2Ad$!qif~~rim6EOH9FE(3Wa6U;}2i%v@+u4bOv0FC#3hQ ztDX$J{%kE1StY;Uffo|_rl?}Ju#H2}_@ZuhB^E;{h76wkkvC5ATR<*s)0rHcoV)h( z+3xuZmUf}{EIjPv(<@#EcuwS5rQd|_q%HW|vw&+~h+fOh?i-5|P;HPEc%DSSq0-I( z+&^nw<8SSz?JY^KRJ*!OyYaphm%emr%O6?9_=Qcboj3J_*5SnJ81k36H2E_@m-0TL zl(?(|q;A>QS9r-N_6;lF|F*Ml;_~fb*B?4&Jm?U)uF{lwCd!vuW)zOQN8L)S>s?`= zsyrRFj^F_AJLzl*lGaxm4RKf0JZ5cd5?E(mwxUb=thn{BmYd(k|L?=Bs66s+S40`- z0#O%Plr!enKopi+R_|fEa9mCW|aeEz{LD=iJYxxw{#BKf+ckKnYqs{rt+k zUd@+;ob@rzE;3~4PbE(PL#CuIomN$FZX;TJZ8jF{3p3)=4tc#`6yGKLyBkEPdXBtX zugA`N;S*;r(V0bP!kO1yK?#zJbrG{bh%row4mX#PzN;o6k2pX0S5; z*Y9s2kT`DUe8_54(bWEU#9xBbfB z4*Vw*g}vXG*x4*K>JL*2MS@Xs^QD-Qf9Uof^ZZ(>y^|kQbyWUM->bX)%{!-H%*U=O)R)%-$1SZNoz$A|=Who8XWZoz?dIq5 zRKu97F|VYUq!LWx+bZ(UISsNXF`gmq=Z?&mzke;FceSknGX zt;BtSH>U4jm9>ONFD6}8RU*eojwXJcw9F)<@V4#ed`Bqk?L#^I3r5M-|JYcdjNitC z-k7*nF#7QHyik)SX1btiwywkj-7Kd<#X2afp*=f*DmiJ>5jZyGbU*CO}u}6 zIA|J{#tV|t>4q$Lx#zZum@2q^NmRcQ_8Z4mAKD+TT48X5wr4g*CMzgV!+HJvZnXk>b<%GUe322=d*?fUd@HX$oZeQ1ITg(|Zi2P)iaQ0HL9XX)^O0 zoRw0(vgOAO07mZsZwq5C(fqcW2Blwa!2@W^aOH*?wVXc>2Za_;Ag{NhrgN^P=skPuid-!0fH(EbOVHG*hk=0hTdazd|_oJ__pbkzW=BOqvW6>glVm~=b8<_KyAew|FMOiK4uNN$xyjw;KvC3(hbZ7{~G1Bs=g zapsOGAqCv82!3PCuH_+9nb|eK!_OshS7Ggw$_b$(N0u*}F7M30&F*nsY!feXtXqD& zvEmp9+_OmS@I@q$jXPHyvt#sy5h;DMCdxMPt@otdyT644KjRrF?B-68)Ct)~q(`#p zA>Lg)N!sdF$_%U7N;2v{w)bD=xMYvQQ3!+7BcC{?+` z>>ug;563d>fI_QRR@4RpmiQPb;zYgm{B`|d+A(+TT2zASH_)sWeWw$=AV9MyV(PQf zPas&si@^hh^={R3NE5eG4CG&%{F*lZO}KC|`Hz6>x#!=q&)?L?R>kaUJu@lfc-7IN zBI82h(kJ4>9Ez6n0WnTWNc#rE!6zw@NHJdURw637kit@Gy{e^T-HILL0a$6a5 zLDKrH;5j$zwZ$3HhPL-`_mZe5Rf`dG)viH}V`F1ivWINRki_VH$Ql&C68M%+kwi>$ zB6sN|GkNYG$NVEV!k+$egWqB6l#BUNud?JEiMDwlWp(kl(v6yEG?yZm=Pn7f#>~Ea zM1Wkn-9m#~hh<%^E0KY~AaSnL<47X}MN+$IsA~gJ+V!Yb=RhWCHV~ZlN)Z{s`e=+- zauyIvkXRfs%r-~35*LUvo%2sK`*Jnw-eFo65x)%*Y5DMx;0xCq7e?KyEg_(FbeG1os zj?}gm*EY!Z>;*^cx_w@8?4i`LWhi3zU_4$3DSgrMRep`-tGvEvw9e+yzEBr9y!ARm zI$vP;lNHYLI$JpH^r$P&jFN5M3Z94;{P@tAsxYi0v%arnTv}aA!x?QfWe`(n5q86* zGG}BneK9RBHL)wsNIU=D&A#xm&~m|<1!8@6u0>t0hzcW3>6)?qX3I&GV9$7U9-S4| zJr%pOJYDrkI}KGgiP(peGE}O|Wx9o=4k?yhr6z0;XRuA3vSgrr(Z%@rTqp{Mof2$w z&=`^prhF{l1$Ud~fDRj`f(uX>J8N;-7G*c%l5>-bZ-`U9-uF*YIU_cM6}=-pB)?MP z_;^7$`cG^A$N1FpN@;ya;*u`t0-C#&<5;>{mL?VD{2 z!|wz9k73wzFU5HrUi|}_Zwy|F!xPZ?(iW6Lf3Fcna9XyzfRar9Z+;D#k)G<}mM(eZ zTZ@M~pY2B!wj@FjR-5y&HiM<1x5y$X_nZ;Y7tq-U=fb0UkE5@5h4y0c^M>mii`lGN zpl8Qu=1!w$fD)^u$14t6R`-|pdc#J$EGBW-I1I$L)3$q(HAyLW>lzKabs+Bq)Dz@f zKY*93%f)o`NMB4vE-aUt*b}qYHzsT=rwm?Denv<&m=bM$R*4~i)x2mgZi%UYkOHb< z9+W=tRgXxJnZf9bXBOsBd6e17#}=paygG9Z@^Alwg!_q69`g>1uADI1>{5A{U2Q$| zK0Se}$-{bR#cH(B*4lDFfn=Icf$<*5E~CgjETnpo;>n-&!SMoGPBO;KR6p;t zU(8=|m(R7Yr#WH0$JtUEF`-(_I zA+xd})?AN@vmJ$eeUVM{@3y7(`InxaghAD7Qz>fLmZc;6c+krG%hnSu9r#YallsHb zDV@G!e01h(M~GgJwDhDrN!X4d;MT81dMl$~k9gwh0V|_>Bm3`qd&PTEXP1V(JtWdS z@QU8&pg)m^^9@ zM=NkK8az)RpqJ?U2ka7<%C>G?(@jE)i25f{-z7`qzRGb<-S2Y^ZVUH_uO(+IRBr)f#*w`tPNe^C>L#YiJ?A(xDhs8#e1;PL}j3y)PrYL z1V`K5{F0ruD1<$ft8!U9BImG@?-ze$!AW@Pk&xHzZ5;UN#5PRXceS{%R|5Kz@7Q|U z>^F++3|GHw4U_k@5&$jIbHkwnUnD(fK{ir0n?kjiUCYY$s2D!WQR2IO!v`V^e`v~= zc1zE7z#N$_&e!}lYtp%Fg~w*)*Um~eayr>I9SU{rN-_{UoCu|QVe)U1y%ULO1AFng z?gIGrY?+EEigfB^R7p^!e+*D^2lwDkz&Cxc& znM$r%phQ`&od-@G?q!to6R>X0*Z3xGrBybcI-cNNMygFq-}i&%;6C+{5EI-#v?qlTnF%col3=D1)jpV-F9 zWYIMo>=z;4Rp;F)7OEWAGLrHlSeW@c3p%pqWNJP4y?sZvW5l=(s!^hYFU15533+Rb zL_#W;7c$b)XTIakX0Gs)#h$&TWo|*Clh#VmikSVntkznbtGygm+ma{gIjH|EE`OWN zqi~;>_V!0D`I1VQMA{?LYwOeWOj;}P2nv5U)Twod4i2mnqmV5@4U*MoQb%3c3A7ah z`)-TLxL#Hx1>C@CVOh2Z`T2O2cEnvC(v#CB_l=%5upjobEHz@moU`-$eNL)_B?ZJH zT+{UmruR&KiPZ{p!mqkwpKXyTvswsIE-SnUIxmeaz?Z=gzDSqApYbL{}d zn-OM`P#Hp8Do+Qy;1RPZY`O0xdr*GJX!>?ye>Bi-C&nu zV1K|G$Z+MG^Ro`d!8UMe-O}<%JJvvj>fOrB?V!2@Tc6vT=?lDN)jeGNOQhY=wvK7}YEFJnwOp?Cs_} zane=;vI{Vb5r=0SZUSb+!@SMX5?UhToe3wm*^XDaQUGjRrKa}{?sdnl-NZZV*Noqv z;J$RX{@V4Dkr)$>@V*qfG#WQNz%4Orw?T{HAlGU+qG~#a*9f{3=Wf@pZ1c&t%>PzSUr#Ai|@9F@L8f{3k@Av&4PtBHt&3iM=_PR4pG7&8%%3h9c zYgmn+dpD<#JYY~P+u-$d*nX67$3;IwMNaXob}@!?GpVRy!J{x9mxrNllsW&hN>ixI zRZ~-}QE1ACPRVAey6jGc1y=<*RjuW8lhF7DxKZBnZwqGHKVQO{(BC)A7`Mixmn+p*!v zNy`Wk@rQ^kq5+Y+bgBhFrT>(Mt6CLybYCuoLCVdlJrlp1pn7X)wbM-PFZ5nEE;}34 z{MYID^`qG>_w_PHWimlGg1n|bx-?z4;=R0+|M~sVd$e_XN1M4)SC{OnXJ(2|xcc~6 z*CX&>THgLFe42)KFV4pM{Q7yGj#DN;Td&vi zcVA4#zC5#V5k%cPkI4S6H(Tv_FgP1XtMt?4(kjFVnT7N_aY zi5CneXI>GvA$&nk)HDP2e3&QKn%e&@N34CLDh{Z&plL&3=nJz|Ey0 zhCW@thd$hZwz{G~iBp;<7xDpgFUM*x{S@+SDhkh3QaxQcmoJmOK^ZIT=>D-PQU2#` z65C9RaVML#P2&Zgg6^on{|wr2LcluSwNUT=^JEB+M@aqpsG{i1MoGU@>#ptiI7lef zvobWhNZ-`tRNLfIw5dblL`Q+jMh4?F5IikP(;UCx_+ zKzo--Uf+yw=F&Z}TK#o>@~kV1$PqI$qxu8BMMZ2*6I3Y%jqBQpc#gyc(PaRz!1^)- zaDV8oMYVY($eD;HE+X|@2EarlQv1x%MMbC(Ve9KJVE&&(_@eU0Tr*ZZ#1Y=yxzeI- z;@iYxj9JEczjV8+V8I*c7oe0*aX_W_{D6{$Wk}w!u2AAdgWyHy1NivE2}{pl_X0OC z2Je3XSQ>Y%YvrlxS@iiX(|w8Yt_!2Y>0=gN)Bx7+a|bpu2F9z}60)MIT1*9nG?<0#+NdZM{w4n@IR7Rc zBp(DTU7>3XPk7hRx_d`ei8I!t_{sGF5;_@#E!ooguwxNHq0@NU%WU}y*oa$yY0M<@ zb!oV^I48b8CFJbiFp_Wi@V_6nTCA1|Du>l&M?(miy!v)xmWeodo~(572(bWz?4><< z1?k9s^}s-%5G+A7(5=*Y8tpQ#(fVLRdE}OscReRn_e@-sc^qWSHq4HYL*e{f)a74rnIFis%}M?6#wH2w z9%-iT7NkG@(gm4WSd7y}5}M1+tEFY|k3-Hbs?~IgZe;7jCEG2}`P`#5bnIR^@T^^=4BLyP<>tQzwN|- z&FA;a7+O?GO-(I+Ii+KQk#>ZIMr}l$wV;q>=Sbl#%%_tjOfyN#6H)mKj}kAOuODcEckJLK=>7<^lKy!Z^J(B z*P4kV2N+dg4ygqD#P83ne|+U99Qd0#yYRKfyr9baSpb6B!>)_5wSEJRw1j?XEe-Ai zb8f1yACzwjs}PrWgJ{M!Y~&k(W?)e){jW31%Gg`v?f?UOb7Gu@b80&-9 z@a{rxgc#NI6mPF1vJ9DzuqN!L znT_WDWwp0scy0Xgr@(8@v*}5~@#i~t?;|OAy`Ek6oo3M%h;m3XiH%2Sp{Zx4^?VLd zx|eHhVilD)H%?Fu)B`ifq20C@3MlbQ8)Z7Yt4(dz4HvEUtV>Cp zlW=u;_67QI>B>19&Qoy94)aG98{jpZtR#;tNDEd6MsH)vjybfcA&tMQ)5 zi4+ahKoDqhf|j~^X5+^#{>hXG-Mg>xq%>NVj?Zt8vW;vfHvZ3 z?2Z;%JnXpCspZqAi(*WDuQ4Zo4>mQuUv|ihE`L$wdjtKxYoF@Zy8!;ZpxQe-XG88; zG#|f^EuI9o6Sp=VINBL3Rt>Z?;D)VjiFF%$sFSOqcU=yp&f}bqiTRT}ne4pnmNsbD zrHjBJkzR!%Jho%gk&Va?k--5{zL)Bk&n)OC4DJ4*aP{vI4vMZyW02;u zENZth1FAc;K|lv4tTr2)Cs}E6<{@LhclOuVgc^|@m9E(@ezWp6M**!%s7ipWmWS2rD|pLzxdPEQ%IZV_f-(p4vi29YhP zE>Ro!Q{X7#*aBc!6D|$`g+#nwUf+N>Dr&JRFuook{cFwXUfIB_8{yY>CDj@i#s(Qi z+1BM!T~@^Rk$h12m~1ACxCnIP&PlmwD*b&J{~xXU7pH*o15iZ?3Uv1K%MQ-@Jo}v3 z(g6LU?>&G_|M9;7^?_PmuvMgIG^{)9qyf@B@sx*gnsbrr*dzSg(b{G8Z zi^x$rp#t71n%dbBH5?^Ebqw<-XiY$xto{eY3YI)s6W17EIc>+A-bg>1Zh=tg0Mw?! zrqtD^PVDs;SqZa|`RxNSt5Y-iMk_EH7J%p$sQ*S%i5BEc=-#g2(Nmc`U9D0V7 z39Of{W((8iOP`u?p~p0|eiT}#yQ}sVhm)$yl0E@J_A4&T{l1JTPy7Vbax9~8rzkQP zF=CnFtI7ecu2rJQ;a&1kMQG|H7&Ua^6;h7{qYT=MjDA2RcqaQ+S1@*y=W6Ga7d`j9 zuVS#}|2yBE+S{vTFH*Lot8FD4&5Q^Fb*gmJx=Brn*@s+?d|6)c2u~s5WBucF_*hq= z$c^!10z%M7uA)D#{f^ETxK&b(UT{ltNL&&}Gn3vP z4b&+O1Wms1%n^m6?xp|RF8xcYfST0P{{aJak7-DDqJVRYLJ~3h5+2UuOP

    q7{83 zp$Z8Tt)IX)sOP?c0w2cVB|6#C)@v0NsXqO^WR0mzdL z9g%6aRa&T8zw{2{4l$f;%$z&?w<@M4z_C~(g z&gml1-McmJe=YFV^b?onOqL>$jP# z4MH2_3-X)R09ma?rhpmnRpd0gRMXxQnMfqU9yx#@A>G@I@GnAI`PMg>0K*cA0s5NH z_OiXQs~<&2YaQ?GQ3j4*E>sc2%dS>VFY0%4GG_<+TKGs})<{TPIkoAx8?6Hm)`ySS zaw%4w&npbgV?|?+bqdbZW5*}s+SyEw0<}`M;f3h7`I=1B z`GIhh@8JNgUOjtFRw3;vQGy5^U%Zgr;JVbH(7isvb;W^PQTkq7)tg@Q-aN{TthZ&N z0Z=kwnX>dlUxw5{w6)TR4-pAsF#Fi1PXmhR{A%1y=&(8efK~ZH4e_0N;+VVSKU=)x z4__g&agW{~Me>(+Y|t;<3R;b|c%_WYEvioaZ}m(Mw!G5n9DQ)yQ%*lMc~}S`Z7~;I zZRUH&NHKNFJ^MjjE>3~&Yry~9)YKt1C4M3iIH!1FI*&F&Guc28zSYQ+v`+MjtCo%D z6Gf629^A`u9WOf8;x7e;#62Thl3s+SSX@T;iTI?*5g=Gbx;Lp2H%!eU;(J$$Nb#d? z?531pScDao#Z@`CD?DiTeLIygcAqB`?iwwI&NscCMBtu{ z)n}&L0X;H`!ybjvYTbJ6r@bQ&k3^byH?!3^YzMTPvU7t;Y5Ib}z=4vSc!=`YP44jZ zBNUIz88yFkO}-9Q;AX`8%NZS|CDK+%_qCgW?5~wb$bpvutsbUHo_)%B(qELN0cHxPh%aJ#-3N5JMRccmNF2Bbu%1$x#Dc(G<53pU{mB0x( z4cYO$6LByA7Jz5uMK9NGa8=Z7>MHhiWY-{K+ygO#@I~R1UurFwmi33mwefY#Kg}jM zN@6H;vdx3dAnVFtZW)29J=?E=ZN*8aTur>L2ji@s;HnBw7#fIwe4^);Xp55#Dc2h{5!)n$vVJK&Syc*y#S2Js5?&qyOmI~u z_2FeGFXpZViK7u{0eG={xz!RJ6j0fnPD+7S4@|}v2?{1n@uFGG_PpyeOYU_YS3^R7|r8|^W>FxIoCzrsBSK!t_BxOX!#$=PNrXF}aav`_6jlTy)97`Rf%3%G$oG6wM^9ft9>@Yk1>N%R z1FS}SlzCLLvCWsW$=|uzF9AWjHJ3Smrq_+=Ste*Z+K#!~&*;|ZB+(fG2d+QJB=!k) z_DZX1zJ%0fk(SsH7?Ol7@?J1i_7ZFO_`2W*0hAYchm&e?G#}73(gxVC*&3Y&W^B`A zPPr-@cVxgCNO$b!2NEaRXF0lTmitI`($%1~fIC43BztQn&)V^z zA5qv6D2O9`*@e3g07}F)Wdg}wXmuusG({NBzU z%1n8ut+iBTkpf}QmQ!zU<~)=e@T&G8ztRoX{~@Gt;^zTre{LNT;e1wUaZyoR_&5O# z%m|Qh1JIC*w^5bDZ?)3l9J zCX|sn6-VN26Bg*4qGYqhAhw%DRNkYqNxm|Io*On21YN}Uaf}}V*r$A%LJ6$ZQ=eN_ z#@Nsgw^ut72i?O;N?z4qCOw0%&s^;ic@)JNsPOE4q5Z!k$mp5h8mc{eF*2HZ+yMyF zKfZTHy9#QbC8gQqf0XZ>+y+3A{G(fT)Vx$h-%CnM>xz(=H&8argJ}f1^6GYL7rHLu z5aW+b+<>sca4N;(aeiMfk-KcNyE#E(sgZT0C@NXLJn|Bskt2T={e{)IA=-ddvAG!qbSso2~s-quBW* z#~d!XLG~ZV>71M&GY6+Xw$RmM$}fu6C0M(#b`Vt!U~c3=VmK0~Ihc+Ra5Hwzx^6-4 z?ly`5gRFJ!lRaa*<~=P>yW32UXL=3ypbu7;?^jBhasMXfLtQ?ai7rv?3tu>@FlutbkI z+?ex(8W%f9Xi{d9-ZB&kAJm_h_LsQ!$CR23;EM{I*lp3=n4g%~ze`vJx&VB6&u1yt zl6bzfqI>U3sppk(oGVHNs`)}N5fYlx8@YjhFxtF|Rm`b7w?Vx_ua+MaK66#=%bi+( zPYa~(PX7z*pIX^Dj(;tduLjkushTm!nfvD;X3+jivGlzyzx}8UGjgb+MJ=7cY>TV(5{JZp2 zzmY+zY6;^L3z`cFJm}e;jseJ$L2wAftQ-73Bl})JMr6uE zFGifkeci&h^wY!Q-A6^bx7#N>`-c5op?R;l=sl7cs9%G zfo7~W*wdxYba$J%NzCi-rE4Q*ki#lhTKz8jm`>vh2mnSAbRqBmh7nmU(&#(SrtDAk zN^Fv06cO2@%FZj-@=D0-%hL8;*{h|Ee*dC+03Vsy6`&9841IjAQcTEgvCnFw9Q@cC zvggB18&A3wOh_CB9WNIhA{!N&71Di#Uay}zY}xwjS_QgQyV>td;LU`sW=$`uh;7Mt ztrAdz>nnz+{C2Jx@NfTPHYsnDV7AD9E+ox5OP!HT_0+Z@z}Xy`DyfA8V*S{RVf8BnTKYO0X?0)ySALfhGG1`D1lP799 z`4_D=Dj)w~QYi4~ir4j=HEZ}Y<%dZF5wc9^SbX5GF$`Sjh;VT%fJ;@jyeN@!Q9GHrFd?uWS! zp85F|gcH_1T9Xmq`-Ok%cUQQ%su;ZjrIe<~c=gQGgnRbM(beFV^wmIFbhQvLeJp5~ z>i25>FH17Fy`Q75-FMB;rT0O;n>SEuvp#8CzO20Sz)AQ=NzQDKcK@Qfr0L7M?gNM4 z?h;l?2t2~d*P9BM-KAVX+`l@0_S*!@|0>OwZ-84CdY}DpkYrL?9^;GYNB)v3j zCu>MJv@>wV`lJYsVjpnzh2MDOFtV`9pO(AxltW2BN6{I@;+$^99|s5sYfC!5UQ3B# z?1Y*Vh|;*hkX6@OfmOcs?8h+#+WH=5)u%?>v)s2@zgpB8Jk3}2W|y%4HLv-A_q(uV zJN{ET`0EijmQJqnye%_s`I^_s!cx7nEER{?TirXVS#2%1TwG0@%yvy;F{I+ zaf!c24yVD087cZJy6J7vnZQfgc-=fBjqkPBiM;(P>Ud50?nA2x66|BdO*DLV?Hx}gM!7D`}!H9 z%-G~Gl5;Y|t7R4^=;{5v`#g9otOouW4?iS`s;Pv*=|$R_oUxkp$|b>S&-ZicOxTCT z)-;kCb*avIab@C;3Kp(M_S8Dk@U7xj9KD5nbAu!Ormf4R37F?Iv&G-3 zfxOX{`imEpg^mdJLgNol8Aepyp77?(+ACl^L$|i{`J$0RDD}$LbN?8!V#8HQ15W&3 z5&m@4T*G~3K)%!gsB~9jZ;5!JRMtNXDk?9w`sRmHU7r}?ka(AgVBj2SiKeIX-#74m z)py{k+Ze*K@bUS+**!AR(C2}3D#?`i8OiT~E`niI3c!tJa+WEUPf zO3y9C?E2Q01_0j&-X+ZPH$9d4!i;Zd zx44eek?jT8yA~_QKgA#Ba3i%xNbp9rvjTiSoW1|*+{zrCOiiy137a}U7p^~Kc&AqF zf4M3wx_hvz-kb@>^2vJgSZlncm3R?l+{nRBjxFpXPxh{1r(n{>eTA{~&Kch;-$~3@ zf*%YDgW&`I!p_8#Q0hXW!b=157{5fx33%hu?$N@^mz=voMT76g37@;NC&PvP{tQ;} zaGV-lK#ddy%%B%*MsT587aJ}*nE=;rVs=aAcXcf#)f6u*I+r0o{MW#jg2g-l- zNVvUMf~qoB%F-1XjFz`b-*8KT|k zZ4aOSfNTFkmD)?Zo@MVil>s6jK^6g=hJdQ8$+OSyJiBu@^^dpH86|g@OP)VCR2Top z73Zeo^@mm-oW^e17bQXd>+W^ASNCP5pIy0pDfU@MrsF%-|7q_#qngaNzX5526g7Z| zl(EslLJuV%UFiu(m)=1X2)&4i1O=4dLnjHnLnw+ydhabLN^jC75qN{njLuxI_w&2f zpAT8j^JL}heRlory${JjKtM;QmB;!THj#hNc|T=j8EP|^a?_@}-pmF%x6Uz(lV__=zL-+_$LbF*(uc{{p*~0xpEoTMJS9M@ znU4Gi_n>aFm$b@@*?HwvjfRXqnicBy&3FKp8E{~;v~K~ilRE-Wz$ce9V znP@z)k{z~dtBW1TJ1SZ2%@D>)KkjH>8NEli}T< z^fGOtdaD>bsxXKL9BK1u`l1%HM-{KjZBQ%FVP>g(97#_SRNRUH#e1wufJ5Fibu22D zje71B=x2ENTm}x>gOLza!bi6*=(H1pTAi#sVCRK$WzD&$_L(E*B)NZR@J}A#^kn7& zZr)H3lO%7&*GAEvxg9l0^aFb+dx6#|y#@{8om^G^(dwOC)SA57G-NWrHhy(vUU(Wn*BoP-mOsXL~N=jdBC= zV5|jn^(ip=C&{zSb#?YtQ{PAE&LzW2smQa%oV4JFSTUwGTYHZA{?VBxl8X_8i|(?}7kzUdR4 zd#NeDYlByet5$0YtJCqO65HLwP93ym^+Q`DX~2Wm`o%S~R*=GF_Wi{U!@3SHnJG=v z>-POmhrFhHz^r_{)FB-m77mX^5B8|42RzOF4sti^>NK<8r4^}vBWlY{Y_<{fR&= zhGWX`&!i+Y%8SY8aelZ~-_=~d;C?LE(VRuR5+Z^|k$(htN^zeki|VI9*mb41w8uK* zpr|B0OfAJj+76xY2x3}%vf)AO!5u!t80ktxo+=(;5)g*Y`YNtPee5KquZ${>GHh6O zx0r%ct$begI%*_#G44*)y8?A6!|UGdhvCMgK&H^X%C1=AC7o}aaj0|j2F8@HStK2* zx<=DZ-gM{i=P&Adq7ciT8FEtnofHjO=g%P;WI{RUoKM4#pFFCJg()m=W=`h}hP5va z++w`^ZU4=udw0qb&=1|8J0^Iqvvtbod2PA`1G=3Vy*2Q*6yBtxCz~|Fsd_IT$E=|K zGs-rgCC1aKh*6{K5SM%%AwQ#y=jLI}o7Gwu@=A!gZ?khdKkeT5hhJ~C!KvoV3z zgmuTd3@>Xz!kDA_n|g}cfhA#(t=yVbkBN|G))f`la~=JWB38coD-0h#Gmf`BQvRT6 zt1Igkqkb_E)OM1TE$YS}Gy;BbS%hrTLFLRni~!ZT?0BQ>=b&ct!oz%=t4wr((V3qt z??+Bk;JWrX%0ENkKtg0iKUv-B{2XxGk?FjIcgqv2%Ft>y!ghR|)w9X5M9M*9@U`}VJ74KV;d+-Dy`z>|N_q$zBnxF*`n_|2cqN&DHN zHmhb%Nskr6Cux^L%-DW-`rlXZPXu@2WU<^#9_D+xUwg3$rA7Dc&7enTT0KFkP^Fzg zwsm+3It#~ph$Kc{X=g|73kvrfynDp)E3gpr+sL*@7-IQ2G)+9WT*3V@k#HtPGtcpmR#6Q_FMU$nJw5r^T~j zV5?!PBnjmw6Hqa_BvU7Fly9%Wocb^F|DlDPt5Myo+9JI|cg+Z>!)<;5`XrYc>n`FR zy=e+glWJcl!yHl>t}`6R)g+BqY#FteU(LlK*BJi_D?f4fgX5s9WF*snf7iVj(##M_vHLo7y?mas(fd9@H3dOHHKcLNjz)V8or4-`G zA1K2e zK?)CLXt1R;x4YR|hYmoB}Xv(9LbGWVzI-t+zwocu~19I+rcF?U`Doe%*T z%h=twN0KML#O`E2V0l_MrN4g3rk#|aTorgxoq!O2%&V(HSmY;k@Gp}AnFr~HmFvtM z%I2T|Xs%qlUOQ6-{jcZSCp5fv4TW}-b%bHdfNvS%hduo3D(KAFV>4cRYLNdr_xb}| zwfWN-nh+^iv60!LbSzzOg@c>j$Mg+@^;Cme;Is!Q~{Zw_rPqU%QUJD`M``7T7(WjgtcIyxI{L845kmm zLtJul|JQaHlzl+$^+KayRNILw_ASnr{$BmxG4J?|FZ9aGFZ`s(bAE;azp{U=pl54Q z{g?3G$XIFske8er$NOZt8SGiBS^WG3p8ir97Y#A=GuHyvB!j^jGJ0>&8)Ha?W@%4u z?r!!-YRr&DsAJ{k{%d*mZ8gS0*Lkd2j64-t^L1-Yp&ReBbQ$7NC1>Hu5&} zf2QG|Y!&26HM%ZArgpZnISdvI5VY;JdzmZ9Bf-VDnqNO9EM;J}!`G!IAK!EF_q?2>%-(8kj3SS3 ze9`3$qh{!c78~L^vc~Kpk4#=lSpMF}0JYz|nvUkike&adwYWOt0e*&Hh69EUKsm!h zNDFz6@~@2K$FnPZQM-XL4!N>{2lQ7dM>pRrlMxw6nQZ^mJdKlR4 z;7INNi3g~&r8shA^Q+46Su?hnK(Q8s*d8$UHf&i~tdgfFh0qQ8r&0XlHt5W0We1RI zu$p1p{B?|pN=m`hAL=PcoRY2PQt)N>HpO3f(vRi1%#VHQywSKK+W;V$*PdaIH|@AG zFJ35I$P<%wEi z(($e4nd73+ttEP>dO8vbu3yku&>t#S!tV^?ru>R-3IAS<7f&B2cwpL6<@6SV6;)Ft z9;Ic>5>+N!K&8-jndVicj+;OttF_Ya0$mX4F$XC-)BP1juD*|U&seP%cY46=&i5kK z!5ImgLPSHluC?YxyAG-5v2!w7cfhCka{a7K|Dn)-i&lnY@s#pZ?vqt@PL_MjY4(?R zs$1|=g(X0g^BqX_x%P@C)(ZAwAYKAAP}00cKcTR36O)Xz*!r!Q;g+Odr9FD6H$xb^ z!IIgYw@&bB*n1b7r_C5Dh+>9eZy03sMmf43!VHF_?hJg{E?rB(2}qZN;+;M_35L+T zI?y%~5b?Buh8^gX=FAAQ#PmxG_W#~MxV(eh$g{7iV91jv-3qMvv~wly9eK-)!jEz+l$Z_$2?jZ zCmNYZIuub5hW$<#L5yc@g}7$2I4!_=(UIQ%D86mO>pjb3teSCYL+=OLpO-&Blt!hX z82lJQxlA-Y2WUVt!S}{q2~aQbD>D*+QCSkuxNiS=c3pS+DAr+Qlduw%SD>=+!p2 z*ywfRybPp`=F&+kk@GPx?KEa5?hVpo^axP{r~CU*85K{`$TpTI8)dR|a2G=i!wV5@ z(y6h$)R9(U#*11yr|9RG4M2`6YrdKruFQ;OWJ8t3-FZ?GQb~elFZG^JDk46g#+1Sq zKH`N^vr49)J)p>1x$bvL@#N#*?P;I4_Y(oTD3YV_>pXQ5=#4rk6pU5x&|v_2ZKdw} zC#mbE`U%}PkBs9TwrrxF&81np^U|&%dR=-(&`h`o9z(Ounw5}Q@;Z-|VUqa@>F?4A z;0dT*Y1fXUjg5IPyP1_x-32gWR>xf`R%jLd3cS$eHinoGDttrs8Z2)pZsG}JxPVtc zQyck$_LXX1HKeMpb+C3)O17tg*%712xt+=xP^@1T_iy8Nt4~nO3GJnWDHtn2t-no~?|4yg@3 z3}1B(JeVR=^Wm!6=DQjGi`e|bKq^7~#N0F@m5`pJk;Q`@kyqLG*+u8q1}VGT9x^&Z z_W7z`|ic9FwlH+}AyHD4aM!LaR0#l%_gsSSLPX5gz(ifZF zf1P&nqX@`os`e#FTm}D-N>atzJqEUPcr#;M-sV{c2ZvSH@XL{}sF|9^(Dv??oS|cJ zXBMvKFJC)gZ|FfE;D%5Wx>LU8qT&0D0Nd&w>AZ#M*0oK(rR~Z6VrbFWdn}jXAUbIe zSiN5~wHGcUFvxE(=-Tx*%xij2d+{FnV2|C{XSEW~h(DC}+eOXpxbMD*9PB6Y+ne;Y zTx}>#XR@E`Tr_bt?NoXiw`Q&4(sYm!8i8n7tGiTBd3L@@_)8<1UFw`?vYGU4lD!o# zT-ON)LH&8BwM^#pQ|XQ7vB!*^SpZ|}9KOz{GcKvBi{dv9%Atq2*@c`JDbAtq4o*I+ zv0D$Tgbuwkx}fhfVvHUUFj^=kwx0!aOqN=t%eRD_cA3U{89PDLtJOXplI||mb@{ts zj0cj8wfZ(Xp*CL}0u98O#2&sH&f0MSZ_gLW{;|Bg?9krPk=~#17;jk*QhOXdG<@&v zI%JmxBP5Jn*tk!~t8M&{yDX82?yw}n$w26%8|>~>I3zYxF&4&g3vUNi>;fmWd>HRx zsz>K+(i&9B0@1gZTw^)UQEfqt78$do9t)G)gVlcMxMh&P%l_%-`Y9>uOcvAPlKi$M=Fb>i^g6XZ zPI$pG%XwrJZqiBY&!f8Eevzmu@NOEgOwk!u+_6vD&rG&8O&?Zg%_-4PZm~Bttx5&^ z>#3z_ujG~R#^LiEwh54{|jBU8oj3! zzAo#UG}DwQs&w3C1A*akA6Yue>@!}&K2Y6e0XThR(ZfzriPAh1gvLWYi05f*VVP&w z^3*ERfn0MEc!y|lST$g%>V0-8*1dj4R{r)#72S< zfFI%YfW^U$n8xB2c0wQFZ143CSLbU=5}SncLyqHaZ$F8Ib3m?Vc@F;d?su%p#Yo&x zoc1Y&DF+w!!hFnl_XII1l|G09r4I9+hr1U+O=CuGmA>P~4;s#;>g^Z7cGMeu8mur9 z#eS%>B6%I69jM^<9*kxiCaMqewl3h=*T|g3SpGG z89)Sl4zeI-r8^g8-zi5+U@2;ZM8e}HN9@RvoEz;ix`p?WZe!a6!5IfxeWs+U1XlR> z)^cN$WVBD~t`vEie8>$y!;B~?H?PuZBd?-W0TNev7@ z#l4Pvidb@}}vTv#TzZ z9@_^mNA*3<{2QVFTv#ao5%qvjjq&gby>d$#4p8e;_`v!M2kPKNsi)>*-I^Gmc;8^_ z`?`~6W&OKKfZtO4S_)LJ5>lBEfdbHG*35Mz!YA=|xe(5Hlq(V&W15Ma*PG`J3;>?I&f>I?7Od zULkqj_N>rxx+XnCNr7C=Dx-Zsfk>*vH?E}_%vvxI1ogXA{{O|h02ru7Q$khdU&1cC zL>Tb54{DR{|{1y(|a zwO@i;@V+&EXRW+J-Z)ZR7m*5P^6DzmCRa0guU7HL=CMZks8ZlnY|cn2)&X5acMHsi zA=>S$LGr!S@EB-xbW>>VI%j?~&$1c=?QO{G_Q80XBtA%emM6&1>+iV(zW{tM|A_;O ze-HGfyMdEaMp+Qdp9%;~%KBokN?s?O>7J>wTG`iG1#<#S0k#-S5mG)HYDpC)$gQO8 zWpT3WrkYxY+&!yF_Rk*tW!fCPBsk3*Ixl#J5cd)0$eGIuWv@}T0pLjcN8Ln3XM+MA2-^w7>6 zYIAIjDv51{zuarqYG#-JWRKQ}c@VvKJOFiMi+3)bg;$I?)?8cX6q-40xsOWZdG|ZZ1@nb2 z7e@ElggkkYK!Un>I;IutQz(3#R|&=`f^x9jEi60tq;<5 z>qa}XlKd2EyU(}SDU8E#-<|vN>DC2brFE;_#_gmTwvfFJAKr=;EQ8fN9TV>y@=#e# z=jsr178N`9ih#AxQI+L-dPmknmZO#2wzDuB@U&0k9HN=;IBFXcqocmf{x@asrAcgr z*F=m`WU9jsVYe7|pHHL$BA4FuUz;*HOq`3pf`+?VzHFv#pEkHI=Ngx3WX2?vi&7)c zeJm^Y>e4lht&L(e-l`sYRt6bxN+Bh$K=1Ox!-fdwk$O@6+MIX>kGz4CNNCs+e|aZg zwqp}EK3~IwWL}hCU8_%EQ++EYdBB3 zaD&=Lpn1?RLa#QpWo#$P0dR|unApIFT(Ki|(ptFi8+liGlwXQ+sz{H0*9pOPJ5DZ~ zh{WG5VMcgoc$Gq-Af{bbNfre25F zBMt*_F!XVzoLQTc9qhclSETLQI~kyZ_9EXAlG}1<3*XvUP>-#Y+I!*0oGOW6UAHKS zP^nBuqAQ#6cO#;;hw_d?S8E(7$|BZDXJ_@?T5cQGpX+8;%V*mIQ2KPrq2#QG;y@$z zP)N8K*TTz_j8qc;o-ox;E*xBr-bx)dn3kgGbF_3st|GpQg&Y(Fr*JlBo zap0S~KH!tl?c==I2fPmUrlU^E2D{Q_f?0yyh{9*myh3g;X2NvXlbFSd^LW7OB?xy} zln3mtPIjbrj~C94SdY6_ibI?v=PSxnbNy1hu2OQm*8vr0F+p~`^6I9BYQk+Gd)d~x zOJmj8boX}u+~}9qtVG={BWPR&h)u+HR+@I%<-rHBPhUCB9ubl3UwVJx&1!;`UC;Aq z#slDVx%-~sY#QI6>DHDp=uS8EU4Ez!u;FVk)`2s7skEJ4uXbwH64BJ0R@EEj#%89G zSliKdw{Y>bK}*_U)HQ@-FxASM&EDWo4(s-6wGH`Zk z*|UEg%V)Z^>p<@c=Cd`86 z1BnqSCH-84%e(K{@(zZRXuF%MYTz%(pnR<^&%avl+H6W-fg^y_t*HBw{djcwx*>W>UQ19`1Wq7 z)3>gCOu;wj5}Z)6M~goYTx{r+Vp0x6cvC|%P`MObjvQ^t{xkZXgEyxKSI$)mYi{UZ z%5r&hA_)7pym*i4;jxc`GEUyvaT}F>o8zbTnEE`MiuLK%JFk4Iqwz^@?4}#F>W;bk zTS>ah?t3m>j;N?$t1Zj;s}nshS~luV0mDVgH`-l1w=WmM$HZgJG-H!kvBiaJ4T zELPECZBiiCXNP6mx@Ne4Ldbnx5q30(o#H7c=0?BrESlj_xXL|IyvMe5#O~*jugZD9 zX-B^rVZT=D6jfzfY6m;?Q-I*~u{5$8&w`9-dc|QF?Zv>B2mn_mK zF|e6+${FoQG>G-a^tf+o6}u}sNEhVmOcvWJ^Q32NM#f&3ca4}U)=Bdt{Ci={>WB}M z`=@wmphaX@J7E~<<5=Ef6AT!WxV}@%3%*4ZbcRu^t)+3o73YB&l3toa!lb$vm@&6u zEj!rKt9mS%#(ye>zIEHTg;lbw%eU`sRTAH@I7Ad;KM(G#X&_~hl=EWr@7IQygW4kj z?(|%;#Dr?(GkdDBBAk+R#k@4Rzacn~`g{?G4yiWvXylAZVx4Nm_QQjQyG#^$%a*N; z3IWzaq5y9#uW9iuY@3upjlLPDc?-`Y$&W@uCvxD$P`pUS^lLaxkoU{F2O<$ph-whQ zvpAyezKR%Na8(c?s?@x0dR=4Vam2hwL621#JD7cK4CK>A2R6zTGwRbagGNhdHd=0d zL0$bb1ARH$+P|+JYvX|3BGT}Hk2zK|HEoc!-#wk4XgNln{S9Ht5w>xrEQ)ZN$V;H4 zFfZVt)D~7>z}8bZL5ahU7+6{w{$j%}=f-4sOuF6Sbe)(jlOR89X2~?JWIIEBw4RAc zbnc1V?s@lt9^vS=Kya1m`n0X-le$!gcL;(SfL&)2MJ@D30fagsA@Cr|4%2W=^%Lga{^?^Qp!ya#av?kCNM!lcyuSJ4z zBET02(qqa9DQ9LWC>OFLOLdO zeR}jRS2uxRre{Q7@M9mjo2k&Z+u7P;D#6Od&|3H2hzS=n zeJVGji5jW^LfsKG!+1Yp(Z*JAwP-R8<|^UQt$Apnwz9v)b5s>3fi?G!Z%4f*9M~a5 zyk{WB9;gZAjXUf{$kZRRgdYsv^?`Y?Svh$IMDAVTM+X}=uGeY%9A)jW>E3~SHC``t zu~q$hY7Wx?vHM!{@O6DIhayLeODA%MhN{Llz` z+^0n(TZq%&P&TgeFyEG!<)|k3A$PLo5K3zKnVNp0+;NzqCXtvUmDas7Q7~TD`S27= zT!BfMZ{hN?r}Hvutu4)of5J?<3#|>OTRK6Oaa3bWzmE^YL zrVKoav`EKhiq5`zAsJ{VHLSYK4*_Ds8v_O?E(_QK`j1F)NyQ{KKvul^K)jB-cJVEI z7XqImv0KL$i}bIv+CTIjDQ#JMhN~rKzJPf;C$7n6i9KR%K!Rum{={3Y$@O2=i zOi!h4MmA#r4sOX8FkX_KM<+rDd($Oy!8z-o*sQ>;Qx#dr9SVUlOGx= zL)@SU)$3vxt2`$0tsyT+{!6W%WSe>?0*EAET@gp8MGm zPI>sJ3FyfaYb)o5OqUBDKBbL0*3D@f95vGYbu|#&Aj15x)u#nSO1VcsF6Pct(i2w{i_k+qMQIqS58WO0H3BHCktaiu;Sc9+(*sGE4Lt;hWCo+H2_0+-0 zFCVowbw%SO3;l~#H5WmZRxQ^>$JVW!a!Xux)(gdq%)H{*z?1Z;AL$6{(X1gE!L8uR zkv68+s8ZbO4-{L5c`Ig2Q(`&oz4T^_=_b$de(z(vH|0PP%7N+r3B6B}PX5Tu(3w30 zHpHS6OcdNG*D)m|BX73uqib!K(_9n=R*07jh}mN=+=5!^qk%lqy_?fQm0RWFRQUe) zAWR>OmtnXuC(K}E_;%bb1lZPoX0}Jqrw9J+c+DkZ zKT6ktJjL?H{&=Jnlk3^5vHOQKB-O-)7Sr+%1}=1siiRl@j-&^gP^>u&jJ%IpnlOIm z(QLVx#o>vRk*FJfCqu(2YtY&5qU?pw6p3!cLgo&i48BVv#iPs7tc7z2#*8wBU{wDq z>_#!Wp&DEv%Rc}KO^MPtl{n2B>4?8dkooSB> z16?C-h?^YAohC-GJFtoxvspSAAhOI0%O8Z~_5oo+U6?>{KXt-57O2kNpGe9aWq_dZ zMV7b-n&d37z74PYi`Fs0-g**TXWGsd>pEM$pO5OB5SSn!tbeb)Ub{3A+5p}{I%vR4 zUgz>C_#$=QpQo$emG-E` zYn$ah_uE2hyoiP>gTC>yl62k#R6>>ri`1sZ{$E|=9|ti0@9HlP`0Y8B{}a=H7ZdF6 b`=b+|G2&bK+5!PYgg+Gpb@?K>dyoDfmbZ!c literal 0 HcmV?d00001 diff --git a/docs/images/minimax_system_diagram.png b/docs/images/minimax_system_diagram.png new file mode 100644 index 0000000000000000000000000000000000000000..35ba777f7fe7c81def2408ed7f7fa4ca6034ed8a GIT binary patch literal 235638 zcmeFZcRbbo`v+bpO1TqdghJdZTiGKctIQ(gm?3+VRrV;QtdzY~C^PHGrh)A2O=Pb) z_WWIM&AZQC-|pX^zsK*7>U50re!X7T>l)AJ^SbU`keAxOk7D1B9Xs})KPRrZW5+(P z9XoJI@pi*I$)Zor!aqB$6{XJZNUJ+C2LBPidilJ8oZJpZc#XGX=Uw9+ILNoa9}4)h zW5=%8ojZ2HzdMolVo_hey3Z?i*Vos$q{ugBKB&FAV~5C&^WvhHZtk4y*&7+8+{626 zS0(LZylF3i={3}Srch6rJk@EL^wp#x#e}>|;T@(!TEXh!Yl<($O~M@+y-e>hPv5`i zweOxA&AzRbsFEc+!{Iu1j+)Y%s2O|V8`Idxkw|PL*7^8Lw$14on{)e_tw-vn6kBYr zwU{m?wK*uK3uzZSTq{N|r2oK5lEL96S1Eb^3y#n~}1P zQCpIq;iV#%3k&F!Vo?pcKYmpBotJe`P07l-(a^RrTbUxs8UXiOr;pzfjJT zAO9~a$05xxRZ`|?PuAdmkKXxMxh#%SnX5Q)yyZrmLf4;c-S_VqKeT+K<*jD2hqzP_ zcM>Vh>eb-}!z2nk0v2T}+5fO*Mnx#1zdK7f9~o7=(n$6YD^NLFftm9V!#~-(_dlFz zsy2%MwvP=ykZk3%8cu^3>K)>s-2W3-jXD zBA!1osxLd^j-h8SGRfc2O`+baxG9H1s`hoXm|79)lC?b4{Q*qD5louOVm1n3g z6$N|k^ZY~O-hPhQ+I_OA;VY*DMB%2$Tr-Mr_B;Mx?D*t9aoN#+Ji?x$%R~R+ENMhy zQ}HLMc0QiSSx^oY{!e>nub=~Rhv+O8`CG0lSkjTs_Jfjx6nVh z7-~9b9IrZ0(LY+FNEgY=^||U~xy{ih{%Lm8w~-U+j1e(AmHr+no{}IhKLN}2t^18Z1nr~B~~9_r>}j!D*VmJVo&1? z;0x;|72^=6_Z2!y@h8+%jlF8|m(%38N%O%)<8Gg=5iSZPu3jd_j{B=C5{nVRxjQ}A z=->Ey%f5I~g6rGUkkAoob>>317r@grSpIrQv8Mqq|AJ)uiF*0M)Mz7jw z9^Y`tdD0Xc9r2VTpIO=A-KnqED?)kb=1i7<;}hyFj;Oiep|QR!p_6q&Lc6H4UQrou zOed309Qktoa)$_MIyrg3kpDdxxswtvzkXAY%uNxcR49gXchfX0S!qTh_+%^Jw;wPn zK+Sr_Y=o>xQ#2;Kb}p}MG3>rqZdkg^eDnQypJ{_VS87M6>*`pdaa|;zJ;5om_$#Pxis7du?zqQvyjO7JCaJ#d{ZB7Mp5P#MW->)* z{j9&i7LXN;xh_L`VJ9hQlG2L=)$n@IO{GkcDJd2|sIT@{zSZ3d z)l|HkjkYxtP5O2sU?#Qo4qU?cD#J-#6-k&5g9w|osoW6FNj1Z$AW2_}Acw`tu(Z74 z@J#bov}tPQ0^?pz6`h*UQ)7kLI#V;AslD0uJ9j=d-y}l+yqbTtR`=Eu4Y`(m#OYQ; zH%itt)22IfOgkNCimJITN+CBD%K*o-g3}!JN5Achad+bA6PlYVym9(^R6iZ*PrvKr zKBxKjQe#QUX;tctC396w#vNBfI?v`UiR#k~y@~|+ zDE{~d7p1A-4W7t&qhmwFx9%wLr4Hhv_>eSQzM)3x!Q zf*Yx)gw{X3ws|i>HniHLAbh4a^b}o>6Sj8j-H904^pZ^n^wl8cM%IGKhXi~d9}ssa zPcTutu3yieEvLj5X*(+*GYEG;4T`C8ojNYB@(!`?|7KACBAV!tvJqDuL3J5Ka=%F}wyez;3r zR~9`}vgH!hR4|!Y-AHa28IT|wriiI^S(^&cEm*ZG3%x4fq{e4h7isfq?}?`}Y(?2B zS#?nYBW}%j{gLcf&hn2vebl%z5~H1}iDS3<>8`NssS>NJaA2}+T8`$aBDEpfE_|t3 zb^LkNl>u~3qI-@8YB?{opv6*3HdpzKtY_bnrQ5PF(z0`Toa=*6&1@^dM$J5xVa@w{ zVf6iz&IP>n!i84#rh6x%LvM*65d2g+-(eV|YAI$FF|x;9n&jkGm-h4I9M)fjh=0c3 z&f6mTUV@A5de+yB^jix*y~1yz^A~(*kv9~qd#4)DuG_B5ym~iZYi=d=#;DZW{swS0 z{Ch~s+e^SGtsi{2a`&hT%Y^>eWg?ryWuBh>4S4xJc|x;q4%ye_v}p+xtxXqVW zq%ud^0ZMLW&1+@2*Dt4L@6N5Hpe&19o6J(khilkG4R`8@5v7_0U9yS3WfZ@yN%W0o ziSh=#)2xAy6omNNB-~z{jcvr$Crq6`VdngcTOPZAUnNb;Zt8G`Lto;^uvqCx$x7*3 zrfK|DLf@3f8Jdn}@2GZ0x^7Hpn_k_!@tS(`fuG=9B{QLWP{3@~F?!5%|G%PQTsQ4I zP1B34uIb}kC9P7oEDEw~zjBv2jPXfx4UI#S9M6#3o0p_J?|bsSu@A(P_xUTB=vw}C z{=~3(H<_A#fKw{xEu%~8n@c72qERaYA;MHnpUUQ2(p|V>qQJL|H7hBwX!-G3X1|v- zhA`1_;`F`p*17m0HrEvy#3d@}ulcZy(#aUb5uf%dUpXFO_HdVsOjckyH+I+VzbeZT zt66e2>SEEAr5)kj7B4^C8Q?gX!s6jb=f)+iALQFVaQ2@0Xl}c%x6^97o_{=ehvIAR z1l;eFJ`l}*Or8Fg=v$Qi%?l$(Q~(R7hZx;1eY0KK7WgZ4e!R4XV?HfW)^&VuQv$H) zsh%5sQMbedY{vuZz!)Mkva;*JwW3)D^~ZA9ikBZBzWs6{t)O)ufiFgeIYa!uTTiGX zj#atJ)KC@pA{8cH&d#aqCV6}BCEnuI@#MkO%L9VtJ_ps#n(n?Uxk2SPd9mh&L#J_~ zgmTA0^@b35^U2i^O-wtFhmG)Z|1I6V)&bHEr33t*zs_Eb>2I&u4BFFuhZNh&&I z)CM89Yhf_=<{w?SRE1c={@fnBu6Q|~p6t54q?o;1^Gym(p{Ml5%~TzZVXXWQ>ZNCt zQ^bedugG}olxxv*f5h`vB_l;wh19*tC$>R3t!So&i8eNEm*FtOD(dcpMz8$|iCDwhgXD6#5Qf8BR-AHL zRG3=pLr*=q*^Ta|EilbHSZ@lxv_3?7L^CawA2lV>zcs>Y!IqpO|3}NiK^KV}lQB3} znVGgX9b)=hnGEMO6?JoE!Ssn(!qLNFXD1jn^KH&MuPw@@7cHW5j;9x|n#`^Tt6Q8u zaAy9(Y~irzu@_m{3dRwQc_FP8vy90p50i)Mq9SyQyDSRKh1h(!92X~b6Y8Df%Ka;g z(I?-Zz<9H<5b02mU#bqe8myLZ`6#&(etgbL60?0+MrZ%L5{O{WZF=L|jvg|yM9!+~ zP_EV=EhhD{PHT$V7?^b(hlc%7&_vXjG*jxO!p-3bOH%&{{!k7>`Rm?`Le8trEzE5d z?F+bGNHktpWYh3rjvbPAcK3_7_O5xxwM+bjTzH^G(L$@$>((UiHL_JQ*2U?bf@_y2 zip1^`0j%izSk>e7l^XfdO?Rc|8 zqQKRvz=4IgWZH!{HQGcg{EjN=hxwQfjBqWE#JFmihK8!fMY?XT*r&X**PQWY%{S_h zSunpPWIy*&H>Odf@zsW&5dt?H;e-MV2;~_2t=|q&CR=bbA~oQbX(@ zPa3e-JJ4%C5|zE!X&N%6;xJRo(^uGje44sV*Ly{bfbZ$$5*h-ucbCWIWJkhh#O{7U znS>L+aWFI@I=v{!btLM_NyziU=kDv=#=Qt8r5>qWlb0uLrbaoj30E+}w33uMF`gOi zG=R0gwU|G>qTht%h$M6`hQm#Vb9@e;dfZ!*OARh|AY5h_gLlwX0@{XCRgiXRj- zE_-eZ9I@#F>wZG#$&3mUO}5l!cq2w$cw^qCP4h;O3Yov2xowaXwKJzS8qBGwQHD*^ zOy%X4H(PP2bXFoMrIY=qN;?0idj4m)H&)w9i@kIE=>G6U-DXY30v1%2d~# zK>f}3vfsnWyvMdpZX&(JrNfSkNzZZdB$909L#kz>RW~vc?P$TcL?2YCn2)U!7~w8i zskLm3uq;-?xZ-$fLEgzwZa)apvzK0-_R=L82OzGw+;yL z+Kx6HtBvI2h_dT6Zb>7NAIK^L2N&692L=*PAEJ?Y&A7N%-CJFW^gY}I-CTd+&6hVP z-9+N?@qQb@KPN$#Qc6F4c-@o`VG({yp>u$i?$HE8V&p^a-kXDknilFBxA>5a7OBQ_ zf-tU7FojX463o0N8LOus!WwveuzWjV-xv|zI)BD-Q5Mcv^FyOxACkcM32!cUvwAS6 z7tU4b#=NZ9+UVPg?*jvQn;*W1sJ$azCM43?1~P8x*`@1ovJu=Yi21*r>Z4mTbP|o% zxF^zZOzh6nN)`0US=@V}+CGOweGOxr?LH{F__O5^dl>5Vo`}AwOF+uPNpZ%(hQHr! zSEF_PnUG|8-Ve8%Q`K`{4Xrp>vRKHeiLd(+OAE#zscO;2VwWf53=X4;PQZQ5FwftH z=L<44r-geUYt0#dGLVv0C;ddyw@Q?>O~YEQgWJ_>e!^bapk-dE!KWCtrGr-J20OT7`BORify z8y`;ypAYcl6vCH_7UVOBkvd$G$bK`)+m1PB~SK$xqt}gfOWw zT&6112d&q-`ZiSzch@8N>055>d!AzEQxH?umsTVBtgUv8x2S4IwQ5+`hUeCIl~=6B z+#I+^T$ZG&X{#JPbC=pB6bZAsYk0)w(~~o)WvMSqC6A?>Yb8iUv%KZ~|3 zT2RDkX2f9xe`!Q7tZDicHThN-|M2317|rJEn}t|yk{s^*O*}BK%F-p2JE0CE8DjdV8ZB?lj zKg@C_Exyzkng=m~!Ekl`!vWXOEPwfMPKLKk<8(};=D9t=#T!$WZG0bZ6l0?z%jQG% z{N7pLHw*Oi2_(9l4+Tqn%;tO|gNMijW65eudSiQrUK3=F)w<>NMg?F>Beo8M-`&*k=`oKbsw>FmQhlwfJ7p+fp%tnr@C4B+k=Gp4iEz9YlR$*<9zkdvJKL)~1_w7PZeduc;=pIbuMdJ;} z!pj0>9$!Q3)oI=4VdCX-p5lP%;Vx!YVAESQ@mzTx7-flG*ZnPN+8qVt8cj%0$NtH>V>Mq&XK8`Wk~u`ir#^1Ee-cI zxA8txS}3iaz5W3fRb5l8ZL8V|?bT}S&4+e<&P&APVsvyjaU3K?jP+^FW$)F*y%(15 z7AV#y5=~;!Bz>xGoi+qD$9crH+THJ?_Y|SW$8$~lm}qMLi@IamYoSGGFt%`Tt`T4z zkr_cIIPtQf?8o--O@cu)U}l*E!#E6wnKg)Rp0enE$q5m5o%*9*b#prb_h~Z;(ie>f z)eE$sJjxp{-F2Ww-L3pQj)(>z2zf#n4__BZN911LJy0K(8?5cJk_~BMJmQB_F`XuP znq&ULTPyWrFr> zSdfk|%RFCKNUGLY71Zv)h0*MRO4F*}z>I-XeK`&4Yli4qKY{7XEQ781Y=sYy!bdRi zRp9`61yiQ2M5%O~I28bh%M}bRA^Yet6-8T{Yf5Halv+crnnvoYRO(BTCCeN4MS}Ko zx6_zIeKZz(i$EG602|{br*4h<{MJ&&ZGhn|F75JZty;}s4XbzAkjL=Yw^xdgcLh}> zGthhR4bG$`1TU{yG{lO^P4#|s!S70ae#pPSCm(e)f-(|w zRuim8IWn`~Vv32f(P`qxyP48nK5m^6u@;=LT2hj7D_!5ayw2n5^1k4VK4Fw=yr=Nl zu|6@+zaHbesu)0+j^D?u_#8j$_rM2Qs~=LVnw=iWDVfrBZ-~iH&>3+^o8?e+yreZ^ z&pa0m-kvW@B~d=g>Qmp0>(<(Q4M+V`h_$pE17s!VO4fR2D7Zf0$sQH3GwX!Cv3g(v zV6gRYU4%tGgJF%2PDA^ByL!YKJA@3RPZ(!BTfBHzCpTThMK8q|}<>_H! zta;kOuJ@9`4V51$b{1qDojTyU{94e-c({2F)lC|S(yL~g1+VroeblMPlidJZs}V^a zj>y{7aMPNmgz&_pDH<`owtDx!vntN*GH6J2@Dge$vhD+ zf3IoRK`(K^2&YdnxTDgDGE832B0w%%!}xh$oYj)zcJ$u%QaYlLYH2nMOn-mtp9s_T zHS)lry~gfAT(VR&Go%bL+>wv)l)MWzqVHpWLwnuEYz4If5$ENTUFNx(1moV5aT#Zv z^xJ%{k5@6OjjFCvmBn%0+M-n-yF+Oay6ae?Ttsk|CzS0O@jb)#-GSF#Ki#7K$R&Ag z6f);v@KnhI&a`e^sH=MFD`)Th6^>Qn?sLKtr&kJ+?tC2Jyx>LW_l)}^*kd%3Egd;R z(Um6k!oZA0`(%J77wT`ihSMH4dNM13yuP@Ww@g*fi*fcJ$!=DZiLV!0WA-!;yxE)E z>tHJ6my#j8F{XgM{-L4`;4otK_6ld(XB^RO&l97=*(#jqLbC{ID9m#ql%8#K=%aHq zImSDWRJwqFYiDfqm)){{V1?r&w6%`qyUbJWXMW8zI6-hCT#xrV57U6g<;S$%3;UG= z`CRv;YU==E%dq=yxfz|z)j&MANI`om^o<@{B^;yR-FJs~D+|+l>tDfNu4Lzf>hM+k z?jk7~oxGX{0nalZ?B&SRE?byq%cSZGll=~|!J+O*8+Z+KPoTBpC%-LnaIj>`J} zj>?k(^x2%e+3fCT7Tf6%7CRb~y6t4h43@mp zZZ(9UfA?9&o`_4ft*|I1%F7|O0o8FASLs_~_XgQgc z*4nXm+hH1ch^#vw(eC$Z13T}%O=JU{g~)3_=geOQ1^+Hc;x!YVtPNCiU_f!w(Oq?> z_c$lRVJvA-RUUZMu075SYtfS*Ux}vZV4?ezoe`fgC`mY+7~>Ldc}P8_P1#Rqb(B;0 zjaNP^kE65xx=-d89HotKGp%4c+#-K0VIF{}r%QdVC1&h1aH3Gw%=@Z3)1zF0DVjw} z@kr)hMIPSLEEc{B|^>8-+?;t zCD9p9dRSukG#U~3hgPaSnU#*{il}^_W*b@f+00lvz0_@L2=1x^|6w8!Bdff z8T~AI#yLn?TEI;JTZl5rgePlZi$| z+e}}$X4_+846NOyAcdK>)K1Oo>4)S z++q4q`uvDNh?tUk(G=xN zo2k2eD%(SrC7D*V*tt#JAS{rJc~&jA@n}Qb*s~{oNv2-h7q#(D`DT-LDmn94h9k>{<^1oGnKx z5aU<3_vVyaNWXn3!x~l~Vb|V{DK4H&AS$GwXEp&(fba6X@OY*JSQE+mH5T9BETr_+ zK`);GNU(1b5crxG&%7M;%P!kZ*&JR+X<2wN#rrRR^NcsE`-Y-!^qvP0*5X5Ur=t=_ zyb9%0q7$a6IVyE^6zVn9R+YzmM>{?x7)IM0oZ_8N6y7lAAN1i+1Tr;gKcRaTARRTf zwX;4lMSYN0XMeausZyXl_O8paBuXJf%^V|&%tY3|*LCWulO3U65B z93b7J?zSg6z1X4CvSefKC|myMmbgiLA zs4M5m+&j7z4{t$M%(gJoC)^eZoP%*^wlTfxmgCJ2o@AuE^bp)Yd$q zqypKGZvhd+r17|p?wZU_ay*Z>HOT>&m3ADTXo+H;p9H#R|tXlS-hvp$r#+-59XlgP6Ar3ZFJ>#N(xu`e`0j*s@Cx!P7`P zgtwmk-<0axJF!7{e;wgvpMtE)o*NJYS%kFdL`s7TMl-!dO{!45AL(E6)?Ry-CV(<8U71M#wZ zfDTbCaoK2u4^cNGDa-pnMJ~h!*xhzkgm=W(j?(Of=qfg5E3!*6b`-eQi<4~)w{E&b z3EZfwr6aWVBvCmRkjha`;0<2m)PC~Qe@{Vio+nppSlB4d6kqJTHf+k%=VXmGjIwQn zAj@Chb>J{N&BnNDiL^I!<~av2C&gu{K0H!i=4VLKW&OfL-ws@=sZxdCX4A6ZMJ`~2 zmG@=a*wqJ9l_)tI#%<-0n=8#J0($`yw^q#Ev^}t@SA&=_*?H9(^N{mm>6DWITrl8GbycwLx4O z!;I9C>*DuoV&VrqP2-reYEF+qJgZidr8~!iI!~v=bJ|z(9!+Z{s{;V3LvB51OYRg) zwdPf7P)zt7EXzO&@qj2BNXY8dLhBD8gu>Kz6Z@sEF`p{ zqICgZFp^^im<8XC`JoVPc|byMu6}%^V0Fy|r%>{E8|Vvy?5sg4=+1qj;UT$@y4582 z9`jdVcF#oI36%SYBh=j|GXF+g(7*T`k=XtCHaDFvAsPoc{rEvQ!rhWjo8UK$0Ra;i`&{W@iBIZpMD{U1_Pjo9FSdVsWwlUOnq&PNP{{wjdApxqTy94J zi!Y4?YKl|3;`u=DHX)hn=EjU`V)PlugdkNdzHLnnqzZBnZrfi)C#`L-jgr)>CT2UI zAgQF{*LLEEcVp%D!lx?vSL^?NakVJO$7E*}x1UW!29G$?m7B}kD9%-;rZ#kYlU8x~ zF~tmFK$-pGM*U+YLpZ}Hc8#vjv+$7FZvS{b6abj%5k0#<`DvW{D7jixXS2&&N&#fA z;EL0TWY?ZKedzeFckbJ!8r7rZ-r;lx3QcvicfztS?1j6+^nn4r>kpn?q=3lCjJu&6 zW6AS?R2n{d9~mJH`on?tx!)e!mk(!Dh&@FvO4)>&dz#_}>CKcXz~yVsizla@VwG8UU(eBt<10&++%N^x+!SlLZ@eyL*maYq^oQ43e`QfJWEx>rWqCr*K! zczL3({2!e-1 zQ$rY(5pw+VmoS;@s?Bu@u-lF zp9fhZThYhR&znHgFpOhR6B=+_HuQm{&*6Xsg&3{KEo8G0xq!pcv}TI+mwTQqx077F zYuK8@^p~HAi(BqX8KRrqauy=BrSpXPb3jmj=zVbKW+BwRd9R*(mi&a!1hKT!?)&v; zzY3fekqf$EXoUOy{C>RAh+*M_6i=W<(x)a)v%oG@Wn^1ijoi+?sO{ev7#tqsDV@&u z#VvRGL4J?Hrql#}S?vFKtji*(rUbcUyTl*#x|fnbQOggCTIKQl)XxI-OZiBl;QFfd zmmn-BT4!;xqh<9CY08)Nh{P)5nz*!r)cBcVo4MxpqCokE;Lx1&uqX425sI>{T(Ls)9Q?FAUSCdU9Qf5*zEuB9gt3d zw07*mFMTm$moGly6a)Ig37Fh*IUNu?%N-S9?T+Mu<}^7DgPKb0Olzt-R2)i=S|1G|!!xZ^T@0ygw1qM8|7SjX10HRJ^sbRY;<>FKuVuAn4piznP-qjK44V@-hk&s=Zua`q#f% z*4KNJjqi@AX7^KYJ}WcT)br({=*GlT*iChE?T147Kp*J1gY$i_Q-UTSj;&~3kwWn1 zHDDaRCsMH+cyLdJ>pP`?cTbrwxr1Ub5us?}>f$nZX1a1MkwQVQz*ymjCAoXDf{NaA z6YsyYp3y+k1C>L1;``q%l+fsyJ5ps8!HFCHs}8*p5vpQw6}3Vn@2QQ&+>+V~ z;kAzXMg*K=Em@!Ayz=UHHYk@90bqdm-YDH-fz2)1VA&S0Bb^CAgDZ6E+}htLNogKP z0J_j@kVdJRQl6l8S-t|mo41v7jli(*7g_u~(5~@NlM9(c_rqBKtpoDOsTg$XCT{w+ z;2)81gUVdt)LldxW;@nMZ&{Cs9pi1=GEZh=;8j$^DPS=EJ9d!ze?Zb1!D%q* z0Hl?~329JhJ6%t<{?)*su#?B-uSsQ`vY85&`{AsAyij=AitYsd2a>q6^19BTrwZd0 zXe*Pm#Tx?70%w6k%WDm=(z_^VJXk$a)1DomNU*JNSHfYI)VYFV=Casfm~+Q_N9{{h zpro-mcsIKNYr@VV_2DfZQ`@8@uhQznJkmJP<%QaS&NZiNmuRSV@}QJK95ARsy)y*r z;dk!wd5^~{!VIIs>i8#8oK{=aE#vk4_>Ox@w1WB}v~uS4wORSHC3QFhV4CBw)Y1(8|ZiA}k2a<&cE9DN&&w z+XXDFRY9czMOi|?QCl$eszKkQ0^A#OAtfnXi%369)&oiCHlRdqxDt*}`35Zw_plwp zY>RZ|&yjL1s!4vDQC8Iq;z%#Q-Kym{YPr1UABtM=X~`SDVff( zed65Jpw#9v+MQA4Xhx=;27){$dweXOqz0%Ez2*FX%jDE4btB&CO(afEE+}9(8M=$) zfWYL7BjOJAXO=!r^?<_Roo)%Qm+Vs_Bj7k$5FNTd<=i0X#AKq{>^04{u(Wi$LJiTc)_19k3C1Dpesjd z@$$;*dw-X}1R#HQ$-a&?xX1h~IW6A=ak~Ubz7u_`t+_~b(S&FdX_br%Y1uhZm+^-_ zy*^MMtyk`I9#PLV)w^zDU&Rx5y^p!}YA+w%*ltR*2fM`dOo0J6dq&huQI??~I4h3O zMb_nQuNGc8f@=V+TJ*p|NXa_Ar6KiB4AJ`cwk^n6)kSu{?>uhudtbfNy^@q%je~N| z0l=$?C{Z(YEs7%)zIw@v?fHULYcnM=04&{dIwDN_>C|boWI{doXwnJ%JqW-R4p6<2 zZ)s(DV0&;e;Sq`r+6aTd%29j3+lL47LVZ}eO@l+N^(TD3(!*_L59{ed5e~Xxba~Lf ze&RlM)x#3;HoDi0;w<%9SOW1D5`>{1v z@iq~&THE04`-=>axM~z&+(0;$jAPZ5D0+C9q zDn(&s`a z8N>|;^sUGf5c;N$RrpiK(CTnieSG&wNuS5Jv3)Y`eIgWGvnt7NFrPdXRU=+4Kt~V9 z9mtuZqQH&}y{TGyXf@U_Qp`T7;$(jT4=+`13jpTfB&f#8 zvTnDU9U4IbZU+Bq^IwGK<5|oE<3GeLcYa9DTF(!netVRu7h`YC^xDcZVFJ>V-`I^I zO5W`jIk+iJh>#TK4p+66~fpXIoYG>52D` zLp5)LYr}0@d%YeJ;1{cpm_oZlm|;ZYZqFU>&p$dIkMPp-icu9va~{IdBx}aR*eX+2 z{&)sCfF!ljUi@vFv**OEwl|R5h+k^ez?>(l~4^w2cf+!^0XQ8l?Hgi7JD9 zP*@LV0IxJZ)-PRI8!pYy%LJhldbR$;q2%P~^m}BygjDZj+hkd@i7g8H4*Q4{4PeKZNN^HdErBV$@U-njD+?0(Yi3R@flM$l2IS0RkeyY_@O`*_=zd$co;h5(n%6#*JgmTk zIx}SEAbFcHAVQ{oAAG;xfmm9&6)JD^vhjXbF2h_3)ZW3ChJ$4fS%_%zyRE9@A$jK7 z56qClRgh^iU_veL9aQh!LYPG`1&3{B(X5s#9VkxjJ?!*4cmTZHJ+Et|0 z6eLGswBLEY`#?Jp`}V(sRN@%Oy`LNNTs~J646=!Nq%9BhvFs8J<~XM1{P<;YL^hN= z%aB%ux`fWiCcyzOIz@eY7X5?18PF-z6`9LU`t3p7x!D@7SSTA~lV9xk{_OuuwPGLR z@Cyn(>vR~lSTf4N1`FjOBO7`KzK?8>BMJS1xoeVUN)1GEy5>CVkpzOaq*aOGh7c_W zhI(V5Iu)QVtafJx3*Wot`X0;T6$3N~ld+`Oqx?2T@4Cn%L?rOPCejkZ8r}$~1GgVB z`L$7&$ZrPzFE8A2SsI`%0ueN`jCQd;SYrXRHU@*h|ET2|AueKs^Mvt-l{J&Tc_dY2S>R5P)iaki-60r~fa%Oyo}_9Y5_$XEq>i+#AcX(LX)Gw?ObO zFCag4Gy4CJ=>OFbxl5R+KL0lV;a@gaMCS^=EcDg*f>$mAOdFB)7z2Qe?o8HW@X_r7=bUDm)oa*p+26-ouy2F{&mVw>;}T*l+^cD{pKd1!(b#zFSMSFL3lC0 z%E56Ig+m%Ff^)wF;u_y&8)(Ko^mqOXs@rQ{^N!9~w{)_cvbzRVKRoKss|aEs3=_k> zt1_IG!-&E%5xR1}i6Qz94Z7cfI->Ie!_@ckDezDzFH<~`k|G7qnRtdQrEso94W@0j zcl$cn?zKr>C2&lyV@S=qEWOJyKHI4+AD#5snUJs{j8tK&)7V+02?c4eXa>C_qK#;j z@!L!IOvfGSj;DJM9e+whC;4=&DS2~hw)!+^36ZoYzPHGcEU6ITNIze&XG(B)qBvQ< zFY1|8zjq!(&x(t*f~@@W@eALK!|1VyQ`zxin?vapNkZ!8>I>BQOK8@3`J>^yu@>9= z6FapJ3Epp28a)q|#cfxsZE3|K7ujd!V~g;_MQ$R(A;j%NH<2*OE%Cv6ujMRV=$^4) zBX0dhTd`9E3L0}jQ1jbSc0PxL(-IPjufV;FY3kci{RQ_&OVbQo8Q@THYQ0~t@p;FM z3{j9A$>)$r4PPjWYdIH52$|!op~$-01+>Yg+C_)jZ{uZlxGMrwA5Q_B_H{cr#evHz zp6fG@qG*(${Oar6CGp-ErY_gSZ-1Y=-EnsF&TJDVKl33H_?G7Ntk~fglH1>px0xQ| zP`-Zi_gy}Df>1rC95WYx^<5pqRI!<~DQj@!K_T6cM&7R!Mvaor_y|K0nuL(wOi_9( zY@=^@Hq5k3i-E;brf2IhFTy1QSrU4t;N#4zq_^dyO~`^_Nf)s7`_6th&+&uMa?eV> z@co^ty*{B26~`O-} zt(|Myt<{6H-y)qm1xhK&av?Z3DE<8#hOcjn9KvnYNO7nLk10yhww3&!Q1ikK_ZunD zW1I2TWW7t_k)&}!6w)n@q+eu6z;Ay3=xoE=Bdn z-y?r5-34rxa&5gT9M^M8dK(Am?wQ*P)?>NReO(F}{lIudt+k;FcfD=$ z|56lx4UMr@xI)mzv7Oq%eQ!qVYk0RIy?F>sUet`$Kii2I8Y!ulSsc=F8O~+eDma78Y(XlRY|!-%>%6zOYj6|6$1}@(cYJKI%hIx; z@_^8?4$|>!)y36%nGTt0BD%qtbFnQ=v&n7euDLmfg>mKQ)!At($H9Vv&JZ?z^D1Aj z-j9ay)i*i0SN_%6(BI&J^d1FI!(L~D?(Hdvc}QX20*vRYW-Ty7##fBkmjJXE|N4Ni zV6Jy*1I(*ol5o>^WJQJxAgz=2IIYEmIQ*=)9rJ$==X|*}?shDMT~$1PTN}we#XE?! zV4122oMR~+?CSx{ExJqP1uX8O0g(IST4eOgOa{Q}PZ2;JM7<@v7GeYl zOV0`|-`(@Tk;#JN*2)0oV^1*B$DLiv-Sx4*lh@Czj84+$93pf@I`=xEakvS(LA56b z-9*HoaN`y~|IM@h))s63{si};cIg~Z_HhxLV)FeBHK@Lg=^Nf%&t^bBIR>rBNmzpI zdKd)cje!hGNjdjQE0HB_V4ot!i5Y_&~b3kREDPvqS2o6=x6#cpkKHr}lVCz8D4U#M_+K2%A5VCIhY1 z_WS@8crRf2y=+ORkGemL-}xAtZB+AtRN9{AgSa1BH%<>#{2_pls+PRD0|IiOAIwvx zcW*ONuQ=Fi{IbXOtBXdg5*V|9oi;Aj0Hg|+h>a@5s^QZ1V*vxJ)06x(2zBU#+)%a~ zkC2XkWNN^kyJ?wKjAO6x*=ou{0pOE4C1ZdihG@TAQ1p5!0Go|4$HR~wMC`mr6&Y56 z4B3(a)x|qC9y;qWKuj-BFXmYw62YzY5n*dWW1L5u&^mn502dM4Hocdkjsnms7jbAq zZY`npOC?(yNWVEJGDfM2A@M05uX!EFR}szvdj{?;GMR_2X$&Sj1X*&JwZG6MZ5%F4 z&tO!AYg6TnQTboT{xwPY?jsuZ;Yw!_aoaj%MA_-Hr=*L!spp+eckvWj&6O+3k5>PX zhk3IG!pShCpG$BaX?@7mN2Xg<_cD4!=y@-m1k}xTbwYa!%_3u@PtXIbleJwPGIk4F zFDx7d_U~`#sM8UUay+c zjHq>lP8m3(7mAl~6k6vU0BL9fiDDQ>mfhq@9{odJU7BPi^i$IT>c8WKe!M~9QvHzF z$|2xO-6XlFcYtvgV63(TR2^sOc4UE!cnmb?sia5Hqc(4dq@Jb&H?%Sj=YUJzcCYi=Pz)GSJKf z5s9Pm>YN~;?Q5xBba_V~(wuLY!Z1xqk1|6VA`A**+!+xLp~|IL<(ImY9|Ax_6%O=h zXGA(*b4ef}C#uG+;L@*Isbzoex>ECae}+Fdeph-N*6dt?T9!e)^J2#dIbd&j{N1V= zcT`*H`A~2%s#-!aZwweWOWWUV5|WKzn=@lT>dGN33Ctr?rr@#AKzdN;7NFy}xUM)L zEZcQ!a}2n|^N6&%!_KxAk*c4>_&c^TgM>#86ke60$-o?^+1nS_Av2c{hZlHM@B82! zm^R@Iw03LaAjOZz^5ZT3(W2cANKft;F&%F}Wco@4<5IC5IK|+z5(4O?hGz+Q``v5+ zf~vQkZ08n|c2<)dT!pSGa?89zz*Sa?8hGR>I!hXO4qTXfq7WlQZ@6`45-KsFdm_`2 z<8LjUeRmPZ3L~y|{VfR`Z&VdQQO)UU;t71w%(wgY;EWSOGO))7k^KFz?>|1>4=Z3f z#-3h^xzlY8)7nhVahT&}hn50~9i0Xvlms`C8E?o)v>L2d!1Z}#M9!27^qo0$r2}35 zb^^g}gj|3=oO8svD9L`+Ii7uUd3LY~!N<8$^K6}JcLOf|T&-nb#1{8yxvv1E83Ri< zZdDyJ3cD~|Wt~a;<1;oh5UNMK5oUfPENL^*np$V)Tf}~2y8BW)?ce386O>Q5D$Gqv z!UvlL{av$HfG;;q&5EB^#ctNxQuQT80s8dljyeQDXcY+B(yV@!<(E?)9u(+mIK1FH zxm@w;6;E~8x8(Nw#vpp@E?UPAxAz_zH&q$QsTAQ)?|;cJP#_XZjuQ=yGMCy1&YV_9 zM%Cv6#}(N z&iF`6y2{<%DwB{Q*$W{})Hzfvwk0M!#^sOL@tYC~qhZRALFD;~qvO!=S_#Elj`Bp5 zK7K7T<^(dlz{Iy0^apj=8^=-NgVYo-;G!iP;k;q^TtO=@LFq;tX3ENXw=`D3>%@?5=HrnDaJ7c??{wlCZ~@{rL1 zs9P-$D0EPKD=eVh;1O!p-vgPUW<3M4j2td;olzJr5V-pQl}4(N{P=`w@^y|&%%f{u zuLHfsjzBTsyPB{;VVZJrQvKa_6W3-BlaqJB!#4EgE~Kp!Y)q9m6^rz7o(K9sp<@Ek zJ(a}=#4mz-c-I$Q0=oK5YR4NW!=d)A>+5aBQr?T%UMeh%(h`>fnOUNVNj(3Sxj7kk z^KXB%AY!V(`wvnWh#yMhO?fW{K9w+zz^uf~EV#k|aO>g7v6`B&Fb-c7A1r9jXe-0^GUbJ{cRq z5meamYvAqLArBj=bA{QKT~Hk;3Wh-W(;8?y56zyrz=McXqH^fjrRNV>F2A68+ay1Aok8fm>w(HqY`k6c2frq^fca`w^o!Cb{z4|JkLgJ+ z+}S-t7I1zkeC_R1p)=_RvF2Tv9GEDRSK0)cnfF#hRV34dm}5Wu{;!|(@!HO-yNbX) zn^y<3U<_(0$T)Ldw_IdoWwU7|?qBxYMJEf!Nrg|He!UFl7b~y{73bm>fi3K3+U*U{#0+H)`B4k^x)N+e zD~rZcn652Ou{w^yF)cW#4D+~(*XYBgRbbkn`R3rRk+e12s#QBfoQu zyzQFP&fE8p*+7rk2pdzAUHe1Qoi03PyU99wv+sodhO^UE_3mORD*HDjET3+&xB`Nb zrtosG(EY$s{=l2icSOJHa_r#s$XA~DCssYEi^SH`K~r>3pq%XTXkUWoxe+YxM+rfx z(Qezi25y){CEmy8MeFxrP2cMAj8bXb-Kc9&pp9*wau#(sF-iUW7F%eea2+x%^l1d! z4xaZ*GR$p>>bVx18^vxKTi5YPna|DR4z0HB(sbGxaSw8m1!}UF{Y-2@qxhMBIYyR$ zx=Ie;;J2Gtjd#|*~jAMVdQ;e>{JZ{o)zVx~orZ)Jb{CWA1Q6?T@ zOw20Avah$VzCfiX-6&!@uKd0A3uyg{eNDTuMBIK}GFq*FP0S>{g;IcJMyB`h^CE@! zg08)MH*3pK?tP}lVRv?*_*6>&ANJllEXuX}0#ioKK}2jyN>W7zF%hIY6^1UQ z1SAG*0Z|F*lA&wp28&MVR=Pn-$!|ZNqmJi2@9%s6{jTf#>u?ljp1ALO@4fa~YZr@% z@y>EYA6^i>-LG;P45fyo@TG=a-Ss#Tg$Ie3w``@H-6*y8*+Uk%9)fb2iN!IJ?HJ_T zs&nh%#XgGVea_-hPr9Vu z%MjlylMu*-x(@DrP0QP~f93roG682$!m-W0<W%e1`DEME~kxi-YW=qXB?@}LYis|9^RZ0Baf)xqwJcexoGD-}7O z$QcTRsI2c~xGAWZZ-<*G`^$LE0V5El75>rJYx$Z9?%q^qR${eJe3OWy-~!ti2sbNm zHJ0G zxRdX?q8gvyO{EU*MCD|YVmH+QhZg#{NlEgtdohxJU!NaLTy+7NXxCF(j*w^mwq8}m zXN{YV1uV*|UCc4-xb@)~y?u6mK(U+3fipI7!OzY+q`bUX+;6$rEuSB;8sQWN{+y@h zQB?$+$8jsMgX$6#Qd6t)oYM;G;Ux{GHxK?tj`Aax5wqPkm|t|3TpP9AoxpNQVhjEa z-=w%&@uC-CRHGd!c^7(S#EBcsTS8~Er&V7cITb`GbqFhiv^i{s^Y zW=oSl-ljx6BkwTYpr_=14{RnZGU^7ZFEcg|LG5iT?u`~cduQ-5<``l_<+1Wc|C0Et z7oCvzSoXHFUrTY5Nj)g)*i62gY1CapiBHRb{nU&A>X?Z)nrjW>o0h|pX-eM5uFxnI z2UZ@wpcCQHkgM-g!g~RSC?Dgk=$7n$j&?SY;1spxBI<-mFt=&UW^O48^$5Yoxs2Mo zrmH*0!kz$EJn!ol1YulP9*^GJqiq!z#(-LPL}F*k{d`Zwrk;SjR?(B6CG9wb6~%qn z%Ti3`^6u*RU$>0gSG>Yko*yv`7;AN(N?8Is=FQ$wG#vt}Az^F7P#a&GUkpyDxp(IC zch@SN@?X{}i$vdF!d+ObuMKEvuDw$BF~9D~zLWV)65|o8F?XpWMU|*W|7Ipxg->`Txh^s;6tbVFfi1w+NHb&v+&4t__#?Qwre#aE<_W!pG z^zX!lr4Tzj;oi3jmMIr$^U(3IRS z=#pSQKSDN}#~09YMX$)qni?rYB#=q8(3mC6(eRIR+?>uv%$MxyL`HIltTspY)fUUo zU9)358%r!msC;LNzM$w99h7>}LDnj+?TQ*NtyN-D0-9NouKubtgI>|juzU)L*PhJ+ zl}kuoP39rdBF-R-`;rBAAbgdlQ%8C*?yVX9S*pFu7_M9vR^jHekKFx>%rY9u$(_m_ zQ;WYYa$W7hv@U7WjwVZ`RGm-YOf!iMb4>Uyc_gpjYUZ1231LxPVWpU+dezr|guB(- z^ATyP_2;Q*N5g{UKijL6LP@&AzTCj7ZTIWp)x>b_O8q)L)o9(o0 z;ff9Cll+*^rbieCS71B(*TT?IJ0||5$|#;7&j&famoUmJnvmp$PLY;Y*b>fHaCtsP zx7SLV;XG9R6LxIv3LP>nG_3KuWrLmIv%q*ehfg~pS*xIRV`J4aC#nA;w^a^kLM$|L zU40ocVYYxp8i5cu}3ew__tGdTvJtTJgBZno;8s z(+*;cgJ5S$(<0=>O}E;IQGVpi``YESKRl(ybo)jPp=sIgPla>qoX^n;G&+hq=B4;m zo8se_;=~%~H-ybzm(6wi>r<|#=ut2UNpZc4d#lv22n0k^;DXx^P1tT0*iINg4fJ*S zdg_BrO;eNJfWi^Be*bqQJqB? zZddJz%{=eQ=8!!~Qt4omP%ygS7Q#PWOiK`)1G@bs+ciM{bpgM~1mnK~Dyqx24U}h2 zY34RKSIG4MM8{#;lfOvmtt&+GfOqICZ6&*yF&?VdOZt%9uxOO891)77B>+qFj?(tG z<=)m?zH(wc@>(_8CJBmY^PYTx2hk_{*1=0|fx_uLv_Dd|+rK%ArstZp%Q9+w3l~8j zuhFnHTpLW0Y`9Pb{7zFawhY_MMDGR3!3d>?@+%H&W>uFS%&|o?n2@N@cy zc<2m%_DNnj6zQ_Tw=x!lL%^&9!^;HUdnFqzHhXm~#5u*kcm2uHoMRwjC z^De=hN)CZTxncs~5w%7X{XY0=TGr3=5X|c!z6-hik%1sTh4!9+_ds+WxH$rdt5%?+ zLnv5!r5sypin-n+==YFYyYTLNWA@i+2DL#qmz{>jvlPk-Oo(&pU{jC;i@Vpv-?i3c zS(GW){UQ*gEujL<0-vdNDBF{vo7oEfJjZ&9wsZ};W0xm8UJd5f^!gl}Rxmhl$K}_3 z;*+=|sIpNmS9xZiE}WiudHnh$*b5So&a_p4tH`Y3+0BnTFbS$w0CQ=0F$x@6&^=AM z3|Oeu;JH)?PKuM1L0h*I{Yq;BPxmacUWdf?JmXR)Yz3UQF0^&8yj!|f`El%1EP~v0 z9|rr8lp=7}I?0)rEaq#R@{FE48yqI2#0d@=wnH|?RP+GbH8c^gv7&QlZ{du{`kSV{gE z0apo=kt-6n0L^|Ib~H@TAZwMmG6j&6CFpW}tCsEsCU1(SUc7va`))Sb2saHCx7%+B zV(d<^P@=WNJ@3|XEwdd>68?P*W0z=UhjuwXFa^zyUGSo)D}3i$+iO5TTVozG{6@3u zqT#P64U4Jb1zDc;pptdtpIe5Ge%8HO{^iYv>Ya>i4FG`Xi4UcH0<$N`BcK@Tks1ht zlQv~ATWDtRo73X>ko#3A2lZjf2ev&Ws7rLYTMxPt=|Fwz0$715TgTB-(QO={rq4I5 zp=9g;yU!sqK49Xy=9_oMo`2B^Ls%xEq`yIQf@5oVT~4lypsjH2`_fcbK@0Q_I$;ME zX^E$@TW$aX-UXsXGbmQXSHIXE&wvUoBk<*B%3BA76pheQ^RxhViFK%#(AD9Na9A8~ z$-ejfYpIq6(yz*dt;SQi1naJ2nJptVeEwgH@gD`ckKgtRmLR`rI_p1yr45b zm6S}_zPVeZP$9Hz@`EL!FN@=v7a`D2^I3xvai!w4sQvWjW_Lu@O^yM5-6w^a+biR3 zR$>CQO#oTSfS8uw-*C?1Ydbhv4cE&uo_PmLUA-~tQZlrF?4b6^Hw$5x(E%_)B}zNh z1mHL84h1?c&Cym0$fZ|u{b)Akp*2Ameax(24W{^GQR<()}f~-(Opuh zY6CzZuGLW}@VdWs#_NF=YTra#YWPHAO*HsVd1!bqyYhW4ev~o05!_S^07Nh!Zanal zsEo7|)I8E^y0lx8IEmn>?FE~@POKUFfX0xR&jWwKw!;>DVa(Ba_ z1XIkDY*>~ovK8tPa0cl9yu$DmK(id^XLds7JOxCM9HUm!Gh-F2?qwN{5TePT1m5NmPgP62$Hbi4|qbF=5$<- zaF`#4W^if;Ob1xQeFVz=!Byy#@0q*sX!k)A_UaV3OdxA?0`Gdr{h&&c>gNrgH#Jdc zWkuO9pl}>?mpQuI&i!D}vyHfzC0K8&@N8wQ?sve6KS*VEZ#+I$bND*Sik;;vJdoSs zFVOmknBFxBvhGguU5tY|GGp#!vWv9l3bNs37jZFLis9^8yWsRCu`iTtoS=G2NlBUFlz!`j8Ne@r{#pW7b&Xtx&CXOpht@wU9X~?*ehs<<`Kf1Y z$8+5Vt=45SrS(K%^v#A?KfA2z@c|43$K6H!?wBN8wLih|UYR?JU_HjnJ`m^bN}HCj z+7~Nxs7-94L!|NhCu>)O-q?jgu8)9a{5*YFj-miNnRYQ~5CJ#0Bg^x-%k$k}P1Ifg z5~xMD#lk_p+Bn{pI{Pu((b*DTOE{7d6~5fVh-hYpWG z{p;sqsi+bzJa9QN=OXx=e|o$nQA>bwoeSG>0RZl)^E4}9XO)XaRWTSd!Kif5e4p%lcKKB$7@=vQ`EAgx3q~*j1@)6*W;xb%8Yls z$=On`<%o&h{IV84VQJX;m&gY4#JEMc}XsKVnBH0IR{Puh$P2NzKQcVCGKI8S`HhB zeMWa`tt_&01oi0Q(tIoJTApbeSE5!yHjvzoUbBi@88h4jK3LusIi(@6i_h_6SE@ST zuJXr1IE*xVh(V9w&S9SMw%|(xS);9 z<5H1%A?d|om*}CcU+vRNq5gFU{XUNV`00ycKO*{VsR4%1X;^iyYr{m(R?b0W5b!Dh z!<=KQ?SDY{BG8^~46#nllmGL|@xs)sa#8m|q~B2c2Ka)vd5#kP_=h`Kd;-aU`+RL~ z4C+~Zu!jE{@FG<$gCJnm-hdo_#p;S~E+%?-x`ZVy9AQ3TI0+VveKk{EU zEFCqg0>M)h939kj2~vMllGIohW|ZQYV&GSB2_voE9FUEv!xz7lvV~B!VGbK_nLFc; zs}#AyVqcf1m_qaa_S~o|I?34CTn8)m*zZA1)!X}Jr!>=VJ=nYe&WqzZ23tu#;0_p@ zMv07QTgUudDx`~n^v`@gpaAt9E^)saC=ThNDUhEYlpNX`<$|XB8x;F-&dVW7GOnxk z1L<4;x6i~X(-}3#y>MNrV#(Jl-@Um4xrG_D#9Gt!uAH^%cXM5Z2)|a%`TjDt7({J3 z_wV12BUkiES#i>N;qDa8=y`+W>MpXuOYoJoA#J*l-sbNs=J44o=u zYss*9xmYvxD_^#zXq~P0;|SPw@JzE<3xG5$0cwxEuz7ce5D?l9$R=S{rJzDr2kMfh z>p{RD4k_?(@A~~s_ysyx-bD`>4*mYK|MJu0$WpKgodm`J5lWO`J^<|3KrLo+#hWP| z$wQQrRBs{;T4?Uqhb{sZ_4IQG1N(6@a72P|h4eoUwq_X0d|Iyv;MS3R7|Bht{Nndk z1Sdob@AxlYO|pJjt|9}d#3x=T!$P+NEa-*ZYL3inp0WknPRLUYVD^V6cxk?eU($p2 zMh5KocChy|$g|$$zS01>@&&3aP?cY{ojyiEJp6$yvMdtGOOYnC_%iDUk2|EVo(-5U z{3ul$(tP3HFY(JIEIta#%BKNiJkPh_fvSsSPM5uzjNx6NM?V}Px!Ey}*^=kaV`d!o z{}(X+_sfT$@Mhf9j7*NuKsOQ*$uEt~^wu{p*sqZf}xfu{`5l+dz8HK6PYBl>jG6C!{0heZ|gpgB34q z&L4gE_q)T|?Q1xc0jYoq{7yDh6>60)k7slpzavWWF>ivQOOG;xO3@C4&-vCgoT?4D zy`nI3^3>tOW#l{e>w*uacPOAoV}caqr0M<3Eimw-GL2+o;DzagUDk4)ffc$0%#ZxN z$_4f_CxrREXXq9FIz3s!Pzzi4Idex6U$69D=+hf@szLGX{pkg;3)-H)Uv!9S1Yj#0 z{j#a{>nn@qU^i}|bZ8uif(015#ieZiv5(Vx_W?CX4{O(1DVCM zF>a^F&$sb<^{+4A<$lrO2+2|uPRGL9d20te>4HfBXlzzN!faRofYI9}y`v--H-^Ln z$8kPE(m!M2vFtt9MZ{e{<5}bL^F~THF{4hqoa@C&!w0^ZlGVxvJmHn2E>a4W!eKE9 zG61O_U8V;=FOkI!%Z~a=_$s{nkAEmd-JWF{a7rVabF}8tQ<}`-+6lD>=$K@y&+g)@ zs=w;%8^=Imb+{|v_EBwkN%PW-*x*dcj0jM7^q>v1V5#>%d|+2eVOnQd_;q|iUa^ZzpIUkw^COb*>(fXEILS_Sj@14Ji=IwZnBxKG-e_>y z_rD%%#R;+%wCq+2oPpO|7cSCIO4%Bqzc9;kC8V#DnSM+f{TgjIK|*q zk_{jabZR4X+9>H;H^7+n7K#|5tC-Xyu*DJPNx#-k{GK1*cO>nB%gmnK=QxrNqJDfs zcVVK4?Yq&-mnZVS|asDyTX9TBOt*_=cHN z7fO~Nw>a!4U3wwMWDD=!4(7S(K2*29=^Azj5=>`6*hQYWnf{JX702*}fi~JVNRaX^ zWbxF(cLuLE$gxvi|MlK)9j9I?x-`1XoPLIC;MYAuX4!&*F^s`c*Ns{dXk&&Hj$QtZ z{NYu1_=xk$oYOIPA4DIZM%xDO;d3l7CNUCG_j*)x$6lrRJ`U4M13KD-`k#sITJb|k0O zzpN>J9i*o;Q#4dXasnW!9EVk_+LYN)l|X!P@%n0;8t}&Co$qB*^jf{s61@GzNx8)3 zAmnDsnU6x#bgKOWy52r`Cetn>9uLfEghr`RAEq_cbiFHdcK;dD*Ml9V9F@?9OngoT zrqgUI4%gxv-e$4!iU=2X`((tjd+A4t3)=)#e}~$uH{ABlSB|Ao{f=&bJr+*R2&u#a zLK_AEFh_#LhUG-t%rxgc0V&d$JjpGaY1A0;tUKS)S z18Ogb=h1ReRDW)JtR|%=)A!u<_2rFY*6PLgvP9QrOL~@E)*U;+BF5*YfL`lHs~@?(h@He0DWp?fN2uo5fPSAzm8#e zZipivqt{;NKVIS1?O!A#v0VLLr+0WdaUP9V>P!>GB%XF6X!&$7i15Aj!?$4S2 z{{2NE{QBy07qD3MYJ-JxpyMoSao*YjV|a;0VY{NTcAx|;Q^GP?>ez@wkxB!}$2i6hJ5Q6Y_z1!K_koCAqG44d zzL}BGwKWIQ)CNXTw;j4*>l2)Ctk%&rINk5%APu({5k!Gsa^Y8|@Ym&3*pXQli?je2 z(|jbnueSi904yftVgaxHl(52Te!KJ4qkr&D;unDy?a&1AZxHcQ^ee1!290qb3UmHJh_GLQ}&0)FR*VpDEC@H8eYj@!mqt|@qYY?e&o zgpM4HrY|;xeAqhsxMawnb&?@eLkl<=QSM5Itiv(&*mm*u$0U!|6ICa=tf^w)0>~dP zbKqylfD$o^Nve$t%%s^Ii#mRDJ%*$6=;%KwCs;vhS?XvqW+gl4n%B3N$|z2EE7b;O zl!74hR#*0^?5Iog>KESLJaF!AuIT#0{x+f6Cfe=sOYip`oeVU5sJgjPw9UQ?dJeGeP_v>s5uV321*W_?RtfTCtr)%e!YO9s~WUt3<1mh)6>2U2hVgn2wAM1GBQK0L8FLuqATZ-*Zw)~zID#M znPQ!WETb&SBrE%>As%!+8sN#|d?-I?WL36{&o%G6&G}PMn_!tRqlON+-AYz*y+{98 zn;ft^C zbwniMWYWbpyN%IOOrk5c^n(toQ&iA0)Y7NrxU7?}B_4S6O7Zx|g73qo$p*Eb%)WnZ zS!&go=^AgD&~zI+UGJGy&RifQy%c@9XnghES9YPx83ta-S6{p}X5?-vqu1%~QBL~S zl5aO<1inl7dj3>sQX5S7lS(DE%$pfH%CKXq{&)4eg>39p#FjMJlt}me z9_>zLl0t9uysCe%weCGQ6*af6heZK~q?~sLm#CCThZ{P*Y-eTnGkayJMniRkvpl%( zcb)$OrFs0o&0V5iI69yEZ2d{)1Vze#O6$Jjb&JtH*Ufjhx#lXL!}eWApLf&RR9)vb zAE^r9;E{KY3+4H&Rj|3y<-T$(CyGVZ@0Cjbt@K-$*0m^h=GN7nFYsVHI$X^yV>7Zl z;P5dhAzfCCpKJCDZy7(ks= zHrs)&>lf-^V6M|YJWwoVG-J=8Wa}JxN`MX;7+LsjhwRECq33(Y8zCCvIH1$nTI|+0 zF$LNDGU%fE0+O?y<#cTZb)W8^c~3?X&H`CviTc;7t{J+YjU^2^z#RZ1>Zp#P8+Q)= zBX^Cb+b`<)8ne4)3xuM1+>mgGR}697FubQB+hYBlp9ht_`!lXHeYnByl1(@6v%}d| zS}%J9Z@nK7FN+-4BuE)gz|qF>d;=PV`DQUy>WrNrKDgy|K!+w@WO5}wRyngx$F&#r z=HV2zhZntO!x5mq9ry~g$yrq0Zrd1MzeoZ3>+v4|v>W2{Hz@>;L>-XlimX8d>=ceW z8Sr5NPx0{KE^4K|tMzU|yHo#h`G5VdKA|*4tDu52EKrE$MxFG-G2y&DQrqyHpy4&5 z%U6@4{WGy(ao!F*zofKnI(M!e_@r{je0f@wt5jiY)FY}xH&RSrI#-Q$L|yuHtSc*0 zJxgJ|i;78jROd$J+sc#lLfqBHBXtMg`rFA=-aR?)wDG<6ogJHMnj0VxAAMg4G-bW{ zcnePH?(gi%oQA-e+v#Ls-YlvZ=!8>JDmBm5?YdfzeJ|eeuT=|{dp$>8EQ^|s#jaWa~=>pq=Zb$=0X9Bb_ zN(qWjg)>vG!0g79;{tSWjP-`jRJ@2T=8P1&a&FoCV8Pm|jg%3beD!Xw)TDU#Zz#25 z%U0B*8^MCLDtSiN$wRyp^aZxApLc|S&!w6Psmf&^Bjo~*8hX4Z2m$%^LPY+g0=r>D z&~Xcncg)*TN+!o*%iG)22b@ccx?a68eSjV^8>!Rs_wgZ4Mlx1#;KKD~`yK-E z2_K}o$3vWOL{J@VqURq4GQ85`+mGO8n+G>HoWg;5P>UQe1#~z@^X}Z@mQm1 z?ou$C*7IWz#Q~Y+S@49C)!23JUG~RY3~RnsfiUhNlEqHVmCwl7c~|1FN@G!XO&?(B z?KfMu`nwUWUw{ISrEf2kWFy&~T+*_l=C=SOu;>BW&tNdWyLWD5?> zT&sRAf9uDP+*8~udDd$JtlJz&5LZ(jp&r4xfVzCvQHF4YML|kZO+&4H5DW4M-KE}sEOXRBs&$yzh7yH zJenjIeLC5+&HGy2AdW*Re(9=teOS>8J6z||={n(BH&S&Z1^JN8X3?BS#2wNK!*BS@BxIuOM=H8w21mKuE8_*Ubh?TAZ7)ZfB8_2oJ@O0 z79hfvzI&fwqi*V`;<@QxzXSgl#hBE^r1I9{rE4|`Hdw^Flx`q|=F za3RpNN+7K{4G<_uO_OxR6Atedgl5^5@gSOYNEE^B{MT7qdD7InKPFxoces zTQAu~fpoyK-f|U89uX=>k`Qnqpw40XCIgSHdmK^>B;CLlbboy=#zmwO3wfdMWMDbk zG9jt3^hx5;-bv&v5A~j|6kR@N0Q9Py&+l|HLFT|_CMNi=RV=j(88D47LEcqWpQv}x z>~KDDAw00OI9=NUXtA`AaCfBs=>LR4+r0%!8w(hacP%OQh51JfXxf>#rycIntbf9F zu_R{`l9e0HrxcdT=ey-G7u<|YhJ<(jBP?`b_)Qe+wW9eHf&J7H&Zc-kDSJJCf*j0D zr0360cI*?`7F(ilf)hp!zJd;uoc-|JK;PW@7o1%KCm0vY=cLHkEKfPbCacaRYN+%N zc{1pzm3xb7Q5!Uf-%f*B(Ru|-ofZ}+V8CS~*iXKQaNyF&u1YC(bOT~&`yzwz>}3vB zpS_y5ttMWRhV`E@uat88g$!=?2JeU12<+(TJ@8(aYLkXwD|N!$j47Z}Egk2}twAPD zt0_FGp064K-AER2a@wIx{c^`UJ**iz?XZU~1KBT4l`3?2?u&zkah@>Q#ZOV?=|8{b z?+T$qU3avBf>WpX-ZZ1rQsIZH083q%*zE-{zA1+SAxt{tTbS0p+olC;U!Q4(`*F0r zkGT>fIW+a;B=c;2GK^-PEX8|lEMmBfZl2Nzn`))!{8l;MEK;AKXngDV^;Y8UgDKhf z@4s6bPpn^AU43cyQQB!d{!vf2-;nM5$11>T?S#JB6)3jz3S>asl^#6NqGF#8a>b9e zg@HRrvL4dw2FNf$G5!`Xh<2?mt}pj>fdR=RA=u*UX4pE=mD0f+L?ltiu4cI8-uF8? zLy4d1o|hHz7F77)Y4#@=o?zBESAXi1_V&kwOV9Owx1oOI>BKDTs}sBKG4S_go~}=t zO0iyDI#{e@mA_fKcmyHuO|dbM#0Yh zvUeT?5z|&0g@?6AE^UbID?>no+0RE?GmRyJMV%6|Ol)t+Zr)WB8S<4c{$6AWrU!9; zr<|T*?kV|9bY>msrh(u+x%yz>lc3Am4i)e7+QYTyNU9Rjvk9NX6~Xrv(uG~CEt$p& z@Xr&$=DQV!Bv@C!I{Can6LoQX+a_>hD)!?G8LS!W+<#YwKYqflU|1jX^3E;C-Qc$! zHjA`C82NJfu$LPz90~?45Vr$oqtPM0ZO4y{nWkfB`bk zgk?7I#Se5`0vU40b#5d+AaBa*aWE8H2E{K|U+i<0{5jh9iIef2hf*%B?!152-<)&} z{1MC=DKi1K2M_qYt6)iO4#mt8aPc&N@g8;dwKfFX69A8WdS71lk53R0X9-9pIQp^s zb*tEwXgfbG&@NI}TjGP2Wt^-L9*i5QfB%mK0_C&wb{J-9gfM>&9Xq8MC#!f;l4l zw1Dis1lUzOIOYsrf6de9mz}DT8rB0$T&`;42K-@;c##~T_Ye-G<{O+gfm!HGc~UTM zc@x;>Jpqp+<85bpMdRN3@x`473wq!!2wLuNg7$rPVZHFcPB_9$*Sn(C+KZhFfHPJH ztOGqnRE$t;Oq%0vf)gK*HHiJ5QohV_dGfHlLR&4~~uJysAY?zHx} znk+VbI{G8RWAfuvD*}^ zxVEnRP@Cf<2-EaYcWR9fW{Uky#d^6glDgAHOlk(B4|OZD8}Ga>g^vHQyPTr&#jX6% zXZu3+8YsbSFx=FZ-Y8m^;F}4@aR(gT7E@h0IY8l#9h9XvBi{P(^l|wpBh6ZLgs+B0 z-Un37#e|5xw|#id@~y6h*KBNX@4QB^sG3}T5l>%V8BJG_H1Y=QtFx^>Ur^Zch71#J zjGjdVvKZerSng2XZz`oW&7pTp_7vnm9?3gtz0`p%#Z zlH18gl#uZtw(X_08$-=u8r$x{Oh?3@{5!k#C6_)V_F`BTi3D~{?xt06A}Zl^S-bmb zW`^bsV+vb@d|7nl$br)ao*A7*PWHL)G-W3M#eTJ2&#@Pb6dti%T>wi2UgwA-nQW@d z;2ANiq<6M*3g)4tf!DuYWKVbgdD+h>BD1hn=`?71KdrsZ0y0JZ1SLIB2{GR1 zdhK)`l@PaRK)u-Jz4wNk-MkE0r|d6>KAtBouA+E;m284?hJkcW3PMJ+ z?y|9@Wtl)dGT{nJT(glrV*o%Q*Xd=1A?{7NEaO-sU59?krCpc{TI3psG3^QF@u20Y zE<=RHQq9b4aU(F#Ywvtue0|=39F_kp36RamB7vf5h;Ayz<2e}4;8|sqvrJl!St~|( z^UXjiSLSy_Iqj(=BQgY-%PIy&#LIc#5JLXh6oQ%~+L+@Eqrjoec9=H^$Wtqu)TVao zEp(WIqj2ZAD~#<{kbUFc=Z90oM*^^th42IKjDXF}ZM-}9;Sj3T$EKFGA(9PMRd8U5 zHe*ddZ%oPo>Ngk6>p?iN)ww|37h7G9UF?R#g}*mY`zK-<4KrHZ_{&t->5&OzF-T)>_|TxqDY#atJ3XB#{Wk6cUjDi=0l(m9# z9Wb`?r|_GsR`XHMxb6hQOrzTh4vSXi?;r0$&@kA344XyrJPCZ-S3c1gIe|1LyJ;VQO zn2;V_zY~?QBeGCzSBHz0R4WFwT(9NGy;2sXy`=#t+Fz2D!~smL*eThPoZ8 zls{g_-BwnvL;#_cuj+5^O!aQ~7CN>a7V^VUwJ92C$JB$IW}Fz8@O;`Mh%cn-xct0i z59s(CwDgR`aMOy)^;WeU45`j&PcH&nyXw$D%NhG=ra5Xq>t%QTMe!$Ar&_HQjGb^# zloa9}aQchGXY9AxL6$x;o~7?|qL z`vhS2ushqLOA_BmRcHK^HfAT?Vs^|@OM!QSQsRlT7Xe6dU*ec21kvj=WS-VN-k{L7 zPhC(8A77<=FojJozRj0`KWMlIv*-*KP-aj8?V&;(S9*#|^~lbaJ6bTV!g%|1Bqe}; zj~+U#r49IL#$dp1rGw7(8r&nCQ=@tU%YV9A?=rT}X0?2O&gFsKgccee#@Z9$#0ZXE(u z2O@`p`{_;FQ2EZ95l__nn14#Uuzi;r=*Hq6-2eqTq?-8~K`M6;7nqG6=oT$=J7GRc zesWs%)e?xQEucR$4WdBLaVt@Z&+2fVS+kyvbAO4r4sWh23MxB`!sG<5ccYL?PQpN{ zT|zNZJ+RjZU#;)_m8kufD(YvBSaEeLYBH|-HgH#^hFsvjOrgz21ji7=LH=8gDIn`v zEmxS`2aMj`fN-+x+fKvs4ZH>N?#zM_R2ooGOK}#+dH$;AqtlF1-d7+)AMZG1;UO9O zD0`nDUmtLu&K~3O;ZGh3<2QZ@$!fLEW$l;>U~Z3Ab+o1G^!e$A!#JMiHd;sN%48=7 zwIZ{i%EiScq;q#@&t9IgGyb#td|R@{ zh*%$>Z0ps=TCz{>`UP%68vXL131WbQt%CM-p31>ks!ITM9PSt>l!_jwkxnxpdR$6Z zSbl2?Het7MT}6v`9b2rOfuoqmjq_dy*FnuKnZ>cPblr%(=CoSLl-OV8z@edMqyZ*;m#PTyTa>0$=6jd3Lj0{0-xRv5w&CsacV{Csb}!c>o%7&r zwQC3T3j2F7yEwWAgQ^M%x-`1!=7tHH_k4pc1KKhw%DkkS$H`cheg-F&WPFg6Nfr-wX%vnH|HgHHk z0AE9w3;t+Q;!jNw+2i_r_~y=_my($r90LeuDL4`lx~V0n8$-ra?#ZABEa4o$3`MR9 z6;ZLbi)lIy2O4&-7S!!i4Lg5oWcN;aFJA^5==vfh@V8hRMwWz^GuoJYhnBOCFC!HR zfbR$IXpstX5!r3%H!2UT@f|1nCr- z?k>{Cb({4S7aah7z>93y!f7KbFcC(oEpalmY728Uov}p~83HpDc;;47GeT9F3kUdV z%>bz`wC{e#kH z+{H%V0$^<|*YUkHq7pQ8-#zG{ZpIbBBuX$~5}HGVB_N_RXZK4HwCcxlsHR2aoE=>R zmskO>&jL7WMLQYic(!f&Ktd?&@%OhDg6$~g{ZYkVJ{2_2HNJR+CIX%Kt(F-N$S;N2 zG+olU1R&rIvxy^^*J!tq-;uVsj7iR4p-Dnai}>+(A)&^YtSeLDtk|1 zW)x=#I@j-2n{tY#F?<&hO3P|~v7S^_!I>aiW>nhvrcUpp(=b2y=1RjuzF>eHT5Wb| z!)VL!$~eIDEfk;;wB9zecx}S;O&$VoY>e0gU3Lj>1l6O`lL3vymz0|G>_`wnnnJg@Mu9+0kfuwXcF_p!!}sXV_YnE)zTc!HqfwWQYsBMpr@G;tyG z4XyS}aJ!Jo#>$tbl%m<8AH9Ug8T(tVbLAaNyH2cT)DTZ$w@oEZoJmb7fO5!!t7b`wDcKuDEJ}1Eds#^x`HKr!NYTP)dylG3!HDDPdE|glRPytA}w`O8sV_3YA*^ z2IZ4&Y{PfIlCYaXplzB~72dV^ID zQYMWluvNFJk4vgQGr@Tla)>DqVi3VzvmJD|O?(e*MEtij+T6s+{%>&>O{N|1;ODPI zmxVYoh1YU&)aY36+U?lJfKKz0(~n4N_R}ZVnmEP z1+C2(s)XaJY@oPbtvv3_I{}pX22?fEOt6^j2f#R~8nuUAVHi}JZ^>y0L){=-%s@XpxN23;6~824KJdyyIcO@i$Go; z+k67?ExW4H6u1%8t-ubmV=#vsvxJ&48dKx-LGzZ#Wv)IUq0Q<&2dZB%V16%m{3I)n zmBXx(SKPkBusT!N*1OI~Y;J_Qx2pYON;R?b*iBN1ORz>uJj|BbaN&?h^?P;L4ND}; z8_wL_8QwlV5wW!4z&JI{LQ203isGSZ z6?=Ts0Q>h8PHz*muVPvlpUJqW5(8VbIUAsz1fn{7nl*@UxQ-Gw!uJ^$_wq)jGDw%KrLkXcwak6F@JLc@GodQTI6B_>=h<8z;FG`?$?UPejqxWV>cX}hU zxY+-+4X`Z1lyNoFkoeE=y^4RIEZJl508zu|_RH2P9e?O|+v`U{h1-e}PPk`p5=^|6 z@ZQNp-J~u#;u0#35M%6_NiN2otEj_fjxKIixh#lM3}YS0;=9LDs-nZtaf4q77uuqt zZ2Vaw5hR>}|68fRLcI--RP9wLr zS04HzTlpRvfOEKI2z9F!zBfDtbx;||3zugeS*3{;E(U-kA4<9}p z7{*Y-0V$7i@jZt`)$}Ncp|uG5fxx~%N5HM=3nKyyY6>!Vo@o$Y)FHHd(G`^1ivI>P zOZYdYDNmDdJXorC96QF8A?mcdYh(S&RK<${yS2Nw`~%PK^D4qZ_5*#pZsj*uwa<>d zSyPU{hP8(u)4dp0vSlaT9xTgsjQ+x^q5sEQU;jwfK#&0uV?W5Ou6*9US5x`D%H?x9p9}rZa)*moXA6lv#hi6n$UQiR zx<9>)y3Z`OQj!_zXmI1aNC_>g&&j89N9-w=*M_QIwV^{5KYpcSHOq6Z{sCZcpqa$% zn!QgjrFl4IYeZRgn5H{!E_@vL2kGfI;L2i1Iop;IVq21@PI4GE9ZNH;{~R$u%j9?` z>q~KIp{973T4f*^nwm@YT*(l}%lZxV5+T;BiuE@bbXabytB=mZ$eJ&>M75M|W6U|} z>JVWB!2`}6S@%3Wm&JE9U3!sg5*2!{*N0U@&f))1O|b1_m4EXXIXfi-xI&5B#t^Eu z{#LvPh4Ygm-iLA!er4o^#`WnAApCzGJd)vn0tN-w3z+ zxG;Zt_eLDIV*ULvFCa`z0%5{V#AfG@zfl)mfhed<-QX0|+eM}^>ZH?u{-6GKUsNK% zH50(bk`73FzSzI4*$EHhY77>1$%y=Q+7yszIj^P?SJ$>pP-c52xEh^A;SqDi$l$4m z2P~`x1VLk7SsaqR74BvZg3?O(BEZroMzss zpqWJOoEuG0QVx5F6~;Fh9$o05w6A%R{lGht`1Qv+DWJSxob#Y~8J9CL%fBsMgqGya{x}$a?dsphlh48J zh}sNDlR2L30^tX1a>=*zsk=GeV>ng!w>0cuqs~9C63@vc3!pTxC?5f3>)jG$y`3IZ z=Z#>twPsC<3Dl2E;OgHAP1>HqvccaP!f1uy&2CVZ&DJB+ zF@QJyhn27@AXiDy-$LAl)Ibo~eLGPl)y^4lB10MzNzlHq1Gqv1+&b0=9&&+eNco8y zAw~$w1#()%Iw}d40+8emm_in1WWSm3ZUUZ9Ch+?i2<`_FT54+!A7V`!uX#^W z80)G<%q#6+_C)Rwz);N~fwe>u@p+^i=`NBQ6WJDOy9xA4-5(7YR*-wDC(6e>g)u&5 zFtBdQez(0hcya{}=K)*8zC_5pQy(eCkUsTys0(`#*KeCqC>PC|AVWP88ZL|5P7^Qz|2xsG@8puL!!Jh_ja_a)>bz( z^*Uln>t+Dytq0jf7?ChG)0L_tMsJe`ecMe{_UmGf3r0|mOiV&^PYCsZUDXDVy^Zte z2;hYlV$KxKeLS$Zcja;AC;+7MkeCgk}UqZ=mA zMVDS9J_>f=;8n0_wE0AT4ou>J!uV>+`|PyU))2(t2=jXJ<2oo>y9E;J!DHp-$xxe@ zITQC&@pVXpWpT3O6AI%*F`m~jf0)974#35(Ay*~c*y(rBnO*D3j40JfMf{k!+%my1 zrS}AV_qgaGdM%q5NV6ZPm;Fl|xpyJ_3+h);=Y(}yf0OZli$YcPIxqJK!YrPMqpoXn zS8~n+E`I#)SF{vPspGS*o`+=YCDb{UxjhR1NGd~G~0T6HT(-`Eu1NM8=?xPSkmgtChw&9#`r85f<5W}03;d`f4$hPf2? zCdPB`ixI(Rgeq^O zmjxWufJHd5X>u8=2}zN_DzqDkXHmpWD%vR>RFK|EK>FiEs(7i$s$+>#DQI#fH@aaNK8Pszh5VIln)Sw=*Ys9(R;SOI%O|Nul7}mW0yz*2~7&Xf-tjKWPFGgVl z-K+Ghg*WQp*?W0)b1;|)1O4#oX)t1t8kvRRSGo0b`L+{M;>pl>vHdp98-CKmsOH+( z|K(M}SKwx$8~;AM&&DG|x6FNm^B2Dx>%Vsg)o_#jVMGSyql$lZ<4h&9!{8nyPH5&W zAe#>7DK)v#n0*$IeQ7avur!^jVvaAZd?V;*rP4ouSh}|>i94ZBC_b@4d5tp_Lx&an z@H$GKohM5rRXe474FC--xA-%{Hlsd^z{x;8XWuVZ<_W!FAygE5a11e?omKXDn_rfK zH27$OSD(cS^i%EBPV{&#Ka%X_FpZP!Is_~Q)Kx*_ruRCbUsO<%K&)%<5)ytj0>lO* z=vmm6rNB@SYc(P3VJXMnYw%(SGN_6b>4)P5P$P2PU4rh-Z8gL7#i}z-aAHG&P*|A@ z<-i7ez!PQh7;vgc3p{Puc~+YzRkG>|V=$RQ)-)VqcB_dY1!VRuw`_N|sU6v9SWUK; zdu0fIf`FO_S5;4(j>4+qY+DR9@U8JJFE?%wTCa}MBLVTB;pgX{Rj`nKKE4GS4dLRK zKl1$iJ$yUnzH1xhCKt`>(I%))^ab#+6WUPL)}Wqp*w8Cg~qxpA32CuO~bhG!xNXwBrzmjQ(#lnBoz z)WnVJ;QiU#m?sGGGoMG)vdub5RiS}x1O}5gT5rk#9nPm;wqzfmRf4Jlp}~E=#tPyw z0LVqLT@?@(ZtlArh^3W^C-iqhF}0d_L7VpO7{w!#2Zxv#tM%?P@l<)bYc1M~Cxi0) zFkA*#6%)*}kqsEK0VWmF40KgWD1?-XJ;{bdef=LnAW$s=;?3cADh}uT5~F|lQmIK} z`))62oy>jUN{`kq=SOO|Yb?3Vh3;P!O_*GlP|38FX zcOaH)|1XEEP!vf<1KHVRRw`ugP4QTTlD!`)ilTvJ@9a%BErrbN9+d3y*y}NV-&?1S zdf)f_{y4+Q;ePJxy1v(Ue1`o{ts*+U{xyhNTI}Odnk(!-NuYMgnnYnq-$l#H+x2`- z?cNt(`z8!J5DgRrh1E{7rBVC|kR_jBXrJ% zfo8ZFMLZl$ssJ%HLP)d?<3RLq8t-GbQaN?g!o}ZI=Kf5NlF&Z#>xh-d&6D!NLP(n) zwzngSj7>OppB`U?FCx&IiT&09NYbn%*0jKZS}T7)rMWk&b)5^w^n)Vxz7bJca2dQT*97En zmU`|eAY%=BR%ai=*w-4e9{atC(3LGhI_QuKDK>0UALE?hY0qH1L0iGu<{%dylG?L%qQRVtX29j6lz-PGT9&&1# z+DJ{pV+B6MJ)Q^7{S;$;g;q}=LPNXXW;3)t3g|4P!WmO>B-E9g%YBNptR0I!30^Ut z%evuLHO>{!p}sqaG0z3BZrOCb*~@0*b-*&o%h%5#l=pKa#fG>(Lv+wt+U$&@oS*Nv>rI**xk9N*OIfa=JXK>aSLK7wGRKtH~pX5>sEbclFW?rVnz+*e^5J@{Kut z8T|;qX+KjbT?VHbAs5ZhR+@=KwgCC#3kCf%&oV@jfLnGR_(zVKoQS+BIPL}BhLNhy ztSznE*{Zfh$Ye=_Bt$|M!J?@EKm+*c&QUjC6L;oby?_V`;~vk!lp z9_xYTJKH@iEC9$V9Bj;Nb>dKolP=Wc)m8GbR5&t@|1+BYmoS_ELb@qRbQ%CeB@ocd z_?%nidr(CSBOZS*I1`Tew2S->lNKwzQ?D`Ot*^)=k1)Cn?$s}~PiD)o)Ut-S=|sz= zd94b(V?Ig>ZuTK+94m?YUqIee2&2$Q`4$ebaS$o90zQVaM#$BC!h>(Ll2XZw?uFbWHS{GRuS@*Q0~t2Rwx~v`TCZ zkx7J9q;1YdPK@p1u_OQS8K0cI2{#v5t(4$ec?N&+9i^>FPD`Ip#sXJ>FT65I=00T@ zfDAv2pYr^rykm7D4lA&3o5*Rio}Gvl$M;VmmLNRGq)&AEG@|R}JbD)L#g#!rEzw(2 zL@E(r6>>rkhotcD>FgX(L!pN?6e*Yxm&6?*9AKlIchc-xLsTU}+tsSXfPxL_(I4+K z`yK&;q!;O9ycyUDjLRIXN}}cKan(cKaHpT#=WqQ6DjdRf3AhtX&hk=O-3l7KwHiyt z)t)zW9g3K8(gZD)o7O^n0Vyfm|2+K9tu5V15aY7S;J!IbV^D2Te75hs$63*lx(@;L z+{R_y^X|#!?AmEz2mR}w|5)h{^LQXplJo_1hQmeA;kOwB%0B+M-z#X6Ul@t_)r{v0 z^pW?f1}6c;pXQa?0t6$)<(37oc38%M$X7ltVE7UH{p`3dxYT+O1jpBbuMV69d`M2c z48XE-pAU%Y6WO_*GYBU_7z2{gG}061zFS1(d_iH4`7Iar^NYI7njlmEOxedLj+?4c zHzVLG2$0s*Yv&4H1c%Wgc~%J&%WLH#T{Q|PSlYfFsePj&w}vC|4Re*xH z&9d0)or{NwlYKiA-re_f zU+JZyz?fPHOMtm0| znnJz+fW&D`k zd(V@BsiN9u0{Q8rx+5Yi|M3HU+&dx1_M%!fQW~EXLL@K49G2~Y>oEgzFi+@0v5oV+ z!?DNZLA^fX{F#J^Q#1V;+f5DA4CCr5gf~Ec0iA$2t_`}tcNBp)#(@1u<2W804^)~Z zc5n+B6nB6EQxA0Q5HA*BhP%S!hfYO33!|76Ey*%WXp$RP`k>(fK-8ltL-hNuKz)*tJ)oJ z4;f^vtf2`RZk{XYb%i#_t#w(;b9%>4dhCb8Tm*Uz%}oFeb+)bFFXp-KfZU>_5f*;b!z;UYfY0#vl5SvYj6B+Vo{EZWv^TIsQjgpeS}kbZyO&u-S2XyHZ2*SsJ0 ze#l{hZ(B30532)F7d4tdTAuIWo&m5mWZzOIS~B6zmv+KMoo+;sfZY71;59LXZ*)mZ zJ~Ej-xC+`Z&ba{H(Sg~E0h7u3%L5gz1)ygA9*ma8@~^Z1m6mUA9)>7xWvGu1-~=JG z-glhpZY8)|e||J6GXc1;a-kMv`4}Q<&=n}U!lmQmalkEVX}h2 z2ao~bRR~!Eo{|p^u~F*8_EMFq$p9`F zH{5SARM(?iVDZ9Nsv(TSj#W_n9rx05j#`nMiT7syLPQ;7z9OA4e%M2gQ8Jv$czF7!u~RFd5^C$#@oOo>pGHr9b` zdPpsnL(x;d4P#M!+oIyveSPWhs>$2!()dm6R57=R20wi2ae{z;tJ|ZE#djCH^43}F zwn>x1ja_x60SS$c29|vf*hH~i(sVCbvEA$8b>Y2;Rnk+Yt7u&zHFX8yC3z6ZlsrXP^@^}Y;dec`KfvWSEG zY*!eM;ncE?XH#idpZjU6^%C`|M2h|b@2#Gz!Ykuft2vEFL&Mo7s@%yvhRO8H*D2^W z47|HA4?VHPM~ke1u@!p4v6=5*dKl77ApLVD)R!1@O1IpJt2UnPjM#S2mGkHv_KDL`*qEp`e$4g79}>=k;7)Cw@0PWj)&6(e?u1@28F#;&-FPS=axY3$`eXq%}tmb`$@px+6-NK7$EXW zS#qNWHKuW|g*ArO%yoW*tS8$cB7b>Iz1g30!2)!TZz#Ly%6BwLmQ<2XEq1Nt)UU2h z4aRd%rus_PJvus*L*lA=z{+LZ&8B5^UH`A12^t=~1NjvEKab#l1~@aBy_%-byvf0( z0Uhenv1`SQ>JP3oW-Qve=&nIZQO;KZ@F)Srzr8O%+`wa3oglnp-D|fll>_&E82I0K zANireY6vZPNVk<=>F!D-{N!N6|6C=$FlnI3+qA@;qrI*Y#3IA=t~C-y zHbm1cq(lgy8lYkvW}<(;sGSoH+E%Kd{uKgvH1;H!ms{BO@MHfdLrM1%1xacd8R<$= zxKm&^7T>JgBKOc6N?BeCkL`UjK;0W$V6HQi!gaoQ?k|}Pt>A6q5o^Im!Bk9T6Dz~$ zs`tV4DE_Kw%I6fZD#hwWnnod(Nm^<09kPL#e6{3;S9?@ZoXNjBUH@(Q*bd9y@)?`K zY@5%{5+?Z2`d;uPGukQSp|!iE+`erCP;H7R`PFSt^kId6`X6r%q=$P)H$)ni%25j} zlie%=v;1v)2Rv`f$|66m9PeB#sy-~R@QR6U=@2`9$|f-if2l9XN`3RAi}AKj>4|Ua z-jnK+2=Y!6Ux zc^hk;)o%KQU;H48=U-tz4@dLng|nMM=0aEPzqHNQ4te61w5MvzZEu}RAmOZBTXx4x zfPJ@K(L`Ne+q=LUo3YpZobU5j+D^X^X?T>QE0&w-6kX<|)#tRU%k*EvNDzAne1FEY8y%y?uI-onZ(4;j5MMKBD~%lf6y9tjJU@Mvdc&moV`|h$m~!OVk4`fi ztk;a$Evj#Pw7b>WK}2#OTF_K4YN?i#p%m*xH#_#1&|JY>ZD7RzOt_Ko)n)(!VgjtA zK0lZ8SLg^if^L1Y&BYVTYG*XrJwA``tV+DTOk*%rtiM?&A1O3EStD7`lCF>-w$*Rz z7vsv8`z2NcxVRE)vz5osC2yS#O}E30hzsLJTEcI9n$X-z578#LTQxVq#;^5nb-J;F(eNd4@C-`)v`i(yaRGUwkso|iy#G~i=I6_lq5ME^b^O0XE%f4{BVyLCpm6y>_BCJTS@=qwn|Ve-mRe{! zw~re0MAvp{C%Z=al$Mm3%S-po!Aj?;H?1cz!PT)wVut-4Bb)ODq&1DJe)e z`^fSLXDWK0MHdch*Gj)uy|Ix<-(5XmWBlsoLHF%Mo-E6}Z^v<|W|xMfvad1Y zu47`CTc59Bl&8-++Vm<1s?$olwHwE2bF)*a+K*czyO^zDB*IrER8d1$+@d*Z2AmdYO(b&7QLtUIsKhS|MO-59MTnHP&B<#86AZbF~5zObuXB_|HAxH^062e zYq5U6lN;-lJkF|y8;zy6z4iFocBmsofA;&iq8p8=PYrL5)DMLB^mcgbz|EFry>t`| znjZJg6)ELbJ25;gWH;;i9#MU-k+Au5)<6wvKL(Nko8H-QdhK{Mrssk#Vkv)a;ok~j zW1l^$#^9vTrfX8I{YYh)D%3Fr1eV9ekCOjhxPV&q<89$a^X3;L zhm|iM~K|M@i6H~@rmLt>nZigZ78EPU{bT4e21Af3Xmu6? z^9CUo-vC{2TO3Fgln>THk3$FeRyF`r=>#MToyg6&KB1O0VP2Sb3ul-jz>}20_CcYm zukqVA@6?@)cI2-5@^`|skC>tiQr zj=cb&;q~6aG*Xhv?@z9L$q=d-d2t<7O=9v8^$9>MPXkklm*6H?&80f6V4U^AW6CgO z6bvaqC@_n3Tr8dzMFzCd0t9rj!zVUZB}fDiZr^nJ9(95kmJ&Agmq6#Q)aa2L{S;{d zQyt;xjQmt1=xjTI%D40iz?baXp@hf<0yt-`xXXETCW)=4I&~Z10r7Lp|1}H!Gu$zS z;)7kBe6B;JE4~tp+}eYEMK$Xy0n*c%svH}@8n;KDz>J;lC7$5nJ&55>$uuB64inw; zSZuvK4YX9^@kBstAs)KjrK2&|VUZjnE(7PKD#YenBuw=qI9PLnQfni`LDcfnb>Q`} zEsO(5Mxm}Lf`2@30H$#Tp#S&D_SM&WHD{iE((o(+ekqDlb2dZDoD0EBlR9XQsG|;& z*^yXlhN%(UW0!u0;NXA1EdS-p5Fvmpr#$4p{P8>Fa>W5f+F^YbiQX*OP%`YJ*_Z%E{$sjS%z-t_*#*iS>oF8)SNd$d{b`a=dJh6 z{u|~1aRkK}w^k;8F9s$Yi<>s{xmF4Kf9gNv!w-_4lK2!?&6-1Grj7<0<;iamC1B|e zxT>p&!yj;vE`2cv#wvP`GKx`A9-+U1^qcrMZZJoo;2Z!O6qB=TRKf!uw?BW%DVty~ zu>}-DG5GY>IBnTSH$3Mx#cL3Z_h|(v(HE|T64D+Y+CR_B?x4aHNE+qTYh~qh&&oAZ zK#Yz2&h8ar13fr_iyl)^82o1u{NjEVr~F{c^Z)=4F$gFGY*M*GB!W8yyUxCQWOQG+ z0q6U$@cL&I`)GD%^kY4^aPa~u7$uKASE`qRqBz0UsK{EgM9DNp*v`nL9Kt1`ua>5{ z*E{eLw}e>8WdPQ^9G0Mhiu!>RZ` zM(buq2+yK_<jndJBtj|nTm*$w&k|@=G$6EW+jT&9EjkbXY9sy`IFO436qeJX zG5kfiW+%HNth+8|X=-H6Nh>3wkRG8_4og@JxAt<#X zKQQiI&_7pNTIjIsr~R>MJR42?M1+S0rkh{W&BNT2c3>O`rS-{T3 z=ghPD`Qr+W|NA8PNDEQRK2(TJYqb#a3KVhN{idRjH#NgGeCG*=N~R8P$t0XwM3k!d z+B=dyAl4?7Uk6XBJ}1x|Uk1b^0pDHg`1()AU}?GxxVe4nS8SQAYk*BvAA#s6+BAz%Li_RIQ$9jb6kK z3?P$(V%Ux4fFQiJ1DA%u@6JpUj4Nn-dSLoq@(@q8Jspwu5ptd}03|Ec&lh3F5E>DHSG_`*o=6BG?Vv;}8o` zQ2iM7I7zl7kmu{BpF8H5`sm~gyhzC66X1*V=A}w9XoX`df+%A{N)%&Y=UgUctz%eI4A9RB*gAFh{Q+^EOEW2vl zLgTf|fr|`6SSQfKyZVqoYFZ~gCi!ZBe5&WF|MGP4Nq=&;F7nH5c4v->_K^UbyWM`I z$+UEVCclCpP&ZqA+epe>|A5l5Un|0-PkMU%1E2@g%%qoF4$bBL&4=PWjNnSA2dW2_ z(q7&FQc6W$r}ia){YyHt6JrUlH*y5!4nebmvbI}%lJvBS8m*M(38Qv7o8HT)-ou<= za~c>j*pjpy%FqYa=j}%YEKNYoT~6)h7$jXN_^XI6KR@rcfdEr9L51;|nN9b^=`XM9 zm$6^WNe%(CC3@iL4G$O32F8xu) zh8)F{U)Owf^Xf`Fc+1JqUjt-6+VdVc{fn2L-H5@LEMk85QPmXey#r3p0>`!wqEZ$d zWG6r-zSnKW+Z8$v$X?jXGi%!wR#fcWwJach+W$P$9Nzkk34Zwy%Wt2$DgSw!*NKx( z{beb?{7LUY6xojeN~=5w8Wn{A#St4K3pxl~oZ`c>BB{5ElfQxlACu(XWHAKKgrv}3 zt6vEbrKY|L<7%(D>Gx-Udx*c#m106Y@tfOWtnz+uZV}->VUBBlDVag=ApA<_*3_;< zMcSNtw!kObxLQ&!^(!_!DZ2Y7?x{^<5McAUz&Swpd2NfYjv+*!07Pl})Zs9AHeWpG zNauo^FmMni@%j)v4q`mDZzN*(NkJFV+(cEp;qxC^uBr~_QmG*sX|gH3t)_5ldf@OQ zm$45A`)-eg*z}fnkrRZ5X|uFe+z9dP@>C_|?8#A8{)Z7~0KB9~3FHbN*oROjJNyN80GazDA~ydwA=d6Ch&I z98;qT6JH&ZFydW{pC2l%mLKp$LO|bBKtvb$1GEDBENH^&fgK}w0xCLz=x>?Y0u1m! zCRx16g?+_=7z}Y#1V;U2fBw2P>+l~PJA35X4zY^wiS()k-v)o=A4f?G) zf#Q@?nuO8%-@5eZRRW;8*&yb?bejMX%Kq%J;hwvrdu0g3QuVIz{(sKokDn#0js~F6 zony+$l~zP%r_i&_W~&xLCm)99{dyVzJW%7^|G59fbDOqnna6&5%TQ8}eNKDoINtWo zM#QS^6assEEtW?BM=C@f?+dd$vrgH+%$ff_kd&0Nr^PJKPf|_9F8!Tfg)HMit%SI; zHPNpk28>Wfq;4{aAg+Rg&?Ty``*nrw-tfOpIP#~mm(iBnn*j1mG0^Z&q9t??9^+drc;IfgHoAYy0e7<05Hs}bjrgw%ax)=6 z4;&MQKt7C<_OxI%6xl#XE>X3X#0*gHgTRNR()yK{D9-=y=bCAt2ZQ2k+i4vuf_`#l zDXCx0z9vPmM7D1LtGf^H5>@|GF8D0yXGm&ff5GjakNdy=O^tKVf^5lH{mXIszjvVY zy#p=~;P8@1u%)96`cRGCJ3`|JKQxmkL~JtlgvBn)k=>=>kNuRSU;=eN(wEZ*7x6TK z?@$Y*y%4`g<}66`ph5d>c5u;t9X#B=ON`7+_zJpQ@M5R$M_O?D+)Yugho3S z-vC%ML3)9pv7*})btv)Eq8&GD$Z>+RPE(d3xKsk{w_ZrU_`e^aJp?ofiM?Ce&2CsW zi(OaBe|&V_11dMiK*PZne8^9U2<(e-_&KDZ)t07sJth`hg&Pw)08vxzvNW6q0DgTQ zk3ydA2lHC`>cM~81aXb=kWBxD_w(R=5W-uZN!-B}%hDq^0Dwx<+QoL>)GcVLGj`o! z{+ALemb4~W9cVRnsN;79Gtx%R8e^uYlSw0-hG6s8YpbOQSUJ;wuX zs}r4B*8!uspVJhc=NjwHS^^p!NJec|XTRo6K8rzK_GyGoj=svf`!ks-qLH5CmM8A1 zqx~A`_mU`4wznai;!eA7!KcKWy7{M)GAn#9Kwz>5VNPQfF{*pNmR*Lr7emQgj`R2w= z{T%OwkEiNWcRU+GmEob1ohAL*qF_zu#s6Oi(d-C1iq`zp$72EY6dW1Hc9w<7ooF>e zrl79JK5W-B{`0i|?cYu*GqqbBfNa^-PTUovPGZWf_H(Ou|l3`fmbnCWU#0rUx-u-JyyYre4Xfm~5?gAUIrApM! zEsi1}I`Apqj*E3i3gj%B=e=@a@a5o=N!#-GFm}1#lr*?YUedu zhQVN!#HyUgoUc)_dZ2P7qy%n0`J^vcz}}JZYVFp^I)X5NKb{chlhS48ue~Kw?DmIXYQ^_R5TRWqoy#C!~YmTrihvDy1o3<5) zxz$8+KXwQkp={_%3W_qe3%|a()GHC(j{+p(qI|<>F1NO$Q_s2TEsy-^RU9LS*2ch8 z!d6qgOgiPVb@Wn{J2wBeXgmZnNtp~#*@uTPyfj2wn-9i-d#AT|FQq26A;adLsWFXrPW%8UTP#K9Ols#S4~YZ zX}077hH~etwef@6xa99e_Sl$8v<QKU=l+w%gHX@1=dJaRpa8)iUfG8-ycH?eSnaCKwnMRlw+=U4=GtqVu8=0Z z4W>Ng3fB`r{r}aTR^5S36Yp$Y3lbg8zN%@gw0V1W<=Jx7#_wzX$1dIZ^3+8p3pFA0 ze|(a6)(QOzZw__*6Wrfl=pS*3Zhu?WD%4RGb6b4rxw>^3J^kK$_CCGH%@&eFoF+6z z{k*wRyQ_*HkN(F(OEcvmpHJ(-^lyKRK0^APCuD%T>f8CQNq_7n4bxfk)#~A4o>$*J zIMfPdgM#Iq|G!rUq62W|$+3K+zvWc4T5*#dhF`j=c?_&Dfu}d`CUxJ&ePy};nX7+-zNF3C{D|UIHg6BN%f?+2 z>L9t;2^p3NG3{)FFe&4z=Sx>E|JNr{mae7((Lxgw6C|JT<=rq;k27WA%kq(=s#T1o zMJ6BzxPf4D^WeyDB@1Y^k*#b@xvA5JlD6^mxkMA zd$KIo_&(gR`fbdsoNBHO$H#pJ=y=1^Jh4a47Jp>HYqw}FCW{o(UCT#Ni4RdoZHu2O zIPWW#```P@v?>TawHSykH6I{NMd^hF*YoXtIwhXxz3tU*apKkZ!j7rmYq&PZXs$Bs z4paFLYi6&WRD`9YG3Y(6i6_(WjKg%aP^ynbX|nIspjB<@;ctgmUDb1p zh@S*8NJ-Y({2tGKeq3}AHzG}}pKF4qmvH+`=2)Vg1_LQxFuG9BP>IJI9^qf7doDku zuKU9$Pm;<@{Ts2TIl>;3eg%!CuO}LJwGrU@8E0x)n&80On_%c>_?Z0UU3(vicv^ORqKftctSr*7LsM}`Gy z<-o%=#l2|A!j>&uFETO3H9wH#M>0Mp1bp_#1_SA&C`Y5~eIAQB&!dKNRWB$=%@r-v zM2Ei|Dk?v7tM8pNXXkMkC!IBRi6=b}JvS7*-1YvU6AS@Wm2Xd_#=gjZSS&Wlsb6K! zyf;BL`H#Em$K!?2;sY~sq`omnzl58Ae>iW-Q)w8 zZvwlhn+jwh%PZUD*!jH|IXy3q8Vfv4Vwo!whSc6t( zZhrC=eZMlttk9K)&$so9=7j@S)8{Idws1nO>kj%WUB+72J5*DRARtng9DeDGq{r7s z-dlN(8NDnQKptcbkq5cy5C*Z~N=o>0G-3Nimq~S4cow*`>)E z;GB3Q^YaB;(8Y6$I*V-9<1(#9FBLn{HHf}T2cbFAe!!g&!J zhAB2GRejW8cer4>k4I2Rmqb%&b3L&xSQth0keaFNeM7>T>rKZvLZqqw z-uH+xdYFvHdKOoUe4BKZ+Axu@D}AT6y=L)Y{r-T*&?8UBaBiX1`>dH~x~?3T+&z;4 z-)-uCKV2!XYVpOenG^t@s`IV;kxY15oiGN*Uy+;Gcl0texG}Z^E?xLpk0dgwZCurG z%30Gi?i6JoQ-O$;=18=@)87$ciK~&?9MJ4?&vm`O61M;Y4`jmPG&^`MC_mokS`Y~h z_YLUg6rXjRP|=d4^V%nZ0!B8pMbzFsc~zOPxWwq-fn)+H4$X*G8F1G0K+<5SFUnEw zKu9IG5^%p>+^OVypAz9ZMNACn@Gl4py9ALUz=Vh2S-vW<`vig=acG3LkEI7{L~1tQ zpWV;bSB{D#^Vmv{h0~DdifBuED7)^!_3XS$M$1hQ{d(@?R2QosPQQ=>k zJ(o=nqv$Y6zmH6pL*=TyHg5oc8-%UR@MQQu;9i4V0#qCjPoD67>lw-#;Y3RG=Oi2Y ziA*VmZMtd%&lFW@!lCPz2fVvp&?t`i%-t*s}>KEJG?i$Q-sf~Fo|6ygWzOH0* zVEfC}n8o~W8wIy521ABel^ohJ6`To*%3@pq7Iql}UhD&H=2L3!4GsI#VO;!38k{yEB*rV(1DZO&>r{UEC1GxW`RBxx#&GWvXhKo$Pyk zXal`Y?c;PK>EDAlpC7ydt3~d^3%2V9V<;-(?bf>hU^24Gaix`$T(Y%?PRoafV}P)^ zMSHQ$(&gAc)-^v03G{xhxc$-A1KW#FvsUUW2!%t#waZ@_ifJ!PxEQW3`o0UqoIp)$ z=&o5RMo4rl&-7VTgG_yjc9t#`5p`wyZHp4l4GI+X^mJ%Q@Mm*3Q|%0$s*33T+^b27 zU-+axI8LuGr39t^!y$o1mo-L%l#VXoN-#{Iy5x+ANVPdjAKT^3jeJ*<^o)#cow*g$ z&EoLkB%8kCKe+> z3@lI}4a?CR4r5%D|Kwcy^a?%}mn+O~a5)@x4y7_fP zcMCX_DE4<-KJ+sci_l7S0gr2 zf`If*&i2&@cZZK&AKa=l`T8Ae_C0LbfCT9EHEgb_7bZOJPYBmnjs zj3FXoLKaQO+6f|KY{db7XQ-vV6pzT~;ode>Bm{mu+)tK^IL6&)jj;p~^t`|dO6OK? z%1zTx&w1aZC`4Q%t!VR_Fh+?wwDx`CXx0X&Der<)l`JTQG7(PzdMS=hmB1*j4DFN5Iy zHHsmyhp~$y*4U-}dDYqo9fIK-Piup2s#qO=#SOu-+2w-wEap>_l~cTIwCdNIU?{M? zrQ7`#<3c|iD*YLk##rg^CI#=`2CT5PR@3e^PKpn>j?z2NtfTyHEM2*it;#okXDTWn z0sxpjK%zzb!)Gp+f6mlx-!uSmu!B+=auVAxN$r3!lw#NRaqPbA`j3d^V~UCGCjA_n4?7>ZBwpgVRyIb_xYN4#FK4U12c1%)1DcmqP^+Ai7)y!1 z@~Xg`X{nnvaO0=;ozkOq@!qaKhx!&Os$uikdGl)t$#DJJ(_2OymrGSEJci{R6%n1m>ZnhCT~a>{H#Qf@?KZ$f0Hnh(jN zXwUa$qy(tSqQBj**z@vk#^Y0 zjb0j?SNUqEdoJe9&xgBql>sUKK~+Hsi*bL2R<88kzG=fZTSJ;N0Xa4+e4DFTAMz%X z?9cUhZS@p2$X|3lphofhqV?Hgz1!HY1=rmd0}AD$P>Pj}$-k|#tnpq9oyxEjqI{13 z#CtVS?R6-y@Cjz}*mTrZ&ew-=O;WxdlZ^1?;#p2-7?Zs3U0pvQm6l`WDgICyohL7y z=xzyMRObr_yIry8!lsOXoCHPj--FJb4U7Iv1H-DNhff{wQDuvfeBA5M3mE3kNX9!F z%B)%&P(PIxY0Oc%{}jo7&G)S^f_Wh?dD?|rtsJ7H;Y~}3afmDhcVG(Mk&Z(i>4PmW zp?SeQc9gL}C4uuHNoMOImjBT2F~)R!+A4u&n{r=4yX0-UlzaXUHv z*;jE21ZgbA%#KL>nl9?P0dt(Db9JSFmo{8(d2p`Bcw>k|KRw@VWR9;j%D$d$=9>}v z;bUKhO;1W(y7VOp*R=iN)+8sModdNf!<&e~lwp{fE87wEy)Wfx-LKUhv?{;Tq_1nT z6vIws?uxyafp>yV(Auo8g3ljnN~h{QE&%++Se#$5Uy1apjW;nsF@6f?iFMB}1PG|D zF{-*FZ!^x+J2t6ef*3R$f^K_#D|4 z-3yzP9P8qf;Ue`%Bc;5z0^TT1SO{HX<=pN04NPGRS50F1aL#f0TpVQEBGgPTp3l!u zXe1Ahx(YjO>y!(9^ij?zW* zOj0LXi@G8$!{I$Iea?n9YN@lRga*z=a!(x!a0YBvRwR>7Uluabp$Tzn3thD3 zo7m7eDnYH-JxxU)yTX;}jbT;@>cw2KyMt9;r81LhyxTP0q(^ z6Lh9afV~PLj(uk;enqeNXktu^CR17ybHzL7t#wqeB$ninX54}pw!+l4lJkN!V7pQr zXTJ1oQr3B^>FjOux-w(YIS~8X3_ZYDy6Z`F1$OgIMLq3M1zq8%3c@lCU&G;Vk92P0Tyymn{`|;D>yuI!?qGd3!#rvVLz2y>fp^cRy-l0@YVo_6!fO!%Sx z&hbMk3BFxAU~U(zA?8j{jKD;1c^#@qA-#wiUi8B(4$nZ^PVDiqokj~~o2j1q2-e*Z z(VX*HdZ$9m*s;jGf{jyi*%ETotVv-!mhKAP0SFkEJ5H&+&Yf2AwZ0%vd2+oK1%r4u z>h)sRWWxkPR590#k=)U_qm^@P&EVzM&40W2PK1}M)Kp$!ptkRHU%KywjzdH`C*<$f z40=-XPZNG&{|j}BCNxVsF1e~@djxY#VUl{f;_!jA6*%PX z^r=)g<3-Y=QX-$RmuYUZ8#AUQcUTCu<>~&KI6^DW>7Yu4YZ0x~mSv7>7mu}G%JrtV zg&e_#4I4$b5-aDI`wlv*&W|fg6e#k&yarzAAjk~4NkcQ7Vusr$pX)#2$JbYPRId$PeMLJotD@7o6Fs01VD#nH1 zYBKk|n3Jkcs4-%?(2%^McF^k(M!j{DuLu;91QEeIP=`u!zhn*3gP2o!LPn2~CvC{o z9Y2Biba{5V$V6Rr4!w=OcPZc6I&RF!(f-AZe+C6}zT8W;yx^IO>vbCTjN4e}^Em6u zEwV9yYcxnkJ)hpRL6oU#csEUPIc3ThWSO<8U~1ctz1^Hf8hVY(vpu9jkQ&yc&d*4|p{vOTS%`qLCVw}WF0 zG!czrce3pZ5f?+_`9&4$q5vU^xO5$IIhMvIqDP}(-MYnKX;`?>HNanlB%PefIgTXf zvqlKg?v)g-H{N3m-#y05e_FF$LCDzPN@2@3SJRsBn)d||(1Cmr$A-4E|` zh;7)`1?rXJ3xSWCl!m(;wZ~3zLr2^cKzu=4 z^tb$Kmfyj*XD!@2X?ilwVToPhVE^Lx_%QHS=s+FRT${MlHqH_Gl22&@?QmpyN7KRt zVqT!`ymce03p8TOP_?+XJ*RJ+vwwf6^I}B-`?zY7>|>V_xu*KzhH%1q`mxG~or_7@ zZiTB&ecs+036&r>r|O>?c&gfpaVag;Vv~BnwQIJXJ*AK>ss5qCnd62W9;==Do$t%NVuw+iY2Q%PtyP4T z*Pq=r8ey95%Ti}`$0i1!ARq4mLDxE!eV=OqU2o@tPGF68NwAsA9@oY}9RppATQ z@v!930EdQ^JElQa>El?Tq|vJAnPBNmeDHrKcV;XD$Xnh7U(eD4@bSqZVD)Un(mG-LZJ?Ga0ZkGZA6R>J=U$!hS`0XNQq4_v`6slHlN&m=M=NVt zU53ZZn`5FA636eq1Ha*>6y;ckR7S&^^cy97ZtuIkaZG1T#UYkNz{?VA31+;x;v0Va z;^g98fmbq>j0!JJ5zV|g`m?#V@J;CPJ1kH5Q|_$qH)4#vJnJME9Tqu2xsLLwHq@(n z@7%xTUe;623n-{=8L}(bH2K8SH+$^9R;$c~PYN>AxRtHCBGg76)H5!Dit$$@Vf~EE zroqL*PFrr2-4yGeMZpq9*zI)A6h`+5r5Vq8IrUCusUeF9wd5&L9*&p3 zvWWx%){$0>-Z%UPJlaFkv{~AE`pFooH5am{8E5e>xV!Fs`Pfzx*EjbY6)TN34*LxO zmX#X|-TEd&2MXL0Qxa24NzP;k`k(E;q}VZhsPM4yHA<UsR;6i21|Z$zA5UvI|9-6Sxi|Y^{@( z{z{s|3;7M)fyJT?zdEERr3IdP5M2spuz@Vqk?CZDysofiOO4Xm@l)Z$>b0(J_H*F7 z@?a(CHNVe2yG)+Qb;>mm1nh(F9^OWGkDoV6cH+4&1^s~Qq(sNv5N=73 zWv8#u|2&TP^<}A>D^;_Z-hi&aW>1JVYFj}`Jr-sJ!cq;XULV^KUAzH5btTbviX7@Q z#zsPMmR!A15-;_q$-IG9c*68d7@>s<*}xe=se-R{*I;DC_Y(L^;lyOZrMR`t{A!-Z z6FQLJ<;jkP>t;nu};d1$REXEV&Lt7uLzG{F6}JCT~yq|l9< z=oo@Zc8a2kL8Z`K<(Fv+p6)yuEqqvXsix6#>`>*z!6M&u=C{I!Rh? z+W7R`v^$ElrEFAlz&yGJ3sc5+Klq6lLKssL`X1y#9`v)*-lhekzrrP@&elbN_d_LnDu-t&qvrM(yPZ_ zmJ-oO{u0`Xs2?7W#DX$C0sr-@YTFEBESd6krvX;%y+zw*x2kY{G=w3xXO@@mHxYsR zR4<_sJ_( zNUOAkQ1#DV?oB9jUVxTq8O9fzy1IF4SunW{K?)A4w9c@m1#JSM?~BtMGbgdbTYTVU zs1(&S84d5p_*a3_rD2*Yfn*9weQ>3x-!E1^ENQvr0!69&kd^4Y?+jN~Ob!?Jg8*Up zss#vn&TM(w<4k{=JbmcJ6D?RCM?EZ1mvfamGMknf^Wey_bbiB$(4qx>HGY8|Z?Y>x zf?WnwuLc)zZSQhodgmL=$Ij0}biSS8$8%n=Y4wGCm(;h=-v8t6E#s=(w)SCNvP40_ z0NIEF21=teD4~df^dh7YX{1wnfnty8u5hh^+iRz8owQ(Mk1c1M4H|s{1|iYwQi{%Pmobq9T)8T^&a^AAeQ-2zv)udMllsLt23-|2%`@+bOVux})QsW7df4qu;C zSBX_Kt z@E5ppy9g#=&1(b`Pb^%(TUbM4BPjjcmB*x}Y8`-++&7A7A(+CC8)%@PO&-1M)S^$(|>>M_f45 zZB@X?LQf~#?H|gta;Na!iPzFLH;KALwKev}cC1}W8~7}B>nwCf>*}Vq?9PnkIR}o- z@l*x#MJ{dhkIzX>-W2O;U0$91Py-a*X}B!20bVS!HZ7b4`B7n`SZ`&zOL61VY^5w} zMr{4&%G4(xEplutF{0DfDQkPZ8Qe1FV<7XwL!IQvel;5p%2H?3<%28mY1cZ*ler&N9w{2~pt|X{@U7NyJo1;^rb26@J&GRa zjrze0;uqOS9I?wFmata|4wS>e)FN0{BEz<}evtXG>&PtEAifNs_W#wO)40W-3SxT} zY%Z^dy%9w3StPDy$dXWYSH_?#vdww7R#?Q; zh`s6710SI!k%F*zIvVe=wwiP|evV=0?Z90WF~FA`JLWfa!0HoMmqv7I<&|@JK7g4s zfVf$O&f?e9g^c34^@$Eh7)1%WjXTe~@#jdDmxFffaDr6NNtNQrXR-GK3x)Uz2n{C3 zZc>^-!rGu>U|ZyY@CU%%pK5yVo+Q$$Vl~s<-q(8CmpurKF+KuHJd$_3*zb)4cRuw$ zvyXoCF$n5QYeN9CSe=mXjZB~_I<#>{A`+eh7hOx!*kvFY*hKQsAHz`Y`oG3R8t5l( zHh;KNwbk`Tac?35)7_7x^8nawQLoRuxt$8E2b0QcXs^Ub^m_A@e;>;%j8G;CDg!Ls95n{L4Dd(qF{0Zj-^ zlh_Tywv8M@DV{k~an1K&*}lsb17*Nf31f34Ja{^)#stn336O%mo=|xPQGzyiC2B5# zph_BUnMejRSqx4wLH2WM@uJuou@(g9h-0#+LIdFXD?k$pd&rUHzIh`!L4jG(5G@Y} z_mTGu_0F=M&c_#=ZcQ!8Q*bu9uU1Y6F;-xVt-E*%B@z`d`!EGhpIMRv7Q^90y0u)3~c%G(!F1I0Jb| z)OhX3$S4;-=*IIMx9>q8P2zo*uMPu$Alzm6(-#E`I9}UlTS5 zx7Bc>)MO*J^YRXRHzJ2kOYC7BhjlzAEp$Y5< zcrJwT$BBm`nd*0i0?b4rJ;yBt=qQ@?$Y4Zc5&7ESQ~|tO-JqK{1!>(OKDGiT27@#J z&r?k|d+&UQbTHN{Qaf^ZSdE9(kDZ2`&UHAOKzFoqeyDrg~#_6b_=ftif$n>+6Z z^FR!^rNj}yC0m-GXc8OWDYuA(Z~z-T4X8M1BVmt6SdS8HN+E$mm?6_GU7KDL6%c;_1(U&-i`jzoPpFtFHW zH_QYLzKt$XW;+`tk&nMSj!m)by&EZ>LDX&P*Fywjc5Q)$36p@cu!A|6xXTggN(}n) z6e%_1$u7YbY!18Dtr#om?8t#HNhLI*0_*d)*rs-6{4!AQ_=_%p$xRD46 zLm|Gu(Zl|FfB>Q44Qsk)If~*bj&l5j%JP#y!AQ(}2sMn@URfpzRyBz`ggi6_&zV_E z9w)&?NL(w}$32(zZ15z=U8Xv_fMMvO=-9J!07d-8pz)dpHW{uoOAZRjQ0{tkwn}A<@G(pnx&~lBxL^;v>7)dvW_g$8p|p*Eier=e%Ccit1!Yct=Op zBM##Z@;90gzmiw{;qHnb_yVno2UhiF-S;uf7(unCrT0oxzgoit235j9J^WHQ-a|7Q+gsk;h3=%LvB15sp$GK7t zvDlfp$nr;?-@gkv6K@F!TnRsi#y&XtfPDR%w%O1a--A!g4UOhR8bTsQLOb2_Lnm*N zv}AuUP;S}SQ3&w~sD#_)2KI4Qh{v>0*ZP#-N}Kx2tu4jVrPR39eF-$b%^W$&q?6Z@v`L{(m_JKVC-lS{m&KScyMl|vv;}dDS z<|D3~b@HM&{6m~%Ic`m6si3u(q?NxzGQ^VJFIH4<7JS*G?VV(g*=Tvb5n>p6Qv<@= z?JiL%+Y;SWIq4L^JOQ99yS?<5R1C}$!Vm1ltZfN3V)cHjirL?mC6L(#8{7(ZWG2dP zvzoAv(Um!PPv8w^dFBp&!)N*6b-tA+V;A$J*gbE%8Ui;)%|k9t=$@0p6%`1ycw^N) zu^G`KNSDj8sdF1scR7cCp8_Vsp*7?3%J#aCA|7g#5{C*~af>|P(?Du-N>MAyg>X2h z<4H16bKtVkTSCm5Py6j^cTlebcRx`V(B(7O{3@|-8KgH!|3Di4qk_NtKsf6d;}w~% zTLIWLBrOcpvJ608dH@xi1~e!W_!Tuy?qrT;JnJ1Bh8U3PFBno%P#m9#cSVrL*UBQn;$h`~S;XrQY=QYvCZ9OeU; zFVU@-|1f;eHbLl2Yb!cdFv&fFtz|8$gDAsCNN@^-wRCXeZvmq@#eBHZX259;aeELS zsA;uYCi@=``Pd5|;ZSbl>D(R@%YYQs0(Sm@2;4hFHw>wDLp1G>?$85ad!^L(i9uDF z4Jv&kCrB+1P%DId;2F!>^26!%#Z_$B)ef3Mq&}&5D*XQDGx1XCi5q_eaQt&=e&21_ z?Bh*xORaHIdEgTlBjJu!Fe%y44dpeDcA76IL~I=IW%zw0LuR-SF9rKa<$7;~>q3{Y}{6=?I zM#Ad({_mXz3CI#_sR^S25}YixaG5`bH3x4M`%oy$5K(_esmosWwA)ci_L1xvgLj|6 zOg$K7(8}++k{v$0@5lb)+m8$KEx3R4`Twy5zn-)|{s(kB;MtV|Ofm+E{Rov9f)z`2 z8l4%vw7}I3nkp~O8IhRE2HxQ#N#Iea25vBjQ5+JR%@7nY*mPI<@$Y0vtQoi%d3dJ# z{AktwSdO24V40J^#P~4r+J6z0*CEsl4CSpkmvIrmlxWKM>;eu zI9g~QlQ=3r`B#?2kB{;9v*aHiyhR3ZH>--c{|0gV^c&D}Au_A~zkj3!?sjb3e@*N9 zb7i&3v6hVjonTrB{-$FMj3T;WKaS}Ah}LKy#Y@(|Gi`tNFyL##(?rZFxy&}#?lk_s zdbG7fS3Wzw!!IBC(=6qO?(H87{Ez?3o`g7UP6g&<TD ztT=5+;K5y&{e5rCMc0`BXjFc(AEVg-`dF4w^WIvU%L8!%!jAG6W07V-*f!iKU`ghH zNCGkBX~Er4{_`Nb`&xK4tYCV*`IGZmLZ7o)9}6tErkH8{`cZ|K-92ZO{&>TFa_!v$ zzu0E_P--FPUJ%loJSW{-d8q0i8ig3M&`O}m##5{M@eW923#7gHa2iKe9;=o{4!g-& zze&o(9xKEhLzvd`@JO8sI7eD-mAfm@Dzes=*PRNUx~bRHe1^16F3N#De>@`gC(rZ8 zQxjfB2Yq6Cg$ogLhHil1`R!)^#9zS*vh~zXUVMklpF2=Xk=Azi{JHRJrw&zb(aq~b z;=##ry{-QoVWyifBhh$&t3bSi%$l~@;Zi;KwTwTi{-6BxFNhiO;>@?Of=5*(*G_Ru zgpkw(b2tpXW@%V>j65JMrNOTmZ?T=(O`6j9%88uWFw2IQ-&0$8{&?K?iASmfsuszZ z4WF;gtez$F%iAd1?-GtC&=M71OZ;Q)GwA)8{GRQAe}Qu+2|>Tq_r}Np4ZU9l)yOWa z7Dw5qom5>`fAIqcqx|)`sEJ6FI$nAFDtmjw5-O7o7F&}OIk#U18^YRnqrt}Rp-LWg z`T;_p4VI}K$+Rc1MON}M{(Q{GR8SkQ$ZT8dXs(^ve7^H_k`Di662-SP!qb=Z`&F(b z9|gglV0EpbQ3&^Z7UX?jRVcb&`eVVdWQWJ`l4i{g=YDJ>ZO8rRDLz;oU7P8lkj3;D z^CeqNpa`CO#HYhAPbne1f7YI*)kxXwuuDC0e--;5FRJj7L)x{v=tjmF_Lr5yrsF7- zIe|Z)+*&+(R{rC~wo)i1X0T+|vbvP>Tbh|zP8D1W~i z*b$#2nL>pc9EECw;5Cgw4idqUL);JH@7Zy#>b1)ur&K>`EV=!4D;&Fk!mgQbH1Jjt zg<~2P$rwj*3k~nQg$`6yw|}wdYr@ARj99ZxMzrDYnd!eP0vsH91&0U{6;e(w+QMjQ ztp|Z6R3PD1v+^u^j$S^o*z@0iQCc6@{#F>!_YaTbeH52BqWXz{>|WYoVmNXAQ-|ON z6fq=f%%#YDOOCug?uVb2;+~a`>ZA`fsr|XP4%5>&N(`%Ayni&b{EtJP;S{|>a!`A` z%qQp@MmuB6UIMjm1K#%^k3CpMcn*!9X;CoFA#Q#hzLKp0dszqW4j@zc0&T+ z6W$s7vC4nDZ(rPrLLC1b)=VI+5Oe)b`j0g~R;`KxKc0vQfn#*51kl{-fXFBlAc<}S zWQARx(Rt6_e-Z`ALngF#%0TQ^!)dq`V8X)=9;==Q`K|8y{px)`_SoNk=x+xj+{~GJ zjV9oQk_$rbuWcX9Z+5(@QFF zRe(lr6iBQw-6TBr!U-wK_f9v9{8-$doc(t%??>O|G;FI*R7{hs#90*{&6w)}cRi3#mouwPb(D8nw-_ z=i0M)_kH>+^%nWSv`a+#lecjy^~Ifmug}3@4Ne-Q6F0$B=Vph}uZMee2&zWA6rOG} zpWW?nec(tHDlUKKM&1?B*1GEFADa4ce*VvrEapQQ*y}uo8vC38y>yf-82y%khm8T^ z$JPZ2u1c{E@+N$ggI0I8f#A(`f#JEk&z>Q^rAVon%m?DQcVM+j;QJ8nZE&5Y zyoDJEI`v<6-`{t~-Rv^h);E3Hvlqz-+|VEAUd}sQTK}rg#siy$=oE9%pa{4;+9`we ztkFOT9-;ep_f>PQK>yR?45{AY91*QYM}U{i4OMvS*>|j4j|xn`pw9_&7FdM+g;nw2 zAEGehp)&&ZoG}XMZW%BOJ5+l4<6Q9IqgJDh!E$$`i2%FJWHpDGoDLXBEKiDw=Q2e; z2eB9uH4fFhi1_N&?(h(ZCPl1y4w9VtWhN1$s=>U*Lg)VI)rSB0AFUznG3F+t&j~r_ z+THWOee;j=`~UcpHb0hKr{-(Sy|?FmrpmAfSjT08&jb6`7_)B*tO{p9Uv*A;*+mN+ zqtn4AQgpdCnTq_ELa3vRp!=yidiCAbyA#adt`%KX@b5R6tMDnZCeB4J!?qW;BQPG# zfkUW(VKfc!`)khI`NS`#J5oXcf@bMo`U+r5^GVHD8{NZ;yWMb5y0s76hWG*+PRjE9fB0^GpV6FW@rpLHy#@ePE-UPe0S@yO0(CVZ zp)MD`^Z-Dm6!;W2pUD-2*!^xZf;PvxYInxeB@4KELAdNaz$6fNZP?1JNGf0-IM*H4 z<(J2QG3c3U1nfe5r^$Z-IfxO{?*!S{Waztb%!vRykbViWUmUsT@a^U0$AH4BlOyPO z=Nw|V19Hb%Q>nU-kX2<29+U`r0{o(ZX^6VZ27;6_VxP{s5P+Cc<`hA$nhM@Nhn!8l z7XWV^qHzRoxDYY=1ZH}%tmBHE24K$Fpt94nVhHh$eHOJW=f)sAu@BfYLyp;xKylQO zXR2W<4u>2R4K3%YT!8d8t30VLEu4}1VR9BaM!;B^M;@5c2bLksn}ab4-yyf>n4THYp=mH; zO93xI?Ex4`O(XIbI4iqBp^yxv(YY-cz-iLBY-bRo)W9Q~#%la5C%C6Oz?-c`I|3%8 zL&hXeBu#n}M-b=WhVW}Wu&yGLsxWFQOKz?rz~Vor#)5Q;4}#v&Gd0S zKd?3@G)P_LOXJ&JhDqv=xAuGWuTRhFk0@{wOoS)ad$=u4V}OW^{#f+;e;_%ups#LgaFrFl14IH%%hr=Id9eD*>-%mCgu(m%B=qt~V$`cOsFSD;#>_ zwS3Knc}q-7Zl(zm>W+TSs1lE2BtMi?f~2ytI>dJ)s0xycY^ppgQz^~njdrc6Loy#jXVJ=-A zdU<#mU#WEFg8xH-abYfz-wM#dDOTj2Prq$IaD4=HS%qm-XZXp9$hPs&F!D3X;G>3^ z^)I#Lba1r=L9MNa=#v0D%E~I-m1?z%1s&HCt#<5m0UkKEks>kY7~z zIHLUWnQJz6WMl~wMO@zbZcJP`#fqtr)WFmC1ZWv*Q3zD5+976Ix>gKu8;^G}>S810 zGvq0?CWB>jZ)(BlX}}0WLJw}kNPKxME_((JuaHV*P%`r1w;+FY*t+U8kI6JUma*Cb zN#C4`Md{$hBZw}^mA$9nO%yaVT&;KqW`3VGmnVivO2IR)Z@t)D1B8}1*9Q^@(sJ7y z@6b54A*d((LZQTk?JvP||CsT-{|qq6#Dn1+vc#r=qj0pq?)tEKVeH@oiH+1L`FQDb zE~O|bL(R*`9q;bh+@$Tf(mHJi*q23~<&*>GO61}R|6iQIf*Z$j{H;pv30 zfuQs3`b4_d3U@RS+|d}u+*|Gc{too(Lc`7p%U~vaSs~)e0K4@* zh1@UG?1pXhF7`G$hmVqwtoRho7EW!~z4H}V4a(zd$igwNxn0DELk*RIBxnaAG-W(F{YMs#0c%X60lw z<9WynGoQir)Y`_Q^Zf`3i8UeNuFN1V(<%`$ZRW(=$FOTx`}qw@pT!j0neQnx{9+C1 z6|W_|vm~&&zSkFD^z?({I{f{drp{a&i2AnR7?q%u8)j;r4s{EMD`PT5w#nR1=Ac~P zeT4Rjj% zWs&@68UFWD?=M#c;v`~TNy(~dvOK;gdGh%WFMwtwT{JM5{md;&R(VcQd)Q2$KwV9A zT23ZC0!8}C6D7^%{Uk;f>{VP0mjIRDcpejDjAmdQ{tox4xDwq3a`fxTp#XqW>^vEq zS1Zr)NYqj%aKf=B;Z*WuNjFebzG|F5p8ivU>Nlo-xiuuF;n5Kx$>3`N?HsZTIIm7Y zbmb)+{nrPrRY|>OpN=j>L{IeMY3reuzjK|n+o`o}wdO=pG{DTS+ozH`_oF=!liP;& zo>NSX=9bBoVvXZ>a3|>HCwql2ZjQwJGUQBp+uup}8;SbA9q(B2i&4U5z@g}%dF^(% zGet$OMs@>UlKjD2=#D-p_K~!|z5KyvVCt7IucKf5{hf-N?K5VksaQoTNxRAhV4yrH zx*!^{axC~6uarv44MgKKyU^@FRhL!a}cQsPmR>s&C47<*No$=Sycf{iTS=G7+cHlo+ukF7X zlQ4mIs!0uL`=&-5&I5S=t?<+>TG^L218p9e?u_&WP@wIu$Z;zOsM`&p10QCC!N1KM ze(m<8Y@rEoo7FZjrRJg$LJJj3kgRP}n(_LAP&ybnb~7rnnu5^$ZO__dl4;S-&ZdAq zREZys((Fb+Bgx`tzmX{1-LIffxST+7@2u5$(?hCjt69PoTZZ64YCMSBtA~AH5WpBf z9AlqqJkz5=Glq8ROck1r4fFsj5G#En@RzWu1zCBfqB`Z!JFxxty=K~*{}SZ$jw<

    9^;{W3;{=sDIeA>Q~k4reLfTYN{`guz8R#FFRxGv9Eh5R`IRZykAz-Fq+n_ z#c0CfYErK~TbsOEtbcR$Zt|-8L7MzLE2{X6<{Cs73$^(JD><>`9m_Uo$Vs(1g^016 z5qeF`XpNryWfuHUqOf1sQ=O3ntBWwF`LUG!=P5XpP0P{!z2a;CN+%Sx2l!v!`t_0D z$1c1f?@tkM4kkIPgD8V?T?-+|i+EK-RNK0byWYi@eiHH-44x|;2mw2uWw=!9Rp;6O zZ_*7PzhEwVmc1$JRZN;K9^29x@|iRXuZjgz5)*nupD8WS*>6BUMNU${(F+s*7Sw$` zySG%<-`@cD@*Sco?@ji*wdO}AZ^o#Vb~^+f0*qX)XX__uHyY__;Vp{U91D|ny#gPR=t_grUfyb6WilZqY@jPR1JDXIMAu&=qbqp zNUL)e3wR&p4spjKgo>v54&a8U@ykVdmS>V{W9@Bs#hRkx z_T|JL0K_Y(kFYXr^CPoB-4lko;8s8EV5w&zlmGL+7iA*sy^*fK&2Y`i>YEobK% zew3rwm3E9W`56Dcsv7HZtL<6kNDknJzv&($JP_*16>0dmyvR%af<-#bp472!=z|Mu z&mr;&Vt$16FAECT6F`&=XU1iXqK8%{l--dq?Y&vz{sH9KQs_j`j%IX8RSytTY$N?K z2x6u6PBtGwkE_(CJnmh5x~6Gm`rRVN1*@;bGy^^9d~>MN<&+mn6!E)sDooLM`WB1x zIg8_+9^0vocg=LegFTHWEc!9$x^`ia3j{@IFWt;4tI-1$`_`tk)c|HhGIPD|0AnBG zKMdZJA2Ki8N-OPMe5?`jOHC9houdz`Cz82NO!xunGqOhA6)TG`0WtnjNd)A0bNK(^v*;vfurPpfiCf;fQjz)^DNn3Yd@A9h3h;<$d>2=;<1x?=MG+Dzv+O(#))V4XkkmXub&QHx z8(nUN<-N1bFn=kfCvF;-b|BNWKFos{szf8mzq{!9egl|hePEAo3S%J0EUK#Xgl?SH zR)iS#WCyO&ihK1-S|?oe7;sp^b%pGfSKX7unV?GVt#n9oBqbESVp%o-BGNpxPPYm~ z^xk#bhE`eV&sK?cn(7IRtp&et;#w}5(>?m_W}8!Rz~uR`^|+8iXME9)MCHOMw4RzL zs*<&-pezu2@cv5;42DosI+9~|M%*T2xJUM$TCLO0^+>W@5io~pX82Qn8^R7ZaFecS zBMkk013x&rWJ0>?70fzD_JHhPj`c5e1gGxaam-w;D;5!TY{7Hzxi zXRJ3b>>-@MjCKM!2g7f+*4S9-gHSHt>cJqd<=bOyPuE^4&H9?Sf<8>MJmE#? zTc@q>U!5D7)5>dXkPPi!BP4A>!PH{SC>@ z-|#6H8OyrlQhE1)G;1WXST=Kn&REY=E`tp`1u3tj-`m`Ku%rb-5%>C-Lj*(tj1op- z-Dj9B;K3A*rEYJw^Q4QG*}nB61pz>x>R0`mY0vowgOaB>m$l#EOQJh0`bW~PUloCL zK-Y+1Wg}r%hOJ@X7SEfk8@usrwaH6>Hra{69vDLm7OYZ0Rc|I zC05nUkm)D4{+Yzbbeiz#;poF97Y830)AN__+H5+s@D3JP8VK8=}M*2Ow^YI?x ze}I!RAvc{#=zsglp{couXZ#=#TSD6n2)b^9KY$h4VBAWDd9e-PQL?xSXe~{^bS0 zIl2fQA7pcQke`(RYF3{!3%;y{x2vdLrSXpmSE-JISSw}@EH;WkB|eY~F^$UmWxW(_ zKZD8Kyne_uu7g3@JVszT`Q_YrxW02Yaoj0WCVH?E;qEK#eTZoXYQNe09ZfN}=pobk&i0iw2AHJ#Bj4@dMA!DI2-t zL;vb~8d*}BD`>H0DU;s~HTecM1GxyY5UMt6VwQh39Juj~h-7a!(NT7DZ)N>ZZ+9Oe zO~rS5t6$rD9c#fY)H@lft~c$M(`&H{JCc~5A$LR((Z3Dma4E3{ME@8g23_7q8I-~{ zF5vR5W(SPht`hA#qAa?|M|DxF;Lh7x8m^g8N%` zle1TW0l++T%tYDh6wmoWz%ot5nusDAX6;Sc3E7XK-p{1}B#$$r-O&Bs^r8oxT({aS)TN!{wDX2s?)1;3u5_Fwu2jR+k9S z4BO(&WcX7BXf3lz8X69KWaYHx4G+v;wv6&epiH1Uq?WkFAv5=9bL zs6f*84I10&wt)&5p(6?<7y`tAGt|Cq(q6=QJ2~44hBfK*9C|ptj*=A7J5;FM4`)j3 z>q+^_XSG%-1G0HvQ=xog!9@p=M%0>D%+h_V_QVve9LSc~95^(0=G{>`d^@WRb~TLn z=_)y9KIh4*gK>4(WqbC9*^QemLh$5Wywt+vv1(N^vF^uN$}4p-&dkQ0vN>Mhux%Ot zm@9yVzEGXAIhQ1Mz+c4sJU2xbMuO)|HXgB4=99_ql`Efwv28<1mJ?LjT)Ibh=48!b zlAyK*&`{-=2zGSc)~bNy$1DDI--^rQU6)&*juQw;?D2^{74MQ`>&0@Vp(Y7Sp*TUD5L^~13dUDMw17#Zz^sB0c%2`5M z9`UmEnYZ|>k%rP4oP{qt!1r;Ku#A!-^LBfn;5R9tbch=$_p=sd`WjQkr!U7diJ)y% z_tzGO00dd-jY9x3D+j20qI}x`@&4@hs%J!&rIEuOQasX9K5%y{w!iXy3PUOfWwlmt zYWhIOZ8}Q$f%4!4l`ak3$TfWQt|u-Uc+)59^&2Adl<*rWK1ns(Yf+b#75ZTBFBB$$ z-5)-#%Y89fnA^PTJ?fpV=W2Ir5g2Rwc$z1Iyp6-46ke9xT*5gp44fkj3B`H>=p>+X2ou>3O2fudzwa{US7z}!26i1a6` zL6x|$nv~E_6`f(BnYTA|Wyh)`Tk~>wj4NVPyS6HFj9nSy-ZJZTiEcqP&v0=yXH<6A zb}@4oLT<~wV&lLUFP9Nac%9by_>PYGFN9a-hN3rOJr*vZb9!dAG-aLIIv77vv7w_7>?kn#n^6ViRtt`i1y z&xs(OXkrU&sUap&_^xVvV4=qZ#Ln~|o)Hg*?!OxqplAQ#Y^vyW)JepSljEYz%2bCB zxiz-gZbyQ$Ir(WS(SDWvSGE_FMH2|cc}OY>Bi+w_TpO;`5SD~fY4wJWBhj}iIFyD& zo=|pufyw8RqF6jpm+G3T=Tve+L1t)b=s!YTu!o5QJsJ0D}_NQ-!UzY^`buWE&z4y9*Bm9QPu3|g3aCt^cO%qHv zz;&z_B|B4+ym^!RhW>s4@B}`Nurw}ZpB#q^+?*&=ZM3W~Y=-y%kRx0$iusrt#0qn@ z%}ZpRD+WLNP>$7AyM`5sg|5DmW}vbi3-No5I^E!7(Sp9Hdd~3YZaRFh<$8{{f~VVc zugeq+kq)hwC4EH&iKb2@c|2RVny?<#?x8HW7AO1zCwL?mQJt#&O_6+V?B? z+kWv~3emhYa9eRj{xJHUvO)(y!4Vv+XpFLG@MA*XkdquIq`4cutm=D_vDhk!J;7_| zBHYu=coWpRC$zg9hUoUkj?Gh5P{05)k3%~ZzlfqpO^R?-@22Q#e3Oy*S=qU1L%|p6FHj6@(&GVsD*`=18|qd=-JV&2RkUBT7iR(bBp)9mYvq#L=lx6mW15-l1gob9VAqlV}ul?f8ht4u<9 zRzthc7aKNCAi@>Lkw8j~Jdgj!-g`zxm2FX@ErN;x5JXf2k*Gw;l2JfJC`2SkR&qv( zLJ>q!KqX7gIfLX3DySei78#YCC1?27u|>MI``-J#@y2+6-XBLwg^G(;GM@z~-u249x{k56 zT1-0{LjQ?BnD}jU^I5-1eO|}0bJq^1-7vXBsl(8GzpME~xZ=*G#^zlA2ovJYF2GDI z8WYMX?u)n6mcR8#CC-R7PcUvY&rew;>rN&{QlV5g`+E+CtahQr*;q^xvyJ-ATP(6N_R8I9Ku9tECmVBh7uF}4X%_oGW-)iqz;x#gip72+JF`zIj<3&s zH`6o{nJ5i7+p#nrDdL|XxD1oRX|qSsFVXwd7_ePGY#GEnIi&pp<|Orw@bUf0U$ooz zICLQ==~;xLZWF%H<|QS+Pj?eP-#jLg_ND|Z7<=Nv;In6k&jIcSRDhUX_X@E>&j`IS zUC~p`c@m^1PaLBn_=q*PuI#Cm?WQ)kP2}^8_*WbgEKff&C z2CpKzSPDR9yaADQI@BmTfOHEmPxu0Dg4?TF&|nThAP!`;(4rrI);D+V4sh-xgl~C> zEHcoxnd*gG1>}L+9wI!CFa$%9w`jsJ69~B>>NDZ(*7}s|*?bA>0V9D|=J4f_=bfB0 zLdOm6`~$pJ`^6)|bGb&ChAn(_jW5lW%6dIX*G>afT>V#Pb@1fxCuI;LPK!Vqfb;k1 zzt83VS=j-<;zH0=p!v~*bQDVj5nch*k%d=N!wLw69XNW5ptuuC+GYrCA1)DX=gZUV zy;Qlr&jQZzSsMo&V<(?xeYzY82%#s!{N({e&EXq2DI+-7f33Cxa86Cw+3AL{3^2O~ z_TB|_6G%N;LqlFeTFBCq<>jMm`{;q~LLP_)K**{7_58h;cL1*U7O*=9(3fDk^ZOJ0 z{tqtgeJEzYNWkROQxf&>ECqo{ z00%`N1EH-&KWN&2LU3&ep(aADj!YT6?$PUbr|)?&XI-=;i$q3O8HXhmq_3Ek`Kra3 zK1>?%9KX45B~p~%Xz?dV>?i0F?tZ)_{`56|?l67N67kQauC{YV zq_ChvI!u1L3yrV~c|D|WCcn*3cEkafMJC@^iIo50{Z)U)dT(Sk*)1u$N{_EZ8Vl~5 zB8LzCdAYy6az2GS52Il^h!Alu+i zJ^iDU_^-eC+YcU4dJ9RfU*X=p1?MUtt>vEca;Ez2?wo^grvRA^A>+xcGKA9AoscFC zr4VOZGfJ2W3V=cMf>|oh%kNO= z$<4;*nSp@-?lRB}<9qlmiVS816C7Lvp!V1akfEz2iI)-XcR&W2KexK`E5(AaDItY0 z2Y0xM=YP_C%9gqd@Iqkfh?QGO6s`w0v(Bh1Po{wNL_%RF+Lb&TVP)!?zkcKlm$5z! z-X2aXhrHjoE{N56?LN_A85jnLMS93*xJ}Q>(R6Pe=7gXYEq=jl>+{gv4~b|Bs4R)W|L>7;;y0tLaPiS}f`*ditL{xo!d zmX<5RbYfZzNye2)4h!Irq*eKV_4k_rwxJx@P(^D~7E{08!e30#n+6LJCI`M#zNvD@ zttG|4qT5|8W+l^HNUlt3EtoqGEII}NR6C*;He##OX%FzK%8p1N2(lV)dCw76iX{(f zWk$eya4dy%db_<7xcJq$T4Tgsbc3LT7I>{31axJ;Ddrp-Jq-FPxyZ@$>VXx&#G+G! z_a-HO7_&2h^XG~w5e2sYOl0C+^qpiEsW<@x>vy_CD3|Na;8o^20*PfvU>5=h%?HJS zOL;LG90(rFa`R$h*jFss_OC&#TE+kvZswLg^4ro7P=gH+97HPmzwN}`_duHd#6{UE zL+B`4^8=l7+N^L3Ih2b`K20TqW`^p@*~xS21S#1*3dyqX0Tpcsjy4y7kOCbgF#^9m zrZPC!KdusX0zwK@5&Dk#0J*;73Toig?BfxzPiHCOrq{O+h9SU!f2X;H@?CJT&>3+G zaooi_>y2NICWpr2y?F8)?BBVLq!VlfL48jrLFyP>QF#en9moRv|&ba0_ku9q>*FGBOB+p z{VIyU;6y`mGi5%vLD^Kbqbb)+31CoZD58^@ zSD(L9)(!B=boH`sgw7|VV@wcw3)w&yc%Xej7m_SZ1h30YI%f-0)C$m5)}CA%{s!=N zY~Xj+M@~+lpKc~cPq7xvK*U}E4i3A$Rf=N+Qx2jm@|Y0>svfZv3trk9#Fs)I-*yGv zAp}QpmyGE5^$&+V-Mm(mzOy=r|5!xp^L6+gf(k64vEu5RM7W46!sGOQ`!mFptFqpJ zHBc>QK!pM-$)uA#1(^^?v1yh;IjRfvODHo|6dqPqp%?lWpm1Yq7bo|MXAvnps?daN zH3qR(!fdXdE4f|L#Xs6LPFza)qZx^A0H$ULgGWo5x)9m^vXE>DdHaDcrS+xvg34?K zMr#5Gp1-0uT026VS1Qr^JxcV!#v|`E+gO>(Ra0@wtME%Gi;;=}ej}So0o#9@6?4wT zJqkd7PpAWNSFHoyed+~E`D+B3t8`>AxVhl}|Z*y!>eooAwi*$9p zkc4W93Syjl++MmnnWgj3jTRmsVSWwm!9v#pKTs3xx#voz3+F~Yur}Xmj1VZ3-_y3( zU0C50SfVV~oD)%;i_a@AT+Y=lN(rDdTXl+9on-Ss);jhPR?m@>Oa?fINpeg=JFs!w zCnCg8Ha~<=2=}*T0;BB5IdN+I+%kfjMMM| zhHRZf)bp;tjq11CvADL9+3e2YQuh2=@XE80m4ay$@RGf-M#f~{D4Cr`<{*D=-tVux zSz$1iedJ{4jY{T?7i_MCXD3MR!J{LT7hYlOYqd7*I(6Bl39Wt?=XdM-^8-ws!C7=7 zm0no#5}=%|p1@?90cO*-xJP300K$K>i0zV)*Vi@yP`qxPqrjjP*A7%%no9r=?gYLu zAX_xB21v4a+Q-Ac+6;n1?9Y{tRiJj>+5jy>B`c0J{_m;gMXczRT=-_vJkp-~|6vjT zaGoL@YmRjdx0^^dT{)xc2z=gM?fz0dnSpt7AJ?I~EM^}De25wN5U$temN)&T3tA)r zq5g%&Wza8vq6{VF?-u$it?X_tvuEqsk`xd*aY2=n;Z}@`0roi@_mQ}rO2qFX<np zq#H==fyn}bv0j;J^zxL%(i%b=(sPaD2z&MpSa6{lQkqqO!1;vD03eznDG55_jYw0< zrq$l~?Mm0x;7hc5B>ha6;!t<5tFix+f{QROPO;6bX+Oav!p8~uC5_QNgpjN7{ud4b z*hhiJ8~P-1Hamy4$pA&(GYt%qJ`*$kVoT63vR*oIrx=KqxZp^z6)HU^t)6x?b>4PQ@6)Lk?!g*ZZkbK6IV>r#*taMfm>sO$7!zDG0e z37EG6Y@C-c6-*lB!T`X*k?+g`-&a58coX`FQ!^U>Z);iQ`X)}ok8x1|ZD$86z27(# zg@8|e8c4+@Egb0gCh)2fp%^f@P};=sW5L0W$tCL-qbR)Hu@E$dAyCotZR?tp zE$!t8O^C+lpU>&hX^}b$;=epYZ(%IR31DVzq6-MQ_LnJ8d|VuF-M;{g7C*dk7C{pl zuWV&^^6&e*!kbXcNqiR4Sa)@~sje;FWY;<=jDQ6rB(Hz!*xn;`PQrn@izzX^oLe#3 znV)7$i91&Itl#2q+GrHz*lc_ydb$PYG(r{;48`}~u3m#wc4X6{?)ctw8oK0+WQ`ET zqydXMR76H!R}S6(vHd7Wu{qL8`?dbz0^8VZ?3H@D`D9MMi--PlfupRE3yrMeKj{i6 zy!YdrcnuuMlAsQe0mCc39hn+t$czDGWy5b`hrM^N#g0&=--SS+0Q{XY%tM@35M<9X zAl?0N*qx^iJya3|a_>`5I<}488Ld>%?2b0UYEl=28Mr$h7{O*7M0G z=IQ$mdj5C@BUqPDZTmCq+xtv-Rc*h|fye95=Pq1&=tAo#Sh!|4S+?tkSAHLUd_5>E z6Wk3;D!hEm63clJw=R;er#ts@M}&j<-F2=wuAheS<3hW3vio1^QsJ~5hY!VNE(b^c z`1z4%DIr?I{rJ~jtERI||Lxg7&O&b`5=~?=8x!r{m{hTViAkVy;rs$ogc@dsUMMccEv4tNS;F zW^X#&9I%+-!H^wX>D{msA|H*IUA4z7gtiKdmN#xV^qB3V+qL6gZ|$dn!w-bpupi4W zvESKn@9tvXiw-|49HK5KKKSL-?%xWR5}cBuR#&AZ`K^Rtf8qkL;@OuPXFvr>)A_=zjqnN-}JVO(?QH%4a^ zYc|e=&%ad>mrBBOiD(-bQAk!5uyJWg!kVz>|Gx+5hK{?N}ycev_ zaQGg;=bGbnT^Mbs8M{@f6U}2*#!|E}J4)%>`)!&G^CyGo{kEByNyh?*nWu`wQ(f$Anw@u(Z|_`L8xfy1 z>o=GmR1qky;np}dSKeUipM#6%*VJ#l>4wNlAr$e{rI&x&QY+xg_21aqq*A9AA9rvZXN$!5@Evb3QjuS~K z!@Zl3uxU9nla|?B8NOqNkEHnD9guU-;cpuXQaD}Y&X=jY?_N2UX;h?cm4Vr;kNC0> z70Oh%4dcVvSf%7*3iR8atwT*?3dc>_l6)?f52{tuY~L^+E>QVwy|Zmok{LPpC$8>3 zJ9ZL<9lKoDC=(2pJEZd8$;nnNvy-16VYutZbmSyGCDaFf`%0{=l z5gxD^-tsz#4#%X8-xg@+=FFO(TfQlg(QB)-6(T>s_Qr9(i|*Ww>_2ezLlpb6B?Od; z^_}|?71oVcW!Q_@Elw&`Smzz(mgllD?xAENPodJHmG+z9oP0>u6WnZZN>Dqh_ko_S zK~Hqf5X1d~bQj*aMAjG#x5I??Y2McrrSl(zB%(turO02+d!9&ZG_l&CR@~KR(4wPF znPA$$S}cHRrghbA43>QNP8OKjNr!wsc6YW1vS&m&S- z$vfP)XkTvi(+2WxmHKmf&lxp}uYYh9lI@@7zD`IhrSGbbHO_uhkQNGQeh1G8F-GF= zsB`$-ik0=BXo*|Xny=o^sk885$vW69y3bw6bD7$8nUq6HId4+`<2b$v>CS5dmL1U- zALchUdFDq_)pVA9^L#HqSamWDNlTs{qeDfS!8Zucom_pHUS8xhOTN~StAt4jvzwur zxu|un)oHDZ*3D35S;2(wa;Vq@53T9U{;6J|!xUU=v#_4?sdS~b2q>AdoW*G53L5f0 z=;pt8J0nK#HY$!uF*>_+`)f{wj_H5~3M-j5Xu@I3u1-|KpT5qsBvH!NHdmgnxvFip zV(rFAW5jt^ys3hk*a))!z;^Q8{Rs1)xBxOduTtG6xc+ja^`2a%Q(-pMpi(3`h{F;6 zZItd-6|pMvJDVaHD}Jm%q=$d)6?|Au zS$_^VVxAoJMzr-T$m<4S-=gDb6NpmrDj3R(aL|4&F@XwU3`Xy1B>bD(m2^&0*zYm1 z`q2H8*iW?V1K~&5&6n>J4Gz-ie;ppx@CsOSGJZv(?$FBOOe<2%a9h3DvWDHXZ^GYt zyz6;`&dram`} ziLa;*-lzze18V36Zt739niC^}cZ275XkXk5~kNv-}7|DRRJ|UZ;o%UdH+aF&^VqR>vvYBDT1-=TcYRo9_w#+ zUT}wkp7=^E+ww_`1FkoTd7ckQ2WQ3V!g!|(owwYN3dagKbRk-KdjY0W*Dz|}7Do7@!PeWz>*je4#OjD76 z_JDi$_sC7*KsdjxxyuPTd=GEepZ6pjMo4jmL%OeF)rytqgCk3$m3?pPH9-yMlK4CB zva*GDa4%|-52GfRHE2EjOdD>{opC`2#m*ZcO)MykvXFm>QqZJwK6!5eAA%@tfXP#Xwk~N}$BrE&t!Yx5+FN15&gMUKdc}FuiYwb( zI)BaP_e^W=)toFv41EkdrobJE`^`VzC-A}{{4i)V+9Go2?%X?D^R;;$G>+NwTD6&Z zQbupSj^8MKrfS*OU}mM_wD1US^fUbpbEoJY)4o#H-jvb|>{|j}ZZV@qEY%xjCAGO4 zv|=1n0qHN2RkWpZA11f>Qe45}lVd16@cg2X7kfoYLXdPrKWcc6{OC_zPYS#v-V|qC%}iCxb7RHv z(9AbgJU3npOF9PHTIfDhP%E0(csypoQfkQHZdf~iQ)p`N^q`b}-X}q`a>JfrzS^+S zm%%gM8-|#i9F1dS7tkwH<_fj~MzUjpPlWuPw*~^$`0~-|FUF->v}>lAp<{76UhRDw z6WhUBf4)o($HJ#%>+kg0;q#>K>4W@> z9XLDT*rz^SZ`i-9yae&I5H{p|=T>+c)O5P${o}60CM*{#Y-h#fyu<$l-lX8Z})vac^3G^&9#PqKBy z%l&pVo6`J{b=ZPPPo2~1IDte2<;5*qUX~o{Rx+DUSwdc) z7Q(yT*5w#fsuT`@E0gLL+A7skG#y~>E0$clAwN}YR(^(3zV<{uCQ%|@=kTyMM#H`% zqZK_WN53@XhWj7)4D6%VSToO~XABMYfJ&jMF zzYIu1{dby6`n~iDp0eO}T%B_1wUmw|gT^Zi+RSRXZ}P=%23?94a5Y#6l&opd=?lft zDVFcSYncmX{wkPhqQWI+IKXQ4qU}l@1!_Kw%dE)KZU!eA;^sK%@N0)jm`qcWtzQkC zO!I!eqOdUa$$?Bcc&X<>#dVBg|2Nf0h3jg$cQelN*k7L?s%LRn$ID%jo<09xLis*syk*q+drLG zV%cRjy@V#?k7&@3Z|LL8uWhDsrLigBBvgmtKw()mQff+}=9#e~FJ zKtzCA!k7Ajc)3!h_$`5=!H-@qB5KPWHl*mv*kdyFAl!4w7nZ$UO2^13Wj<18piuzB zFwv zJ@w6Ptd^K1BH6egr;@cCz0SVj>fj#am}@hPQD&9<*NBv00xRSMuUS98?^ThlYv$XB zunC%6%p8DCO9lWKG%#jWP!qH7nQNWq^iTnejCO%&NY(3QnQB#by63|7#|yO&FYC`2 z2m<1Y0W`b3VVF-e`O+=>tgpO(W{x1i`1zI6QDAc{9ZVP2T=1xWpjGa#o>_5WL3it9 z!xt#Uw=9j*i4J@-jAFRXZa!z6~-Ct!@@1f3*)02zKk|H$0p_m*Be3v=Yx>yk5aj%4~Nt1 zfL@J~@>iLvV)# z5YFO&vFd<&Y{B z&VS`(UAWG<@=$%FwR^FSuS&9YR@B}jJSk8j%~GKVzWB~*uG?w!;l%UnXB^YF@m((N zdTS8ZSTFJCTeM+8W#{;)&nh?=kF5iyw-QCyO)bJ$3MVROhTh=Pl@7{pH0DnOTpL51YYGsNSB! zJH`56*X7WIeW=pQc8f_E`L+~!NqBmrc`t!k>5$1j)Mm#GL@n_Z-%OzhNcVmzqHs8N z*?KBS`_rqM9L5v0!2C8PVcEXWKGX!2W65monyX(HdBkC{U4-x43F>uN>GKspTdx7I zbO_Kj)(j}qfoz=@ub5Q02+{Wr{n$ZOy88%4wHErm^qLmKpBaw##)mreEW62TYL&3^{nkA$1B8`OxK z2hw|^E6~TmRD|7*=;H|508`hKAsOyNt%s_v>_h#58Ttf$5!(#Yo$amjffN&hPGtn? zXK$D%jRvqeLx$Ca{SAODHIH$Q@MGdLRjA$UF8>&_rO?J1Yd(=(;7Rl|VIe4F!t!r< zL4BWWZMK&3+{_832eMTKG$}@>{Ps4}J{TXmu%&jxDY0cBJ*;V<=!bE_Lr5eHv^d=z z04+i1LYB`PW@@RpuF9(sIkTKkyO-{lgJa2hG+QwTUP8QdpV1GaqM(Kf`c@utT&@UL4!Hr)12-+3*uf;6?;?7qHzR_A}bvTbMsziK;;Kt>2op=UatK ziLYy~hYN<^j!Q{495&@XXLhCV{?A*vj#%=xXMBax_^j%`mOO-War9ZwnecM`Wsxvg zA+NvPKiO(3SXTX&h%&VCdDx}&cg*T9FpR1{ybCP3jtOryYCeaKoFzS-T;uOM&p% zyO~Y|yho6Vook*{{$bpR?Em6xMV|+2r0AnVg?;iq|HS3#(;Aq|MjSbKMdBimKJxZ_ zV+vo-S%u2`(!Hw^VVASgU#4Xmo;}tHnLx^}c*)<1WeG^P;~LbqH_ozyIl#a2i3>WBs(kV`2Y#sQjDDAmE2+ z=80ZS#P5dwA65w2MR(4F1^#kP^#|B5cew!mHy`!??)CrIYdTlG?IA|XI)hvP#YvpG z1Pd*E9hVv^--LA1jEM7p{rZwb2#3M-a3Q)EZ%l~CvgaZz@h_OG+F>Lc z+U>x2g)HGdaaiic_X8aWJ+5_ehdm>Q8S>%+~ zn1oFCe{@a+hmP%2uKV~+kcTO#uSCMQ`_e!#<f2_OR2+7Y||>^~X&->kXHVqZ3i*6O)8;Ojn>{T8zE5Vf-ms*C-m?Cg_s25{f8{}Dz!$#O`?Lh z@~r&Wp-6qAKulPpc9F9lVock;81rx6?Vm3qJarr%*wNYd#I4V|$hs9Gkahj~{L!pY z$gGjNYLRrt&nWaCR$Ma@f?&n>7u9{B(<1TeLi4ek^8lkY-`=$4V@@6q3V_lJsVPz2 zK09N9g90G_mEp4OF$T!9WphXci5F|1qfCMQCJ=D*T}CgboNTTfyyv=m|H9r#h%Rz{ zYq*~**WbqI5(FF3%%3+xMm9o7CNLk@es&#q|6=ccg{O#DVJL-$k+DwjgC`kjGLagL z66=KmO9i!T54MHO_U7939Dviw>-y}L^9)*P5mf=bZ3Waz7%dyL1744)bIZCVV#6*r zGri+Q9kBVtb#8z7iAc+mhJgEyjpXI?|BESZ6Ez(UMKIm37?os~XZkh<4k!G@g5884 zIgTCw{Q766>E`?95#_Bj4R`{GW)vTX@U4h|WY7j5>hUl&qjD`m4c(O2lG!6qm zCqTFKok-@LtgGc9cMY7{e$3k9`49_tK${jY~NbV}N&2bo;$HF<> zDGmeAG@ujEAXDVHRa^>eu;#1Z?loq!RBX0H*nk*iJPeIQ4D&viACg!D%6Wt?e_1(C z%r03rJ`T9zWMPUhGpn2(5TX7EE4u~X{m+lAnvBq!a~~aCuNes@m@bSq7OY5RP6GmZ z*>OB?d`+j?`|V^Hz;S7T4j;jFRfn3DU+m@*lqrRIEAzRpV#z>!&kr&W=9{Yz#s?}Y z$UumV;i7EZ^+MEAISl4Y0u>Pz;CG4gS~nE3G{tN|HJe60Mea-fgOj4eykRDL&50NW zI14R@JAa)a`-dZxLK~YF6+#UJuuo7}YA|jG^aGWXy`%M^bU=Aoq>rTw_~o*W@j7Tu z#=-eC4_w{0>8;a(z!+N+jyW888g+FF=(rarF*h^R%O>)yfu`Jg8yOF;aSzcRb78h{ zrF~@Bma5pua7B&?zbDs#3ZUQ*@JUbso1pzU1T00bK;!&6R6H0p{$KQua{03m+!{6U>^h^f%^3gaLW@6K>Cdky8I&HtYTYj<)FqIOF(xVCs-d^)Vpu#p7WHAZDeHY=RRcteY2_M`3WZP*Kt{ zqtfW$9=x}i0E%vB^o{?9vEAJWdp-tvMG%6eEFqUFYWV&m@F6I2;L4=h8eWZ7AOeOU zz4(W7UuDbfZ8MJ`vuRMb1ZZynbm=5ARExBs+BXm5!x8{#rM~yhMZKp`kJoxipNKgx zC7w3Qkhrxjn7tMr!^MzZo`>J=hKjBN$NWul`)^lj0+>S6A0^|aA`jzdZM1;kx1Lx_SYaVB;dw>l77Ohke1w3r4XX0xBxKU2Nohg~>Y^eXV9jsp>fKf>s2&mZf znBVzMDJre^{HfRCfK>8>S|mK_QF%?+QX6 z;A{PB8}{=uuD*FvzE315mm^iSi0k0ukz@Fjb#mFsse8U0Vc& z-Il+2CgkNR&}Vj5y@dgnZk=+WT$*A9miDo}l7hvlu2v(rd@ju4Ao%Lg-Dc?#Z9ym> zJ;$4(pXj5lQM&C-S>RxDTGt1`Uh}h)XBSbH|MK|gd|)Sb{aeT6&L{f)GF z((KyUR_l%_NAn^~SLG|GE(}OnW>yaH>|YNhEoF-8VNTeXVQE@HTpfRrXgW;K7oy|NccY3rFfX^ex=055aSeL3B;V)){meV zrTppr)#Bv^s7Nba@dJS#rR$3_O9D2ic-P8pM0hIgEt z0)~?oiROhBl8~c4Fn;BfYFfw$o6@6Q;?=P>O7)vj&|QcjzRQV_4n!`AgJT&24*h}! zL;ygqi2X}$U?!*|^ttBnH|5aUWq?pnMn_79^Hm~K7$9SSv-P!zrX88HXQlb>cB5d! za;IWx{?_BDvjTQ)IlMCQ(x^W?E@bQ0 z8!DRh7B&kSwIsB@>r*1p-#E)-jx9yd9#;jU3|#?Z(nc05j+O&3RTro)xtQeQ_b3Uo z4fDvNhOMBuaUJHg`AwRvV%;)OnG7k%Z*z|pFBSspv}fCwrQ+UbfAOV?Sg_?!aJ~)a zVbpU{fv|)@Ii3>)H4rh57217?oX!|r6t<@^_8>1Mn(@Pr=F%Tn&yO51@aH^!)^h{G*lRxtpXoAuq&#oK$C&L z67fvI);BN9Q}osLwuV$Ef1&m@q-lk2NM$~sO?Otw$CAV)GwbcE464w4R6(d56pa|x zAZ+WjUhZyDPlSGu0fRSS(wYXCv6x$_#XZooWr{GOdg6Wk>mt_#kN%3Ht$@bru@QxI?8`Gl>2-*z#hij%r)0->HzG8KD@*xR5LEE(!Iep z5i8R@qf+Y**2~cnq0-Re>7aE()D)D4R=T=6Y;=I6ioYfr=0INoJDgj-g3U*mxgP%Z zrPA7`)v&)zu*nj_R+EiIIHDw>A&zxyp+q&a7Pl;47&pc_qqd8y$wI1ku{i94Y@AzL zrp7?9M@vqAFb2CB2$OnTY3-Pmva4odVb<2CUlGns0j-tU0T%E@3w0%Qxx8r#VfB7= zUNEmN_SlMSMhnTp{=7#~1&Vvx2ig9IKojbmG*QH&dBq73$Ff)T3HA^0==;|~bk{8( zf;}ev&C&}a`c{~j!i$Vg*YA*3DH=`ZV-5jEjC&Z^h?C*hsEQyM>{m;ZzdB$ynSQ?} zy4V~7`LEQ^uElF6!iHgL+RDqYX@FybXTFZTu^LX+Aqi0?m?@;b8PSUu43xm^9KAW} zHkOpMrVUo(52$##)plTH?m7@{>Oh{)J4rlT8%Tn>)E;#CK@6fjW6cY`+8ohH9rQv~$nOV;a}VV= z2!gu@$$890EqP;>zai=%$ZUcqCaav7H~dJG^%A1YqgxgM$O;z7IP%;dNt0TWMNRi` zF7iVsSu@eWsdQ(1!+dE%e&D@zfAZ0kcA2fGrkKQ==Njn;u$$qj*>;Stl?+0#TrdhC z5gc3KYUoSlZ!NAsv`iwd~jAke6t*zrk+$I+_ zKx_nE4a1zDK+9&6T;1>LQh(}>b?f_P4SnKU9=HjlgM|+Urz{~w5x|YKetXPOpAr%0 znoa_Du#WoBjy-@7WP2vVVFdMcrN_2p+?T}^ekSl=kn+3uyWf~1{$bTRDZT+eg`0wA zTN8Z%gkOFjnIn!!f~DSt)31eg6W*((6`r`9PM%WZdHkuu+*vN;AU!u)Q$rn;gTDUc z+|{RtZ99zOYj(C~ovhXUE;$r9z*O=O&}^=}TnR56jX+OPrq8bdj{utf>BQA^<&=-4 zLXM?+>;u$IfKIUM*aCgC{O+XInR3nd&tqyl&)BKM?jc5s|IGnrec7v z`eDkmiN{Eu*fAfnnc@lJ7v@}fxPc*YM^=Ou#83 z$^>)BT7lrfC3r2zJ}b>e=mXQFKHC%kF49Gy2cK4imeNk(EK=~Zx*5((1ChiWHT7hE zNa<#;MPm)5%F`J@sWZ~0^E#t4M}&5F8J({q{Q`*#&wic&+$T=mRc;%5@-zC`{ncG} zf9Sa|=t!BUe1k))bIZV23*v@&IBU%z!RX@p29$?x2yt8njN!*hg>s>C9UewCJ@3r_3}Cc`EXQ_1H?wbSiO)S7243@PQtgrv2xq3Da$q5LdOd+jG)Hixq762@*DjaWf zfvs`|6~jRWX5N@Vtim~N1H?}GzN!5xs1qIN4v0Z`9M*HkoaidH*HaVOQf6@{%FNN@ z4ll+kjt@+SukqW?R|99ViSHQP*m;Qfa>^0l&C17Q<_+MBtpNt9A!FU1x`Ttf{CwJb zX-=H**Xy2Ir?9fF-koX`s_jOF#u+|6EZb-=X*OJJHJ`d>{aJMZl@&K~tkz+qURh+J z9tc_wcP_Lm;QG~(zTv6ZVXjq3_bsVo*R*K2JwZ?HtR-M z(bX5!m$RF_=?LOhwo4zcFW4s;%>9hBp0rqb#G|3!?Fb3W^KR4LVtKC6{3Ga<*}y5K zoE!TV4kW53%eKTk7`#7CjCW3e{z>_EcSM8ANRQbBpJm$x9{qBq&T4y=40l_Rkjkk_ zcZ(4fhY2>LFIP);>_VOz% zEn_(ZBPwaao+04{`EH)cpw**%HK&naJ!7ntlK7IG2Qi}V#_w;WA5Au?@(QU492W^R zvZ~F}`uHiOcv2)f!10&x&s7mQ?&K%ndLja` z64_AIkRxs1GHjPZQMP|dgeNZRgv5Z9sh1T-@)4>Pyf8_P7aqpW5lcBMPzVV7-NZ2ZG9 z+BI)=9S#VsDEskFhI0G5){!y~BpDOlCJixTTLuxml8Ic~%98=~{+QQmk^$F4M5ca!>h7UN+@`O5N9;HjgRJUFGhdAK_)0I@jnj;8nf16(L49R-5<`MoT=d`{A$6mw zFF2LTZJ?=`zB_*u5{lXvbI15>WXr-PyAkRC9uIJWK8;E#fnL7SPH4=Vt>2Nf8FmP z$GwlZCVA*s?2~$oFpEM^j0HnRK2Roj$Sl0mH(9YWCi+_4E5>dLj#E36d5BKX^#d8F zw-)jheEIEHUvC<0AMl5U&G*ufT`dxLJ0N>o#^}vvwxR0*ls9M;P=UkIfB!oZcD8YcbM(-cyx7DI>IHwl?W>1be)6 zdnH*Sy@1VhW!9-+UYY0Qp9@%U1DgB=H$n!n<%RtOH$u7NZR-nmLPBo%D@&OB+dhe{ z&3&wQd*MY2Ex9ziS*c2>nVtdRmvQL0qy4H2t8&Aew5#zC9P_o23O!`=a$}gp_QR_U zOtntGjMaGOtr^yKzC7g-ZNI|HAMA~!iR3tsTjohrx242OpS@&uHI>_hUU{Efs-GcY zs>0cJ!#JT3vlvtmuYADv#b1j=(6w@u&CvqBZj1$sKEmzcs8>Lz} znv@?;CPs~hTDaAb9$DGmLYZ-zMKFXO4nt-putw@{D_WF^NrfKRIyr#1kT9{}L#4Ae z(Vl(3i_I28ea}!ct)fX(4TUG4)0RN?@xVf~IB6@P$RHH@=iRz8XfnDpGRZZ#EQ2%jix-{7>^wl{!~m#w~&;GdI}-Q@m( zitx_01Y+)`<1d@ATVa2T+bSxX9eMJW9;`C?_S|N6=c`dz4X#;Oi?v2 zis}n8146JXUyh(cuPt2jFqUsrlzLjvMC&elPFYD-C5Ug%MMAXgMnK5jRVQ}chcs@6 z{5r;CjH+W@gc*rX>Ciud zx?;mcF4mezX??JWzs-sb!Jx3^jh=C+sUBHj3S+D`9y zOAwkE_{wprbJqm19y7sz(3@L+!MOV>e``iKGJ9y2*EA&OVKLfR>+VMvF0bCW5<-2G zjz>=Ma!s^yu;uDlv-Q+%lkSH!uDpCrS{n;3OAP{_Gzk^Mhh+5l92{$@-IiI2H7d^W zcPf}zW&V39m|_Muj5|`^tUFgiH;9rzt4?pS7UlfJA|dC_g{LPyE~A&z99AP%fWu+h zyiEFMNcJO~fN^ICl5bzhuVBIoVN8;6FaJVKJ28JaUUDNNf8jK{_Up#(uc|^W4>ZHq zge626^bA$jgriMHIU6(z?=tvIEM}CU1!`iHyIM5#b()tGo|aZL9S%R!m96b#9)Dwv zR5Oq>NvTzEmhYiV?pFh;tA0Y_q4Y_lsHJ*t)C+5;Xo`(J$xsAgGu<*)Qja*oS~ zb95INkE;)JZskglFOhI=o7866RURn3Le1 zs~fEi?_r;g=*%|?3@R{Zs?#G+g zmL@AU5vtvF(6D`8j$x^f?%zkBqFJJdIZHzD)Zpfi*|0)OGLUG5eF{qRHC(q}3!iJ1 zD#!$yAL*2!Mj!Xjhx9pHfm^&LF(IcLG7cJ>M-^7sqxw@g&Y=u41C~^EP|nnDjMy3> zcPsmu1;fEFi6EkAxF5H=g$NtozOs1Dr_Gp%CbXw-4~}JN%|m?V_;HoF_g;Ct%6ofu-UL>M8C6@S&%!F< ztu`9a6Fa%oiT?a(mOgV$SXWxU<3x?0q~U_enDFsxM!)3c^^KvLmM`&v+Fvs(x+kI& zr6coOc{3&6<)zB|-Gh?l!qHr(fWLg}U9bMd0H%$DiH9N0T!XaW`)cRnGH&4Ls5|vU*oG+W$VIK5MSfadr)K=u` zhFK#$-8id7f>eY~q7+}y%Mq2@?RR}G{blN2=?~0`%v;9iKK7+zNh`AXOKP1N2*s}> zcbeh3NxD|DFzhG${<(zvyQlFk2F(_k62fFFwJa^kDg}q|{Vxd1cN`t5?ldx` zgnPuhb28)Q1ezlK)cWA-d*(xpLc@Wl*+q$z^+FV`(n*Cb@jnpm}j} zsV{6#F?~1H-t|8O#A97KiF7=<(7c+@8^1iPvUJl=6xg#IH=10Ufa|VyV8mh*Qerx4 zB2g%ZF*P6j6`Ck~2`Q*Nquc=Ga5t?({j+1)Q0Y08`7xOz<;hHSP_rkKO3EueQnhmz zu5ifEG(4tdy^S$=vBm}TU0s;^u%?c=n?d>f-CX65)~?OFe5s7T6MgjyUQw%kPq({~ zuCd(JmC35|CkNy2J7Xc`R|=Aqs5+b8^V8?hb%M~Lp_f$1c(qT%@uE<$ne)yXMq|Ex zteI`B&!#VQuYUMz$&?@li=kfc&DO?Bfu2USy>&YA6+arulV937j5}r2(bJq8cR-00 z`f1(@3&)>l{X&mOk%7!wVrDO4NDQ2@c4*z0)wA9yl(yxMZEKD^#)){ZbK}+P*7q~z zD!!rk3Cuy(+zWNC((x!GQm3rC+@d)W928}|!h2Pt?p%GHxX%M1xq6et}Qo|4#K z+hpf+{PgO6EsJ_FwMfOIO<`ud^rq+odT51$1ci0c3wqvWX@ot4&~T!$LdCzv~!76~M|^ps!`;XP`${^tBO zZ9Iju_y16K9q?4|@4rq&NoY_cN|7zI%u__sL0Q=e?%t z5Q?%Xd-H$3)?MBEyZ_gH-Ph|}_k6$S^Lakc=b7*4c@7xXjZ%c7s4iegUc=Ce%3h&? z<1>v45)}}Af(E-B6VNSkv2)pFa`wUDgx(ftIQb2r4p*yfcd{FZbGN!0!8DrBf{=xP zw)FGGT<(oaVrYph1{e|!iLQ&Pp@rT9Z;{rhS+zG;RC^w{Wv-UeS$gF{kNq2vM!5P$ z@0>|TCN3k?=+7}!>@_4M!%=WLq6J4|+>xJyxgl!%&CA8Gy(xdGn(wRp`9PsXio-WT z%mk`mFrQXtvhFFqx)SBAzw-3t#8;!}3Z*{DLy6}3^m=YbcoI*mlp?xxWhuisAW}Y<|CNj2 zjR&{7333Uc6m_)E1t)T==>gFaAbR$8J5hZe=Ai1YboE*QnQ@93pN(baG5z*2vV*;1 zQd%v983QH-qWq@lots>^QY&84LKrY0x9@iUqrTgYzA!ny0vxS)=s#|+4k?5<-9oph z6DlI*OLcLwCk;? zmr~4cLg%Jn&_@GKQ|R!-k8aioSAe@P*rx;h!hV%t+H9lR)?av;hO0k!+66&*KyYH3 zb~0l=Vg?TBoF_5x{TChCmxsdm%ch?B}aTTNn4Y4 zWij^B>}+?)yJ&X{c3okX*dUG+@hSf$*P&-|0t`H^d{!|TmEsga3=vS@g!u7<%aDN& z6OO&~WcspA9P{0UM?MCXUd&0Rd-xbqZAT%SKeBW3ddp4b>neeg0=c#$#_YlE&y82I z%UoW5^7h+F5qz`nJ~zvhW}8Mm{`zo>&DfHo(c1`7*LFVoHt(*!qU5+sPd`;A#ruwD z^+^^W+PE@~Av~^rF-#y3y(<|PCP?tCpPAx5w7-Bven2ofoT6b$RNyEtgEUe|vzayd z^Cz$>cKL-hC0YBiexz)!^9dF5dD3^=9<#T`U^l%nu{%o?PCX1r=PQGS!b~t$CkAew*pLr^UIh3t7@QR`19I6g6uG< zQc^5oUh}Tb>LsNUk}3XCyzV6Rbp}wCnD${;ftxlmO3sL?mb6$MJK1NcuR;E)TT*vN zguJT|st(2RQ5?05;dtRX&^;P4x^B1 zv@n|IL&0++jgI``)6`+Xf;*}`*>%ID>U$oNbrfUFW5EStvJp#~H9CP_jCo7d>AJaJG@NW>=H; zLkqmXo<2(=B)}hW_I9Z%k858yml^$KAwF}-)zm?^gxFSHoQ8mrQWTX}Bg&wL8~H?uorwt&u{J2T|aZXlZpYk#6}@L_k~;tprrxE2fst+ zrUEMCSYWvN3;pWQagUDqy>(LKkT<&o2ppEG>h3+3b^~T+00Yy942q6QL{@mAQXhXL z@eW*-Ox6cY*OW6yH-d5xya1Lc`Ayn7EK8a(#BoQ~=llw%B9}W)#mzOsPvqzgUTrgc zN+n4z`5pKIMIbi^H!K2o*7zVePNc@%pR&@D(OEZ`X%8k7affl{@y5fQye`A ze(%=!u+rO$&CB#^A~QFqKy1YWvXvR~=774J(^c1{lq6i=f^^G*2*A{j{v>_B}f6jd7>|}sr%Mdm!|5g3_18!C< zp28hwS3iNVFxpMQ9}7Lgu6$%m1EkMw_&@9f+GN!|&DQ{?yQccHv3FO*tg_}if>bA( zixp;vE6D}P7ffAoXRLc3xQ+rGVD^^5bqqBHp{}@vRVm@lQ^_AZ?m{(syFU&L+ei(c z9~C~UyJ82Ltk*hyk+MYX;C*l_E5PJ?Z%wSw-BfFITa-o#4$~EYD%-#`S)ffce8JZRc58@y4ixyRTjhN=fyHK=Dqv)GP02 z?GxJUex~097RxQfBO0bxZuqZdIOb>N=wdb(~FVlgnTR#NN_e@cB*0l@^u&F&6>$}n}zyHDyR#gt!4k^Q* z7bZW1W(K!@YiH5GQ)1YrY54kTvmOy-lcOFL)_v2(9ZLSA-}z^Ejd5(Zah@T|-thB# zg_!sQCZuC4blV&5Kpr@~?SO6F33goStAq3Whvka2V{HLT9Hnh~UOmRklujQrXLAD* zQ@mBSD6#{I3elM}6NJbd$tl2nN!HcQVXI~9#;+-0l4cs@sgh0mLe+SCr3($W&Xh*Hcbxn{C_13%fSi|XHaVpB>0e?aCr69BFyZF&=cQwJDX*O0pT}Qs24EX|LgrI$rG&?rsRn)S zNPy(T8Sfj0rj`ZvkOnfW&Cd)&GVDfOVzEJ&UhV^VH?^k~l8;l4FQIFGiLb{S7%Fp* z1ICB$s3*GO`@QeSjV!=i`~66zb{Y9}TKm9gC@A}-TRsIGnwiz_R@Tt4~WIskvUEIl|6H*5lkR6%E%e@u9^SikYKBD;V)Aio;`t7-JswWZjxQ(7`` zbjL}ry)YMmSr;NY&CFgy!!I%-J3lSWL7()IqUR10d95bt$$^YcW1r6sw&23|9Of++ z(*inDh;+kHrNd`!;-JAmmDw^N*u~1yloFpbK`1GLtsa`E@56Dx#BR$p^U$rj&GQ$a zG(|B-DS`!cj4h`7x1$m+*&KqF`5t~yL*Ov_bh$6w$)qML^|oOZvu_QFI)%i~yKy$I zJA|dUgBt&_yo2=VUDXGTg5zOU&6(z!n#3P<35xEXk4zuR&x#vxPK2LtpQ z;$7Bi9GgP&VGyvEG01W&Gu<}~&?eS)X`Mi60Z3l4Xk2gdMi1Nce>C?r-e(4?N*~L5 z?IjZPK(gTouuU~h$DIAa&~O&>UHNdq8u1RSxC8>m50oVAPEif$g1#vL{3$g$kg>^k zP`wY)CgE8hz6*?T#B9$k<%icWG(GsW(%Y`P$E_E5bIxqqAHQ4hUku5N1wIOOyELla zYOJ(DDgxkCJBZ@ZWi7j{a%-%V@Qk^!<{ko{n5Qbs%7}&S=#NGSw4u!POZl z01c9>492Th`_?Gjr-Rr<>co=$S(h%p3;~8o@cY=j~ z{x#HlyciuN2u#5I39~>$yGfV>_eroiQrIrtx#}5fAO`RAE!FZ&btl^*2&y;UmP0Ac z)1*W__wO>3noi_c4h+|2*uL0<+OrnP#^ z94-Pzb_`NQ4?xEgfH$Hc)W^*lMLCwLyJ|)WIO?WEnDQIfy=On$boQ8-Wr5Jc-3QZG z6%GrSJ`1ezi(iP(BK*pRd}bVt{Cq#gpMG~s%?*rMYI#mpA&K;GBFl_UCW2rA1_o!@VJRm zUpjMxdO)Z7s5VE4I;P+1%5O6ZL8b`pii0|%U~oI+0L6Ksqj`7*>A{|5*_<3?JYk}I zK80#82u5a3lOQxkuYm~DxteZJbxTCQGdfceDIGv?j91!RAKWAq*v^c!nIDf(9cc7q z9x&}NKw;+?V|I0qRer#x=2(6GE%RxTD*ByvtoM|L^&FPus6W)@mkjXZuUbPGb$Luu zmu7leet$Rg?j3We#%U!5B@oVMc&tb;Dh`;vipp}!LDzQZ-b-Sh-@i#Gv=-8-hpQ8l z`VV7;*7cp2Bx={b6K^hsLH$Dz0*MsIm!yl}FZ_f+oB96J0&R(ats85;64p}8W&S;N z)Uy?*$pV0j&)L@Y3<$VvR~#}F8iQ(vBAX`*ApeRZDHBwt@nNr2Ga66G;w+M5aXkdu zXl9lR0_T*oTFp{42+6i%pW8C8bb%%xkd#cJ&NQw+?=&-F!a4FA<4P#6fe$Vr**m@6 zUL$Q0+-SEBK9cb_BfFuhI%e0`6}CPo^XTHLjTFf;FIfR>W9O9nelXr=PM`soguC(j z(pR`K?~X{geO5it5F`Y!On|lv#LeR z-6}A5Nje8|J8%9zCF1-!M0I7$MKJx7S%oklw1(uTdIbQAnb_u~nb8g-O**5Rux4gM zld4I&3(z_GmQAm3X(yO&fWD`#_pueRA9-UwL@3D1Yjy&idHk~)A{fGNVj>zIsA9M{ z6=Ux9*M;d1a^vmt#ZeE(ZbJE|;8~?vC=ro!p zR-}eqi|33dyhvj1+ecbVvK_Z4*R(nLn)NFU`cIt`TC1FZ=~Tv>zGW^mi@)g_1+_`V z_y_w&#k+wi$uuNxS1-jsmrjd)kTBKtl}dQ#9_gu?L?Ty(x1z;~eiFxhh?M=ADYkES z<&VEu#(0t2gHm|=gStx>&+pqe(cN95ZYY?Mhx~Kf=$&URPga!k$Jvivm*+dXFM*P% zN`G8RIW$V_nA!XY08My;aFq@%om7llPJ^o60)pzd2$AQed0j7|`w&&rt>dN=ON;V;OW%C=6OZCGhaO2+n*jL%t?tY7cVcK^8HYtI4; z7$wE$ClK=cn&ssP5B_nbb)QfHoKwhj!-w6s<A2j{P&42eRJ_v}y$c)UsjhN%VZRQNw%rmA?Eo!|-@;bcM+tM8LE~;iaEN zO8U(l(ABkb=aSj|)j7Y_i*j5!`U?#q@1B7vA9DQtq>RME6=9fRn8EpOzQ!^kNKGdv zCzrJ8GbBbDbypW!ZO2P!j2a=Rsz)nqYYcsBj2aZ&7aDD0Y?0yi*FV(GjyFJ|UgJ)> zTmA-Oop!C-jX&IG^YGIxnnvc~(D`g(GI}*az4#u0>*LZ~zO&bi3ZC2Go+P8~0pi-C zdDd#m$B5mOr%jc@mSZsvhSS51iH`(2V7?lvfc?#zK}k2QMjtt?$pa{&cFNE*slGv~ zRKH|FG!q#zisqo5ds@MaY!z4DtqUE&9Ruzw{q9%we?|zJYd}iyYVfWpsDR|iGD)ej z|8(7xBWbcqcg4mBU4JNdB|zrrC1l2Yrh@*+(wIYG-lvuB`Hv0_Fvlzi{GNgXWf2en z^QnGOAABl$dBjMG$WzC@%^-5OhW5zaRSn9b>S*iI!-HnJiyAFs=9z8Gxb#Xk^=-p2 ze=+d*HNi@#VQB;!g#_z34Z4pGdT%hNVmB(j_hqYtZeF|t2x|6M4(F86%^7?~3C}0{ z6N4rBLT(v*z;^}MC|gsrJxT~1sU{`y-v!d{M4J4wV8C|Kcs-6RT$xS_r`vZf90JW( zCtXhNH+%h7dZD9xB{=ibjYs@YPQ<}(OytRxQO(Qhky<#@Zl;ve?OW={5KSDPn$_jh zcKtXH-#tkgU)rE|(>TlcM_jo{jZgz<=CH?M73u;Kwe8zZibx+r3%1umx@Ky=)?K_4 zNM+K95i(C>8{QNbc1$pGiAKR7ZpIJ4z!#2swfiJ0StpDyk!SchY|ZW%Woz!oy+&7Q zwh3=C$lnf`|1tWcJ!Wz|_0N zu_|7h)SSMsQNxwTF%8hoAQ5~JKC!tk_f6u1RAmFcW+>j6o4teQ2x#W>c+0KUuA!E{ z427ex?9OM@9M}A#U_jXvR2Re_RK0c#_ikZGeWtF*oUAc`oyC@oc`j900oaX#oP8_yV{vAdMS2d0Xx$v-)2Gym%wkY6#q^qz8( zU@M6KPrtdbE=A()+h3@ll@ao)wUL3l`Y`;~+L`DgBK z!)olKWYmx|lGOaKOG6k(1yq~{v6RaHP`ZEGsOcpbU&I+nZvHPCtcYRy_N{9e3o-4LKJ9^h} z|KIqJGU@UcaJDeeDa3KI6vj)-LdmbZWx=$n(-$nsRsE4i^BEt7*=wHju`GjTHtWhv zWSMAciORGVMfo!OCk~&BYHv4AV_JsE(XaqVxNKpG8S%JZZfoP$P0@NOxJ+3vFLhHm z=g0lrBi{Xd$Q=jPE0gw}uMbV277`P^(3Ggr`A$Ub+bdk>V2lNd=xSx-G^2v~bURE5 zL$+$FhKMR>3;g3>v(wrkqq&GxbsWFjHmj(`!~9Dzv(+my^g%jq^WVcpoOS`QMR9z! zq8WyCk%^ft94|bmlJ;XowJCsD!j6suJJ*h*JQ9@F=SW`if0M-i6j-PUl;A9wYEZw) z;5t6UZS<#=iP^ik)GzU|WdDz)iW4FC1Tj5czlF9AuLh<2U*QDEUs)3kp&1og z?~4)$dv$1Y*C}hm0`n&w5-;EXHy-^aqX$_f^_$Ty6BScnq&3?XO*XZJc{H6t*)gv3 zg$4;Xx%T>*=OBEM(is2t-}qwUBoAoV*B_vJ%|)sc8dWD1%y9-X=1htdD97BRzzEYn z+w0avmOl%KKu2NCcG>k?^~)1tG(v&rU~Dmdcl(a8L^JE-@V*NY=EoHkHs3czTTinl z0{uy_pGmxC1{+x@KDJ*(-l=lHYBcS5Lh_b)`s0VX8Wi;Ru(f{x9TXB&RMq3d{aOe^ zu>aKQrY<5SVyuaJ+S4eLE@8cUP1DtB zFZ_KkF+#+HSw zYjfz~C;Xkax2}ASK2QJmdk;TL;|B;)M%7g!#>_jjAJEVXUpY5SGwVQ)#A*zA@v z0o`A#R%|ioFxK-RG+WItGBL4XFekqa#Vb4LbS5t*oe$6k0n*^cc|) zbF$5s_gG%~zqW7F+z{Yw6r6p!0bFZK2pQT&V{r9-6t}w`iw0#bm)3$&BtKok@Y-g2nDIA#O> zTzsRv2u@7~Rjs7iae=UXJ4$!b@ppisk;dAmRA`I|0Y=eN)2NokVd{%!P5bMBhz2M zy0sYNQ}7JAU9Mg5|Kk}V?22#QCFFcI9;(wuHR>u1rg|!cEojMF)?7B}S{f66>a=+u zr;oiUe*Y|3oj2Z26=Ra(dzF^&h4?}|%`Z*h8_WGIH>Y#Vlf9z-r-28LQcH%LoGx3LZ6BL0kpCpyq<5qYbdW>+gMQ2VKA7l`gr~=elQn;E_7R#sw$14{zw=1;A}jg!%apBtrk&28(#D`wHSA| z@YH|SGhi^R=@u0=?Vv8m-@JZHb#Vroq!{|OTNY67W2`%fB~AA+Lm!pXN9%wy)bOn$ zFg%>$+zZYfdf{qeU^`h5i7(XqF-umr)(kdLAtW)j+J0DyZizT%@lP!I2MVEQcRxNV zCTyP7Y2WEgN!ZV*7>F9rVrada9}GHO829G4-Hzt(?DsR)?ZU#!1R#=W82PDjLrkGL zcY`DT$ml}&7L{pLzm(AER(mj+kt3?xq~c^qwFk@NJqNy5!bpUMXRJzE{sL4->XCZ8Pr;mE(9TMGLWW4X45QDvQSE-jI|j#E5|Als1{B(78o{b8%&1I4u$K z)~`ONnsaHI;RshlHxmGBygOilg^;Qc`BBj`oe;Gigv8?LSkbHKZCD*@ufkb+`%lM8 z9Jn00#_6k}rY2|%0{l}0FZ~kA@4lvFRE$QA&%T3HMFMB$r{mL56FX+yW0tk*^T2SN7H?g z4qcaO#TRb{b525gq00cfkf-9wS(>Qg&NS!nVqjqEHFAzO@He@>2!3zDx3b6O`%XhY zQhtD@O|5ZyIpg?Z_v*Vhauv7L{1}2R^wmK`l#85Rn7<}UOoYtG{4gV}#Jkr8GwmnK zo;)YD2egL8Lb;6iQ|ZTt-hNt;y+z$of!aJHge7dxF``P3i>Qe{E2TU$SNp2%xO{Dd=vJ_u3}345G|WaDVz}f zO=;|xSe>5&Xl8t%yf7;UI@ph72H4#$m~LUuOMUt<>$8oVhCz@Dbq7C6fazxc+5U(? z!fk+IeB|}c{QM@R%x{4!`^!B%B;%)iEnS!*-4+H1p%doVAhePv-Lih`P!^@Xe>O7Xv@ z=H||lNWcc2wxXWKZW;;TlTsTtqJP+KCR( z3_)BFjCtL4pF;eM^BA~V!qDe7V-P|ze9kixo7}ZYt<9HjoZU68nAM&~99vfnu6hYLu@!y0(l0igu){0JEM$}QdompCey)ZyrEFol%K21mx3_y#~MK3|v_4M%PuKMcU?F>OqgSROpE z{l)5H;cB=6uQ~h0Onn)!el{2Os}IJMh;wH1!fwk7XVgVO%u+*mAm+X-t!wq^vbaNg z-2ilJ0$oGpVGEWTaME&_CTI7bi%WT(E?ZS{u(nKbhH8(-DJn+VbKyb3ACInP6)v0G zdmU;J_XxGbAw8J~pUQDv@;qX5(JcI${01-n>h?hz!{Y!X-Tf1_6St(>%I4yRyy ztmfHYJqG9dqicE1Rz@w|?@7YZm=_- z8K{c&KIS9x3cmlZK~(s0YWVTGMBV=1f1LRZC}IQ2njh&^T!Y}F%gY?GmdgG48`)%w zuV9BW*kcKx))S9Odc;AVkcFHQJTsX6G%E{m8O=|xBH ze8HL@6>o>24@n}!>Rj32Y~iZw0%UNbju$2W&F}aNKd@FX4uQE5LM+&6+j!LMoWjwR zGU4y&wSC2$BZltFVY}7&prP~p6!e`dd|eEI=?Dn^HjUkcxH@N^yBdxC)EteL)j18Q zfTb04vTcV_rw*~DW1dHz0(LwQi3#4&-Et8Bfh)x!K{gbGtmAl!k!Wt-RFidX9>Wme zK-Q$ufkP5*E(=H?igWHexBK%k$P81c_1Px#K|s8-!+6=(~zn%G8$OP07-H zOa7~y`1|htZ*P~Z?q4D@b#o<^GORsh`cCB0NNne;6vz$0lZs;4+aJ%el)P-P0qjKL zodY4?gEhGjZ{yAt&jhFv?ucK`vso%T>ik;L+}9T{aeb<@IeAhVT8 zs2xknz$yf!{tx-RsUjf;1P-!w7=QW|qU6ZYjP_}PO2;~d|8b=@m+?&-%-F8ln3>i{ zv;d!*YDQ}A{gz|@@!_Vo;MfX_=i~+}U`-8{U#p7ED>s?`$A|ed;o(y8RVUq-sOIt| zbzfoAXEGTGr{tmbC&V3~_+KmzU#+Wxy*tknoe{;_ptNgN+M1WE2o=2laRrp+e(2RD zeigZ0{K07bZ3SS>iZq+FNs5#hvaG$LjupF$#m-bnWHZFEdES3LS**oDzz%3Ss>sJo zq9SWh?QP4#-B+Fe-DLZnr_i`vq?9-U-wQD!A0pwvU3k951N03i{=pd9Ji09f*@Q)o z;f!~#Vj+sYW@mf-=&+i6?LV#cFK-3B*D_t+qLV_*4DoTM0dPjNuSgnD;JhlN*JV~R zYQGp4T`vm<^9)P%4~HS!F4Uc zY8q*y|MR(xrNpctMtt|#=9j|Mr$SiqpVNhz>jeHQHz%QE+=)9rJ=lh&r8=cg<_cl@ zbn1hBW^s;T|Abx`()lu!k|zJ~$Dl}K8Fja13;HIjXVe@tKisNR&F{*PpI>KI zevi9*Uiex^O{YI0^5OHKR%pFKWA{(@wCOck!A+<;uq;{KiX1ZhRVXn>&cTYCc88^j zH%N)yi9i*%v8dAw%f((Y*kHC;QFOv=MW)P zFyj6D1m{D+rVB=^?i4(N-P^CI_uDfHS}|*_y>w|6e_u<`ZYhuaL+Lpp zk2qY`)SEBMm)zz%{aSvTTz*VZV*3j+n)s=ZR;Ao5)-$$O8u902o)8B|7ha~Hm2V|o zl4C?I>6-0c@GI-Yw9eFo-IH21my%sxVBAQa|Cs>jwA}~Nzd^oPuCw9E`Q^^5IkLMb zjW1$!cnd!slXd)LBs7_mhZp>IK?CM>98LOy|UH-4h3vldr2OylZ2ipx2H5k0?@WbW<_+QOMVe*qqF0D=hZh>Or~4GHJ=&xd zfyICISf8#rt<#ycSiPW~*=)jfFSWxF4L_G0-e#L6z&Go7P(L^>nzZM!cGexB6wWZo zRqP2XG-G4F^pbUHM37Kc-^oFxBx=*aJWBuWlAmqu&%?J?%|F3N!m zSW~Ur$ya2!_nqP26m-XkCKwpoDHAWTZ1TZh7YE-oRf$PapUTa3$K@K+Y)28;>zaq< z)n?PQ4|8q`q^1y1eG9_Tn+usrcwcPIS|a&}NWsPmjn80H^&5sO-;tBxiU8MziD7Sz zN*&Q8>`n68q)m)AsDROZZNJ#m=2z~bJ3zrFw6-m`Tlf-S(Z;$l8LiPD$6DPU-P#hf zista5W@;6NzY~4)FKEf#w#+YejAl$OYDF@mV>F3`Z<7yIi+48M`90KwX9v4d`#9aE zc-v&AKN1TT@KSwRIM;QN^9^DsgUYBMR+P#5am{QqYY#Tne^^k_Zrc+XIx?B&*A(el z&@mmv!Zw^k<(ZvbRwP^Z%2*A|#XKp&tv~$YgW_{^eA_fzXKMP(b|0WyBu&zklk#8e zfMibCS^EnRC~L%~RRfv(tS9#G8|BBNPad9OYfFfwvWwb=J$f+dTbOc%pjf*x$tDv} z{-;&tpqmT8Uq^jlb@|HMXRL>g6@t?d&}nk~%WvMKAv5Qde^s#3a90-JZ5 zR9O?KO}GG}bBq1{_95m{j8WIa7jWOi8vFtpiM|j+wXrmd8dX z5@{HAnw{-_B|8c@IIP71W5-C-v)#U)ssdZ@5rQ1q=uTyot>yjwuqa8fJ=M-7jkZ8Q z#S~OYYjV$^hIyL(mi>(9sw#&L5aD3Li zGLvNjkQ5V09y}659C-w2@kKJfZ`OS_as@F*++a4k=qSK>hl#TbYHe)AfP9QPvVMYx3?-xAxlHSUT*#@EL zc{}0W_>_i4%qr{p|3xqFE+%JG3bWYlU1qYHFG^2XavOZsTo|@<=CER6WDDP#46oaP zLR!W-!Q`n%$i8UpQx<^Q4QR}bdfRj5lKKtrlVSH$kI$I`Sj+_Ge%(uB`;bAt9mSe@ z#k$jHSnXJ-7^B1tSU-zm^bbY zVXH_0_#Mnp-|Xby4T-vNRKQtfD`U&3?PcqKE#Rj7LEIzufx9)dZqc`XJ+-1(gbFbE z@hUM&7O5ISy@m+021k7=>55Mo#O=e6>+$nGN`+zvlNy26%K%GFEip9Pza7b4>S9JSG1EU_A|L9_@Jv#WXkUJAKSTw+xOtSGdP{wW>A0^M(`V zOMODD!Od&)J?TCBm>Lz-M0iKDn`9boc1T>!uiVhK02bq}b!X}n?2VloHLAVL{m~hh zBQqZLCF%Q&9C8sHpYK{%_lySW=b|AR*!Ly5+@&>O--qW&_F$6d=V|()-zX08=)EmN zvSGSJIX`RLf7rcT!GNeB;D zq(3aRV?SD-DJy$@5vmo8Bpb@?%tk)FRd;4<>lWN1MGzvmKPSs%BGUnsT(;Mz=@ugO zNy+qlW7Lm?2Cls5N-AWLb%U zkTXNQeE`cXMB-iQeD99s&qvLLs;$edXV-|U=#5^|+1Dn-)Y}D)Rv@vdYPv3w;z18v zZXy+OI>FaZwdFFsw>8{RzIwbanNn&R{4zsHX&Firvy%(d>O{R%=d`I#(4JEZh);ve z4Q|x%EMRpgz4Rus4NC~@sn>(_cl#DDLOqebWS;vsCWeZ8+)O<91~WzM$yAo{q_|?s zD7es%V*@(*J#yV4I-1^HsM#TrY6eLy_nAWirpnZ(*qCh0G=ewdPQqM@Z6Y~OTFw0A zcNddPXE)yNFweq3Po*J!&PP9}84mUM+CJM%=$@o237^)qED#vY?65dll_z`~?p*>j zo^wbJJ49;$&e-ktlSd2c1ReQD`tBDo{{m?tA^j@l*By0_J?huCqFR z?oGkI#Zr^(`ji`8q&1e<%%%YCx9 zd!<%k+4?qv<~t!{o$KpK8GbMHiIaDrQC%aio!+>S(F!S+))NFE&|aSXyWK_#M-7UNSg zBx5uP)B9@D?_{>#J&!ilDh$@N$k|JBKU2-!d6>zczn#)gpye7sT}W@Z!#=i3m(N6^lB()Z&e}kJD-Ix&s$yvTbxJ$D zBx?Nn%v`UZLSmGYDZnS)OrMR<-L6fS@Q?zrA}$| zB8&y>IAX^wed;mChtr~qFt6ov@AlXry$f~5`g%{`g@-uxZQP*RKYc!UT56lglo)EU zH|z-Z1(j{`mWg1i1x8R=jZKcso`Y|oag33<_NcP@36a&QxB2~i9(D@{!v+Orv%4=^<#XDk(PI4PNxi}Az&}$?QMyCsS z#Nn+Q>6gsNarfni&ZiP1Sj>6AV&>A(>N21*sNZQe1RSza-Eczy82-I*nX{XY|H*?W z#O6T1++7_hBD4%C(sH*|#P~?r6A;UXO6j4J8l~GMT6xEf zZR=kobYp#Ft0kbCv5g6($E3*5`|@|B$hzQUYt<}`yoq2`(`K$`)CkHUX%A-tIpNOF z)s-Ks^5crsG=~7-%(f>yW3K&KkHECIg3EE&nT%)*B<0QQNeqahq?zl*T^F8pl%(hH z&KH!3ccA7c_Mmn>O^JPRMaNu&#xJi+-efufUC+%C-Qa~~cs=R9u5~u~e!~-hvaQAo zRw|-5awBPU$*KV=L8mH)lGJ3UsuKg02S*GN7qc+W(Bfp^(L-xMA}mAZ zkkl$ueKnYV)Wb3-o_NdXAMXTY>(#Z|%5_%&I|HbfOiO(-@Ja!P+=v{;Ab?esWcq?pTKx7UYsZ@DzM=!_p_aznJpPcVVPp{&!0WHHh*8Qz9lmw;%}xq*f_cSy<&af zDi+2c3d_;*M#qt+`oxr%AJM2i$vaZQB+cUt5RC>XUJX0WgU^=wVoc;C%{bViX&Fgt zpr3VPMl3p2Fz1k~SKt(?`zj(dKbmnzOAtM-q<|NZ%@W_|ScMSU2NqH3KM<1KYjE zW0}MsyN*B7y*0g;a7NW-FuiE6NZc8r#0VSNT-bcZ>rj#-R_Jy}M>ZZ*(+cRU5QXaH zsp&NL0;}wi9SX6dWen~vGbJ?$_RJ!0G;`#BPeQl_(BCH7b;iWP*#!o0eum`#AQ0iB z_^G3IO;_4HU-tQEUM{A;0|=t)K5ivurU=q#fdJKu`s!yjZ$;JE&kzn3p_f|Sm(>s} z_BP=($CB>KM92XLQ};nRT~nQ|dhatK5BK#KoI&G%wqG+RG&iWdn+W>N@MP`y2a%5# zp7lNv;UcHyKNZg9MqVKcX~(cD5tYHA_=DI3PvB5PNo$e zDt(zCr@z>93PZhB2GQ2&v1dxmNq$0OgCM*(N`f2YCn)O8GiEMW-5{c08IX}M?Mk@QgVfE=JUlm^{_(O5k3Nx}w71{| z9s?uUJs4fmu0=@TMw7GZXekiojlr&2bv{O~OeHE2eZSy+xBfS?2|%dGBQ@eq6BX2H z&l5&(9GXNhOeOSpIOT|fp#FNr=(fLU)%5Ad9{?PnD;SzyHwNYMf@W3V-L&O9e1^{i>_}`VYa*bohVdh=6hc>s) z(MGB%+7EBRDKX1o3zqX(jzPE9L46iOc2c~DZ_aWHb%=0zDjr<6Ul1EPcGy>l^ofYF#?k4 zhjyE)`nB7T?Q^M%gA$um6Ix|4$hZZy2#mvdYCIq2GbwwTz|lUI`P6X(5rSSO`}Tbq zt<$C3yvi<3v8yQxZK+%4!l3M&<{ixGbL?Ah-nTgmF=!$m4A2IBYybdvQjayQKHRS^ z*-Kb{zk`Bf#`*lMecfNHDqAOlnieNhWeVK)Cb!+2Gt}sBe0J2noxJ+hU)N_K>vQ$bV9TG-K4)Q?<7szoQ8x-QmnTY=wldZqe5_YD_uBwev|T(sf+nJ- z9y--^cIff@lJh$oNoO971Xv4oBgHk<9Ajf0kj7~w7y7K2Z&qPT^c@~Q!ajE6QA|GR z+-v?bmH_>#Nq;Q;^Q)!XWVr3ev;F)=#J(}b<)O6a{Kv!QDiqwF67O^GrRy$F*DDRV zfdoGqp}*OchJn4KPad|SO- zna<8_ShKq_7!%49`}r7~hCPB3>*o>d0u3Q|Pm(=3<~QG!(e4#avc0~yY0@%Ct9@g7 z2h={t9^GCCO_Mk@HDz$SgOqzk?WU*IjOF|!Bw$hy{+HuhwZ=UIz$~i5mfrKDS)5J?T6Rv7o35@ z2ht~?Y~NrIZXc_zHz47lg6iJ+-mFfW53vDX&q$wm+#RcW@QB8ZBTNe=ma*e`<~a`} z0-X!DqY2HK(FtjcaSyBefo_YdXf(~?O{4vv`b3$LyypV-Q^**MsWP2h?g$> zY^QLaD=HEHM5C5h!5kiUr`Z2nz�%Zx-frqdm*QBJ^zvtDvKm@rzKs7wNij zAJt=;b5Y~*3Pg@zZk?*{BW-y6rRth$`O>vdSwD|pvn=iT#p5sg&u|e4mHEaADIymN!JU?N?~HxkH<5nv&D;Kx zJ7C@p0$4S61R<&-rlqBR35D*ZKA~_SUvA4A*QO7l#O_Joyc1ElJZf2!=1qG1$qKzq z`LQ;u$$G#4qwFoivfR2ZP)SKa1r(GP1OZXP012g}k(5U11}W)qD;+8!NGshf4Jsks zU4nFX!LQZl-c*IH`O#g;b4h&*X}>(}`W$4agO`EY=5==C zf-cBHGh?4&St$MSMqKYpQJrTY7Dd^Z2d*q$qekYzoKhtmXYFACcepveRs)`+cJ4Bn z&DXX&PoA|4U8q4$%%m}m$3ukgr?2@2uE01!z7OS(Xv7UHEexK#2dW&aBahJGD9j)? zVtN6mdRN7ddYIO$_)YFw1C##!uV?QKtJCpH2-miS(&Y{aYgy{3m@;9Nd{FGZC#?=w z#e#WYGoSj)g_zdzLG5N6s{d9RlqhL|nBy!|*rKb$mW^O)(1s|gBte1wVe;C#DVAV| ze|^I?h*Ri_GC;m8L(?~hT)`j{{d530pt+-TIP*Y@lqs$Pq$1?$*G~`iI&;68$Di$@ zXcf~%Jv&BGJ%5u+?@DXwzhb}Md|UsEcioMdRrhOR=wh`plcphA&WpwwnE}i} zKso$!_iGsCSp!l^it#gf3`k0vwlA#L2KqJZmnsm+O;b*Oj-|bM6YrxCrkfmairVH_ z#PNjAe(9^rp01(fnmAL#uo}m}%U_^KbK@4O;6%%JtXfl2i=n}+nuIV7u10>c`xTY} zG+IU6jc2$QWD55_y=Q+_==GgRWxsUZPobbij-!Mrt1>rg$@?WciS;2vR+G5l=W)A< zQ2Be7YTr!OC43Ko+WP3-j#4@Cl4hTAj{ZKY@EaC`@0u1h1h&U6jf5JPG?_m$0U9S zgAbtzx7oZ%!oX85A)|2M8cbX`h`t(m8s3mJVOM?8d97bRrdjy1-+L?^I+rZpvskq5 zZIG8Yd^$Dp`$QQ$M2QKudOkp#%*x9V%tR{^V4`uE^oDVY;H%d9z5bAM?$6I=U;Og< z!V+OXRayN||A=Q2xHuL%%v?<65z}$=W91!}D!&+7D)V$^4@<%{gJ>z4WhCkOAXd6b z_;B8HXGa7;cj}kK#Jd>863G2Q6}N(A`{m=fD5OOYN|$dkDT%ASv-iN;Sq0O}cH8(S zYV<@)v@k63Dc%PYR(S46#D$IyIwG+V-d5KV|;eACGaeWKQG?d7@7 zUNtMpTXRaP-dAx39qB=(>b3he-fC>`ZQYiSFbBMBV|QJeD%E+K`Dzi_g@?*`0sUSrWvMI`KuROosy2Wan%T@(^e z$>i6;KI}^34WVmc@XJ@c^8og7haW&rH#Zjti?aJ*;7W)*&g$m6Z&0Q)?{x`tRu=YG5? zosHp0_3v3$eo%=3?o|6XlYcV7REzymB5#FPpVkZf+|o*V>F7un-?;BEwDL`nl;)iU ztoQZOiW_Rq=|cb(<(m5N#rj{mechjuuVSIXUhSf@FRzHqkNc;|gX^yA^gjT7k>lG) zJnHvcI4^aF8fKO##bb89UL~2ArmiZ?plTy!*JCc!G-plrT<@a$;{X0Adf6dOT;XGL z=-j~3)_=N6ziCvkY_WA1Q!P>NVU(|A18~vc%iOO{2R!lc6~SQYuaZ!}7%$!28R5w- zI0AMqBk40<7pr2ZM{>2&dfw>E7u>u?8-(T#>Z05Tim4nrJU&$$w6@AUAuGPy9N(8v znm94)Rqx>h!xtaxDS+aBgG^L^t52KYIDkmzWH11qT5G}c)t%*&UrgBSEeAMPm?CRj1!J;*I0T=qI+5Vmh@rCA1p$;HmnZ4iIj3`D*Q@ z=Jl7}I~_N~&@UD#BDfSsmJw5@!MZTdjYq80DV_4hoVtYS>t7@)$TRi*6P@dBb^jNl z0MKTEnCvh9}8~jjD~<`!8~t>k(m>Z|w+a9AlLi@Kd_R=93){oMSfw zo|s{reMPM3v`vB)NaiVj68JagpoiD|DySc(MLJy%>RfdipQ@X5s)U1K!BfhHr-N7w z2i;jNA&Z5OyWC!;5NN0)*LidzkasJKDcXy`5rOx)KsL8iPMSxdELAV5a;Jt>kMz#O z^#w#oPIr*>%eE6#R$}^*mcq!Ki`>xg;|$s7%|g!V&cn_~{kgx8C*6||HBiRxrp!>! z&e}CGJV$o$5+7IS-+M#mVK3iBNWY$2uxPz6Wegb->~azWA96nEtYc7EuB~Z)G~gf6AzG9O&xx zvJbku5HIb3R<98P;K>c$Bxc;gowTdmSHHF3fLBf-;ALG1!(8NybEzAvy_H7=yE4a6 zvveXCNHds+?EM@VTQ5Pvs2K&<5$N1W1-HG(5;(mI(7l*GU=1! zJuy6PSLm08%CAG+&GtT!dWwK&qiq9e2CuK)=h0XPm|98nSlo80CVYffy1Z|bfPm+j zmYAk;iuV2Z7ZFE>+)k6h!kmj~C;Ri)Me?iC2*P#WQ6C4C)(V_nq(mKI(yJyPJbn_W z1xjuV$cXyuaWN5Ao?n2A$uLAgkuim)!kG1Ty6rb%ki1CY*Xi&dji6Z-vn0K3rnXh8 z>+i$+%@v`VMjd%Qb%3t;VR!Y_4gw2QSd+rQ zf#M`&;h`|-B(2+d^U7y5f_p)dMVy!I9ZkLYzdn~&p-DD}s!w_qlgEQ}qVHwgB&|W7 z%s8&M!7<#TVdH1I5zK2Cwv6(trq8!l%RrSc>7pKQ#iu@l5B{>~xKh#g?v?`zrA%mm zr^IKw&^N9wi?>Qn%2Byhkd=T%kD&UwhuRE#$ixdGMK#FlaXt$|u-b6>cLU+)Yz`}r zw~squ?`@iFV-W>VPvB}^6ee6jpwP`tqz17}zvKDZas#ft=2x3u>#p_Go;v&65EI#( zXF{DF*6-PfdveJc;t?M=cgyaHc3M#4k`$b!qie6aL_X_1p9lZA32X5wg2|UkN#s&=1LDN*v zz25e6b^A>dd&_>4yyn~=+%=67+nM|aw=P#+Lhjgp^DpfYJNR5|-oiBDlVP{7sV_%U zi~JrRDUqwXI=9r-EIRyplZ{P~MBj$_NM>|!*z5ZsGPjXV3RS-jo2l}6OV*4)@N-(q z{d{Ib{pA+k0hqqzHN8Wwh%FQ<4;k~~$$krf^4I~=ddz1&9X>-))3 z-mbdeMdie`5y>`cEM?+;x(yMRQ>eC?;cSy5bb`YcD6-<7`+waHL`>}TDf=b_9PS9Z zS+az}>EhHr7cS^|d$^M@&|UJ_x~5V?oa2x+kydU-Q9Nm5S<)^;`mutAL31Pr{{N;K zQngj^U$OcGJ2UUf*&5X>Xc-&Av@6MF6+c*jCUeJ+~uzBwf%OM1e!mdQeOj#%Ky#oe`LzNEo2a4+joYgC)#!%+6{x8^c}d&>faS=&eES5Ot~oHD z;82SftfXPmmmUfsxBCaCt{VGSaY*?}I~jOqgicvIwfT;iVq_&JvKLP=kq3@ z`@6)IF|SccAMu0Qd`L9)KYS);y*+G?()*!h(55c>EV?sr9j=q~9pwZ%Gn?krkPKTs zv{8IS<-!*r5nn^+LK%oxUxOXu)BAquuKS3~ajzAR&6QsOjTI`0C*7To;FBF%CDknH za`cX^13)!fZrCqQj%Hr>-du*?#hWyXpC$r$MlygUux$`z3K|q7rD?@=+`})?wg8iQ z@OJS;swVj%6`R@DX0=rpQ&0mSUex{ox=5GKPQySKCr68M%T|tjQFNOz>SY`$P?y>xeyb?DNb^?honDfA|4p z230y|6}w>@49`%nt|f^pgg)~A`A-IY20fy7O5yLD z13J~qIO#y*QMTQ#KPR90mEZ>8Oxnc?#mXe%3)&nWBMjv>3owaC&z<$U_!xFfrb=&V z+DIy=jL;u$&LYk|C|vrJ^=d8!ka1pH1dU@Jwi1O^OG!HosyENR1~nrSqoQ=k#CHk2 zLh;aWYLFDO8?qEZSZm@X=dhm_!gcjVSIJ7Fjw3+DqBY~l@H7qw&1#$(y(&{-x z-;pOu()sUG$xn_WDC~JLaS^)3O}fi8xHT~3xd?;UmlAef=U){!rzpL;Kl}L4m)$+* zl^)0P;xq_b^drd=Y3M!8BsODHKO(~Z2ioZAkB&O;-%b=Q<(a_0~jiRXCbcaI2JlJtzKgr<#tUkvUJIj=QaaVpzeP$usXA|1cI_ zrInLU-N3f%!Pud0p6Dg+lr#tc&_ zvtb~>K|tYQ>;;Tyr|p`W0AAzorTw{AN1Ul^CL(8hKbnROQAJg5*>#e=2-P%k{G|uU zUw}cwumRJO60c_~WROWnfa=4#*s`;;N32}XY1-@n(GXYCbhf#M*M1MA6c!?Z_;>_j zZc11`9#NU?l)M2|pN;DIVIrwq4 z6GVw|eNSp7^k+h2xT&eNA zLpRqc4JS^Y5nuhaQ{9(Hyq-8}?~IJ+0)Hp*6UxIBf}g?>Ahi+yksa*wj7nha5jh1x zSKUE+fTtYpIyy;2GoTZOc~x_pBzQ3pxj>A4lG8aCJO**Amv$Gz6;G;iUeK}aV)`?m z{}b0RlBIe@oN@81Ar=i)v>Es)`Rh_tR9ko%W5@ZKAI!=v^< z7mh~ZdAnUg3fdv<=Thk_oju)dkgw}?`hVBExoCCTp-2P_x-6( z3BtgfSpg;E_e=q=H-t;9DG}tZ2dtsbm9T?yw?#ejt%a$wO4>G31R~W`ZgU{CxYK!F zV_b3Zt3W2_4;~}MXcEFaKuC+m%BJ5AkQ@~nWTQIH{>(R5X}5P)C-Nx$5e4P+7DQhK zp{LR9hf}u=0$mRX}L9c?;Mi8=Cp}jus($ZrC45chh2x!`Qwr9I^`~0(3`&T{bPrj#Jxi63dN~CiGiGtjJ}1X z89F0)?keZKC|@B=jC|nuB_hdLGdJ)xIuXH>TuFb{8Q;8)1K?3eo+h`Ji2ww>e-Wu- zI4sA%f^&2Bn@xmN^pkUeh~I4hi!BMjuUva)JWJB*s2pjX0I#hBc)8-Z7p|K?^ZhCK z2P;FN4|>U(e2hVv5x2m&e9*E$Zmb zFca?iMIIqMA+*~jTSRpwFLvaPgbECZi8jDh~ zMSQ(GyvbJ68ABSDUnI+voTj4)%(c9>`)@j#>zK*SNmGps)-|g@7t~})%}LMaQX}S> z%Y9`{g1V@gA|ZDb9MgcWb$fq^Y!sWQL4W;~&KV=;Mha<6~Q&&faR- zu76ECpBz>IO6@MkN`qV~zlI68uC6>G3vO97XZ_b@MAgQEdiLT)_YX-iE)JpU&j2Zt zI^K;r4ku5$vkw*O$UB-bwpWKLX2p}arz+|nAo2Jkl7U1)%Wf7by`T8*dyO2)C+W$D`gZZas_tbwjVX z3zPG?B?!cyyL5?(t7g)Nn5pUXDMkKSppViZZlrg&uJY}A8;o+4evRgFlz30Z{4OSj z74i#DY_hJVI}Q}h7r}={5dfO_*=@kSv~&ILB(>oQA4^UI94l-L^-xdxYYua$zQU|E zDwM&W>5_=ud-4v`2zsrqAws?+-@p>N`A&B{n^4{>UbypX*P-q^U)~atu+e;bl%v-K zWJ2gy3e&Z|=e9sle0;vm0di8G>+=QO>G^<9lzi(+RTczmJ8IzfSa8N2=CE}e5w5km zY_Y#3Rm@X2bXx&=lKX@&#Qnq=u$%6D`q)q%$8{zu!ey#Dfi3mXs0O1Z*7c9IlAuxf>K-YCe~u+ncFQukCf65BwU|;C#=;t6zskUOVH$ z<~^|Bxc#fy-E6y>@!R{9 z64je+Q}36zFphEJ8ns6AEHcN!%@Q$VpB|5r1#K4p;uwziA((Gf`u(9@VUo894M^fB z3ORAktGYQ-johgB&p&r;z0hqouUQ;(j!MN;i0tQjS>P;;R*oLrTUJZrdwbV4)7nh? zt`mAkN5@AHOp8bPtf&k&tGh)?4h|&+K6dC2Wce#)l8+T_Ne*wPTG`@G-wORA!0&oG z_W0-+gu50{X2V`=_9s-@#Z#S_OuGnHV&21w@9Y5hCIq! zNnG=c{BM*o@dC}!qb65#n)dQ*TXxI1W{-pbuECu z9=j=Jylbt+-6^DKKv|dc=e~n!t(gn%9D|h#9OB0yA)J0~`N{QIl=v1mE~ZPAnuR#) zxXbI`XTz>+_D?dCtw^}V%3W~^v*`&7Am=W@Ozscdw4)$8@zkjbMAaeOoKX1Cda}!Z zJ6;L%=P4SsGY&WI2knB87e_x;k?9~knaM?lwns_l3FJ2(JBKK7Fv#X9M^r_BFTO0~Od#qZT{ML($%{yn4 zuUxtE&Lla4v}bVY4V`!sH5ivw6eppQXH{1XWU;6gq7O82q0h-{)#a3T^BxZzn0W6! z(NN9n!ogaZAs?c9K@rQN$n+{aaOeCai=Jp>m6=$2w$ zpYhMVkmh(CJj|y0KtXGBuKUf~w-a;GdU3rmD2%VEW92O8nQ?$e8ONg)RiRv&4u)c^ zj^wyl|GaT?c_d$X&w%)_C}*UuCk#Jhv&=KADu`O(3OD!Q5qaQICc@!h119APIR~x4 zFJB_}Y~nG^u_um(q!<6S0|}2%d#ug53Q9vv+1ipf(H^GF+#EUeD1$rccG1>m#eDF} zrQ2p4a=sbrrHVZns`|WrY86h!QN-U!nP+)MR0$N1%NG?3HQHB~2a8($)cIz5bLKZz zAZs8}HxFs>(+q4M(>QFqBLj6xO1TRAC;7yDFvo5hE>OoO+$=Q2@Un#$(4=i6pr=QV z5T5i&9`+~8y4V%hM$HF+30H54|va<(mIm)mCUM?|J040cUT8yVBTT7e1F z)lxUiS4M8hdDhD#!%fe=tJOVwj(dfUF1%j{Iy=S$)&JbL>)l^Ud7GaqsMe&{Ke9!}B1E)DR zx#S{~{tk3e5@KR|0+u3*^rJAxH#jEuhIb(8k+WaOsnY@7;RKb=1!;#ey`E$-QAtUb z_3Swv{S=c@#JKJSZ#l}>ku!6fiByC>s2SrOI_lMhMm8D7vX_*trYcjrgq6clmmV6}A)mqEioYyNar%sG*jlh&e^ zRR0}o?c8!?dc0ioc`75r74UWac@AA*a5q5|hauKa(ZUttVb?p7q$wQ_jQF5ag0rvmkwwDs`pJ@D{R3ky@o^T!Gid)hRc`zt=v;P7CN4L#`t`x6Ml<1- zlpU~;cy!nXXI#KPJr;ps5_m6n5L8(`dnO@QQ&W><+%ckY>99yz@g+tS*Aeh3*dVi@ z`vD0t6OY3NLw5(LB`w;7kXAa&U@;4dh`eXK&KdM%uyBq2h9ak`5f;Dv21x7^E zvQ9e5XFLj!rc}+CxO?%}fg*kk=Qi?=<>%VADw*c5!rmq*Q4z}QjTF32ef(hh*C8>v z81n9G=>P>s#+`+cC`gFrKHE2;w{VPGC!=AA5+Ju@87}vr+T8 zJ!k?2!@cAW&UELQZglM4e;&IPIdAmgxMf|zRyirScTAPV=mNtX`JxQvJmFD^l%|~w z?FW7%iKUKy4d*Aw#a=jD_@XY(fg*oR`yT9B$ z^J^t&W97WE)QUS2;z(y28&#GE!o~V8v^-;us^MYrAx0!^?| zu5?l}My#2pn=QFuKG$F2Txvz!c=DY54{-^p$U4G2Td4niEWXy&T|$qVYgV57QWDVSm^P_jq6 zl5sSDX5?m<^>HJ2(eU?YByL4OBREMkMc>^Rb0@O{+X z9fvl{3gwXe#ZgPOGKXWb3w4Nx8_x4Qucd(k@eQFr_P_@KnR3t-APLAF-;6_p^`3BTV>NX*U+&nz$pfbxz%59sj24a+Yl*^uC8LCC0 zetvj)i%?buWvZ7p029R^rhHFryJTW=X51BoT-~II zE3;d1iMcwnF?n=FE5igw8Gjupc_seJSfIb(EDeW1Jv&lfs;-ZMLfqzhp6=?^!HZnD z?zgl1M(R;f3#hV9`q_HJ?qA|CpaM77)R*j5IF(o&rUUY@$tLEOAbC@>x%TInCPSS> zma6p1<$oO0i|#Nbqs+0M#C~ZC9?gXOknV7yKW`FltD7i(WDirT{UQwIeWlP6`-PC6 zha%&?2>7W=PIW}{RN_WAG&hHU^S77F&TyQ->9@r4?C({=uiiErVt~%O|1h8W4qz+I zrzaG*&JNc__>?E^YmOAFM=^f@DrQzav>;h5EiI?e*rQ}M(5!6+WbjM6jV(c#@4cI{ z={QuZ%pd4n(*ptn!(0z0DB~v(Bv2=js4I42y_HWBM&9Y(3iUL&O_p6fyvgN2DvBB* z+NCt>)HIo(_Hf#**02ZF|c78g{yBQHzb*@<{nNin()f z@u=>(UUTIiH1|GGfwm7<#=$H6OKM4Uj>*0+8n#hKnB!qeq)7i03ij`N>%?X{O5Nv<@shzbSUt>Uu@XaYY)7e#?tfs7b`1*0Hy* zw;na96NIRAh0&_3Gm`Q)DkOQpEwT}l1wPehn;(26CC|_jUw&*&wgFbm5#3}}DgE4= zX!Ua>Jg&C3h1mDHwM6kRW|+8XX=p@;dr0gD2L}^ni*t4#`$TeUc~)qq)ER3TOGyQM zYl#RCCnf4#YavY?}k3-t^7t$D=bCK zw|WCsUEwSPuGCFo$s4$Lh|tSW_g>yTJQw4#+H{Re&iyD|l98oqJvm=AyT@d>q}z$H z%Sbv#c@IOz`PC!B6&%1`xO4SeDI7PI^`q#3Ur{j645AX5u19G{SOzFRm(acKycc%) z^>*5Sr=BS2Vz>tU$(T;jnI97pz5!q)U8I^&G`~AoXlkqx!n?DNCMzo|s5u!DG3P4O zqOMW7v14)ECcxg@62;x?w|=2=M~FWFiiJk_eRH8kxdX4ZN{Sy&l3xWp;TTd`f{sEykY6Si>$k{XFj0y2E+}F^{w!QS8 z-2=SF-ZLBDJr;+EghWhHF*Hqb*?D&0A>B9zhg;uW@n(0l(^qcazCEsf>h}%g-bS(_ zsPxKKA8ji$?kcvC;{B@O$KFV)GNz5PoHx9`u)W$g!*%k(Zr~vv3GEjUD?Q&y9VKh= z`5$b(h%OkDZv?s?Gh{#^fDsLt9J?MbAB#4IvIr!tZ=jzXiDUtHs1HA(?!S5SrXF3{ zv-b;Yl39EFOT_hg$t;#%+mRL*tIMqlK$qqVr!o>RaT;Bjot;$-u|pTbhO#p_Lb>++ z%fhbv>3{CexyMzyLd>%L8n=TB42 z&;gH!cuJh5k#bdy=if1k$9YnL|RM|pbzb%|NX|v2BbPbFRQ)@ z($%5K$;m617-(@y+(wvM#>>MZ2OkSl!JFOpUx=$FIn%mTw(Whp?an=s=Hy@51b{WsU7@FrJdZ zKhv#Ws<3gPac?2%4^VWqx`jAuij9wNR0nslH!EfcNGBbo zcFSaU*d_mzB1c3F3=BcAaX)b_<40SXe)Mn?;9ve}=dAJe?OR%VdwX~*PqVa9+#?9@ zSkDAOjIT3Oc`;Cq(zqo|LTG^nZt}wC&Yu@LKI)W}m)CI#Y|B*VmYa6{b9R!kclaFG zc%*cyQuyPt2#&PU+y&*Hxg_WPgS-+#Uevv&UVXm5tft+`pfdUZBS4xFO88M8dMB^R zJ^O4Qm?5QHrWn1tdWSPLQS!amM6Lf9Jtw@6dKeq}=21p20Jv>=Z$!`aWW0w95WaKU zev`6Fp$X%+Z{J!Jm}v=RsJOUPAS2VAYzW!1Eswz>EV%plWY-g(UpXdP$Kl$NFKpY2 zPMH$Gi&*-(!07k+V>b7i{$B0BvR3|wSV(x#f{`pF88T1b@up zo?owiIq&-K8~(2s{qv&<&!LXQzEjLlc_~;q^N%YW^(F{ihn@BgbK)q?4h!Y2W!rJA zr-hS@CHdJYjp-kyUA|+%zqY=fac%j}yLpJcm~X3Ey=&y)wRBG!;3YDtDjC1btlU@Jq?}l*&XlXP!+!OjW67CvGIvHMhN81%(xf|K8+zH`aezWmQu`uDF0KckK?M!dgmejT7B z-0u{f1u{flT%0fDAxP=Mf418k>?|npw@%~UCt2Yw-8h1KUKypCYaWlVdzu7vd=zwC ze}9E!e0R#V=-%4?JE#B4O8nSvRUQud)SBHUsn6ZrBS6 zA3uH+ZV~$MnngxUjk(lz@fr2=WxihFYgob5vC4VcPhH++Z^BLa3`Eu0w)8kpzO{@U zz&kxrJo^9V$2l=tY|K1jWM^0Pz3U|G<|IbxI zr%@V%q9U`}@r4fZMP*P>mU#qY6)@WR5;}gbPf4}dWf^(-D_5l=gB&*JL?1lxaMn?t zUnKu*L9JH1Q0`i`t#na?>($SA4bQ+!MD-$>fR*g$o%H|u>2tDG?}ytjU6BiOo)|hJ zG8p=llQw~_QhX;FU9`$k(|);>DvDl<9Rm1UFsyC_()RLzMvp_U6mJG_ScMDChE-Cx zfvNK$=OP%YrpldFqH9z;*D%2)6%{n@KjWf)LZSmRst=KQtAvCL7HM-NA0@IO0l z`29pzViPeUQ4}3AAtP_6dW3`8O}v6hHa$71P$U{4QiVjGGI!^#0A_hr1oB=31&{{N zrAZrBOuju<9{Aks^NBKZbFkj;gP)rvQ&rvy}k zo`Gyd!%X{!jA7IYV-m^5Buq8H*n4y)8tdzWq@uXKK^Nr*khmZjW9ZG13PXl|_4dK0 zM(5~YH~0DKQ^S$6;*I%U>CDO_2gA{d^7+1!bVwpUNoA2zGGd{NQw2w>SCgGz>?(KL zUJiYT<&)uP<8X3vLy#lQrn2DspK$IbjoxBBG)he6upBDNR=9df)ZcLI?b$ZT->aXT zT6J{ZQ|Nffj^wc4&Ew}k5N9vHdkDm0dU*0EAR1IJ1zjn}qTD^+LPd9|hy^(U8823f`B0 zEByo1@^4kgLSDyF{ca#S4&`ruYr$kW}E2B9&tD-hl#t&k4r66QE*YGaI@yJ6c)EtX|TU&W!+XN#VCRljUB> zaLy-5(H_8nR3k98bW!hc_N^mE^lHP=yv`-xBvSPvOgEG&_d>o6rJzKD;6}Ja1c&Gm z9D+s&bV6vNQQ`bH!_apU%%gx_e6#F8xuac*Qdac0fK>N5vS0wf)sQ~^z2k%ROe@t8 znD2`?QU~jY_(q3v=uSP^%M6#~^qqkg;K%zfN{(e2-4%NjCd%p2J$4s5roTYr%`A7% zNKcbxZ(3dxyd(pg^FGqo%7onF%!8h3rODBdzP`S=o?apz#l60J=Qr=}Lhm&~4-`;h zeA(NExRi8jRb*vpDtDXBuUx%a$EAkgCA;7foB$=oROA+Bod@rStJ2XX9;=)EsfV4@ zqjX_vPowPv)YW|zzO|$)CvG}{4^1HuOFyGG%;z2Yv9|<)7~BJx(7){Mo$~zR%k5=; z)Is$yFPz_3<3DcvHLkhX?kydELOOEmC%>yA6H{HfnMj@xC z5BGPFJz$RIJ3rHk`MVA3K}VRXyu)!v{sp~x>EbQNWPl^yKz-Zr?xwZ`;S7)vsF4Bl z#l;>EDUneqRg^zj#Bms|4ToKWLF8jOSvk2$U}}fh%=d(c=?!-A%cTv$Tt}jF)R8*7 zHy4^<_bkj7bC8NUZp=JE<+m}iPc!I|!td*HlgD4{icNqSu$}>C;BG}?pXr_*1Cp^g z2q}h_n?jP6NC`I%duOe2kmB)__GT2%LGYt4E^bF2yn4}rm>XDbaF!H}x)`V*Kt*A~I&t@oW3lwc5eu`w4mBZGL=`(Zef?ekJ&X(X8F1$$74l|66cCom&vtr*WXm-&gE_o2-J4L* z(N9EP2a=@n=tBH-ZsM!nQw%@_DCmSZs+WszbO=PvJVjr<@11VFX2T`nRg%cxzSX-_ z0RJnVJEU*fw~9wZ{K+P9&J^4lf#Dn5+74nS#m8y$8&UjeT$2R*ZynJD%SnxyP%VJo z>xCIHZ?%HZa1d7wdf#kJuTd~JqPNO9vhU*4j0i8f?u&U%16*U`p#i1nTx&otjUUvd zwE^J9JF3mxy~iOrb5^44uXv${Wt%q_KhL2xq8iGoX?A&YH@W%{nvaFp8@;D&S;*8< z!^6sj^4RC|SeTd^b@Y#507=JVg>yx2u-#?GN{L;3ew{-G8OoKkNanTU&`!a8X%u9l z>mg8}zU=&!Xx%+u+4J`mWy1*05+z;JLw{wRU9tRX^63Yqb?2XLPgOSHa`R;;z0aza z>)<#!k6vz4Oj+RqoOk41=}(dZ36$oNDdv$_-s!>tHJC#>T4s|@B4r0ND5FHZ-dSHc zSy{DxBmOqXH`V9!<2ygAPQo}*E{XcGV(lYSCA7Bm6z>cZj|`Q9Hz;VBp8_luua~Vn zU?1KuG#wPoJ2bqJjCtEoxP8qBljkm4OaqQshhZ>U2-JY>3m>6+Ifrk6`|?4o0)yJi zl|=!Y7lWIv4KM?*>jq*jPwe59s6Pg>OO645i5_Opm_)NMDT8*-d@Nb|5KM~EGbJ7s zNANfn;*oLG^S;M?Fh{f$QQUd0Wm?px)&c0kjpH@Gy5nEH6Af+JQ-;up5BIOo(h86; zKh_$_aOT)iWM?&pYgy;+;TOsyt<(8Rx4H9d&X}qumG|lw@YZdTrjmv*sg}8(ese!R z*JB~@_u_N10H0eDx0Cm;62OKb7KPgNTbZ1hv1Ryyd5g$ezq|RBQiGz)ca>#4%g4KZ zzbKjWsGIY2V()kKo@q=RJ+oT(4Ogt74>CND1imV)=dd;JJKNNNKwT1>^l+i6YMvxo zY$%H=Jr`$bo`H;BETdnPJD=%5z9q#|R#lX^EuZBJ96JwvF>w!pH-F?}I$KokxIMtM z`HsK;!S>?dRRF-wMKx$7m4bIlIWx2@tjiT5tJQnBoQrCg)bv?yn+|k=-BMOcL3!b5 z)QE!_YKGB1&Oh6`W!v_;G3xNKdgn%pt&)a--iVR|QS#nf-}(k*>s1f2+$B zoHZ`qC!f*Z#>u^4FHg7y%K8sDA9g8k3>aV{gmI+}(iB!_4Z{=(+bLk@4(mvD`=z)y z0BmorAJAi!n3#w;kTGO)uc%@23GK;lb%Sm)wZ+*P)W4RQlf^<5>E-IUNQD6$Te6lht!xz4CUeZ?i<_xl&tFx6IbWbM-ZjyliTcSWcDQjJRwJ+v_8ujL?$81e~44%@&i-iYv}GdF;l8Q$9|2tCj>w zaa$^8uaEe}>t3!kI=Mk&djJl|;53ptDL?;;WB)1IcQmUm=}h7Wq@GDyvOO43}*}E{zO2_rqQ?Foyq**cD)y3Dxr|i8^ zL1uNT3ZS5k_#W1JN?W$sw`wDzqnCIV4d)fz3~X}Q=S4ZttZw*h^Ntr2<0%ZB6kb|L zcTGZa;N(&~dFX#Z9-q~aXKUVjtn{VV@Qp|y1^LPb0BLOVoKy@fEG%QZ)fy=%@nFYP z{EZ>mT?fX%&ph{m_F)4!iYBO++R_K*Lr+mp%${jx1_XNqs>WnYC6Znj`snShJLc4y z_`l$168G5F!utC46CYXDLLL(H$}J(LQ;t z=ygPdQ9!rhRN&>ivU;H{5J@oai7*5fl=O#Zpg2}8Xk}zAly39VtXt}0^#AN}z%v=C zR2|A$NXGbYoq#02leqtZA~>;(;peVlZgzK%MLTaPBlFv0x{aMCE-+tHHroTavr&d4 z7taAMYZ=wcmoJ}z>r}Yki;=g+GSk3)r3HS@E%!8p9NQ1w&$^L<^rl!an|%wTMuYoj zsf^^Z-T3+4=Xgsg9A>Dw?`#0yKNvL5_{*x@$bt+HcUk6J&8k{3>=X1TV3CxT08Cb} zDd)~(;=cXQ+T*mhQT=yowOAaF&s@pod7PUM9lD=;vih9GbUT4!+Mhr8uRE2BFcftJ zFMed~T?tt}rUzv4Jt4)?Nm?*d9{er#0$`%R%U*`(1FzmUKufNUL}oQPs?*4rjY5!m z1u7VA4*{ED7=>cQ#>UpO0{ARrSdiU|EAIeia5{A8%V*+!uK?~pvJ6-8R<7oS`mfWy zOw)lbO56Pjr0xXvN{Tmd^O^&h0<|tOeIJ!K6w@C+K`jQ6OkGR8 z?lgZFBa3NJegi~PkmHLX2>1%3iE!I47$o}lrc4$X#3t*}<~NL0ztA1}WSLlE@%;gk zbQ_AicwjSU5H+Rpq7s_=2IMrxy_p4Jhb8>6A(~l{e0%D{`eDR6$*$38#P}mF8|@;C zk~XAN1JIE!25k>X%_F!d8{iEsxG3X7k{*Ki=_z(^Y7FNX@mDB#ke6llhi~rc>*^P_ zOgI(gvKy03gC4dl8K=>Yq03#m%cJfi>AClLNd5mb&lO-`As@%!@<|E&Usp&ll^?1W zH38OZ*PeCHr>?==S3Qgk&?6}?Q<7d-r-TcwuwM{p0zJrt`4E+573lftZY}h$!BosU zD(U8(6qK0vw@lfb4SO@A3~bwx($2P!FYPTj#QFS8hdwFP#mP0eu6?^2e?OQ^rbMrq ztQh7;Bx6%3)`9V?;<>jq9R5V%^!?Znb5`Alwq+%)*O~INcvUrA4{f;RlFumS9e`Ss z|9De9=|WX~Lp*RlzD%_S>GgN{#Uq4u9bMgUeNW!E9HqG-i(P}ZHgyBpPXtQ8{Un?y zc3YkPAb*wH>PE}5>6d*wDpJAn2VK8o0*L)Sohw26)foL(ApAQ@RPmvg}-&Yp6_|{5k0+?us8k`Jr?s zEj|7^GDOoP{-QqLT!rI{rm*m<;K|T8Z)l*|oo^7-3k^ph_C3R?cDTfPZ$?met6C2w z%@s=REhjcX>tfWHU22F>F8ii#G=dFEs$ANvmw0B{yk(U$?16w6_{_#C!U z-!iI=THH7*{a(1d%5=kY;(j5x=>kBC!AtHTIOU*E6@e|-^@3yO8PursGmp?JIOP(} zicoKIx~&4YB*kK%GiL(Mybe%S>G_?Ze(s39kJjmmkEdStJm*`sNmOt6Ci_M0Ltu&s zy?DU+T+`R#&at2BbVh?C+_*}UPLt#l_=Khq5$sFB58(IRnrwr@=GDV>_v*hnLx1lr z*mMUhu_!kXn(65{6gKplzRS$=_u&@q28FY0`Jbt=f>~p?L-Z~b`bbqqM@E_`NbkD< zx>292+X!r8RSvlX%qXKNF@1{!y#>7BY9%y!w2<7M(*s4<<~c5^Cj}G8CrZbdW!VPY zS5nkZcPPW0Gp?1vKN=}84*SMHc79#``RCc(+w-erM3nW9iVWLN?WgMKyFk5HX-tK6 z+y$R{EL$~R(oyZUJs}XH6JjO8#ZcY# zk>$VUIlo^4=%$tRKj6fr!~lbn=V%#a+mCGqrXHg+qnMwH*{;ajTHi0#%T>#H`qIF* z184;*!0%KtdPwQK@fwNGdWVQvtEqm?EJoQjL2E|m`<#=0Tvr8re1Ulj>HnF1XyQK4 z<92wVV`OYW=7-okM|9c%C|xMkXm7+{hG|`0emdU$xV1ifdjN^s_=jyI!1jrUurmXGK?D8*% z(LOg5S{ttF`}MW3)KO3c1-1v;*#FN}x`I)5{|1?k0w8r<dTN|Ri zzZxiOuTdVaQpUGhiA`eo`1%Uvm2-(h(h#Zehx6ec1UE6qmFs9x|E-St^JfQiZK(=O zNSkBW5dR(xf7g^&+&NUpJS-N6;jpW;3eV4-3dtt+3J3@VY`q>%%fd~S`){{=xHK;E zI(?d9daMgv#CTRQ3yf#ao-Ocen-)V6gkE?$>|M{vhJT3Ke&5Jb?|+1#|M6Rzhx}-Q zQ8kx-Ob^wvp-%okl)VR7&insAu3LA~ptMMIBcwDmY2IlMw3im`rBq5w-5Dh$QmHhx zr?z&P4eh-{d(+-ks!Hv&rFK7#GgBOYF(`T-$`9eMNSe0zaI8eqTBE6Q0ku_&Nd1$K35 zwGJ2JKBhiW-u@asXkMBe`eJK_uN(n5=xhRokfFqEk?TkQcG+-z6w?H^oj4PlH9nqA zw*s66iN>Of?TAZfJ>Y#5apOHJb-A}+e$77m%Dj~o>BiJHBQ0u`;}!&c#|9%qKsxU0h+RbN=fKRuY-8 zhZ}v(^u9=QtFIIQEj3Ckl=gJ%Gxr6dPAKir2|8uPUX(1j!VnQ8 z3iY6V+9wOQYydU1<#3wP!NlHh!8LDw?kG+y;#O?i1e zD+J-mW7*MmgL->|<$*)a)s#xOc3&MA{ApUl&r)(6t>5>@7jHc+vON+JK$u|NO`6f5 z$4rEZgU#-DoQi2YRBI0%|GGR|B$ zGpuvaiTVnP+kYT4w&T&Os<+mE^)Z!XCeU_#_x^qR<(ZioLjWiujcn9zAP;!%rO)Ag z4>%cU&TPV$sHX|$4+-3qx3RS~U(U*!(#OF;uc)lFdE{PdPqS+aGeIbVxQywjx|njR z`1^9<-1SENI!Aso4Ss*wpR3SEMM&8;RQ+mD@E#wk$2l6Dcq&%^4q3b@4(zpQXHLo( zSItz6~KgAdO|cT3p~ExK{w*a*M3W^i}Ru)w9l#4%cV)Dxrf`1NB+6y z&zSqFBXWSIdWwmOX<sGCoaz%|gDQGk72K_^@vn98ryB0aP8##wOr^b} zF@2PEBSC-_9o0G>{9fU93L{}t4i@@CcB4%dH#&~X21rRu-$$S=)NDHA=S_fyWn#_$ z%jVud)_;wtzQl`?P-3uFFN^%iA_?tQr29`B8zF;&lqm4^>$59u;CT^hCbF4SKvaOD zQuZ03^+g-kh9RfX>~aKfjiPW-s^?r5*)_yiNO~T*8||qEBV;-|(b)>n+LC$V&| z@W?&0y(2Uzaz^>?@$hCrAymZyj=ydQH~P!hud?W6=!-l&9o1|{8uxam;oj!bt3#B% zpSg5CeA$avjT)(Tm!np2(<sSzUF}$TWJSZG0pH4aB^Y0C40v5C1h9um#AQkIG;(Mfftk4#TYj%c88- z{kiZ6!z-&^p>`m9BuOId;G~Pcw!jeCf~C>ATrcW)GwQm;!!-46B!~aD1@I)n;#siT zsX4`K^Ow&#evd||4-|~d&L)6Q?!UtR#g_v1{0WrV3YKFu#7-f|LLT!LIXI#2+q?%C z80-{p#Y%PPkr*z~0F_dC=7~ww^S@dgGYMojp**S2dkZjIKQ{gd2f8Iyn;9`VyDrVk zhj`p2BOeBssF=8Tdr^CdTL}U~NX;a3L^&@^`h(I~0l~$2`Cm3?8q_(WgkDS8ov}eP zg^*@ttVpJwJ|_9MggyQgVp`69eaNX*&8C?p#U=l2Ul0mVJQ+}|Y3+ml4il6&Qg0}e z#mnMA8AJ-&N?O|Luk>(q{Pv%+ePj#>0k2;E;+(D4ZRlwv0POTbw$E1o7(+G+n6-P$ zR8el^Z1O=5BmCQWzg)rl=MC6?xcD=FEDWF`$xpBYp9O4FhU6ij z;RmNY@xaFKHnEqgzGm*fy1J@ZR^7tW^Y${9Q+H@^FrUB?WbN}tw?hj&VT8xynIuMk zOfw=%LxeeBS4^Md=3m{eCb^WH9HkYe8rlvF%*;uuHMP>`CG|e~9Eyn@In1uNqg_;V zXkR^rC*lHS(QIitMfu7R5nsuu-$IUl8#DpUTgw5-dK`lE-&Udzl$XTTto?`e-w`V_ zaGHqE-@hrfW6M4M&%rdi5CQw!Q0Qpex>O9!Y9#Q2BRNfV*73VR!l~d!V6cHv_{?u2 zG3bm}?Vb!aAw8D1)d4odq~)vrN1yiFj~+bmixlB!m(HSm1orOSi?dqflGOFsYB?o1i$ECWwoX4XlLE9;c8H7r);_v3yYk zql~umb}YISY6vWh44VO&A*O)0%6d>F$$+jz>o7=6Fap8@IZ9jmUqZM#&1O?F`~22X zvPjf`At8&^U$!jqmC>JMV&hzDcOwx&YH8L4ULVV zNr3NWKx<~39S0_j&D@1JItK!Q0v^TP#CX-Te9rS0t?H2PFfEV&#%HGjHfWlf&n?Lp zSPhNyRhsnIu_QmahX4CG@abF?A~G`GFwOeQ)gZu^0D`|`Or4Mok2GT$ut8#R&vU(v zdK*u>fE2<}c7t-V`;Q;z7C5giep6h?E_!koz`N&cVbzM%$_Iys^|gAl%+KHb;q#Ps<{gorBG5PMRIDI9o~ zRz4=eI~mxkSEG`aUjwX+GFu|JWIb1R3Vt!)Uw&}b|tRcjN4XpySXL!1>9(kG=#bF|FHDmoBkUVa7Xf+oq%sh8v2x+C&bYKn1vS z>c`u*chvCz?d0x2MKVS-KFBqxf76@|x)l;RR=ty&lJ84UfK%QyZ9BNiW#UF-F@0({ z&=I(d)P_gb!HgLIA_;kxQ}XKSj6lW=gbjAy@$K6;V4&~6?{nhC8;L7hVTKEH=_Rh> zB0D<)MRam-scd4k+_gAEUIKNyL1+tNWiG$)s7{IdI_``tU+BF3^Q*LW|pD^CoR0K$mzvgp;nwAutJ|qJ9d6`Pn zP{ON}W3A*jmqRnv49Ym3WM3BFLq~mzZ?GN5gt~(9rEKhdc=ZFt754r2+_48x->x5h zbkNJr@+8OWT&bBX4ts-bpz)va(V_%of6`DeVL?hcCVD>9q^DP*wl@HRQ54l365qm0 z0SfhrdzN4s5opHOSemkF___m?kdVOPvS=Ly$Gm6`nyv_-Ee|>%$_V<^mMP2a|neCT;a;j1M@J2y-vXvM(rapP$-r(6H0o*DDi#S9bZ#1ys zpO;U!8s`gj&K*drb=p2F|9FGT5zwIF&@0{1im^*HBHh=V>_X|b`-Q;VgUh5(I|~+? z*z69)KzF?%_QJuBwAAE~ph55#4PA*;5g1(~Dzvd`%iq@xakmsP(tw9xbz)Q>K}EvM z);|my@f~sZcmk3O!Kc3u>^nF2Hn*2;_u8iOohBO2)DHi)oP1KMDM|F-yTu#3O{Vog zsY2^eZFMy_?jF#nQ|FY=t6|fc0l0Mn$@*_7?i4K&DNQ{TkLgpw0c}zmV*jKCK*|RI zc<^~qpe{C!zSSbCCc|~bKI7Xtk@GH&fI@ugKd|nb3O!64P=Eojx3bql*yIs; zN8I!0`#ED=079(2D-AHiT*rf<@wmlqMSO2PqtsLXD`9>C;Aq(XvdDd zW9olt)o$!=N>aOo0IHUz-|iI$QoMNPx9Tv#%Mw0xrYTtF!2qi--+MQr=EK@6dWZ5Fu z`yy{z?XABqekswXZ$@7)7uctOLKe*%yO~IbN3s{!o~rJ=sg|lWGWvo*&9v?XK`lJB z1FpZ{z_!GEP}*1nVzkpbg>8A>17zfFuddui*tJo$>`B8kR9x{Oc^cco)Uy?v;$Bfq zySfi|q)eJe1&R9Y>l%yS?2Rp|VUZ_vpq|R?&3k0HWo@#Y1IuQ0@wyu+<3nd>fD-%N` zf}4i=OJ`B-`ZO)e%k7YSJJx_l$9%_6-T}QzhSt7Zt6o-mmuX!8&`?2(Qi3azzatf3 zXpiMxOvNjvWf&rzzA+(&#<5dL$;$d6^i=S)TF11CoU#!pYLrOxvwW)O!fwSTMry|O zeRY6dmw{L(G)ZH-a`Z;AbOA3){9{+EcpO9QW?kY=p!-7mr0cwAtH}YyG^Ktsy_g|j9;x5FduR}#c-A&%)D=Y(7%s3{u|iA>a6dO&!kdh z0d)ZcfH{B#V+VEz%RtO!)>LlL)ZAxkTtEI1fHhA*9)VviQ%e-JEbajX`3DQcXr#I8 zrN&vjtqXvYf-#$1Pg8mH#s9fRzen*d7%~cqV-Sb?14N_1DM{o30)G&B0L`oGmM`~# z6I*rhbG7>X=5ACY7r*RKP3QpV#Q8!p3_DGZ?R$m=SP(*JZ-7E~;74di4uGzJFM=O| z;*3^jJia7$<5nFT8G<3Vfk;B_B_K?A$_l{mg45VqHJlVuPv4iyq5q#{W|tFWR8+^2 z?loX?y;=FIfHW_6?=>7HKDA6bd>nY{1Dya{e`YoTz3gX*szB7sC9ibKJX9#6wX@?{G^FnbY*B2CN261OQ}mB%V+6P0R80A9oat>KwfK3DT`g|Jdc z_7GtkMRoPzjteE~5pHg?Q@tfkfh1b|irdEIjh4IHDbFRe(aql<@6TWZCY^_^Ughz14n~`f>JG&nNCCmsSj0$AXSc-J@P96S8bIG)iyiE zOooln!ok`H??42+s}Pi!D>vryPDvP7$_DwSAGDL44-!*RoOM}z<#^4|(9oX`!m|p2 z+}WjH`U^+3q5LIPvU5M;H2?W{TNV`4w-c}WDx%v_2?@Xeu$iBNtte>aD|$tsm!ebD zfq$9%T31`ohx6?yxZNBgW_&4yxh4c+)rGCOR=OEej@RRl$?wKN zwY9Y!fJQ$BDT5#kNMSf~q%1FJUq@@FRdRE%L;nUFc@rAn@Vj8lGWIqPFRyEb9-Pj`VRFV7ESi!&3 zk|hH8J1~VItHQ-3Wled33MPy0IS)p-or*{X*7KUo`!o3kikc^+tkV%kJ;!S!+uuI4t zLq=7+d+3k*$?e51BBBmWa_8t7%N2GW=B^rs0`7%*S<;+_zSG{%Eu10h^oI(gmyz}y zHR-F-`j?($T4Vl^K3Hpfs45^b6mZ9YW2^NDG@1~vDHotn-O~vcuytN4wZFgrdBi-n z-KSZ+go_>GZ_sCYWN_lc_6@_Q?9+XVr}2X|{@1fXR*9mWO=$mOXEvB;JLJu4%_uCa z3k_Xk4{xXWaZ?oi*RNki9i_Nic&k-kn>900oV_eBPc|(O+-?n_MpWY!;LAn~;sc=2 z5XkTiz@(TN2i-RJF$1J(1+hnv-bbkPNI_La{C4iK^kG}=ogIK%q_SQF3KU&{%&BmEZlE@M^6lDpz^Hw!$%?Id{9vt7#KK2d*RBJ$kR~SLJE5nzA=q^ z&E*vsC`k#LEfQ(zMqe~^oUPO&M>oxo5R(Bepe0cJv5^@A>Jsvj5w zU8GGsv)X`FAib9cyb$wn>tv+%1mVqN`_t}~7>ZdRd^wm!T#=Dd?`9B>6 z+4K`E1P(F4Y&b!td|Twwi?g3ciwJa~3@fLas%6UY{l;Y1CRbWPK_PYRO=ML2JkoH{ z0jdYdSq4%r1O)N|G%};w)q#^Nzk;G2ju#mVK5kd$friq{2gT63hy}2|4fs#YS`?}S z+5s$~xU%H>zW$|T^`csu4hvjwCfl>L>il-Y^09twN?xQ}X2bomyW2#b;IDJP`5&$v zu?D)+z4vGL+Z8g>&&?c8a2pP4o>Ls2AHZff*{Z`5 z%2b!lpj&7z4ElPVeIU&t=kCw@Akp?TL;^OT*d+y0h{sO&CEBh6Gu#`se3}+6FOb7( z@$<|hOs-!~Pf7|VzxnOU`$7M(uxnwLY1My+lRY)oKyBJl>J6?`bk$mmvdf z14Qz$Axiu~hW!=6ZF2~5D@tr<#4DoTPS5`ax1}X02>6l2vEDT3hSQBuBSB%|k|uX9 z{ZONW&1#@xxcl{JUF7ZIthrl^P;CJxlJmSTF(dYB&WA`516oQ%mU5FgNO&jAUC0yC zb~-pfLk*xEbX=ag4;rEMfkua_%gfnoSxF-vK7PEnNW{@8x;>P8zw+(#pnRyw)R}4P zP(7VoYd}hJK~~oG+SMNlg~)S*C?<%0lX-Xl%T@U8Wmb~V$av_mq?1^}hkSgVb zYty!kR{M+|Qb!snlU?oj_m+?A_3{&pZXGggJkmL0QuDa=cV~rIswOW4e;19^MN=;G zQbcFnT66MqREp!>qs)~VZsfjAh0r$S4q z+0*o`hT=bD)UeG|jju-1e&P`j-4fPHM0d$ZgSp}1yi zg?`9m+VD;wm#Zz~#BB-KpYY3YZT6Aow-%dO509YEr!Y5WnJQ82hqEVHpZ-HHU~^*< z*}zrIk(zE*jsAVNN_U}X*<%{djLGf{vUAcWMIqFIJUltt@MrpfAz zWuTt%Nj6*(Q9rDG3DrOUb6C=&-OxO9Jm=6*D(h;b0VmMC)mS-VC_qTk&f9s)R(5Rp zVP$P??#}kPr5M*Fi+YQMU>4+`%1djb04AVl8O8jpO>G5T4fFIoG4y7Kznzw#PI0ty8I}wO> zX^@9bTx6@1Mx5(WOxB5J z(}Q>$M8ljfp+!6x^RmwP8NtKRnJOpykr~;zhiM5>wI@S4dYtTIVq${y?5mw*ennX@ zYZ)ar_MuzNxY+QZC>&HJQtUa(h&7 z*N^=5UoWUU4QiR+ITY2-PnxlQxRWTqVF8f)l~5Jj^qp#uE+%!-3dDpb+|;GT8lqK3 z_)88tq$|MLl!14MhT}MBcqmb+j{cm@JFp(L9!0e*uCeDH7_Hg~WzJ?^sWwR#NeDjF z0ZR{F_H@}${B;Y*)AsP-ukPT)~_kVf7CNyy- z^7U;36PQ_`1ce2qvF-x!?E^3z&ktU&0_qJO_UplBye5Z8>Od*sD=@i*o*;E$NIIzW z_Ns0Jh5u*&K$U@Zzo8*5OBxLznQSbJff%j5DbxjeQAoj;B%C(B2YA&Sm&RPL!8DEH zPp<91;x4idRDEQ(pwrMv9Q1wxLhuwq&zyKc8B@ z=(*))|I>AChtaR_%Q0rs)3-EnaJ4F73KpGp38bDCH8lZ9hS9sR?!ZY}Cd*dDu(qz?9e|)gyA-eA;*evnUwv<2B6O78Bb+YA@HVY4i&7ttU z=fKfc9fqeZ@w4m6zmEOZz5kbIK^7A86#w~Ep{J?#ERSx0w2%#?MZuYXQB|;?=unY0 z9^`^F)CU!k^FUxG&HItXOoD5;i}<$8+rM6mUz?2lB$lz`fY`C6(bm4U8+=Yl2<8s- zZ%h-B^rZxdltxIb88AQ5dAT63C;aA@-O%BGMRJI*X%A2HAsEnLsR4qQMLL1POZ^l` zX%C7(GT}n51|aD99W(&intMi=kp2&XK8FUfxqWw`VXAs;6Ax4Q@U_|L@y^^!2(4KIV5CsJ)`4zb3`D3= zB9%}{5wxsAGz$hAFA4i2l3-=Gcf~-@esB>H1O!P>nL|^BdtfSB$?)smn_o$Tc&)-L zIrK+K-orj^rot^4wln%eTx6+(p@UyoL5r)?@t|dcuXUGDP{5c_@dMRrm|Q5WTfZer!eVH z25^I>IhQvY-7m6q)WE{A(7KGChEW~`+xLD4XtrRx1hJ**g4tnkSq4)JgT$s&v<{F6+;&f$$%Rru#5d!1X+C z?xnM?6->o@i{NavT<4mR8CbGrW+Q}UNL3O>E1dvTQD(kH4-Jc^K@Yw2_!&eKOSQ7z z@eop)tH(Bw9^`4WJgt!-25fIZM<`iesI!4qSJ9)piMtEurYg~%n#QNj&PgggF8Zoz zPP(m`^oK+Jw;dA;z}gv*2G+OSb2n<&0+7wQW*EiD0R6z>xMoZ#{o1f3rrLYuJaq#aFB zom@N3AUS~vFT!VVA+)cTLG7Uu5&h>dGqWCpS*}Nd6|ytLIrTvrf#5P^lMlYbY?!zB zOlI`J$}`CDGMjI%e3s~RSooG!YW^q_1syF&a{#(LyrMjzERwdc0IJ|bhToIrbZ_Wd zx`CJ*Fq)l6x!6^2KLvTl3;fQ>1iKMoXDV#+N3(E5;&PX#S%6Zv8fUo>4E{Ocu5ug4 zeJq(^EI@@zDW1U743lQ`N;}6Q2(;4_uIlSoucb`{KuUY#>=-lVm5{8#*Bqj8^$)Ck z+-vv7Ie0Su3SmMF`{KrU_pTSp&Fvq*dG^G3nG82P@MxgKN8=c&DfMZ{sm7AZ_Pujw z;bEX==V{WLLi^(%Zx9y=mYEVKql{*}z{j@X*LB^<9{!0N|1>`3B2?97Un4l=ybz{6 z@cG>De%E&ZWk6Q_ebc06oN5l2m+J^WD5yMPn0(tZ$rkBQ_-{gH1LDa5iV@|)&%qcc&PHc(06#lAPSuX;0 z#VUW~ghMr)V-sZjrd&7?k8*txqI8=(v34C0CK`y8U|lqdN*90Zc1#-W_UvDQ*GshI^2~#M%+5$+94;_u> z%s=$*VfnW!@~?mAYl_N~tTF8H8n$Y{M>?M#j754fQ%ZQ(i3pVK<=D<64-T#Cw`aDd`fUN<|$R$A;nZeDOhVL=KD; zB_}#oU=lGB8-4O=u7(UM)C&Gka&q!~x&E+O30L;kbFae$!rQ?si|n~T72N@8krnTe z@Ji}@fR*s7vRs*>ryK3YmKE8kB5d>Os`$i9Ec9uJAPoV&XZ~%^^|}RL4>y%((#m+T zkuuP=MR|x8nN#jaewf7+_SnIvwhcev;}gNAg2u!fJ=Ed9pS1t<1C*zMu|l0Z6xvkJ zR23C?M5)S_-oEXMbU}EAACnZP@;BXmIk8r!Qe$7y3wS-`oJ{#Yt4hW4BK_s*wnqq%qgtysso~mlZ#)lk$y9$ zsIL{_@P7M9PoI+jwe}7ugibEM)+qyZ_6zj4Oke_#@8x_gyhk9bniGC&xuUtY>giavSu>HdmDPB< z4?kO&rbjn3aJ#Ildeg$_GVMz5J-3yz zp+e*6A%5ri@xI3KzXE3e<(^2>0DisUe&!HCCC3O$%;X~(91dbD9+orfkbdM3ybff! zYy=hOX%>e`?U3?X`6_Sc3FoS0=aq^@(A6?(Gvc-C8K;99x8@ z`V;Tg_0i^}2|!+`zP8pAD5AlqO(fko!|ZGdr4Lc<59wO?m4V!yug(QNusP@2|Ji8g za=(qj^Tx6uG}ZKkE-g1vCF7{u_d0v)wTQ;NNkw6`C@-2mnkOe){T9Vfc=L#w_p6@$ zPtMxUWy0E^QOClbI?KpWDo9bh_6GcofAlOA?9nVg&3`Q zMV6C5xl#iuWM42Fir=B}WLFimF%!O#1!^2Ew-!I|;s;G@1@i+sg%OG@oN43fpDUq^ z{x;Nm>E``?$CK?w8d7^;$B(rv?5%0Gl~0h5^jJ~8*9;%@dcxX`u})2Vfq2iJ=5Me? zqsI)sjgA^bV;g+^&ka+=uQ)$Un^I3cm`y%VjrO{o=92z9r z2eFS@_xBN6{{ZFIhS9S3!Ue|Bm^$zk+^@|f#(P0gVUVTh7UE(lQk&p_g9t%m49JjDFH9#RqYfcaibo&9I10twpnvU z`DMHh>_0*EktDV9)-x0ft)%0XJAh%M_{4v zD&TbMeMsyF&W4}b1!CP-bV{do(Yui4bVj1?zSW_7zniVx7WgQ!<1wGT7f##t5I4+@ zj}(6O+$LYB@h+b47T#1{;*T*L1@b7}B!|z?(YlBLjiR#|UM(rtR7;Fl zc|ULIqx{Ro{D(Q$V1i;?fKb~iZHqXKwgIAtRImp0bqi$bI9APvZrcQ%(iDmkL3uR*PSFeNi8a&2amwsk07t+rg8T_nHOF-nh|uA($&2&S}Af z$ZdjlNdDH6mU58fZzdofNm^V!uJJyjvIj20x@XrsG=0WF)a!aD=sQ$h58Ii(TdS)& z*{-5(ba+(J^J=|H;5PdE|5^yF2%Ocd2%#kgpB0E0vTX(`^Y~5Lm(J*aSe%R<$cN)- zkh=kiV>_bo0shK*g;_P3-SChK_R8lsOEW2ujXJglLa)6Y@^nj(nJ8XgA|Su=6@Xz5 z#xQnNA3%4pY(Pjh%UeP!cy1Eag#=L&!{=o#0iVs4RW{M%l)u6(q=~#N6ELLvyppdW zbc<0W`|9&xux(Wh49;$C0@1i}Pz2-NaMuznK)fC#l2@I=mJiQ*bA&Nfr3mp_wDlF! zfNdQNwRd&x!|jsOWdg0HEO_c(haNXkTtr-`eK6!6H4f^%Z)r18m#8dUK-zAlCjAL7 z?if?>+>(-oNYwGYXipvI;dGURXSvRbTM}0P$9;htZ$VZb&RA&St50$Tov4;}ZLjd^ ztEp9ZDJ#Hg%}yYa|3H{OlbdD>4(oMus&*$R;Etgr{QVnINMW13WZ>`!t+I$V*o1~yU=1t9F218L(5-x%)pq6X)OOhN!RI^<{7iu%F z$|@m6<6_~|O;TksZ%*_ddmQizMk=Nf2Ok1r-4dJWRCjo78#+sQ)v_omNo}Yud(A0i z#P9#d(WHi>i4k67@L2H$nR)|s1DB{0&h*3SNtg@{sF{tJwS9ZO_1`3}RwtnnUbK3xNihU-IA zQVCJx{xAg3t}|#^5}cwXR0i##v6cTR)bo)2^Mf0RqyuEd-whu&FBX!deDhHe3CbVR zcIY+)J*q4?Rrym1(ACb5h26G*0>zx`UV`3By~aq6OL6gR8QPr{09ehfProe-fy!@0 z)jiEvi?ULpge&L9bHN_Ja!Aua33k_@jVwP-tT=M z%&cV6IcT)ukgm}yI_}SmuUqlEYSBN;+duw@z7$ax856Yk1Lt@jiC5*jz{&;(>%Q)= zar)P_I!?rhw2I;NWQCXas`{U^7B!ov18kdFCGi;&ie8icuD%?*oTQd!FWbc2rh@nK zK9SGE82SHNp9IAM&G88uaJjQD4kG8->nJZtpyhe=wAXgePI#Ql3?%9LTv&5nFbORAp;gu zGV~Ik1a~4sSq~b@c*m4-Ht}&}GVV)9pAmihuH@dE_FXC5`Iv7H>h1{#lHtCxJrq0V z{fgdh?3Kx}E;9}9Eg`DOyx1ZY^caB11)uSs=f0`17HCH;t*z*0e3V@Iu~hw@*!tDi zrxwIYDdLqAo}TK|&!=LsOnVsa$dZ_LI&itNCF)a}+j3*L7Ix5IO~(|P0>tN>l(X66#{}V|Ip!v7N$kIS_HYD4swrWx zzdt_muD%Qqr1;^>W)f23BSAb`^3>$?^%gaVQ0lc9f z64#2JO2Jqb&HF*CVXch;9O7 zJ`LcKIMm=U+lC(!{3|;LiwQ#A^=m(C`Hxhj{{mJ+aL#}HY2pflGhNnWtO&JKv-69@ z70Q0&il2P-e=B%PMZ?D2|0mHOgF_{hj|X`>Gx*+#MdnOlnR*74yN-RDKKsuxNWUL9 zM+8~6*F^cx8oS~V-*F4qifh7D5?wp-@Yk7r#~04EPt4H$N81AP4ccTUdVV1YHd`CG zJG&V;^>sNqFYez#3PUZLP!%q5e(m;6JNwW7N5<(T4VUQo>(_shCZtlZ85#4(le(Km zt$9JVi$`A;-9&G{mdU<=<9?NGoF&Qb55Z7>;p}Pi zZ2IpHi2VZa$P0z1SpIA*?xvA4ASe;b@t?*uU{HkbSbWPJ1fE#F7i9l27xLfE1cWiq z!7Mzw|KPzNTZCIBmr|xK5zF=yChDTX!;d1DYH|*Xw~rd(Z2kRFem@fM_J4UYX>uvC z6D49*_v6+Qw$Be;h+!E0hjT2>@mW687q> z38=*GLlGJP7rK~J*Pu!NW#-RK@I8V8xe<5bHQsz=Cu5XHeJh;#*2E%TPD1_jnTevY zOm1p214TsrZw=}QDamlkHOCVQ)$kl$NH$dgkxbD`7LL;E#rQv8C0ez#ifI3s_KYT5 z%ZfGNQ=%VZb$V9{TV%tZG*WDxa!ej7VcGSR%7Sz%b;>b!EFWRCnUHj|YeU&Hq3x{* zItrOhtiY+|KP~m@B@A28{_l#rtntapU2uhkD~<_%ONtM-vsbSR7^)J^m@kLZ9|b3o zV=;bc#WQiyUk(SYk_>P2YkcY9h7Nl2E&Re`A!q3enHTp#(d5>T_1uX^-Lb*D-z~3y zbaD+Q4g96veu_eBe(C}k1Vi~QLZ|~tVA^M7GCD6El;Yct2@RgZ)i^L;I?VdV-N4|~ zIXK?gu;hc?cSWUt!b{f*aaDW>pYp>P!=ne4@E^~dXp^XDhTpk>s(5kvX}dj-;LZ=* zpB=3U3$OgZR(NYQH{TqChpIO+^qSAdx4VA=Y~BTQROqg#BGwC@(z#of9`}rxVQeAJ zAUnhca}AK^rYC)qmyT;p#2&&N+NK+D$YZ|l6qBXgtK6F}XdS*O1CRSeLm1Y1duaU= z{gGoo?i|JnRc~PAH5gpj9jXbBDvhfDgs#`}lh%(-+p-y0CNL!Y${ZbXchL)QEkBV- zeW3Gu$ipe>r?Y(`N7#J7!EBqo#9})6eE<1?7kz4o&fR{dWnu#tcaxVLxbkBsgD1MEZa)0^ zid1zkQuDe?@N3uBhuVH%-XkJ*i&fT^WhRx_$Cre%#Xsh^|Sq*qr| zEH*pw(FN{<*3|>Sjoi7;;^)!rtl#RZqi#QlI@<==hK=3qy3;A!C1eIkOw*J` zde8k0p9d-r$Ka+N#v4qzIUB!SLo@KfelRAZy75@4nzpHu&d$jjv8}$mc-3>k+v7&a z7|C9w9JwhxZ8i@GFbpw%?cTB#bo;&ZZEx|7@ZE~DO5xns5&j#n$a+NY_@~}oF40Tm z+!k-d%%L>MF`Cg{?Q=BGZQ0aQ8PqNZ3b%eCcko=|TFz6nwpgtxyl+8;jDQ zz>F>(R#~e1{x-BoQ0RA;hebh(oP*RfASqgfSj{KafF~UqHD^aJOdZCxR z5hRn!?QrUYQR=DHb~MY|5x)VFD6~&g9{7FU=n?#9=j@=wb)?1`EIo%db}Famn;6yL z12O4_J*bhBzQapL*CeDK|5z+3VVp_kn7Fg{@+05F5RB#Q7UQRfKjtj4AKt$8erWZp zLVn?gU4M(TF%L@tx!axbeLb$~*(rSp+qu^scLDxkCHyK(ifB;Rg9dP{VNWb@7@z<) zJ!c2|&;lJ|*!8DV>N-5n*}g|f9Kvr<0*5B<Z+$9%_^j=D<7T>7~jZn&o?{}FOa zd1I|xiu%@~yPipaiNKx3_ZUj(TRR&Z%k=9Rg$Utv(AVW1-~Ov#FYxOpY#G^fZQ)Rf zHoHVEBWyJVdUc3tV?vIO*mawNMwE}O_t;aq_`8|gjDP<8msEhe+q|G90$PQ21oO0T zQfy=EO>o1IYdq@lX8x}a!M(IzajEr`G?7+Vp8y3cY)OQ~q z+GPy!74raS>Npkm;){p4lFbZ5Pd~KEcnm8kLx9u=;yEimL}$)vIol`FZj-sNDJ z{K;+J^6BE)&-l#B@$!iUZ`OW(A4VF|^d{E-1LnDU39BIrfTau8jJ@QLCg7pHM?+f+ z!YjnfBB9cn0n`q1y5NevPm7O{-F5F-fC0NT0FBz1E|&cK{`ik;WZ(@JGw6Ht@X7x; z8wSy(ARK22psfS#4t$SxpuX)M?rx>?O%M+fh0o>Fs5|+aY53=&meb-uO0@Ese-*Q4&fM#-KAG@smtk8qJ~ZF{Oo1bzu4@>B z;g{(^V`+oIbgu0Svy_?Bo-(exW) z;6ORydm6o?i1s7wXO+*f&@hL=zoBmHOR5s5kG~FL$EmGTu(TEnEHqzS8B`b`JP&3nIO*4SK2AD$Rf(FnH}?@KTvWtYiKA zgmyhX)%Zcw4NOM{cA6iN`7Ni~?&uysL@Ol(PVoH}3;fSI2Qh-6;kx@Zz{2q9g(If= zBA9Ad5CA#ATetZIUkStju!@Ot&+u9f>^oPwLD%{7A$`w5N@-M+85Q?$PyJtihgF9E zQ{w$Q@&Edz|9MvUW$Yk;g@e!4dWUVNQ~$3Yp|JbiA&e$8(kZ}QJsAC<^+Wa?8%v14 zVL`m!9(d|&#Q!sN1TBeS&*$9xl@-h%HU)an1ml@M>crSs;-8;4^7ZA$|LeoGpCof| zHwoe}L@D!FwCSxZOzl1UVb70K7{nA2Zo1q)sO#Z2zZ91dlGW>TEZV=A%iy@6 zK0Py$`9ne#n?pEk9CmB|C+uG;x)d-J$F0vIIVFNYUnmB zuE0re0X=m5tl+K_h>SHA#7<7^y{0jg2;#bSM2$7&xN2fol zK)>H#>|y+U`!JgE@<)=8z6?XKr;z*)>x|2yksp*_OKz+c_(i~guVIkHwlDiDXy`5< z5_lqxi-cahtbt$FUuP@2V>OB26}>CugMN)_L zgNIlk3ZomKmQo5gZYdXl|YYj__GQ{bgvZp_(sS$YYX7K&4aW^+nJAa z`d~&g!FOvyo9ft67Z@fo4{)^2FaxC7H+=#ytdb#p;xTJt4B;^JM)Dz|@c7nv@1%Qn0=HF);57#UgLD7N&y>V?0ApA8!Evw5phUch1 zfUw^{Gf_Wr;p=1Je1|2(N5WHeF{va{CM3RZ9O}CVz{9Hm8aVSP23nW)otC*8c{T$U zpqVlGI+Y%9@GwZTvBznQWd-DLETF%x3hh?j8gNMi!KT73j^?n-^U!QBNY#P1A(EEp z=7QY5b_fJ1p>_?@_CTY1JpqE*O_0BgY8l*w)87g6puQx%d;~tV9a=oyt8r`4$83YJ zC)*V8;6rQ}04?}@@1O+`vhnmo&gkFk zq(`tj>jDd1wmWN_`4QCq8Vn1Xv+s^Wou}I00z%WB#jXW{05rA-$RY)XHsaky9^yif zcV|6o4KWJdTV?i0j|8gZguZ$h@w@9qtB(gC(q;wF`uEVrl|`6iq7KVq3HmH8z>x1f zXC;aYK!($f=M9C;?8&PIrhI-aAUT@4Ga&@P8!8IDxjdfz*_EkW3-1epbuNoU6bS%e zv;>Bu9R?zgvkDHquhS;ZaSs0JtX`TH~LzU-oGW5>ui3oh;2Z0Hn1r0o>ux9SV{1{|uzI zjc$e_W~kT3V1WGq}U|8@|F$g(pnlir}tE9ulr6KzExt^0JU-L!NS9WPHiv| z?uou{W$KV&rBX{>ppH`oNa_!Jh$g8d86ks2pf5HQ{}Dn64Q$zMFn>H~&rw17S6Mv| z5P%P$DaY=u!^|{j)vlfASnb_d3n$*A;y$4NSkMSM?qPEv=Y-$^K-MfN)V{+s2f&{0 z`MdTW&H|i5Hw|6l=|BWA?z%J(I9UvDp9Njf4xk1|uE1T}Icj#YM`(q2cut`#5`w`j zfR((XWIT|_*!apr6c8dtG$e`1aR}A{4&o<-LmuqM#j~+*s&JrJHQut?oJQO3N!6kg zAT~lc9ze^ms0Ad69hN|g%vD3`28^HvK@{x|h7(nqE6aS#!!K`m{b6{287cS)t4aiE z6*E2a?d{?%YzWSRTzkTA}1@r>@Is??OMs2JwI^Slh5BJ znxthxyN<*Jc0g@r99}S&x7Vf7kf^BwfpLZ^QNM!_K5y*y?ZPgRbu6X|3K=tip}FzW z42Bz4!N3d~!`!rm^JU?hZ?rV*H@-b~YXc;LsU+QXF?WvugAxe%wdA85zJL$}d$vQf zids;Bv|8Q31>WxdPbQy@-V)^Kyzn*5Y zav})Pjo#}Uko~>^zK>8KZOskX+^EtsfcCX2+=Q8hIaN+v15HCi&e@t%NnsplCQ)QG zsw=>K9h+U@o)u{eku#E^H1BDNJZ3^HiDOyh`fW8;c`Ee1iy=JK8_zl?ft`f4j(5Oi zm_p{+GJ)W*`7MYTCjfrkDGcI<2;}#{9{zFmTmh%Z^LOQL_@8NFhUMz$04oFs-kB&F zG<5CW^~pDz0&*Na%O?Rqgzp1yGzk-5FPv#{yIi|=iNy-A#)-7Sjn9Hq41udR56$HQ z%k}MR{68xa5rJf=^5Ftsf6q|Ck*jB!JRv@#3x8 z?||$Uj*dw0fJ~$N1M(bnRjN278zj9jb*JZ%ym-|o=I1t$Kd>@eO>@KGE+V1+dH<9Q zm^yt8DGe5zBRK7V$E`T95Q{#$tW56DBJ+pd0zqgx^rBce_hED#6KA4_diFzCV6nO^ z5axakM{_R*15~R(v{E(sQ#8}Jn#-wAwAST~R1CFARaS>GAx^!Qe@FlKE&bD7Q|`xY zq^hXu-K)a>Q7XWFBA>3s4x=C49dm_MyMSj1(hT$3U(jAH2}&OwPB%%wt5|}Uj=XrC z1eZ*)2=}o2xmj!Kq3afgmCtrN7lY2#m0HI+s4$Eb7{ReFhSdF5Cm=qDI`{TWqgd9) z7@}%5b9$;?EW@1BAP#>CC9d2%%sTAjz!9E59tqVdJe2V2kjYk~JDFnUp?* zS&$2wDU7H+_E?~y;B>z+*lE$jLl{&$BfD`89d9xqa)%smYE}vae4!r)ncKzqL?-4q zwSD!*XiwEd<%(wmb`S#C5BRGEdz8xtoF3^u(H{rM-O=ccu5<+AjiIeAj42jZ;_|UR z-cj0WJ$LRG+w{-f0E5Q^45i=;m;BOy_vKjC-B7$JdnBnbZ9m?21(uzdUxo26%Tq7a4g+mxrq*WU(BAdg<%F+2 z_5!Y|@?*##N8F1*Et~sQvyK34Z3AlILf_?GF_pULe)|<4Pn5SMcYAF@CTDGXl};JxoGm;g0d{)m0r!+o!5{& zMMz1B$KsrNyweSIalAysxXlBIKM8!WQ{hFAPrO(JS4t z0s*0G>+z`Cb=eBxmGMkqBX|a7Pyn*J(~x$)`?QHZAZ|q@Z}o=49MjGOjm%h;RIPT) zkNj3^J$yeJTmQZ{WNEN1--r~q`#qa)DfQHygA@ircVr6z)fVD8=W_m2#C>3ayYh%x zUza*u04>FCn;E_ZfB-G~I3Fb1)5r~Z;q;zw_pvR+S^%7#ELw!CET_RO>JMrZoj|u4 zzKEn_O~=n1QUd)8&+1n|8Dw?4L) zGxEemUIX$-d^MzVc;K)f2rl(0rMfRK;)XhZP8Yz@=e#H zql7evyqpK;dDyA^qQEz?PS8O!m0pnXXVQQiwfpIJ&=GZL7xkZ;fvBCU$oW2_lS8=k z+(&!(K$_ThB>XU z0iCwotJE1xpY_QFJZUmiuUkNMw~^yJ45zz+mxl01&t&Vi7iO!Qcy zEnxi7Tl7_e-m?c>RqIA!KIhQ2Y=eq@yn*<=YO?j_6)RA+ZG%#a1+Yt=`r}mjlA|;~ zECWxNe5E5woOlKD6HCYmS-mb)y<@(4p?wWT8&apIUevk%o@$c0=`NC-@Q>4TQH}EX z903#vQG78_g1cG)j@{S8jJ`4&s>p6&UH|{J_uhe6_V4?6?kG{BNRcS1?2IJymZ*@u zXJv1)x4WrfRJL%--kWTdQV7{wC3|J(w!Y_8(Rk;1zdwKcet$jBUmm&d>$+aoYnroUaH!S6BzK)4bwZvdkI{r%bP0pX=Hbev-dW zyI=2vt`p%!J0nl_eJ51DLxLC-Gq&cby7OvYr$siDMhM(_30^*zB#mi#t;$&~yCFU} z*lP7(;6os86`iEg!-MANDHu*WEFPk7&9YP$)ucOwD*`)267I} z4mWD?Lx@+Tl&XHJm8ampUrZE4J~fWgvwMC!hrQIW-~v6rx-?l-UJvGeoluNDSbCj$ z1SH~uv-P6S@3jBI2>-WFBws#12R>UVQ!eiBG2|9S2!z{`J=bRu` z#Lv3ey&wwkFC%(VFDyEMKs?iwH^o9wWFABXN*lFA61pe@c^b^Hu7F?T$ zp3$$|`gz|Xk4bj{3tJ+O?NB{7dTjH<%o82AgS<9}q*3SBnYMY-^2;!QUYqe-F|Ah&}zR;e!TU^wBRjf6%{w8Cnn41^trMy14Gw`TEP9{q~x_J_Yb2+F-`4zkj~0W+ry z6LP*uK_-MVkLkwHnZYN0D8$M!BAR4FTY*tT5?dzYiQjPOO^1qOM-DX0Bha{=Z2Rt1 zYa4}%<`t;KJxO4Ukn3O~hPzcHP*;g$6L8bJ)YjT`eRJ8g3fsUP2+;{-L$ya6%$hOq zwM6)Y+bJ(a*0&fs3KR=RB$58aFI*9*eUSLXs&a$a3t?PWnJE3E%4B_i+e44A%|+1o z)6LsBWMP>J!oW;u4O`)`guKxFck7fh9q4kw3Sqb1#4ju@`Vqmbuo&4M+jb zpwx{Ajtee=S+*t0lF_Ms1NR*X5`EFmL`QW?FVsD3U`Cz?v%C-LFQa!I;Hay-wVuMd(98dV;&c>vBC)@dHFa}kvaT=oE&62jvYG2Efxjj;{~%lQ`2hj@-& zZFhoNVsnU0(^8SVv2<~O8YWtu8PYvI_5r`$$mc*2#6!!}9_XKWsKe~Mxnr*x6q?T8 z5ng_dq$5TBff`4IC|Mt}gnA%FH2YAD;m(Q4Q|l2s+`i9^*wB36>M zGPztbJ)s9lGm!qX#J00O+dW1GdtjQIB3@Gfo(g}M1p#7LDQ*Ffg-&?vN&d;Xc781S z90h@fNyydo6CdW-Jr?hS$%#$R;RV=6q!t#TZ(*U3iQ2~x*FKcvA;ow|!i*OWkc%FY ztz&}lj$dt)%NT^PuMZdJvHxj*fSqtUVybNq3|7Hq*_9jb!Rok8aQkGj&>FBt?chy2 zf=YxGy3eD^+_VOCkvqSx5%Wdm1<=Tt3jv38cx&Y+r?|p9sB098%aG`rwyHqGg7FDA zz|adO6VEM2pV8?v1F@N>6Pw3C!AW2GPgf=f(ZC-AMF6=5up+1xVN;iZXL&UO&6vxg>=|%H2}^>bG{xh z$)$;?wK(PSA@JMk=FmT~0u3wm&ePcWLk8%8?8WV$UfmxDubOB^E4j$kSV)aZKh%SU zs~xt;QO+c$>}~-29Kcj9AB3oFWfC==M5Rh!k~w1{9D1A(dQO2yiKfv*c&H6#;yA7~ z+fXLR$p%iyc_>ru0Pb--E0n2i2DAMT;j;y(uD*EHH|Ub+0vAm6Hn`TxK$DNts$PlcV*Y3*O+2KF7ncyO>y4@(Z*FC|96cLl$Cv zHPbEP!4VUSgZ1l2gL1C4(fGs~`jAuSTy(`LLcKin(}DXn%GiZ(uQ!oY)@lg&7Sq#R zRyQW?)qRTLaA^T0PSA7t5H2LA>~&(5O#e_1F+)Lr-P~^Kl53FO7mQ&M2P=mY9~2Ip z4-iujYv%;hnF!Ar!*IJmVy`CoAs__pFflzPOS8ke>bP!V(&k*?D6gSc5b%=nAnpY* z`!kmC9s>1<6-Y`PBwsNrJP9;unLFd_%tUJ;*{R!;b z7RD|FrO*^KD9+FpN?)Dg7thfRqiSptBL|;4C4}Ce-MzQ?spqPmKNx(3a%yg;Tsf!# zzF@}EEyCz8fdQ?;OoXeITc6wg%;6rflJ*KqO4&~u{f1OdvtY|A-?cDEjjhS`6bG~b z+~`SL^1tigeLZx!8W8D^*TcBefVJhLgHK4~Twfb3~#%3 z!@s|UbLjbdCHy0VLt4uFXi7S#*u;iCs0WBW5KxZrc*em5kk%-+ z9Zpgqf+e2v?0%e0x4^(Q1L3$7)cJat)t$9s0n}H{x6qojDvy7ynWgvEQ#W(MAIL%f zcihj6jP28(SiRWMV^AJXshBhvby+#RCY-<3JZVy0SxEDJ)(jH>J7J7_6JeXNg@Y%J zW`#cJn^A+T$54B+5u>5LKrEdf6O3w8osDxIq9kD4-@P^pIIcBb@ZRIn-1;=i+YH}H zu0F>l;!3|vM0>jGXGcJ%sHN@rq0yjq{(iKe-H9_QT-}B7CN@OzK-}yhAfIgpJlzyr zl34+zU3v8uQkd=r=ZnRxmp~_pBABK8gd@I%$Or+`J8Lj`v0^-!eNWT{YQoRyi1vS( zhQZ+*U~%E5=Yp>{sIT_h*;FMY?ir?~1NUfQGZ!$*xi)AkLm)F!rc^>F5iTqK&Wb_P zGj?r`$VwAD!QO#2z{)~;ppptUnzM@0>HY*Y=<}!RXk2AU?Q)&e+0s9k(`_G*W zk!nLSbd8K~MaR8B@zZanA$oym5Qy$6JZNX&9nk}leHK&=Wu}q!M*m8XVj!S7jS8J) zjy@bR6Ae}&e0B}Qs9nu**IkOAnmr~WRXQ23vRvmg|%O3gSPed1kg zCJq30<-y+lL4j9o5NxFnQSe9U5Y(BHPXv!}hHg#r zGd+(6d+d7R*QN)1INX?`!>?aMf^rAd=Z^rJMr^|h*v-NAs7rqsF@*AH*33s-1d7D* z5N!4lJwlkXQ384>ARmi6uAy%uI4(_~&* zv$6_9TRV+h6QzpIMc;SCSnq|9Dy?JcS4+V?lWel?BL0gVCtgNhkIyY$hQ=)~wf0$c zt!e2+5My@~J$^f**(UPJ`fSJp=%8?Ux4;+ArGjXNNzlNUKzegOS_x`9f6Rj^Vsf9d z9qNApqkh5Z-jL(z^UhR-S(Xl)-(GQr15ZliDH9E!Eg_f$UwJ{nnxFi%JQ*;WqH$fH zGp{9p=FU5QDfF|D=n97$ZdUdVBI=rW#+eV^q8`;`2a_H{{%wWm-F%?<>Kx0$L;a1b)z>fNI3=;vBam_4;x_Yr+D0{p$QbnomQaCE`H< z(gy4BQjm3wLB_J1e~W(aEJ`!SwvhAbEoQ_HN@HH|>+(8{d8aeibdX;w$4Jf;Tv|G5 z1spSO!J1Ix1-eIHhh}XTJ6A#${%s~il!(|9IWX~}6{{0n7#t6n=vN3p2G%|7vD`3W(>E@oXXb&_Di8v`oh)eU?)EL) ziEowzZ9V=v(LuH|YJhV0*~yF`4wcqzhDYYp8jsIt+Sk~u)^R`VtlEczM5S?dM!p}s zOS1q$XPDmiQ^Z=q0@A}8J;i^Vd7LU(&?{T)(DX(>4^XdK3N=l$J}?^yeSEJUM%*wF zdmar=?dtkuO)wOU+xhV0J(P$H|2VgTfmC{mYUT@aze1=!Q9utMWNG-dNQC^8tY-E~ zhJ7gxdk)fFd67_}eLIzWgjXFMRNIdQ>}dAbu?P)N)Isw;4%2T|X%`79;D!oxM575Z zZ}MojCAiKgE2YNWoDUwkjzL9J2|oEd3$vci1MTaet&2cVWAhY4UmQ@{ihVLGaFw*u+s<9~O z5A+-qhCZo9vkj7$LKYRlF?zK6EOq@3d!piUKv1rLW3B^9)n^wFf)+Vhg53<;=orY? zbP@TUOFo5F%G0qZB4uC2;6YB82!)mb$jSs;(+_YNoWt>WIi`U;sdUi;&^&us&)aIE z$e3+5xRwjo(eIDuTztH~3jTIzK>0Ka_JbIKJ5zQ%>SVx_l+MR0{U9v^z*KiQbkIOk z=P2XFe%Kd70^PveCmk~5Isyx(dU?!^p6PyVQADjR@&xS8?d)>#OFXm)j*avXZP~?E zRTCHc+Ab1sxel^a zmFTSE`)h+$t4jOX_oLF-75H=Y{Wm07@#SM-rI)XOYPS;_#dwj3S|Lvw64D60n>#|GJal^LjRuY z(s=$i+zS&Jl{V$WI1>?gsdajes6yUAMt@FmKuCXF0c)6W4G{2WXew5PwEmYRg+y%) z)6jqAM~;@Kx;re%Smu+ACux6 zn$}aRlJAjTENmBmH>hO`OQ0N!{vgYGIE7A3KU6{I`hBV9?fSk7eo45~#tlamjw$FL zMWijpj7zpGL^qqCA)=6E)y${hgxO4RhVf;}J%QIgG^SNw49@aRzPqP@thK%CI|zPr<>pwlTAXPcdKfEHCO1< zvMlv1y}KHpNP_spuRc6hsjQBr4)XuU1#yNgniYg`mk|jOYlnR$V{e{j zQyFL?pgDE3CTUU@xwyjT{shXHu=oDOLJDhn0cR9@ zWtfPTo4MqzED9-0sH?0-DN2Z)*JkARoT7c@JPB$l0>!tVcY5Kp6^~ukf4+jiRUfi@ zbl=H@o<2e`P;i~2P7HVG#EHRKt`V>&pWdZ1d3!bL9FwH`+G30K7ay`JS1? zM*~9twDX`ti4egX>%%ntiau7N9*D-&YTlioj=4T`br)7iP`|iIMf*NV(1v&(3UCvu zd6FJPXYO0f0~uV4iLp%yYIv5y^zA6uT$v2;`l+BCSiRFGPG^AU#?JIbQ|Io98k;i| zTy7E7XO8%6FevSZ{Z`kC+4!Fv_+2pb$2$Al|N3zP`lulO&ij{;>X*6ma~A#i6Q&6% zZ=y1s*ub+kZs(uI2j*#v6xOI}qT;_?F-rmqU{woZq7;V&FwJfI!K;go4sYNLbP(LQ zMNWjzlyY~T{l{hiZ_Z<&8lnxbsF}>|U5K;vg)~6vuDKci{22J20K1~8qy9Vf{_|3s z{v?mI=c;;JaQGhytiPO7ewr}&k4Gg(|I3K|$H=0q5J#RfXTH_fZ+c&U`-ex&vfKsl z8u|XGTRlf_1-|qQMaYzli7W>Vj~Qso0kLeI9X%*`b_+(606AI##k6ZhOv#Xz z4Vt4zao3@!Kppz1AjKaD%d9g45oIthI(V0%Fk zvJ$Za2Iaq916a3y6u5o@YIkj+1p1w(fYE2E&^f-rBClOPzws{;u?r5EJ+HR9JhOB8 zAms9MGQNlUKsJiyBIQ|D>)*=5kDf(Z)Y}8)(y9nq+t$zrwrn5H+W(wz4dPg%j@g?( zwDQ}B z9W~lbzu09YF}^DkX(oXvVxjaZ7>W&R3mqmc0H@9XnCt7v=l(;heMOKan*;ba57NuF zio1Kw5f2OqE57k_z;TpLv4R7#78m4qOVGRktPREOU+y~n%=#VK?pWsDE%T!?F@lJuH+_10G*Fl*t6SR{EGhZ|S>a*SZ^GRMefD1m3uVym!9wjT)E9LC|G0OW4O}R5e1&jt!iylNONE6!4;e_&l99#iy&nCNO6?sE z(l((3Mr)61oL;X9L872}3(+j%ay>q^; z#?NdSGxfdvyWQW81(}!WbY=#q`PPH(~U%8dnT2MdJom@6a@Al z+wat)Oatrg9s)mLp_(0#wuLEBD`TO97ls(Lj`x&wI%wmCBfSnf>qZ5aEAyeGjPeVw zX$woQVDYZGHF%jk5X`75sx2%_etfak3l&vwm73R3!jA*W{enp!U7!=!qdbBqcNqVy zC#l!HyC=3;i{P^o$W>i?-jL<^^ushe79U@KN^b1=aRK5lJN7fN$iE@k5kp9~NUg^0 zl!brx!ob|5kVR*ANQmvD2>x!aNL@RegN8ujzKrZR{zTS^x6?;<{B+sk zJ6mWKonpgA@_<%YNDFH4hW$iI7$bc%!y#MfU#apgTkdBczCpWu*&bxqKD%2?sYUXj zDkr{H>S@IM{Jbrcy~P9cBxJv2d)Gtx)9N5Y>B7`XZ*W7x;QQNoGol4iX&M zPh0oq1mOA7g;129o#AR;z5SpSx)4g`8u@z{Ym*c}6DV=I+g5=Oy1it7ac+K_@8Ax& ze>@~F9G&++;JnD_8?F-&%QqEpFhnGh_4qb4)`fTvN7gkRCtZ90k|D@{Ri>j;|Bc__ zVPc&VE7g-INz1iWW~+oyuoQzvdAD|?EJc!2wB+1|_~P4d@CkDw%1dL*3=5-Y7lal) z3@9+S33*&?)g}0)FBm(zjCx=61&PMDzZD~@$eQM`8SYA z6zB?M5cZ01AXv8nxWI_jFL#9X4!?udDd09?6Df&gFU?X|j!uJ`2l;>)n&s64(lFXS z_83^;8Us2(a1(C?nlfmjc&64CiXLMi3{O|cAk-G;i=?1WU&Dm_S z61W}Qsy|Q@q9)5sQX9^Hnm7vD9J3(;!uFksWCBmXL!l6)16d2t*bFXsx6A2nYbLZ2 z-_LP2PVVZy=O?H&9OggX6E4j5zHD|=6#4{4lrD8o#_e_+5^X;YKvs)ww_U6LJV*4E zo4wOE6iS^((O7Z12+UNs_n>}FIW?w)(Brj(putcVIDck`+|b=te~Fbt)CT`Nef^*;s`51P9dSBNgKTPkd$YwZh(xme6l)GS!-6(ETtj&AmD!VA z1_M@ao8#fO^7c%DMs=W7F*JQ}=CQ_YVy7MH}9f*JZ>J-k$)&-eEZWtIj4_ zzX!%R{KW^Tpsj{K;Sew9W{0X={WAxtBqda8Pf{IDRn-RtuX@!PnE6`m z%r3Ws_d}FSg93Zn3TI0DC$-vNr(npo`0T;731F#~;9RdZ*ySDKsC&CGSW5}btw)aP zCO-FJ8uXR=xDk~y_0p3D=Y0l61Z1wNeGMKywc9q3y~E4fapJWNy#e{V(fbF)`kVrS zCTd@u5?p!mcIjjI`uJkqF?MUTHY<^K3_nI3#?hVeq>Hlze@U_qNjG+VWe6T2Eld4$ zI`Ay=ZrmgQwge+h6^{!U?ttdd?tEsAk)8Qy&U;tELk+jqt;7EJW$inoY2cL$7v8B!yt z1F|i~GG3C5+8#IH^j@Ng`Uth5_Te)mq;wf+X#5f8mr;upkhbz0r7beJ0)_z(1$)bzqopeOS zjZ%7LxT^GnGNw)%@e5%@b(Cnq@|&ICpDXiYxfzM!N!H|l(rPr^SYO(dURz$u-C14h zADDFFjXfT5$@o<@W9rEAWUO;Z8Q+inmDFe-CQ4ME>Wo0F*0H#p$t4!%UX}$hVOZ1k zdr`p0>jpKy<{XndGPVfC0P(6krFqMvI%^UD(K_zik9Zw4f@Y%;v^-&-9JQHVK3&Xo ztlR1zP|4TXFjv_^QvcFtS_(M&7^+N`}=gMz!8 zmE8@(?=z7oYUd}~IT$X%iAEPT{sqx{@PRBPs}-uj9+E(T{20aaddw8N5;+QgWY5sV z89wVz71wB{Tx%ke`)?3flFvP1bW{zcUq3&JJw1)8d(UoNKTt@AE@7;rC99(jb%hDm z^Q_O2@3X;38Z+@iBH&|HFh>q29S#bDI98y`(nS!4Rq}0xJ`u8^C$F5n7cx@FY<=wFp8aB z(ugDd=tM0KRXCTSbf}<eJ7IuAvSMIaqFhM}XOG!)NXD1;;=31@U? z3&xJ1(8cGqREbmK&hEj=!q6*cYSXn#_{vhQOo|B^QneUUko|Z$I1Icj;b(7e|2#$h zcskM(>PP`0BPt?1psIah9!j;x9;r^OAlvr!o0%!olkN*eIS|{X&rCrsROo63n?(P( zGtWJv=m>XP`**PR6oram*6B-T1?@KbBrem^t=4)(z@;2-yIy+muK8P(+84^8v6=#l z$QpE1iR)b+I6VwabrP8IJJnWv$cETah>o!6;7IT4eExI+jeT2Garm9qB4Uj4ZOLld zt_ts`kz8SITC~T1Ou<=d4Bm=C%r=QXpJii8(#ZywgS4$tUt?yluCDggwSItfYx8t3 zE(+9>S3h*vN8(Kkjm-SDXg!MOS~Pxno=f`Am~ib{cNw~XUdUmwZSS^QEQm{ngw0>~tL9H8U^*A)9cI2~DOM=avrPm9@^kxxBIjHYHoPVR5(CxMKVG!-us zoM(SuPCp+oV;F@*mWZIICGmv8lmMX5S$%->>MXjl2uB2^8Eh+oQksUy)!F#vtX(8t z``(Ke*ib(sldP(Ha)xWE2_|k>1T>$PHuVbO1@2;uoXbWC`c^5metw?O(T+zg=b$7S zK2}kaCX9RU+hFOj)M<9RUYhM<0A5y_LD2Y^4ZON^6mr(7gSi*qOC3 z-t#lkhMX+O96(K8R?TN8g*DyRV4n-Mc=qx3w0krU?6=%W=_BipJbN6%omJa>@AYG3 zeJFlDvaBB(Ut~A`SO&RBT=6-tFW4I-5@2U#a_?jx@U;<<5>SQ3W?8xOX@Deix?1+J zZYZua2FTDDz`y=clN)~hd$r8S--}aTYPI7%n+yEK#dPS-#P``xRKBJvBmO?9ekE6wge-Zc`l?+zhp0=XB~5xdOJM2AOuF%uV`gL!#NMahg>W73 zgrmkqfkmL1G7F3#nDOk_Urf)zCQ%!E5@Lrx`5<4_GSvB zwM@eUzTj;U4<&q@yY`=?BPO5)F4j0XHBSP9J4qN^75j8r#}628gPLGl5SckBJXXLb z{ER|Mhzo&rZQ$uNdpr+yTRHHX7-;HnO6)>TNr|AA^KdMq!9)rR*$ZDoLx0D>38;f= zx8aS5p3+nWf_;AZo$%heX7PXscbIyS|F@}E4jrZgSZ|cR+NTi%)*!-y({Jb*1ATlm z`amxoAp{3ddgJ42wjvh9vq8<1JI$M+pZ>#su8%Y}H5~0yj*u*?cfgQm*J%SSGTM|` zx>YPEvsB2|LC9d)1uC8!pKqsqa;Bmt3@ETSXSK*mcitgWSU417wBtHUC`0$YHm*+X{24VVKhhI;(I60<Owa?+GNbat6DdM%WG> zyB!A|LC5eQm32mZB$xM4G~ap>xpEPk8m)+H^xS;08o6Wg2pW_4+Bxjpo84t5n`)&B zV06qBACuMI_tezBYLFkSmYp6L+{~6=2nICrEc}N=>IhFE;aGbc^i|5}`B(;iJ;RCd z5mkXoK58d>XCfB#xD?VBZctl^K%9Vf;47Jh^iKki3~AWE%&3IjT|3;D#;dEJIp8q) zyIQ**|hQTVKi6V)I)Qu_IkWR<9Qdnx@F4ZY#V=xMg^lX5W==z@N1R0^@HyzE zI<(r`3X77nr!`qzw2Ef!`kgxQ37gnSdxurE-9@W`rmW^-vm7#{wswQPgzEiyyLB)C zR|uwoOE?-Hq0MjR&)dWH?jZD|%C<@?{Dbtp+m$44F7zW)CEO?NQZ z>oD%ePbTzww|TCIYJa($^SR5=?PAzpLHg|oblM$viP;qZHM_r^nsacwErXw#Hh)6Q z9N~ilT*nZl^=Ltr@gIMPC4@wPGAd=)AHV6!&uCcPwp`6^)Ni-a7%df!(~>01Q&{27^S9PO#cG9YSKvU7qW4y7e~LeilyRBrDaa7}-HeZq>5B?6FX zPeL~>>{HqmM-i!^v^7KYd(fi?*IX6ZJW!eS zL)Vm_hU_h=;~Pl=z(WJ9X?c5IrZv9b#Z9O0> zl-JPEC`c!xD*~dm5gc~{SWfF0WQt&83ACZf>MVd^aX>$Qv$zuo#M`1ggeZR=epi*p|6+cO;&qG596W2j*uQs zcHEY8_3i<`-ld*}I7l&5x>gA9wrsmxecT*I&JZ6-J@xJ`3fb^8o|qurvW1fO;yDq; zT{n?8fGhAkaBgW)c+2f|AbI`58@Do>QVynfjli_HrK|TxO?CSkqLTvwE@Jnl z6)sgOO7F!dG`m?2JUNnzWs^nDl6?;(u?GN)51UFG+Phg~Ss#yY53Cfo%|H#eSzCtC zDYhGj(O4VI*R)Cpq0*&P&8yp*plxw1@cSju;#2w1<5IYX0<>$d?m+!Z68@Tn5}PCE zzXjcBNg~y&r3{u~(x998d%T-^jFjuzD=Ak28kH6X2slNY!9)%ntuePK?;s5d>E!Z{e zI10S}=OKW_#)s@~|GuJsUlHgQ{{LG=Cll-nlj8rF^nabUVMh?7g;}4<(8wTNSdNMnr z(ohF!msV@xkA*qp2KaJM*>>kmH!JQC@R(@!xV1+zlL@}|ggSP9!xo|s%INqYUTjWhIHs6#&Mt`J3gyPD2%DD9XzwkUlSe#la%&KM5Ho{ z)X{e`fxMxRJlnfAn1kN?;F+vW)9@eAe5feAgEJF1v^RGCKYzG%^yJkFucV#|&bm5i zlkE)Pw!-+MIweXZRxCe8RKayD;?z`4qFWE#0Z%3)!g!$pV^qHP#YwQbX0W=tE}TE; z;E%^Vg1Vs|6IlpdGdGC^PDouIs*h@~QH5?TC-M)7q(T{Pzybs@XzT?{a4LJmB_PA` zsTebWbBy8TQ*%^Snh!0NckB@LiAs5Ui_@qW&ck~wAtKebTI?kC^O`A(2Lql&~#Eg%R3%=iF z_hae`(!n9NH6dx8@Az{J@zr&(si~|GJlMBOJxvv+0>cfg62aAhddGdQxGus#qGbqc zfa2OkDMQw19pl1gRr$ZP9x^V&!xZV zvqtRY1x-{Vl-y+N55VSyOS(PX-w$=68g^Zf?jYY#zYMGDormx2j|uRX;AmsVsMkdS zyY|+A(0f~2$$ZB(eo#mnK@W~AJdbfeb7%@#tjFuSV6$m?CgZ&MB;L5r5C=TGkVI*X ztJz@o9rsUrBQJoG?13mUhmrd`3-`xrn{B%}f%x=nLn7LhsBQJZ2mTU;CWMOWX1f@ba#5OBZ6@mls2+k(k$JQ!`oKu3!h zi0RW4m(py$zXbFk++=tW?>{it>o+;1f?DL90T&KKK2QNk5eU(WdUP1tk@ z!R7<_jSqhR;(xkJ%q3D@5EwQhbGakev=PcOLYwuR7B6j?llwk&)yRhFx|Ms=oaN^t zG}?v%i_18KupE^*fBt+2m~S9}DIy`<9Xa7~kGKUzXA> zyJe!0>U)|C`A`#ZWI~ zrK+T~<~XDC4J7(8r*S^$am$j9-s^6LG2-6@iTL*@NJ&{+G)y+ab0he1`4Q4!j_DO86}Qn4 zo!EZM5>30B9|s03_r_P2MPI!s%jTx{l;QQ|W5168p;;kX z<9wo-^C^?fmp-@hhZu&zn3=$p^w-KsJl~!d*)`(!eI4P+Ze;JZ0d=F?;I~5)os6Ws zRpAX#hx6A{w7w3%XG}Gm#AB>;cz$gOElgeTaXlEv#h1@Ij0ITqtfX+qm|LykpEUXQ zE%lc%Yboy=Xl2|1`rb(7T@06m$&4k7o?`oc-=-)sQn%P9XP5Lj^vl1G$Im_fw@33PcVq|* z)@gY|)hzbgKm3<(sZU0&0fnsO;l0U|{q?M&V#{DbJtd~rP9@ryG=F`2f84FJ8$9LO5$t`h~s^`my(&oauzf4TonDXk_*LjEPTUzD~ zjQj2#(P)8=pZ{fOez^^_FQFz`dwpI)A@}{EXAPYSv90?u)Bj`5{?icTJSjJgJv1px zyZq&~{5pE? z7)jOYg3rvp@0dv4OX^MBBmGdC*?iTvVKleln^^jP9%&;I%o}_0*EZ_oHBn9r9n;=v zJ^XxpuLEjVu(LSZ;Zw!QqX#!dNdI=rLvolP$y^;d=kOk-liZeWC3x?bU&zbaUNBz5 zHjGBcwpzrt@>rI2NKXC;xi@*lKgI*yhp)}5lsh)w`l4aUJGM1a@_GGf$;naHwcUdk X)d;2J^$u)-|1OG2pG!M?`TG9@QCNsy literal 0 HcmV?d00001 diff --git a/docs/images/parallel_dcd_overview.png b/docs/images/parallel_dcd_overview.png new file mode 100644 index 0000000000000000000000000000000000000000..3644b825270722c2f97c9768f841ae08dd510730 GIT binary patch literal 132713 zcmeFZcT`l_@&<|sqA+FzB?~GUB0SXh;*NG_O9Ah^;OkZ=gk8pnX|+c#CUjkXYb4Y{tyq3 zSQZbDpyBi>@SEZ_ZBy{q3Fn70ckv3lsTRRMemB*+Z>Fe-#{zynjd$XeB_09pE8q_W z_=AUcGUWu`N$~dx-0xEGk3S`rO*wh|GeHCHi!o5)n|OGVc=vz5qxR&)$^=n@S`!*? z193k*HT$BMqRs0&4A1n7pPrSsc|_hMXS5OryJcll+&F3UF<8qekhRWSsS}PHappkam%DPd%VBdLhBdSQSYhV%O{${FG+I@AI` z&x^Kuzr4cJ7c3q@zEo|6v=Yi;@(Q;HDClBIf<@WfKYLWk1XL6M8S<-p1xtHi1i4?e zO&v<|wzjx@yIa*qChnmKs**d}_wm|w!4lz6+g}qKZkH*=grhSo4)-`FH#zY~O zRg2}c`8pNtcx`$PGe%X^_A@lAd1ikq{S8?&IYX44k%v+ovRm=qVtGo%J`Xum983(F zd4BOgffebp@ut&zugmaf?=a{>?;wNrWff8l3#~jYvW-U!a4paO9l2`0HZEzfpJU z*H;8FV+5jt>pwh8|LdY6ko7$KH+0ygB>B_?T9Z!k`>!9+)1f;QzrJUJxWuP;ZD#KG z`Ck9JtG+6powYOFmeo50`4-%c^C+lC|^92hxeEb{n6E_Fxe*G9q zK20C~^*z(|t`HE&^4G}c-ud-oh%<~|-?QZZ8^eG^ojyLZU3WeouERo3rLj<<@XuHSofv+>-Dvy~0l~e0=!Y(~O zG;(?c+QT7holDtO=c9=)1(5(`l?-*ErtvA z*-CrwY9k!hC+mzST!-0PB?N58U5E7@*Q(~pc!iLWv@X4O2rYTyi$14#j#%Vx58WHQ za_8F#;^hM8PjTg04#&5rUf6t3MB8ZZcQ__X%~Zx7eoN86*to?kd!ReE zOVi~u)h*s3kNH1G&wU3faVZTeXVV4r?UK=LbD5s-ZQE!5G@bAHbOOB-K(~&-O1I zxughuPn`G-sAir*(BV<-&L{aYnk4+&0wa%}k2v>;LBm+AyDW(Q zU9q^|MAS*fDP}4si-$?4mHy=bJALpoN0JT;tH{0z4swfi18l7M<9P`^HSD0Kp_uP> zJfhJmuYfFoyJNlvmU&MXpbpCLP4mD)IQk9}5VWpNglrYFvHI!N8|0SFY?& zB>lCAW6Ss|xRi^8WI7h_7RD;yvDVz~mF-I|oi3?Bo;Gt^vHGCUF{v zwAekF_n7%4FOGW5!ScdLvGJYivp;uCO!knQ(R$n*H7g|)tovZkP1np5E|MEOc}&Ku z&DTF0ZAjyX*-lO~l{XC2bS*xHixz7@a&~K)T8Ce^D9Ll_s)(D7iuJ<_Cbsev`&_P` z>a-`g(Yb;_((q9E11Tp@-WhJksYXxc`ug@9o!FN)eiVc*8c{DT5cvs)20vSy9K)Xt z_#`F_Qc2!hZBmR1Xn$Y|b}F8>XU-0`2<4&U&#H%vj*5vaLPs;s!X<$jUs8gKUJlO^Wj?dw-?UO~%yv&p!H+C9*^ zuqtywvty8?eVDXg<-syWIv+?~`A<)m}ViSJR~LKA~Qh zty9sz5bJj|O>5l2UJlTevh(1o}t z6W@jtM2(;1ITHs36f=@C)oBIn^i;|yQhlTt!+sVJCE>&a14IcdheNvdgF5pE3)A@_ zn686tnS!AFENYzN+3t~$h+pzK0iKB$d_fe=FUV zZx90Q7G^a7E@Z?k-gauW0>L53Hme+>YhTAlLZ_hGQffO^PNzH=WS-=4izM3{$T{~` z3%GZJ*r~G=q*+OxRyLJiUJ@Q!k7cj#hR}#rr=Xdc`1kGqe1B>aC{aU$fiNhuyP1@# zx|6#@EpyoYkcFiD?oWcgWcWpk{!GOnS2VcH|I;!w#98S;U>Qmy6_nrd$3S$#foV(N zHPr`J2VV}tmBnMlIDkhS0d_p~R-U8n86yez=&@gp&}umS_U0*f;cqAY(JtYTws~Ue zbHBYWul70&F>eXmY(dC`7isjr4$07}KU}L3e$HT>dv8f}sXt3O-Pqji8o!MeKfW3A zbL&!zq3fW!mvEmG-}{`~##b1finbKqL8A~b zIzev3SYQYo&TaN!zgx4OKg(QFEYFB;G@Rtq5bRP=b+ud&?~YwX*erOe*={G~XVIdH z%7%2L-iPr;dZ3ZREpgU`vFHX^Ci>H{1!i2jK&Q=OIsg6m%xTdnh9~;Q>h!JM1Pl706Rq*YgdYYW;}4V@u3^j0b5+^2GEyvXIcG?3~;!cqBX zX9dplu;H=qeMkb2j$9w+&1J^s33r6PvAP?=2bGC{`1SK@wps1J=I>QwJ@`%x#CXq%SEsxBwVx0KJk0h zdte3JcLtfe_;;-b6BGhpfQydiTFsS?)AWQSYb6~Vz%$7=zI#POk;9W&esfm-PCyPr z?!l)F`Dh!%(PSEVjw+M%FxGzF>p&obu5p(R!enUPgfYl81JsqQ7^E+TxU4LyB9_y= zI#i(Bq9`yC5t*3DybnU!)Oo+XdwpEYS9bvNV3UI_qy|g@NwCW*Q$MpjSW`o^(0H-M z*LyX-zNmuM93Uh@+ouhz+QUms`M1pCG(^fo^!vqjN772HkQ4Ro&kFtuOh5eaKm&hK z<};?p;qUbzJZsa=3(Y57263sQJWtaT;DsRI=G1RRmxMqIFa=#ZI5@ zGn)Fesh?NOv}{gQNs~5xex`P_nG$~fC~vL`_2az_!98>@&&N9p{f~b54K$dvUUdF2hiqlR2kde*Yr)2Kk7%>iEmo)?+6Iz|A)L0$W7S0%5ijLNEx=#f#VWhh7 zX5(d?tI4=Ns_8+``ZM$%5!t|6RQneF)Ki0n4>-c+WXw+1vwL#z7Hv(e8S}CV-fa>} z%MHlVbCUG>otm}kyhmgi8BsbcYqi_mT9z4+Q{c=Mir!laP~wr)KDNW4>mWho8b}QP zCaM?)2x;q+^*U6u1ZFct5PP)E5R@I`YbugpqMekQB4Fv*~QmnAR%(kp5=t-eBlx*6jsYCNBs5T7on_TG3;6qYHG zQ=KlH!CYajGo@``fS2~yvQHLVNrq)0F)!n zs|YXWtM_kCG4U=BWuTB>q1CbbJ^{~9Yl|H~kVi&ej(!qkt zh>aEBp9$A;>62pZ&p$qzayK9OuRgpla{e1Ke6Wr`W?P}pZz~t4bW^deZcsy4>Tq{; zSyY)~-x2Jq^%=`@ry!At>W1;sh7b;khb!{G;TzIQ%Pw7qVe6R+%KwN9?wK#J6kfR> zerwjiQp#hJ`o*4Z5=I%M`yQrl>)=w^<%h)sQ3~iC2$I$6G6yVk; z>2W4KjG;$ly>wV_No50v!%0O){2jmt`-AQC zD1;x(X@dAW)J=)NmkwpiD0lu|YB)ff=nm0kGe{JJxqG zPPmDG+~H%Y4>UZqJ2vys7ii8HnEoa}q2~SK!|ECoXh*Z?xMy_REt!Vonuebu`qiW3 z@K5(MCAhNV+>m|^$u6JEq2FeJA!$O#pJX3&4<_;VRVayoWm^9J@f!Qt zZ~M!2wQluP6YI$@%wjIj<80gA_;|i3DZw_&P0jF=g5=_>L$dEKSu%%^KsHKeeE-{d zftEqyYy)U>{PkKcRX5S%Ond%0zQCY_Il0FBp5F%E8f zA1r1;T`cq&=ruUAGwE8rlVWrH6k(nE4(|!Kr3UtDK>{-zV<1*cx~hEQPgk(Y`s!|L z+VArHCq$4zu4Pv>JLS(>m@Vpfz~xE7%#Dof1{ql~01=%IneW=4JF}jh_^L&3eTr(D z{q4@~ZA$xE6_*irgpX!~hUQ)3(7#d;3Xt)d7-x$xetXvK4t&esTLE-o`Q$RXxCyHG zElFN$*Kp>RXu4WHW~ZGs0{*WXy8epR_cf`O04r>gor$i(5gh3z1FzL9;f%_K^{i1k zRyj6*w3Ft;L6jYF=~pD#>TEy#HpjWBJR>zVwOl`PF@U4mV%F1lHh{CfwPk~&>r%@7dgJX?DXl-d_rmN?|#Gpzs?KQ{p@_+&=NRx0exvQ zG9ahUyG33A+iGVegaq#YW*N~EtNqC$d@%a)$Cp+xYz7n_^;;JQ3iu&@Ert162NkO*dERuPDF#Q+MJ`)o%Wcv^~rY>W$}+^O7uWyW&2 z5J%eH{k_uCSu80fCB>iNihMTdF`uYq8mT_@frK+~*w9aG+50|&cqVfySYfj)B(g_z zTLJmesNv~}u=f1=QoD(2&xaaAz(k`nvYvL3j%)Q7KRQMV$y#)Sfr3iEK)W1 zrfUOw%^k)nJaLDJn)JNb&VxG#HS6`1z=Jyv@LHJ-24>j!DbDBWM8C8!9Na4SnM>?F zBCuL=w|Ol-8P~0<4$VZLRSC(RPZ&gs`yBi}-Uyq|X0oWTu?7=XX$JR`0?O#gHxBw70vFZHx*s>(`xs#_S5M=Xp+4()@@nI=2+?X-r)AcTZ5%4cB@6}yMqH}KkZ}j(n6dTr`e)C2V712Lk3i7 zZIH7een-YTYZD^4G6h%^WeF`J+2O;PL8vLZhlK4a$UYAN z2SLEDX0=?)x}ePJH(`bOn7-Cm?7HQFzy`%Wm3H#6=Y-!PJc9w%Lyi#uf$U{G(`P}p z6hiGj&ZQ*0m@fN1eBX4m)H2K$uG?Sawe7LhV`->2*_ z^=P$MAo1h(Qd7j`X2wT*n_O&<zqg-jmV18I(6bv=#RSd9yGdLO4i_uV5_5z)n=WPf!O@J`p47py=Lv;f{-j7GxibtvtAT4?LKo1%9h*pWfQ zQ=JXqDk#z3>;@@7Ear<}Tpbgka;A3TagPqtd@1{o65X5|xqa#Q*j`uwUX(uwDTW%q z42mjdiHTeQ55iqWvJ#x+YIjGii-CHERkn8YaS>4u;5@k6#2^nRw3E9Jcp^3rpkf*7y^}+t9({!2~Vcp^i|j+mb9lp2G1d9z3|70lcyT{l_e5`K4|;ww1YC2 zc>L0hprk60w)FhTGCP7futEHG!sJJ=cC5LS`y>(awB}U<#%3IIIP7s=bVG6nfIVT? zCAI16i6ZX7VDYz-z(aARw;+!j#~YNp`CxN8L?S6s1kRNq{BHXgqAuw^YL8&%vKz0o zAznEe7?e1K_ZR_V3FgqRHc1of?iONHO1R2dzh?x}Mpk&lxaHF73o;#$!H%QV(p7G% z;>{7!dSA%PFOkcyKUnGm*gds&Hev(ROqxLgM2<_!`034wF$swodSix$tMSyATaF)o z94K(nxz~lDJtG9Fx5UM&xdt-sz@42$ih z9Mvqd5Yij)T=l%>oY&AxbhqX4#bvrKMR)$QP@dvJPF-vzy3a5B1*(pP zle|ZwkR+<@U~-RvHTk#@L4u#d!#R-#3ZzVaxajK_G!XhP5!(WTKRY19AX%~xn=JkQ zp~YQ%%b8YCQ{sC^k;n2qHa}XsAd9kCH0Zhus(2*neTF}wIR&vkQ1ao$zd?yA`@C`t zW8@$%;aWDo7}NS^YCiD4Ajh zWaWA|BL7{nOtDWFDdeSBe?Q(4oXnwzf6V3Fjh}%m$u?X8FJb)5@x%UAh5wFqATP-P zd8+qA``@5S%5i`~z|4%&PX+MrkHGzgw-hjsUXp2u|NP^h$>M<%5XDVK_q)f6``?$Q zyaW5^Fa2}tZ|whH^>wV}|024x>|p=zUPLPYtM&cMLHgkq5qbc$wzVbxs|oz)D=t`A zfc>9k&-eZt`=0^TKY69KKYn=f&qVvf*#Wi-_<0tOXMa{b`Qdv%n(j?<0Q8-l7=L;O zfBfPS*o%c%T0wV(<|# z8J(!3z##7rpz6h32r@cc*EEn)fTJGdztV+a{g-3B2!&l1E_AR#6>xQtQ@>rkpkM70 z>%~=?@O85wHv^q;Dp0|~rSwgJCp9l#6;|}Bzh?q}QQTuOBRaT?e~j$_wN9H?SiLlU zHWm4GeJO#2<1*2kO)6zoN)0|cdhTs}mHtm*GU~VUFIkwJ@C%{X#~1X`KD}3?MizbN z=4lgr42PoMb+x0t=@9dIjS$PVq5#nUY5|?eYq)}yLOg##m4N2qX3N=SZCCX$*l(zxZiz-5H*8h z(yVwD?@agIN0|h3iSCoJ?TnXC;AfCC4bHQm`Z8eU-Lj7~2*zc4{G)0WwHpv=ncm0& zAG-@1C=0vF_E9bM21G_K#n73EWJ%*AeU|6G1IDZ@EZwoJ_&56fKIqz(sf!*gq=i|# z@IRaYOG*yXh^99Mew{a1lHxsI(2{2t->?Zo%^ZHYF|L_OMDwH}8W!YyKOqtiB2C6L zNbV6du8_sa02W*eG}Qv&gg9EICKlJ>zmy2piD-7Kg$N3kmP<_tZ0gpK>y^t`OlRL|uH#H4- zI`k!BN>zr}k=9I*CBNu{b0d6iaKC7+oDTo0p8Km5OQr2MB_Ex_zg(Yq!PwAK%nmlU zd({bPdlQ?Wb_$*Cto(Gw9x$RRL5;+Cvxz(;-maRNq#^G8@<%&-0y(x&{1_DIx9$N# znbGG4h-2H$=!nWwN`Qw0X(nfC0>VMa@i8$%7=i`q3oEDmwQBo~9QzFm0~Ci>wUq`{ z$?*xC@I!*VMR!H55PJ*#A=96qpAQBEm1F=9Oz9q+qV_UPNg*a^S4|;J&f7xuC6eii zVz?JFwJPQ_XWzWjAeb|PgoQ{@$RH6s;^;jFAcM{2lqvx+EJ_xp6Ow(S7I6HhkoqIthQ3M>2 z1sBlV=%Rl_>oIfjec0I}vxlM(p^LJ2-hzU)2miCd7%t-y6@&S7(H0P9Q*zEkOa;_t znx)pCW!wOLHC%5X3&CiJefqieWts*zM2Ji_aWQ@!s_vF7>bbL^1*(Pfl?sFx=x&?i zoAwq9n4fjvg_yDR_9o}$PRbbiVHut=#BW;P?~kjOBz`+?zPyO;T1$(^AP1h^h2;8$ z(DW6lzBUe!j6Jh3%nT(-6+TP<0J;7tC`6AFfX0X`aw)z94?|}R;@4G6)+yUlX!%G& z6S92w+AAVJ;ZGQV!YIYO#LTraFbGkPzbGaQ8n!E9@sl;*J8>yr2mq~y7cDL%fPNds zK-sO`MzC~`u1=<3&2kZH>$$G;ElNR}FSjCOvMkGjsNheA-^ZTm43zW?$wcKIF2we| z&B{45gQdyn@jq~l8y!?dR4Gcry2ss`=DT<2upAkg*YN+NHV3P{p{a*vC3%x zr)Lm(`zSkfL@w#Kc=bqwaLX^i9j@uOoj=;9oWe!0wLW;{Rl&9NIe~!UE*&h^=`#wSYJv!{@q$yUvJ?*a5ou`tC`1>d;Or5UzoLLDJP9z%)d~=N& z@W7V0q7`^f^V&m>y!fz_wvz>e-&gVoKhR!$I1Oz^OW`W4L~YIuB7 z0>8xu$vWa0ELqWrwx4s|2bMs|lV`qd;AEn7F@YRMC}=E5utwG51CZ0e_U*GZHflB= z$St~1Kii@K)23Swbk%Q2(qRp{VD9s$!pvyt_CXEMY`FN|t8>mDkpt5BiU!|4jrV@e z`z=I8+${~Z`e_K2Ih~@Kdag!+)e8Eu!-F1`4ogUwr{@b#hA4VCU zi3pd(@FimNwUk_O=Ic-RO`d|UNbwbQfynHJ-|6?;y&k_%H=H@bfv`?9`}zcW2(GhXZUyppo3cTTK9D3= z&b*a38pu}J9EM@|aNOb;UdxA6pp}bjC0rV8xBLauBAELfP!gR}=e2M?(t-@1`2sh) zp$}3$zixMQv-jS4sg_`BXzAKSjSS#Wrs`JMZGzrd3y!B=30OBJKA68s0>^grlk>x5 zc!EJTYQ0Q_2lo`Fum;eUh)&G$Iox#weo^C;%$zc;nZnvc)jn%=KG{z^c_sE&R1k>E zEdg5P`?~fg5WvIW!LiX-ekY=GkYuaf_`sLsGN`WE^-0zDuNuRDxv}IvE>p%@r@zFB zy!R>o{$g1_bFkplYn>i76_v16Yd~fz@L2gTJN@e;CR*SdW}VpwzD_ex+R)N_<7UpJRH z)CP^RydiJ5f7;nk7YRuFA;5=iRgqbJwU(P?SH(PCX0IiK&iGXcU+-Br-&sJ8c>^d1 zTw|ik`A~Yo``zlcLlkV4&LShdYZnB4mJZu+S|s zpf4LVX!I8R7NzGt;W`c2*+IZ1Wq#sC{zCrRi3E;k{va9;4=AQH5&tryQXt^Ja#?!= zj5!VH8i7SHj4+N!+Zw_K2OWN#XPX91{Sb`;ovy?OI=?V&Xpst@5x?+4_3O-Q3=afn zl$lD2aaY~~DD1S9lhw?emGG8O7C1Y84Ujp-02cNy`T`3b+`Wg84|Q<|E1MB=#1zJ^ zQ~H{3SPxn4Eb3N^0Dtubs+=gId*IPb=PKh$4cMMFL?!vsFgDx5TM z)1DRMPLA&h$$6hgS^Nq@k+cPxu@1Lx0L;JBou(s?e+Pc!W{+vT>ZZpAE-kx)>!4U_bcin^k%eAkBG@ep4NcVbV%m}bcc>Om(ww{IN33qZ2}(nty-v+vz<&ysiP zny6vv;q0?wy`IQa#22rox{wLOsGxDjM~HlC-L(o^?IGoif5e0v5HCAQDt+>`P_c8P z9yPId_ZQG8mXNC1TvoHCtFgE*rtFg}3|nG`wMfofun@-))`Fe<>nxbyXQL+sec|t{ zXQ$yU$%%`qiLT@D$d(LlgM(pt9-oy?Gle@b3^cx0X3(C8^)K1HqFCs8Aku)>50~AT z^&aIFp2o4B?!El&jcvsQ|2Z%&;8bX*J+O+S%>YI%JC5Hj%b=Hu*AG~W9KgTT$)@Sm z0{~XsuOw2;7Lb>53Fys25CX=&==tF$;TM?T4$X469)veEt~;ZOXafVTqb6v^Xey1V zo=m;tGYyyiNBn7EJ1x4|d(yE5Io$o>V#(Lz0?{V#E#g@=3kDHW(uYJ8@PNS_g>K{c zC8k(Z&3bt(rNi~0I0>l7wrS7OHY^FFxwAj$=C~h=#HK;P1~tZ*B>*KQX7lK3kUG!2 z?QIBd$Pi4N)l+mNhsG0|j#oNv0)uEN8#bU+%~W^;%4RH-7gbVBf=uA0v*3GPIF>2= zLOYJB#{bsDS8f%sZ8&$~=E+Z1id(olc;C)Xdadg_KmJ;FPtiU-bMht_-kKH!ONBmd zGwZt@MAs{e-@#F(Fz!BxuogZ1<4x7!aL8o9E1BN{tk)~WGZnffVCSm%hja>bsYdLv z6d_?hl)D=`&m=gGhZVn{t1~XX868)3a^~A$@F2m?vf9aF(3gR5leH4b*#QQXDAc5(d&~du~fh7{^K108%oac(ps2&2xDI=7<_Wu8td=)b{BC)E`-u%_t+t zMF#2Lt5sw>aBLaCRB@OAJn&I4$3zXHpJsZ)Kxu;OkTx9Z_x;QQ$pkUMkBZ93`12)~ zHGmYZXclbt)h+83_mjbbS(WN;yfkn}D5HurljkvIk2wQAeJujuqVpU+#J(OQj9Cvo z;e1D)FAVoP3OP{ty4S`M4gPVKG4yF*KAW@{m(|Nq)TF0F=N3(ep|h@tkId64!06#c%dyojnZb-mUp*qT8{^> zaL*pv`=uzY0s{1Pn#$?-Y(4xw6*)Xb00>4DJ!LNMK#fL7b#0~5WVU;Q!-$(PAst*^ z9xr-rU%i-#IX32RZf7kzLg zaBFWsRk;bchvZ0MhtCZZW3dvmz{1FAy*KW-td8;>?5@TuO85?_$TWerD6YKNW6tV{ zf%o`o#71FWe+71r)-HqN8|Ue@Ry;ztm)7=>} z8-8{;thSaDopceQ97meyS1h+ug|;h8Vu^I0M%cIQ()MBh)Ozwhc}?D0Y{5^Y9&^jR z$Pe{dR>{}^t;{yU90h(0O8{dOiy7h?3ZR6zGkC zU&Ze4t?UJP+FYnXF6ufQJ^%QsIlM0}Nev0I8sRLdBv)dgtV4y|j( zhqq&q_X^)&XS$OG6vVC3PF*9Wd5QkNk+3}aUfcmOLh|z;&fOu1Rk(mS*2O!)<46Ur zHYU%8?nk@bnTs8fO;ec^+sk}{l;E79F-|ouq!l83Tv*X&Ry>*c0^JG<#awH3whkIg zNkr!ZrN}tfqU)FED5qA7i=>wMZKnjVs1^!qht0*T#)xsBX)~YlfWev~edDCjOEMu> zX&7FLB@Cka$3LH7b<<|o=o)l?CAQl>ShvUXF*@0g>^X<91gqWWb!_N_m-v%m>dhka zq+oI^vS*=m#Hi0Sh|l=d1j;`nO1FNiG~5}r6P@H?#gC8CS+>7QiA=(z6}A7Su94*B z`TK8v3y}Pkd%+^t=KypW69Hr7S3OaPsbZqF07v`!i#V5lMYxE9HUE=P`N-oeEzkXA zP39(Cx3^src#qApzM?e4tUdMXFM#*k0!9ruc;Z!$T1b?tZ`0InzY|=XlF;Z@QrCB% zN$Y8xpOy$Q)zsqmts3IYTkkHH*#8_}xmXepMd}l+#MNL8a+|#YCAocX7lx3oKn@2c z>7qKfM8#GXW#^x44_1_b;Ni)Mar)o^KUgDLCkx<2_q$Vt*C&&r&Q@_6`0WQhVU5Oq zbfj%Il4mFDD5%tPS)JrN*cr=AIq$8}uEcmWw`@PPE-*n|RKH zxUOp`04ZR|nb)6i#ACI$aCLLzb^g&Ntq%ojLAg=kk-G7#I=9z^WJ_9VJJHEL*M^fM znB=itw5QB$<=^LGJr4{wrz8vmUyv`8<`uwiiqz-s&)g56eyhsaC2s7Ba3~nD=m_#} z?`9Mgn5ud*1^J>4f0g7QyT&tA3oqQc#;W)pGvX7{RAV0)kh936!IACuqMNlbO2qb} zJl1>B`(jf3Ye^})(HW?-t61in0r#l9*yN>=o!&6QE%bIWw(}VQC`gE?MlnCw%!oE< z$MDFe(Ub%D^|=!FvH%cbsd00{cRYP@Nm~F3XYFm!DGOC@PjUxC5;&SVl&+Pu@(JML zlddRJE{^Jh%6l;ub4b2I3Gz#jq%+6^SKdM&E-X3Gqh71*S;JF|1G6?)38PVYTk^@N$!c;gKfU z4Z+kU?kcb4QCwiX!S2njZqC*5;DpuqqWo53C{$7pZI&7+;4a~E2U#Zv@cl<(&xkNpx zUNbCVv|fpv9Glwk@uOUJ*A>_mbV9_4nmVam8?zl^v=f|<%g0#G8~2D_j9)90idwgi zKKI5DHCew96S1)aOL~Jm-0j?*@Iy`%4>tFQRDCIoH^w4pNmcz~;#Tu8%ESu_!~FWw znbm6ff@limPQnEcJ1q173%p<-JKkIn3B61gNsK)#XSm*0RPoQ$cU*h%fN zHLo>^&2CC#<43T>l2xR@@|EEUk<#o5+Ij&FegjK58#-iOpUR7RD?U|}dE|EF;rq(C z!!Y+rbJ*U;Dst%>ERw1{w7vg@HgD`T6H5KvnIdQuJ8okU>qeWI5>-HP=z0sI|zBXtn zO%En|jYVL+(hajV_)-k#8PB7ICA1?t;(QaGsxB?5fSUihoZ_LCTpkTmtOTZu*06}# z3!Yvum?M*(Qe!xezjaRROYcIN+Y9R5a!7jTBl_A=#|xYl@E<)E>Cx+VW|WR#eL{rE$x%*JM{_9S3?7y z-9~t>uC`nA8jWuVvS`B0X_GNR*;d!KT@yInkF@)&Ul~NU)!5d04P~lV7hXcuIVhWY zA|rh~*W*b>y;(HY`}i=sD~a$m(#GnROfS&yHof+UnDVSe#4_f!8+<`BI<>Ujf-nf{RY@lrehEW)T%wnT> zDa>raOA5v>F$;1NdgEqJ%S>72aAQBv&#k4#uc&>O_E_4PzA~BTSRhBQ-!hM2!SGsg z-e_SlXsmsnqU7j!#-Ivc&hJZO66E%IQ$6cD2CBZ97k^u?G;s*5L>5=hq>~uU&T9If z7vI0_xn86A8}+7af%>==Y!y5NvF2mJan983}<=K z#*0&U-g!ALO{}2s-rw?mnr^oZ9qOa<*#JeLJVF_cNz3BO&yCJd;R#o`tz+vNk=d-C zR|~Ru5HmJ7I>c*bLOgW@$0un@K8H2tXxPpaT&!Shmg`W)_M$@`qnZs1t8VA%9xy6q zlpQ_nFdefTXdjKD#QHjyv-&Ldc|BibI4g!q_G^BM6%ix}R)B=M8+cl$OAeWgxX0_4 z7-Ge@cG9%*Ol@c7{-Q!LkuB7rgZs>f7tyM>{V*{yLoon*Dd|mm*gp4a0l7M8YX!h z^&MHwZX2Tz@da6Z6z2RuO=ctZHSDY)R~zHJ<}EXRu<%-N@kriiKT@OH=iUCDbjhZR zA*4hBbIOx`W9;;VGMRmp$~8Ofg3Il9iqUpnO4#d~b~n1xP-CnDPMv<+aO3&9L-S=z zldfTdBVaCrkpOjEWuT>ux;eX@6*PTrM1UlC-naVN*TS@FDBl-Rkhpxd#}q#3;r zZI2MwPfj7yl5%tsTtQ|Pk@(a%aY{bbPzHdryt~AZtqZLoOmHy9%tn?bYQ<+=wzuEj zAv2FfH!RkVS*;l+b9>0~`#)mw81wZpTP5I$_1a$L58x>j7JEo%sPnMylbh#^jFh+6 z%2;uSXN}qWFw4?bRSq$shjeoVABkGv&%4{DLljyv`z~;bpi4^Q*m+dl+GFquP$MQi zY=TZ&uo&@z{+z6JUDjhA8Hk#aB5z1$!n1Ug^rPw_ItKacfc@IY~x2(75PEQ6iN{eY< zyD?HCwT3}PEcQRv6k}3+lbGqC)DfAbPe!yeQRL)Kv3OgtsP5zS8WV5V^@4Tbpl)|uP%+K1Z=G%jP@jP;IMwprH=y$WF3{1isSOA04mh}c&d z&lc}T@>5SN2qTpKK|-_T@P(^Hr7YUa@ojG$IAbgoXw}GfRT|e_n?2XL!MU z+(%iJsy7BI*Efl#^02;FNIGGXgY?j+J=d#Lc=}6*&v~gSc{vS##J*-$&1y6fYN_FX z6*#GN(CF8+)QhZLAxh}IURZDYAibdnC4r%fudhw{e5*VRE9G`MuYBne(;LZq*r}=; zK@OL2W*ysbf8LOa8E+H1nQX$qYBypXpyd?1$RORFohvbO5T-2V=_#=#r>_y9>jVj- zCg01U6$(nN{4%FIBb_tO^=YRM^;^}=@@0WKS|P=NL}@9i{l>JgcFVBktZvTz&y6nU zW9uEfgvkXf$4(HL@u@@`D$SPY@hcV_jGm(nY<<7zsX;-+wrO2B05C;b{(yR;;u6$T4KR3sb^e z`BEi>t0l%;>6W)CLeX+|!i&6-PQ*jjeW_A+R9{N$m>i+=OBz%A1Oyb_2WJlg>{{z(79!#wdbWl?H~u1()iRVWYDv6EjoNozslEfV zSy~h+I@IMied7N-Hnzks=HOn!L5(W32bVZLT8N3)5rL(vToV{I`Y3490@22s zld!8|EJqmE9Iagw&pJv=_U7n_#-}`_?ycC0v0HZUklq%xcy(_owk7FJ|2YZvHCkuG zH6D#GuChewVLVu;>8h}2^xGXuhig|^u}-ss%au@QN)xZxUeYTitZ3CBhi+3FOe}*N zT?q~DqSe?x3W>*V0&i(sve?!i%sV=phEl?wFreVud71_ks?6|w?JXrAcaUh@GxPPU zF6Kv8iF_<5U1od8V%XzZeVMX6bBN<~vs8uMgg=et=n`q*lTNM{wNo-2V8{^H4!=T3 zV&L4x7pbb1r9z(v8rOL>fa)g`Cnv%V!=l%>+oXET6C9;gHjLvz^#i>0R>J)TIF2}$ICop@?D zz_|Os$JAklTb_;j{Ml*&lFmFGC*n-ewUtFx4Vww)j%S}+z3cZyI%zd$GD~DpftJeIzE_1X*gq^$P$8{fo^Utk|(RvRP+>~+K>998Qn#n zXqtNZ-VBP^pB?UB|FX?sk?CV!hjNQDEq+0s_gio$ZFrLM{sirGDT7jQJl09S)N&$o ziR1XSF@SG%LF>`GK#t$xQR$3$x@*Vqm1NcxBR2IG`)`^Z-_aFV?Wt1xO*2lYWHfz zLmHdj^BeOME_Shs@oMm@B|jUAj7{Y#W>~+Er`52&fHL-ukFOubBQZ#T87Mz%-Dlx3 zyx8)vEZUcXxy1Hxw-~9^4@Y`iEgvIVtjx0iI`+gS8z z3WDy*0}}j29bP_Wea&EzwMv`xFFA08I01z|T@ZKWzGr;nsgp!4LzH&cn0t#|L4G?7 zU4-@@6!=xDFL!mV_wV$`bJnZzzu5h#EOyDuwnApR-1{+h;^j<0cwJ^6asNOcs7y19 zV|J$7o}CPEZ(TF&sjhu8Xwnfr0VAuoG)(xkskD#owJ~Y3w%_;L+)iW+atO!2h4n-g zXzz-_5`$}=Aj+d?pX6K0z2SY~D&FLtQyX7cqoOA?mHPy}DnL&tTz+mtAZ1B@T8_enn&(hDZ64% zV&)~ksVYWeXfm4V92(Ba5JVgof>+&pMA95~NS#Ekx?LpQnzzp}@Kou-ZG0KV6qdID z4NBo#k~Xu0P)+$|6H>GGI*)VzqX`2)nW?>j_llunPuL+_sY8BcyH(e+E@%4FM&7$q zV=tbF<=T|;tD_tgY@QzvLZklZL`Yx2yuZ(H8>dfb?3Nse4>= z_kUBbkiHl*I?uA-X<_Nsr=t+SUZ&;kKDZ8QY@*bbajSg8$vdK6AIvXL1z&VG>l&>M z&m87xVM9}YS%D?+HRXq+so&kMeq6QKM?}+6va4OGysb8ov8#9@Sb$7g&-)NrF^w)> zuejP&qJ}A5D;S;(=Jj!tV?JG65RqA6P1ISX1}WXKMTQ1x$w8%ChOR-nkEcJpcVZ z@4Npw=iZDn_qW&ju63>JBJ|GZ_3Nu)u}xEf=~8TlliX6v+V{&G)@Ij8;zea2`WqF; zn$Z}Wd{r8!_$^ZWNrNKwQJz?*Vh5*Lo=QjRlU= zc}{>8Q?bC?s5dG+!9yYnuj#fqtPX6MK$X!Terp_cARbf}vSO4q@<-6~|((FEKT!f#q>CxHX={8}ayh;4QSwIQp9 z&<^l<*r~@R4&ATK<&?>ZMxDn4<>dS6jN)^|uP{O9dY1<4`|V>X++)ny7v!!qJY&Mu zL>W^wC6^{&x?7?Q&K{#YGs^M)NYQjgWZuJ%e}?eS*|thIGG1;(NwK%`&tX?%#;iml z<=>+@yiYrf?M9z89;uKM;poGza`s^#6`3#3e9pP9v+!|3Rol$XxxKNoB6_!^c*Awt zgQ6>}bwZVSRxv6OKO{wkHP4DdZ7WEwh0BSo*`$?Dl~eX6E?<=KAk0jltFbyRoZ1e? zO5RQeHbUzaH@hjiH{8yKcfxzc~tHTP{j+elSW0QkDGZ+7WD4yx6roR ziO@)3NwpVwxaP8=pE*zEvmLf-OvhOG6-6q;cj1oq?Ro7$%x`zrdMa6n6M$dbnPErb z?cIy5NpUO*PnvlQ93(}%^9=^4@@dTJ#t9oA9SqxY)MA3Ti0h8gzt=Xq@-V*3g&?E% z;iJ(X-%{NO7R1TjC1sYv+kT4tMMd0EeqRr&dBIAQ;_(NP#zCK++2IQtOBg6ov`ay& zclHIaB-}l}>+;3XAJZoCHSA=ztue_=GbvVdO`{A=c|06NsY5T!8tExW_PQalQyZPb z(Nn_QNW!+soN1~aEN;0y-Yd%J=t#2+gb9Z*MmjohKR-da&>+|j!{$P zR=U9B*@($KHYa0^C2B^_lFXnl!cH#%<_jZ zYESed47b*o{7lR2AqQ8SGZ}s`?K&gnq~Ij!?f(8jKaurI;=+Og+S_Yh3$3JFmG$x3 zubAmw&b4j|<&_PLOiPtw;wpk#lU^O1agop0oQk!amn(`%aFqm%gA=x7CRfB~XuUjI z=GoQUrqBWO=0alkl&UyZ-NgEfeRKX+oa($IY==w@Li&UDJXN1E!lt2nsPX3P&~LRg zWpu(;A8t{3yFX(luEIRq9SS21ckN!!_$t^@ZTb6y&4uMa%%ZkY;Ek-gi_;Y;9>$Ft za@E?|ttu+PfC5aJ^36zY@V?q~8X;GnY@h{1F>?LTh;c_as*W4`J|!Pu=0=-L z;&t6erSE5=N!y5!iu;7>27J?C{sk(na|bdDwP&1{cPlD8w{@c2YSk8oQavWAny{E< zg6Y{OTypk=r5G1W&rCgS*W{5^p+nhiRUTs&MQE8`g*d(I`vTC|`BgKD8h^5Wel>q% zq6!MeFcfjGB+I?a#J5Ghc43D``L&JdqZi~pW!BhrKZ5g8RMzzA*T*~;82tROeKxkVW%Fmd%R0ALir5FOJ7RJS zKfJ3XZsKn)vE1dguO5&%^6E=L|LK$12uMJWh;~~!#fm@Q{i6PC_)Q?^!(;E+2k_r1 zu>g6qg|cqMu|;Y-)f^LL-Tv!q;QC54nmQD3RZU^v@az~asFRbu#VIA@8SAxN$2_@6a}E%On|c*OZZqkGL@pL# zws+Zslx3ar%iWKGT>q8M3|F(ji1$quc zVae()gV?hkqOz;60?DplqA|ak?))V@F(=lmnKr`b^7WB-OnL=pw-RMeK*qO7vmCLC ztuLCaqd7%aXkUvZ9%@s(uYW9eay7ETf{c!up=|AekVEq`kNbs~8)S=+iH=XUBAbj<+3m8=FgQ_#Se^pYldUvg>EN~idjNkBvX_aMAuewcs&A0N>Jbe7 zP^sc`HWTiizgK*$`vU22!jo}DP~>IfGTAyBp2jXIqIyi36e6PnwW=zyrt3ik{>b_|5K_yImBD zX&UR7_s*0!d#m$gb2G+ar?<4A@@RqI%lKvy5m|wW_qD76Pp1}bHoqi_&uf=16_}T; zvP+{?Sx?Cw?ssyg9=$B5NbjGE6DBTcwf zEZthnj023y_4}=D6bbqE`dYwYo!FS~C#vas=K;w)*b~R{@q8LW(g@F#aYc#*XEKAv} z5SIbGM=USDrg))sPT$bW;d6DOjA5dB#E~I^r!s9tKW8HKn77{)S zVh{ePmdWoHnZr)6&N%Gvq>C!e$lE!v5bDM0{bJYBV)aGFa)h{pbhWnW<6hkRybPPV z*hKL4g`FBZcjk`>JB84H5aCLU&(xSNa*lG{{@nfWwLRX++H5)AoJqzwMm$Q|xVL|` zL0e-t=Zi|OKJIDaXI_aNLWh~;;&_bHxh-KPpLNHNAF!8*q(x3KN2BjfYn4LgLagAh zLk8}T`9j&iXX66>cB}%I-Lr|NxG!I^aG(xigZ=x9$JrIl64GqF#3S%}%yQ?5J*Vo=Owy zH^08|q+?WgIL5d$zT#O;(@zM_=jf?zCEn$&zmq&8MA%&Zq&G02xPGp?;Ed6v+UEQf z$Ln;i{f2Al!)fu&P2$g_Gb1|tA~_uCDaWXzO}d-#EHP2nfpf zM{`~?o|@y?93^2(Ybg=dxs7`@&O-Pr*H#kTmrKjTA8+RJP>)`WG&{j0&U4ETp}SEl z*NqP4O5@!cEkboR1+!`eXn3*n6CbB7u;lfWs54RtL;m!tY7a&g)FL_)Sn|+TT_Q4T znXaKt;#kcG;X94JOCCJY4l$Db*e32?T94J8OFJ%~bacDYJB{`Qe0oGccXPT@X$y4_ zcH&&tvb6)1O(dM|?RVYO)1d1o6tX41oZD!JGty0fc!024b5n0} zo*i#ER^~C8iFYF8eJmXf^?$l65MdZ)jJcP1S$SC}v-l$ErqsT`W>JFD>WexTlBxO3 z9BsZK36eK1V_6bI8Nc6uASv2)KYrr&Z+a=FF(KNIHq4x+wp~?ASJfVVx##@sXbxj_ zWJVz?>1$fcmEi6#IHhhR&lahC2TPNlzZy3)uF4#9Vje5VH+S0dW|ZqkZh!HwC98g+ z6(6c&HB|C;niBV+z(9Ua2J+H9-tZ3wp|k3u>g3F`k;aiMG183cEvV*`qU-p+4>MqO2ww3*Sp2AQsub0$0u0F{GclCgfm|&k1P)Ar@wIS( znuFnu%TDFE+Hd+(YeJWKMrDkjpE0Zv|5&ovP_LU@86p0Z%iePZb7K1kj|XFmI-?AO z>yRV5MkO&h-){qdHa&+2t25La_ld{p%er5Wo9}j}zkPMq!5J!J$q5PvwWtFYWd^IO z4EBALWCPA}`P==X?P zos6$0Os3(Kx(e_%HzOS}b~lkBaJJah*f!dhu&i$6H=HlnvU6P>>qXOVb?&!Rc~DXN z$t~ZAFVA5ie`9R^q~L6+1+7(Ua`zO^JFz*{sAk0NEYg5m(V=GBCemixqAh6~7ZeAv zLdhT0(o?{b>viLqz$)=}$c6T*VxC4jlIK!5Gf|iMPnPw01^TD(y5k&mCSno!*X4|t zYZWF(?{ZPibw}>dUrxAtQDZ(5(-yNnTNC(#OgLBnv3}2Z<#-j(ynyA9YGeH@2YXh- z3AK=M()hE!4E$w{GrL7wI+hJ?)rN}FSk==vvZ8`QqZdOWs7u~^^F-T5kZi5eJmS*U z@*jnENUrV74y-pz?7o5kMys7@H>Wel&Un^AdatW~B5cRHm)?!-)D<*$8kKc9;-iOJ z6v=8;65O^`AE?_U^Fb+Shq$P%BL|Q>RzuNR)Ah$Biy&OqpjOkw&x`eC5EW?xmc_BPN>U_Su?JnZl$2@KisI*>I(R>X zT8(+M5YiUItG&zwvOFc==nWNrBRvU49!>E7^~My90BjfL$d7cQ#HievjfT3Wo<_F) zh#S)u`=q>Pb6Sxj#m$|LRPl;M){ddtE!-t{tk7r+W$;|TX<3&;%k%CcJYjO1{pGt` z&n@t8jfMur3fo9K}9d zz~=dk1iLdf9nd|8UXHcgBG~g;m`?BT`kW}NIOm${gsoQm#zIXe*Bi@S2 zZtHV~%-Uuw)!d4lXwk?c7Wt*ucqo+GuD@m3Q)yJyXWgWH1)rPNXIhMjYo6Y9(|c=N zrP!kviNW9WtdBMrQyVG~)j5sDP7VEzWuRoapKTjqTooSS8lq(t-Vup7I&gx68#fUr zMxx$lGLCU>HDNiF zF;a<>l^ThiI&LhS*H+%X-|?)^AwQ>KAvf520;%MD;yBSVZ7a+J_1k zizgNz>yHsXqiH*6W ziN+EP(UW?`V;kn0j$d?!-rzWWuaC#%IQ9lnw$^w)<9kH%u5{ZT?FoH|*I%We*r?9o z@*2+}4mkH_m2U!5krZ_HmmEQP&1?@Wy>^#yzt`2OE=4IOj2AqMGHv)))`t^iW_! zb+L}jG-mxOl;Dax6yzwWINcAhi7YTsl?ar<#rExQjS4J26yZ&-o;-F9wdl3TY@|F8;~8Qccu`Jk)C{G%IULTZc|zM z5|Px{WcTM?U#(+3O!Tf}S!V6~oaY#X4Ov_>W9P5(W{M7&goar@KQsJlJf^`nxaa+N zesM6nNhgPWQ%B-tGwx7aGj{oR>$2t1WYUc7n|*h`N*kws71xtr8kD}IqNOgh$+w=7%F9;C#iMA}$f-^4dp-Ef{c(JVBuOzYT8 zAbY8%E-WFtFEEt~<15R;u+OrQ4}!NYE5Yn|6OYLx#n%rcaCZ}971wW0G;n_bC9E|W z{+B$z>&3J_4Cf~+7C5#sIoF-yI2O%iw_ZE2G3Hr{8Meq<%20DOR-Xyt8*nOSzkS77 zak+W0G2a{fxquyFtnl@@630JWRF=2503GFXl57WW<_DomntE6J$~UDP2&YI9kx-|x z6XkHB@3=s_tzn8x{2?Bb`}&rQH# zH1Tl4n(@?h{0rKX=+d!FZo;zpi-(Qes<=T7F5z7TzSb|MdfN4M6QcJwl%zvAsh#e6 zJEL1`wkPKgt>G8Mwrz4^Pq~q;vo}+wKV$D@6`Z|%-zQi%QaHb9wRH21=}^UfxjGD0 zGbs%OjITWoeKr&>5ow4f)b4YpbTf*)fbW}|<{qaFCXSR{H~W1i|HGWW_`?PqT6Dvr z&t;`?p}rq1T;o-|P}cB(;7BCfYQ$5mfY+lzmWQEqQf5Ri@}$8!mN>6_?| zH+uV|D$h=ne>0<4`bdxC#T{_;dNRg%`OTx-W#hLW}I& zXBSD*Nh?|@Umu`My@frxS&ico3M9U-t-cx=eYgOI$!}=HcNWOk7y^D+WnM!wbBO6w$MnVth5xTHJKPU*>;zjcFM*kR+|16*)qV)lT6SM*TL*%zUT`Y)E2!!35YZ2Em7 z*QDb0F)aB~2~5vvBii}RrQ+4yjN<1FY-MhwdhOg#UXL91Fa%g%l zeu7wFflIG3`683O$@z)7(+{v`s8f%bIr+M5IF47CkT%ow9#v;Lu%;uxr)A*req|Fg zUZXMdUh2s8v}NlNibG+E^d8;LD%yJYcleyO3>x#!r8ncq3LX|UH{nCKA~4+|U+k$2 z4H7<|vDdUnzh~cFmbG-?nQq>#F)v6iTnFBvVfIT4t>gu=d_4w*)y7;pB8_%2nQhBG z#=S_O#z<>5bcQT_zKo!9`+mcfQ%tQX4KP z^1-N>X&#@V8D!ThJ3dWO>L=x?Q|eKis-d{d|LMq8{pJeAG7h`Bu9wTA0G1B3#GJ{w zNxQNXuNUm7>T|=j)5kVcU6eGe&n2-jC$1#u?9l@)0!Q!V7>m85m{<>#aigB*5WY57 zI2v<%ddZ(X*5PaIz0zjttWQ0H1~>fK*3^0JcE8Z8Ibwx22FK1-Q#@t&Qq(XQcRa@6 zbngLCqjFBZj$PBJM^PqaZ$_4u$_pRWE}lnc>`X&!>$UP<5`1X`=2bdOAu&-wDtBqy zx1EWUvfnM8?rLcCfmzzrh5~ikcS|=s@L3{_PfzSROeYu|ObD_4`bN4JE!b#RmhxlZ z=#6$q(`M|`vMfqzwjQl4HRy%f+WT;*46t_07u*oq>dN7s-(IRZgXR<-GITqV*?s|N zR*E*`r%ZC-?q_hrFIktXbXkEGYC9Tjr9b#ENkAOuwBlu5>Tqu2iub)^FeN1lJP6F!gm>4?Ip~_h%MOr6^SDPcIth<07H#zqFArSMABspLR-% z^Bq28OPqQicLD{Gw?;c(@jY#IFYTT5ZraP#M<$zgddZ6lUk9B` z(*n&S^2XD$2p45^=vjjcN^OlL7yVTS$c^Z^c@*MwsVlD#TlzYtPkxXT=(y!!e34Uh zHbAyit|_rwc7;%zFT+F@oZ?J7&+^V^zI}y%}-*(L=KNjn< z{=tlO#-@TN`wM^uDH63>S0NypU>~ExLZs0oBvnr?)w-1{BKNG|AvN83G~T~+cD0M` zk<&!&ZJa~mXIk7PZ@)e?urX|mcQe!~Fphsk3(?}W9NT>Iv^kRmem3mn}A#CVM zUQfmaVx}E36giCIosK^ga^Ki&pi0feEf;vG8BL#gM#^;9?tD5_M_4wxs!CvY2-*E* z>)T$pFTxeNS@hrzn&Pio^O41`u6`75)tl$G=-BeU zJb&?`+Fq6L$gZSm-Du<7D$`EySm8~1S*yeQXhSS2hAg7;;QVS1Z{TUO3 zCt7r8-fX!6#>U6*Nb^%k!T5~28Cv&A62;5}NAJigw%I5>8`EUR``Qt@vQwJwJP9jT zn|Y3=76Y}7;RiqSqvTL+AXtqE7J&1_)2b^uT;=11=p4|@T7qbcU(7B1`s~9-QBD`H zwMr@-Z~Fn3ZSLI5Z$=fQID!d&c}$~1%Plt-ATckx#o)rqmUm~0df&Kt=QTk<&$TmO z{_9~x)>LXVb$g_kTjomR#2s93y9Q0KX> #j2!GiE1{B5^7Seck+C6w$#GeN={!! z?`HKqXETg^v`w(h8Xbf~wZ=$Xm_|?3fxk5=z>6dIVX{`e`r1OG$5y9mb%o%(Pvg^# z5OuTKV|XlPzj@J`isf_-wt7xuo;p;a_e{+XE&#I9+%&z=z)6B7!yhK5lEyuAJSjId z);33jI>Q~y))&1TX45-!VnLsiTV=D!lY4x|+HUM@aoWt}Ca;Xaj@pAdiC4e5hoClW z(^H%L=+iN1b2XZIxQ@!YH|>ITE-#hIng{_g8JWoL_x z_KH^vixYPjs!}s6P^28R$z>Zs9g4j>O?~s9*6>~h+Gvge20^|}&VDXR9h1rA&HaW( z9a(~PbDcTHf5v1K$xZn32^bC^9_eIiwiW7R8OK4q|GH#!Y_)ZC6TjV`P`sNdrB`;Q z-!(Yt4HJ6#%lhw^`*X4e-;jz^-^gNOspKnNSft^WdQee0K-$nbiyFTTWS>-%m zLgmF(>hM+!>noHPtI%C(D~)sTb23!QH~D;l%7>UzE$(JD3uo%eqUuSkXQ)IetwqVA z__CUV`@J-IH<1L}>5}%ZJD=7>ybH9QQz7BJ^|?R?-yrN~l>B^CMv5*ZB7-HYAO|g% z@^n1u1?A=p&%OZMqH)%|5@`_0>!_)Z0>;%mPGcTa+Na_E>c$sfz)UBeb%AJ+)U$l- z#5EOiM)^)uG~q=_Q5Ej}G13t-cIu5AP7*DW#JBDAcQa}t#U-V2chJBsWjU{GUqz;( zmCAN(`n1-i!5dCgx`IlLFlKp8ZgHWXls-HwEnx+-|L{UfH&%6Y7fa6AO){W?Qf3Eh z<|!!Aj!|dC(mQ4-(*HFsFnYbVy%4+un~coXVpT>g=i%2g18$FPP;}%Ea}j|B`w`q( zi%*9TZl8rjQ38wUJd|`(WBRFoGDyo8qaxf=)9 zjLPr--Fdj7%l3uUc~m>cWwWk#4saVr#2LMT1`)iUWgH^Xqz8U*@|7)q1n;K;gI#T6p2U5ah3~ywF2i>fdyF$n)*M{}Dlf%Z>_Mc9Mf*6y&}o#^UIa zz}HPKlbg4Qs{9xj5S3hOHDL8Z_W{=mTxbrcn~P`t{JuOd3K%-I`Yd;~7RI^?BBpI> z1`bx+(QG}HyZVy&+fV+mGSIqHBydW5M8A>z-p77mf=lboNS}Ldo?Uwp@#PUQ>lDlz zph|Q)0*a?lW?Fg=h=GP-SdEuwdkaja63_h{xqjx$%~4Ag`4OtWeZo&F=DB&~Y7%=w}Fm)A}S zxAA3nSdJ=k!kVK&HTCvPFz1~Yi(5pUyeOUI@|WcKmgi#4F~T_#+8oK;S@2xU zmSlOld)fXCZ0sai;=B}o_id+bE9lqs{`0rx%Mf0v^pXF7Yw5?7nEU~pQ9d?7e|P5RBLANr{+^k?m)Za8d!`Mj zYnO5Syhz-eQ|A_(Yeh1QqN+>Q1--08<$F6Qu8DZj)ZA@0eQC^u-aUeH z@wZbe>8|^>dX!G2Y3`X6nYaf;i`KlN_B5C;)aq-f4I_bF?@rfeysr@`;JAs~T!(|T zQf#?Cd?L7;CKvL71Vxl0`R#=%tetm)led1W9Vm4+vOy>kn8{f~esh;3+54tSwwb|q zdeuN8$nuoUGy;T--#Fm{{DjK4pD5jJNUdN_x6lyRxwjM$G zqcKbPQ~nUR=tpf7M#L7kYiZ;$!A)*Ehw!18!N$eN831#m-#*jQc7Js3e*eMStMhnK zJ-X{w`utx7fa#3x)ZQ@6PV6jx=YStQ%npQhYE=M*(-=5rMb6lg{_Q9F)z2LKW+#ub zzt-_$C~96BJFs*5q0`aB--blKT{*a%f_SmnW~X?F;%e`0fTMgHEjwoqr6*z@@X_i` zCWe&9wVq63aaX?$vEcCOb`?Wt*`v~xj_{3F%rU0{O5vEdkqSDpw@{O`GUFza(C6Il z@5+YqH{78@`U=}@#gg2sX6)lw%^T{s$dlfS2}s}yBNW``RJIvOzFVBS6E&HY8!MiYa-MGX-La>&?XZltXS{F{ff!4~IGWjIV66F~ixEg1 znCcc+)Q0Qw1X`5Fh;H1`dTr%TrjrkK1{?JJylpay{2t*YeS5&imx!5e)5a%Te)k&( zUUD>LO2x<0(HYU(sxX~+c-1|E{MvBMRi@}vpH*iC7~&=n`?Um0`l=&W8;&Q&x;dEv z;&K7Vo;w>3D&9KG^HfZL8CU37b;d~lpWFXBBM+&%iku7Ny3UrrTgkbw^PWs#{)yy8 zF+gHYMoEv4oe7PJj!vD7vW@L~ATGFP(A_;rDDVURwEnmJAL9ns@a*$iu|l zZDcXlORd%sG5p1TG2-&YstIBXD-$*Vk)}m}QcVv%|1!g1s}3$pI5E~ z2K6Rh`y&G`^%DgyyJ6;bYoC=D@5=iepjclH^vfYCBz8ts37JmR%>BL3%jLnLIWD6@*0g@C6$81IH)o@Tq+g{N)EFPnxQfyV^9`MSIG5JemHi6xZz` zMs@f-y)Aj}C#B#g(+bMAFY1mzJljIod0}4wM^T(9-kX}2^$~&<2AJ_VpZm<|g4^UinA6&@wQ1r1K(H-f{ob+=0Fa2rN2eMydO#m=@Io_T*;9kGv~%es8Qg^Hp3%bvF`)_K^bnqjdh!`AnY zCt|83Hnwr{s`-ss@S(h9THV&b7361{x=)y6&NE+=w;( zrV1?b5T(4lOYlaoQWas_47hlAQ}p6g$zJ^f8yegjzZt=Q`@qTs*WxR_lK%-;==UE< zxecEDDck2MKV18NxrFSQxc~CzU~(AUGZG2^i?1l3M=tCFH>aQ0 zzxWC(P%^6BivElBUmN}J{vu`oHzt*;yDb0iAN=c>i=6^q(X=;B@;?QO{^!e9+!f#w z1-9+))cgNsJpav`<2;d;-8D-3S=ainP5I})$e#h4zFl)*aLrWo4>cs z-`nQzZS%La`PR&``qeH8g=jA`UQUC4-&k^0;q~M7vM{-})3}}~*Ibior|I(BB;otah z|J<;LM9>JIminHs!wATD0SfckS9>k6w-wDm&wPj*Qd7A9)N^qcpa5_8MRdxYKvGRe zzGSsZvQ=IP)od;KJmKF+LHQs0&GPo(|4nB|@h06HY!aOJhTO7ne!pF>129(U0cQT; zKvaj+f7m@n`B>z~a&+=i7y&OzRdImE>^LaMJ|5d+A0G|X88Ew=&UE`2XjAzLE!H5a zGuc<>;o`a{H{-+G@zOlM`(hKI(88`+>Mkv?|A&+N1TJFV^Ke>hm#s!>kVt|qko0*Y z@t+P5tyu4$cMDb4g+cdIR#7vEV6{P18}$5yDXZ!1HMb!lZo%zp&*}ggIF@i@t%9(F zE|h0PWAwUzX|IT7l>@QXX(-IlSx5<}7j|A{FBD1N{y^q-95|shB74n$EjW1#)Li^@ zx}wG$yL&fLz$+voRp_VYVj4S$KYDy;@XUK$|3}WY||ww|F+cK3^C#Ei8eV zM|$?b%ul?CSlrw4yApKDR<05%DsQ}fY2|(hrb3JQIpkkC4@w4o?myak`aKb>z>xENWBDmF?GpgyVhu)G~s{4fn$`uHw<4i>tDo!rI zv>G-+S+L#qq#W843P<=A^a+C>o-xT_qcPpGlW)8@JrOAyja3L1eqTxMl825^^IR&% zNB3m6f>iv?pB3u}>-$_hO>#jV!tUM91_M{>x+nVc@DCU`@tKt>Qz)yhnqGQtz5w*& z6aX@A>Jd%f7qD?VDEXZZVM;Ztp0ikbY;#&?S3rZ`a;U-qAc+@y9})k0yD@oKh=smv zeHl){XzN!-4w8Y!g;X&iuEoXu-$teV!ex%mAmi?WB8s->$T5*}>%Mvh2QJ$pFIPu+ z3eip>tYuNUv3&)6WCipxuv7yyY!YiPFucl!_>pDHjf27L(}MU7*`-b3Fyi6T_=hEO z*gct`lK7d4?XS!uKSxIW)MEy%RN=${WUX-R>x|!=l1qm}K`fp0J!iWUSqj=2bu~Zq zgB@z++-twhYfBARtJlibN&^~;hCJDZU^X}l)8%4C5YAow-okIOT(^q5(9nEepatboLn- zkKgVvJbg}H^Nd>YvA)JWF94zdaew^V+GBzJNXjG$El11j!ad2?ohl$NacBut^Bqw#G*;x`(-R0>kvZT`0~m0-S1gO5Pa4Q zrG6VvlN*lddeq`;QDHca)6LB9F}V=A3&&Ok9HMkr8!q3hjdf93Qc`{7q#f7CR)j+H>3>V%A}SH&OF1hfKj4gex!~lm1&R zG37nRi?UH&W6fOXje4_Pp_VJ9ll{f^`5A4Zp;~ro`9(mS!3~tAA$7Vxp#i_S8-ti% zd$;xtaX&M;njZqrps0D`F+>0D()4s>S!lzun3|>Gh1Q8Nk-X>R)y?D=k0id$CoanG z!nQP-ck&OB6C-VJC_;Hb`zoNa1Vkz=f$-U&*A5P~0LQdn$zFLDcdN2wHQ5-+c8TlX zcPC7T$&I)u=Mk&Lw%;|{_f!K?RO6ydWv)A>6nL2IsZZpcmv7=tj1tOgGqg=d1f*B+ z)#~?-*GRZVA%`zO{~|92mZT}weEsr?nQhvOtL&xlY8O-Fb7w?v6)+Mm*+MG6Rz=cV zF=zY~kCXm|yx^ajbMjosA)3IPhw!uKApyHyLgFei0{qA!I(>Kc4j3HQW1v>ZuwpqH ztm+F?t($(X5g0+4&gj-CK$tSex>zAvh-E;9VAc=p)ih8cXS6-cuv;?#IgAZ%P>t2E z{|DKwmT;8fX4P0wzO7+&&@XWySQQwXOrZqx_5sdstD@&bSSzoQWUmRVioMm7ckXh6 zX>(A1yX00|&H@oeL?LYFG~i#JL6*#LP{l(}mvZo)sWBXH{HVr>bA0V-DLMIw%3aU8HL z^7Yg9j_=S&&K&t`3jvjoA)L@*i_(3*OXhxiZOr{qdq5WYk_D(=e=Ab>N$RUUx?!q{b~P8 z&TDgY&kmCLg1S&%Rzdr(junZ=t8xRLs>c(P_7vOvojD zPcKvaATJ2!i$j>(cIB~fgaE&Ddo0)-!gBJog85p)w1VR9;n!)i%~?J6f0YB+m&Na>}T_ z?4eh`FZy3^KgbonR>{LF^c z9msTG{t8;OpM|ojjqp*xIY|*0O|5~{vsO>~G4VKd7*+%BS|uZBux-a*5gPRZwF zQ0~d0MB8?}TJ~w;QD5C9;CCy_-S-z4QNw-{qd}1geHEL5!V&+bcAlr8>d2I1o#!*7 za`KxldOz^Z$9agUZflQ<6cHEl6QzKLwoSY3D-zoPFzn19SVP1-h&26JiIT%2kmVPi zq44>@sfmnfg*>9$viLYcM)%K!OGK?c2&e&Hz>5o$`u0K1$uK0LkJEOy#2)g=6nCe1 z7uD1VNn^1wnUUSCNDkh()-8C&-3k%%X1k!n0Sm3&YChRH{?L1RUP(DqqZ5y{ z7OykP6Hq>GS|=FQ2(fqfl<>YCGV+%0h>U`wobV}NnrssHO?ztbSh>W9A@rbq0wVWd_lbq+qS zXqAY1qv*U?_Nqvx2`GPmKi2L?5I;Xk6B29*stbnhsyNL!6y))E+$m_oo7#gGtqX9+ zF!#SL+xwoVjl3?`Wr}dK^N@^~i%)$+qx2>a##G?2N*0m8jq;VGlk%t3Y+?SDJ1?;Q z5hvp=UJAHKvn8end&Q~W8ohaH&p1_?!#-WPUYwaYj8lP+FMUf-G~!%7cYTMvCrq6rTRXIFcU>5kSk=e0NDZGS?)A|@^dpKoD@voZYR@6nLgY*%|>BHNFw z6;~>Lm;EqdJ50aDRS|vbSCz=yl%EzDX8q?&|IZJ(TwriqnXZ3ZBLC-q^UDv~TSnA~ z2A>`srs&rO_@BNFWJ9R)z5D?`m2CdUb{$XR2G@YM~GW&ba z{C%4J-*PmEWmQe3{ueVdam78eY?BrAiZ53V@5N1WjG9@-lH019MR$cDpm7?Bvw$*G zd|_992fxv~UluTq6x=NPv-vlXON>|~_@CyjU3(w*GY%HD9i2FKXHWyAuQxCKS6fKUxMre=wO!rfcI8G4UdAZ`EtqEg(B}C~}(+@+qI_ShR`&<%f80TP7kd!nb zb-J~!mv=J;b)JTBGLG;MrdB#fY;$blBC6)!FE~s zLi-fWLdVbla#=u)k@IUWeus~**DR-&0tr~0hU>34y|4hnPT~M;CE4WX3wlga_Z)MC z3&<>1T+JWF6bs~W_aN>>olx5Yx4Ccjt3?u=U$)m*28#SR_urvJS|bm>)3f^?GZ_+e z{5gmaZ`~1hP=*L_f}=8QxQe(S-)&&rKBy&^T;^D7e#o4k*A8b!gf5p9UCh@u(bszm54AuvFR(0Yj3pb@8uy zQ=+2Z??bM`LZku0ANZ0!r!P>%`Vzub*N~7q5;JKK-QCWD*s6FD|G>j|A9bVHu{A~O z5>djW9O!LGv1k1;1Bki{(|a*sh<4?eLtJ`KI}H&BLO5;_ZI9*-EE0u7aHsW1-Mt>@ zdT^+P1pi!jK5T;`J+3yxZku^g5Pm=LSlCK91=Ac~1e>MYw$`>*?FYQ4j)#O62ye0m zoTgnRg8kj z*XLAKs8UW`ZET~T2SK*lsh~3NHkBtMK-WQ!-DtGPPN4&OPv9VNiF2y}sgC8M&Yh_zWvN3n65$b^X~RFy&RGJ1ggg_eR@O_na5)p zh*q;%a+ZwqAVJjI_4w0KjS&Xe)DT!ZufY$}pYH+HsqTFuT-+DQ;SD32@IrvOim4t+G;>p>>L3p?^1g*0?N{DV~2t^M39>JZ@YC$5&E$~m=Y;+VY?20 zZMh5Avc*bGKZp|rLN^rI0A5%YHLb9rVwI1Cz|3hhitgPvk?@C@E0ulFryqeQYNbKI zNqa6U%Ju-AEHG7eI5vZ>-|c-tNDWyO3_QnErlk^N*69bIo>v0zB_(sTjlv9ecZYj? z*Jxg35OZm+dLLPw0#T(e@n<5l)7<%8@`oj7N-G9rUS7Po3oK2dm0fz?bj~M=v#Nbd zJGKpIa#7vM6-MPp{34FIhPk=j2>%o|&8Fz(P z{aEjDoN(TD-nzP{2hOTpK7O~7-yemXv>8M$obAfisb<87Y%-o)xrB006LWTYYSzrx zO&9SGkvv9sM4*}0b@}rb2xlDuFt9eDfXCUdV<9|Sn+>92%%4n}0|s9ZTXxM}lZm>< z4d?OZi-ZkqkP~_%w1x8L_FX|@YXg42tw8uy%;3Q#mJd_*&3G>-O>so)YxOd<> zlTNb~93y#KdV4IwXMo8%IKcO3U*b_n86Og|3dDj={d15`W{I+@W9imrtoES^J(64s zB8&O&Q?HrXMQyIl<8>q6BSxruC9`x@-A)7_t}xGqU%c}}Vw_Qa3l3mYtWW!Ecttzo z;8GH``kgbdG~`85BB4`>>Fv2!h>m1wotcfJI7+1J_ZmS@zk55XRw|Kirg!wa1$ZL% zfY=Gbqm;noZH7UDfv2xMzTXAXj~@Xy92i4CJPq{W3@SRVD`rJDhKnP$Um!>$Gr$S? z?8zZRoh{%I(At6hYc#$T&a$$z$P$_RYs{En%-nj^Wsp|YqD1^y3vGjt;~XNl*2z0P znm+YxaR+M`H+!A0q6$~fC9l;Oer5le$huTm9(Uu}M<4!y(?vFuCv@e9R5Z*_4L~+k zt?qN4X=e-M2K6_P+++Rbb5cZ8WTqEFyaq^1Ms_fAu>(UH*sZx@%1*T#zf9f*2yo2B zYY6bMBQwcD>_WdG#5%fQr~}P@C{;cl(43I(OFrGpj9lv<<+EmsaSGoY3&7M=27QZg zB0JLkkNc2DKK6pa&EX0n{`LYXhv1>Q7IrM10Qst1*3ea+GbcITm7~%z=H>>aa>aGKMXj`(mYLb#utwRUid017YU7-?KNtYd8!t<3sFsAjtok9TE5Z2qN}I zYO$Q}$77sZyWcI=vB6wLPfqOUas^qMq*vEd>9}xfq;)t&G+ey0m;c(Ez&LjZog4>% z_%+4`WXnmz{JGEvPqyS}z^dnAgGE2CkaO_4$1G6pc0UrJq11@Wqd%vip*mw#`_pB5 zB4&*OG`I76z~bK=Z@OA$NCy$qF91U@53+$wJE*BGf0p=?meXXC%jtUZMJK}C7Ek~w zUNdAN7O~EE`p#uSzOz;Y()=rzot7umR(F?iaxWqu6&Z(&PCn%+SKtaveyNIV#l5M) zz`ls|{BuymN#&#z{7v3$U#vt9$(B$=d&QpWl4SXKYg(&Fs3C9Gd3%VF6l&a2({&GV z@^5q2e?{|g?%VIc2C<^N2YRWW=H1qc&l5&!13y8aw=;=9OoqRo#|&y^Yd5c*Ut>v( zYJKQ>B#BnY;qf{UaDRk?2RbA=9ktf4`O$aI0upmF**-VT<{_5)C0JGa&VXD54-1m^ zxT7`XBBhviGfKBqcN1C1kiIEP7zz1a1_Z`88Rnqs=)6c#)(d;+-=*Lu&GA*OB7W4) z4v-UZS0G*2G-02!PKJ*+Ahh4mEH!n4l&4cs-@0B>lfC$d5r?((rW`O2zhm790V@dn z;2L02XcB~>PWBVB$3TYq)7claCy=86n8Tvqfuywu0#GP{UG*2x`rHwF;PPAR=VYcW znk)LZ?>W!r1)dWEc)*Vi+#o=N=?s)H((8lb#J%A zc_!Lt*K6+6>hNuaG1tfl|KI{Z#+#Aky3Haq z=7jJ?m^1aTdL%8sW0-&GFEO0<`4xJ#-(8A-o*55A%aODof-#xNM|g{n12<=0@5vV= z%8Q^ofX&@xjr?0t?Tu7z%3sgO0@E zKq5}A$;|HYPGrwhF=L`Ru(QXVGhX4q%cIh5dHc=vYkMPR`l+o$*R?&l1O+K(Ig-~K zBJu-IyPSc1i7Y^y$$5E#`~W2NkW#2ODY^D&JbKslMBh53OD}%#M-mK$Q480c;U;xc zXl@g-J`t-t;nw1|F_A&eR=l_cV>M%>z*rx4B1Ut}AZcRU344LPcEM>|;C0$g%z zVXw1J??6C-J$f#_mreWqY@Y;GlY(q86WATAo;RhjWV7ivoY%&el1hg>dYi`YeqkR~ zas)%fnkdt=b^Y@IHMB}3vfdgLInUoc63XM_6W%-I3kOWOxCo7=N8>KcF%Cn+; zh#^}ORz+I!Yq$J3TE9DAKM#Z-59x=LG@s~ikDAX&^6qS$dc?+|YLtrD$hTFMJH!sr zSsk}*_1c_6Ikf_?Jg72G){KCk3*^6hn9_`UgF27ctn-(%n|l9P0ZpSgu_L)L0=6cL zRQoT02eFD8<7E4B*8MQzKb(x8R&MWsRXNT&fx-;d?o%eV1n!_`Vm>4lF~-ArRloe> z|N4y|{rlENu~|~3Kc48%T(|1UHO|XSe-4UuT8!GJL%h}h;SZRT1__H@IfkZjw!I1t zXq*yiA>w4F@b|pjfA?;rM<%LqGy4LbGMTOoDqKYcvk&$t)~3gs{dWWSvjg8QyU(rQ zr6foe^+7xf!l3%|^MCi?E<{vE&uARhE2J)^fFdHf`2Vr@mSIt~f7_@csH6=dEhR_@ zNHZ|XtwL60Q;$T38(VGUCuUels}{K#x=lQrWeImfi2u{39aZyG$roO2Dt$gLzcpM0f}*15+a>&~~})1Sv(^?1*X`=(^U!^T0}HITz3=KA%l zTfg;K?g%|1@Ng>y8H^v+9}NtdkPnX(?oUN&8c?7Oyl$7ygpt+)94eIo2y!?851crR zOF6v%^53j%P+B~vCm?8dtnL;09;Ggo3}^e~-KTlQeUL1>hXx$uj5=!4ivMe*S`ed!8~CHu8|GGJ&G* z{Opk(Oe@M?=ZyZlGvN?8zI(jyPXQ?N=~Ts7%HlC+hL0~6VRAMyN&gV=-QrKkSdDu= zoBbo`btwT3)G5Q-*cD^cE8?4?4*SyWh%8&OIjs}~n<0*~tKEk>ATw{r=6pd|oae>}O-%_~z=;p5pyhmhvq zFN#%EHzBo_ggf=TyDb}TThcfx*yr+II!WmJudNFrt-gcTT6lATL&NUxfypXqF+9%- zju}PdhMr>Qh>&pTk8|zPQ~0iSoUag-89P5KJn1gMJSrxkoTydnCSeO#D1G-lpTW#T zKkzASkALe?@rz-I!6uY`Xg<%t^_SspPb5Mh{t;poFaRzbXU_A9 zGw*DX-{ztc=-rT@hWNLWSp`KfZvn6YEw;?8%%Q*`P-3m<_BblTF9zl_81rX37F5Lx2a*NtY1Q>)n zNDsTE(#hav)P>FgkEQSS<{{)7acKxoVkPWMlML|1C3U(q-h#(k*$A3t%Yesj_J7?3 zbjtFZ14~x=J_MWlCx2)XAE|P6MlGoq;hFk_co!w|e>{ZLIHsKpl7{GQBhKYL89TZy51uVya z5BHbi!)#1KH0tBZTrnDL3)28^h^!!b>SGw;jdc^TW3+Ro$q>9v;Hkay1uwC8EWkLuLGALL^$_>1iPjA)$V+Uv;op6NoCV0E=M~l_g2G3oZoeewG%n+PmER`$A5Q@2$ z@1|@jm;nO;gCcaQ6?m#0L46f(OV(M{-trG6W4<(!z#BX1cia5E0TK!<`uhR(toJ3S zUG>)bA7TJZ{({%$*E@|Q@0A#(kQ+N8^epSn;W2P-k_ZgqB|zVZ#0<-5x{H^EWG`XW zc5A@Mb3QXcIj?%EFHz3*<9z5{+dBJ-SV81WXs~-$)wRR~CH0E%7S3V+>>l#Y?tS2q zxF+Vx85#mp50g>Z3;0f=p%%uV*ntb(QwHCKy;DbCFIr0X1$>QF@bnzSxs|&)Ycw~A zi6dk`Y`y3Ih;FizKk!s#wt4?8U`7O-aP_+?1CRv>4+JyN^>NOUa5fmM=95DWbMN=0 zPyO-i{be5keBjSF_qP{)H2@aq2m`^PKOVq8&rA3sRJ>g&bcgvbZ2j*Sscb>-3wQF+ zueJQg4_3;cU5iEO7yjBbMwUUS*sZ`rhyK1@2+%HTuvY%Me(88mCCFctNf93UdxLcl zf+8Vo-vfG;zq}}E39QV%1ExcNa^HU65TMHrH4j>=3H}Px{F)X2rw^_Ws%q)P;X{Al zE-dhh`V3;=o{fU(<_vp~5gi64twq7q81|z$H%%dm>7g@`ElXu{`}#!+)>WmhXur zD(_`K(G9$R5t@N=m#a+c<9Z~pD87FUK25q!v z<#1f`MF0{9G*2cO83NFyXXW50!u-QBL%o6IC&poW#fLNb>eJr|rax_qEf9^1#L|rc zO#bD}sHweABklp#JgD4Dd!ZaZoA$@8;@#21(g0g=d0MN1SxFQ|TiQx7zOIa?QW9AR zX;v8u+R&G1%(76VM7*t-Z}IcjH1w8S-?|=!NbtN)R-@UQvMENuJ8hZ0)Z*=t;MDL=Qu-VWe)@T_LD z_pUM-o8DW)L>3PLo^p&ox+d&Mq?{&u&f7#k9-F*fx#oRtWl4s}oo15feI*;_6OV|; z>!j$;W}o-8E0CqsG5zeYM)a!VbPx4I!9a2ywt<=rmnl|9T%J~{G$e3$&#&$+D_RufY>l{C^D`i<+XO}Uj~Xl$?F0;bY0}y%$gbeg zb6u_-?@UWP7pZ4847}?;o4TL!ijLcdoDGlj^iuidynvvJ05TMD}G}h{`)-=ISlz#c6S6h@ZP)8 zMmQoWxfU9PhD|zE(+uavK~MU4=|c)K$K!J;{L<=UxZzf#ts@i)0b#+gOSm!W*+2vhj2>kFdz;fw;37&7H{G1#u zEX=Tl-(Wt$nqho*a{iV63qsNCiUu7ficOybY2|=kG+h{>Q?SVPRzbw&_Vsp6L*g`p z{Z5urr@%@qIgsa+S;Kuf#m**c{K=TsP9k1t(5u{LIhy=c8fFsHuBo3O;?};QTH1`Q zR?w&)WO{Zm&*pLlJD|dYDr%9y^pQT+`oe%+p=;%HXVV?%g;g5&Xmz_hnD>0h@#N z;wG%B=hxeQE!oOPSWH5K@IF=z4qfvtfdZAP4IJ1{=hOOiO=|h;M_CQMXVOhaj>dM< zwuotes*!?HcOpBFrNfIbwxy1bko)l~?SFYn5c_=qtAp9D4m{ zXVnue82qlvmVCMx6mU+%ps3!~7;YRV*+RJQgRNE$+cDGB;^XRaUp1{7zcLigrbO0h z>J|H}8y83yUEvEI7e#F=J@XqM=t@|x1+2FT`)P4>@0sv)%b1MG2XqEonw&J2SEDT! zBfsfA9?001YN<(6Vb^`5d9QzOa-Vclc81-|%g;u{cD?q>(C07Bf%NxEIV*N+)(2jf z&`{*i1@0Ss3EZDh8N!R~Ef0t#Q`dcH5{cO1v?f=g^8AIV_+^U!*B_7)MCPpml~Z?E zP6iJ8?&A=(X^GynN)r2DayxR#Yu_A1-YYt~T*1et&43WyueB2|j=}~h+&2-cx9wu8 zG~o@yOKfGH4D7)jd~RfuojBPa&aOyuB$DQQq_&aDJA zm-#MM?Y=%KOKp>~{-8;dcCw-PnNx_aXWiveJ-qq}RXk-L)h=$4=p9g%?7h1x!-Ulu zaL@sYF14o*jy=)}7HM;%jEOU-Oo%!}hziXKPl#u)R4rP29# zluia6vBqCo~Qrum?oO_-xL~5t`~Dz z{>%&%d5&MU2tbV*Eg$G8pdIR!{fcj5Z5M9j&FVS(JC1pup>N-qEB)Lzu|r{bUvTLfJK;oMLn1T^F&1I7e_+tni7 zttIYk>&~ZHZlm1$i}+wUuZI=OSlw@iP#sgdNl3Wkpg9^AhEBzk4H;uhwO0DeXO6g! z(ocj=;Gh&HeCZz5sF*A2~%^`I2g6B z*wN8=mpZ1``25x)trS-NTEj*I^6FNyJi3SYwRrPbA8f9j`MWv1p1bSm4XPt-hCZwQ zru_>p$G;|KiL=b#;X8V}YGa{$d{p7CQQXlSd{C(JArmfZu}wpBl&Qzp4WD!aDq!X0 zVs~r97#nr7tN4jSpM8AisAHD((P)+7ptSytOUs4=>gyne#qSuM`}OSwzr;rfyDGl@ z#s0U_3m?~GAKg@x}VW>#2r z6_|PpfK#JpO-f%>n;sEev0O}jI1FQ-u(vYkI9ZtKhy#1rV_t40M-hz3S?oy3aKCX3 zHv4g{P2>E>^5x@W-UoL|XZvEqwuW%I5}xa|Qk{~xdm^yz$tlQwAmbi&7}+P~IqQq+ z%&GavtfW8*&{QxC`bxa#-;V4(T`;mkHw9l@Jj!jCH_0QbKMDlmC^3c8O639l1>;k) z;Wg0Nf|1og!FuIqRnN3fo^`gl-!ar~NA>U`FAQTp zUZJLRe7S1pW`U(xzQciH$#K5pt`pm#jeEF<_*HX0UXZmBb=gZtMk=aTHCLgA(qF=wAjKDIYZjv4j zW2o3u3bX5NbLin~x-vH3q}F-7pp#D~>Z+r0zwP9v^ulYtcw_r$pxto?co@{u{+$A2 z`$l`PEm=*f1;Hi1AUb!YwV+z#M$6i)V#Nn6;lkuN>ayk;qYi`0FxJ^P7w_|~n06!J z3=EuQsMxX-S;8Oicvj!5X073u(q<~}Ap6rY{(0g}fpP(U_l)<-)PcqHtFS4D_g4$P}n~AJjortZ%V-#pNzOceSN8a44r*Xu!LG&W)t@1nkW~rZMAP zF;~neEVFPoW`1bOby~2G#L-jCHdouX);evd-l#a_U=Jm0kmue&7m#_bPuJ^7?&L1! z(H*K-sePjoy4r2~YQ+bqdCPXN+Fs=f>+^HgVr_0Q%FeacjjpiSn9Fg#^XI_M-ZAdS z;B(%$a)k29az||+scvSKs+Ql*;qhs_B+ZQO-H#4m>+uQ&>%Xe!|NEgoOyCJbG1|^# z4g!7l@b(%F@y@y1c`0ReuQPb7_mB?T&3@$Tyrn8#wZQ9!y&&_aD|rH=2^K+^icHtE zhK-lTwz5;q$17d7c)&e7Hz?>6ThXuu`b|D!6XYVajJ9jl;abo+gqRHh-!h#qyTT(Ms$F)qA=yO4mcF$mYAzZlzGDCc&md1=fS+Yh}YnfdD zeq-2XUE8O5kem{vrmu?H3lj!&)9KVDtj}#8v(Qv8qOQGBV?SSwoY1f{j9y8*;h z^!&cQei@|MNCHaZZ>1^4D@A8{+=+0`I^G5r*SG~HQ}bN!JpYglbX2kW^{Ra7*R|M( z084+0<$MbQ0|YgIS!*ey{`LakS)5}8K_G$)*G~O)82J#g224q=%(UVE!6hUoL3ID{ z^Q7Z{82*2C6!eDh7G#;fHwS_FQZ5Y+iomH|D@ z3B$7PzOFh6g@4}+(1eX;d@Y*pjfL+J>G`#c9hLiWv#Y#cBvU-8LWX`|Jm%cR5DKd65CizHDS9a!&Qx^<{DS|^m>tOdzJ)Vo)A={OeMac^gJVYh z9h_;2kK~1n|Jo&u=g^3~2v8^g*+b+Yr4}fkf{Jp6^qWg<+iF3AbCF~_#eT6kE9S?{ z_CK{!`Tk0<6BC@#4=l9(cAINSqmXn_38khX0l~9*Dh$S(jLxOFF-klshdFc&!^%o~vVt$PloQY^~@3Lg23}ybg zDG)v)8bmz5Wj&?yLeArde`lqgRJo$^ZDXCl`{D{L^ZAY1zn2C__&z}=*bZt-!~%gLb7XMbGTlINTW|4C ziy3Y}u!vf=>=g1mCudAR`1N`c3>}8GQKO3DlubGcSg5MpO@}Tds{_@EL8ea$m|KWq zOPS9$*8!P70gJf8r6WHF2f(4vG;L{m`&Md}73Z-Nykrbr7QDz^V10vg{9Cjf@ZTIm zdM`&0U+%5F{W_RXj`_>3+64;)lL`=y0A2F>*`-rR@0@GdFuKphw&`GKmtL(^nUx132 zb8MlVJBiAJBpIm_KbMXeOK7W)3Xi3j`f-{wj4H1PP$1AX zT;AWh4<@J+Dtjs{_0WwML;!^ZCdB!aXTP5np_G(mxVp1xFh5w6a9sTjTo?Vgur2#s z`lBDE2}$%sgJl*@IqD%Y{$gh?J-?4pwv1c!hZm6)x<0>!nKod33D|w0*;{rLg*~gP zYmfplH*;F<{dU1s?jrkqU!w=h2&kmpKor%hBdGnI=Y&n z@vQ*aR7QDk@j~}tS=miBPXbraSy#qaS6%-}eGf7jDoI5cd20-b7NPd3OHl3G1E&^< ztN#1O0-q<;**%wNxcoAkeBlY4m9)_1`K{K4f2||H3zdRk+mhS#R(~gs>^feGi)a`y zIIf-nmj`{9T)Lx5oC6*sos?o7YOTyyIIT?rw&N6);uF8;!Hv`RmR@-e@Lo9Cpqisd%Kq?ldwLG_wC+19TsL zToIaaqMY9Sm~k9Mi$U;B;$-x3NU<A_~rZJQ;gaTk8N)~(Le$0ZCkjUN2pmmNvZ}mRELd?}Cx6YAf0@h90c~F)%$eDfP zfU;7z#Lg(ebB0)+x6(!HE|Q3o#JItAqIYGQ%J7kFIJ>6$z*#nTq$^sV9n?M+G#_=u zfg3jam~dRao}`P;2G`EE>#;|smty|8afpKu%)gdPz_z1aC9SX?%OD}snONZ5LEOo^ zr04gPRd?50-%rlopb4h2wh7c0uNS-QtgY##d1L(tonew_$3)e4Z`1a7N|ibMXk3yB zUwS8Yz3`eY2aJ8*^~pbAaL0S@e*kiN3>2bMp(_9#)(5v$N5XB1>kN$&yw{qIJ|$f< z%ZkGMqrDSUwtk8^m=|Otm>`G zC*qvmofNx0&?qY-H;Yn|+2Ax8l0zw-m9O1Q)5>3C1=2t|)wn{o9!rp!Ww$s@JFN4@ zIQz1j@(rl0&?)NsRQ^?R?2Nql~)T@tl9yBx)o4y<)0m$Fagz? z=K^NY$Wj&oH}tj`pJu;z!E2*0PDl9BPefZ)>BU*$jE?}_B7*@vjBVJv^00t;?Qnre4?bM|$A2}ik8CbGfno=5s356_Hq}Chu98lL1KxUMjU*|3+J{0-Y)FW3{+og6X4t_ zDFkZ3C0My1#)ES!{{yJ9pbl8}b17XKbB|aQ`|LE|QgZXY>CVDpG(3fP+u)8R_Zta9 zIbIdwv5cp8FgM50KmnApz?-;GH|aF!b2jL6VL7+$tt2giqJ6!z&zp@ta(NU?JE!lJ zz)?$ToQyo|fa+}L@y89ReJT8g-=gF#8*sqP6MjdomRKM__8(MV0FDNx{atjm(gWrb zVsl9jpybf}K?0#npKoi1ksxuMGk-$tD(!P*8-$p(OHaLtej>GWCcAQ1;p2;xni2*A zQIfj*(;jy85u!4!j`2--QDMN6OEM?AFv7cIaHn7~)3}#gWeI5CbU7bzL}q#upH)g4 zMkn4x6L-3jpmUurs=$#tKH1%KyORjj!y)IvF%btxEvu1ehGVDI*AIt5IP8n-Q_?32 zm76rcP^o`M*d~jY2!IW*r}P6LR-lvwTvMhZi>H`x_ITLd34yzk);YTX@`q+on$8hb z#R1Ge=HR!DxYz3CMSwDIkm+RSXJoXqpl|W3HL=gZZw4Xp7(v$1O2d}n4A5>h#SDJI zODfr*-EUlewA&y_51qK+TH_pFBmAD0_^eh^UjoK7N2-eb-1e;(bjTXg2#jRPieNKu zW6X(Ah+*u^ivnd49))NLQ7O7~U@@n|C1fU17t%I{xh)SbK*yjUkHdPqFuS0{*LQEP zqqFs`D~jLCB$W|Oh+gRI6-k-3eOSOxm3@y`0xIm)wrFkN{54PMux8ROa8V1)RCFH@*+eC2`9z%d#9-w02hW1$*E!|MMLKEXVbh zADSF}gk9ElUO5`pn`#UMMLz(@?VIwL(7yB-%$o(muh+ic=&Y;K)=OVau6jwR-&oS` z7-p%oHT&p3rb_I~>VJSo1GoN;K+a0Zdg&_2z&HR(b{zmmsXvc?*&|ZCo7MKB(vUx| z6h2d9P#Jc-=Mfx2$a;RJPGf}-vXdxl2nPdPUvKQdGei;*m0oTxb?fs($T#0H)0AfH ziu{UOMPDAQ3u;etABIrO-yqZ9QOw7CV&Gowip#xLS-xMTp}u{COrv})>AJ-3IO|8m z!(rcV6#1rOzN%~aX-a>Z52%*({Q-=gRmX96izTD0)?VEr2Is8e_r4bu7UkO&>!jbZ z-aId4qgDFl0cj1i8`BBvQ9+5YD=yvbwfCow28bfCR<>#wU{AdHu7Wq#hA8T;v_D;`2~(0o*G)b=J;P?simVs_QL`dX(Pu_51L=@BI!cN(|UZ z1@{_pfMUO#%HKP)CKrWx(j}Jc+TtV60Tx2w_nU~esC@P5162+GjDS+rT0kyA_4Bw4 zpy9#Gq-QOL971XkLgk_kYt8I|Kx?WftcCNpY4_VM`f17i>nBreD#C}uUN9du7y7>-|GN)UOe&g(!_wu+>P|mN`{P6!yEa8vOd3Oky)#pAP{~zF^j-S;4;RPeu^f$8Kzk7-5_koK*a*y!F zuc*VHKagAov4GqJr@sxl1LKbq4q8TZ^})rz8Q34+@rNYA{u6@xPYCW`A-Ml6n?Ekk z|9>o-8cXgAKe+(@i$O#Fe-_1V(>QGQ?;y4|62Slbm#guT5ah{ac@$P&Q54-A&K?K| z4gw%BBR$NkFtYFqX2N$8H+dBl5K>Cuoyk0!wfJ76A!-KxgJ$`=7^6WS)Gk21AUy7oXgKE&(^G7lo! zdfZr(RWdrL6WMnM`C-vr;=ET!<@Gi9O~I0B^$99NFIai|yk9v>nk`B6JA9XP_yL3O zl%;>0P%Ii~#!v&h^5FtPz?M|cY2J0f1dv!0+uDbkJk2T@k!l7x=&cq<(&m4?nvn0W zhT3)s)$wfBTQPC^syBEmi~0-_=;$Iv7&^K)Ig1tea;&D`m|vym?d05witqh&hK`)6mDY-=?4) zsx;O70&5C}dlDLVMg-Ejsh%(FN*%AvV~9*UiT#nNRF8)*e6-!)UX7+NW}nf+_6zt9 zo7X(5Iml&1$2TrhI*zU8M5fLq_GQdlyWp0}m!00*Nhd~n?b>s{-SD`l6A-ZM%q?%d zmj_Do8UTvBHzVKc z7Y)hDdx3DWXWoF=zb2GPdN9w`Na#r=e}da!CTyY}pA-Z$m&Ep)jDr#<5(u>m@f<;X zT(v;e)-WnwxG4oFA&di3H06h^0v3~gTveb*OIA4+@Xk1{B|f9lbGW7j=&uOnI!yUB zQwZJ~dNRj3ef7Vi<5mA7h0wx!65)UVlz27w4WFog*OHE({%V$!)k z9}-t}0AP7kyKyA(CU3S6rmOarNNH?xuLbL#Wz*hg8UT}|%gVf{2SRthXmSkqeN|(2i788h`HhIzHZ04Cc=w#ES+6 z=H%!Zql0Yrm75DKaQ)(b`yaKvxf>&}Um~#`4AsidX+`sCZ{&f1DGAFLloFEB>^oJ9 zmBNRukc%x(`)=n~mFLaj@t{*6L&U|bfVaH(A=aLL5Sr8-P91*=2$Fc}Yypi& z?-Xx#khDl$HJ$q~-~OD8lvR`jRF*yr7z+mqAqfzXiO0-gSF3TGInrZQJ~SzQD<6-A zkEcMA6D*7lQ1$2K>{4UZD&X6c+#WQpd~)pxADmxF8MdiDK)^DIGbvA1lQ>FvEA_(W zp4X-zo|v~{TW5Q~>S4`HEIGOR*!r!~_3S`?={hMYz?@l}yT2yyKVCh`L8fO1&$!ho zv8O8HH+3i4==2yFwC_JWmUFsz*!W@Ynp|s1nZBN=W!Kcn)>DBRUYoro{IxZHQPgsI z$}?iQ|7f|sJXgc(-k*HiKFzUzmpMcd=Doj~Hc-J^y7D!7Sl+C7z^9}ER_=SSSu$?; zYV2%+-+1rJ^%;8*;qA~n7D+DcS7ol{Hy1M?u@)^max|w>u78CJ^T;9*Av)~4cVrxQ zEa7;bCYlLzz-5!a8k)P1FQt?pFSfboFsbMsxVPjdA+a+>YBcUob=*1&<+Oc(M3`ph zITw~V#QlW18BQ+#a+H4E?>y$>gomorA@>1jd?I&QH$syl*=EdUd zg9Cr)Pa1o=kC?S;l*nsR)^fKG{j8m5Joo9LF369;eZfYza6tmSs>fbkkSe5>*2yk` zZqD&h6Yg|f0~MHukX^*v3F>0j2$E~}li&XI-Jm#`Bw$Hec~!m7nm$IUYjXA=?66g?NK67bP3yQnC>5H9tNMUl)Y~jh3_z5@s{;X7O$}58lLIbAZ zE~{KQ;q1y3ez^6-TEKo8L$)(~)h{ZEd79?yzdffoDzVc;Y7|mQAxm(XCVO{zJ;!vc;lR*m znmYRaC{aNx&guH3v-nYX1IN+JLS&)dE3eaP>=3kZC=Kt>5uuJ@$JvMDb>ST+5Q`{V zk;#p>*EDbt_-a$%jV4snJ@<9iJ85e4H<(-M4{JP|drv;OOy zFdpLTwxyEsAs>@yZ1Xi|b|nNJUj>PWbwl9*dERo#GZiR-6=@l8zO~G4s1UE?0ld@@QJw`)J%*6Aj}K|jBan{v-6h$VcHl=DF4x~i-47i z>8LPI;iv24-O5tre8<;7lv~NGXdeinQ9%r|xj;#qZ7B6xD5M#l%3exU(q0_nXjc`_j044cQ=UQd+tsqw?_jh!s<0ios^pi@x;HozAZY{Wqu zoxrAvZWqPg)z0SV6EC)igXoSF;C*q^oKFullHmK(``|o}pD!x17r~X{sWh~A*vEU( zqa=JMSMwSZN%!#BsFm&&y*A@@K9<&+?EFL@unIPb4b z3M-SZAxu|$-}WYYjKVs*(&sb_9U=(GxM7dCa-*fXkC?quP%d6(MMm>6u8hkhP&6hw zD2eQ~nS^;yxH2ELsR}3dlECrKvb0i2-b>8wxz8G&z}_8SKY9hsr5qLqK5W0o+{bE? z%UM^iILGhu54&#Fj#sSbkWm>r=>v*V_nL=WhM(syyqfvkXHDiMv6z+jWT;PCB84tZ^_Zbq&y;qK)LKy&Af$P7@?V6QM-MO6tB48ydr&ySw94q+0((zDRIt=sJ!|7NYF0uJ^-kL+-Ixdcq|X@gddFNm;ZSzz<6lePJc zLDdZfzvR>4=Sb?lG&y&OvXe1LO6cyM)6ind1WH^?I09}gi)WWCyMHZ(I9ItOMJwGy zX2`t(%nEvd>MbAR2B-v*-J5`vk5GB$xWaeIPQZGZsD z28EV`y)nFGvqkX$iQk@-SxZFnu?`oHMXjZjNF7dYxsL`W_HlW$9jR&yC?^n0WTKG9 zeK>N0Tcg^C+pt7*Kb~aQ0uHb z7N^f}gccu3+!xfXp-Um}<@v1EsgdL8lx@l4O{0;-v^jUwPh$H~?mV0H_2p4# zvu;}8TFE+?+63+0Jh+^hx2s$L#@9D|Wfm{aOp6%S9xx-fkelvvNSyh$eZmDv=RPW) zu=8@rm=P#-YXLqLX>&pF^^$k9V*Zpb4f%+^oxt>lAl0$X749wxth$|qDTJz|g~Cp^ z&{+MAoz@h>-=-@GCuEijk_0OVgw!fG($h==VG*j7pl~#eeFdK<6f~wGl1Y66QTpWq zGS@VFno3DPtt-1?wy?@~^pcQp{U}{GJ%x$=OOz7drIk<8Clteg4chSh0du#+3}8ft z@(`?*^-wBxoB_WrQ^uKtK0oK#b6ci;%^i^cvB5MNc<%?&I^7DhSh&W!RFIOeu{P)! zUI%nV4NhN;IU16h!}mcD8|MjZKZ}RVCb4x1Cp7dN>)B8q@{9tR2djQ zDM?y3h;HKc7y`wYV?GO5>-k~)mgMOx8t566o0DZ9GI!gWeED+9FNWG9o+l(HFAWU@ zLpgzdR!FVAZ>96wFHKj$anFC0zrnKR(UWnI)s@vO+VT2ak2VintlEBa=6dayCYMQW zAw#K!?N;iKfqr5WJuZ%;Ir!3z9vcVKvQ-V;j!W*|42TgMkp*`>%D|rBYblOin8mzi z>;2hS`v(6rN)l3@44Ba?BK(SH5n&SR%K`+g_^s;B+VtpAP8ucQQ5W|$p`+eHWWfsE zw4U3zXw3S>`JtORDK6H9CidU_YB>4vI{M*;XVl>vZTNnZ5PwMXh!zljHixuyCG1SX zRBAaEqosUH<_3$bArra=9BA?TNj88+hXN>P&2dRxvM%-6=hNEf8-ns7;F=T7kj7Hm zi7R%pRmbW+9|p|zCyhJz$*VQ%mpvih@v+piAhm^Psj3h^MZ}C0m;>Qg^DJ=NlR1E% zRV2=}PrF%&uXB535hTOpy3VdMz)H`lX^sN?C@OeXtk;+9vY!zEu~{a*ofTVjE;t8r zD&ER1ci#KVdn;Qd#%5h0JzGT@-LQlBr9uWzx=2VN^5s5dYhEKg;t|NbiwX=c#zq?% zRg(i5DiVk=h3&D$%_LR5vEj3b$Y;_PfDzm;!OWFR@0nkgqM`4OS$+=iMFO{j4)oH_ zb4B4A#{hdt?}T|Cdn4{(j~$QP=}}Ym2M~*l`aF|%mzK}(d(>Ae*%xNc$DW6Oy7KsLOT=;vo8_v{ zf-ApL5Koy3@Exe}t7Cc-&xLh(sh7MK`a^1f0AHU?lDYFxN~;%k;Z&cFbIOJ-))hRo ztpmSc(SD4jQ`ml)MUmD?99QF4JW$3`8`B717CBfdDQC@TKR6R85VHAHIuv+tK?5w{ z&H0`-rKb94ZzlD8Q`uI+5GzbpNZ*jNBz14|d#0x6!F~GSjV~Icjd^0~Y4>NvKp128 zMMjT5a2f#~#6toA5Sj|)4E#^8z?!#`uhYq|aX|bN69X;3q+XNg2rQQ$k{<~eY+)lW zeDd)Yw<2I2zi0W4S3RF+NdS8vkUUtk=gc34!ct=5t=2sHbIS2zf?xxoVZy#!14OUp z0dIV`DRCt)kt|!L#XsfmQ7h~0uWbbmYgiRIKzk?z^x}ZD=>@y$1MC%`?)WK9IWZT6H3FAL z$}PI1)0L1N@fHcKs05mMSH`~56zh-oaq@*ET{&YHW$#ZpcRue(MM$3EqE72qGnMZK z#wI~#gkMPtOY<7gc;Kv%@KWHP+OW>4J_kYw@hf%VcyfSEpVk?R@@G=BvZ?eZWVqg!Y z8MtxaCo#DjtN+Q={KI`;+54s&a%r=MQJ>jX4^AKG*u9D*(2<{xj6{ET(B82ki@h9z zq$Gl1>kPnf7_ge{62L?5(!`06F-n%5`@k_UiiNE=G9;u(u~-Al|ML2LG^p#2Y~C1# zy|ut$_gA|qrxEIglAUy#`BQ42RpGqHc6m=?7Rc1n6SWxHXFUbl$JbZ*R*+Q5&b^+}rM-L+N$yh(16+x*b=Pv+^vIvKBAzZ^R z4l?FW*Fi~*_k#YjVyGg$^aN#(>@)n#GSheot`d!irys};kA-_HJMpMjf$?dvXJ-8w zWb1heUx>b<1gN83J2r!>`GeGG%~2`f42^sUYEw+aQjig(+*bwev17mTjd`!9$X=)f z6@sy`#+}chz@Z0rfLi)XgHbsk2%OHXA-T-r=jaQ-VbKP&l%#pW5#Fi5t1}I6O!@$+ z!R}S1MC&Zx;luJcf_OzBMDi}aRLWd8Y8fo{^<7*5k$rQ^+8SwFH{xcJ3s*DRuyc;6 zC`%QP)29uvR3bXba@%;?+5qS$eSAoJrg|Ho=+ed}g4i&|S4Ig)+hXUR%!Mz6b`;ob z)INyJCVFM)=-ltR^)SD(j;C@X%&7jFqsVfK0Q!Mh*z1g(y)8PUYD0zr)jn<>P=1Jw z+%;nhc?lS-mp&Pn1zhgaI!GfQo4Gos!^{+W`Vr(_Gq=zlVNxpUNJMK6_8h*mgHba<9rAweXVP{CvIJxN# zpv@>w0U>nDDO86-kWS3H9(ze2SMNFi6X}C*@OBZbrpZDYzHeE+-Fj5WMER^(yQUJ!8PdyO&-N5{#EyObpv}+F`Li0b}N~ z9dV-5(c-V-2i(V;0joOYL*@`b8!WQc#H?$kC2!KUm!MoUQW%WwJ9dIzZ&=W@DYPG8 zCW3Mh6LMZn?m&XXLd^&4=|hhDtBsAo{G>MA8$^3ul#zbs`Gl3SIsbkt$3TMD?ts$s zbjza+5}3QKN7xmjPAQByA#h?jCRs%q&BFD$MH=k{EQX$D#*VtWmvf45yT^!uMR}t0 z0rN3(9U+`cdvspQr)adjL8tsA`&5Q@WZguNb|x85PxP(?$Vl~1#)`MSaamkY>NcNk zZOz%gtf8N>*g2!l=B|u8dcmhBn4S>RY5+G`x@cOO>+0yTQ zlFmwNQzTjjWPh37EvI)15)U%lO`y1)@fCGMbjP4r?)<}zo|0e!7QdB;4F9yyqdLSe zbpSItrr}DSXA7K-=TLADiuu_{9HaNRo4loph?b%;{zNYEn20ipw>5XWF1)jD(S08l zdmRv;4|Pn$oR{30>MorNvle*ad$8M}V;D#Mr9WbRp0Bh17{h)^`!j$0Y8XcsuJ9^M zlIWI!UoGA0r&?u7oRf>yxYyUA!WNr@A~cQ;zl(kbUrZJ6ju`AtZExa7tUU=VqNa^* z384`-xv*TA(9kttw(@0qBKY%GW4JPmCY-I3syQobI=dWyO4HC;o~yh~)gRunXtG)l z@=(QDUUEg;L_ssHbDvKX(lAEnOS5f(Ca`lf721!XNE165&PgURJmt+6y_%CfuoXz} z`(^;Mf1|Q&`d};mTTIL4Cj4lL_T}a;LfyBHmPAwbmGOQ-b!3vm(3ZASC?)G~uh9vV z(k!S#luTR!6JS;|eekS9;#JK0grsHai@oG%%ICR#<)@Ask^S=Vo-;$N@%6Kk!X{BpuTkP<{v>y*vIuO~5YaNupJ__%6;-XlhH|jB&HIX{sJ%r=h-a7~} zzn0|rF*1<|m=~H=8NDu(W5YgsT;1j*XZM?5RC7jJmxU4VoPaGPXD+Ah?{4wo_XSc* z>~vDQFwz?I=dDlZa$5@xC?*W=t_vJAnVnC{?{Nz}4fq;5=fX+ham z33nCFQ5jriz9&RC=8~>il4_Ex^eqN%&iaY};ZSCaJF9>n!d0+6d(6Nb*Fif}sL)UVB|HO+ao_t-(2-dbY(mev2PK)4$FR+HNG2fqjG8 zZa0wD@@Rqf2y@E8Qn>lz`+qh8?p^~=J9qe3rNSBH)%uUn!nctp;ZYFQ1X53akPzt$ z)xEU4AAkwg_)Vj|W#KBhyH3?+mi>*E-Tu)q0wy53u=X<=;G<|4?Bb{41k zb$Mi$PCRYnsZ^@|G6?*iE~`7QfVKq(ZY5BvZS8XBaWV0D7?6=C88|%KPNCRs2k>x< z1!~Vttl(dAr70eRz?TF@w&kLTQ|-tERbxE0`{*Xz$ZEG_BCoNrP7pnMA=akYg4SzW zRvQ&DvYLE`0+YKXwO2pyxU`&eDN;63fIV?-Z{f1-uytongWP3q!{O@B0horZs&W~u z7!gyk1ojns9GykHk7b^B_e()Et*W=D6H_R!S$f;6vntW~QXqDTYbV$rMO1q-Zh0pr zuKFr9>N`#wP7GKyQl-TkY{^^KB_I|;?|6bIK3m4GlyYSlk2XZ*bvD|S;~D9F7Upfb zKaP!DZmoz~>9QXx$P}k`f|a+TW@kVyY#4j$ zYyy=Sev92y%{uKv?*=`H+eUwUm!8dCm9H!iv|qYWX)qEK{1H@AfE_35CA{TJB=5Yh zsfN5J|9Xk0rcCrsHy_8I4?^BTU_A|y6e*t$ex6w}kBNxJKV}YoydiV%dBqg<@ETug z+y1=4;Ps@-z#whwSp&Jr8x{cbJ)fz*_-b9xqCs>G@;nEtbr2dSM%deTy@km`7v|`B zdZ#2U`x4}OfZfDPQ9jG zKXcWaN;yQ_S`FdVZp>i1#YLdo1NAGvdvvO72)3giyJ`)^aG-Tc#5fj9T)Rf(q%W?PoI*%BoMNUO$g+kyr49BKwuF!!L&YId3>wxz+W_ zv~`mn5SqQrGVf}I84Cc6x7q6Wq|fEBi~K0d^Xp{7o=_F#5tt3t(=gLk=)=Zb+(s_X0f6_WA+Zc)?AB1eLS~}91y296q-;5YoIS-`lrd} zMuHZ4t-tuqXHL7EzpuKyBRL?{8S?6;XOl%0xn_`(4*U8I*&ZKd)IbS@l^9sh2Rc>O zjWx#MC!cq+y6erkh}sLS?38_EsIUp&rvN$kPD(2V)NP4{JOpO(5q1kLi^^`xE8yLe zt3-}kep2U(Ng3ocm2 ztU=N+x8nj-5yDVtN#$zUw|z)CihzFr74FmvVpV-mJ@Wuq@YS$oUSafKuB6rRZPMgC z9V&>}Hp)>A7B!|Dx4ux)51#iUkbQYGUBe!QvXML+!gNVjMfPg7D2?q2)dt)(W!eM4 zQ1YF~!=G*vv_!@U!}U#l9luz}mU1 z-b#g^CUnz-RqjmLEYh~2$noU{O6?~cPZ4=bUT_y({rK{z;|ha`^o0na`)PS&S$wx} z>V#1|7FV3c>?ftXrLOMtDV%)4E&CazT62Z$36=2bD}r}R`yIC$9}y_cHv)Qs5nEE%CW$spzF(is`g9VJDog)u35x#(ReM57Ww z^~dP|_pz6}OIO7bEwwYZUYw~d67qUnI*^o|q;8x-H(2q{KT9YAXZqp*DP+VSzwR!A|hPL#*fEa{JibK znZ(0;?sdae9G z=J2yNHEpr;^ObYT@2)-W@erkPEbvXWEIyh!{jzkfB|mB(6x-I03QV{zF7_K_%s+1- zywJuqagE&qpWTkmWK3<|rsy{xOF zLxMg9Q~Cy)ggTlljNjD8H9w<8r5RL@1*rc~_Nu1hi3QrU(StmyfxTienDGr^4ZHfH zgGNrxr9{PC?zb8iwUth4&l(=F^(zeBU7bmm-9N#2+ER<>(k)*6F2%abD*(E}29#Nf$fUM0+oGB_d$Ea7z=k=wjl)4SnrYquqq7 zX`;3M6uKk~1l)+0slt_I`pgfn-SjHr_8AZkd2`MrC~u2&E?#QLG`M;X`6JGZfidBETP=TX*MQ} z%3sPWLT^B3P1<=ZSXWAH!7DM=I_8+jg z6Qhfj6u?@c^g%CE4qpuI5IL4=?y~Bfwe&ifVlr)_B}8PO!@;(YhZE5FYqGth^7I?wz?(=` z=|sE%a}a*57|-pc84h+~4GstuMvZnr|CY)`wZ_?^bnzysgp_$eB4{C*$+BrC{V71k zpi**s?8;176-^6<6lfvXUhPcZtPEw^<6|rhJ@rUJHHny)=S$pF&1{&an|Yz0@Euca#o`<;Y=a zo1FpAEQnvD?Y>Byu(NvL{KUWGfbzZ8~gjO6KXv=BwA2wuwWPN!I;OEaM>0xE+(JSRUzyE_LO zCCrk_s|jse`;+%oS0=s7 zd*p^!lI|D`2v0&cdGK?)JpmfgjZwm74SQh1ghLHaTtDe@73vwQtIuM#RjmdT$lQqv zC?T`!SP)-(BD+WsaJrVsqI)=zeg z?6+i3mZ#}zOIE}^OK)l$V|(V!EpI6*5w|R{iH)FW*9PHvqXwy>M0g zDE{u+w3jr!UB?=OUB97ozJ#|UqTtCapHs9G2W_#)(EvHc?k`_BS`WA$#>fY;r@q00 z{uzg_Mb2OXo!g_Fp*y}m63(#O+ZV3wbGo-L{R5Dc@OClr(0%*5--6V?AAX%;J9(FQ zf8wU$1=R`J{wEUj!{m5H_T^{1{5QSnhUfe^xw{VzU4uS%&~@Zh*y4ddw51n)TS zOKcMI!#CUdsnLwUcQAHMuW)}w0hA!FfR13tTD4pAjnJEJn}_vZUe+!Mvh$Hv#iOQK zqSRJ5b63>d`3t)DKW!oMFK+rz_HJ2c}C9c%AY36_YzQO5szA@Ytck|K9w&t9w}sx>9OhNU-JZyurhVS0b{ z-pH_4pM*Xr<1A*jxR+x(_nt&(i(~by{-#y=kFy{$2NR~ypi!fKaL5pOa^cQ>eVFlioWk^IF6fB6pC2tN{O{XHQ|m0JqRCs00P+;R8* z<2Zf2mA*c7!|ijlqqMk9je0nyuc>W#l;k%@Yj+j{JeZZF(@-V4h{Zosv~|{?=_?L- ztC{_0)!e)ZLwzNmM}gTThx1BKcIwUXXKugzdcXK4BM9A(V+4EAT9il)@?<-s-`*E; z@P(y#QrZGVnmqla+C~iz@^5t-{PVVd-yL-*B!(qBDL{k(KTOw8a`d-%5}hyT8q_j4 zq)TiMr5~Nn8vffL|HpBvW_wG&n-0GV5eE6nT+#gdw~ zhha4*&p=1bRowcTwhIC2Ly-0jOr<9Irz_##%*NXpe7l=>KLTEx3{rG=kYwOg6mB(P zFv5iXRp^kK$aRHhYS;E&< zTmY=1My1q>Qa#wLr3$@?RXe= zU(MvDY=IRXd0ZuYR!?)zK6V#~x}W&+`_3EKtJt>B+x^?rtUnM z&*YSQIu@7FutTH#6_-_&y7X?9u^1twQT`qNbVJvwUiMYWsHubc++Y7_N24k9z(I2E zJgFn&TjH-bI?XZgB6U?G52dFm*0;k z8pn(oEpDqyPA}#cPy~39F{w?x>g#kh*mKgLh@kUH$Jit0u z4XrK|!cbhGD^Zv+$|Z8EN56ZKI>R1fPRr9ta?Tf|x&9u~*c)E3AfnxfzRrQ*lh>d3 zG`|pWN*!o3t+NLD?*Z+~islP93gQ0SzkV%oOD40^C5kDR!#n6iZp{;X86drq_z~Jr zpJTYOWGA0lI32yeeqRCR&QmLPuw3;R$#z-*5i+;cNOLBy;O+M>LqKd6sqbZU-Kh!8 zW(_2!d+X=|IH(H9#?VTzX5T!IbFk$it2))WHWRlKy@-7-PqvVuVC@xC}|HXdB>bi9^= zbmm}1%nDC~Rdrd41|Gwq#MVI>Oam0M#qY4mUyN46^O{#nzz+IE$yLwg++cU{-+gm= zFyL#%W^Izd?Xa^7(RH@JKO37B&5(cm=EZ1nJf<_8tV`Ldai;n~a60T`^2JL*&}1wk zcL3k*GrQxt>o%7TVzD+|;A}c$QJ{O8gevqzn4|RARE*ex5fcG3-&a`YT+Wd1NL*LXD6(F-(L<*ct8tO-k7dcOUGj z6^}j?HOjrR3T4^k+`DV|Nhhz@Mad=c4&ivZK9irA*^4(Eyc2fF$R#?IxVS_s z94(8NZK)}r#*tf~36&+a<`IUs3@$3ms|IuDKIBUW*T`NJMkvwJfe4qFXAlP@LGMu(ab2 z`3VV&j6_!v2E-%0s7wiq;i*WzGn?VL0l=G-gEZSwpxE%{0 z@CReAdbvZ_K;hFSLu+oWC}Qg2cG2ieyzJ$3H>zz}l=?4qyi@|iIg6)OG7^}wK^X(8mW$Cx@LJZt zyemfO>$^e>n`hj7A(|6h?$iA#B%tUk6Wt|D*3$EE?n`jT%;eiktH^ve#b3LL)J43c z#OVF2$j8!mMFe1|I$ScoT+TTj`yG|Wq?#AFHYNS*Cn@bvz z_hwBK{Jmou@vQ#$l>d#`|JK<5m;Hii^wjvq1@Is56aD{(9eT<`BweGZS~gw$=X?=A z`*n#1eaPFczhJud&4_IdkbpH45WW->5oteV1MD7%unNW;qQi3aSX}+(+_Y5j$RO_wfmV zq^*ab#U{V;|EE)`)$M{}jMCe%8g5sFtTq zjq@5>^#;L@Z|YJ{6kF>z94eX0YSFCXHoH|kZK+6yI*s5?`=Cl*R}7I5WtXgey9=Cs$z4|qa5Vvzu0rccq%`G*63?E2>ZZGl(9#Ksl8#)nUajnL z6L!+)KRAH?s(5en=>Ft@f zAD@&mEI5a7RI=>&1ndfj5s)7XP^#6&Ka=j<-Sok>E3jwxP1|+1^ zx!q`0D(f!usfbAjMj_~n8Z0cFEovOMdS*fRyT(+cjKEIu#g%=nMKAeBQ-~|5EF{m* zzP)5#e{S$HqzhHOhM~p2I>+lM$MJB5LU|XQiCB|q0$|9QIM8G5IFs?^T=CqtIU4?Z zziAlonVG;c_;*+}wgRY+9Jp^)=6U`o{nKW7IH5m4PMS&A<;i*5JG^nwHI&{x@vt!q zYk>6l6rviv0jfm;fV(kF*<4!M&4Qw|d70luc-R{`Z93Nh z)R^1RYxz3%+GBV2K>+FW*(oZBV|$=axYe|&DPGFRI*R%?cPHI-RfxchMje_R%vzy^ z18Eq{bSj1ptPHStqN*y#EZ-I$QML_4bGN)Nv#TAanu>=?lpX&bvg6i^P3ZkJWJzP_M*R+eYAW^+nu@hw^ z_*MNcX6ik9iu7JMBIR=E71s;frw_T^T8$DYs{?9EkGnItM6AgOnS^?cv=`IW9F{|G z9TbC0taFRX$TZIz%>d@v1M&hnhB&a^kAOI*xMkkq63s298XdO2-`(j+2!~(6x>e^g zEN<%Fm7aynb!UJ9n!nt>>vB_yJ3w5mjNzEUu5&XiFeN-`GO~zrYpOPHH`4sf9J)VswXcnivl{F*xYjTJNz-AfY!RBN zIyKBYe|w$QA_4y1uF4i#FfJLjoHaU^UH~ahA8vgT7q)MqmaD-N%#R=d^dIw$J%6o* z6vh4MMax0)K)cfS7zglrgjo03V(yk6xsC0P6BswidUrJg0WhfBK!Y!4JuZV4PNfXM z$z3JH69i@u5A`u%{V*3yby%v=tk-Z}{NSd%2h~JygndIg_KNLp1;JKf-NRW80!GQBn2It9wnKl#X8 z8~~@HMAA#MDjCU=gtHu^EY&PTn@Fu4bOz_=N88v>w_7d`)>A*b`HbMC^pg;j*AqC# zwP3eov}<_;7Ch3Ot24C+m&|~)YY}P-s8@cFAT(1j6J>8)F$R7szaDE3_F|i?T5jyS zjQP4FmEEwxI4!ivTLG=5`ane<;vdMJpIT_Rg6ZWFMRXWM=y!iysQ9#guJBNb=U?mi z%xmjtVP)sZLxR7m=SElTp;CBwfgj~fbJloATG&f}2{!WKWZ9aroHhu+5CSUQTE5Z2`MS3N_<*<$|{k025QS!Lq@E(5AfOX+PtLBZqgnF1B_Ha1&o z-it_q1W-y<(^I)k*NQ}s;$gzC$j(=$sS0fA(Ih&;SI>6#p7q`A^FW0i&3U1|o+Dzy z_ul6FZrB1Y%<#oi4!_XdY>!3G;TMDdnX4?>s2gq95SI*Qdn z&pnb5p6Bnwyy=Gkby)e>M?61#fcf-WEK(;|n17-FkM7P7_W=J18ba-ie)z!6%h6n+ z#{}f!k4LoZ-t@4IZy3E7Ia2N>iEp-$*G*iMQGNk}nH_p!D_aeiNIoJ)qN=pE`2$e> z`HD#hEJ#+~eA^9QPvjbMaey4^hl|np9W@9HWBdvIhGv2d7b4%j8d2YI$T9rE2+EV` zpO({6?M&*qLH)z@YmyaJ&ziCB2b2jBRC-N?^QDZZ8f0sexhPhB5gOxT1jd^M{WI`tvy& zDq03LsTCuwI@>5T+#CP#0jNSIhk~Q9lQ$2i`LaNQMKzGZiSfms7Une@_ zZ;7M7r(60K@cLl0I+!dzK675RaDHKBD7c0Zf2qHx=I-tszTTm!aQ_}@SbtVGdP7Pp01AMgiAYeSPD1U>x_LnKFL1?@Q)5+_n{ zwJAF+2;d8!E)afS)B%#Wrax9GtIy0vCFocsfiBeap06;FbOxR+LVB0wBgj*{fHso1 z+p;Ny6D3Z1n+G~Lm(R=DNkzXRXw1%sW(r+ofiz_s{W^2J+0elFyc{FscDSo^FCPf` zwSXz&5yWuZ3d*QJKFE#rffxNmovQ-Gx-Po>y#$fe3L4NLty{Yc{|q}!w9CZq+IJE6 zM?bYgrIl|2(pl|5MAZMmjn*4r<-*0#Qd$lvLNcWK9sI`-BlNh4HNEtr6&8w@jtKKI zfmTCrU?JsfN(rc=5^vwhf@U2Z4&jGvHOdYzZEC zHI@9%&_d8MNoRMvL2hRo(EoA{Kx|Kwa%*U#ssG`&8~zk9nIj1zyS|(K|M&wu z23)-C#O43^$?p$xNrFpKmYLpu|IL5?A-Wlin*?i;&=!0A7gi;D1TIN-BDCCe&}|sy zUku2pY&hj)J(7Gj9&-PFNn!z9k`#saFK)t54-!Wb=l`3U|4q&RcFopC@c;N-V-Zkr zac*-N+vN_S*`0Du>OGYiFc(x-*q`vm&IT+;X|So>cNeh;!lE zWCUwZ+?`J;g%Z>^@;=~C00;^pV5cxu+qp%~Mi9|EZxF&={idUb%&AKOS>A9`xB$@Xn*n0E#>MXN;%Jnq$8b|ZLuQT7>i9dGeoeF5Of_r5*q4=qkxpp> z@>%xA=SEF)ZwOE*h}_b79jUV8*T*6hyzx$i-%{EH+jm($bTo6oHw9x>vV=Zw*>DLn zwy2r5D50R6!Op^8lSz0`|K@fVKQB97uWz!FKb%J$AWex4ayQv@lw6;|JDtYKw{t&v z$lT08vf1$S5+voIW!gBlk_?heOUi8a%UR_uSdT=l5`o)RHP8LZyb6UEJOKQ zqZgm|8&s_GnikEcn~x1wTn4(@jA%4zq zP#lro<@mX0*VLe}25rOnDarFQAGxNMpk4ErQ3};-(S1I1yyO}w)@VhVf`t!@o#W0m zBe;c8v$FT{J4r@p4|h6Stb{nfT2hf07#_c7{>qH4Hph8z2p8iBZ zdaxf(b}-jRJIKxW2ru*1FP4Q)K_`n8}Ca*-(AMh+* zTh=XXY#ucroG&yjIIUs(p=D90Ba0;w{l0pKwnw@N3=gfZ;=AX)a<@K$&J!83USE5^ zd%1=8BVvzNL|CHFm;Zd`?L1+C{R-e1YP$birc{bX(6mEHSp>qEtUG&V#|8R><3A;7 z7Qq@BO1c~6QNSF39;9qZE1ip-rZ9sN>Q>iADRV0M{q9YmPvL*0w)}V@V^};KHK%De zNUPQD)oOxCrJY=PoVR?>=s3Yp-21%A$@%*!T~V~_avZm&tK6bIsauy+a@H~n*N#G9>9eZczbiR?ZqPi$qUkG=FW9p{c*89Vh`X!R}Rw8YNs`!l4U-!A1 zvjaH}k_C|DcsGsdr7~(lKYE-k*hu!CCtSnO`@-a#?aHM3V%Fb4Ai1C|M z*|@U=mEJ$Uc+p)>$9L@2qB9@ZAI~mg>r_1s#=DZ0{E6A_gR%JjdQ^|60!B8pNz&VFv~fO2bjbNbqTEW6Rk26w*o{Hctl&*`FJ69Om?$?ZY+ z3FZD=VACmbm@8Anj0G}@aiK1qA6>-s>FKR=3lwu%CEQ~Yxjo4`|0saR{2c@7BQxst z)T1huwTlx1kNb){*GYpTENn?n@UQ0w_q*G(8wJI28YSL$mL;vYx$t^H&_A$4w=I>3 z{AljOE2Nryy4C|wZX!x1IMObz9|vKH7qoE_nJ>qLkDHRo*8UHq!)>zdWL)vlhWM!y8}mR1_}0e zkA8;Ab&J(|nPO_Aw8E=7qm$NMvQ)8JQDlLgbE~$h~#tGZ1xC5tU8u)Trz+SD-72Dvt7wA#PsFZe#Nunf*QZiea z{fZ~@*1Z8A{*Il6RDdjg*t9@xwNN7SedQHGIdM(f-cK2ywTQ^96PRL43I_L2&COX? z#YYy;KNUvFJy@9N7D%>MuyS6mN|^Gv@9ak)x?clzn{?Yb%|8ss9j+;pi<06UKCn81 ztHC>CmN!=3K0(J?M@33@kBTyHgPDo*t=0ZC=QrlP!fd8a z)A2)GqNBt0as#!RxDSdMYZIL$KDZ7I%{n|<8UEFei?^rUu(R(AX=lBRaX|tmgomH^nJ-oEbA4ZZuMW)vxjtKO3qMwP@>z1)=hUu0+@j zMY^dNtbTGY(dm?JeQn6fw4=WeXPbMbXo1l09xiOB=zt@$6Bkvd1V;5%B zF{@53ONwL(<3hvth@TqV)^YlLveV>n{wpToCHZ38pkBH1njYtf$7E*O~ETx>P%96>PL+xT~n)Ve_7L+-U`*U$gS=b08oVx${mO0f# ziS6Vkes0YAh$oT$+%kpVxW#6~FCo()r{u*{@QgZ#RdsV-|ATkf(Nv-)BSoWYuY3~J z=fn4h%e-=y+ObWTP3P>`*!0kv59$h;>YWTFlkt`gK^6kO_~8`IsLysU)&@FFb{giK zU(8n2^f}8|915Hj564++_Gz#MG2zn}uddek#S`{|Emb91Yuq`S)e*!SJLY3hFyUOX zu5fInzgZWB?piUzHot$?z_g7Qb+b_c7+5l!5}eVpzns9%bkt2PBP1RLO}Sn?__D&PG$d$GJazUKLyTN(Hob$IN! z)r!$@0waHtHv+m zLlEVg=cQQ#j>}n*J6{-d&gE$&iB#E9f%?HFZ}a=1;SvO{qN=-%)TaD`731f?h#b$y z`vuOe&$K%Cy(*+-!LQlZQ0^;}USvX&&}QmxJE=M8VIj@ONMo$c{8`9i_7OYFnybxt zOIsLC_c`B7JNXb`>x8CHvhwpqKs8D&fw7F09jSohCaC$Bgl9RZFB#(hV|n zM_o^ZkFzm(d`$OVwEE&WZW~jxXdp2dnWSVZJSH%ZY(H@j1dmP8t7rFju@Y?)oY+-C zCMB0sHET6dRnovINn`oZC8NT+7(14C)%Mu+%p3Kw=08PVUyKH%)&QC-+K{o?i6zEk z;`%@_ns&%X__Ixy!yGHj$HxW6ite+$k5PM8tJ_K%Kl#3=BtU9cO&| zxH}9eh63A;9zLC#qxHckPRXck&qG= zp)A|IH2qj}jgM4C=hZ5zuy=5xEw|edgXvGUA3IobZaIsEQm2|QEWT!ag?d4coKenE zU}7$tH@SN?Is8f(>FS4-FTNh5?A?((XYZQ&3t>(xdcAA5=T|;r%vbX(vTYscjgOqd z?$&#E^k+MX{N@%K$_MpnPx?3gIXWIchZ+AuJZNTzjd!M(& zsI+l>o*|;m>S|hqBeh-Wv8nlRvx;%&VtOZwS_3~$Qw`f5`Kg$G=c7dEU_e^w4$S0vp85@k>!(sD&()`#7?&Zk02wi=KNCs;{!%qe4NxXrFmo?Zcfux z31xH2sVrx0@>43gT#X-2O4+4%_u=wJY0YadROre{ChT>OP@XQ3!{)^AG*b}`tLK+; zbzF%uGb_kHF=`Mtp4DYPS~Rs@2sn?sQZ?T8lOc{~%MIEK!lVf_YvFvB>90KoPxqF< znO~9C+rhA3VCt>09KR3llw?hhT+sRJ)9KD@Mo4>n!ro54O{d`ZogM84rR1@{7<-;? z{5!N_GS$3)KG#j>~$$ygAfM>s8XbzR*`JBWxfPXl-wmupe)H!P&Jbfv=>Fn! zmPg+has8uUO`Etn&TeepFgpCrDp5Td?Yg!26Q!e~gjx{^xbuE^+Nx~nq@3oFwq43o z2_Cor)I;NZ;Ud#8axN+B9_TXi7aaGbWZyjbf15))1PvN#+v63s(Rtg)^t`2B71h5y zLz~{fxy$tW?y*sEx?5rH=UC6Gp|#c4ofUOn*$i8SEiVu4@Oc!T_JYZ7!4#)p*RwPr zRn{K|;Uyj9L+dI|HOs#c09h4DXw#)$z^9A?(mHLRu$WIH} zPWCrRjrrjiUmQ4Tf4TZ?2s^Fx`kD1RWtx4W_O+Fjc`r+e-qf!+3ihgJPBOJyb($Fm zw*<$RN!J7knEDZ1(`64^#Z=|Q^jXX}v31^Qozg3#KI>_!D=-!KXbLmAcD%EsIa8us zJUO-b<4|Na^Yu)<2(y??d4+GoDk4LykIUs3lq$8^xV%K7CkSL;szKa-JZjx8Ts?Q9 zF{F5{->i(vqTY|%qB+;cJDZe(ls>ro46oWB{VcccjH>6<-mFRW1`;>nVl6|LqC!5f|zwk%n!iz}(;WXe(0tA3K#_R9Wh zCMh`NoCACh(Q=5@M@Y zB5_SOx}Qmg5w1+zvw6|W2p;#pMA#_#Dul@cr6L>urXQ4;oUEppOmtd+Nw%^S45p?mRYBT zFaOLv@|j!mf1Hv_kWRgr9At)}-ODIAG!3APK9 zS#6iNfoMJX;vClc^csTW*6osKyKyfRcgIr_OYBuTF1xcVjfLs{r;>8q=>aw!H8q_n z*KmRp@sb|tFFuREm@8LBl2gYCmaVu=tZUE$NreVjZm$Ik8R1|Pis!3$l^mXx6*fyZ z6{w^kQ5pxt)l=!gI1&%md>a<^MB%8$0u)xoW-TgrojAQ{Sp{*nGsdD(TnjmzcUv!N z7IwIi*wyVGu+r?3&z}gi7RCw+i~iEdi^juqBXJ_T?G8fDVn$CL@RrzbuN_ZxDECss zXzKG!V|jS-<`4&2s3BZ~SpS5}>Yxz-rYse_;@*LypfI?%Fe=5nP+`-q{&uK9+cAqs zbLNq={|)KQJ+Rblx0wD`$^5l@xA_&C-QbdCJ{h}T zHd1{vT$``4M4%;P&yW1l#%=!Sw^tas1eZ8IVT3N}Z^QB*e<+~)jKdZZpGtP;GJmgm zR^a00F0vDB5APkK4%YU}+j~#oUC0}KDl1P)axX5uy;XU-hpn4g+$l(MYu^wM2Ulr5 zJsM0b=M-I9OTj6Xcxh>DycCQtzbT2qr0lc6v*R(~kWTa1II)ZtR zKkG}`O}L7GpQK8+yZBWOwU4*rMEb>GxRCA^^EKWcl=Sa+-nbw{gz!&@4QNMR{x9Yz zOGMLlpPg>D6{@ zg$&tn-IrUs)D!#CVQuH_OkG4Ito^d=b%<`u|K?@^nK3cnl+tUdt;e!;L@R@uQx(|z zfAhg2tVD3_Cu{?@U84I+@f}{>_;G`UO7}?KSDQ<1zqVn>wU-D^h@nsA)&uWBv$UU7 z_i07vj;+gWo1gFEut&_dw$0(u{3KrT1Uy-+gktlEZf)n@UWd`Tw687yuzsGl&g)UN zeHWkq_O)zy>YZ`&UWbREbFxq@-srb?^X?<~hMX6sL0QU+oJ;r3>e7t%yQfzOh@#FN zX$N)+Od=xy)L##|c471yl#EidwLZUpl>ty5yx$xW5iGtT^M&cZTPw<;bxxL7WFMf4 zXk4<0h5#FL`1ON*r)BP|{NDYd;|UB;UYG_@yca?i)K}&a1C5GdJFO z``$(qFN#97cw2Wa5YHL^lLOJG~&E8cs4Ll^tfyDTp+ zE9Dp~&p%EBVh3hby41FtDQ(q}jPHKEDn$nO++4%;&s8x-y@7vE7(^qXL6ZXzU%pgW z+mhRMC*KB$s5~|>iRTGB9NH8nWu*9h{DzcVN=pI<)~qxt&?#r}@KqSDDX7=zS!S*1dr%3GCN z?2lX7$fK5^4->TwIy(mqkDUh|7EXLZqwMGC)AYt+}s#nw+HE=ItBoZrxKfx|K`&Q4|cL(|~yF#vvIx;}T#q=O(;*Ot)SwL%Tcn+7ti? z%I^^n@uxUb2Wq~D?yEWq1d>_Pn@8HI9t|DD2t)Bv=(O5P!JXK>()3*s;Rk;#ai>ec zr+gfgv_d^a`p>J0etFwpUKKqK2W+QlBEf%gNhx>(3hjUH-ilY? z7Z1`e1eYm?(s0Yd%SM;>>qI1;fzz9W_gLw7llJpxe*8@b8#tBY4^)2q{SO1R@ziIQ zgk|L__3-w;eCCHgx#WN=WnB0d672r$vi$Nl|C^fsP0jyy&HwGY#wCq}RHp6$M*rPY z#D{EdYZHaoTl|jF3mbDl`{~_KR3w8s9oulk9eMG!_;opDkch5sF;2+E-2PCCc#eezVoK^nOEF(E zV&%@jQ0}xo6Xi2o_&L0HRo6<&?&>#h@0Z5&ySa7=I}8V$ToXvt3(dD{XMm>m8|W!e z^4p&9u?}<`1(AMku|0taPwt2(l!c;SJQ1`;;)VlO9r)%Je z_YcR#!}ew9;4yJbLpQc3!UWfJxiQT%SRJh#vo3c>r-Ne#==^+uIquE$ry6bO$<3(a z?R-*=n)Z0qAW7mGq-u3xhDc%{xhX)mAzws>F*Za=C@n?|uqQ3!(PuHI6Jk~P( zy-@tR1RU2!2032YI!F$-OtE-6PzAxufpH&$#tB%Y_&#nopT*%Rx7|Ar5J7>2xzy4AP*(9{Wvz-)+|FZ!&p#+goxilF&2*eD5j`8&Ia-PGL!}DyzVxt$8 zfxU$&IAVG!L3oX6^elC1Q5mVKrTAX{-_2#kL8>z)AXGnrPu9O5#K&k60u(CSkAClf z;AyeW%!ycd@znt>gzKP|qvB1v(_FIHH7i$tv_}`{yY)l6ekD3Yvg+aR?!f>X+X`yy zH&DSH1#THe!WK%$$lIlRl#infXHgM>B(BKD@|`c(6Gx7VNG zF1v?3m?t{UAgTdEJP6Z*7US^IA$2sC0otZR$4}S-sZGTKXr%ScSh84)dyloIs>Z7C zDOoIICyk?-?|U=UP?$4j&0;Hn6V9di??6creJV zmX#1ETq+Rdvchp zsbQoVGIy9_*?T%2k3bT_pMYpjpc{>yJ|;abf(l5^YC_Ig=pxOz@_YOd=Bp<_-cdJy z!{#}hS~>#PCbF8OT~(=Z;~33~+qtHpoUnpQ#$6u!hz9;ED(ay4GBcQZ>%F5@Vm-vp9xn=nGWT zc=Svz0ilZ3F83s#DkWmqsnB3o&0$II@t=zSv*Kbt%%gT=>^0YPFCn>7J?4GA1~(0O z&O-9<5JA*~mR1#`)G?<_a!3Rs)&!f;x^FaK#tu`*>mdxxNo6O=*wjL(lO2Jxq5}9* zR+PZ4F@FAP=a&N%IB?`s2ES$O{(WU`$IC{-&5KSuTGcG-So{M%D;O?~YU0A=)w|PN zq!^Y+I(mUijWBo$$w2M#X9~@@P9enJb=qP7(KFGvu|qS^{^%pYUc@bc<{u}8jWS*z zewLk-UE{?_OpB9Q$QVfT97xMC{qRQzHHL>_d~c{X(d1ZTwozLDvhoHk};AOk}76#aotQ5<#jE0bfU;|&K9~OoO z!pc++T!jAK1ZMTszs`{S9CzP*3>K3xw64_DYk2pktkDm`{H4oqgn8xmIth3<0qv6s z`UsPYDderCgq&etx-ZD}X)z0oXUR?VjA1b``S?{2CP_7w%n_nu(#P}-=LVnf!C@O? zzc8^c`7(v}vH8*)ou}pX2Wk{lP2?Ro5wT)t+?q$P(bRES(CeFi)iUMm3_Z~(@fIvA!)a4RL045IOF zT)MRnFKX#n2;AUpv3ZSO)lwGJv*v?a^x*^NDl>v4+%Q>X0)$)b%)x0RMLrIzaJp~b z@AC2_-awKzoKrv`Wu1~yPNB^G5L)xa&{L1h!MV9Pgx-x?bpd<_gIMi! zMOMV!0j@2*Tx;N?>+4TbFDwUIJAcQL#XgUiXY>VwwV}j_q~27-12Wf$W5Mj2J)hB2 zcEgQvr>S>z-@K;--e2Z&GR@gY@8%w$=gZVXG?BDgQ;>h=42$~I~u3OUamxFzkA?_(F%n9#43An z0MeGn8R!uQf<;Jz3{JL6?^Y+MCuo6?!J~YOLDHy%KVJvL=0W_}SM-2v_~GEqNLXxQ z0)LRA-6akUr ze43>&YW;D!1&lfhJ#61WG6T_rOmKt>qAOjFsKYuh%C#TL;(3L5A;|GF)?zs}4~DBZ zO!viBVDcZ%I+vtFiK>2{)>ZEB9C9&No7JAG=l2aGnbeyW0u=vGdtV+-_1g7c4uwvI ziZYav%t?ka@9LN{B(qS+JVdBWwd*v=X&;|{r&EHSogZu@L8V~k`!&95$L2THki?HT~u@?MyP1A#@3|5>oODF zn$ZSo&>aPiV~%ieTm75OP0JkyMv?Ob4p}t~-O<~xA!<)`CsJKO7YKPm{%md>x zTad__nm89_Z({o%S)S_gx`r`YIU8kV)wOO(>E%59EwBu}6;2z@jza9tY)Fi-wa4+f z+1~wgn}nE6!D#bD^z39b_wxsLL@vAZ-579abPy>NWqlbJ$<&bI!(qa&YUPmn?MONi~9D-PTB-4Fy&>08mxZL8O7G}KfrYrn-d0@r>=++)W$5tb-=KTZ4 z>eu=9oon099}HU}f7eQ@KE;>gNXCOeW9JYW?lry8e%urMmEF=_oBaoO3A>C9oCVt_ zy{Fuv=dAoI#2M%r$g*be?dWIJBy|7?1+*v3L+fBX`|cm2UPPrrG(y|PeSCF=(1t97 zKM+6X4%XQ?mXxCbUD6EJ8xdWJBivD$kwPyI>}A#U0k#Q+>xV^U;*HW9QUc7&N>t_B zhWQ4M@3(Be2V7k|X-sPs+&(~@q}-=WNmM=l?}{rFzHwRpQ;K;vk!^9LMg76+yb9Km z@u~Os%fKqV$$Em;C)`G>*0GS3@^r^5(e{>M7K0+X+pH>Ukn4L{TSyvw1&62$Oglwu z^=q5hAYst7FF|^#+#RNg!dO(+DLMCq`tu7@qwneZ*qFRKngg6P5Sxk4%fs?&3KYS8 z39PIIp+1F5lF2-GyffbS^vjFPh%$Lbx)YR${&=m-ny*0|}^X5KmugmU?3JIR(j2#XZd z+;AG{5{byJ9hQdz|nc$o>54Gd;49dGOh#kht{9EoFRS z%waVZPOB0rFkNa|<^Et8F)D_r{37B4eimB~q3Vz~v9!NB6Dnpu7FaytaPC`#VJz&M zxqM@R^QgsG!pB$r{joyCg$bh2J3YPi6_LcdjWO9 zy&=|wa2Wsufco?BxlVl4xa#*x(VCs;dSfl9A->47m}d-1bgf%grty4e786@RFg85p zg4+Rr5sjV<^6P{GkZ}x`IR|>_ZGMIKbwV-WGyqZPt@S4-+$2l|NpdSin&5{ZM!)OC zN38@=(N?*Sxr-t%gWK!s5wxjwYdTJm=&)%}TH-PHwMCh^pd&a|(SupBHd_ug61RzP z9Wd!E(bRQ|mb{EIDAzN#gGJOye}rgk*Wn&Wrq5_I1EH9`qr$2>UEz*Xx=Na~f!MI2 z-33C*tyWW9Byi8ZV~o;GWoL;k8O8-A@N7>E62d>QN;Fy>>->{kn&n>iqc$^3Kd+(z zvVO9+aWml;5*e{?oDiO?x7uHpa6J}Yg&jlaFZV2+MNDInyT74~-j86&mOA*~-oTy? zE4KHNh0nqZ(HPVMito;eSb&KPbh^RQ*LbhOxBzy{N?Ff((VU@deGX?tNv=pv?AaAq zrw#AgYYH#&CnYN9MW=3l4TV9Mgavxg?7|ll5&7d_77l0LcjJL@ZlEKIWt0K121Rk1HeGZl zr_z%82XPVq8v^M@q}lDZMdCmZx)}oZPdEkLi2WcbB`ozC8VLd&!VK0JHAGj>$1?3) zy?!g>zL%#C4L~<$V`$Jrg+&ekn79wSbdWogo~g6_H3hMiQVYVkNl7)VdxWTrEgc_S zBpa;D-31hE+23F$IqfgDexEAlGf7nlk0(q-M8QYw9qv*WRNMzMwu*(ak%~w$vvMb7 zhBceX8!K4~CF>CdshM%Bj8X)EF;*@e>2pgJM(p|xUR3F;_4mK)jP0j>2%J}4lW(Ja z!DQRlE}Ujs8`Z?WpJls>6bT@sI`kTJ6o27o5Mz_HIM7p<#+{sG(Gcsgfz2dF6hfrwPm?&e-n~U#UVxGJIs6T@G>!X{m^24zBV^U=sKKmY|NHmd zUjHfVo050S@B)u9ZW|t&7 zVecyYG43^%y$ONvRM*K*D08q=PUDMh#&E9&GlT#H!<#L1(#;1K0((qcAAz*qm}Y5^Y$<@yRP(qdtPdTE`lTu5ISsZAN51!8fu&vJ@V=SH z#S~6AkNb-?^XG~11CExoUt^%zsp2};Jv)6>9=OEo`sM;YAKX5;*a7^m6rfQevC5H< zP0GcHXo#}P$M=*Eb>xN47&2Husv~`^tqVeP_tL#fETRjRwC91Idpsmyt5jYIa2nMh zrz*dS(bLnTefRkCn$Ogyg8%C43{>Cie`P5GT8!WtFTLTSTfRviOgkI6VCG)_;KDp{w0&Gu8$%>FW2FYxFgFh46cFfYimQ_QD&mf3W4|zqDl@o0wsz*mCq6k*vr!O#*2PJ)!bLf}0_M5AlLkl+W`@Ri8VYGMz#Qp+^_80Lsxr@k@ z$pv^G0A^Y%!|X`I`dRnv!a@m1)n&Gc^g2I0JKTkAh}#OGyPpF9*D$@NgID7e@!KcS zex<0a#w01jytz8J20@ztKsMbCtXtDeNx)M1KoVlEk22)4JfR&i|C^c!%~zP2S8WJ% z!yN)W0c_*OBJzq<Olw-Xf-PB@ZnX#bJ0wD7FOv(u<25vWa7oo8bQ6=3C2Mr z>3v;-36&QW?~1UZL4YHBX4E2QE_<;-S&VP2^Ze|9Y##z@4MB_-nWo{cWkf?05QzFj zvEMMyHAH7ojgvQNu0tOUou0#y;Tcp)lJ7)(x0&nQA)sm{smrs-@jZO}pJJ?(2Tbz* z*@FI({Bxh@09kbUXSbukycQt&4FNOZSuobdH3lgn^`z`c7t5(U=w*>+^mt#pRr-EF zNJMzOM!5yxkmYi5Lgfs#QQ{~$J&bY!AWP|AP~V%I(CDQVm2ohFH)EE@vyZ?>BRHwa zn;<}-Bm9BzlwoPeW=0YuH(9O&(E8bqk6qBE;u+1Vx3$roADzAqohhnv!hJ2>S4a)u z7GH!)8p+Qz&}r`+%1s$Q!nTkZJQ&51Pg|#A+HV>@`ra|(?0qr6C8wZD)8HSVss(QmL*l2N3gsq4J4+KjvBGi zs+Ew!4`SVyqBD*h{vZq(0(8)hs8_zi|8g$sv32<7e!dxi!IeTy4jsZbx(=O=<`T1c zZYneW;`Tpk#2o>MXa?BNk{_(>e;@y&>8pa-fc`B230I05E3Dn&VuEJ69ZwX>Q0>Z^X9&p zcwnw?HRSVxr}F_3G~DIMTMBKquIFKPYdv5VRds>gu+d;+iioU)WL&FNLvHsX{)( zr`N0!%0?#Z4y`YNM3&Ay&4&nyTOB1X`UUd05wte@CWWC`q-$C`-DJ~!UHj+Hr!#j7 zITAm-y|fQ58Rtl465d{dynC!#9y|nySPa}z{sA=W5ljPl09!LRK0xvp+RPWe96*I9WZzI@|L=G$h z_C5pIwW@}!xD%s&`S#~(UB_RW4T2VS%aphrx$9`$T&l}abNoT_bW>B)$(ntFAqR`L zxqdwGx}JEZ2iE`+?L=STL$DgSHZp|WXo0*_+6js2kp{cxQgnj!pKu2*rMdV@&Ya~w z_VB^;&yT&~Lc*Qef{xK}ZXBKUAn)M8u*!a?d{;<+UO$m+lTcyEN*r{6b)o_-xh^q$ zQI1Fsr<-kK2i@6Z+@@z|-;P66XdA=<4Jh71eOELCn7459x26zu6Thv)YftTvTxJ&D<|FK?dk4uzK4>U1&DPMc^ zVDHD*Pr7Z-f6=`M05m%Q#bj4xCQrMea>FTQ87lr8d%8^04c+UZeS02<1pk5B3<=uWi*_QMDmF>4PZ zEN0q7eGBRUF&Bc|SqDhzjfd_fC2{E|cp`K`>p1G(nN$RqYUu~}@x0*v3F5dwc2rPr z&u+0OLU~X6xVK>QwWO5XyIS1C{s6S03)hVh%4 zI6`8X8a3WTspFC}ZT8QbXAt}sT~)A(pCCW`bPmrLG6@(zQ&D{(x=t|8q6>6eyk9@51e?? z9UXChP|k(2I*gyKT`&z&|9ob3I4yu$wWsHqS)T`$Q4_!bK+O(r;WQ{;V=W7FuQjO& z-_$p5b7`&6HT5&$+&Q=C?`xgXeR%4Gw}QjKN647BrZV*}YT)ucP@==R$EjrYibRT~ zLZ|9tF)=g1LK=I2y|PSj^Vo|xh0qqbs&zM+$Pcsw!bx{DC6xG097hh;+D6u2g*t*Z z!yZG0(oGH&#Ir3w=sb2Qa6>rc`@c_z_;}_oryNL-g(daT0O$Yw-~Qu+%5{)6)u6^@ zxMB9-h3(aBVAcBNAHm=@!cmN4zw?Vg8Z`VKYkpiX{haXKj|lbUPj?By#<`{B=1zi6%~d9GpB#1YZvj`41_<6HZJDO#h8VE-7K{>rY=0gg|7!56tfa z^HV9oQ*{4-9T@0h6(#Os5_1fqHfz0_sVI63=4#^i&9dwzA)+Kz^g$)`jyBTcpKE)r z2VlLMq!vLLLJrfVVv5FqZ#hRE?+z^@8frIUj@ONYp%)6FwEu_^{EK9gZZ6MP zGka~GaRBd7{~H(rjeAZ!6CcQceMyPH<=O)%2$BEwQ&kSrRT$RAd0K+ZJvK&*jMMo| z<^ko>O2b3oM*uAX!pwRBCo3)B(^-$zgz+a_m$3xy1unnlH`FqW{{KxwK(mt};^D$w zqM$u%`JD>@?%ME8s136rL_LO~3wRkMffM#&-WkqJIG!rt|S5caVq4DOS20{ zllKD~0gIdFk2k=#zhK{x4KTOnd$aw!dDwNE*ayg)ZbkNcfo!mn&(VvRKD#L^+QKop zX|sguS=z6V^$zbm(0HoHeDw=tOQB|*Pbh==9U$uX?AW!Xfp9hCEeFzoq?v8D83jnPQ<m|L zP=4!h+#>&Bes7le;713i`dk=?-7+qER)|n@hsjG3Q51_$DQHP%UbyH#?>*W@?BLCx zz=aR;D0;Lvm^iu$Uk}Z#Bt{=y^e5w=E`x9@h#M|?W6ob?J%YJm^TE!kt~%@yNOZS< zSOIozP*fFaoXc3`a|omo8b(9h&;LOJK*&fd0`90SW060BowJvhpOrB3y@Z@BObKTU z62p4*4iQS)g$-Pg?jGbA=$ptB;elLu_rjC_Ui;JXkGqoS`KYY#~uV8xl<>8al&!z`5 zqr^;G-rg*b{1TP9dpvK97OV_e^JU!sbtWrURtPQkX2$I0Wr>Si>e|M0Z}F~ZOavL} z&i3URnJX*uy@H&UVgFduQu$f&FVp-Z1tU|nZ5ifYhBgM1R>F30#tQ!z9~GSC#vo zw)VZc8c%`=)na4Zg$u@d>pZ0i($p)kV`~~kJY!o~+HD6Z30{k59ph%;EWAJOm9eM` zeTFtJB#hZIC%$IwPh7q_MG}&L4$=>V9Xl{Me$qBk^vj|~8($K2G7?wAVRzn4W{o!- zr1_cH>!P-7Z$Fy-N5nggg%w(~eE`IQHFq#vzvyW_uDWLJoi$rqGrPOYi9L;ZK}ilp zZP^KnR%|!5Syi>4^_>P*?@=mZOLhUnMh;Jt+|?yi2G}9miNv{|?qKPeB>K~)YvAFm z2VV%2JXjkVGwUnm&CeWZc@6hjSe6cjJ`qYCK^h!fF&=t^^@YB7nhPBVcB(b~~EMr*SKOp*I^JtfyY zKVXjxL0zkz^pi#KRs{cd--W@(h@XBrO3p4 z{hH>$8}WYXstd7T2b9-cF!ouca1?*^b(=__qT8At`qJw_PL3W3W!j8<#p?kdQxBQ& z+^*rM0~av-BvAFP0O@Jvz+H-!7ps7IM~TkB)vgUP_2G90zq5w^U}#anEZhkzu5dd( z0*H&%n@LoNM<;`QE?Z2Z1x>TMd!<%2H~I2p=gP+@M4SexwI5WvM(pw67+9QCy`4o|O9 zw~8`+SRkHVtYrl@!7XmlMUKDk&auU@a)D{?(Le-r2dZnt>oExwJyOiCo}bl>3F_>~ z&(IUa-!r^-Vl*k>Bm`%?B^cl{vM69SyLTD3DdlsB1%L;)tGH`ceb`MTM2QI2;M#9{ zLI5*#zE_OMH45+N_TLq6#*|U?$l<=GgmiX@W$Nh?9S_oFm(xt~$HTmu#FLKqnNDj` z=&V}xqf_QPuxfL|p&_OdaXQ5Y1$V^<2=yucn*DG_$*ADcEoSZ~`RTU^wpWD*F3G_Y z%jliWegTfq%2h|`e)xHU7vpWv3akZ4ki~kUU^T+K7cCoRB`N97HG8h*;=krUW~BzV zfQCy8L3sJ+k5Gax)4p(^aXDkt#2C8pE)*K=S@o> zk|rGhdZrJOzXb|Jp)Ufux1K*LmEeowOBv+ojlSBP0$s$FBiicQ#BUPS{-4j0!Bpyfb7L2Io0&9^fDQB7QwodCouPwZg<_tp+41{^g@c>OPeDa~AnW6Na7eYY?_EY-+Dg8F*ceezf!$Af>UL61o z8jn(_ZsU2K)~d7JTeVyUGaH!BynH@jjo2IoAa!gNUsJ_fB<2PbuYEwYrpO}WvoXj@ zqxt}m8$#tzxIPWNV)Pdl23-tT8)`)SI8~`xSl^YtUbx3E0~j7-5&FLXm}~kFRTVPm z`;iw*{}#3!D{1O zPuQ=N4g&@(pWfqo3D*dmR|-VWKM3%|){eI|5ur8Ct#ZCoS<&w zZ#(Brx#;P66^)S~t7Z^mZR;9SVo4u;L&VX-X%OmT z0=qVx1UzE7#FxQr1A1WJ>+bK1av72fWK&LQbc3C&2L{E^CPcYf@3;NX-TqEQO7*E8 zGV@g=*a)jMxi&x;rjI2Obyn#}hiSMGNsF z=M@sMmsem(z<{SU+v7-BTHeO?5gedcATt%rdk64qdy$XDe>6Y|f^JR)xMv|YIlUnc zug<-V1Ik49wNPkbpOQ~~>k0W0p(u|%DS1%uUk~hrVTHi38ImM4sSmDQ_?{C7MSvwP zoi&<4^DKy9dWTsjBA?tbp|zX(EO)l3Pt8r?^t2*4!fGS|)DvP;?_wIDBIj74z!!JK zYV<+NP^4|Mg3=i$57y+WlfX*IYwJ`sju?{$a#AzK!25l{WBpNuU<>gq8Qfu$=oEfVnmd72XIOFTUUaTayr|B*LK>Eg_D`G7lq~*y}2KIZv(pk^6}- z;zCc1|E4ygcuTIY4;a)wXs4jGUh?ay9Ou9fsYYcc2Pz-F%`TcA-1xc_cs_XwW~SbA z9Q1>#nG-@uz2k-B1sNbH34$+<@#o11lOjGu5pk%KF>iTrLYr!@65#A}wIhLYG!`+L zb^w81t9$idIZ%a@(4=V~KHa@-R@M%*4Szi%)EWXr@_=Yki`?v2KqIHT6gGJWv9a=A*(Lbfdna}8x z-xHvq`Zr%y?;3-NgIW1wa9F}JzaRj!Q}n9>(V6ob@@HqhwElUsw>LAUgTdzEi5w25 zF{X68(&|Ws&>K0QfLKf^_0LJrJ`tDoW0Xb&OLZ3oHy0M>I#=EB`APPaU_y3q43xQXFz-Tc>x^^a_4G+d_wwy*L9 z0Bt=2l_>G#E2TX?*970t-W&2P3eI(41NAX4Wy@ZmL#&M2w6fxcM`|+bN?36h=?4bqku_U07|REN#`>jx=pd*$Ia5&SqU16X> z6`&kVaZIo9c%)-9hb`zP*t~E?-Kkk**njD_}2u~2`oLwC#>(btWR zjqD@w?X?HOgn0N`LVkOXC`t4YdV_{oRj$p8@R+i7_`w$Fn$iYGHvMEW z!;8N1K}5JA4mvbgw-63+Yg!u=2{=tPM9&1hsf~`-FmM~4yF(^+_zAned} zE^wM(rZ(~8tGJD-$3Yd=PHnU2b>yr6 z+){P<%$Yu@L7M{3FYP8TV6%${<7PWzP)7Dd%{`WD&ZrE7J|sEUgDC8nIUAhJ943*q z_yILc?Z#_i6fyZwDr&vz2EVD2#=a13n>+cSA$cO@sg1cq?fvXQ_i$~i3~AFhj|=Af z;=Rw%sT>Z}%l#1h9BDD~J(*zCIs_D;d##})sg*4Y6&{Y3mY(i z`&T|H4>7rIHT1^&kAUP%o|NfF>)ri)pmx{Fq~y(zdXNzmr&u6+nzLgI*pOMS=Xx(d zT$bRQO1stXR^^^BQcO${i{rGrI`jhC{uWz{$fXOy8tIZ&#VYq~?-$KDPrY7$UaQ=_pD2UaM@g8gI}O1? z0A3#qdeD*71T*z9BxV-WMG?H^bl$i=9WqV&9Stt4RA5Al&p73JLGYT6rYlko2VDdeGTXK$$neGc8*PB~cx6(&+=%9*c2jGfz?MV=0 zG2`(r04iLR3`kQjY8?clC@G}<;T3|zh_@)^Sj?e5U%4XA%+E5yW=XrgC$%Vf>Qpbp z*F`o9laThyIJ_xT^Cug944_|Z~yXO#lgvH9|4I<00Mu`l1%CrFCpb{0gmLNnJMiR0!d$sW{)|w zp48r-F`SBcRe?>_plY2|1o)NFmv?p-Bi2yEu#Z|}F0UV7){Z1!qXb}KZ@@vf+rF2C z_6@adxlk*b0EAdsqiMTFQF)zqNre(pj9NWJ)g-P7mAs0+nlnzqmzNA;^6c?;f8S(-u0FO;lt9MclcxrEx zobFRLEi(Xj*8#|?;~^fAKt5RVr>~uaNx2sdzp8X%nw^LCHWqfBD7n4Qq>9sytpk@; z04f7Bpeu@Tx*Yh1{K&l4-oH15(f8w zc(QO8%O%}QtE^W~8e^zs-g2S}^>|$)Ra)(zYUNbTxrKjsR@_rBU%pGPL7Ch@6u(x| zPqOs)A%Fhdz>Pb5Ndmq9@UCy$%Z!maetDCItPGQ* zT9%~4yShp{2BNQ63AGYnR?lwkC+wKl$o`=JZc;q3?`D@K>ERbi5Ln;9B~?oj(vtG_ z`Ewy+H{bjfIR62+3xYM_Eyuodu>I%Kkdim1!+MJQ`eZF5&m{=)g65}hb}VCHQ3#W9 z&LLxIaa(c~6Xjsrb_vlfV_>6@(ATbwiLqYtg-fmm2H`yfQ**8@sIx^I@k5|C(G*tM zc`K7O(O~C~0%ielntL8KQp*^aIC$gDpWm4+BK%x(7XJ=jB@i}K(Q5b7YI@0e^&_xV zl#ERKk7FkG7i>5r2Q3XxQ!n|#C7%LeCJ7i76KVU2z%O3s#+|U48zMZHeBt6RV_ zGfJubJ~IDzj|_WMF|w$XYN92J&PFz{kdSR;FG}h?lYK$25tWG(-d5k_^J6NAt0Zl- zC%!UQVZ?4fbj3Il(f!+r#Icc5a?imNY_NHEEth@|W-0B8?V><*E!qjdII5NZtdAN? zLK)QA^eIobcy~WB-pL6qI1Sr8YfJEdh~1|cRaiAbnY#mo(c&rs+2|6CoDbqIkPQ2j z=Fxw!%3mP7T&%3>)Zi4~@nFG!;X8j4um_c|GC67Kq#4Vh=C39eZq5M;tIq2AZ%8!kD>{#~H}q=Z}R0WN84uQ1HWO#L6QsFlkYPan>OAYeW8 zuhva|zjGyt9xqjug^bGH$Nmtm>6==0*0BG@{cS|h1d#sPwogFd=807mQ(Jc?A!jUnH)6VlLHC|WoSKTN5B8n@zCrie^5|ONMg^f?$+}=oJvNOP(OCK~zhw;nykK`w3J3^nOx2zGZez0tUFdA36uc?2P)R&* zk6*w%0=svOMDcfmz8;z5VJAZ>l#Mcj{#D_1hseG@Yd*AEpRkto-&ZU;t?=s;MlRGc z>e1hBI_zzViO6I#mb;SpivRmQZbYVCc}$Y+w|@l61;ZTa^QUT?{LKHQ_NaS8+I2=y z!D{@$ffss9=_Y%Cr-i+L-Ej8XKjO?tfCbB{zp3iJnG}QO^uPI&PrOt`F(n@?2)Xp; zm41&*u|el&ehnS@Z?hVq!UDVR4Y{243sfxjTvr`@Hx*UaYoz0J*hy^`%&krS^$nVDoRn zd;#6w{x*&|9!~s4{%Hv{|uN9c+IaL@ehL=01O%(+O~{=t%HG`-Xk0S>rOj) zhBAnvo4y4>W^sP1OMVarFlF^4W^-at;w84VP6Fot|72qvP1jyqv4ZTh++PROezwcM zH|g4io_xlPqZ@wRXjw^OOHHF{m> zX|OK*YZ>M77ckS5u38ptk}>BB=;`V8Io%|;iW1RqOP)&RGLrESdw*M*FGi~>*t4f= zL(eZ~k#aUA_H1Xky#}%jt+(>@_=bocrGA)SKqC5T$^aw@vXr2veUeXnb}<++IJY zp!Lsy)T1I3AL?J6=-ogRl7jc}yc`HqX{lLsAMl3ZyphN3b&_YBRhbgu$y(N;tRr<- ztCW0|;~>hVU=^wy6w>BX*JCRmo!V=(tid?4f2>m7|0$s+Gf57UcI?i^ps)JoUk2(; z!#?qK&JX#&KVU_uw6rXqw7|QE9kkV)`dTn-*Z$wHm|rWb&BljC zBLlVBv_4*0nVbE*0^>Yq4f@`v$JM&W)pp1E5>2HqA;0g|VL9Ti4jri!veBIq8#}Vr z%+6!*&2DaH, + "api_key": +} +``` \ No newline at end of file diff --git a/docs/parsnip.md b/docs/parsnip.md new file mode 100644 index 0000000..ccce24f --- /dev/null +++ b/docs/parsnip.md @@ -0,0 +1,131 @@ +# `Parsnip` + +## 🥕 `argparse` with conditional argument groups. + +As `minimax.train` is the single point-of-entry for training, its command-line arguments can grow quickly in number with each additional autocurriculum method supported in `minimax`. This complexity arises for several reasons: + +- New components in the form of training runners, environments, agents, and models may require additional arguments +- New components may require existing arguments shared with previous components +- New components may overload the meaning of existing arguments used by other components + +We make use of a custom module called `Parsnip` to help manage the complexity of specifying and parsing command-line arguments. `Parsnip` allows the creation of named argument groups, which allows adding new arguments while explicitly separating them into name spaces. Each argument group results in its own kwarg dictionary when parsed. + +`Parsnip` directly builds on `argparse` by adding the notion of a "subparser". Here, a subparser is simply an `argparse` parser responsible for a named argument group. Subparsers enable some useful behavior: +- Arguments can be added to the top-level `Parsnip` parser or to a subparser. +- Each subparser is initialized with a `name` for its corresponding argument group. All arguments under this subparser will be contained in a nested kwarg dictionary under the key equal to `name`. +- Each subparser can be initialized with an optional `prefix`, in which case all command-line arguments added to the subparser will be prepended with the value of `prefix` (see example below), thus creating a namespace for the corresponding argument group. +- Subparsers can be added conditionally, based on the specific value of a top-level argument (with support for the wildcard `*`). +- After parsing, `Parsnip` produces a kwargs dictionary containing a key:value pair for each top-level argument and a nested kwargs dictionary, under the key `` containing the parsed arguments managed by each active subparser initialized with `prefix=`. + +Other than these details, `Parsnip`'s interface remains identical to that of `argparse`. + +## A minimal example +In this example, we assume the parser is used inside a script called `run.py`. + +```python +from util.parsnip import Parsnip + +# Create a new Parsnip parser +parser = Parsnip() + +# Add some top-level arguments (same as argparse) +parser.add_argument( + '--name', + type=str, + help='Name of my farm.') +parser.add_argument( + '--kind', + type=str, + choices=['apple', 'radish'], + help='What kind of farm I run.') +parser.add_argument( + '--n_acres', + type=str, + help='Size of my farm in acres.') + +# Create a nested argument group with a prefix +crop_subparser = parser.add_subparser(name='crop', prefix='crop') +parser.add_argument( + '--n_acres', + type=str, + help='Size of land for growing radish, in acres.') + +# Create a conditional argument group +radish_subparser = parser.add_subparser( + name='radish', + prefix='radish', + dependency={'crop': 'radish'}, + dest='crop') +radish_subparser.add_argument( + '--is_pickled' + type=str2bool, + default=False, + help='Whether my farm produces pickled radish.') + +# Create another conditional argument group +apple_subparser = parser.add_subparser( + name='apple', + prefix='apple', + dependency={'crop': 'apple'}, + dest='crop') +apple_subparser.add_argument( + '--kind' + type=str, + choices=['fuji', 'mcintosh'], + default='fuji', + help='Whether my farm produces pickled radish.') + +args = parser.parse_args() +``` + +Then running this command + +```bash +python run.py \ +--name 'Radelicious Farms' \ +--kind radish \ +--n_acres 200 \ +--crop_n_acres 150 \ +--radish_is_pickled +``` + +would produce this kwargs dictionary: + +```python +{ + 'name': 'Radelicious Farms', + 'kind': 'radish', + 'n_acres': 200, + 'crop_args': { + 'n_acres': 150, + 'is_pickled': True + } +} +``` + +Notice how the `prefix` for each subparser is appended to each argument name added to that subparser (e.g. `n_acres` became `crop_n_acres`, and `is_pickled` became `radish_is_pickled`). Also notice how the `radish_is_pickled` argument became active, as its activation conditions on `kind=radish`, as we specified when defining the `radish_subparser`. + +Likewise, running this argument + +```bash +python run.py \ +--name 'Appledores Farms' \ +--kind apple \ +--n_acres 200 \ +--crop_n_acres 150 \ +--apple_kind fuji +``` + +results in this kwargs dictionary: + +```python +{ + 'name': 'Appledores Farms', + 'kind': 'apple', + 'n_acres': 200, + 'crop_args': { + 'n_acres': 150, + 'kind': 'fuji' + } +} +``` \ No newline at end of file diff --git a/docs/train_args.md b/docs/train_args.md new file mode 100644 index 0000000..17262e4 --- /dev/null +++ b/docs/train_args.md @@ -0,0 +1,125 @@ +# Command-line usage guide for `minimax.train` + +Parsing command-line arguments is handled by [`Parsnip`](parsnip.md). + +You can quickly generate batches of training commands from a JSON configuration file using [`minimax.config.make_cmd`](make_cmd.md). + +## General arguments + +| Argument | Description | +| ----------------------- | ---------------------------------------------------------------------------------------------------- | +| `seed` | Random seed, should be unique per experimental run | +| `agent_rl_algo` | Base RL algorithm used for training (e.g. PPO) | +| `n_total_updates` | Total number of updates for the training run | +| `train_runner` | Which training runner to use, e.g. `dr`, `plr`, or `paired` | +| `n_devices` | Number of devices over which to shard the environment batch dimension | +| `n_students` | Number of students in the autocurriculum | +| `n_parallel` | Number of parallel environments | +| `n_eval` | Number of parallel trials per environment (environment batch dimension is then `n_parallel*n_eval`) | +| `n_rollout_steps` | Number of steps per rollout (used for each update cycle) | +| `lr` | Learning rate | +| `lr_final` | Final learning rate, based on linear schedule. Defaults to `None`, corresponding to no schedule. | +| `lr_anneal_steps` | Number of steps over which to linearly anneal from `lr` to `lr_final` | +| `student_value_coef` | Value loss coefficient | +| `student_entropy_coef` | Entropy bonus coefficient | +| `student_unroll_update` | Unroll multi-gradient updates this many times (can lead to speed ups) | +| `max_grad_norm` | Clip gradients beyond this magnitude | +| `adam_eps` | Value of $`\epsilon`$ numerical stability constant for Adam | +| `discount` | Discount factor $`\gamma`$ for the student's RL optimization | +| `n_unroll_rollout` | Unroll rollout scans this many times (can lead to speed ups) | + +## Logging arguments + +| Argument | Description | +| ------------------- | -------------------------------------------------------- | +| `verbose` | Random seed, should be unique per experimental run | +| `track_env_metrics` | Track per rollout batch environment metrics if `True` | +| `log_dir` | Path to directory storing all experiment folders | +| `xpid` | Unique name for experiment folder, stored in `--log_dir` | +| `log_interval` | Log training statistics every this many rollout cycles | +| `wandb_base_url` | Base API URL if logging with `wandb` | +| `wandb_api_key` | API key for `wandb` | +| `wandb_entity` | `wandb` entity associated with the experiment run | +| `wandb_project` | `wandb` project for the experiment run | +| `wandb_group` | `wandb` group for the experiment run | + +## Checkpointing arguments + +| Argument | Description | +| ---------------------- | ----------------------------------------------------------------------------- | +| `checkpoint_interval` | Random seed, should be unique per experimental run | +| `from_last_checkpoint` | Begin training from latest `checkpoint.pkl`, if any, in the experiment folder | +| `archive_interval` | Save an additional checkpoint for models trained per this many rollout cycles | + +## Evaluation arguments + +| Argument | Description | +| ----------------- | -------------------------------------------------------------------- | +| `test_env_names` | Random seed, should be unique per experimental run | +| `test_n_episodes` | Average test results over this many episodes per test environment | +| `test_agent_idxs` | Test agents at these indices (csv of indices or `*` for all indices) | + +## PPO arguments + +These arguments activate when `--agent_rl_algo=ppo`. + +| Argument | Description | +| ----------------------------- | ----------------------------------------------------------- | +| `student_ppo_n_epochs` | Random seed, should be unique per experimental run | +| `student_ppo_n_epochs` | Number of PPO epochs per update cycle | +| `student_ppo_n_minibatches` | Number of minibatches per PPO epoch | +| `student_ppo_clip_eps` | Clip coefficient for PPO | +| `student_ppo_clip_value_loss` | Perform value clipping if `True` | +| `gae_lambda` | Lambda discount factor for Generalized Advantage Estimation | + +## PAIRED arguments + +The arguments in this section activate when `--train_runner=paired`. + +| Argument | Description | +| ------------------------- | --------------------------------------------------------------------- | +| `teacher_lr` | Learning rate | +| `teacher_lr_final` | Anneal learning rate to this value (defaults to `teacher_lr`) | +| `teacher_lr_anneal_steps` | Number of steps over which to linearly anneal from `lr` to `lr_final` | +| `teacher_discount` | Discount factor, $`\gamma`$ | +| `teacher_value_loss_coef` | Value loss coefficient | +| `teacher_entropy_coef` | Entropy bonus coefficient | +| `teacher_n_unroll_update` | Unroll multi-gradient updates this many times (can lead to speed ups) | +| `ued_score` | Name of UED objective, e.g. `relative_regret` | + +These PPO-specific arguments for teacher optimization further activate when `--agent_rl_algo=ppo`. + +| Argument | Description | +| ----------------------------- | ----------------------------------------------------------- | +| `teacher_ppo_n_epochs` | Number of PPO epochs per update cycle | +| `teacher_ppo_n_minibatches` | Number of minibatches per PPO epoch | +| `teacher_ppo_clip_eps` | Clip coefficient for PPO | +| `teacher_ppo_clip_value_loss` | Perform value clipping if `True` | +| `teacher_gae_lambda` | Lambda discount factor for Generalized Advantage Estimation | + +## PLR arguments + +The arguments in this section activate when `--train_runner=paired`. + +| Argument | Description | +| ----------------------------- | ------------------------------------------------------------------------------------------------------------- | +| `ued_score` | Name of UED objective (aka PLR scoring function) | +| `plr_replay_prob` | Replay probability | +| `plr_buffer_size` | Size of level replay buffer | +| `plr_staleness_coef` | Staleness coefficient | +| `plr_temp` | Score distribution temperature | +| `plr_use_score_ranks` | Use rank-based prioritization (rather than proportional) | +| `plr_min_fill_ratio` | Only replay once level replay buffer is filled above this ratio | +| `plr_use_robust_plr` | Use robust PLR (i.e. only update policy on replay levels) | +| `plr_force_unique` | Force level replay buffer members to be unique | +| `plr_use_parallel_eval` | Use Parallel PLR or Parallel ACCEL (if `plr_mutation_fn` is set) | +| `plr_mutation_fn` | If set, PLR becomes ACCEL. Use `'default'` for default mutation operator per environment. | +| `plr_n_mutations` | Number of applications of `plr_mutation_fn` per mutation cycle. | +| `plr_mutation_criterion` | How replay levels are selected for mutation (e.g. `batch`, `easy`, `hard`). | +| `plr_mutation_subsample_size` | Number of replay levels selected for mutation according to the criterion (ignored if using `batch` criterion) | + +## Environment-specific arguments + +### Maze + +See the [`AMaze`](envs/maze.md) docs for details on how to specify [training](envs/maze.md#student-environment), [evaluation](envs/maze.md#student-environment), and [teacher-specific](envs/maze.md#teacher-environment) environment parameters via command line diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f1de7d6 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +numpy>=1.25,<1.26 +pandas==1.5.3 +jax>=0.4.19 +jaxlib>=0.4.19 +flax>=0.7.4 +optax>=0.1.7 +chex>=0.1.83 +wandb>=0.13 +ipython>=7.34.0 +GitPython>=3.1.29 \ No newline at end of file diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/baseline_train__8_seeds.sh b/src/baseline_train__8_seeds.sh new file mode 100755 index 0000000..a9730f9 --- /dev/null +++ b/src/baseline_train__8_seeds.sh @@ -0,0 +1,72 @@ +DEFAULTVALUE=4 +device="${1:-$DEFAULTVALUE}" +layout=$2 + +seed_max=8 + +for seed in `seq ${seed_max}`; +do + echo "seed is ${seed}:" + CUDA_VISIBLE_DEVICES=${device} XLA_PYTHON_CLIENT_MEM_FRACTION=.40 LD_LIBRARY_PATH="" nice -n 5 python3 -m minimax.train \ + --wandb_mode=online \ + --wandb_project=overcooked-minimax-jax \ + --wandb_entity=${WANDB_ENTITY} \ + --seed=${seed} \ + --agent_rl_algo=ppo \ + --n_total_updates=1000 \ + --train_runner=dr \ + --n_devices=1 \ + --student_model_name=default_student_actor_cnn \ + --student_critic_model_name=default_student_critic_cnn \ + --env_name=Overcooked \ + --is_multi_agent=True \ + --verbose=False \ + --log_dir=~/logs/minimax \ + --log_interval=10 \ + --from_last_checkpoint=False \ + --checkpoint_interval=25 \ + --archive_interval=25 \ + --archive_init_checkpoint=False \ + --test_interval=50 \ + --n_students=1 \ + --n_parallel=100 \ + --n_eval=1 \ + --n_rollout_steps=400 \ + --lr=3e-4 \ + --lr_anneal_steps=0 \ + --max_grad_norm=0.5 \ + --adam_eps=1e-05 \ + --track_env_metrics=True \ + --discount=0.99 \ + --n_unroll_rollout=10 \ + --render=False \ + --student_gae_lambda=0.95 \ + --student_entropy_coef=0.01 \ + --student_value_loss_coef=0.5 \ + --student_n_unroll_update=5 \ + --student_ppo_n_epochs=5 \ + --student_ppo_n_minibatches=1 \ + --student_ppo_clip_eps=0.2 \ + --student_ppo_clip_value_loss=True \ + --student_hidden_dim=64 \ + --student_n_hidden_layers=3 \ + --student_n_conv_layers=3 \ + --student_n_conv_filters=32 \ + --student_n_scalar_embeddings=4 \ + --student_scalar_embed_dim=5 \ + --student_agent_kind=mappo \ + --overcooked_height=6 \ + --overcooked_width=9 \ + --overcooked_n_walls=15 \ + --overcooked_replace_wall_pos=True \ + --overcooked_sample_n_walls=True \ + --overcooked_normalize_obs=True \ + --overcooked_max_steps=400 \ + --overcooked_random_reset=False \ + --overcooked_fix_to_single_layout=${layout} \ + --n_shaped_reward_steps=3000000 \ + --test_n_episodes=10 \ + --test_env_names=Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9 \ + --overcooked_test_normalize_obs=True \ + --xpid=8SEED_${seed}_dr-overcookedNonexNonewNone_fs_FIX${layout}_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr3e-5g0.99cv0.5ce0.01e5mb1l0.95_pc0.2_h64cf32fc2se5ba_re_0 +done \ No newline at end of file diff --git a/src/baseline_train__all.sh b/src/baseline_train__all.sh new file mode 100755 index 0000000..83b5a58 --- /dev/null +++ b/src/baseline_train__all.sh @@ -0,0 +1,8 @@ +DEFAULTVALUE=4 +device="${1:-$DEFAULTVALUE}" + +./baseline_train__8_seeds.sh $device coord_ring_6_9 +./baseline_train__8_seeds.sh $device counter_circuit_6_9 +./baseline_train__8_seeds.sh $device forced_coord_6_9 +./baseline_train__8_seeds.sh $device cramped_room_6_9 +./baseline_train__8_seeds.sh $device asymm_advantages_6_9 \ No newline at end of file diff --git a/src/baseline_train__holdout_sp.sh b/src/baseline_train__holdout_sp.sh new file mode 100755 index 0000000..af2b080 --- /dev/null +++ b/src/baseline_train__holdout_sp.sh @@ -0,0 +1,71 @@ +DEFAULTVALUE=4 +device="${1:-$DEFAULTVALUE}" + +seed=42 + +for layout in "coord_ring_6_9" "forced_coord_6_9" "cramped_room_6_9" "asymm_advantages_6_9" "counter_circuit_6_9"; +do + echo "layout is ${layout}:" + CUDA_VISIBLE_DEVICES=${device} XLA_PYTHON_CLIENT_MEM_FRACTION=.40 LD_LIBRARY_PATH="" nice -n 5 python3 -m minimax.train \ + --wandb_mode=online \ + --wandb_project=overcooked-minimax-jax \ + --wandb_entity=${WANDB_ENTITY} \ + --seed=${seed} \ + --agent_rl_algo=ppo \ + --n_total_updates=1000 \ + --train_runner=dr \ + --n_devices=1 \ + --student_model_name=default_student_actor_cnn \ + --student_critic_model_name=default_student_critic_cnn \ + --env_name=Overcooked \ + --is_multi_agent=True \ + --verbose=False \ + --log_dir=~/logs/minimax \ + --log_interval=10 \ + --from_last_checkpoint=False \ + --checkpoint_interval=25 \ + --archive_interval=25 \ + --archive_init_checkpoint=False \ + --test_interval=50 \ + --n_students=1 \ + --n_parallel=100 \ + --n_eval=1 \ + --n_rollout_steps=400 \ + --lr=3e-4 \ + --lr_anneal_steps=0 \ + --max_grad_norm=0.5 \ + --adam_eps=1e-05 \ + --track_env_metrics=True \ + --discount=0.99 \ + --n_unroll_rollout=10 \ + --render=False \ + --student_gae_lambda=0.95 \ + --student_entropy_coef=0.01 \ + --student_value_loss_coef=0.5 \ + --student_n_unroll_update=5 \ + --student_ppo_n_epochs=5 \ + --student_ppo_n_minibatches=1 \ + --student_ppo_clip_eps=0.2 \ + --student_ppo_clip_value_loss=True \ + --student_hidden_dim=64 \ + --student_n_hidden_layers=3 \ + --student_n_conv_layers=3 \ + --student_n_conv_filters=32 \ + --student_n_scalar_embeddings=4 \ + --student_scalar_embed_dim=5 \ + --student_agent_kind=mappo \ + --overcooked_height=6 \ + --overcooked_width=9 \ + --overcooked_n_walls=15 \ + --overcooked_replace_wall_pos=True \ + --overcooked_sample_n_walls=True \ + --overcooked_normalize_obs=True \ + --overcooked_max_steps=400 \ + --overcooked_random_reset=False \ + --overcooked_fix_to_single_layout=${layout} \ + --n_shaped_reward_steps=3000000 \ + --test_n_episodes=10 \ + --test_env_names=Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9 \ + --overcooked_test_normalize_obs=True \ + --xpid=9SEED_${seed}_dr-overcookedNonexNonewNone_fs_FIX${layout}_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr3e-5g0.99cv0.5ce0.01e5mb1l0.95_pc0.2_h64cf32fc2se5ba_re_0 +done \ No newline at end of file diff --git a/src/config/configs/maze/accel.json b/src/config/configs/maze/accel.json new file mode 100644 index 0000000..bdd7c22 --- /dev/null +++ b/src/config/configs/maze/accel.json @@ -0,0 +1,73 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_cnn"], + "env_name": ["Maze"], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [true], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [256], + "lr": [0.0001], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.995], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.8], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.1], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [false], + "plr_force_unique": [true], + "plr_mutation_fn": ["default"], + "plr_n_mutations": [20], + "plr_mutation_criterion": ["batch"], + "plr_mutation_subsample_size": [4], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.0], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [32], + "student_n_hidden_layers": [1], + "student_n_conv_filters": [16], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "maze_height": [13], + "maze_width": [13], + "maze_n_walls": [0], + "maze_replace_wall_pos": [true], + "maze_sample_n_walls": [false], + "maze_see_agent": [false], + "maze_normalize_obs": [true], + "maze_obs_agent_pos": [false], + "maze_max_episode_steps": [250], + "test_n_episodes": [10], + "test_env_names": ["Maze-SixteenRooms,Maze-Labyrinth,Maze-StandardMaze"], + "maze_test_see_agent": [false], + "maze_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/maze/dr.json b/src/config/configs/maze/dr.json new file mode 100644 index 0000000..57c7c98 --- /dev/null +++ b/src/config/configs/maze/dr.json @@ -0,0 +1,59 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["dr"], + "n_devices": [1], + "student_model_name": ["default_student_cnn"], + "env_name": ["Maze"], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [true], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [256], + "lr": [0.0001], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.995], + "n_unroll_rollout": [10], + "render": [false], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.001], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [32], + "student_n_hidden_layers": [1], + "student_n_conv_filters": [16], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "maze_height": [13], + "maze_width": [13], + "maze_n_walls": [60], + "maze_replace_wall_pos": [true], + "maze_sample_n_walls": [false], + "maze_see_agent": [false], + "maze_normalize_obs": [true], + "maze_obs_agent_pos": [false], + "maze_max_episode_steps": [250], + "test_n_episodes": [10], + "test_env_names": ["Maze-SixteenRooms,Maze-Labyrinth,Maze-StandardMaze"], + "maze_test_see_agent": [false], + "maze_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/maze/paccel.json b/src/config/configs/maze/paccel.json new file mode 100644 index 0000000..10694da --- /dev/null +++ b/src/config/configs/maze/paccel.json @@ -0,0 +1,73 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_cnn"], + "env_name": ["Maze"], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [true], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [256], + "lr": [0.0001], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.995], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.8], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.1], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [true], + "plr_force_unique": [true], + "plr_mutation_fn": ["default"], + "plr_n_mutations": [10], + "plr_mutation_criterion": ["batch"], + "plr_mutation_subsample_size": [4], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.0], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [32], + "student_n_hidden_layers": [1], + "student_n_conv_filters": [16], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "maze_height": [13], + "maze_width": [13], + "maze_n_walls": [0], + "maze_replace_wall_pos": [true], + "maze_sample_n_walls": [false], + "maze_see_agent": [false], + "maze_normalize_obs": [true], + "maze_obs_agent_pos": [false], + "maze_max_episode_steps": [250], + "test_n_episodes": [10], + "test_env_names": ["Maze-SixteenRooms,Maze-Labyrinth,Maze-StandardMaze"], + "maze_test_see_agent": [false], + "maze_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/maze/paired.json b/src/config/configs/maze/paired.json new file mode 100644 index 0000000..ff0a370 --- /dev/null +++ b/src/config/configs/maze/paired.json @@ -0,0 +1,84 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["paired"], + "n_devices": [1], + "student_model_name": ["default_student_cnn"], + "env_name": ["Maze"], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [true], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [2], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [256], + "lr": [0.0001], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.995], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["relative_regret"], + "student_gae_lambda": [0.98], + "teacher_discount": [0.995], + "teacher_lr_anneal_steps": [0], + "teacher_gae_lambda": [0.98], + "student_entropy_coef": [0.001], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "teacher_entropy_coef": [0.05], + "teacher_value_loss_coef": [0.5], + "teacher_n_unroll_update": [5], + "teacher_ppo_n_epochs": [5], + "teacher_ppo_n_minibatches": [1], + "teacher_ppo_clip_eps": [0.2], + "teacher_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [32], + "student_n_hidden_layers": [1], + "student_n_conv_filters": [16], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "teacher_model_name": ["default_teacher_cnn"], + "teacher_recurrent_arch": ["lstm"], + "teacher_recurrent_hidden_dim": [256], + "teacher_hidden_dim": [32], + "teacher_n_hidden_layers": [1], + "teacher_n_conv_filters": [128], + "teacher_scalar_embed_dim": [10], + "maze_height": [13], + "maze_width": [13], + "maze_n_walls": [60], + "maze_replace_wall_pos": [false], + "maze_sample_n_walls": [false], + "maze_see_agent": [false], + "maze_normalize_obs": [true], + "maze_obs_agent_pos": [false], + "maze_max_episode_steps": [250], + "maze_ued_replace_wall_pos": [true], + "maze_ued_fixed_n_wall_steps": [true], + "maze_ued_first_wall_pos_sets_budget": [false], + "maze_ued_noise_dim": [50], + "maze_ued_n_walls": [60], + "maze_ued_set_agent_dir": [false], + "maze_ued_normalize_obs": [true], + "test_n_episodes": [10], + "test_env_names": ["Maze-SixteenRooms,Maze-Labyrinth,Maze-StandardMaze"], + "maze_test_see_agent": [false], + "maze_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/maze/plr.json b/src/config/configs/maze/plr.json new file mode 100644 index 0000000..229fbf2 --- /dev/null +++ b/src/config/configs/maze/plr.json @@ -0,0 +1,69 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_cnn"], + "env_name": ["Maze"], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [256], + "lr": [5e-05], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.995], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.5], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.1], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [false], + "plr_force_unique": [true], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.0], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [32], + "student_n_hidden_layers": [1], + "student_n_conv_filters": [16], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "maze_height": [13], + "maze_width": [13], + "maze_n_walls": [60], + "maze_replace_wall_pos": [true], + "maze_sample_n_walls": [false], + "maze_see_agent": [false], + "maze_normalize_obs": [true], + "maze_obs_agent_pos": [false], + "maze_max_episode_steps": [250], + "test_n_episodes": [10], + "test_env_names": ["Maze-SixteenRooms,Maze-Labyrinth,Maze-StandardMaze"], + "maze_test_see_agent": [false], + "maze_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/maze/pplr.json b/src/config/configs/maze/pplr.json new file mode 100644 index 0000000..071cab2 --- /dev/null +++ b/src/config/configs/maze/pplr.json @@ -0,0 +1,69 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_cnn"], + "env_name": ["Maze"], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [256], + "lr": [0.0001], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.5], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.3], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [true], + "plr_force_unique": [true], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.0], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [32], + "student_n_hidden_layers": [1], + "student_n_conv_filters": [16], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "maze_height": [13], + "maze_width": [13], + "maze_n_walls": [60], + "maze_replace_wall_pos": [true], + "maze_sample_n_walls": [false], + "maze_see_agent": [false], + "maze_normalize_obs": [true], + "maze_obs_agent_pos": [false], + "maze_max_episode_steps": [250], + "test_n_episodes": [10], + "test_env_names": ["Maze-SixteenRooms,Maze-Labyrinth,Maze-StandardMaze"], + "maze_test_see_agent": [false], + "maze_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/maze/s5_accel.json b/src/config/configs/maze/s5_accel.json new file mode 100644 index 0000000..f2fbf76 --- /dev/null +++ b/src/config/configs/maze/s5_accel.json @@ -0,0 +1,78 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_cnn"], + "env_name": ["Maze"], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [true], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [256], + "lr": [3e-05], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.8], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.3], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [false], + "plr_force_unique": [true], + "plr_mutation_fn": ["default"], + "plr_n_mutations": [10], + "plr_mutation_criterion": ["batch"], + "plr_mutation_subsample_size": [4], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.001], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["s5"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [32], + "student_n_hidden_layers": [1], + "student_n_conv_filters": [16], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_s5_n_blocks": [2], + "student_s5_n_layers": [2], + "student_s5_layernorm_pos": ["post"], + "student_s5_activation": ["half_glu1"], + "maze_height": [13], + "maze_width": [13], + "maze_n_walls": [0], + "maze_replace_wall_pos": [true], + "maze_sample_n_walls": [false], + "maze_see_agent": [false], + "maze_normalize_obs": [true], + "maze_obs_agent_pos": [false], + "maze_max_episode_steps": [250], + "test_n_episodes": [10], + "test_env_names": ["Maze-SixteenRooms,Maze-Labyrinth,Maze-StandardMaze"], + "test_agent_idxs": ["\"*\""], + "maze_test_see_agent": [false], + "maze_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/maze/s5_dr.json b/src/config/configs/maze/s5_dr.json new file mode 100644 index 0000000..5f688c5 --- /dev/null +++ b/src/config/configs/maze/s5_dr.json @@ -0,0 +1,63 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["dr"], + "n_devices": [1], + "student_model_name": ["default_student_cnn"], + "env_name": ["Maze"], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [true], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [256], + "lr": [3e-05], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.995], + "n_unroll_rollout": [10], + "render": [false], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.001], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["s5"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [32], + "student_n_hidden_layers": [1], + "student_n_conv_filters": [16], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_s5_n_blocks": [2], + "student_s5_n_layers": [2], + "student_s5_layernorm_pos": ["post"], + "student_s5_activation": ["half_glu1"], + "maze_height": [13], + "maze_width": [13], + "maze_n_walls": [60], + "maze_replace_wall_pos": [true], + "maze_sample_n_walls": [false], + "maze_see_agent": [false], + "maze_normalize_obs": [true], + "maze_obs_agent_pos": [false], + "maze_max_episode_steps": [250], + "test_n_episodes": [10], + "test_env_names": ["Maze-SixteenRooms,Maze-Labyrinth,Maze-StandardMaze"], + "maze_test_see_agent": [false], + "maze_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/maze/s5_paccel.json b/src/config/configs/maze/s5_paccel.json new file mode 100644 index 0000000..d61f0c2 --- /dev/null +++ b/src/config/configs/maze/s5_paccel.json @@ -0,0 +1,77 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_cnn"], + "env_name": ["Maze"], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [true], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [256], + "lr": [1e-05], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.8], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.3], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [true], + "plr_force_unique": [true], + "plr_mutation_fn": ["default"], + "plr_n_mutations": [20], + "plr_mutation_criterion": ["batch"], + "plr_mutation_subsample_size": [4], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.0], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["s5"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [32], + "student_n_hidden_layers": [1], + "student_n_conv_filters": [16], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_s5_n_blocks": [2], + "student_s5_n_layers": [2], + "student_s5_layernorm_pos": ["post"], + "student_s5_activation": ["half_glu1"], + "maze_height": [13], + "maze_width": [13], + "maze_n_walls": [0], + "maze_replace_wall_pos": [true], + "maze_sample_n_walls": [false], + "maze_see_agent": [false], + "maze_normalize_obs": [true], + "maze_obs_agent_pos": [false], + "maze_max_episode_steps": [250], + "test_n_episodes": [10], + "test_env_names": ["Maze-SixteenRooms,Maze-Labyrinth,Maze-StandardMaze"], + "maze_test_see_agent": [false], + "maze_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/maze/s5_paired.json b/src/config/configs/maze/s5_paired.json new file mode 100644 index 0000000..451bd40 --- /dev/null +++ b/src/config/configs/maze/s5_paired.json @@ -0,0 +1,94 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["paired"], + "n_devices": [1], + "student_model_name": ["default_student_cnn"], + "env_name": ["Maze"], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [true], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [2], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [256], + "lr": [0.0001], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.995], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["relative_regret"], + "student_gae_lambda": [0.98], + "teacher_discount": [0.995], + "teacher_lr": [0.0001], + "teacher_lr_anneal_steps": [0], + "teacher_gae_lambda": [0.98], + "student_entropy_coef": [0.001], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "teacher_entropy_coef": [0.001], + "teacher_value_loss_coef": [0.5], + "teacher_n_unroll_update": [5], + "teacher_ppo_n_epochs": [5], + "teacher_ppo_n_minibatches": [1], + "teacher_ppo_clip_eps": [0.2], + "teacher_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["s5"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [32], + "student_n_hidden_layers": [1], + "student_n_conv_filters": [16], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_s5_n_blocks": [2], + "student_s5_n_layers": [2], + "student_s5_layernorm_pos": ["post"], + "student_s5_activation": ["half_glu1"], + "teacher_model_name": ["default_teacher_cnn"], + "teacher_recurrent_arch": ["s5"], + "teacher_recurrent_hidden_dim": [256], + "teacher_hidden_dim": [32], + "teacher_n_hidden_layers": [1], + "teacher_n_conv_filters": [32], + "teacher_scalar_embed_dim": [10], + "teacher_s5_n_blocks": [2], + "teacher_s5_n_layers": [2], + "teacher_s5_layernorm_pos": ["post"], + "teacher_s5_activation": ["half_glu1"], + "maze_height": [13], + "maze_width": [13], + "maze_n_walls": [60], + "maze_replace_wall_pos": [false], + "maze_sample_n_walls": [false], + "maze_see_agent": [false], + "maze_normalize_obs": [true], + "maze_obs_agent_pos": [false], + "maze_max_episode_steps": [250], + "maze_ued_replace_wall_pos": [true], + "maze_ued_fixed_n_wall_steps": [true], + "maze_ued_first_wall_pos_sets_budget": [false], + "maze_ued_noise_dim": [50], + "maze_ued_n_walls": [60], + "maze_ued_set_agent_dir": [false], + "maze_ued_normalize_obs": [true], + "test_n_episodes": [10], + "test_env_names": ["Maze-SixteenRooms,Maze-Labyrinth,Maze-StandardMaze"], + "test_agent_idxs": ["\"*\""], + "maze_test_see_agent": [false], + "maze_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/maze/s5_plr.json b/src/config/configs/maze/s5_plr.json new file mode 100644 index 0000000..05a9146 --- /dev/null +++ b/src/config/configs/maze/s5_plr.json @@ -0,0 +1,73 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_cnn"], + "env_name": ["Maze"], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [true], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [256], + "lr": [3e-05], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.5], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.3], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [false], + "plr_force_unique": [true], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.001], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["s5"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [32], + "student_n_hidden_layers": [1], + "student_n_conv_filters": [16], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_s5_n_blocks": [2], + "student_s5_n_layers": [2], + "student_s5_layernorm_pos": ["pre"], + "student_s5_activation": ["half_glu1"], + "maze_height": [13], + "maze_width": [13], + "maze_n_walls": [60], + "maze_replace_wall_pos": [true], + "maze_sample_n_walls": [false], + "maze_see_agent": [false], + "maze_normalize_obs": [true], + "maze_obs_agent_pos": [false], + "maze_max_episode_steps": [250], + "test_n_episodes": [10], + "test_env_names": ["Maze-SixteenRooms,Maze-Labyrinth,Maze-StandardMaze"], + "maze_test_see_agent": [false], + "maze_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/maze/s5_pplr.json b/src/config/configs/maze/s5_pplr.json new file mode 100644 index 0000000..cccdce6 --- /dev/null +++ b/src/config/configs/maze/s5_pplr.json @@ -0,0 +1,73 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_cnn"], + "env_name": ["Maze"], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [true], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [256], + "lr": [3e-05], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.995], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.5], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.3], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [true], + "plr_force_unique": [true], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.001], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["s5"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [32], + "student_n_hidden_layers": [1], + "student_n_conv_filters": [16], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_s5_n_blocks": [2], + "student_s5_n_layers": [2], + "student_s5_layernorm_pos": ["post"], + "student_s5_activation": ["half_glu1"], + "maze_height": [13], + "maze_width": [13], + "maze_n_walls": [60], + "maze_replace_wall_pos": [true], + "maze_sample_n_walls": [false], + "maze_see_agent": [false], + "maze_normalize_obs": [true], + "maze_obs_agent_pos": [false], + "maze_max_episode_steps": [250], + "test_n_episodes": [10], + "test_env_names": ["Maze-SixteenRooms,Maze-Labyrinth,Maze-StandardMaze"], + "maze_test_see_agent": [false], + "maze_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline__cnn_asymm_advantages.json b/src/config/configs/overcooked/baseline__cnn_asymm_advantages.json new file mode 100644 index 0000000..6999ad4 --- /dev/null +++ b/src/config/configs/overcooked/baseline__cnn_asymm_advantages.json @@ -0,0 +1,63 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [1000], + "train_runner": ["dr"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "is_multi_agent": [true], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [25], + "archive_interval": [25], + "archive_init_checkpoint": [false], + "test_interval": [50], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [5e-4], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.99], + "n_unroll_rollout": [10], + "render": [false], + "student_gae_lambda": [0.95], + "student_entropy_coef": [0.01], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_hidden_dim": [64], + "student_n_hidden_layers": [2], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_agent_kind": ["mappo"], + "overcooked_height": [6], + "overcooked_width": [9], + "overcooked_n_walls": [15], + "overcooked_replace_wall_pos": [true], + "overcooked_sample_n_walls": [true], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [false], + "overcooked_fix_to_single_layout": ["asymm_advantages_6_9"], + "n_shaped_reward_steps": [5000000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline__s5_asymm_advantages.json b/src/config/configs/overcooked/baseline__s5_asymm_advantages.json new file mode 100644 index 0000000..0a7961e --- /dev/null +++ b/src/config/configs/overcooked/baseline__s5_asymm_advantages.json @@ -0,0 +1,69 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [1000], + "train_runner": ["dr"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "is_multi_agent": [true], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [100], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-05], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.995], + "n_unroll_rollout": [10], + "render": [false], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.001], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["s5"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [64], + "student_n_hidden_layers": [2], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_s5_n_blocks": [2], + "student_s5_n_layers": [2], + "student_s5_layernorm_pos": ["pre"], + "student_s5_activation": ["half_glu1"], + "student_agent_kind": ["mappo"], + "overcooked_height": [6], + "overcooked_width": [9], + "overcooked_n_walls": [15], + "overcooked_replace_wall_pos": [true], + "overcooked_sample_n_walls": [true], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [false], + "overcooked_fix_to_single_layout": ["asymm_advantages_6_9"], + "n_shaped_reward_steps": [5000000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline__s5_coord_ring.json b/src/config/configs/overcooked/baseline__s5_coord_ring.json new file mode 100644 index 0000000..d1b1eb9 --- /dev/null +++ b/src/config/configs/overcooked/baseline__s5_coord_ring.json @@ -0,0 +1,69 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [1000], + "train_runner": ["dr"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "is_multi_agent": [true], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-05], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.995], + "n_unroll_rollout": [10], + "render": [false], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.001], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["s5"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [64], + "student_n_hidden_layers": [2], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_s5_n_blocks": [2], + "student_s5_n_layers": [2], + "student_s5_layernorm_pos": ["pre"], + "student_s5_activation": ["half_glu1"], + "student_agent_kind": ["mappo"], + "overcooked_height": [6], + "overcooked_width": [9], + "overcooked_n_walls": [15], + "overcooked_replace_wall_pos": [true], + "overcooked_sample_n_walls": [true], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [true], + "overcooked_fix_to_single_layout": ["coord_ring_6_9"], + "n_shaped_reward_steps": [5000000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline__s5_counter_circuit.json b/src/config/configs/overcooked/baseline__s5_counter_circuit.json new file mode 100644 index 0000000..ed74216 --- /dev/null +++ b/src/config/configs/overcooked/baseline__s5_counter_circuit.json @@ -0,0 +1,69 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [1000], + "train_runner": ["dr"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "is_multi_agent": [true], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-05], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.995], + "n_unroll_rollout": [10], + "render": [false], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.001], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["s5"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [64], + "student_n_hidden_layers": [2], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_s5_n_blocks": [2], + "student_s5_n_layers": [2], + "student_s5_layernorm_pos": ["pre"], + "student_s5_activation": ["half_glu1"], + "student_agent_kind": ["mappo"], + "overcooked_height": [6], + "overcooked_width": [9], + "overcooked_n_walls": [15], + "overcooked_replace_wall_pos": [true], + "overcooked_sample_n_walls": [true], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [true], + "overcooked_fix_to_single_layout": ["counter_circuit_6_9"], + "n_shaped_reward_steps": [5000000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline__s5_cramped_room.json b/src/config/configs/overcooked/baseline__s5_cramped_room.json new file mode 100644 index 0000000..c82f066 --- /dev/null +++ b/src/config/configs/overcooked/baseline__s5_cramped_room.json @@ -0,0 +1,69 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [1000], + "train_runner": ["dr"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "is_multi_agent": [true], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-05], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.995], + "n_unroll_rollout": [10], + "render": [false], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.001], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["s5"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [64], + "student_n_hidden_layers": [2], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_s5_n_blocks": [2], + "student_s5_n_layers": [2], + "student_s5_layernorm_pos": ["pre"], + "student_s5_activation": ["half_glu1"], + "student_agent_kind": ["mappo"], + "overcooked_height": [6], + "overcooked_width": [9], + "overcooked_n_walls": [15], + "overcooked_replace_wall_pos": [true], + "overcooked_sample_n_walls": [true], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [true], + "overcooked_fix_to_single_layout": ["cramped_room_6_9"], + "n_shaped_reward_steps": [5000000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline__s5_forced_coord.json b/src/config/configs/overcooked/baseline__s5_forced_coord.json new file mode 100644 index 0000000..2d436ef --- /dev/null +++ b/src/config/configs/overcooked/baseline__s5_forced_coord.json @@ -0,0 +1,69 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [1000], + "train_runner": ["dr"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "is_multi_agent": [true], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-05], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.995], + "n_unroll_rollout": [10], + "render": [false], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.001], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["s5"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [64], + "student_n_hidden_layers": [2], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_s5_n_blocks": [2], + "student_s5_n_layers": [2], + "student_s5_layernorm_pos": ["pre"], + "student_s5_activation": ["half_glu1"], + "student_agent_kind": ["mappo"], + "overcooked_height": [6], + "overcooked_width": [9], + "overcooked_n_walls": [15], + "overcooked_replace_wall_pos": [true], + "overcooked_sample_n_walls": [true], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [true], + "overcooked_fix_to_single_layout": ["forced_coord_6_9"], + "n_shaped_reward_steps": [5000000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline_dr_lstm.json b/src/config/configs/overcooked/baseline_dr_lstm.json new file mode 100644 index 0000000..9b97792 --- /dev/null +++ b/src/config/configs/overcooked/baseline_dr_lstm.json @@ -0,0 +1,64 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["dr"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "is_multi_agent": [true], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-4], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.01], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [8], + "student_ppo_n_minibatches": [4], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [64], + "student_hidden_dim": [64], + "student_n_hidden_layers": [3], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_agent_kind": ["mappo"], + "overcooked_height": [6], + "overcooked_width": [9], + "overcooked_n_walls": [15], + "overcooked_replace_wall_pos": [true], + "overcooked_sample_n_walls": [true], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [false], + "n_shaped_reward_updates": [30000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline_dr_lstm5x5.json b/src/config/configs/overcooked/baseline_dr_lstm5x5.json new file mode 100644 index 0000000..6a1af77 --- /dev/null +++ b/src/config/configs/overcooked/baseline_dr_lstm5x5.json @@ -0,0 +1,64 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["dr"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "is_multi_agent": [true], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-4], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.01], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [8], + "student_ppo_n_minibatches": [4], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [64], + "student_hidden_dim": [64], + "student_n_hidden_layers": [3], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_agent_kind": ["mappo"], + "overcooked_height": [5], + "overcooked_width": [5], + "overcooked_n_walls": [15], + "overcooked_replace_wall_pos": [true], + "overcooked_sample_n_walls": [true], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [false], + "n_shaped_reward_updates": [30000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing5_5,Overcooked-ForcedCoord5_5,Overcooked-CrampedRoom5_5" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline_dr_s5.json b/src/config/configs/overcooked/baseline_dr_s5.json new file mode 100644 index 0000000..91c7864 --- /dev/null +++ b/src/config/configs/overcooked/baseline_dr_s5.json @@ -0,0 +1,68 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["dr"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "is_multi_agent": [true], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-4], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.01], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [8], + "student_ppo_n_minibatches": [4], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["s5"], + "student_recurrent_hidden_dim": [64], + "student_hidden_dim": [64], + "student_n_hidden_layers": [3], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_s5_n_blocks": [2], + "student_s5_n_layers": [2], + "student_s5_layernorm_pos": ["pre"], + "student_s5_activation": ["half_glu1"], + "student_agent_kind": ["mappo"], + "overcooked_height": [6], + "overcooked_width": [9], + "overcooked_n_walls": [15], + "overcooked_replace_wall_pos": [true], + "overcooked_sample_n_walls": [true], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [false], + "n_shaped_reward_updates": [30000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline_dr_s55x5.json b/src/config/configs/overcooked/baseline_dr_s55x5.json new file mode 100644 index 0000000..7a45833 --- /dev/null +++ b/src/config/configs/overcooked/baseline_dr_s55x5.json @@ -0,0 +1,68 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["dr"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "is_multi_agent": [true], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-4], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.01], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [8], + "student_ppo_n_minibatches": [4], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["s5"], + "student_recurrent_hidden_dim": [64], + "student_hidden_dim": [64], + "student_n_hidden_layers": [3], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_s5_n_blocks": [2], + "student_s5_n_layers": [2], + "student_s5_layernorm_pos": ["pre"], + "student_s5_activation": ["half_glu1"], + "student_agent_kind": ["mappo"], + "overcooked_height": [5], + "overcooked_width": [5], + "overcooked_n_walls": [15], + "overcooked_replace_wall_pos": [true], + "overcooked_sample_n_walls": [true], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [false], + "n_shaped_reward_updates": [30000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing5_5,Overcooked-ForcedCoord5_5,Overcooked-CrampedRoom5_5" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline_dr_softmoe_lstm.json b/src/config/configs/overcooked/baseline_dr_softmoe_lstm.json new file mode 100644 index 0000000..82de0f9 --- /dev/null +++ b/src/config/configs/overcooked/baseline_dr_softmoe_lstm.json @@ -0,0 +1,67 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["dr"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "is_multi_agent": [true], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-4], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.01], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [8], + "student_ppo_n_minibatches": [4], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [64], + "student_hidden_dim": [64], + "student_n_hidden_layers": [2], + "student_is_soft_moe": [true], + "student_soft_moe_num_experts": [4], + "student_soft_moe_num_slots": [32], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_agent_kind": ["mappo"], + "overcooked_height": [6], + "overcooked_width": [9], + "overcooked_n_walls": [15], + "overcooked_replace_wall_pos": [true], + "overcooked_sample_n_walls": [true], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [false], + "n_shaped_reward_updates": [30000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline_dr_softmoe_lstm5x5.json b/src/config/configs/overcooked/baseline_dr_softmoe_lstm5x5.json new file mode 100644 index 0000000..663ab72 --- /dev/null +++ b/src/config/configs/overcooked/baseline_dr_softmoe_lstm5x5.json @@ -0,0 +1,67 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["dr"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "is_multi_agent": [true], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-4], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.01], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [8], + "student_ppo_n_minibatches": [4], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [64], + "student_hidden_dim": [64], + "student_n_hidden_layers": [2], + "student_is_soft_moe": [true], + "student_soft_moe_num_experts": [4], + "student_soft_moe_num_slots": [32], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_agent_kind": ["mappo"], + "overcooked_height": [5], + "overcooked_width": [5], + "overcooked_n_walls": [15], + "overcooked_replace_wall_pos": [true], + "overcooked_sample_n_walls": [true], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [false], + "n_shaped_reward_updates": [30000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing5_5,Overcooked-ForcedCoord5_5,Overcooked-CrampedRoom5_5" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline_p_accel_lstm.json b/src/config/configs/overcooked/baseline_p_accel_lstm.json new file mode 100644 index 0000000..4a3ae9e --- /dev/null +++ b/src/config/configs/overcooked/baseline_p_accel_lstm.json @@ -0,0 +1,78 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "is_multi_agent": [true], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-4], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.8], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.1], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [true], + "plr_force_unique": [true], + "plr_mutation_fn": ["default"], + "plr_n_mutations": [20], + "plr_mutation_criterion": ["batch"], + "plr_mutation_subsample_size": [4], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.01], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [8], + "student_ppo_n_minibatches": [4], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [64], + "student_hidden_dim": [64], + "student_n_hidden_layers": [3], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_agent_kind": ["mappo"], + "overcooked_height": [6], + "overcooked_width": [9], + "overcooked_n_walls": [15], + "overcooked_replace_wall_pos": [true], + "overcooked_sample_n_walls": [true], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [false], + "n_shaped_reward_updates": [30000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline_p_accel_lstm5x5.json b/src/config/configs/overcooked/baseline_p_accel_lstm5x5.json new file mode 100644 index 0000000..cd95ad1 --- /dev/null +++ b/src/config/configs/overcooked/baseline_p_accel_lstm5x5.json @@ -0,0 +1,78 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "is_multi_agent": [true], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-4], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.8], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.1], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [true], + "plr_force_unique": [true], + "plr_mutation_fn": ["default"], + "plr_n_mutations": [20], + "plr_mutation_criterion": ["batch"], + "plr_mutation_subsample_size": [4], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.01], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [8], + "student_ppo_n_minibatches": [4], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [64], + "student_hidden_dim": [64], + "student_n_hidden_layers": [3], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_agent_kind": ["mappo"], + "overcooked_height": [5], + "overcooked_width": [5], + "overcooked_n_walls": [15], + "overcooked_replace_wall_pos": [true], + "overcooked_sample_n_walls": [true], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [false], + "n_shaped_reward_updates": [30000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing5_5,Overcooked-ForcedCoord5_5,Overcooked-CrampedRoom5_5" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline_p_accel_s5.json b/src/config/configs/overcooked/baseline_p_accel_s5.json new file mode 100644 index 0000000..3913395 --- /dev/null +++ b/src/config/configs/overcooked/baseline_p_accel_s5.json @@ -0,0 +1,82 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "is_multi_agent": [true], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-4], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.8], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.1], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [true], + "plr_force_unique": [true], + "plr_mutation_fn": ["default"], + "plr_n_mutations": [20], + "plr_mutation_criterion": ["batch"], + "plr_mutation_subsample_size": [4], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.01], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [8], + "student_ppo_n_minibatches": [4], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["s5"], + "student_recurrent_hidden_dim": [64], + "student_hidden_dim": [64], + "student_n_hidden_layers": [3], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_s5_n_blocks": [2], + "student_s5_n_layers": [2], + "student_s5_layernorm_pos": ["pre"], + "student_s5_activation": ["half_glu1"], + "student_agent_kind": ["mappo"], + "overcooked_height": [6], + "overcooked_width": [9], + "overcooked_n_walls": [15], + "overcooked_replace_wall_pos": [true], + "overcooked_sample_n_walls": [true], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [false], + "n_shaped_reward_updates": [30000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline_p_accel_s55x5.json b/src/config/configs/overcooked/baseline_p_accel_s55x5.json new file mode 100644 index 0000000..db1b950 --- /dev/null +++ b/src/config/configs/overcooked/baseline_p_accel_s55x5.json @@ -0,0 +1,82 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "is_multi_agent": [true], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-4], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.8], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.1], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [true], + "plr_force_unique": [true], + "plr_mutation_fn": ["default"], + "plr_n_mutations": [20], + "plr_mutation_criterion": ["batch"], + "plr_mutation_subsample_size": [4], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.01], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [8], + "student_ppo_n_minibatches": [4], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["s5"], + "student_recurrent_hidden_dim": [64], + "student_hidden_dim": [64], + "student_n_hidden_layers": [3], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_s5_n_blocks": [2], + "student_s5_n_layers": [2], + "student_s5_layernorm_pos": ["pre"], + "student_s5_activation": ["half_glu1"], + "student_agent_kind": ["mappo"], + "overcooked_height": [5], + "overcooked_width": [5], + "overcooked_n_walls": [15], + "overcooked_replace_wall_pos": [true], + "overcooked_sample_n_walls": [true], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [false], + "n_shaped_reward_updates": [30000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing5_5,Overcooked-ForcedCoord5_5,Overcooked-CrampedRoom5_5" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline_p_accel_softmoe_lstm.json b/src/config/configs/overcooked/baseline_p_accel_softmoe_lstm.json new file mode 100644 index 0000000..1bd059a --- /dev/null +++ b/src/config/configs/overcooked/baseline_p_accel_softmoe_lstm.json @@ -0,0 +1,81 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "is_multi_agent": [true], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-4], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.8], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.1], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [true], + "plr_force_unique": [true], + "plr_mutation_fn": ["default"], + "plr_n_mutations": [20], + "plr_mutation_criterion": ["batch"], + "plr_mutation_subsample_size": [4], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.01], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [8], + "student_ppo_n_minibatches": [4], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [64], + "student_hidden_dim": [64], + "student_n_hidden_layers": [2], + "student_is_soft_moe": [true], + "student_soft_moe_num_experts": [4], + "student_soft_moe_num_slots": [32], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_agent_kind": ["mappo"], + "overcooked_height": [6], + "overcooked_width": [9], + "overcooked_n_walls": [15], + "overcooked_replace_wall_pos": [true], + "overcooked_sample_n_walls": [true], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [false], + "n_shaped_reward_updates": [30000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline_p_accel_softmoe_lstm5x5.json b/src/config/configs/overcooked/baseline_p_accel_softmoe_lstm5x5.json new file mode 100644 index 0000000..2bc84ac --- /dev/null +++ b/src/config/configs/overcooked/baseline_p_accel_softmoe_lstm5x5.json @@ -0,0 +1,81 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "is_multi_agent": [true], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-4], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.8], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.1], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [true], + "plr_force_unique": [true], + "plr_mutation_fn": ["default"], + "plr_n_mutations": [20], + "plr_mutation_criterion": ["batch"], + "plr_mutation_subsample_size": [4], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.01], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [8], + "student_ppo_n_minibatches": [4], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [64], + "student_hidden_dim": [64], + "student_n_hidden_layers": [2], + "student_is_soft_moe": [true], + "student_soft_moe_num_experts": [4], + "student_soft_moe_num_slots": [32], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_agent_kind": ["mappo"], + "overcooked_height": [5], + "overcooked_width": [5], + "overcooked_n_walls": [15], + "overcooked_replace_wall_pos": [true], + "overcooked_sample_n_walls": [true], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [false], + "n_shaped_reward_updates": [30000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing5_5,Overcooked-ForcedCoord5_5,Overcooked-CrampedRoom5_5" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline_p_plr_lstm.json b/src/config/configs/overcooked/baseline_p_plr_lstm.json new file mode 100644 index 0000000..52a4e45 --- /dev/null +++ b/src/config/configs/overcooked/baseline_p_plr_lstm.json @@ -0,0 +1,74 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "is_multi_agent": [true], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-4], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.5], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.1], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [true], + "plr_force_unique": [true], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.01], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [8], + "student_ppo_n_minibatches": [4], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [64], + "student_hidden_dim": [64], + "student_n_hidden_layers": [3], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_agent_kind": ["mappo"], + "overcooked_height": [6], + "overcooked_width": [9], + "overcooked_n_walls": [15], + "overcooked_replace_wall_pos": [true], + "overcooked_sample_n_walls": [true], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [false], + "n_shaped_reward_updates": [30000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline_p_plr_lstm5x5.json b/src/config/configs/overcooked/baseline_p_plr_lstm5x5.json new file mode 100644 index 0000000..26a9653 --- /dev/null +++ b/src/config/configs/overcooked/baseline_p_plr_lstm5x5.json @@ -0,0 +1,74 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "is_multi_agent": [true], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-4], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.5], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.1], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [true], + "plr_force_unique": [true], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.01], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [8], + "student_ppo_n_minibatches": [4], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [64], + "student_hidden_dim": [64], + "student_n_hidden_layers": [3], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_agent_kind": ["mappo"], + "overcooked_height": [5], + "overcooked_width": [5], + "overcooked_n_walls": [15], + "overcooked_replace_wall_pos": [true], + "overcooked_sample_n_walls": [true], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [false], + "n_shaped_reward_updates": [30000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing5_5,Overcooked-ForcedCoord5_5,Overcooked-CrampedRoom5_5" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline_p_plr_s5.json b/src/config/configs/overcooked/baseline_p_plr_s5.json new file mode 100644 index 0000000..6fc3dca --- /dev/null +++ b/src/config/configs/overcooked/baseline_p_plr_s5.json @@ -0,0 +1,78 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "is_multi_agent": [true], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-4], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.5], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.1], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [true], + "plr_force_unique": [true], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.01], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [8], + "student_ppo_n_minibatches": [4], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["s5"], + "student_recurrent_hidden_dim": [64], + "student_hidden_dim": [64], + "student_n_hidden_layers": [3], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_s5_n_blocks": [2], + "student_s5_n_layers": [2], + "student_s5_layernorm_pos": ["pre"], + "student_s5_activation": ["half_glu1"], + "student_agent_kind": ["mappo"], + "overcooked_height": [6], + "overcooked_width": [9], + "overcooked_n_walls": [15], + "overcooked_replace_wall_pos": [true], + "overcooked_sample_n_walls": [true], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [false], + "n_shaped_reward_updates": [30000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline_p_plr_s55x5.json b/src/config/configs/overcooked/baseline_p_plr_s55x5.json new file mode 100644 index 0000000..43abe81 --- /dev/null +++ b/src/config/configs/overcooked/baseline_p_plr_s55x5.json @@ -0,0 +1,78 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "is_multi_agent": [true], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-4], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.5], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.1], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [true], + "plr_force_unique": [true], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.01], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [8], + "student_ppo_n_minibatches": [4], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["s5"], + "student_recurrent_hidden_dim": [64], + "student_hidden_dim": [64], + "student_n_hidden_layers": [3], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_s5_n_blocks": [2], + "student_s5_n_layers": [2], + "student_s5_layernorm_pos": ["pre"], + "student_s5_activation": ["half_glu1"], + "student_agent_kind": ["mappo"], + "overcooked_height": [5], + "overcooked_width": [5], + "overcooked_n_walls": [15], + "overcooked_replace_wall_pos": [true], + "overcooked_sample_n_walls": [true], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [false], + "n_shaped_reward_updates": [30000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing5_5,Overcooked-ForcedCoord5_5,Overcooked-CrampedRoom5_5" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline_p_plr_softmoe_lstm.json b/src/config/configs/overcooked/baseline_p_plr_softmoe_lstm.json new file mode 100644 index 0000000..1803e44 --- /dev/null +++ b/src/config/configs/overcooked/baseline_p_plr_softmoe_lstm.json @@ -0,0 +1,77 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "is_multi_agent": [true], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-4], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.5], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.1], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [true], + "plr_force_unique": [true], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.01], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [8], + "student_ppo_n_minibatches": [4], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [64], + "student_hidden_dim": [64], + "student_n_hidden_layers": [2], + "student_is_soft_moe": [true], + "student_soft_moe_num_experts": [4], + "student_soft_moe_num_slots": [32], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_agent_kind": ["mappo"], + "overcooked_height": [6], + "overcooked_width": [9], + "overcooked_n_walls": [15], + "overcooked_replace_wall_pos": [true], + "overcooked_sample_n_walls": [true], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [false], + "n_shaped_reward_updates": [30000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline_p_plr_softmoe_lstm5x5.json b/src/config/configs/overcooked/baseline_p_plr_softmoe_lstm5x5.json new file mode 100644 index 0000000..29f21e5 --- /dev/null +++ b/src/config/configs/overcooked/baseline_p_plr_softmoe_lstm5x5.json @@ -0,0 +1,77 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "is_multi_agent": [true], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-4], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.5], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.1], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [true], + "plr_force_unique": [true], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.01], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [8], + "student_ppo_n_minibatches": [4], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [64], + "student_hidden_dim": [64], + "student_n_hidden_layers": [2], + "student_is_soft_moe": [true], + "student_soft_moe_num_experts": [4], + "student_soft_moe_num_slots": [32], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_agent_kind": ["mappo"], + "overcooked_height": [5], + "overcooked_width": [5], + "overcooked_n_walls": [15], + "overcooked_replace_wall_pos": [true], + "overcooked_sample_n_walls": [true], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [false], + "n_shaped_reward_updates": [30000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing5_5,Overcooked-ForcedCoord5_5,Overcooked-CrampedRoom5_5" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline_pop_paired_lstm.json b/src/config/configs/overcooked/baseline_pop_paired_lstm.json new file mode 100644 index 0000000..48bd630 --- /dev/null +++ b/src/config/configs/overcooked/baseline_pop_paired_lstm.json @@ -0,0 +1,86 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["paired"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "verbose": [false], + "is_multi_agent": [true], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [2], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-4], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["relative_regret"], + "student_gae_lambda": [0.98], + "teacher_discount": [0.999], + "teacher_lr_anneal_steps": [0], + "teacher_gae_lambda": [0.98], + "student_entropy_coef": [0.01], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [8], + "student_ppo_n_minibatches": [4], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "teacher_entropy_coef": [0.01], + "teacher_value_loss_coef": [0.5], + "teacher_n_unroll_update": [5], + "teacher_ppo_n_epochs": [8], + "teacher_ppo_n_minibatches": [4], + "teacher_ppo_clip_eps": [0.2], + "teacher_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [64], + "student_hidden_dim": [64], + "student_n_hidden_layers": [3], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_agent_kind": ["mappo"], + "teacher_model_name": ["default_teacher_cnn"], + "teacher_recurrent_arch": ["lstm"], + "teacher_recurrent_hidden_dim": [64], + "teacher_hidden_dim": [64], + "teacher_n_hidden_layers": [1], + "teacher_n_conv_filters": [128], + "teacher_scalar_embed_dim": [10], + "overcooked_height": [6], + "overcooked_width": [9], + "overcooked_n_walls": [5], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [false], + "overcooked_ued_replace_wall_pos": [true], + "overcooked_ued_fixed_n_wall_steps": [false], + "overcooked_ued_first_wall_pos_sets_budget": [true], + "overcooked_ued_noise_dim": [50], + "overcooked_ued_n_walls": [15], + "overcooked_ued_normalize_obs": [true], + "n_shaped_reward_updates": [30000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline_pop_paired_lstm5x5.json b/src/config/configs/overcooked/baseline_pop_paired_lstm5x5.json new file mode 100644 index 0000000..72eabdd --- /dev/null +++ b/src/config/configs/overcooked/baseline_pop_paired_lstm5x5.json @@ -0,0 +1,86 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["paired"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "verbose": [false], + "is_multi_agent": [true], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [2], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-4], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["relative_regret"], + "student_gae_lambda": [0.98], + "teacher_discount": [0.999], + "teacher_lr_anneal_steps": [0], + "teacher_gae_lambda": [0.98], + "student_entropy_coef": [0.01], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [8], + "student_ppo_n_minibatches": [4], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "teacher_entropy_coef": [0.01], + "teacher_value_loss_coef": [0.5], + "teacher_n_unroll_update": [5], + "teacher_ppo_n_epochs": [8], + "teacher_ppo_n_minibatches": [4], + "teacher_ppo_clip_eps": [0.2], + "teacher_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [64], + "student_hidden_dim": [64], + "student_n_hidden_layers": [3], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_agent_kind": ["mappo"], + "teacher_model_name": ["default_teacher_cnn"], + "teacher_recurrent_arch": ["lstm"], + "teacher_recurrent_hidden_dim": [64], + "teacher_hidden_dim": [64], + "teacher_n_hidden_layers": [1], + "teacher_n_conv_filters": [128], + "teacher_scalar_embed_dim": [10], + "overcooked_height": [5], + "overcooked_width": [5], + "overcooked_n_walls": [5], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [false], + "overcooked_ued_replace_wall_pos": [true], + "overcooked_ued_fixed_n_wall_steps": [false], + "overcooked_ued_first_wall_pos_sets_budget": [true], + "overcooked_ued_noise_dim": [50], + "overcooked_ued_n_walls": [15], + "overcooked_ued_normalize_obs": [true], + "n_shaped_reward_updates": [30000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing5_5,Overcooked-ForcedCoord5_5,Overcooked-CrampedRoom5_5" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline_pop_paired_s5.json b/src/config/configs/overcooked/baseline_pop_paired_s5.json new file mode 100644 index 0000000..eed05b5 --- /dev/null +++ b/src/config/configs/overcooked/baseline_pop_paired_s5.json @@ -0,0 +1,90 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["paired"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "verbose": [false], + "is_multi_agent": [true], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [2], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-4], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["relative_regret"], + "student_gae_lambda": [0.98], + "teacher_discount": [0.999], + "teacher_lr_anneal_steps": [0], + "teacher_gae_lambda": [0.98], + "student_entropy_coef": [0.01], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [8], + "student_ppo_n_minibatches": [4], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "teacher_entropy_coef": [0.01], + "teacher_value_loss_coef": [0.5], + "teacher_n_unroll_update": [5], + "teacher_ppo_n_epochs": [8], + "teacher_ppo_n_minibatches": [4], + "teacher_ppo_clip_eps": [0.2], + "teacher_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["s5"], + "student_recurrent_hidden_dim": [64], + "student_hidden_dim": [64], + "student_n_hidden_layers": [3], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_s5_n_blocks": [2], + "student_s5_n_layers": [2], + "student_s5_layernorm_pos": ["pre"], + "student_s5_activation": ["half_glu1"], + "student_agent_kind": ["mappo"], + "teacher_model_name": ["default_teacher_cnn"], + "teacher_recurrent_arch": ["lstm"], + "teacher_recurrent_hidden_dim": [64], + "teacher_hidden_dim": [64], + "teacher_n_hidden_layers": [1], + "teacher_n_conv_filters": [128], + "teacher_scalar_embed_dim": [10], + "overcooked_height": [6], + "overcooked_width": [9], + "overcooked_n_walls": [5], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [false], + "overcooked_ued_replace_wall_pos": [true], + "overcooked_ued_fixed_n_wall_steps": [false], + "overcooked_ued_first_wall_pos_sets_budget": [true], + "overcooked_ued_noise_dim": [50], + "overcooked_ued_n_walls": [15], + "overcooked_ued_normalize_obs": [true], + "n_shaped_reward_updates": [30000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline_pop_paired_s55x5.json b/src/config/configs/overcooked/baseline_pop_paired_s55x5.json new file mode 100644 index 0000000..09ef6cf --- /dev/null +++ b/src/config/configs/overcooked/baseline_pop_paired_s55x5.json @@ -0,0 +1,90 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["paired"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "verbose": [false], + "is_multi_agent": [true], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [2], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-4], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["relative_regret"], + "student_gae_lambda": [0.98], + "teacher_discount": [0.999], + "teacher_lr_anneal_steps": [0], + "teacher_gae_lambda": [0.98], + "student_entropy_coef": [0.01], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [8], + "student_ppo_n_minibatches": [4], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "teacher_entropy_coef": [0.01], + "teacher_value_loss_coef": [0.5], + "teacher_n_unroll_update": [5], + "teacher_ppo_n_epochs": [8], + "teacher_ppo_n_minibatches": [4], + "teacher_ppo_clip_eps": [0.2], + "teacher_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["s5"], + "student_recurrent_hidden_dim": [64], + "student_hidden_dim": [64], + "student_n_hidden_layers": [3], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_s5_n_blocks": [2], + "student_s5_n_layers": [2], + "student_s5_layernorm_pos": ["pre"], + "student_s5_activation": ["half_glu1"], + "student_agent_kind": ["mappo"], + "teacher_model_name": ["default_teacher_cnn"], + "teacher_recurrent_arch": ["lstm"], + "teacher_recurrent_hidden_dim": [64], + "teacher_hidden_dim": [64], + "teacher_n_hidden_layers": [1], + "teacher_n_conv_filters": [128], + "teacher_scalar_embed_dim": [10], + "overcooked_height": [5], + "overcooked_width": [5], + "overcooked_n_walls": [5], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [false], + "overcooked_ued_replace_wall_pos": [true], + "overcooked_ued_fixed_n_wall_steps": [false], + "overcooked_ued_first_wall_pos_sets_budget": [true], + "overcooked_ued_noise_dim": [50], + "overcooked_ued_n_walls": [15], + "overcooked_ued_normalize_obs": [true], + "n_shaped_reward_updates": [30000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing5_5,Overcooked-ForcedCoord5_5,Overcooked-CrampedRoom5_5" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline_pop_paired_softmoe_lstm.json b/src/config/configs/overcooked/baseline_pop_paired_softmoe_lstm.json new file mode 100644 index 0000000..c497a5a --- /dev/null +++ b/src/config/configs/overcooked/baseline_pop_paired_softmoe_lstm.json @@ -0,0 +1,89 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["paired"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "verbose": [false], + "is_multi_agent": [true], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [2], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-4], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["relative_regret"], + "student_gae_lambda": [0.98], + "teacher_discount": [0.999], + "teacher_lr_anneal_steps": [0], + "teacher_gae_lambda": [0.98], + "student_entropy_coef": [0.01], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [8], + "student_ppo_n_minibatches": [4], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "teacher_entropy_coef": [0.01], + "teacher_value_loss_coef": [0.5], + "teacher_n_unroll_update": [5], + "teacher_ppo_n_epochs": [8], + "teacher_ppo_n_minibatches": [4], + "teacher_ppo_clip_eps": [0.2], + "teacher_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [64], + "student_hidden_dim": [64], + "student_n_hidden_layers": [2], + "student_is_soft_moe": [true], + "student_soft_moe_num_experts": [4], + "student_soft_moe_num_slots": [32], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_agent_kind": ["mappo"], + "teacher_model_name": ["default_teacher_cnn"], + "teacher_recurrent_arch": ["lstm"], + "teacher_recurrent_hidden_dim": [64], + "teacher_hidden_dim": [64], + "teacher_n_hidden_layers": [1], + "teacher_n_conv_filters": [128], + "teacher_scalar_embed_dim": [10], + "overcooked_height": [6], + "overcooked_width": [9], + "overcooked_n_walls": [5], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [false], + "overcooked_ued_replace_wall_pos": [true], + "overcooked_ued_fixed_n_wall_steps": [false], + "overcooked_ued_first_wall_pos_sets_budget": [true], + "overcooked_ued_noise_dim": [50], + "overcooked_ued_n_walls": [15], + "overcooked_ued_normalize_obs": [true], + "n_shaped_reward_updates": [30000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/baseline_pop_paired_softmoe_lstm5x5.json b/src/config/configs/overcooked/baseline_pop_paired_softmoe_lstm5x5.json new file mode 100644 index 0000000..3ec6692 --- /dev/null +++ b/src/config/configs/overcooked/baseline_pop_paired_softmoe_lstm5x5.json @@ -0,0 +1,89 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["paired"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "verbose": [false], + "is_multi_agent": [true], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [2], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-4], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["relative_regret"], + "student_gae_lambda": [0.98], + "teacher_discount": [0.999], + "teacher_lr_anneal_steps": [0], + "teacher_gae_lambda": [0.98], + "student_entropy_coef": [0.01], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [8], + "student_ppo_n_minibatches": [4], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "teacher_entropy_coef": [0.01], + "teacher_value_loss_coef": [0.5], + "teacher_n_unroll_update": [5], + "teacher_ppo_n_epochs": [8], + "teacher_ppo_n_minibatches": [4], + "teacher_ppo_clip_eps": [0.2], + "teacher_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [64], + "student_hidden_dim": [64], + "student_n_hidden_layers": [2], + "student_is_soft_moe": [true], + "student_soft_moe_num_experts": [4], + "student_soft_moe_num_slots": [32], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_agent_kind": ["mappo"], + "teacher_model_name": ["default_teacher_cnn"], + "teacher_recurrent_arch": ["lstm"], + "teacher_recurrent_hidden_dim": [64], + "teacher_hidden_dim": [64], + "teacher_n_hidden_layers": [1], + "teacher_n_conv_filters": [128], + "teacher_scalar_embed_dim": [10], + "overcooked_height": [5], + "overcooked_width": [5], + "overcooked_n_walls": [5], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [false], + "overcooked_ued_replace_wall_pos": [true], + "overcooked_ued_fixed_n_wall_steps": [false], + "overcooked_ued_first_wall_pos_sets_budget": [true], + "overcooked_ued_noise_dim": [50], + "overcooked_ued_n_walls": [15], + "overcooked_ued_normalize_obs": [true], + "n_shaped_reward_updates": [30000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing5_5,Overcooked-ForcedCoord5_5,Overcooked-CrampedRoom5_5" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/paired.json b/src/config/configs/overcooked/paired.json new file mode 100644 index 0000000..6a1bb4f --- /dev/null +++ b/src/config/configs/overcooked/paired.json @@ -0,0 +1,83 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [1000000], + "train_runner": ["paired"], + "n_devices": [1], + "student_model_name": ["default_student_actor_moe"], + "student_critic_model_name": ["default_student_critic_moe"], + "env_name": ["Overcooked"], + "verbose": [false], + "is_multi_agent": [true], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [true], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [2], + "n_parallel": [100], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [0.0001], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.995], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["relative_regret"], + "student_gae_lambda": [0.98], + "teacher_discount": [0.995], + "teacher_lr_anneal_steps": [0], + "teacher_gae_lambda": [0.98], + "student_entropy_coef": [0.001], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "teacher_entropy_coef": [0.05], + "teacher_value_loss_coef": [0.5], + "teacher_n_unroll_update": [5], + "teacher_ppo_n_epochs": [5], + "teacher_ppo_n_minibatches": [1], + "teacher_ppo_clip_eps": [0.2], + "teacher_ppo_clip_value_loss": [true], + "student_hidden_dim": [32], + "student_n_hidden_layers": [1], + "student_n_conv_filters": [16], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_agent_kind": ["mappo"], + "teacher_model_name": ["default_teacher_cnn"], + "teacher_recurrent_arch": ["lstm"], + "teacher_recurrent_hidden_dim": [256], + "teacher_hidden_dim": [32], + "teacher_n_hidden_layers": [1], + "teacher_n_conv_filters": [128], + "teacher_scalar_embed_dim": [10], + "overcooked_height": [6], + "overcooked_width": [9], + "overcooked_n_walls": [5], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [true], + "overcooked_ued_replace_wall_pos": [true], + "overcooked_ued_fixed_n_wall_steps": [false], + "overcooked_ued_first_wall_pos_sets_budget": [true], + "overcooked_ued_noise_dim": [50], + "overcooked_ued_n_walls": [15], + "overcooked_ued_normalize_obs": [true], + "test_n_episodes": [10], + "n_shaped_reward_steps": [5000000], + "test_env_names": [ + "Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/plr.json b/src/config/configs/overcooked/plr.json new file mode 100644 index 0000000..da68e10 --- /dev/null +++ b/src/config/configs/overcooked/plr.json @@ -0,0 +1,71 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [1000000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_actor_moe"], + "student_critic_model_name": ["default_student_critic_moe"], + "env_name": ["Overcooked"], + "is_multi_agent": [true], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [100], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [5e-05], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.99], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.5], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.1], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [false], + "plr_use_parallel_eval": [false], + "plr_force_unique": [true], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.0], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_hidden_dim": [32], + "student_n_hidden_layers": [1], + "student_n_conv_filters": [16], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_agent_kind": ["mappo"], + "overcooked_height": [6], + "overcooked_width": [9], + "overcooked_n_walls": [15], + "overcooked_replace_wall_pos": [true], + "overcooked_sample_n_walls": [true], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [true], + "test_n_episodes": [10], + "n_shaped_reward_steps": [5000000], + "test_env_names": [ + "Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/config/configs/overcooked/plr_s5.json b/src/config/configs/overcooked/plr_s5.json new file mode 100644 index 0000000..425944e --- /dev/null +++ b/src/config/configs/overcooked/plr_s5.json @@ -0,0 +1,78 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [100000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_actor_cnn"], + "student_critic_model_name": ["default_student_critic_cnn"], + "env_name": ["Overcooked"], + "is_multi_agent": [true], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [128], + "n_eval": [1], + "n_rollout_steps": [400], + "lr": [3e-05], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.5], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.1], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [false], + "plr_use_parallel_eval": [false], + "plr_force_unique": [true], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.001], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [8], + "student_ppo_n_minibatches": [4], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["s5"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [64], + "student_n_hidden_layers": [2], + "student_n_conv_layers": [3], + "student_n_conv_filters": [32], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_s5_n_blocks": [2], + "student_s5_n_layers": [2], + "student_s5_layernorm_pos": ["pre"], + "student_s5_activation": ["half_glu1"], + "student_agent_kind": ["mappo"], + "overcooked_height": [6], + "overcooked_width": [9], + "overcooked_n_walls": [15], + "overcooked_replace_wall_pos": [true], + "overcooked_sample_n_walls": [true], + "overcooked_normalize_obs": [true], + "overcooked_max_steps": [400], + "overcooked_random_reset": [true], + "n_shaped_reward_steps": [5000000], + "test_n_episodes": [10], + "test_env_names": [ + "Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9" + ], + "overcooked_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/eval_all_xpid_against_population_in_all_layouts.sh b/src/eval_all_xpid_against_population_in_all_layouts.sh new file mode 100755 index 0000000..fc7bea3 --- /dev/null +++ b/src/eval_all_xpid_against_population_in_all_layouts.sh @@ -0,0 +1,9 @@ +DEFAULTVALUE=4 +device="${1:-$DEFAULTVALUE}" + +# "Overcooked-CoordRing6_9" "Overcooked-ForcedCoord6_9" "Overcooked-CounterCircuit6_9" "Overcooked-AsymmAdvantages6_9" "Overcooked-CrampedRoom6_9" +./eval_xpid_against_population_in_all_layouts.sh $device Overcooked-CoordRing6_9 9SEED_9_dr-overcookedNonexNonewNone_fs_FIXcoord_ring_6_9_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr3e-5g0.99cv0.5ce0.01e5mb1l0.95_pc0.2_h64cf32fc2se5ba_re_0 +./eval_xpid_against_population_in_all_layouts.sh $device Overcooked-ForcedCoord6_9 9SEED_9_dr-overcookedNonexNonewNone_fs_FIXforced_coord_6_9_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr3e-5g0.99cv0.5ce0.01e5mb1l0.95_pc0.2_h64cf32fc2se5ba_re_0 +./eval_xpid_against_population_in_all_layouts.sh $device Overcooked-CounterCircuit6_9 9SEED_9_dr-overcookedNonexNonewNone_fs_FIXcounter_circuit_6_9_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr3e-5g0.99cv0.5ce0.01e5mb1l0.95_pc0.2_h64cf32fc2se5ba_re_0 +./eval_xpid_against_population_in_all_layouts.sh $device Overcooked-AsymmAdvantages6_9 9SEED_9_dr-overcookedNonexNonewNone_fs_FIXasymm_advantages_6_9_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr3e-5g0.99cv0.5ce0.01e5mb1l0.95_pc0.2_h64cf32fc2se5ba_re_0 +./eval_xpid_against_population_in_all_layouts.sh $device Overcooked-CrampedRoom6_9 9SEED_9_dr-overcookedNonexNonewNone_fs_FIXcramped_room_6_9_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr3e-5g0.99cv0.5ce0.01e5mb1l0.95_pc0.2_h64cf32fc2se5ba_re_0 \ No newline at end of file diff --git a/src/eval_random_against_population.sh b/src/eval_random_against_population.sh new file mode 100755 index 0000000..6a35889 --- /dev/null +++ b/src/eval_random_against_population.sh @@ -0,0 +1,11 @@ +DEFAULTVALUE=4 +device="${1:-$DEFAULTVALUE}" + +for env in "Overcooked-CoordRing6_9" "Overcooked-ForcedCoord6_9" "Overcooked-CounterCircuit6_9" "Overcooked-AsymmAdvantages6_9" "Overcooked-CrampedRoom6_9"; +do + CUDA_VISIBLE_DEVICES=${device} LD_LIBRARY_PATH="" nice -n 5 python3 -m minimax.evaluate_baseline_against_population \ + --env_names=${env} \ + --population_json="populations/fcp/${env}/population.json" \ + --n_episodes=100 \ + --is_random=True +done \ No newline at end of file diff --git a/src/eval_stay_against_population.sh b/src/eval_stay_against_population.sh new file mode 100755 index 0000000..5beec1e --- /dev/null +++ b/src/eval_stay_against_population.sh @@ -0,0 +1,10 @@ +DEFAULTVALUE=4 +device="${1:-$DEFAULTVALUE}" + +for env in "Overcooked-AsymmAdvantages6_9" "Overcooked-CrampedRoom6_9" "Overcooked-CoordRing6_9" "Overcooked-ForcedCoord6_9" "Overcooked-CounterCircuit6_9"; +do + CUDA_VISIBLE_DEVICES=${device} LD_LIBRARY_PATH="" nice -n 5 python3 -m minimax.evaluate_baseline_against_population \ + --env_names=${env} \ + --population_json="populations/fcp/${env}/population.json" \ + --n_episodes=100 +done diff --git a/src/eval_xpid.sh b/src/eval_xpid.sh new file mode 100755 index 0000000..6dbdc81 --- /dev/null +++ b/src/eval_xpid.sh @@ -0,0 +1,6 @@ +DEFAULTVALUE=4 +device="${1:-$DEFAULTVALUE}" +CUDA_VISIBLE_DEVICES=${device} LD_LIBRARY_PATH="" nice -n 5 python3 -m minimax.evaluate \ +--xpid=$2 \ +--env_names=Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9 \ +--n_episodes=1000 \ No newline at end of file diff --git a/src/eval_xpid_against_population.sh b/src/eval_xpid_against_population.sh new file mode 100755 index 0000000..26d4510 --- /dev/null +++ b/src/eval_xpid_against_population.sh @@ -0,0 +1,10 @@ +DEFAULTVALUE=4 +# Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9 +device="${1:-$DEFAULTVALUE}" +ENV=Overcooked-AsymmAdvantages6_9 +XPID=$2 +CUDA_VISIBLE_DEVICES=${device} LD_LIBRARY_PATH="" nice -n 5 python3 -m minimax.evaluate_against_population \ +--xpid=${XPID} \ +--env_names=${ENV} \ +--population_json="populations/fcp/${ENV}/population.json" \ +--n_episodes=100 diff --git a/src/eval_xpid_against_population_in_all_layouts.sh b/src/eval_xpid_against_population_in_all_layouts.sh new file mode 100755 index 0000000..15523ee --- /dev/null +++ b/src/eval_xpid_against_population_in_all_layouts.sh @@ -0,0 +1,14 @@ +DEFAULTVALUE=4 +device="${1:-$DEFAULTVALUE}" +NAME=$2 +XPID=$3 + +for env in "Overcooked-CoordRing6_9" "Overcooked-ForcedCoord6_9" "Overcooked-CounterCircuit6_9" "Overcooked-AsymmAdvantages6_9" "Overcooked-CrampedRoom6_9"; +do + echo "Evaluating ${NAME} against population in ${env} for xpid ${XPID}" + CUDA_VISIBLE_DEVICES=${device} LD_LIBRARY_PATH="" nice -n 5 python3 -m minimax.evaluate_against_population \ + --xpid=${XPID} \ + --env_names=${env} \ + --population_json="populations/fcp/${env}/population.json" \ + --n_episodes=100 +done diff --git a/src/eval_xpid_all_cnn_lstm.sh b/src/eval_xpid_all_cnn_lstm.sh new file mode 100755 index 0000000..1fa1474 --- /dev/null +++ b/src/eval_xpid_all_cnn_lstm.sh @@ -0,0 +1,19 @@ +DEFAULTVALUE=4 +device="${1:-$DEFAULTVALUE}" + +./eval_xpid_against_population_in_all_layouts.sh $device DR_CNN-LSTM_SEED1 dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +./eval_xpid_against_population_in_all_layouts.sh $device DR_CNN-LSTM_SEED2 SEED_2_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +./eval_xpid_against_population_in_all_layouts.sh $device DR_CNN-LSTM_SEED3 SEED_3_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 + +./eval_xpid_against_population_in_all_layouts.sh $device PLR_CNN-LSTM_SEED1 plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +./eval_xpid_against_population_in_all_layouts.sh $device PLR_CNN-LSTM_SEED2 SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +./eval_xpid_against_population_in_all_layouts.sh $device PLR_CNN-LSTM_SEED3 SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 + +./eval_xpid_against_population_in_all_layouts.sh $device PAIRED_CNN-LSTM_SEED1 paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +./eval_xpid_against_population_in_all_layouts.sh $device PAIRED_CNN-LSTM_SEED2 SEED_2_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +./eval_xpid_against_population_in_all_layouts.sh $device PAIRED_CNN-LSTM_SEED3 SEED_3_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 + +./eval_xpid_against_population_in_all_layouts.sh $device ACCEL_CNN-LSTM_SEED1 plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +./eval_xpid_against_population_in_all_layouts.sh $device ACCEL_CNN-LSTM_SEED2 SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +./eval_xpid_against_population_in_all_layouts.sh $device ACCEL_CNN-LSTM_SEED3 SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 + diff --git a/src/eval_xpid_all_cnn_s5.sh b/src/eval_xpid_all_cnn_s5.sh new file mode 100755 index 0000000..83461ab --- /dev/null +++ b/src/eval_xpid_all_cnn_s5.sh @@ -0,0 +1,19 @@ +DEFAULTVALUE=4 +device="${1:-$DEFAULTVALUE}" + +./eval_xpid_against_population_in_all_layouts.sh $device DR_CNN-S5_SEED1 dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +./eval_xpid_against_population_in_all_layouts.sh $device DR_CNN-S5_SEED2 SEED_2_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +./eval_xpid_against_population_in_all_layouts.sh $device DR_CNN-S5_SEED3 SEED_3_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 + +./eval_xpid_against_population_in_all_layouts.sh $device PLR_CNN-S5_SEED1 plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +./eval_xpid_against_population_in_all_layouts.sh $device PLR_CNN-S5_SEED2 SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +./eval_xpid_against_population_in_all_layouts.sh $device PLR_CNN-S5_SEED3 SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 + +./eval_xpid_against_population_in_all_layouts.sh $device PAIRED_CNN-S5_SEED1 paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +./eval_xpid_against_population_in_all_layouts.sh $device PAIRED_CNN-S5_SEED2 SEED_2_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +./eval_xpid_against_population_in_all_layouts.sh $device PAIRED_CNN-S5_SEED3 SEED_3_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 + +./eval_xpid_against_population_in_all_layouts.sh $device ACCEL_CNN-S5_SEED1 plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +./eval_xpid_against_population_in_all_layouts.sh $device ACCEL_CNN-S5_SEED2 SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +./eval_xpid_against_population_in_all_layouts.sh $device ACCEL_CNN-S5_SEED3 SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 + diff --git a/src/eval_xpid_all_softmoe.sh b/src/eval_xpid_all_softmoe.sh new file mode 100755 index 0000000..d9df838 --- /dev/null +++ b/src/eval_xpid_all_softmoe.sh @@ -0,0 +1,19 @@ +DEFAULTVALUE=4 +device="${1:-$DEFAULTVALUE}" + +./eval_xpid_against_population_in_all_layouts.sh $device DR_SoftMoE_SEED1 dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +./eval_xpid_against_population_in_all_layouts.sh $device DR_SoftMoE_SEED2 SEED_2_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +./eval_xpid_against_population_in_all_layouts.sh $device DR_SoftMoE_SEED3 SEED_3_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 + +./eval_xpid_against_population_in_all_layouts.sh $device PLR_SoftMoE_SEED1 plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +./eval_xpid_against_population_in_all_layouts.sh $device PLR_SoftMoE_SEED2 SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +./eval_xpid_against_population_in_all_layouts.sh $device PLR_SoftMoE_SEED3 SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 + +./eval_xpid_against_population_in_all_layouts.sh $device PAIRED_SoftMoE_SEED1 paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +./eval_xpid_against_population_in_all_layouts.sh $device PAIRED_SoftMoE_SEED2 SEED_2_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +./eval_xpid_against_population_in_all_layouts.sh $device PAIRED_SoftMoE_SEED3 SEED_3_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 + +./eval_xpid_against_population_in_all_layouts.sh $device ACCEL_SoftMoE_SEED1 plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +./eval_xpid_against_population_in_all_layouts.sh $device ACCEL_SoftMoE_SEED2 SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +./eval_xpid_against_population_in_all_layouts.sh $device ACCEL_SoftMoE_SEED3 SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 + diff --git a/src/extract_fcp.sh b/src/extract_fcp.sh new file mode 100755 index 0000000..a4bea62 --- /dev/null +++ b/src/extract_fcp.sh @@ -0,0 +1,14 @@ +DEFAULTVALUE=4 +ENV=Overcooked-CrampedRoom5_5 # Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9 +device="${1:-$DEFAULTVALUE}" + +seed_max=8 + +for seed in `seq ${seed_max}`; +do + CUDA_VISIBLE_DEVICES=${device} LD_LIBRARY_PATH="" nice -n 5 python3 -m minimax.extract_fcp \ + --xpid=8SEED_${seed}_$2 \ + --env_names=${ENV} \ + --n_episodes=100 \ + --trained_seed=${seed} +done \ No newline at end of file diff --git a/src/make_cmd.sh b/src/make_cmd.sh new file mode 100755 index 0000000..60d0d1d --- /dev/null +++ b/src/make_cmd.sh @@ -0,0 +1 @@ +python3 -m minimax.config.make_cmd --config $1/$2 \ No newline at end of file diff --git a/src/minimax/__init__.py b/src/minimax/__init__.py new file mode 100644 index 0000000..ab54186 --- /dev/null +++ b/src/minimax/__init__.py @@ -0,0 +1,9 @@ +from . import envs +from . import agents +from . import models +from . import runners +from . import util +from . import arguments +from . import evaluate +# from . import train +from . import config diff --git a/src/minimax/agents/__init__.py b/src/minimax/agents/__init__.py new file mode 100644 index 0000000..0c4ba41 --- /dev/null +++ b/src/minimax/agents/__init__.py @@ -0,0 +1,15 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from .ppo import PPOAgent +from .mappo import MAPPOAgent + + +__all__ = [ + PPOAgent, MAPPOAgent +] \ No newline at end of file diff --git a/src/minimax/agents/agent.py b/src/minimax/agents/agent.py new file mode 100644 index 0000000..d05b15c --- /dev/null +++ b/src/minimax/agents/agent.py @@ -0,0 +1,40 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from abc import ABC + + +class Agent(ABC): + """ + Generic interface for an agent. + """ + @property + def is_recurrent(self): + pass + + @property + def action_info_keys(self): + pass + + def init_params(self, rng, obs, carry=None): + pass + + def init_carry(self, rng, batch_dims): + pass + + def act(self, *args, **kwargs): + pass + + def get_action_dist(self, dist_params, dtype): + pass + + def evaluate(self, *args, **kwargs): + pass + + def update(self, *args, **kwargs): + pass diff --git a/src/minimax/agents/mappo.py b/src/minimax/agents/mappo.py new file mode 100644 index 0000000..78ace4b --- /dev/null +++ b/src/minimax/agents/mappo.py @@ -0,0 +1,449 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial +from collections import OrderedDict + +import einops +import jax +import jax.numpy as jnp +import optax +from flax.training.train_state import TrainState +from tensorflow_probability.substrates import jax as tfp + +from .agent import Agent + + +class MAPPOAgent(Agent): + def __init__( + self, + actor, + critic, + n_epochs=5, + n_minibatches=1, + value_loss_coef=0.5, + entropy_coef=0.0, + clip_eps=0.2, + clip_value_loss=True, + track_grad_norm=False, + n_unroll_update=1, + n_devices=1): + + self.actor = actor + self.critic = critic + + self.n_epochs = n_epochs + self.n_minibatches = n_minibatches + self.value_loss_coef = value_loss_coef + self.entropy_coef = entropy_coef + self.clip_eps = clip_eps + self.clip_value_loss = clip_value_loss + self.track_grad_norm = track_grad_norm + self.n_unroll_update = n_unroll_update + self.n_devices = n_devices + + self.actor_grad_fn = jax.value_and_grad(self._actor_loss, has_aux=True) + self.critic_grad_fn = jax.value_and_grad( + self._critic_loss, has_aux=True) + + @property + def is_recurrent(self): + # Actor and Critic need to share arch for now. + return self.actor.is_recurrent + + def init_params(self, rng, obs): + """ + Returns initialized parameters and RNN hidden state for a specific + observation shape. + """ + if len(obs) == 2: + obs, shared_obs = obs + else: + raise ValueError("Obs should always be a two tuple for MAPPO!") + + rng, subrng = jax.random.split(rng) + is_recurrent = self.actor.is_recurrent + if is_recurrent: + batch_size = jax.tree_util.tree_leaves(obs)[0].shape[1] + actor_carry = self.actor.initialize_carry( + rng=subrng, batch_dims=(batch_size,)) + critic_carry = self.critic.initialize_carry( + rng=subrng, batch_dims=(batch_size,)) + reset = jnp.zeros((1, batch_size), dtype=jnp.bool_) + + rng, subrng = jax.random.split(rng) + + # Notice that these are different to later observations but they resemble what we need + actor_params = self.actor.init( + subrng, obs[:, :, 0], actor_carry, reset) + critic_params = self.critic.init( + subrng, shared_obs[:, :, 0], critic_carry, reset) + else: + + obs = jnp.concatenate(obs, axis=0) + shared_obs = jnp.concatenate(shared_obs, axis=0) + actor_params = self.actor.init(subrng, obs, None) + critic_params = self.critic.init(subrng, shared_obs, None) + + return (actor_params, critic_params) + + def init_carry(self, rng, batch_dims=1): + actor_carry = self.actor.initialize_carry( + rng=rng, batch_dims=batch_dims) + # This is for evaluation where we throw away the critic + if self.critic is not None: + critic_carry = self.critic.initialize_carry( + rng=rng, batch_dims=batch_dims) + else: + critic_carry = None + return actor_carry, critic_carry + + @partial(jax.jit, static_argnums=(0,)) + def act(self, actor_params, obs, carry=None, reset=None): + logits, carry = self.actor.apply( + actor_params, obs, carry, reset) + + return None, logits, carry + + @partial(jax.jit, static_argnums=(0,)) + def get_value(self, params, shared_obs, carry=None, reset=None): + value, new_carry = self.critic.apply(params, shared_obs, carry, reset) + return value, new_carry + + @partial(jax.jit, static_argnums=(0,)) + def evaluate_action( + self, actor_params, action, obs, actor_carry=None, reset=None + ): + dist_params, actor_carry = self.actor.apply( + actor_params, obs, actor_carry, reset) + dist = self.get_action_dist(dist_params, dtype=action.dtype) + log_prob = dist.log_prob(action) + entropy = dist.entropy() + + return log_prob.squeeze(), \ + entropy.squeeze(), \ + actor_carry + + @partial(jax.jit, static_argnums=(0,)) + def evaluate(self, params, action, obs, carry=None, reset=None): + value, dist_params, carry = self.model.apply(params, obs, carry, reset) + dist = self.get_action_dist(dist_params, dtype=action.dtype) + log_prob = dist.log_prob(action) + entropy = dist.entropy() + + return value.squeeze(), \ + log_prob.squeeze(), \ + entropy.squeeze(), \ + carry + + def get_action_dist(self, dist_params, dtype=jnp.uint8): + return tfp.distributions.Categorical(logits=dist_params, dtype=dtype) + + @partial(jax.jit, static_argnums=(0,)) + def update(self, rng, train_state, batch): + rngs = jax.random.split(rng, self.n_epochs) + + def _scan_epoch(carry, rng): + brng, urng = jax.random.split(rng) + batch, train_state = carry + minibatches = self._get_minibatches(brng, batch) + train_state, stats = \ + self._update_epoch( + urng, train_state, minibatches) + + return (batch, train_state), stats + + (_, train_state), stats = jax.lax.scan( + _scan_epoch, + (batch, train_state), + rngs, + length=len(rngs) + ) + + stats = jax.tree_util.tree_map(lambda x: x.mean(), stats) + train_state = train_state.increment_updates() + + return train_state, stats + + @partial(jax.jit, static_argnums=(0,)) + def get_empty_update_stats(self): + keys = [ + 'total_loss', # actor_loss + critic_loss + 'actor_loss', # loss_actor - entropy_coef*entropy + 'critic_loss', # value_loss_coef*value_loss + 'actor_loss_actor', # Without the entropy term added + 'actor_l2_reg_weight_loss', + 'actor_entropy', + 'actor_mean_target', + 'actor_mean_gae', + 'critic_value_loss', + 'critic_l2_reg_weight_loss', + 'critic_mean_value', + 'critic_mean_target', + 'critic_mean_gae', + 'actor_grad_norm', + 'critic_grad_norm', + ] + + return OrderedDict({k: -jnp.inf for k in keys}) + + @partial(jax.jit, static_argnums=(0,)) + def _update_epoch( + self, + rng, + train_state: TrainState, + minibatches): + + def _update_minibatch(carry, step): + rng, minibatch = step + train_state = carry + + (actor_loss, actor_aux_info), actor_grads = self.actor_grad_fn( + train_state.actor_params, + train_state.actor_apply_fn, + minibatch, + rng, + ) + + (critic_loss, critic_aux_info), critic_grads = self.critic_grad_fn( + train_state.critic_params, + train_state.critic_apply_fn, + minibatch, + rng, + ) + + total_loss = actor_loss + critic_loss + loss_info = (total_loss, actor_loss, critic_loss,) + \ + actor_aux_info + critic_aux_info + loss_info = loss_info + \ + (optax.global_norm(actor_grads), optax.global_norm(critic_grads),) + + if self.n_devices > 1: + loss_info = jax.tree_map( + lambda x: jax.lax.pmean(x, 'device'), loss_info) + actor_grads = jax.tree_map( + lambda x: jax.lax.pmean(x, 'device'), actor_grads) + critic_grads = jax.tree_map( + lambda x: jax.lax.pmean(x, 'device'), critic_grads) + + train_state = train_state.apply_gradients( + actor_grads=actor_grads, + critic_grads=critic_grads) + + stats_def = jax.tree_util.tree_structure(OrderedDict({ + k: 0 for k in [ + 'total_loss', # actor_loss + critic_loss + 'actor_loss', # loss_actor - entropy_coef*entropy + 'critic_loss', # value_loss_coef*value_loss + 'actor_loss_actor', # Without the entropy term added + 'actor_l2_reg_weight_loss', + 'actor_entropy', + 'actor_mean_target', + 'actor_mean_gae', + 'critic_value_loss', + 'critic_l2_reg_weight_loss', + 'critic_mean_value', + 'critic_mean_target', + 'critic_mean_gae', + 'actor_grad_norm', + 'critic_grad_norm', + ]})) + + loss_stats = jax.tree_util.tree_unflatten( + stats_def, jax.tree_util.tree_leaves(loss_info)) + return train_state, loss_stats + + rngs = jax.random.split(rng, self.n_minibatches) + train_state, loss_stats = jax.lax.scan( + _update_minibatch, + train_state, + (rngs, minibatches), + length=self.n_minibatches, + unroll=self.n_unroll_update + ) + + loss_stats = jax.tree_util.tree_map( + lambda x: x.mean(axis=0), loss_stats) + + return train_state, loss_stats + + @partial(jax.jit, static_argnums=(0, 2, 4)) + def _actor_loss( + self, + params, + apply_fn, + batch, + rng=None + ): + """Currently the shape of elements is n_rollout_steps x n_envs x n_env_agents x ...shape. + This is one more than intended for the actor and critic. The extra dimension is for the + env agents. We thus need to merge it into the n_envs dimension. + """ + carry = None + + if self.is_recurrent: + """ + Elements have batch shape of n_rollout_steps x n_envs//n_minibatches + """ + batch = jax.tree_map( + lambda x: einops.rearrange( + x, 't n a ... -> t (n a) ...'), batch + ) + carry = jax.tree_util.tree_map( + lambda x: x[0, :], batch.actor_carry) + obs, _, action, rewards, dones, log_pi_old, value_old, target, gae, carry_old, _ = batch + + if self.is_recurrent: + dones = dones.at[1:, :].set(dones[:-1, :]) + dones = dones.at[0, :].set(False) + _batch = batch._replace(dones=dones) + + # Returns LxB and LxBxH tensors + obs, _, action, _, done, _, _, _, _, _, _ = _batch + log_pi, entropy, carry = apply_fn( + params, action, obs, carry, done) + else: + log_pi, entropy, carry = apply_fn( + params, action, obs, carry_old) + else: + batch = jax.tree_map( + lambda x: einops.rearrange(x, 'n a ... -> (n a) ...'), batch + ) + obs, _, action, rewards, dones, log_pi_old, value_old, target, gae, _, _ = batch + log_pi, entropy, _ = apply_fn(params, action, obs, carry) + + ratio = jnp.exp(log_pi - log_pi_old) + norm_gae = (gae - gae.mean()) / (gae.std() + 1e-5) + loss_actor1 = ratio * norm_gae + loss_actor2 = jnp.clip(ratio, 1.0 - self.clip_eps, + 1.0 + self.clip_eps) * norm_gae + loss_actor = -jnp.minimum(loss_actor1, loss_actor2).mean() + + entropy = entropy.mean() + + l2_reg_actor = 0.0 + + actor_loss = loss_actor - self.entropy_coef * entropy + l2_reg_actor + + return actor_loss, ( + loss_actor, + l2_reg_actor, + entropy, + target.mean(), + gae.mean() + ) + + @partial(jax.jit, static_argnums=(0, 2, 4)) + def _critic_loss( + self, + params, + apply_fn, + batch, + rng=None + ): + + carry = None + + if self.is_recurrent: + """ + Elements have batch shape of n_rollout_steps x n_envs//n_minibatches + """ + "Same as in actor loss:" + batch = jax.tree_map( + lambda x: einops.rearrange( + x, 't n a ... -> t (n a) ...'), batch + ) + carry = jax.tree_util.tree_map( + lambda x: x[0, :], batch.critic_carry) + _, obs_shared, action, rewards, dones, log_pi_old, value_old, target, gae, _, carry_old = batch + + if self.is_recurrent: + dones = dones.at[1:, :].set(dones[:-1, :]) + dones = dones.at[0, :].set(False) + _batch = batch._replace(dones=dones) + + # Returns LxB and LxBxH tensors + _, obs_shared, action, _, done, _, _, _, _, _, _ = _batch + value, carry = apply_fn( + params, obs_shared, carry, done) + else: + value, carry = apply_fn( + params, obs_shared, carry_old) + value = value.squeeze(-1) + else: + batch = jax.tree_map( + lambda x: einops.rearrange(x, 'n a ... -> (n a) ...'), batch + ) + obs, obs_shared, action, rewards, dones, log_pi_old, value_old, target, gae, _, _ = batch + value, _ = apply_fn(params, obs_shared, carry) + + if self.clip_value_loss: + value_pred_clipped = value_old + (value - value_old).clip( + -self.clip_eps, self.clip_eps + ) + value_losses = jnp.square(value - target) + value_losses_clipped = jnp.square(value_pred_clipped - target) + value_loss = 0.5 * \ + jnp.maximum(value_losses, value_losses_clipped).mean() + else: + value_pred_clipped = value_old + (value - value_old).clip( + -self.clip_eps, self.clip_eps + ) + value_loss = optax.huber_loss( + value_pred_clipped, target, delta=10.0).mean() + + l2_reg_critic = 0.0 + + critic_loss = self.value_loss_coef*value_loss + l2_reg_critic + + return critic_loss, ( + value_loss, + l2_reg_critic, + value.mean(), + target.mean(), + gae.mean() + ) + + @partial(jax.jit, static_argnums=0) + def _get_minibatches(self, rng, batch): + # get dims based on dones + n_rollout_steps, n_envs = batch.dones.shape[0:2] + if self.is_recurrent: + """ + Reshape elements into a batch shape of + n_minibatches x n_envs//n_minibatches x n_rollout_steps. + """ + assert n_envs % self.n_minibatches == 0, \ + 'Number of environments must be divisible into number of minibatches.' + + n_env_per_minibatch = n_envs//self.n_minibatches + shuffled_idx = jax.random.permutation(rng, jnp.arange(n_envs)) + + shuffled_batch = jax.tree_util.tree_map( + lambda x: jnp.take(x, shuffled_idx, axis=1), batch) + + minibatches = jax.tree_util.tree_map( + lambda x: x.swapaxes(0, 1).reshape( + self.n_minibatches, + n_env_per_minibatch, + n_rollout_steps, + *x.shape[2:] + ).swapaxes(1, 2), shuffled_batch) + else: + n_txns = n_envs*n_rollout_steps + assert n_envs*n_rollout_steps % self.n_minibatches == 0 + + shuffled_idx = jax.random.permutation(rng, jnp.arange(n_txns)) + shuffled_batch = jax.tree_util.tree_map( + lambda x: jnp.take( + x.reshape(n_txns, *x.shape[2:]), + shuffled_idx, axis=0), batch) + minibatches = jax.tree_util.tree_map( + lambda x: x.reshape(self.n_minibatches, -1, *x.shape[1:]), shuffled_batch) + + return minibatches diff --git a/src/minimax/agents/ppo.py b/src/minimax/agents/ppo.py new file mode 100644 index 0000000..63b6ceb --- /dev/null +++ b/src/minimax/agents/ppo.py @@ -0,0 +1,304 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial +from typing import Any, Callable, Tuple +from collections import defaultdict, OrderedDict + +import jax +import jax.numpy as jnp +import optax +from flax.training.train_state import TrainState +from tensorflow_probability.substrates import jax as tfp + +from .agent import Agent + + +class PPOAgent(Agent): + def __init__( + self, + model, + n_epochs=5, + n_minibatches=1, + value_loss_coef=0.5, + entropy_coef=0.0, + clip_eps=0.2, + clip_value_loss=True, + track_grad_norm=False, + n_unroll_update=1, + n_devices=1): + + self.model = model + self.n_epochs = n_epochs + self.n_minibatches = n_minibatches + self.value_loss_coef = value_loss_coef + self.entropy_coef = entropy_coef + self.clip_eps = clip_eps + self.clip_value_loss = clip_value_loss + self.track_grad_norm = track_grad_norm + self.n_unroll_update = n_unroll_update + self.n_devices = n_devices + + self.grad_fn = jax.value_and_grad(self._loss, has_aux=True) + + @property + def is_recurrent(self): + return self.model.is_recurrent + + def init_params(self, rng, obs): + """ + Returns initialized parameters and RNN hidden state for a specific + observation shape. + """ + rng, subrng = jax.random.split(rng) + if self.model.is_recurrent: + batch_size = jax.tree_util.tree_leaves(obs)[0].shape[1] + carry = self.model.initialize_carry( + rng=subrng, batch_dims=(batch_size,)) + reset = jnp.zeros((1, batch_size), dtype=jnp.bool_) + rng, subrng = jax.random.split(rng) + params = self.model.init(subrng, obs, carry, reset) + else: + params = self.model.init(subrng, obs) + + return params + + def init_carry(self, rng, batch_dims=(1,)): + return self.model.initialize_carry(rng=rng, batch_dims=batch_dims) + + @partial(jax.jit, static_argnums=(0,)) + def act(self, params, obs, carry=None, reset=None): + value, logits, carry = self.model.apply(params, obs, carry, reset) + + return value, logits, carry + + @partial(jax.jit, static_argnums=(0,)) + def get_value(self, params, obs, carry=None, reset=None): + value, _, carry = self.model.apply(params, obs, carry, reset) + return value, carry + + @partial(jax.jit, static_argnums=(0,)) + def evaluate(self, params, action, obs, carry=None, reset=None): + value, dist_params, carry = self.model.apply(params, obs, carry, reset) + dist = self.get_action_dist(dist_params, dtype=action.dtype) + log_prob = dist.log_prob(action) + entropy = dist.entropy() + + return value.squeeze(), \ + log_prob.squeeze(), \ + entropy.squeeze(), \ + carry + + def get_action_dist(self, dist_params, dtype=jnp.uint8): + return tfp.distributions.Categorical(logits=dist_params, dtype=dtype) + + @partial(jax.jit, static_argnums=(0,)) + def update(self, rng, train_state, batch): + rngs = jax.random.split(rng, self.n_epochs) + + def _scan_epoch(carry, rng): + brng, urng = jax.random.split(rng) + batch, train_state = carry + minibatches = self._get_minibatches(brng, batch) + train_state, stats = \ + self._update_epoch( + urng, train_state, minibatches) + + return (batch, train_state), stats + + (_, train_state), stats = jax.lax.scan( + _scan_epoch, + (batch, train_state), + rngs, + length=len(rngs) + ) + + stats = jax.tree_util.tree_map(lambda x: x.mean(), stats) + train_state = train_state.increment_updates() + + return train_state, stats + + @partial(jax.jit, static_argnums=(0,)) + def get_empty_update_stats(self): + keys = ['total_loss', + 'actor_loss', + 'value_loss', + 'entropy', + 'mean_value', + 'mean_target', + 'mean_gae', + 'grad_norm'] + + return OrderedDict({k: -jnp.inf for k in keys}) + + @partial(jax.jit, static_argnums=(0,)) + def _update_epoch( + self, + rng, + train_state: TrainState, + minibatches): + + def _update_minibatch(carry, step): + rng, minibatch = step + train_state = carry + + (loss, aux_info), grads = self.grad_fn( + train_state.params, + train_state.apply_fn, + minibatch, + rng, + ) + + loss_info = (loss,) + aux_info + loss_info = loss_info + (optax.global_norm(grads),) + + if self.n_devices > 1: + loss_info = jax.tree_map( + lambda x: jax.lax.pmean(x, 'device'), loss_info) + grads = jax.tree_map( + lambda x: jax.lax.pmean(x, 'device'), grads) + + train_state = train_state.apply_gradients(grads=grads) + + stats_def = jax.tree_util.tree_structure(OrderedDict({ + k: 0 for k in [ + 'total_loss', + 'actor_loss', + 'value_loss', + 'entropy', + 'mean_value', + 'mean_target', + 'mean_gae', + 'grad_norm', + ]})) + + loss_stats = jax.tree_util.tree_unflatten( + stats_def, jax.tree_util.tree_leaves(loss_info)) + + return train_state, loss_stats + + rngs = jax.random.split(rng, self.n_minibatches) + train_state, loss_stats = jax.lax.scan( + _update_minibatch, + train_state, + (rngs, minibatches), + length=self.n_minibatches, + unroll=self.n_unroll_update + ) + + loss_stats = jax.tree_util.tree_map( + lambda x: x.mean(axis=0), loss_stats) + + return train_state, loss_stats + + @partial(jax.jit, static_argnums=(0, 2, 4)) + def _loss( + self, + params, + apply_fn, + batch, + rng=None): + carry = None + + if self.is_recurrent: + """ + Elements have batch shape of n_rollout_steps x n_envs//n_minibatches + """ + carry = jax.tree_util.tree_map(lambda x: x[0, :], batch.carry) + obs, action, rewards, dones, log_pi_old, value_old, target, gae, carry_old = batch + + if self.is_recurrent: + dones = dones.at[1:, :].set(dones[:-1, :]) + dones = dones.at[0, :].set(False) + _batch = batch._replace(dones=dones) + + # Returns LxB and LxBxH tensors + obs, action, _, done, _, _, _, _, _ = _batch + value, log_pi, entropy, carry = apply_fn( + params, action, obs, carry, done) + else: + value, log_pi, entropy, carry = apply_fn( + params, action, obs, carry_old) + else: + obs, action, rewards, dones, log_pi_old, value_old, target, gae, _ = batch + value, log_pi, entropy, _ = apply_fn(params, action, obs, carry) + + if self.clip_value_loss: + value_pred_clipped = value_old + (value - value_old).clip( + -self.clip_eps, self.clip_eps + ) + value_losses = jnp.square(value - target) + value_losses_clipped = jnp.square(value_pred_clipped - target) + value_loss = 0.5 * \ + jnp.maximum(value_losses, value_losses_clipped).mean() + else: + value_loss = optax.huber_loss(value, target).mean() + + if self.model.value_ensemble_size > 1: + gae = gae.at[..., 0].get() + + ratio = jnp.exp(log_pi - log_pi_old) + norm_gae = (gae - gae.mean()) / (gae.std() + 1e-5) + loss_actor1 = ratio * norm_gae + loss_actor2 = jnp.clip(ratio, 1.0 - self.clip_eps, + 1.0 + self.clip_eps) * norm_gae + loss_actor = -jnp.minimum(loss_actor1, loss_actor2).mean() + + entropy = entropy.mean() + + total_loss = ( + loss_actor + self.value_loss_coef*value_loss - self.entropy_coef*entropy + ) + + return total_loss, ( + loss_actor, + value_loss, + entropy, + value.mean(), + target.mean(), + gae.mean() + ) + + @partial(jax.jit, static_argnums=0) + def _get_minibatches(self, rng, batch): + # get dims based on dones + n_rollout_steps, n_envs = batch.dones.shape[0:2] + if self.is_recurrent: + """ + Reshape elements into a batch shape of + n_minibatches x n_envs//n_minibatches x n_rollout_steps. + """ + assert n_envs % self.n_minibatches == 0, \ + 'Number of environments must be divisible into number of minibatches.' + + n_env_per_minibatch = n_envs//self.n_minibatches + shuffled_idx = jax.random.permutation(rng, jnp.arange(n_envs)) + + shuffled_batch = jax.tree_util.tree_map( + lambda x: jnp.take(x, shuffled_idx, axis=1), batch) + + minibatches = jax.tree_util.tree_map( + lambda x: x.swapaxes(0, 1).reshape( + self.n_minibatches, + n_env_per_minibatch, + n_rollout_steps, + *x.shape[2:] + ).swapaxes(1, 2), shuffled_batch) + else: + n_txns = n_envs*n_rollout_steps + assert n_envs*n_rollout_steps % self.n_minibatches == 0 + + shuffled_idx = jax.random.permutation(rng, jnp.arange(n_txns)) + shuffled_batch = jax.tree_util.tree_map( + lambda x: jnp.take( + x.reshape(n_txns, *x.shape[2:]), + shuffled_idx, axis=0), batch) + minibatches = jax.tree_util.tree_map( + lambda x: x.reshape(self.n_minibatches, -1, *x.shape[1:]), shuffled_batch) + + return minibatches diff --git a/src/minimax/arguments.py b/src/minimax/arguments.py new file mode 100644 index 0000000..53e0c81 --- /dev/null +++ b/src/minimax/arguments.py @@ -0,0 +1,1023 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +import argparse + +from minimax.util.parsnip import Parsnip +from minimax.util.args import str2bool + + +parser = Parsnip() + + +# ==== Define top-level arguments +parser.add_argument( + '--seed', + type=int, + default=1, + help='Training seed.') +parser.add_argument( + '--agent_rl_algo', + type=str, + default='ppo', + choices=['ppo'], + help='Base RL algorithm to use.') +parser.add_argument( + '--n_total_updates', + type=int, + default=30000, + help='Total number of student gradient updates.') +parser.add_argument( + '--train_runner', + type=str, + default='dr', + choices=['dr', 'plr', 'paired'], + help='Algorithm runner.') +parser.add_argument( + '--n_devices', + type=int, + default=1, + help='Number of devices.') +parser.add_argument( + '--is_multi_agent', + type=str2bool, + default=False, + help='Whether multi agent env or not.') +parser.add_argument( + '--n_shaped_reward_steps', + type=int, + default=0, + help='Number of steps to use shaped reward for (linear decreased).') + +parser.add_argument( + '--n_shaped_reward_updates', + type=int, + default=0, + help='Number of steps to use shaped reward for (linear decreased).') + + +# ==== RL runner arguments. +train_runner_subparser = parser.add_subparser( + name='train_runner') +train_runner_subparser.add_argument( + '--n_students', + type=int, + default=1, + help='Number of students in population.') +train_runner_subparser.add_argument( + '--n_parallel', + type=int, + default=1, + help='Number of parallel environments per rollout.') +train_runner_subparser.add_argument( + '--n_eval', + type=int, + default=1, + help='Number of student evaluations per environment.') +train_runner_subparser.add_argument( + '--n_rollout_steps', + type=int, + default=250, + help='Number of rollout steps.') +train_runner_subparser.add_argument( + '--lr', + type=float, + default=1e-4, + help='Initial learning rate.') +train_runner_subparser.add_argument( + '--lr_final', + type=float, + default=None, + nargs="?", + help='Final learning rate.') +train_runner_subparser.add_argument( + '--lr_anneal_steps', + type=int, + default=0, + nargs="?", + help='Number of learning rate annealing steps.') +train_runner_subparser.add_argument( + '--max_grad_norm', + type=float, + default=0.5, + help='max norm of gradients.') +train_runner_subparser.add_argument( + '--adam_eps', + type=float, + default=1e-5, + help='Adam eps.') +train_runner_subparser.add_argument( + '--track_env_metrics', + type=str2bool, + default=False, + help='Track env metrics during training. Can reduce SPS.') +train_runner_subparser.add_argument( + '--discount', + type=float, + default=0.995, + help='Student discount factor for rewards') +train_runner_subparser.add_argument( + '--n_unroll_rollout', + type=int, + default=1, + help='Number of times to unroll rollout scan.') +train_runner_subparser.add_argument( + '--render', + type=str2bool, + default=False, + help='Whether to render.') + +# ------ AC-specific arguments ----- +dr_subparser = parser.add_subparser( + name='dr', + prefix='dr', + dependency={'train_runner': 'dr'}, + dest='train_runner') + +# -------- General UED arguments -------- +parser.add_dependent_argument( + '--ued_score', + type=str, + default='relative_regret', + dependency={'train_runner': ['plr', 'paired']}, + dest='train_runner', + choices=[ + 'relative_regret', + 'mean_relative_regret', + 'population_regret', + 'neg_return', # aka minimax adversarial + 'l1_value_loss', + 'positive_value_loss', + 'max_mc', + 'value_disagreement' + ], + help='UED score of agent.') + +# -------- PAIRED arguments -------- +plr_subparser = parser.add_subparser( + name='plr', + prefix='plr', + dependency={'train_runner': 'plr'}, + dest='train_runner') +plr_subparser.add_argument( + '--replay_prob', + type=float, + default=0.5, + help='PLR replay probability.' +) +plr_subparser.add_argument( + '--buffer_size', + type=int, + default=128, + help='PLR level buffer size.' +) +plr_subparser.add_argument( + '--staleness_coef', + type=float, + default=0.3, + help='Staleness coefficient.' +) +plr_subparser.add_argument( + '--temp', + type=float, + default=1.0, + help='Score distribution temperature.' +) +plr_subparser.add_argument( + '--use_score_ranks', + type=str2bool, + default=True, + help='Use rank-based prioritiziation.' +) +plr_subparser.add_argument( + '--min_fill_ratio', + type=float, + default=0.5, + help='Minimum fill ratio before level replay begins.' +) +plr_subparser.add_argument( + '--use_robust_plr', + type=str2bool, + default=True, + help='Use robust PLR.' +) +plr_subparser.add_argument( + '--use_parallel_eval', + type=str2bool, + default=False, + help='Use rank-based prioritiziation.' +) +plr_subparser.add_argument( + '--force_unique', + type=str2bool, + default=False, + help='Force level buffer members to be unique.' +) +plr_subparser.add_argument( + '--mutation_fn', + type=str, + default=None, + help='Name of mutation function for ACCEL.' +) +plr_subparser.add_argument( + '--n_mutations', + type=int, + default=0, + help='Number of mutations per iteration of ACCEL.' +) +plr_subparser.add_argument( + '--mutation_criterion', + type=str, + default='batch', + help='Criterion for choosing PLR buffer members to mutate.' +) +plr_subparser.add_argument( + '--mutation_subsample_size', + type=int, + default=0, + help='Number of PLR buffer members to mutate into a full batch.' +) + + +# -------- PAIRED arguments -------- +paired_subparser = parser.add_subparser( + name='paired', + prefix='paired', + dependency={'train_runner': 'paired'}, + dest='train_runner') + + +# ==== Student RL arguments. +student_rl_subparser = parser.add_subparser( + name='student_rl', + prefix='student') +student_rl_subparser.add_argument( + '--entropy_coef', + type=float, + default=0.0, + help='entropy term coefficient') +student_rl_subparser.add_argument( + '--value_loss_coef', + type=float, + default=0.5, + help='value loss coefficient (default: 0.5)') +student_rl_subparser.add_argument( + '--n_unroll_update', + type=int, + default=1, + help='Number of times to unroll minibatch scan.') + +# -------- Student PPO arguments. -------- +student_ppo_subparser = parser.add_subparser( + name='student_ppo', + prefix='student_ppo', + dest='student_rl', + dependency={'agent_rl_algo': 'ppo'}) +student_ppo_subparser.add_argument( + '--n_epochs', + type=int, + default=5, + help='Number of PPO epochs.') +student_ppo_subparser.add_argument( + '--n_minibatches', + type=int, + default=1, + help='Number of minibatches per PPO epoch.') +student_ppo_subparser.add_argument( + '--clip_eps', + type=float, + default=0.2, + help='PPO clip parameter') +student_ppo_subparser.add_argument( + '--clip_value_loss', + type=str2bool, + default=True, + help='ppo clip value loss') +parser.add_dependent_argument( + '--gae_lambda', + type=float, + default=0.95, + prefix='student', + dependency={'agent_rl_algo': 'ppo'}, + dest='train_runner', + help='GAE lambda parameter for student.') + + +# ==== Teacher RL arguments. +teacher_rl_subparser = parser.add_subparser( + name='teacher_rl', + prefix='teacher', + dependency={'train_runner': ['paired']}) +teacher_rl_subparser.add_argument( + '--entropy_coef', + type=float, + default=0.0, + help='entropy term coefficient') +teacher_rl_subparser.add_argument( + '--value_loss_coef', + type=float, + default=0.5, + help='value loss coefficient (default: 0.5)') +teacher_rl_subparser.add_argument( + '--n_unroll_update', + type=int, + default=1, + help='Number of times to unroll minibatch scan.') +parser.add_dependent_argument( + '--teacher_discount', + type=float, + default=0.995, + dependency={'train_runner': 'paired'}, + dest='train_runner', + help='discount factor for rewards') +parser.add_dependent_argument( + '--teacher_lr', + type=float, + default=None, + nargs="?", + dependency={'agent_rl_algo': 'ppo', 'train_runner': 'paired'}, + dest='train_runner', + help='Initial learning rate of teacher.') +parser.add_dependent_argument( + '--teacher_lr_final', + type=float, + default=None, + nargs="?", + dependency={'agent_rl_algo': 'ppo', 'train_runner': 'paired'}, + dest='train_runner', + help='Initial learning rate of teacher.') +parser.add_dependent_argument( + '--teacher_lr_anneal_steps', + type=int, + default=0, + nargs="?", + dependency={'agent_rl_algo': 'ppo', 'train_runner': 'paired'}, + dest='train_runner', + help='Initial learning rate of teacher.') + + +# -------- Teacher PPO arguments. -------- +teacher_ppo_subparser = parser.add_subparser( + name='teacher_ppo', + prefix='teacher_ppo', + dest='teacher_rl', + dependency={'agent_rl_algo': 'ppo', 'train_runner': 'paired'}) +teacher_ppo_subparser.add_argument( + '--n_epochs', + type=int, + default=5, + help='Number of PPO epochs.') +teacher_ppo_subparser.add_argument( + '--n_minibatches', + type=int, + default=1, + help='Number of minibatches per PPO epoch.') +teacher_ppo_subparser.add_argument( + '--clip_eps', + type=float, + default=0.2, + help='PPO clip parameter') +teacher_ppo_subparser.add_argument( + '--clip_value_loss', + type=str2bool, + default=True, + help='ppo clip value loss') +parser.add_dependent_argument( + '--teacher_gae_lambda', + type=float, + default=0.95, + dependency={'agent_rl_algo': 'ppo', 'train_runner': 'paired'}, + dest='train_runner', + help='GAE lambda parameter for teacher.') + + +# ==== Student model arguments. +parser.add_argument( + '--student_model_name', + type=str, + default='default_student_cnn', + help='Name of student model architecture.') +parser.add_argument( + '--student_critic_model_name', + type=str, + default=None, + help='Name of student critic model architecture (for MAPPO).') +parser.add_argument( + '--student_agent_kind', + type=str, + default="ppo", + help='PPO vs MAPPO.') + +# Placeholder group for student model args +student_model_parser = parser.add_subparser( + name='student_model', + prefix='student') + +# ---- Maze args for student model ---- +student_maze_model_parser = parser.add_subparser( + name='student_maze_model', + prefix='student', + dest="student_model", + dependency={'env_name': ['Maze*', 'Overcooked*']}) +student_maze_model_parser.add_argument( + '--is_soft_moe', + type=str2bool, + nargs='?', + const=True, + default=False, + help='Whether to use SoftMoE.') +student_maze_model_parser.add_argument( + '--soft_moe_num_experts', + type=int, + default=4, + help='Number of Experts in the SoftMoE layer.') +student_maze_model_parser.add_argument( + '--soft_moe_num_slots', + type=int, + default=32, + help='Number of Slots in the SoftMoE layer.') +student_maze_model_parser.add_argument( + '--recurrent_arch', + type=str, + default=None, + nargs='?', + choices=['gru', 'lstm', 's5'], + help='Student RNN architecture.') +student_maze_model_parser.add_argument( + '--recurrent_hidden_dim', + type=int, + default=0, + help='Student recurrent hidden state size.') +student_maze_model_parser.add_argument( + '--hidden_dim', + type=int, + default=32, + help='Student hidden dimension.') +student_maze_model_parser.add_argument( + '--n_hidden_layers', + type=int, + default=1, + help='Student number of hidden layers in policy/value heads.') +student_maze_model_parser.add_argument( + '--n_conv_layers', + type=int, + default=1, + help='Number of CNN filters for student.') +student_maze_model_parser.add_argument( + '--n_conv_filters', + type=int, + default=16, + help='Number of CNN filters for student.') +student_maze_model_parser.add_argument( + '--n_scalar_embeddings', + type=int, + default=4, + help='Defaults to 4 directional embeddings.') +student_maze_model_parser.add_argument( + '--scalar_embed_dim', + type=int, + default=5, + help='Dimensionality of scalar direction embeddings.') +student_maze_model_parser.add_argument( + '--base_activation', + type=str, + default='relu', + choices=['relu', 'gelu', 'crelu', 'leaky_relu'], + help='Nonlinearity for intermediate layers.') +student_maze_model_parser.add_argument( + '--value_ensemble_size', + type=int, + default=1, + help='Size of value ensemble. Defaults to 1 (no ensemble).') +student_maze_model_parser.add_argument( + '--s5_n_blocks', + type=int, + default=1, + help='Number of S5 blocks.') +student_maze_model_parser.add_argument( + '--s5_n_layers', + type=int, + default=4, + help='Number of S5 encoder layers.') +student_maze_model_parser.add_argument( + '--s5_layernorm_pos', + type=str, + default=None, + help='Layernorm pos in S5.') +student_maze_model_parser.add_argument( + '--s5_activation', + type=str, + default="half_glu1", + choices=["half_glu1", "half_glu2", "full_glu", "gelu"], + help='Number of S5 encoder layers.') + + +# ==== Teacher model arguments. +parser.add_dependent_argument( + '--teacher_model_name', + dependency={'train_runner': ['paired']}, + type=str, + help='Name of teacher model architecture.' +) + +# Placeholder group for teacher model args +teacher_model_parser = parser.add_subparser( + name='teacher_model', + prefix='teacher', + dependency={'train_runner': ['paired']}) + +# ---- Maze args for PAIRED teacher model ---- +teacher_maze_model_parser = parser.add_subparser( + name='teacher_maze_model', + prefix='teacher', + dest="teacher_model", + dependency={'train_runner': 'paired', 'env_name': ['Maze*', 'Overcooked*']}) +teacher_maze_model_parser.add_argument( + '--is_soft_moe', + type=str2bool, + nargs='?', + const=True, + default=False, + help='Whether to use SoftMoE.') +teacher_maze_model_parser.add_argument( + '--soft_moe_num_experts', + type=int, + default=4, + help='Number of Experts in the SoftMoE layer.') +teacher_maze_model_parser.add_argument( + '--soft_moe_num_slots', + type=int, + default=32, + help='Number of Slots in the SoftMoE layer.') +teacher_maze_model_parser.add_argument( + '--recurrent_arch', + type=str, + default=None, + nargs='?', + choices=['gru', 'lstm', 's5'], + help='Teacher RNN architecture.') +teacher_maze_model_parser.add_argument( + '--recurrent_hidden_dim', + type=int, + default=0, + help='Teacher recurrent hidden state size.') +teacher_maze_model_parser.add_argument( + '--hidden_dim', + type=int, + default=32, + help='Teacher hidden dimension.') +teacher_maze_model_parser.add_argument( + '--n_hidden_layers', + type=int, + default=1, + help='Teacher number of hidden layers in policy/value heads.') +teacher_maze_model_parser.add_argument( + '--n_conv_layers', + type=int, + default=1, + help='Number of CNN filters for teacher.') +teacher_maze_model_parser.add_argument( + '--n_conv_filters', + type=int, + default=128, + help='Number of CNN filters for teacher.') +teacher_maze_model_parser.add_argument( + '--scalar_embed_dim', + type=int, + default=10, + help='Dimensionality of time-step embeddings.') +teacher_maze_model_parser.add_argument( + '--base_activation', + type=str, + default='relu', + choices=['relu', 'gelu', 'crelu', 'leaky_relu'], + help='Nonlinearity for intermediate layers.') +teacher_maze_model_parser.add_argument( + '--s5_n_blocks', + type=int, + default=1, + help='Number of S5 blocks.') +teacher_maze_model_parser.add_argument( + '--s5_n_layers', + type=int, + default=4, + help='Number of S5 encoder layers.') +teacher_maze_model_parser.add_argument( + '--s5_layernorm_pos', + type=str, + default=None, + help='Layernorm pos in S5.') +teacher_maze_model_parser.add_argument( + '--s5_activation', + type=str, + default="half_glu1", + choices=["half_glu1", "half_glu2", "full_glu", "gelu"], + help='Number of S5 encoder layers.') + + +# ==== Environment arguments. +parser.add_argument( + '--env_name', + type=str, + default='Maze', + help='Environment to train on') +env_parser = parser.add_subparser( + name='env') + +# -------- UED environment arguments. -------- +ued_env_parser = parser.add_subparser( + name='ued_env') + +# ======== Envoronment-specific subparsers ======== +# -------- Overcooked -------- +env_overcooked_parser = parser.add_subparser( + name='overcooked', + prefix='overcooked', + dependency={'env_name': ['Overcooked', 'Overcooked*']}, + dest='env') +env_overcooked_parser.add_argument( + '--height', + type=int, + default=13, + help='Height of training mazes.') +env_overcooked_parser.add_argument( + '--width', + type=int, + default=13, + help='Width of training mazes.') +env_overcooked_parser.add_argument( + '--random_reset', + type=str2bool, + nargs='?', + const=True, + default=False, + help='If random reset.') +env_overcooked_parser.add_argument( + '--n_walls', + type=int, + default=25, + help='Maximum number of walls in training mazes.') +env_overcooked_parser.add_argument( + '--replace_wall_pos', + type=str2bool, + nargs='?', + const=True, + default=False, + help='Sample wall positions with replacement.') +env_overcooked_parser.add_argument( + '--sample_n_walls', + type=str2bool, + nargs='?', + const=True, + default=False, + help='Uniformly sample n_walls between 0 and n_walls.') +env_overcooked_parser.add_argument( + '--normalize_obs', + type=str2bool, + nargs='?', + const=True, + default=True, + help='Ensure observations are between 0 and 1.') +env_overcooked_parser.add_argument( + '--max_steps', + type=int, + default=400, + help='Maximum number of steps in training episodes.') +env_overcooked_parser.add_argument( + '--fix_to_single_layout', + type=str, + default=None, + help='Fixes Overcooked to a single layout instead of a randome one during reset.') +env_overcooked_parser.add_argument( + '--dense_obs', + type=str2bool, + nargs='?', + const=True, + default=False, + help='Ensure observations are between 0 and 1.') +# -------- Maze -------- +env_maze_parser = parser.add_subparser( + name='maze', + prefix='maze', + dependency={'env_name': ['Maze', 'Maze-MemoryMaze']}, + dest='env') +env_maze_parser.add_argument( + '--height', + type=int, + default=13, + help='Height of training mazes.') +env_maze_parser.add_argument( + '--width', + type=int, + default=13, + help='Width of training mazes.') +env_maze_parser.add_argument( + '--n_walls', + type=int, + default=25, + help='Maximum number of walls in training mazes.') +env_maze_parser.add_argument( + '--replace_wall_pos', + type=str2bool, + nargs='?', + const=True, + default=False, + help='Sample wall positions with replacement.') +env_maze_parser.add_argument( + '--sample_n_walls', + type=str2bool, + nargs='?', + const=True, + default=False, + help='Uniformly sample n_walls between 0 and n_walls.') +# -------- Maze* environments -------- +env_maze_all_parser = parser.add_subparser( + name='maze_all', + prefix='maze', + dependency={'env_name': 'Maze*'}, + dest='env') +env_maze_all_parser.add_argument( + '--see_agent', + type=str2bool, + nargs='?', + const=True, + default=True, + help='Whether the agent sees itself in observations.') +env_maze_all_parser.add_argument( + '--normalize_obs', + type=str2bool, + nargs='?', + const=True, + default=True, + help='Ensure observations are between 0 and 1.') +env_maze_all_parser.add_argument( + '--obs_agent_pos', + type=str2bool, + nargs='?', + const=True, + default=False, + help='Include agent xy pos in observations.') +env_maze_all_parser.add_argument( + '--max_episode_steps', + type=int, + default=250, + help='Maximum number of steps in training episodes.') + +# -------- Maze UED -------- +maze_ued_parser = parser.add_subparser( + name='maze_ued', + prefix='maze_ued', + dependency={'env_name': ['Maze', 'Maze-MemoryMaze'], + 'train_runner': 'paired'}, + dest='ued_env') +maze_ued_parser.add_argument( + '--replace_wall_pos', + type=str2bool, + nargs='?', + const=True, + default=False, + help='Teacher can sample same wall pos multiple times (resulting in variable n_walls).') +maze_ued_parser.add_argument( + '--fixed_n_wall_steps', + type=str2bool, + nargs='?', + const=True, + default=False, + help='Teacher samples exactly n_walls wall positions for each level.') +maze_ued_parser.add_argument( + '--first_wall_pos_sets_budget', + type=str2bool, + nargs='?', + const=True, + default=False, + help='The first wall positional index determines the wall budget.') +maze_ued_parser.add_argument( + '--noise_dim', + type=int, + default=50, + help="Dimension of episodic noise vector injected into the teacher's observation.") +maze_ued_parser.add_argument( + '--n_walls', + type=int, + default=25, + help="Number walls the adversary can place.") +maze_ued_parser.add_argument( + '--set_agent_dir', + type=str2bool, + nargs='?', + const=True, + default=False, + help='Teacher chooses the agent direction on last time step.') +maze_ued_parser.add_argument( + '--normalize_obs', + type=str2bool, + nargs='?', + const=True, + default=True, + help='Normalize teacher observations.') + +# -------- Overcooked UED -------- +overcooked_ued_parser = parser.add_subparser( + name='overcooked_ued', + prefix='overcooked_ued', + dependency={'env_name': ['Overcooked'], 'train_runner': 'paired'}, + dest='ued_env') +overcooked_ued_parser.add_argument( + '--replace_wall_pos', + type=str2bool, + nargs='?', + const=True, + default=False, + help='Teacher can sample same wall pos multiple times (resulting in variable n_walls).') +overcooked_ued_parser.add_argument( + '--fixed_n_wall_steps', + type=str2bool, + nargs='?', + const=True, + default=False, + help='Teacher samples exactly n_walls wall positions for each level.') +overcooked_ued_parser.add_argument( + '--first_wall_pos_sets_budget', + type=str2bool, + nargs='?', + const=True, + default=False, + help='The first wall positional index determines the wall budget.') +overcooked_ued_parser.add_argument( + '--noise_dim', + type=int, + default=50, + help="Dimension of episodic noise vector injected into the teacher's observation.") +overcooked_ued_parser.add_argument( + '--n_walls', + type=int, + default=25, + help="Number walls the adversary can place.") +overcooked_ued_parser.add_argument( + '--normalize_obs', + type=str2bool, + nargs='?', + const=True, + default=True, + help='Normalize teacher observations.') + + +# Logging arguments (All top-level arguments.). +parser.add_argument( + "--verbose", + type=str2bool, + nargs='?', + const=True, + default=False, + help="Print progress to stdout.") +parser.add_argument( + '--xpid', + default='latest', + help='name for the run - prefix to log files') +parser.add_argument( + '--log_dir', + default='~/logs/minimax/', + help='directory to save agent logs') +parser.add_argument( + '--log_interval', + type=int, + default=1, + help='log interval, one log per n updates') +parser.add_argument( + "--from_last_checkpoint", + type=str2bool, + nargs='?', + const=True, + default=False, + help="Begin training from latest checkpoint if available.") +parser.add_argument( + "--checkpoint_interval", + type=int, + default=0, + help="Save model every this many updates.") +parser.add_argument( + "--archive_interval", + type=int, + default=0, + help="Save an archived model every this many updates.") +parser.add_argument( + "--archive_init_checkpoint", + type=str2bool, + nargs='?', + const=True, + default=False, + help="Archive the initial checkpoint.") +parser.add_argument( + '--test_interval', + type=int, + default=10, + help='Evaluate on test envs every this many updates.') + + +# Evaluation args. +eval_parser = parser.add_subparser( + name='eval', + prefix='test') +eval_parser.add_argument( + '--n_episodes', + type=int, + default=10, + help='Number of test episodes per environment') +eval_parser.add_argument( + '--env_names', + type=str, + default=None, + help='Test environments to evaluate on.') +eval_parser.add_argument( + '--agent_idxs', + type=str, + default='*', + help="csv of agents to evaluate. '*' indicates all.") +eval_env_parser = parser.add_subparser( + name='eval_env', + prefix='test_env', +) + +# -------- Overcooked eval arguments. -------- +overcooked_eval_parser = parser.add_subparser( + name='overcooked_eval', + prefix='overcooked_test', + dependency={'env_name': 'Overcooked*'}, + dest='eval_env' +) +overcooked_eval_parser.add_argument( + '--normalize_obs', + type=str2bool, + nargs='?', + const=True, + default=True, + help='Ensures observations are between 0 and 1.') + +# -------- Maze eval arguments. -------- +maze_eval_parser = parser.add_subparser( + name='maze_eval', + prefix='maze_test', + dependency={'env_name': 'Maze*'}, + dest='eval_env' +) +maze_eval_parser.add_argument( + "--see_agent", + type=str2bool, + nargs='?', + const=True, + default=True, + help="Maze observations include the agent.") +maze_eval_parser.add_argument( + '--normalize_obs', + type=str2bool, + nargs='?', + const=True, + default=True, + help='Ensures observations are between 0 and 1.') + + +# -------- wandb arguments. -------- +wandb_parser = parser.add_subparser( + name='wandb', + prefix='wandb') +wandb_parser.add_argument( + "--base_url", + type=str, + default="https://api.wandb.ai", + help='wandb base url' +) +# wandb_parser.add_argument( +# "--api_key", +# type=str, +# default=None, +# help='wandb api key' +# ) +wandb_parser.add_argument( + "--mode", + type=str, + default="offline", + help='Online/offline or other mode' +) +wandb_parser.add_argument( + "--entity", + type=str, + default=None, + help='Team name' +) +wandb_parser.add_argument( + "--project", + type=str, + default='paired', + help='wandb project name for logging' +) +wandb_parser.add_argument( + "--group", + type=str, + default=None, + help='wandb group name for logging' +) diff --git a/src/minimax/config/__init__.py b/src/minimax/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/minimax/config/configs/maze/accel.json b/src/minimax/config/configs/maze/accel.json new file mode 100644 index 0000000..bdd7c22 --- /dev/null +++ b/src/minimax/config/configs/maze/accel.json @@ -0,0 +1,73 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_cnn"], + "env_name": ["Maze"], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [true], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [256], + "lr": [0.0001], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.995], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.8], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.1], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [false], + "plr_force_unique": [true], + "plr_mutation_fn": ["default"], + "plr_n_mutations": [20], + "plr_mutation_criterion": ["batch"], + "plr_mutation_subsample_size": [4], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.0], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [32], + "student_n_hidden_layers": [1], + "student_n_conv_filters": [16], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "maze_height": [13], + "maze_width": [13], + "maze_n_walls": [0], + "maze_replace_wall_pos": [true], + "maze_sample_n_walls": [false], + "maze_see_agent": [false], + "maze_normalize_obs": [true], + "maze_obs_agent_pos": [false], + "maze_max_episode_steps": [250], + "test_n_episodes": [10], + "test_env_names": ["Maze-SixteenRooms,Maze-Labyrinth,Maze-StandardMaze"], + "maze_test_see_agent": [false], + "maze_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/minimax/config/configs/maze/dr.json b/src/minimax/config/configs/maze/dr.json new file mode 100644 index 0000000..57c7c98 --- /dev/null +++ b/src/minimax/config/configs/maze/dr.json @@ -0,0 +1,59 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["dr"], + "n_devices": [1], + "student_model_name": ["default_student_cnn"], + "env_name": ["Maze"], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [true], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [256], + "lr": [0.0001], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.995], + "n_unroll_rollout": [10], + "render": [false], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.001], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [32], + "student_n_hidden_layers": [1], + "student_n_conv_filters": [16], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "maze_height": [13], + "maze_width": [13], + "maze_n_walls": [60], + "maze_replace_wall_pos": [true], + "maze_sample_n_walls": [false], + "maze_see_agent": [false], + "maze_normalize_obs": [true], + "maze_obs_agent_pos": [false], + "maze_max_episode_steps": [250], + "test_n_episodes": [10], + "test_env_names": ["Maze-SixteenRooms,Maze-Labyrinth,Maze-StandardMaze"], + "maze_test_see_agent": [false], + "maze_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/minimax/config/configs/maze/paccel.json b/src/minimax/config/configs/maze/paccel.json new file mode 100644 index 0000000..10694da --- /dev/null +++ b/src/minimax/config/configs/maze/paccel.json @@ -0,0 +1,73 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_cnn"], + "env_name": ["Maze"], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [true], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [256], + "lr": [0.0001], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.995], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.8], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.1], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [true], + "plr_force_unique": [true], + "plr_mutation_fn": ["default"], + "plr_n_mutations": [10], + "plr_mutation_criterion": ["batch"], + "plr_mutation_subsample_size": [4], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.0], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [32], + "student_n_hidden_layers": [1], + "student_n_conv_filters": [16], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "maze_height": [13], + "maze_width": [13], + "maze_n_walls": [0], + "maze_replace_wall_pos": [true], + "maze_sample_n_walls": [false], + "maze_see_agent": [false], + "maze_normalize_obs": [true], + "maze_obs_agent_pos": [false], + "maze_max_episode_steps": [250], + "test_n_episodes": [10], + "test_env_names": ["Maze-SixteenRooms,Maze-Labyrinth,Maze-StandardMaze"], + "maze_test_see_agent": [false], + "maze_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/minimax/config/configs/maze/paired.json b/src/minimax/config/configs/maze/paired.json new file mode 100644 index 0000000..ff0a370 --- /dev/null +++ b/src/minimax/config/configs/maze/paired.json @@ -0,0 +1,84 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["paired"], + "n_devices": [1], + "student_model_name": ["default_student_cnn"], + "env_name": ["Maze"], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [true], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [2], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [256], + "lr": [0.0001], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.995], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["relative_regret"], + "student_gae_lambda": [0.98], + "teacher_discount": [0.995], + "teacher_lr_anneal_steps": [0], + "teacher_gae_lambda": [0.98], + "student_entropy_coef": [0.001], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "teacher_entropy_coef": [0.05], + "teacher_value_loss_coef": [0.5], + "teacher_n_unroll_update": [5], + "teacher_ppo_n_epochs": [5], + "teacher_ppo_n_minibatches": [1], + "teacher_ppo_clip_eps": [0.2], + "teacher_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [32], + "student_n_hidden_layers": [1], + "student_n_conv_filters": [16], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "teacher_model_name": ["default_teacher_cnn"], + "teacher_recurrent_arch": ["lstm"], + "teacher_recurrent_hidden_dim": [256], + "teacher_hidden_dim": [32], + "teacher_n_hidden_layers": [1], + "teacher_n_conv_filters": [128], + "teacher_scalar_embed_dim": [10], + "maze_height": [13], + "maze_width": [13], + "maze_n_walls": [60], + "maze_replace_wall_pos": [false], + "maze_sample_n_walls": [false], + "maze_see_agent": [false], + "maze_normalize_obs": [true], + "maze_obs_agent_pos": [false], + "maze_max_episode_steps": [250], + "maze_ued_replace_wall_pos": [true], + "maze_ued_fixed_n_wall_steps": [true], + "maze_ued_first_wall_pos_sets_budget": [false], + "maze_ued_noise_dim": [50], + "maze_ued_n_walls": [60], + "maze_ued_set_agent_dir": [false], + "maze_ued_normalize_obs": [true], + "test_n_episodes": [10], + "test_env_names": ["Maze-SixteenRooms,Maze-Labyrinth,Maze-StandardMaze"], + "maze_test_see_agent": [false], + "maze_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/minimax/config/configs/maze/plr.json b/src/minimax/config/configs/maze/plr.json new file mode 100644 index 0000000..eaeab54 --- /dev/null +++ b/src/minimax/config/configs/maze/plr.json @@ -0,0 +1,69 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_cnn"], + "env_name": ["Maze"], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [256], + "lr": [5e-05], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.995], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.5], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.1], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [false], + "plr_force_unique": [true], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.0], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [32], + "student_n_hidden_layers": [1], + "student_n_conv_filters": [16], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "maze_height": [13], + "maze_width": [13], + "maze_n_walls": [60], + "maze_replace_wall_pos": [true], + "maze_sample_n_walls": [false], + "maze_see_agent": [false], + "maze_normalize_obs": [true], + "maze_obs_agent_pos": [false], + "maze_max_episode_steps": [250], + "test_n_episodes": [10], + "test_env_names": ["Maze-SixteenRooms,Maze-Labyrinth,Maze-StandardMaze"], + "maze_test_see_agent": [false], + "maze_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/minimax/config/configs/maze/pplr.json b/src/minimax/config/configs/maze/pplr.json new file mode 100644 index 0000000..071cab2 --- /dev/null +++ b/src/minimax/config/configs/maze/pplr.json @@ -0,0 +1,69 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_cnn"], + "env_name": ["Maze"], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [false], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [256], + "lr": [0.0001], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.5], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.3], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [true], + "plr_force_unique": [true], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.0], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["lstm"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [32], + "student_n_hidden_layers": [1], + "student_n_conv_filters": [16], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "maze_height": [13], + "maze_width": [13], + "maze_n_walls": [60], + "maze_replace_wall_pos": [true], + "maze_sample_n_walls": [false], + "maze_see_agent": [false], + "maze_normalize_obs": [true], + "maze_obs_agent_pos": [false], + "maze_max_episode_steps": [250], + "test_n_episodes": [10], + "test_env_names": ["Maze-SixteenRooms,Maze-Labyrinth,Maze-StandardMaze"], + "maze_test_see_agent": [false], + "maze_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/minimax/config/configs/maze/s5_accel.json b/src/minimax/config/configs/maze/s5_accel.json new file mode 100644 index 0000000..f2fbf76 --- /dev/null +++ b/src/minimax/config/configs/maze/s5_accel.json @@ -0,0 +1,78 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_cnn"], + "env_name": ["Maze"], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [true], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [256], + "lr": [3e-05], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.8], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.3], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [false], + "plr_force_unique": [true], + "plr_mutation_fn": ["default"], + "plr_n_mutations": [10], + "plr_mutation_criterion": ["batch"], + "plr_mutation_subsample_size": [4], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.001], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["s5"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [32], + "student_n_hidden_layers": [1], + "student_n_conv_filters": [16], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_s5_n_blocks": [2], + "student_s5_n_layers": [2], + "student_s5_layernorm_pos": ["post"], + "student_s5_activation": ["half_glu1"], + "maze_height": [13], + "maze_width": [13], + "maze_n_walls": [0], + "maze_replace_wall_pos": [true], + "maze_sample_n_walls": [false], + "maze_see_agent": [false], + "maze_normalize_obs": [true], + "maze_obs_agent_pos": [false], + "maze_max_episode_steps": [250], + "test_n_episodes": [10], + "test_env_names": ["Maze-SixteenRooms,Maze-Labyrinth,Maze-StandardMaze"], + "test_agent_idxs": ["\"*\""], + "maze_test_see_agent": [false], + "maze_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/minimax/config/configs/maze/s5_dr.json b/src/minimax/config/configs/maze/s5_dr.json new file mode 100644 index 0000000..5f688c5 --- /dev/null +++ b/src/minimax/config/configs/maze/s5_dr.json @@ -0,0 +1,63 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["dr"], + "n_devices": [1], + "student_model_name": ["default_student_cnn"], + "env_name": ["Maze"], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [true], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [256], + "lr": [3e-05], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.995], + "n_unroll_rollout": [10], + "render": [false], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.001], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["s5"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [32], + "student_n_hidden_layers": [1], + "student_n_conv_filters": [16], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_s5_n_blocks": [2], + "student_s5_n_layers": [2], + "student_s5_layernorm_pos": ["post"], + "student_s5_activation": ["half_glu1"], + "maze_height": [13], + "maze_width": [13], + "maze_n_walls": [60], + "maze_replace_wall_pos": [true], + "maze_sample_n_walls": [false], + "maze_see_agent": [false], + "maze_normalize_obs": [true], + "maze_obs_agent_pos": [false], + "maze_max_episode_steps": [250], + "test_n_episodes": [10], + "test_env_names": ["Maze-SixteenRooms,Maze-Labyrinth,Maze-StandardMaze"], + "maze_test_see_agent": [false], + "maze_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/minimax/config/configs/maze/s5_paccel.json b/src/minimax/config/configs/maze/s5_paccel.json new file mode 100644 index 0000000..d61f0c2 --- /dev/null +++ b/src/minimax/config/configs/maze/s5_paccel.json @@ -0,0 +1,77 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_cnn"], + "env_name": ["Maze"], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [true], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [256], + "lr": [1e-05], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.8], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.3], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [true], + "plr_force_unique": [true], + "plr_mutation_fn": ["default"], + "plr_n_mutations": [20], + "plr_mutation_criterion": ["batch"], + "plr_mutation_subsample_size": [4], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.0], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["s5"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [32], + "student_n_hidden_layers": [1], + "student_n_conv_filters": [16], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_s5_n_blocks": [2], + "student_s5_n_layers": [2], + "student_s5_layernorm_pos": ["post"], + "student_s5_activation": ["half_glu1"], + "maze_height": [13], + "maze_width": [13], + "maze_n_walls": [0], + "maze_replace_wall_pos": [true], + "maze_sample_n_walls": [false], + "maze_see_agent": [false], + "maze_normalize_obs": [true], + "maze_obs_agent_pos": [false], + "maze_max_episode_steps": [250], + "test_n_episodes": [10], + "test_env_names": ["Maze-SixteenRooms,Maze-Labyrinth,Maze-StandardMaze"], + "maze_test_see_agent": [false], + "maze_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/minimax/config/configs/maze/s5_paired.json b/src/minimax/config/configs/maze/s5_paired.json new file mode 100644 index 0000000..451bd40 --- /dev/null +++ b/src/minimax/config/configs/maze/s5_paired.json @@ -0,0 +1,94 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["paired"], + "n_devices": [1], + "student_model_name": ["default_student_cnn"], + "env_name": ["Maze"], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [true], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [2], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [256], + "lr": [0.0001], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.995], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["relative_regret"], + "student_gae_lambda": [0.98], + "teacher_discount": [0.995], + "teacher_lr": [0.0001], + "teacher_lr_anneal_steps": [0], + "teacher_gae_lambda": [0.98], + "student_entropy_coef": [0.001], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "teacher_entropy_coef": [0.001], + "teacher_value_loss_coef": [0.5], + "teacher_n_unroll_update": [5], + "teacher_ppo_n_epochs": [5], + "teacher_ppo_n_minibatches": [1], + "teacher_ppo_clip_eps": [0.2], + "teacher_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["s5"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [32], + "student_n_hidden_layers": [1], + "student_n_conv_filters": [16], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_s5_n_blocks": [2], + "student_s5_n_layers": [2], + "student_s5_layernorm_pos": ["post"], + "student_s5_activation": ["half_glu1"], + "teacher_model_name": ["default_teacher_cnn"], + "teacher_recurrent_arch": ["s5"], + "teacher_recurrent_hidden_dim": [256], + "teacher_hidden_dim": [32], + "teacher_n_hidden_layers": [1], + "teacher_n_conv_filters": [32], + "teacher_scalar_embed_dim": [10], + "teacher_s5_n_blocks": [2], + "teacher_s5_n_layers": [2], + "teacher_s5_layernorm_pos": ["post"], + "teacher_s5_activation": ["half_glu1"], + "maze_height": [13], + "maze_width": [13], + "maze_n_walls": [60], + "maze_replace_wall_pos": [false], + "maze_sample_n_walls": [false], + "maze_see_agent": [false], + "maze_normalize_obs": [true], + "maze_obs_agent_pos": [false], + "maze_max_episode_steps": [250], + "maze_ued_replace_wall_pos": [true], + "maze_ued_fixed_n_wall_steps": [true], + "maze_ued_first_wall_pos_sets_budget": [false], + "maze_ued_noise_dim": [50], + "maze_ued_n_walls": [60], + "maze_ued_set_agent_dir": [false], + "maze_ued_normalize_obs": [true], + "test_n_episodes": [10], + "test_env_names": ["Maze-SixteenRooms,Maze-Labyrinth,Maze-StandardMaze"], + "test_agent_idxs": ["\"*\""], + "maze_test_see_agent": [false], + "maze_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/minimax/config/configs/maze/s5_plr.json b/src/minimax/config/configs/maze/s5_plr.json new file mode 100644 index 0000000..05a9146 --- /dev/null +++ b/src/minimax/config/configs/maze/s5_plr.json @@ -0,0 +1,73 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_cnn"], + "env_name": ["Maze"], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [true], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [256], + "lr": [3e-05], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.999], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.5], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.3], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [false], + "plr_force_unique": [true], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.001], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["s5"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [32], + "student_n_hidden_layers": [1], + "student_n_conv_filters": [16], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_s5_n_blocks": [2], + "student_s5_n_layers": [2], + "student_s5_layernorm_pos": ["pre"], + "student_s5_activation": ["half_glu1"], + "maze_height": [13], + "maze_width": [13], + "maze_n_walls": [60], + "maze_replace_wall_pos": [true], + "maze_sample_n_walls": [false], + "maze_see_agent": [false], + "maze_normalize_obs": [true], + "maze_obs_agent_pos": [false], + "maze_max_episode_steps": [250], + "test_n_episodes": [10], + "test_env_names": ["Maze-SixteenRooms,Maze-Labyrinth,Maze-StandardMaze"], + "maze_test_see_agent": [false], + "maze_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/minimax/config/configs/maze/s5_pplr.json b/src/minimax/config/configs/maze/s5_pplr.json new file mode 100644 index 0000000..cccdce6 --- /dev/null +++ b/src/minimax/config/configs/maze/s5_pplr.json @@ -0,0 +1,73 @@ +{ + "args": { + "seed": [1], + "agent_rl_algo": ["ppo"], + "n_total_updates": [30000], + "train_runner": ["plr"], + "n_devices": [1], + "student_model_name": ["default_student_cnn"], + "env_name": ["Maze"], + "verbose": [false], + "log_dir": ["~/logs/minimax"], + "log_interval": [10], + "from_last_checkpoint": [true], + "checkpoint_interval": [1000], + "archive_interval": [0], + "archive_init_checkpoint": [false], + "test_interval": [100], + "n_students": [1], + "n_parallel": [32], + "n_eval": [1], + "n_rollout_steps": [256], + "lr": [3e-05], + "lr_anneal_steps": [0], + "max_grad_norm": [0.5], + "adam_eps": [1e-05], + "track_env_metrics": [true], + "discount": [0.995], + "n_unroll_rollout": [10], + "render": [false], + "ued_score": ["max_mc"], + "plr_replay_prob": [0.5], + "plr_buffer_size": [4000], + "plr_staleness_coef": [0.3], + "plr_temp": [0.3], + "plr_use_score_ranks": [true], + "plr_min_fill_ratio": [0.5], + "plr_use_robust_plr": [true], + "plr_use_parallel_eval": [true], + "plr_force_unique": [true], + "student_gae_lambda": [0.98], + "student_entropy_coef": [0.001], + "student_value_loss_coef": [0.5], + "student_n_unroll_update": [5], + "student_ppo_n_epochs": [5], + "student_ppo_n_minibatches": [1], + "student_ppo_clip_eps": [0.2], + "student_ppo_clip_value_loss": [true], + "student_recurrent_arch": ["s5"], + "student_recurrent_hidden_dim": [256], + "student_hidden_dim": [32], + "student_n_hidden_layers": [1], + "student_n_conv_filters": [16], + "student_n_scalar_embeddings": [4], + "student_scalar_embed_dim": [5], + "student_s5_n_blocks": [2], + "student_s5_n_layers": [2], + "student_s5_layernorm_pos": ["post"], + "student_s5_activation": ["half_glu1"], + "maze_height": [13], + "maze_width": [13], + "maze_n_walls": [60], + "maze_replace_wall_pos": [true], + "maze_sample_n_walls": [false], + "maze_see_agent": [false], + "maze_normalize_obs": [true], + "maze_obs_agent_pos": [false], + "maze_max_episode_steps": [250], + "test_n_episodes": [10], + "test_env_names": ["Maze-SixteenRooms,Maze-Labyrinth,Maze-StandardMaze"], + "maze_test_see_agent": [false], + "maze_test_normalize_obs": [true] + } +} \ No newline at end of file diff --git a/src/minimax/config/make_cmd.py b/src/minimax/config/make_cmd.py new file mode 100644 index 0000000..9cca193 --- /dev/null +++ b/src/minimax/config/make_cmd.py @@ -0,0 +1,287 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +import argparse +import json +import os +import pathlib + +import numpy as np + +from minimax.util.dotdict import DefaultDotDict +import minimax.config.xpid_maker as xpid_maker + + +def get_wandb_config(): + wandb_config_path = os.path.join(os.path.abspath(os.getcwd()), 'config', 'wandb.json') + if os.path.exists(wandb_config_path): + with open(wandb_config_path, 'r') as config_file: + config = json.load(config_file) + if len(config) == 2: + return { + 'wandb_base_url': config['base_url'], + 'wandb_api_key': config['api_key'], + } + + return {} + + +def generate_train_cmds( + cmd, params, num_trials=1, start_index=0, newlines=False, + xpid_generator=None, xpid_prefix='', + include_wandb_group=False, + count_set=None): + separator = ' \\\n' if newlines else ' ' + + cmds = [] + + if xpid_generator: + params['xpid'] = xpid_generator(cmd, params, xpid_prefix) + if include_wandb_group: + params['wandb_group'] = params['xpid'] + + start_seed = params['seed'] + + for t in range(num_trials): + params['seed'] = start_seed + t + start_index + + _cmd = [f'python -m {cmd}'] + + trial_idx = t + start_index + for k,v in params.items(): + if v is None: + continue + + if k == 'xpid': + v = f'{v}_{trial_idx}' + + assert len(v) < 256, f'{v} exceeds 256 characters!' + + if count_set is not None: + count_set.add(v) + + if v == "*": + v = f'"*"' + + _cmd.append(f'--{k}={v}') + + _cmd = separator.join(_cmd) + + cmds.append(_cmd) + + return cmds + + +def generate_all_params_for_grid(grid, defaults={}): + def update_params_with_choices(prev_params, param, choices): + updated_params = [] + for v in choices: + for p in prev_params: + updated = p.copy() + updated[param] = v + updated_params.append(updated) + + return updated_params + + all_params = [{}] + for param, choices in grid.items(): + all_params = update_params_with_choices(all_params, param, choices) + + full_params = [] + for p in all_params: + d = defaults.copy() + d.update(p) + full_params.append(d) + + return full_params + + +def parse_args(): + parser = argparse.ArgumentParser(description='Make commands') + + parser.add_argument( + '--dir', + type=str, + default='config/configs/', + help='Path to directory with .json configs') + + parser.add_argument( + '--config', '-c', + type=str, + default=None, + help='Name of .json config for hyperparameter search-grid') + + parser.add_argument( + '--n_trials', + type=int, + default=1, + help='Name of .json config for hyperparameter search-grid') + + parser.add_argument( + '--start_index', + default=0, + type=int, + help='Starting trial index of xpid runs') + + parser.add_argument( + '--count', + action='store_true', + help='Print number of generated commands at the end of output.') + + parser.add_argument( + "--checkpoint", + action='store_true', + help='Whether to start from checkpoint' + ) + + parser.add_argument( + "--wandb_base_url", + type=str, + default=None, + help='wandb base url' + ) + parser.add_argument( + "--wandb_api_key", + type=str, + default=None, + help='wandb api key' + ) + parser.add_argument( + '--wandb_project', + type=str, + default=None, + help='wandb project name') + + parser.add_argument( + '--include_wandb_group', + action="store_true", + help='Whether to include wandb group in cmds.') + + return parser.parse_args() + + +def xpid_from_params(cmd, p, prefix=''): + p = DefaultDotDict(p) + + env_info = xpid_maker.get_env_info(p) + runner_info = xpid_maker.get_runner_info(p) + a_algo_info = xpid_maker.get_algo_info(p, role='student') + + a_info = a_algo_info + if cmd != 'finetune': + a_model_info = xpid_maker.get_model_info(p, role='student') + a_info = f"{a_info}_{a_model_info}" + pt_info = '' + else: + pt_agent_info = 'tch' if p.get('ft_teacher') else 'st' + pt_info = f"-{p.get('checkpoint_name', 'checkpoint')}_{pt_agent_info}" + + tch_info = '' + train_runner = p.get('train_runner', 'dr') + if train_runner == 'paired': + tch_algo_info = xpid_maker.get_algo_info(p, role='teacher') + tch_model_info = xpid_maker.get_model_info(p, role='teacher') + tch_info = f"_tch_{tch_algo_info}_{tch_model_info}" + + xpid = f"{train_runner}-{env_info}-{runner_info}-{a_info}{tch_info}{pt_info}" + + return xpid + + +def setup_config_dir(): + config_dir = 'config/configs' + if not os.path.exists(os.path.join(config_dir, 'maze')): + os.makedirs(config_dir, exist_ok=True) + + import shutil + + this_path = os.path.dirname(os.path.abspath(__file__)) + src_path = os.path.join(this_path, 'configs') + + for item in os.listdir(src_path): + src_item = os.path.join(src_path, item) + dst_item = os.path.join(config_dir, item) + + if os.path.isdir(src_item): + shutil.copytree(src_item, dst_item, symlinks=True) + else: + shutil.copy(src_item, dst_item) + + +if __name__ == '__main__': + args = parse_args() + + # Default parameters + params = { + # Not needed. + } + + setup_config_dir() + + json_filename = args.config + if not json_filename.endswith('.json'): + json_filename += '.json' + + grid_path = os.path.join(os.path.expandvars(os.path.expanduser(args.dir)), json_filename) + config = json.load(open(grid_path)) + cmd = config.get('cmd', 'train') + grid = config['args'] + xpid_prefix = '' if 'xpid_prefix' not in config else config['xpid_prefix'] + + if args.checkpoint: + params['checkpoint'] = True + + if 'wandb_project' in grid: + params['wandb_project'] = args.wandb_project + + if args.wandb_base_url: + params['wandb_base_url'] = args.wandb_base_url + if args.wandb_api_key: + params['wandb_api_key'] = args.wandb_api_key + + params.update(get_wandb_config()) + + # Generate all parameter combinations within grid, using defaults for fixed params + all_params = generate_all_params_for_grid(grid, defaults=params) + + unique_xpids = None + if args.count: + unique_xpids = set() + + # Print all commands + if cmd == 'eval': + xpid_generator = None + else: + xpid_generator = xpid_from_params + count = 0 + for p in all_params: + cmds = generate_train_cmds( + cmd, p, + num_trials=args.n_trials, + start_index=args.start_index, + newlines=True, + xpid_generator=xpid_generator, + xpid_prefix=xpid_prefix, + include_wandb_group=args.include_wandb_group, + count_set=unique_xpids) + + for c in cmds: + print(c + '\n') + count += 1 + + if args.count: + print(f'Generated {len(unique_xpids)} unique commands.') + print('Sweep over') + grid_sizes = [] + for k,v in grid.items(): + if len(v) > 1: + grid_sizes.append(len(v)) + print(f'{k}: {len(v)}') + + print(f'Total num settings: {np.prod(grid_sizes)}') + diff --git a/src/minimax/config/xpid_maker.py b/src/minimax/config/xpid_maker.py new file mode 100644 index 0000000..6d140f6 --- /dev/null +++ b/src/minimax/config/xpid_maker.py @@ -0,0 +1,328 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial + + +def _get_base_role(role): + return role.removesuffix('_tch').removesuffix('_st') + + +def _get_runner_info(p): + n_students = p.get('n_students', 1) + n_eval = p.get('n_eval', 1) + + n_devices = p.get('n_devices', 1) + device_info = '' + if n_devices > 1: + device_info = f'_d{n_devices}' + + return f"r{n_students}s_{p.n_parallel}p_{n_eval}e_{p.n_rollout_steps}t_ae{p.adam_eps}{device_info}" + + +def _get_runner_info_dr(p): + ac_info = _get_runner_info(p) + + reset_info = "" + if p.ac_reset_env_on_rollout: + reset_info = f"r" + if len(reset_info) > 0: + reset_info = f"_{reset_info}" + + return f"{ac_info}{reset_info}" + + +def _get_ued_runner_info(p): + info = _get_runner_info(p) + + if p.ued_score == 'relative_regret': + ued_score = 'r' + elif p.ued_score == 'mean_relative_regret': + ued_score = 'mr' + elif p.ued_score == 'population_regret': + ued_score = 'p' + elif p.ued_score == 'neg_return': + ued_score = 'nr' + elif p.ued_score == 'l1_value_loss': + ued_score = 'l1v' + elif p.ued_score == 'positive_value_loss': + ued_score = 'pvl' + elif p.ued_score == 'max_mc': + ued_score = 'mm' + elif p.ued_score == 'value_disagreement': + ued_score = 'vd' + else: + raise ValueError(f'Unsupported ued_score {ued_score}') + + info = f"{info}_s{ued_score}" + + return info + + +def _get_plr_runner_info(p): + info = _get_ued_runner_info(p) + + plr_info = f'p{p.plr_replay_prob}b{p.plr_buffer_size}t{p.plr_temp}s{p.plr_staleness_coef}m{p.plr_min_fill_ratio}' + if p.plr_use_score_ranks: + plr_info = f'{plr_info}r' + + if p.plr_mutation_fn: + plr_info = f'{plr_info}_m{p.plr_mutation_fn[:3]}{p.plr_n_mutations}{p.plr_mutation_criterion[:3]}' + if p.plr_mutation_criterion != "batch": + plr_info = f"{plr_info}{p.plr_mutation_subsample_size}" + + plr_prefix = '' + if p.plr_use_robust_plr: + plr_prefix += 'r' + if p.plr_use_parallel_eval: + plr_prefix += 'p' + if p.plr_force_unique: + plr_prefix += 'f' + if len(plr_prefix) > 0: + plr_prefix += '_' + + return f"{plr_prefix}{plr_info}_{info}" + + +def _get_runner_info_paired(p): + return _get_ued_runner_info(p) + + +def _get_env_info_default(p): + return p.env_name.lower().replace('-', '_') + + +def _get_env_info_maze(p): + see_agent = 'na' if not p.maze_see_agent else '' + + placement_info = "" + if p.maze_replace_wall_pos: + placement_info = f'f' + if p.maze_sample_n_walls: + placement_info = f"{placement_info}s" + if len(placement_info) > 0: + placement_info = f"_{placement_info}" + + return f"{p.env_name}{p.maze_height}x{p.maze_width}w{p.maze_n_walls}{see_agent}{placement_info}" + + +def _get_env_info_overcooked(p): + placement_info = "" + if p.overcooked_replace_wall_pos: + placement_info = f'f' + if p.overcooked_sample_n_walls: + placement_info = f"{placement_info}s" + if len(placement_info) > 0: + placement_info = f"_{placement_info}" + + if p.overcooked_fix_to_single_layout: + fix_to_single_layout_info = f"_FIX{p.overcooked_fix_to_single_layout}" + else: + fix_to_single_layout_info = '' + + if p.overcooked_dense_obs: + use_dense = '_DENSE' + else: + use_dense = '_IMAGE' + + return f"{p.env_name}{p.overcooked_height}x{p.overcooked_width}w{p.overcooked_n_walls}{placement_info}{fix_to_single_layout_info}{use_dense}" + + +def _get_env_info_maze_ued(p): + see_agent = 'na' if not p.maze_see_agent else '' + + info = f"_{see_agent}_ld{p.maze_ued_noise_dim}" + + placement_info = "" + if p.maze_ued_fixed_n_wall_steps: + placement_info = f"f" + if p.maze_ued_replace_wall_pos: + placement_info = f"{placement_info}r" + if p.maze_ued_set_agent_dir: + placement_info = f"{placement_info}d" + if p.maze_ued_first_wall_pos_sets_budget: + placement_info = f"{placement_info}b" + if len(placement_info) > 0: + placement_info = f"_{placement_info}" + info = f"{info}{placement_info}" + + return f"{p.env_name}{p.maze_height}x{p.maze_width}w{p.maze_n_walls}{info}" + + +def _get_env_info_overcooked_ued(p): + info = f"_ld{p.overcooked_ued_noise_dim}" + + placement_info = "" + if p.overcooked_ued_fixed_n_wall_steps: + placement_info = f"f" + if p.overcooked_ued_replace_wall_pos: + placement_info = f"{placement_info}r" + if p.overcooked_ued_set_agent_dir: + placement_info = f"{placement_info}d" + if p.overcooked_ued_first_wall_pos_sets_budget: + placement_info = f"{placement_info}b" + if len(placement_info) > 0: + placement_info = f"_{placement_info}" + info = f"{info}{placement_info}" + + return f"{p.env_name}{p.overcooked_height}x{p.overcooked_width}w{p.overcooked_n_walls}{info}" + + +def _get_model_info_maze_default(p, role): + model_info = '' + if f'{role}_recurrent_arch' in p and p[f'{role}_recurrent_arch'] is not None: + model_info = f"{p[f'{role}_recurrent_arch']}_h{p[f'{role}_recurrent_hidden_dim']}" + + if p[f'{role}_recurrent_arch'] == 's5': + model_info = f"{model_info}nb{p.get(f'{role}_s5_n_blocks', 1)}nl{p.get(f'{role}_s5_n_layers',4)}" + + activation = p.get(f'{role}_s5_activation') + if activation == 'half_glu1': + activation = 'hg1' + elif activation == 'half_glu2': + activation = 'hg2' + elif activation == 'full_glu': + activation = 'fg' + elif activation == 'gelu': + activation = 'g' + else: + activation = 'hg1' + model_info = f'a{activation}_{model_info}' + + ln_key = f'{role}_s5_layernorm_pos' + ln_info = None + if ln_key in p: + ln = p[ln_key] + if ln == 'pre': + ln_info = 'pr' + elif ln == 'post': + ln_info = 'po' + + if ln_info is not None: + model_info = f"l{ln_info}_{model_info}" + + if f'{role}_is_soft_moe' in p: + num_experts = p.get(f'{role}_soft_moe_num_experts') + num_slots = p.get(f'{role}_soft_moe_num_slots') + model_info = f'{model_info}__SoftMoE_{num_experts}E_{num_slots}S__' + + model_info = f'_{model_info}' if len(model_info) > 0 else '' + + value_info = '' + value_ensemble_key = f'{role}_value_ensemble_size' + value_ensemble_size = p.get(value_ensemble_key) + if value_ensemble_size and value_ensemble_size > 1: + value_info = f've{value_ensemble_size}' + + base_activation = p.get(f'{role}_base_activation', 'relu')[:2] + + model_info = f"h{p[f'{role}_hidden_dim']}cf{p[f'{role}_n_conv_filters']}fc{p[f'{role}_n_hidden_layers']}se{p[f'{role}_scalar_embed_dim']}ba_{base_activation}{model_info}{value_info}" + + return model_info + + +def _get_algo_info_ppo(p, role): + if role == 'student': + lr = str(p.lr) + if 'lr_final' in p: + lr_final = '' if p.lr_final is None or p.lr_final == p.lr else str( + p.lr_final) + if len(lr_final) > 0: + lr = f"{lr}_{lr_final}" + + if "n_shaped_reward_steps" in p: + lr = f"{lr}_SRS{p.n_shaped_reward_steps}" + elif "n_shaped_reward_updates" in p: + lr = f"{lr}_SRU{p.n_shaped_reward_updates}" + + return f"ppo_lr{lr}g{p.discount}cv{p.student_value_loss_coef}ce{p.student_entropy_coef}e{p.student_ppo_n_epochs}mb{p.student_ppo_n_minibatches}l{p.student_gae_lambda}_pc{p.student_ppo_clip_eps}" + else: + if 'teacher_lr' in p: + teacher_lr = str( + p.lr) if p.teacher_lr is None else str(p.teacher_lr) + else: + teacher_lr = str(p.lr) + + if 'teacher_lr_final' in p: + teacher_lr_final = str( + p.lr_final) if p.teacher_lr_final is None else str(p.teacher_lr_final) + else: + teacher_lr_final = str(p.lr_final) if 'lr_final' in p else '' + + if teacher_lr_final == teacher_lr: + teacher_lr_final = '' + + if len(teacher_lr_final) > 0: + teacher_lr = f"{teacher_lr}_{teacher_lr_final}" + + return f"ppo_lr{teacher_lr}g{p.teacher_discount}cv{p.teacher_value_loss_coef}ce{p.teacher_entropy_coef}e{p.teacher_ppo_n_epochs}mb{p.teacher_ppo_n_minibatches}l{p.teacher_gae_lambda}pc{p.teacher_ppo_clip_eps}" + + +# ============================================================ + +RUNNER_INFO_HANDLERS = { + 'dr': _get_runner_info_dr, + 'plr': _get_plr_runner_info, + 'paired': _get_runner_info_paired, +} + +ENV_INFO_HANDLERS = { + 'maze': _get_env_info_maze, + 'maze_ued': _get_env_info_maze_ued, + 'overcooked': _get_env_info_overcooked, + 'overcooked_ued': _get_env_info_overcooked_ued +} + +MODEL_INFO_HANDLERS = { + 'maze': { + 'default_student_cnn': partial(_get_model_info_maze_default, role='student'), + 'default_teacher_cnn': partial(_get_model_info_maze_default, role='teacher'), + }, + 'overcooked': { + 'default_student_cnn': partial(_get_model_info_maze_default, role='student'), + 'default_student_actor_cnn': partial(_get_model_info_maze_default, role='student'), + 'default_student_critic_cnn': partial(_get_model_info_maze_default, role='student'), + 'default_student_actor_mlp': partial(_get_model_info_maze_default, role='student'), + 'default_student_critic_mlp': partial(_get_model_info_maze_default, role='student'), + 'default_student_actor_moe': partial(_get_model_info_maze_default, role='student'), + 'default_student_critic_moe': partial(_get_model_info_maze_default, role='student'), + 'default_teacher_cnn': partial(_get_model_info_maze_default, role='teacher'), + } +} + +ALGO_INFO_HANDLERS = { + 'ppo': _get_algo_info_ppo +} + + +def get_runner_info(p): + return RUNNER_INFO_HANDLERS[p.get('train_runner', 'dr')](p) + + +def get_env_info(p): + p.env_name = p.env_name.lower() + env_name = p.env_name + if p.train_runner in ['paired',]: + env_name = f'{env_name}_ued' + + return ENV_INFO_HANDLERS.get( + env_name, _get_env_info_default + )(p) + + +def get_model_info(p, role='student'): + model_name = p.get(f'{role}_model_name') + if model_name is None: + model_name = p['student_model_name'] + env_name = p.env_name.lower().split('-')[0] + + return MODEL_INFO_HANDLERS[env_name][model_name](p) + + +def get_algo_info(p, role='student'): + return ALGO_INFO_HANDLERS[p.agent_rl_algo](p, role) diff --git a/src/minimax/count_params.py b/src/minimax/count_params.py new file mode 100644 index 0000000..cde9707 --- /dev/null +++ b/src/minimax/count_params.py @@ -0,0 +1,133 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +import os +import json +import re +import fnmatch +import sys +from collections import defaultdict + +import numpy as np +import pandas as pd +import scipy.stats as spstats +import jax +import jax.numpy as jnp +from tqdm import tqdm + +from minimax.util.parsnip import Parsnip +from minimax.util.checkpoint import load_pkl_object, load_config +from minimax.util.loggers import HumanOutputFormat +from minimax.util.rl import AgentPop +import minimax.models as models +import minimax.agents as agents + + +parser = Parsnip() + +# ==== Define top-level arguments +parser.add_argument( + '--seed', + type=int, + default=1, + help='Random seed.') +parser.add_argument( + '--log_dir', + type=str, + default='~/logs/minimax', + help='Log directory containing experiment dirs.') +parser.add_argument( + '--xpid', + type=str, + default='latest', + help='Experiment ID dir name for model.') +parser.add_argument( + '--xpid_prefix', + type=str, + default=None, + help='Experiment ID dir name for model.') +parser.add_argument( + '--checkpoint_name', + type=str, + default='checkpoint', + help='Name of checkpoint .pkl.') +parser.add_argument( + '--agent_idxs', + type=str, + default='*', + help="Indices of agents to evaluate. '*' indicates all.") + + +if __name__ == '__main__': + """ + Usage: + python -m eval \ + --xpid= \ + --env_names="Maze-SixteenRooms" \ + --n_episodes=100 \ + --agent_idxs=0 + """ + args = parser.parse_args() + + log_dir_path = os.path.expandvars(os.path.expanduser(args.log_dir)) + + xpids = [] + if args.xpid_prefix is not None: + # Get all matching xpid directories + all_xpids = fnmatch.filter(os.listdir( + log_dir_path), f"{args.xpid_prefix}*") + filter_re = re.compile('.*_[0-9]*$') + xpids = [x for x in all_xpids if filter_re.match(x)] + else: + xpids = [args.xpid] + + pbar = tqdm(total=len(xpids)) + + all_eval_stats = defaultdict(list) + for xpid in xpids: + xpid_dir_path = os.path.join(log_dir_path, xpid) + checkpoint_path = os.path.join( + xpid_dir_path, f'{args.checkpoint_name}.pkl') + meta_path = os.path.join(xpid_dir_path, f'meta.json') + + # Load checkpoint info + if not os.path.exists(meta_path): + print(f'Configuration at {meta_path} does not exist. Skipping...') + continue + + if not os.path.exists(checkpoint_path): + print( + f'Checkpoint path {checkpoint_path} does not exist. Skipping...') + continue + + xp_args = load_config(meta_path) + + agent_idxs = args.agent_idxs + if agent_idxs == '*': + agent_idxs = np.arange(xp_args.train_runner_args.n_students) + else: + agent_idxs = \ + np.array([int(x) for x in agent_idxs.split(',')]) + assert np.max(agent_idxs) <= xp_args.train_runner_args.n_students, \ + 'Agent index is out of bounds.' + + runner_state = load_pkl_object(checkpoint_path) + if "params" in runner_state[1].keys(): + params = runner_state[1]['params'] + elif "actor_params" in runner_state[1].keys(): + params = runner_state[1]['actor_params'] + else: + raise ValueError("No params found in checkpoint.") + + params = jax.tree_util.tree_map( + lambda x: jnp.take(x, indices=agent_idxs, axis=0), + params + ) + + param_count = sum(x.size for x in jax.tree_util.tree_leaves(params)) + print(f"Model has {param_count} parameters.") diff --git a/src/minimax/envs/__init__.py b/src/minimax/envs/__init__.py new file mode 100644 index 0000000..6bf1d79 --- /dev/null +++ b/src/minimax/envs/__init__.py @@ -0,0 +1,15 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from .maze import Maze, UEDMaze +from .batch_env import BatchEnv +from .batch_env_ued import BatchUEDEnv + +from .overcooked_proc import Overcooked, UEDOvercooked + +from .registration import make, get_comparator, get_mutator \ No newline at end of file diff --git a/src/minimax/envs/batch_env.py b/src/minimax/envs/batch_env.py new file mode 100644 index 0000000..5d342c9 --- /dev/null +++ b/src/minimax/envs/batch_env.py @@ -0,0 +1,74 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial + +import jax +import jax.numpy as jnp + +import minimax.envs as envs + + +class BatchEnv: + def __init__( + self, + env_name, + n_parallel, + n_eval, + env_kwargs, + wrappers=['monitor_return']): + self.env_name = env_name + self.env, self.env_params = envs.make( + env_name, + env_kwargs=env_kwargs, + wrappers=wrappers, + ) + self.n_parallel = n_parallel + self.n_eval = n_eval + + self.sub_batch_size = n_parallel*n_eval + + self.step = jax.vmap(self._step, in_axes=0) + self.get_env_metrics = jax.vmap(self._get_env_metrics, in_axes=0) + self.set_state = jax.vmap(self._set_state, in_axes=0) + + @partial(jax.jit, static_argnums=(0, 2, 3)) + def reset(self, rng, n_parallel=None, n_eval=None): + return jax.vmap(self._reset, in_axes=(0, None, None))(rng, n_parallel, n_eval) + + def _reset(self, rng, n_parallel=None, n_eval=None): + # Create n_parallel envs, repeated n_eval times + if n_parallel is None: + n_parallel = self.n_parallel + + if n_eval is None: + n_eval = self.n_eval + + brngs = jnp.repeat(jax.random.split(rng, n_parallel), n_eval, axis=0) + + obs, state, extra = jax.vmap( + self.env.reset, in_axes=(0,))(brngs) + + return obs, state, extra + + @partial(jax.jit, static_argnums=0) + def _step(self, rng, state, action, extra): + brngs = jax.random.split(rng, self.sub_batch_size) + return jax.vmap(self.env.step, in_axes=(0, 0, 0, 0, 0))( + brngs, state, action, None, extra) + + @partial(jax.jit, static_argnums=(0,)) + def _get_env_metrics(self, state): + return jax.vmap(self.env.get_env_metrics, in_axes=(0,))(state) + + @partial(jax.jit, static_argnums=(0,)) + def _set_state(self, state): + # Need to repeat the state + state = jax.tree_map(lambda x: x.repeat(self.n_eval, axis=0), state) + + return jax.vmap(self.env.set_state)(state) diff --git a/src/minimax/envs/batch_env_ued.py b/src/minimax/envs/batch_env_ued.py new file mode 100644 index 0000000..497b0c7 --- /dev/null +++ b/src/minimax/envs/batch_env_ued.py @@ -0,0 +1,134 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial + +import jax +import jax.numpy as jnp + +import minimax.envs as envs + + +class BatchUEDEnv: + """ + Wraps and batches a UEDEnvironment in + its private methods as follows: + + For student MDP: + Manages a batch of n_parallel x n_eval envs + + For teacher MDP: + Manages a batch of n_parallel envs. + + The public interface vmaps the private methods over + an additional agent population dimension. + """ + def __init__( + self, + env_name, + n_parallel, + n_eval, + env_kwargs, + ued_env_kwargs, + wrappers=['monitor_return'], + ued_wrappers=None): + self.wrappers = wrappers + self.env, self.env_params, self.ued_params = \ + envs.make( + env_name, + env_kwargs=env_kwargs, + ued_env_kwargs=ued_env_kwargs, + wrappers=wrappers, + ued_wrappers=ued_wrappers + ) + + self.n_parallel = n_parallel + self.n_eval = n_eval + self.sub_batch_size = n_parallel*n_eval + + self.reset_student = jax.vmap(self._reset_student, in_axes=(0,0,None)) + self.step_teacher = jax.vmap(self._step_teacher, in_axes=0) + self.step_student = jax.vmap(self._step_student, in_axes=0) + + self.set_env_instance = jax.vmap(self._set_env_instance, in_axes=0) + self.get_env_metrics = jax.vmap(self._get_env_metrics, in_axes=0) + + partial(jax.jit, static_argnums=(2,)) + def reset(self, rng, sub_batch_size=None): + if sub_batch_size is None: + sub_batch_size = self.sub_batch_size + + return jax.vmap(self._reset, in_axes=(0,None))(rng, sub_batch_size) + + def _reset(self, rng, sub_batch_size): + brngs = jax.random.split(rng, sub_batch_size) + return jax.vmap(self.env.reset)(brngs) + + partial(jax.jit, static_argnums=(2,)) + def reset_teacher(self, rng, n_parallel=None): + if n_parallel is None: + n_parallel = self.n_parallel + + return jax.vmap(self._reset_teacher, in_axes=(0,None))(rng, n_parallel) + + def _reset_teacher(self, rng, n_parallel): + """ + Reset n_parallel envs + """ + brngs = jax.random.split(rng, n_parallel) + return jax.vmap(self.env.reset_teacher)(brngs) + + def _step_teacher(self, rng, ued_state, action, extra=None): + """ + Step n_parallel envs + """ + brngs = jax.random.split(rng, self.n_parallel) + step_args = (brngs, ued_state, action) + if extra is not None: + step_args += (extra,) + + return jax.vmap(self.env.step_teacher)(*step_args) + + def _reset_student(self, rng, ued_state, n_students): + """ + Reset the student MDP based on the state of the teacher MDP. + """ + brngs = jax.random.split(rng, self.n_parallel) + obs, state, extra = \ + jax.vmap(self.env.reset_student)(brngs, ued_state) + + obs = jax.tree_util.tree_map( + lambda x: jnp.repeat( + jnp.expand_dims(jnp.repeat(x, self.n_eval, 0), 0), n_students, 0), obs) + + state = jax.tree_util.tree_map( + lambda x: jnp.repeat( + jnp.expand_dims(jnp.repeat(x, self.n_eval, 0), 0), n_students, 0), state) + + extra = jax.tree_util.tree_map( + lambda x: jnp.repeat( + jnp.expand_dims(jnp.repeat(x, self.n_eval, 0), 0), n_students, 0), extra) + + return obs, state, extra + + def _step_student(self, rng, state, action, reset_state, extra=None): + """ + Step the student MDP. + """ + brngs = jax.random.split(rng, self.sub_batch_size) + return jax.vmap(self.env.step)(brngs, state, action, reset_state, extra) + + def _set_env_instance(self, instance): + """ + Reset the student MDP to a particular configuration, + captured by state argument. Used for PLR. + """ + return jax.vmap(self.env.set_env_instance)(instance) + + def _get_env_metrics(self, state): + return jax.vmap(self.env.get_env_metrics)(state) diff --git a/src/minimax/envs/environment.py b/src/minimax/envs/environment.py new file mode 100644 index 0000000..2719d60 --- /dev/null +++ b/src/minimax/envs/environment.py @@ -0,0 +1,201 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This file extends the Environment class from +https://github.com/RobertTLange/gymnax/ + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +import jax +import chex +from typing import Tuple, Union, Optional +from functools import partial +from flax import struct + + +@struct.dataclass +class EnvState: + time: int + + +@struct.dataclass +class EnvParams: + max_episode_steps: int + + +class Environment(object): + """Jittable abstract base class for all basic environments.""" + + def __init__(self): + self.eval_solved_rate = self.get_eval_solved_rate_fn() + + @property + def default_params(self) -> EnvParams: + return EnvParams() + + @staticmethod + def align_kwargs(kwargs, other_kwargs): + """ + Return kwargs that are consistent with other_kwargs, + e.g. in the case of the student env, other_kwargs may be + those for the paired teacher env, and in the case of the + teacher env, the paired student env. + """ + raise NotImplementedError + + @partial(jax.jit, static_argnums=(0, 4)) + def step( + self, + key: chex.PRNGKey, + state: EnvState, + action: Union[int, float], + reset_on_done: bool = True, + reset_state: Optional[chex.ArrayTree] = None, + ) -> Tuple[chex.ArrayTree, EnvState, float, bool]: + """Performs step transitions in the environment.""" + # Use default env parameters if no others specified + if hasattr(self, 'params'): + params = self.params + else: + params = self.default_params + + key, key_reset = jax.random.split(key) + obs_st, state_st, reward, done, info = self.step_env( + key, state, action + ) + + if reset_on_done: + if reset_state is not None: + state_re = reset_state + obs_re = self.get_obs(reset_state) + else: + if hasattr(params, 'singleton_seed') \ + and params.singleton_seed >= 0: + key_reset = jax.random.PRNGKey(params.singleton_seed) + + obs_re, state_re = self.reset_env(key_reset) + + # Auto-reset environment based on termination + if type(done) == dict: + # Multi Agent setting + done = done["__all__"] + + state = jax.tree_map( + lambda x, y: jax.lax.select(done, x, y), state_re, state_st + ) + obs = jax.tree_map( + lambda x, y: jax.lax.select(done, x, y), obs_re, obs_st + ) + else: + obs, state = obs_st, state_st + + return obs, state, reward, done, info + + @partial(jax.jit, static_argnums=(0,)) + def reset( + self, + key: chex.PRNGKey, + ) -> Tuple[chex.ArrayTree, EnvState]: + """Performs resetting of environment.""" + # Use default env parameters if no others specified + if hasattr(self, 'params'): + params = self.params + else: + params = self.default_params + + if hasattr(params, 'singleton_seed') \ + and params.singleton_seed >= 0: + key = jax.random.PRNGKey(params.singleton_seed) + obs, state = self.reset_env(key) + return obs, state + + def step_env( + self, + key: chex.PRNGKey, + state: EnvState, + action: Union[int, float], + ) -> Tuple[chex.ArrayTree, EnvState, float, bool, dict]: + """Environment-specific step transition.""" + raise NotImplementedError + + def reset_env( + self, key: chex.PRNGKey + ) -> Tuple[chex.ArrayTree, EnvState]: + """Environment-specific reset.""" + raise NotImplementedError + + def set_state( + self, + state: EnvState + ) -> Tuple[chex.ArrayTree, EnvState]: + """ + Implemented for basic envs. + """ + return self.get_obs(state), state + + def set_env_instance( + self, + encoding: chex.ArrayTree + ) -> Tuple[chex.ArrayTree, EnvState]: + """ + Implemented for basic envs. + """ + raise NotImplementedError + + def get_env_instance( + self, + key: chex.PRNGKey, + state: EnvState + ) -> chex.ArrayTree: + """ + Implemented for UED envs. + """ + raise NotImplementedError + + def get_obs(self, state: EnvState) -> chex.ArrayTree: + """Applies observation function to state.""" + raise NotImplementedError + + def is_terminal(self, state: EnvState) -> bool: + """Check whether state is terminal.""" + raise NotImplementedError + + def get_eval_solved_rate_fn(self): + return None + + @property + def name(self) -> str: + """Environment name.""" + return type(self).__name__ + + @property + def num_actions(self) -> int: + """Number of actions possible in environment.""" + raise NotImplementedError + + def action_space(self): + """Action space of the environment.""" + raise NotImplementedError + + def observation_space(self): + """Observation space of the environment.""" + raise NotImplementedError + + def state_space(self): + """State space of the environment.""" + raise NotImplementedError + + def max_episode_steps(self): + """Maximum number of time steps in environment.""" + raise NotImplementedError + + def get_env_metrics(self, state: EnvState): + """Environment-specific metrics, e.g. number of walls.""" + raise NotImplementedError diff --git a/src/minimax/envs/environment_ued.py b/src/minimax/envs/environment_ued.py new file mode 100644 index 0000000..bab01ff --- /dev/null +++ b/src/minimax/envs/environment_ued.py @@ -0,0 +1,142 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +import jax +import chex +from typing import Tuple, Union, Optional +from functools import partial +from flax import struct + +import jax.numpy as jnp + +from .environment import EnvParams, EnvState, Environment + + +@struct.dataclass +class UEDEnvState: + encoding: chex.Array + time: int + terminal: bool + + +class UEDEnvironment: + """ + Wraps two Environment instances, one being the basic environment, + and the other, its UED counterpart. + + The interface extends the student environment interace. + """ + + def __init__(self, env, ued_env): + self.env = env + self.ued_env = ued_env + + # Default reset and step centers on student + self.reset = self.reset_random + self.step = self.env.step + + def reset_random( + self, + rng: chex.PRNGKey, + ) -> Tuple[chex.ArrayTree, EnvState]: + return self.env.reset(rng) + + def get_monitored_metrics(self): + return self.env.get_monitored_metrics() + self.ued_env.get_monitored_metrics() + + def reset_teacher( + self, + rng: chex.PRNGKey, + ) -> Tuple[chex.ArrayTree, EnvState]: + return self.ued_env.reset(rng) + + def step_teacher( + self, + rng: chex.PRNGKey, + ued_state: EnvState, + action: Union[int, float], + ) -> Tuple[chex.ArrayTree, EnvState, float, bool, dict]: + return self.ued_env.step( + rng, ued_state, action, reset_on_done=False) + + def reset_student( + self, + rng: chex.PRNGKey, + ued_state: EnvState, + ) -> Tuple[chex.ArrayTree, EnvState]: + """ + Reset the student based on + """ + # ued_state_ = UEDEnvState( + # encoding=jnp.array([17, 6, 3, 23, 4, 21, 2, 3, 16, 12, 9], dtype=jnp.uint32), time=jnp.array(11, dtype=jnp.uint32), terminal=jnp.array(True)) + encoding = self.ued_env.get_env_instance(rng, ued_state) + env = self.env.set_env_instance(encoding) + return env + + def step_student( + self, + rng: chex.PRNGKey, + state: EnvState, + action: Union[int, float], + reset_state: Optional[chex.ArrayTree] = None + ) -> Tuple[chex.ArrayTree, EnvState, float, bool, dict]: + return self.env.step( + rng, + state, + action, + reset_state=reset_state) + + def set_env_instance(self, encoding: chex.ArrayTree): + return self.env.set_env_instance(encoding) + + # Spaces interface + def action_space(self): + """Action space of the environment.""" + return self.env.action_space() + + def observation_space(self): + """Observation space of the environment.""" + return self.env.observation_space() + + def state_space(self): + """Observation space of the environment.""" + return self.env.state_space() + + def max_episode_steps(self): + """Action space of the environment.""" + return self.env.max_episode_steps() + + def ued_action_space(self): + """Action space of the environment.""" + return self.ued_env.action_space() + + def ued_observation_space(self): + """Observation space of the environment.""" + return self.ued_env.observation_space() + + def ued_state_space(self): + """Observation space of the environment.""" + return self.ued_env.state_space() + + def ued_max_episode_steps(self): + """Action space of the environment.""" + return self.ued_env.max_episode_steps() + + def get_env_metrics(self, state: EnvState): + """Environment-specific metrics, e.g. number of walls.""" + return self.env.get_env_metrics(state) + + @property + def agents(self) -> str: + """Environment name.""" + return self.env.agents + + @property + def name(self) -> str: + """Environment name.""" + return self.env.name diff --git a/src/minimax/envs/interactive/__init__.py b/src/minimax/envs/interactive/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/minimax/envs/interactive/manual_ctrl_maze.py b/src/minimax/envs/interactive/manual_ctrl_maze.py new file mode 100644 index 0000000..d20554b --- /dev/null +++ b/src/minimax/envs/interactive/manual_ctrl_maze.py @@ -0,0 +1,219 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +import time +import argparse +from functools import partial + +import jax +import jax.numpy as jnp +import numpy as np + +import minimax.envs as envs +from minimax.envs.maze.maze import Actions +from minimax.envs.viz.grid_viz import GridVisualizer + + +def redraw(state, obs, extras): + if extras['is_ued_maze']: + env_instance = extras['env'].get_env_instance(None, state) + maze_map = extras['render_env'].set_env_instance(env_instance)[1].maze_map + extras['viz'].render(extras['params'], state, highlight=False, maze_map=maze_map) + else: + extras['viz'].render(extras['params'], state) + if extras['obs_viz'] is not None: + extras['obs_viz'].render_grid(np.asarray(obs['image']), k_rot90=3, agent_dir_idx=3) + +def reset(key, env, extras): + key, subkey = jax.random.split(extras['rng']) + obs, state = extras['jit_reset'](subkey) + + extras['rng'] = key + extras['obs'] = obs + extras['state'] = state + extras['n'] += 1 + + if not extras['is_ued_maze']: + metrics = env.get_env_metrics(state) + print(metrics) + extras['n_walls_total'] += metrics['n_walls'] + + if not extras['is_ued_maze']: + print(f"mean walls: {extras['n_walls_total']/extras['n']}", flush=True) + + redraw(state, obs, extras) + +def step(env, action, extras): + key, subkey = jax.random.split(extras['rng']) + obs, state, reward, done, info = env.step_env(subkey, extras['state'], action) + extras['obs'] = obs + extras['state'] = state + # print(f"reward={reward}, agent_dir={obs['agent_dir']}") + print(f"reward={reward}") + + if done or action == Actions.done: + key, subkey = jax.random.split(subkey) + reset(subkey, env, extras) + else: + redraw(state, obs, extras) + + extras['rng'] = key + + +def key_handler(env, extras, event): + print('pressed', event.key) + + if event.key == 'escape': + window.close() + return + + if event.key == 'backspace': + extras['jit_reset']((env, extras)) + return + + if event.key == 'left': + step(env, Actions.left, extras) + return + if event.key == 'right': + step(env, Actions.right, extras) + return + if event.key == 'up': + step(env, Actions.forward, extras) + return + + # Spacebar + if event.key == ' ': + step(env, Actions.toggle, extras) + return + if event.key == 'a': + step(env, Actions.pickup, extras) + return + if event.key == 'd': + step(env, Actions.drop, extras) + return + + if event.key == 'enter': + step(env, Actions.done, extras) + return + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + "--env", + type=str, + help="Environment name", + default="Maze" + ) + parser.add_argument( + "--seed", + type=int, + help="random seed to generate the environment with", + default=90 + ) + parser.add_argument( + '--render_agent_view', + default=False, + help="draw the agent sees (partially observable view)", + action='store_true' + ) + parser.add_argument( + '--height', + default=13, + type=int, + help="height", + ) + parser.add_argument( + '--width', + default=13, + type=int, + help="width", + ) + parser.add_argument( + '--n_walls', + default=10, + type=int, + help="Number of walls", + ) + parser.add_argument( + '--agent_view_size', + default=5, + type=int, + help="Number of walls", + ) + parser.add_argument( + "--screenshot_path", + type=str, + default=None, + help="maze.png", + ) + args = parser.parse_args() + + kwargs = dict( + height=args.height, + width=args.width, + n_walls=args.n_walls, + agent_view_size=args.agent_view_size, + see_through_walls=True, + see_agent=True, + normalize_obs=False, + sample_n_walls=False, + replace_wall_pos=False, + max_episode_steps=250, + ) + kwargs = {} + env, params = envs.make(args.env, kwargs) + params = env.params + + is_ued_maze = False + render_env = None + if args.env.startswith('UEDMaze'): + is_ued_maze = True + render_env, _ = envs.make('Maze', kwargs) + + viz = GridVisualizer() + obs_viz = None + if args.render_agent_view: + obs_viz = GridVisualizer() + + with jax.disable_jit(False): + jit_reset = jax.jit(env.reset_env, static_argnums=(1,)) + key = jax.random.PRNGKey(args.seed) + key, subkey = jax.random.split(key) + o0, s0 = jit_reset(subkey) + if is_ued_maze: + maze_map = render_env.set_env_instance(env.get_env_instance(None, s0))[1].maze_map + viz.render(params, s0, highlight=False, maze_map=maze_map) + else: + viz.render(params, s0) + if obs_viz is not None: + obs_viz.render_grid(np.asarray(o0['image']), k_rot90=3, agent_dir_idx=3) + + key, subkey = jax.random.split(key) + extras = { + 'rng': subkey, + 'state': s0, + 'obs': o0, + 'params':params, + 'viz': viz, + 'obs_viz': obs_viz, + 'jit_reset': jit_reset, + 'n_walls_total': 0, + 'n': 0, + 'env': env, + 'render_env': render_env, + 'is_ued_maze': is_ued_maze + } + + if args.screenshot_path is not None: + print('saving') + viz.screenshot(args.screenshot_path) + + viz.window.reg_key_handler(partial(key_handler, env, extras)) + viz.show(block=True) + diff --git a/src/minimax/envs/maze/__init__.py b/src/minimax/envs/maze/__init__.py new file mode 100644 index 0000000..ddc8a93 --- /dev/null +++ b/src/minimax/envs/maze/__init__.py @@ -0,0 +1,14 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from .maze import Maze +from .maze_ued import UEDMaze +from .maze_ood import * + +from .maze_comparators import * +from .maze_mutators import * \ No newline at end of file diff --git a/src/minimax/envs/maze/common.py b/src/minimax/envs/maze/common.py new file mode 100644 index 0000000..3965887 --- /dev/null +++ b/src/minimax/envs/maze/common.py @@ -0,0 +1,109 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +import copy + +import numpy as np +import jax.numpy as jnp +from flax import struct +import chex + + +OBJECT_TO_INDEX = { + "unseen": 0, + "empty": 1, + "wall": 2, + "floor": 3, + "door": 4, + "key": 5, + "ball": 6, + "box": 7, + "goal": 8, + "lava": 9, + "agent": 10, +} + + +COLORS = { + 'red' : np.array([255, 0, 0]), + 'green' : np.array([0, 255, 0]), + 'blue' : np.array([0, 0, 255]), + 'purple': np.array([112, 39, 195]), + 'yellow': np.array([255, 255, 0]), + 'grey' : np.array([100, 100, 100]) +} + + +COLOR_TO_INDEX = { + 'red' : 0, + 'green' : 1, + 'blue' : 2, + 'purple': 3, + 'yellow': 4, + 'grey' : 5, +} + + +# Map of agent direction indices to vectors +DIR_TO_VEC = jnp.array([ + # Pointing right (positive X) + (1, 0), # right + (0, 1), # down + (-1, 0), # left + (0, -1), # up +], dtype=jnp.int8) + + +@struct.dataclass +class EnvInstance: + agent_pos: chex.Array + agent_dir_idx: int + goal_pos: chex.Array + wall_map: chex.Array + + +def make_maze_map( + params, + wall_map, + goal_pos, + agent_pos, + agent_dir_idx, + pad_obs=False): + # Expand maze map to H x W x C + empty = jnp.array([OBJECT_TO_INDEX['empty'], 0, 0], dtype=jnp.uint8) + wall = jnp.array([OBJECT_TO_INDEX['wall'], COLOR_TO_INDEX['grey'], 0], dtype=jnp.uint8) + maze_map = jnp.array(jnp.expand_dims(wall_map, -1), dtype=jnp.uint8) + maze_map = jnp.where(maze_map > 0, wall, empty) + + agent = jnp.array([OBJECT_TO_INDEX['agent'], COLOR_TO_INDEX['red'], agent_dir_idx], dtype=jnp.uint8) + agent_x,agent_y = agent_pos + maze_map = maze_map.at[agent_y,agent_x,:].set(agent) + + goal = jnp.array([OBJECT_TO_INDEX['goal'], COLOR_TO_INDEX['green'], 0], dtype=jnp.uint8) + goal_x,goal_y = goal_pos + maze_map = maze_map.at[goal_y,goal_x,:].set(goal) + + # Add observation padding + if pad_obs: + padding = params.agent_view_size-1 + else: + padding = 1 + + maze_map_padded = jnp.tile(wall.reshape((1,1,*empty.shape)), (maze_map.shape[0]+2*padding, maze_map.shape[1]+2*padding, 1)) + maze_map_padded = maze_map_padded.at[padding:-padding,padding:-padding,:].set(maze_map) + + # Add surrounding walls + wall_start = padding-1 # start index for walls + wall_end_y = maze_map_padded.shape[0] - wall_start - 1 + wall_end_x = maze_map_padded.shape[1] - wall_start - 1 + maze_map_padded = maze_map_padded.at[wall_start,wall_start:wall_end_x+1,:].set(wall) # top + maze_map_padded = maze_map_padded.at[wall_end_y,wall_start:wall_end_x+1,:].set(wall) # bottom + maze_map_padded = maze_map_padded.at[wall_start:wall_end_y+1,wall_start,:].set(wall) # left + maze_map_padded = maze_map_padded.at[wall_start:wall_end_y+1,wall_end_x,:].set(wall) # right + + return maze_map_padded diff --git a/src/minimax/envs/maze/maze.py b/src/minimax/envs/maze/maze.py new file mode 100644 index 0000000..1c7f9c2 --- /dev/null +++ b/src/minimax/envs/maze/maze.py @@ -0,0 +1,521 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from dataclasses import dataclass +from collections import namedtuple, OrderedDict +from functools import partial +from enum import IntEnum + +import numpy as np +import jax +import jax.numpy as jnp +from jax import lax +from typing import Tuple, Optional +import chex +from flax import struct +from flax.core.frozen_dict import FrozenDict + +from minimax.envs import environment, spaces +from minimax.envs.registration import register +import minimax.util.graph as _graph_util +from .common import ( + OBJECT_TO_INDEX, + COLORS, + COLOR_TO_INDEX, + DIR_TO_VEC, + EnvInstance, + make_maze_map) + + +class Actions(IntEnum): + # Turn left, turn right, move forward + left = 0 + right = 1 + forward = 2 + + # Pick up an object + pickup = 3 + # Drop an object + drop = 4 + # Toggle/activate an object + toggle = 5 + + # Done completing task + done = 6 + + +@struct.dataclass +class EnvState: + agent_pos: chex.Array + agent_dir: chex.Array + agent_dir_idx: int + goal_pos: chex.Array + wall_map: chex.Array + maze_map: chex.Array + time: int + terminal: bool + + +@struct.dataclass +class EnvParams: + height: int = 15 + width: int = 15 + n_walls: int = 25 + agent_view_size: int = 5 + replace_wall_pos: bool = False + see_through_walls: bool = True + see_agent: bool = False + normalize_obs: bool = False + sample_n_walls: bool = False # Sample n_walls uniformly in [0, n_walls] + obs_agent_pos: bool = False + max_episode_steps: int = 250 + singleton_seed: int = -1, + + +class Maze(environment.Environment): + def __init__( + self, + height=13, + width=13, + n_walls=25, + agent_view_size=5, + replace_wall_pos=False, + see_through_walls=True, + see_agent=False, + max_episode_steps=250, + normalize_obs=False, + sample_n_walls=False, + obs_agent_pos=False, + singleton_seed=-1 + ): + super().__init__() + + self.obs_shape = (agent_view_size, agent_view_size, 3) + self.action_set = jnp.array([ + Actions.left, + Actions.right, + Actions.forward, + Actions.pickup, + Actions.drop, + Actions.toggle, + Actions.done + ]) + + self.params = EnvParams( + height=height, + width=width, + n_walls=n_walls, + agent_view_size=agent_view_size, + replace_wall_pos=replace_wall_pos and not sample_n_walls, + see_through_walls=see_through_walls, + see_agent=see_agent, + max_episode_steps=max_episode_steps, + normalize_obs=normalize_obs, + sample_n_walls=sample_n_walls, + obs_agent_pos=obs_agent_pos, + singleton_seed=-1, + ) + + @property + def default_params(self) -> EnvParams: + # Default environment parameters + return EnvParams() + + def step_env( + self, + key: chex.PRNGKey, + state: EnvState, + action: int, + ) -> Tuple[chex.Array, EnvState, float, bool, dict]: + """Perform single timestep state transition.""" + a = self.action_set[action] + state, reward = self.step_agent(key, state, a) + # Check game condition & no. steps for termination condition + state = state.replace(time=state.time + 1) + done = self.is_terminal(state) + state = state.replace(terminal=done) + + return ( + lax.stop_gradient(self.get_obs(state)), + lax.stop_gradient(state), + reward.astype(jnp.float32), + done, + {}, + ) + + def reset_env( + self, + key: chex.PRNGKey, + ) -> Tuple[chex.Array, EnvState]: + """Reset environment state by resampling contents of maze_map + - initial agent position + - goal position + - wall positions + """ + params = self.params + h = params.height + w = params.width + all_pos = np.arange(np.prod([h, w]), dtype=jnp.uint32) + + # Reset wall map, with shape H x W, and value of 1 at (i,j) iff there is a wall at (i,j) + key, subkey = jax.random.split(key) + wall_idx = jax.random.choice( + subkey, all_pos, + shape=(params.n_walls,), + replace=params.replace_wall_pos) + + if params.sample_n_walls: + key, subkey = jax.random.split(key) + sampled_n_walls = jax.random.randint( + subkey, (), minval=0, maxval=params.n_walls) + sample_wall_mask = jnp.arange(params.n_walls) < sampled_n_walls + dummy_wall_idx = wall_idx.at[0].get().repeat(params.n_walls) + wall_idx = jax.lax.select( + sample_wall_mask, + wall_idx, + dummy_wall_idx + ) + + occupied_mask = jnp.zeros_like(all_pos) + occupied_mask = occupied_mask.at[wall_idx].set(1) + wall_map = occupied_mask.reshape(h, w).astype(jnp.bool_) + + # Reset agent position + dir + key, subkey = jax.random.split(key) + agent_idx = jax.random.choice(subkey, all_pos, shape=(1,), p=( + ~occupied_mask.astype(jnp.bool_)).astype(jnp.float32)) + occupied_mask = occupied_mask.at[agent_idx].set(1) + agent_pos = jnp.array([agent_idx % w, agent_idx//w], + dtype=jnp.uint32).flatten() + + key, subkey = jax.random.split(key) + agent_dir_idx = jax.random.choice( + subkey, jnp.arange(len(DIR_TO_VEC), dtype=jnp.uint8)) + agent_dir = DIR_TO_VEC.at[agent_dir_idx].get() + + # Reset goal position + key, subkey = jax.random.split(key) + goal_idx = jax.random.choice(subkey, all_pos, shape=(1,), p=( + ~occupied_mask.astype(jnp.bool_)).astype(jnp.float32)) + goal_pos = jnp.array([goal_idx % w, goal_idx//w], + dtype=jnp.uint32).flatten() + + maze_map = make_maze_map( + params, + wall_map, + goal_pos, + agent_pos, + agent_dir_idx, + pad_obs=True) + + state = EnvState( + agent_pos=agent_pos, + agent_dir=agent_dir, + agent_dir_idx=agent_dir_idx, + goal_pos=goal_pos, + wall_map=wall_map.astype(jnp.bool_), + maze_map=maze_map, + time=0, + terminal=False, + ) + + return self.get_obs(state), state + + def set_env_instance( + self, + encoding: EnvInstance): + """ + Instance is encoded as a PyTree containing the following fields: + agent_pos, agent_dir, goal_pos, wall_map + """ + params = self.params + agent_pos = encoding.agent_pos + agent_dir_idx = encoding.agent_dir_idx + + agent_dir = DIR_TO_VEC.at[agent_dir_idx].get() + goal_pos = encoding.goal_pos + wall_map = encoding.wall_map + maze_map = make_maze_map( + params, + wall_map, + goal_pos, + agent_pos, + agent_dir_idx, # ued instances include wall padding + pad_obs=True) + + state = EnvState( + agent_pos=agent_pos, + agent_dir=agent_dir, + agent_dir_idx=agent_dir_idx, + goal_pos=goal_pos, + wall_map=wall_map, + maze_map=maze_map, + time=0, + terminal=False + ) + + return self.get_obs(state), state + + def get_obs(self, state: EnvState) -> chex.Array: + """Return limited grid view ahead of agent.""" + obs = jnp.zeros(self.obs_shape, dtype=jnp.uint8) + + agent_x, agent_y = state.agent_pos + + obs_fwd_bound1 = state.agent_pos + obs_fwd_bound2 = state.agent_pos + \ + state.agent_dir*(self.obs_shape[0]-1) + + side_offset = self.obs_shape[0]//2 + obs_side_bound1 = state.agent_pos + (state.agent_dir == 0)*side_offset + obs_side_bound2 = state.agent_pos - (state.agent_dir == 0)*side_offset + + all_bounds = jnp.stack( + [obs_fwd_bound1, obs_fwd_bound2, obs_side_bound1, obs_side_bound2]) + + # Clip obs to grid bounds appropriately + padding = obs.shape[0]-1 + obs_bounds_min = np.min(all_bounds, 0) + padding + obs_range_x = jnp.arange(obs.shape[0]) + obs_bounds_min[1] + obs_range_y = jnp.arange(obs.shape[0]) + obs_bounds_min[0] + + meshgrid = jnp.meshgrid(obs_range_y, obs_range_x) + coord_y = meshgrid[1].flatten() + coord_x = meshgrid[0].flatten() + + obs = state.maze_map.at[ + coord_y, coord_x, :].get().reshape(obs.shape[0], obs.shape[1], 3) + + obs = (state.agent_dir_idx == 0)*jnp.rot90(obs, 1) + \ + (state.agent_dir_idx == 1)*jnp.rot90(obs, 2) + \ + (state.agent_dir_idx == 2)*jnp.rot90(obs, 3) + \ + (state.agent_dir_idx == 3)*jnp.rot90(obs, 4) + + if not self.params.see_agent: + obs = obs.at[-1, side_offset].set( + jnp.array([OBJECT_TO_INDEX['empty'], 0, 0], dtype=jnp.uint8) + ) + + if not self.params.see_through_walls: + pass + + image = obs.astype(jnp.uint8) + if self.params.normalize_obs: + image = image/10.0 + + obs_dict = dict( + image=image, + agent_dir=state.agent_dir_idx + ) + if self.params.obs_agent_pos: + obs_dict.update(dict(agent_pos=state.agent_pos)) + + return OrderedDict(obs_dict) + + def step_agent(self, key: chex.PRNGKey, state: EnvState, action: int) -> Tuple[EnvState, float]: + params = self.params + + # Update agent position (forward action) + fwd_pos = jnp.minimum( + jnp.maximum(state.agent_pos + (action == + Actions.forward)*state.agent_dir, 0), + jnp.array((params.width-1, params.height-1), dtype=jnp.uint32)) + + # Can't go past wall or goal + fwd_pos_has_wall = state.wall_map.at[fwd_pos[1], fwd_pos[0]].get() + fwd_pos_has_goal = jnp.logical_and( + fwd_pos[0] == state.goal_pos[0], fwd_pos[1] == state.goal_pos[1]) + + fwd_pos_blocked = jnp.logical_or(fwd_pos_has_wall, fwd_pos_has_goal) + + agent_pos_prev = jnp.array(state.agent_pos) + agent_pos = (fwd_pos_blocked*state.agent_pos + + (~fwd_pos_blocked)*fwd_pos).astype(jnp.uint32) + + # Update agent direction (left_turn or right_turn action) + agent_dir_offset = \ + 0 \ + + (action == Actions.left)*(-1) \ + + (action == Actions.right)*1 + + agent_dir_idx = (state.agent_dir_idx + agent_dir_offset) % 4 + agent_dir = DIR_TO_VEC[agent_dir_idx] + + # Update agent component in maze_map + empty = jnp.array([OBJECT_TO_INDEX['empty'], 0, 0], dtype=jnp.uint8) + agent = jnp.array( + [OBJECT_TO_INDEX['agent'], COLOR_TO_INDEX['red'], agent_dir_idx], dtype=jnp.uint8) + padding = self.obs_shape[0]-1 + maze_map = state.maze_map + maze_map = maze_map.at[padding+agent_pos_prev[1], + padding+agent_pos_prev[0], :].set(empty) + maze_map = maze_map.at[padding+agent_pos[1], + padding+agent_pos[0], :].set(agent) + + # Return reward + # rng = jax.random.PRNGKey(agent_dir_idx + agent_pos[0] + agent_pos[1]) + # rand_reward = jax.random.uniform(rng) + reward = (1.0 - 0.9*((state.time+1)/params.max_episode_steps) + )*fwd_pos_has_goal # rand_reward + + return ( + state.replace( + agent_pos=agent_pos, + agent_dir_idx=agent_dir_idx, + agent_dir=agent_dir, + maze_map=maze_map, + terminal=fwd_pos_has_goal), + reward + ) + + def is_terminal(self, state: EnvState) -> bool: + """Check whether state is terminal.""" + done_steps = state.time >= self.params.max_episode_steps + return jnp.logical_or(done_steps, state.terminal) + + def get_eval_solved_rate_fn(self): + def _fn(ep_stats): + return ep_stats['return'] > 0 + + return _fn + + @property + def name(self) -> str: + """Environment name.""" + return "Maze" + + @property + def num_actions(self) -> int: + """Number of actions possible in environment.""" + return len(self.action_set) + + def action_space(self) -> spaces.Discrete: + """Action space of the environment.""" + return spaces.Discrete( + len(self.action_set), + dtype=jnp.uint32 + ) + + def observation_space(self) -> spaces.Dict: + """Observation space of the environment.""" + spaces_dict = { + 'image': spaces.Box(0, 255, self.obs_shape), + 'agent_dir': spaces.Discrete(4) + } + if self.params.obs_agent_pos: + params = self.params + h = params.height + w = params.width + spaces_dict.update({'agent_pos': spaces.Box( + 0, max(w, h), (2,), dtype=jnp.uint32)}) + + return spaces.Dict(spaces_dict) + + def get_monitored_metrics(self): + return () + + def state_space(self) -> spaces.Dict: + """State space of the environment.""" + params = self.params + h = params.height + w = params.width + agent_view_size = params.agent_view_size + return spaces.Dict({ + "agent_pos": spaces.Box(0, max(w, h), (2,), dtype=jnp.uint32), + "agent_dir": spaces.Discrete(4), + "goal_pos": spaces.Box(0, max(w, h), (2,), dtype=jnp.uint32), + "maze_map": spaces.Box(0, 255, (w + agent_view_size, h + agent_view_size, 3), dtype=jnp.uint32), + "time": spaces.Discrete(params.max_episode_steps), + "terminal": spaces.Discrete(2), + }) + + def max_episode_steps(self) -> int: + return self.params.max_episode_steps + + def get_env_metrics(self, state: EnvState) -> dict: + n_walls = state.wall_map.sum() + shortest_path_length = _graph_util.shortest_path_len( + state.wall_map, + state.agent_pos, + state.goal_pos + ) + + return dict( + n_walls=n_walls, + shortest_path_length=shortest_path_length, + passable=shortest_path_length > 0, + ) + + +# Register the env +if hasattr(__loader__, 'name'): + module_path = __loader__.name +elif hasattr(__loader__, 'fullname'): + module_path = __loader__.fullname + +register(env_id='Maze', entry_point=module_path + ':Maze') + + +if __name__ == '__main__': + from minimax.envs.wrappers import MonitorReturnWrapper + + render = False + n_envs = 16384 + + if render: + from minimax.envs.viz.grid_viz import GridVisualizer + viz = GridVisualizer() + obs_viz = GridVisualizer() + + viz.show() + obs_viz.show() + + kwargs = dict( + max_episode_steps=250, + height=15, + width=15, + n_walls=25, + agent_view_size=5, + see_through_walls=True + ) + env = MonitorReturnWrapper(Maze(**kwargs)) + params = env.params + extra = env.reset_extra() + + jit_reset_env = jax.jit(env.reset) + jit_step_env = jax.jit(env.step) + + key = jax.random.PRNGKey(0) + vrngs = jax.random.split(key, n_envs) + key, subkey = jax.random.split(jax.random.PRNGKey(0)) + obs, state, extra = jax.vmap( + jit_reset_env, in_axes=(0))(vrngs) + + all_sps = [] + import time + for i in range(1000): + print('step', i) + key, subkey = jax.random.split(key) + vrngs = jax.random.split(subkey, n_envs) + start = time.time() + obs, state, reward, done, info, extra = jax.vmap(jit_step_env)( + vrngs, state, action=jax.vmap(env.action_space().sample, in_axes=(0))(vrngs), extra=extra) + obs['image'].block_until_ready() + end = time.time() + # print(f"sps: {1/(end-start) * n_envs}") + # print('return:', info['return']) + + all_sps.append(1/(end-start) * n_envs) + + if render: + viz.render(params, state) + obs_viz.render_grid(np.asarray(env.get_obs( + state)['image']), k_rot90=0, agent_dir_idx=3) + + print('mean sps:', np.mean(all_sps)) + print('std sps:', np.std(all_sps)) diff --git a/src/minimax/envs/maze/maze_comparators.py b/src/minimax/envs/maze/maze_comparators.py new file mode 100644 index 0000000..5a2aa64 --- /dev/null +++ b/src/minimax/envs/maze/maze_comparators.py @@ -0,0 +1,34 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +import numpy as np +import jax +import jax.numpy as jnp + +from minimax.envs.registration import register_comparator + + +@jax.jit +def is_equal_map(a, b): + agent_pos_eq = jnp.equal(a.agent_pos, b.agent_pos).all() + goal_pos_eq = jnp.equal(a.goal_pos, b.goal_pos).all() + wall_map_eq = jnp.equal(a.wall_map, b.wall_map).all() + + _eq = jnp.logical_and(agent_pos_eq, goal_pos_eq) + _eq = jnp.logical_and(_eq, wall_map_eq) + + return _eq + + +# Register the mutators +if hasattr(__loader__, 'name'): + module_path = __loader__.name +elif hasattr(__loader__, 'fullname'): + module_path = __loader__.fullname + +register_comparator(env_id='Maze', comparator_id=None, entry_point=module_path + ':is_equal_map') \ No newline at end of file diff --git a/src/minimax/envs/maze/maze_mutators.py b/src/minimax/envs/maze/maze_mutators.py new file mode 100644 index 0000000..47907e4 --- /dev/null +++ b/src/minimax/envs/maze/maze_mutators.py @@ -0,0 +1,110 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial +from enum import IntEnum + +import numpy as np +import jax +import jax.numpy as jnp + +from .common import make_maze_map +from minimax.envs.registration import register_mutator + + +class Mutations(IntEnum): + # Turn left, turn right, move forward + NO_OP = 0 + FLIP_WALL = 1 + MOVE_GOAL = 2 + + +def flip_wall(rng, state): + wall_map = state.wall_map + h,w = wall_map.shape + wall_mask = jnp.ones((h*w,), dtype=jnp.bool_) + + goal_idx = w*state.goal_pos[1] + state.goal_pos[0] + agent_idx = w*state.agent_pos[1] + state.agent_pos[0] + wall_mask = wall_mask.at[goal_idx].set(False) + wall_mask = wall_mask.at[agent_idx].set(False) + + flip_idx = jax.random.choice(rng, np.arange(h*w), p=wall_mask) + flip_y = flip_idx//w + flip_x = flip_idx%w + + flip_val = ~wall_map.at[flip_y,flip_x].get() + next_wall_map = wall_map.at[flip_y,flip_x].set(flip_val) + + return state.replace(wall_map=next_wall_map) + + +def move_goal(rng, state): + wall_map = state.wall_map + h,w = wall_map.shape + wall_mask = wall_map.flatten() + + goal_idx = w*state.goal_pos[1] + state.goal_pos[0] + agent_idx = w*state.agent_pos[1] + state.agent_pos[0] + wall_mask = wall_mask.at[goal_idx].set(True) + wall_mask = wall_mask.at[agent_idx].set(True) + + next_goal_idx = jax.random.choice(rng, np.arange(h*w), p=~wall_mask) + next_goal_y = next_goal_idx//w + next_goal_x = next_goal_idx%w + + next_wall_map = wall_map.at[next_goal_y,next_goal_x].set(False) + next_goal_pos = jnp.array([next_goal_x,next_goal_y], dtype=jnp.uint32) + + return state.replace(wall_map=next_wall_map, goal_pos=next_goal_pos) + + +@partial(jax.jit, static_argnums=(1,3)) +def move_goal_flip_walls(rng, params, state, n=1): + if n == 0: + return state + + def _mutate(carry, step): + state = carry + rng, mutation = step + + rng, arng, brng = jax.random.split(rng,3) + + is_flip_wall = jnp.equal(mutation, Mutations.FLIP_WALL.value) + mutated_state = flip_wall(arng, state) + next_state = jax.tree_map(lambda x,y: jax.lax.select(is_flip_wall, x, y), mutated_state, state) + + is_move_goal = jnp.equal(mutation, Mutations.MOVE_GOAL.value) + mutated_state = move_goal(brng, state) + next_state = jax.tree_map(lambda x,y: jax.lax.select(is_move_goal, x, y), mutated_state, next_state) + + return next_state, None + + rng, nrng, *mrngs = jax.random.split(rng, n+2) + mutations = jax.random.choice(nrng, np.arange(len(Mutations)), (n,)) + + state, _ = jax.lax.scan(_mutate, state, (jnp.array(mrngs), mutations)) + + # Update state maze_map + next_maze_map = make_maze_map( + params, + state.wall_map, + state.goal_pos, + state.agent_pos, + state.agent_dir_idx, + pad_obs=True) + + return state.replace(maze_map=next_maze_map) + +# Register the mutators +if hasattr(__loader__, 'name'): + module_path = __loader__.name +elif hasattr(__loader__, 'fullname'): + module_path = __loader__.fullname + +register_mutator(env_id='Maze', mutator_id=None, entry_point=module_path + ':move_goal_flip_walls') \ No newline at end of file diff --git a/src/minimax/envs/maze/maze_ood.py b/src/minimax/envs/maze/maze_ood.py new file mode 100644 index 0000000..fa9a125 --- /dev/null +++ b/src/minimax/envs/maze/maze_ood.py @@ -0,0 +1,1111 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from typing import Tuple, Optional + +import jax +import jax.numpy as jnp +from flax import struct +import chex + +from minimax.envs.registration import register +from .common import ( + DIR_TO_VEC, + OBJECT_TO_INDEX, + COLOR_TO_INDEX, + make_maze_map, +) +from .maze import ( + Maze, + EnvParams, + EnvState, + Actions +) + +# ======== Singleton mazes ======== +class MazeSingleton(Maze): + def __init__( + self, + height=15, + width=15, + wall_map=None, + goal_pos=None, + agent_pos=None, + agent_dir_idx=None, + agent_view_size=5, + see_through_walls=True, + see_agent=False, + normalize_obs=False, + obs_agent_pos=False, + max_episode_steps=None, + singleton_seed=-1, + ): + super().__init__( + height=height, + width=width, + agent_view_size=agent_view_size, + see_through_walls=see_through_walls, + see_agent=see_agent, + normalize_obs=normalize_obs, + obs_agent_pos=obs_agent_pos, + max_episode_steps=max_episode_steps, + singleton_seed=singleton_seed + ) + + if wall_map is None: + self.wall_map = jnp.zeros((height,width), dtype=jnp.bool_) + else: + self.wall_map = \ + jnp.array( + [[int(x) for x in row.split()] + for row in wall_map], dtype=jnp.bool_) + height, width = self.wall_map.shape + + if max_episode_steps is None: + max_episode_steps = 2*(height+2)*(width+2) # Match original eval steps + + self.goal_pos_choices = None + if goal_pos is None: + self.goal_pos = jnp.array([height, width]) - jnp.ones(2, dtype=jnp.uint32) + elif isinstance(goal_pos, (tuple, list)) \ + and isinstance(goal_pos[0], (tuple, list)): + self.goal_pos_choices = jnp.array(goal_pos, dtype=jnp.uint32) + self.goal_pos = goal_pos[0] + else: + self.goal_pos = jnp.array(goal_pos, dtype=jnp.uint32) + + if agent_pos is None: + self.agent_pos = jnp.zeros(2, dtype=jnp.uint32) + else: + self.agent_pos = jnp.array(agent_pos, dtype=jnp.uint32) + + self.agent_dir_idx = agent_dir_idx + + if self.agent_dir_idx is None: + self.agent_dir_idx = 0 + + self.params = EnvParams( + height=height, + width=width, + agent_view_size=agent_view_size, + see_through_walls=see_through_walls, + see_agent=see_agent, + normalize_obs=normalize_obs, + obs_agent_pos=obs_agent_pos, + max_episode_steps=max_episode_steps, + singleton_seed=-1, + ) + + self.maze_map = make_maze_map( + self.params, + self.wall_map, + self.goal_pos, + self.agent_pos, + self.agent_dir_idx, + pad_obs=True) + + @property + def default_params(self) -> EnvParams: + # Default environment parameters + return EnvParams() + + def reset_env( + self, + key: chex.PRNGKey, + ) -> Tuple[chex.Array, EnvState]: + + if self.agent_dir_idx is None: + key, subkey = jax.random.split(key) + agent_dir_idx = jax.random.choice(subkey, 4) + else: + agent_dir_idx = self.agent_dir_idx + + if self.goal_pos_choices is not None: + key, subkey = jax.random.split(key) + goal_pos = jax.random.choice(subkey, self.goal_pos_choices) + maze_map = make_maze_map( + self.params, + self.wall_map, + goal_pos, + self.agent_pos, + agent_dir_idx, + pad_obs=True) + else: + goal_pos = self.goal_pos + maze_map = self.maze_map + + state = EnvState( + agent_pos=self.agent_pos, + agent_dir=DIR_TO_VEC[agent_dir_idx], + agent_dir_idx=agent_dir_idx, + goal_pos=goal_pos, + wall_map=self.wall_map, + maze_map=maze_map, + time=0, + terminal=False, + ) + + return self.get_obs(state), state + + +# ======== Specific mazes ======== +class SixteenRooms(MazeSingleton): + def __init__( + self, + see_agent=False, + normalize_obs=False): + wall_map = [ + "0 0 0 1 0 0 1 0 0 1 0 0 0", + "0 0 0 0 0 0 0 0 0 1 0 0 0", + "0 0 0 1 0 0 1 0 0 0 0 0 0", + "1 0 1 1 1 0 1 1 0 1 1 1 0", + "0 0 0 1 0 0 0 0 0 0 0 0 0", + "0 0 0 0 0 0 1 0 0 1 0 0 0", + "1 1 0 1 0 1 1 0 1 1 1 0 1", + "0 0 0 1 0 0 0 0 0 1 0 0 0", + "0 0 0 1 0 0 1 0 0 0 0 0 0", + "0 1 1 1 1 0 1 1 0 1 0 1 1", + "0 0 0 1 0 0 1 0 0 1 0 0 0", + "0 0 0 0 0 0 1 0 0 0 0 0 0", + "0 0 0 1 0 0 0 0 0 1 0 0 0" + ] + goal_pos = (11,11) + agent_pos = (1,1) + agent_dir_idx = 0 + + super().__init__( + wall_map=wall_map, + goal_pos=goal_pos, + agent_pos=agent_pos, + agent_dir_idx=agent_dir_idx, + see_agent=see_agent, + normalize_obs=normalize_obs + ) + + +class SixteenRooms2(MazeSingleton): + def __init__( + self, + see_agent=False, + normalize_obs=False): + wall_map = [ + "0 0 0 1 0 0 0 0 0 1 0 0 0", + "0 0 0 0 0 0 1 0 0 1 0 0 0", + "0 0 0 1 0 0 1 0 0 1 0 0 0", + "1 1 1 1 0 1 1 0 1 1 1 0 1", + "0 0 0 1 0 0 1 0 0 0 0 0 0", + "0 0 0 0 0 0 1 0 0 1 0 0 0", + "1 0 1 1 1 1 1 0 1 1 1 1 1", + "0 0 0 1 0 0 1 0 0 1 0 0 0", + "0 0 0 1 0 0 0 0 0 0 0 0 0", + "1 1 0 1 1 0 1 1 0 1 1 1 1", + "0 0 0 1 0 0 1 0 0 1 0 0 0", + "0 0 0 0 0 0 1 0 0 0 0 0 0", + "0 0 0 1 0 0 1 0 0 1 0 0 0" + ] + goal_pos = (11,11) + agent_pos = (1,1) + agent_dir_idx = None + + super().__init__( + wall_map=wall_map, + goal_pos=goal_pos, + agent_pos=agent_pos, + agent_dir_idx=agent_dir_idx, + see_agent=see_agent, + normalize_obs=normalize_obs + ) + + +class Labyrinth(MazeSingleton): + def __init__( + self, + see_agent=False, + normalize_obs=False): + wall_map = [ + "0 0 0 0 0 0 0 0 0 0 0 0 0", + "0 1 1 1 1 1 1 1 1 1 1 1 0", + "0 1 0 0 0 0 0 0 0 0 0 1 0", + "0 1 0 1 1 1 1 1 1 1 0 1 0", + "0 1 0 1 0 0 0 0 0 1 0 1 0", + "0 1 0 1 0 1 1 1 0 1 0 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 1 0 0 0 1 0 0 0 1 0 1 0", + "0 1 1 1 1 1 1 1 1 1 0 1 0", + "0 0 0 0 0 1 0 0 0 0 0 1 0", + "1 1 1 1 0 1 0 1 1 1 1 1 0", + "0 0 0 0 0 1 0 0 0 0 0 0 0" + ] + goal_pos = (6,6) + agent_pos = (0,12) + agent_dir_idx = 0 + + super().__init__( + wall_map=wall_map, + goal_pos=goal_pos, + agent_pos=agent_pos, + agent_dir_idx=agent_dir_idx, + see_agent=see_agent, + normalize_obs=normalize_obs + ) + + +class LabyrinthFlipped(MazeSingleton): + def __init__( + self, + see_agent=False, + normalize_obs=False): + wall_map = [ + '0 0 0 0 0 0 0 0 0 0 0 0 0', + '0 1 1 1 1 1 1 1 1 1 1 1 0', + '0 1 0 0 0 0 0 0 0 0 0 1 0', + '0 1 0 1 1 1 1 1 1 1 0 1 0', + '0 1 0 1 0 0 0 0 0 1 0 1 0', + '0 1 0 1 0 1 1 1 0 1 0 1 0', + '0 1 0 1 0 1 0 1 0 1 0 1 0', + '0 1 0 1 0 1 0 1 0 1 0 1 0', + '0 1 0 1 0 0 0 1 0 0 0 1 0', + '0 1 0 1 1 1 1 1 1 1 1 1 0', + '0 1 0 0 0 0 0 1 0 0 0 0 0', + '0 1 1 1 1 1 0 1 0 1 1 1 1', + '0 0 0 0 0 0 0 1 0 0 0 0 0' + ] + goal_pos = (6,6) + agent_pos = (12,12) + agent_dir_idx = 2 + + super().__init__( + wall_map=wall_map, + goal_pos=goal_pos, + agent_pos=agent_pos, + agent_dir_idx=agent_dir_idx, + see_agent=see_agent, + normalize_obs=normalize_obs + ) + + +class Labyrinth2(MazeSingleton): + def __init__( + self, + see_agent=False, + normalize_obs=False): + wall_map = [ + "0 1 0 0 0 0 0 0 0 0 0 0 0", + "0 1 0 1 1 1 1 1 1 1 1 1 0", + "0 1 0 1 0 0 0 0 0 0 0 1 0", + "0 1 0 1 0 1 1 1 1 1 0 1 0", + "0 1 0 1 0 1 0 0 0 1 0 1 0", + "0 0 0 1 0 1 0 1 0 1 0 1 0", + "1 1 1 1 0 1 0 1 0 1 0 1 0", + "0 0 0 1 0 1 1 1 0 1 0 1 0", + "0 1 0 1 0 0 0 0 0 1 0 1 0", + "0 1 0 1 1 1 1 1 1 1 0 1 0", + "0 1 0 0 0 0 0 0 0 0 0 1 0", + "0 1 1 1 1 1 1 1 1 1 1 1 0", + "0 0 0 0 0 0 0 0 0 0 0 0 0" + ] + goal_pos = (6,6) + agent_pos = (0,0) + agent_dir_idx = None + + super().__init__( + wall_map=wall_map, + goal_pos=goal_pos, + agent_pos=agent_pos, + agent_dir_idx=agent_dir_idx, + see_agent=see_agent, + normalize_obs=normalize_obs + ) + + +class StandardMaze(MazeSingleton): + def __init__( + self, + see_agent=False, + normalize_obs=False): + wall_map = [ + "0 0 0 0 0 1 0 0 0 0 1 0 0", + "0 1 1 1 0 1 1 1 1 0 1 1 0", + "0 1 0 0 0 0 0 0 0 0 0 0 0", + "0 1 1 1 1 1 1 1 1 0 1 1 1", + "0 0 0 0 0 0 0 0 1 0 0 0 0", + "1 1 1 1 1 1 0 1 1 1 1 1 0", + "0 0 0 0 1 0 0 1 0 0 0 0 0", + "0 1 1 0 0 0 1 1 0 1 1 1 1", + "0 0 1 0 1 0 0 1 0 0 0 1 0", + "1 0 1 0 1 1 0 1 1 1 0 1 0", + "1 0 1 0 0 1 0 0 0 1 0 0 0", + "1 0 1 1 0 1 1 1 0 1 1 1 0", + "0 0 0 1 0 0 0 1 0 1 0 0 0" + ] + goal_pos = (6,12) + agent_pos = (6,0) + agent_dir_idx = 0 + + super().__init__( + wall_map=wall_map, + goal_pos=goal_pos, + agent_pos=agent_pos, + agent_dir_idx=agent_dir_idx, + see_agent=see_agent, + normalize_obs=normalize_obs + ) + + +class StandardMaze2(MazeSingleton): + def __init__( + self, + see_agent=False, + normalize_obs=False): + wall_map = [ + "0 0 0 1 0 1 0 0 0 0 1 0 0", + "0 1 0 1 0 1 1 1 1 0 0 0 1", + "0 1 0 0 0 0 0 0 0 0 1 0 0", + "0 1 1 1 1 1 1 1 1 0 1 1 1", + "0 0 0 1 0 0 1 0 1 0 1 0 0", + "1 1 0 1 0 1 1 0 1 0 1 0 0", + "0 1 0 1 0 0 0 0 1 0 1 1 0", + "0 1 0 1 1 0 1 1 1 0 0 1 0", + "0 1 0 0 1 0 0 1 1 1 0 1 0", + "0 1 1 0 1 1 0 1 0 1 0 1 0", + "0 1 0 0 0 1 0 1 0 1 0 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 0 0 1 0 0 0 1 0 0 0 0 0" + ] + goal_pos = (12,4) + agent_pos = (0,6) + agent_dir_idx = None + + super().__init__( + wall_map=wall_map, + goal_pos=goal_pos, + agent_pos=agent_pos, + agent_dir_idx=agent_dir_idx, + see_agent=see_agent, + normalize_obs=normalize_obs + ) + + +class StandardMaze3(MazeSingleton): + def __init__( + self, + see_agent=False, + normalize_obs=False): + wall_map = [ + "0 0 0 0 1 0 1 0 0 0 0 0 0", + "0 1 1 1 1 0 1 0 1 1 1 1 0", + "0 1 0 0 0 0 1 0 1 0 0 0 0", + "0 0 0 1 1 1 1 0 1 0 1 0 1", + "1 1 0 1 0 0 0 0 1 0 1 0 0", + "0 0 0 1 0 1 1 0 1 0 1 1 0", + "0 1 0 1 0 1 0 0 1 0 0 1 0", + "0 1 0 1 0 1 0 1 1 1 0 1 1", + "0 1 0 0 0 1 0 1 0 1 0 0 0", + "0 1 1 1 0 1 0 1 0 1 1 1 0", + "0 1 0 0 0 1 0 1 0 0 0 1 0", + "0 1 0 1 1 1 0 1 0 1 0 1 0", + "0 1 0 0 0 1 0 0 0 1 0 0 0" + ] + goal_pos = (12,6) + agent_pos = (3,0) + agent_dir_idx = None + + super().__init__( + wall_map=wall_map, + goal_pos=goal_pos, + agent_pos=agent_pos, + agent_dir_idx=agent_dir_idx, + see_agent=see_agent, + normalize_obs=normalize_obs + ) + + +class SmallCorridor(MazeSingleton): + def __init__( + self, + see_agent=False, + normalize_obs=False): + wall_map = [ + "0 0 0 0 0 0 0 0 0 0 0 0 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 1 1 1 1 1 1 1 1 1 1 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 0 0 0 0 0 0 0 0 0 0 0 0" + ] + goal_pos = [ + (2,5),(4,5),(6,5),(8,5),(10,5), + (2,7),(4,7),(6,7),(8,7),(10,7), + ] + agent_pos = (0,6) + agent_dir_idx = None + + super().__init__( + wall_map=wall_map, + goal_pos=goal_pos, + agent_pos=agent_pos, + agent_dir_idx=agent_dir_idx, + see_agent=see_agent, + normalize_obs=normalize_obs + ) + + +class LargeCorridor(MazeSingleton): + def __init__( + self, + see_agent=False, + normalize_obs=False): + wall_map = [ + "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0", + "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0", + ] + goal_pos = [ + (2,8),(4,8),(6,8),(8,8),(10,8),(12,8),(14,8),(16,8), + (2,10),(4,10),(6,10),(8,10),(10,10),(12,10),(14,10),(16,10) + ] + agent_pos = (0,9) + agent_dir_idx = None + + super().__init__( + wall_map=wall_map, + goal_pos=goal_pos, + agent_pos=agent_pos, + agent_dir_idx=agent_dir_idx, + see_agent=see_agent, + normalize_obs=normalize_obs + ) + + +class FourRooms(Maze): + def __init__( + self, + height=17, + width=17, + agent_view_size=5, + see_through_walls=True, + see_agent=False, + normalize_obs=False, + max_episode_steps=250, + singleton_seed=-1): + + super().__init__( + height=height, + width=width, + agent_view_size=agent_view_size, + see_through_walls=see_through_walls, + see_agent=see_agent, + normalize_obs=normalize_obs, + max_episode_steps=max_episode_steps, + singleton_seed=singleton_seed + ) + + assert height % 2 == 1 and width % 2 == 1, \ + 'Grid height and width must be odd' + + wall_map = jnp.zeros((height, width), dtype=jnp.bool_) + wall_map = wall_map.at[height//2, :].set(True) + wall_map = wall_map.at[:, width//2].set(True) + self.wall_map = wall_map + + self.room_h = height//2 + self.room_w = width//2 + + self.all_pos_idxs = jnp.arange(height*width) + self.goal_pos_mask = (~wall_map).flatten() + self.agent_pos_mask = self.goal_pos_mask + + def reset_env( + self, + key: chex.PRNGKey + ) -> Tuple[chex.Array, EnvState]: + # Randomize door positions + params = self.params + + key, x_rng, y_rng = jax.random.split(key,3) + x_door_idxs = jax.random.randint(x_rng, (2,), 0, self.room_w) \ + + jnp.array([0, self.room_w+1], dtype=jnp.uint32) + + y_door_idxs = jax.random.randint(y_rng, (2,), 0, self.room_h) \ + + jnp.array([0, self.room_h+1], dtype=jnp.uint32) + + wall_map = self.wall_map.at[self.room_h, x_door_idxs].set(False) + wall_map = wall_map.at[y_door_idxs,self.room_w].set(False) + + # Randomize goal pos + key, subkey = jax.random.split(key) + goal_pos_idx = jax.random.choice(subkey, self.all_pos_idxs, shape=(), p=self.goal_pos_mask) + goal_pos = jnp.array([goal_pos_idx%params.width, goal_pos_idx//params.width], dtype=jnp.uint32) + + # Randomize agent pos + key, subkey = jax.random.split(key) + agent_pos_mask = self.agent_pos_mask.at[goal_pos_idx].set(False) + agent_pos_idx = jax.random.choice(subkey, self.all_pos_idxs, shape=(), p=self.agent_pos_mask) + agent_pos = jnp.array([agent_pos_idx%params.width, agent_pos_idx//params.width], dtype=jnp.uint32) + + key, subkey = jax.random.split(key) + agent_dir_idx = jax.random.choice(subkey, 4) + + maze_map = make_maze_map( + self.params, + wall_map, + goal_pos, + agent_pos, + agent_dir_idx, + pad_obs=True) + + state = EnvState( + agent_pos=agent_pos, + agent_dir=DIR_TO_VEC[agent_dir_idx], + agent_dir_idx=agent_dir_idx, + goal_pos=goal_pos, + wall_map=wall_map, + maze_map=maze_map, + time=0, + terminal=False, + ) + + return self.get_obs(state), state + + +class Crossing(Maze): + def __init__( + self, + height=9, + width=9, + n_crossings=5, + agent_view_size=5, + see_through_walls=True, + see_agent=False, + normalize_obs=False, + max_episode_steps=250, + singleton_seed=-1): + self.n_crossings = n_crossings + max_episode_steps = 4*(height+2)*(width+2) + + super().__init__( + height=height, + width=width, + agent_view_size=agent_view_size, + see_through_walls=see_through_walls, + see_agent=see_agent, + normalize_obs=normalize_obs, + max_episode_steps=max_episode_steps, + singleton_seed=singleton_seed + ) + + def reset_env( + self, + key: chex.PRNGKey + ) -> Tuple[chex.Array, EnvState]: + params = self.params + height, width = params.height, params.width + goal_pos = jnp.array([width-1, height-1]) + agent_pos = jnp.array([0,0], dtype=jnp.uint32) + agent_dir_idx = 0 + + # Generate walls + wall_map = jnp.zeros((height, width), dtype=jnp.bool_) + + row_y_choices = jnp.arange(1,height-1,2) + col_x_choices = jnp.arange(1,width-1,2) + + rng, subrng = jax.random.split(key) + dirs = jax.random.permutation( + subrng, + jnp.concatenate( + (jnp.zeros(len(row_y_choices)), + jnp.ones(len(col_x_choices))) + ) + )[:self.n_crossings] + + n_v = sum(dirs.astype(jnp.uint32)) + n_h = len(dirs) - n_v + + rng, row_rng, col_rng = jax.random.split(rng, 3) + + row_ys_mask = jax.random.permutation(row_rng, (jnp.arange(len(row_y_choices)) < n_v).repeat(2)) + if height % 2 == 0: + row_ys_mask = jnp.concatenate((row_ys_mask, jnp.zeros(2))) + else: + row_ys_mask = jnp.concatenate((row_ys_mask, jnp.zeros(1))) + + row_ys_mask = jnp.logical_and( + jnp.zeros(height, dtype=jnp.bool_).at[row_y_choices].set(True), + row_ys_mask + ) + + col_xs_mask = jax.random.permutation(col_rng, (jnp.arange(len(col_x_choices)) < n_h).repeat(2)) + if width % 2 == 0: + col_xs_mask = jnp.concatenate((col_xs_mask, jnp.zeros(2))) + else: + col_xs_mask = jnp.concatenate((col_xs_mask, jnp.zeros(1))) + + col_xs_mask = jnp.logical_and( + jnp.zeros(width, dtype=jnp.bool_).at[col_x_choices].set(True), + col_xs_mask + ) + + wall_map = jnp.logical_or( + wall_map, + jnp.tile(jnp.expand_dims(row_ys_mask,-1), (1,width)) + ) + + wall_map = jnp.logical_or( + wall_map, + jnp.tile(jnp.expand_dims(col_xs_mask,0), (height,1)) + ) + + # Generate wall openings + def _scan_step(carry, rng): + wall_map, pos, passed_wall, last_dir, last_dir_idx = carry + + dir_idx = jax.random.randint(rng,(),0,2) + + go_dir = (~passed_wall)*DIR_TO_VEC[dir_idx] + passed_wall*last_dir + next_pos = pos + go_dir + + # If next pos is the right border, force direction to be down + collide = jnp.logical_or( + (next_pos[0] >= width), + (next_pos[1] >= height) + ) + go_dir = collide*DIR_TO_VEC[(dir_idx+1)%2] + (~collide)*go_dir + dir_idx = (dir_idx+1)%2 + (~collide)*dir_idx + + next_pos = collide*(pos + go_dir) + (~collide)*next_pos + + last_dir = go_dir + last_dir_idx = dir_idx + pos = next_pos + + passed_wall = wall_map[pos[1],pos[0]] + wall_map = wall_map.at[pos[1], pos[0]].set(False) + + return (wall_map, pos.astype(jnp.uint32), passed_wall, last_dir, last_dir_idx), None + + n_steps_to_goal = width + height - 2 + rng, *subrngs = jax.random.split(rng, n_steps_to_goal+1) + + pos = agent_pos + passed_wall = jnp.array(False) + last_dir = DIR_TO_VEC[0] + + (wall_map, pos, passed_wall, last_dir, last_dir_idx), _ = jax.lax.scan( + _scan_step, + (wall_map, pos, passed_wall, last_dir, 0), + jnp.array(subrngs), + length=n_steps_to_goal + ) + + maze_map = make_maze_map( + self.params, + wall_map, + goal_pos, + agent_pos, + agent_dir_idx, + pad_obs=True) + + state = EnvState( + agent_pos=agent_pos, + agent_dir=DIR_TO_VEC[agent_dir_idx], + agent_dir_idx=agent_dir_idx, + goal_pos=goal_pos, + wall_map=wall_map, + maze_map=maze_map, + time=0, + terminal=False, + ) + + return self.get_obs(state), state + + +NEIGHBOR_WALL_OFFSETS = jnp.array([ + [1,0], # right + [0,1], # bottom + [-1,0], # left + [0,-1], # top + [0,0] # self +], dtype=jnp.int32) + + +class PerfectMaze(Maze): + def __init__( + self, + height=13, + width=13, + agent_view_size=5, + see_through_walls=True, + see_agent=False, + normalize_obs=False, + max_episode_steps=250, + singleton_seed=-1): + + assert height % 2 == 1 and width % 2 == 1, \ + 'Maze dimensions must be odd.' + + max_episode_steps = 2*(width+2)*(height+2) + super().__init__( + height=height, + width=width, + agent_view_size=agent_view_size, + see_through_walls=see_through_walls, + see_agent=see_agent, + normalize_obs=normalize_obs, + max_episode_steps=max_episode_steps, + singleton_seed=singleton_seed + ) + + def reset_env( + self, + key: chex.PRNGKey + ) -> Tuple[chex.Array, EnvState]: + """ + Generate a perfect maze using an iterated search procedure. + """ + params = self.params + height, width = self.params.height, self.params.width + n_tiles = height*width + + # Track maze wall map + wall_map = jnp.ones((height, width), dtype=jnp.bool_) + + # Track visited, walkable tiles + _h = height//2+1 + _w = width//2+1 + visited_map = jnp.zeros((_h, _w), dtype=jnp.bool_) + vstack = jnp.zeros((_h*_w, 2), dtype=jnp.uint32) + vstack_size = 0 + + # Get initial start tile in walkable index + key, subkey = jax.random.split(key) + start_pos_x = jax.random.randint(subkey, (), 0, _w) + start_pos_y = jax.random.randint(subkey, (), 0, _h) + start_pos = jnp.array([start_pos_x,start_pos_y], dtype=jnp.uint32) + + # Set initial start tile as visited + visited_map = visited_map.at[ + start_pos[1],start_pos[0] + ].set(True) + wall_map = wall_map.at[ + 2*start_pos[1],2*start_pos[0] + ].set(False) + vstack = vstack.at[vstack_size:vstack_size+2].set(start_pos) + vstack_size += 2 + + def _scan_step(carry, key): + # Choose last visited tile and move to a neighbor + wall_map, visited_map, vstack, vstack_size = carry + + abs_pos = 2*vstack[vstack_size-1] + + neighbor_wall_offsets = NEIGHBOR_WALL_OFFSETS.at[-1].set( + vstack[vstack_size-2] - vstack[vstack_size-1] + ) + + # Find a random unvisited neighbor + neighbor_pos = \ + jnp.minimum( + jnp.maximum( + jnp.tile(abs_pos, (len(NEIGHBOR_WALL_OFFSETS),1)) \ + + 2*neighbor_wall_offsets, 0 + ), + jnp.array([width, height], dtype=jnp.uint32) + ) + + # Check for unvisited neighbors. Set self to unvisited if all visited. + neighbor_visited = visited_map.at[ + neighbor_pos[:,1]//2, neighbor_pos[:,0]//2 + ].get() + + n_neighbor_visited = neighbor_visited[:4].sum() + all_visited = n_neighbor_visited == 4 + all_visited_post = n_neighbor_visited >= 3 + neighbor_visited = neighbor_visited.at[-1].set(~all_visited) + + # Choose a random unvisited neigbor and remove walls between current tile + # and this neighbor and at this neighbor. + rand_neighbor_idx = jax.random.choice( + key, jnp.arange(len(NEIGHBOR_WALL_OFFSETS)), p=~neighbor_visited) + rand_neighbor_pos = neighbor_pos[rand_neighbor_idx] + rand_neighbor_wall_pos = abs_pos + (~all_visited)*neighbor_wall_offsets[rand_neighbor_idx] + remove_wall_pos = jnp.concatenate( + (jnp.expand_dims(rand_neighbor_pos, 0), + jnp.expand_dims(rand_neighbor_wall_pos,0)), 0) + wall_map = wall_map.at[ + remove_wall_pos[:,1], remove_wall_pos[:,0] + ].set(False) + + # Set selected neighbor as visited + visited_map = visited_map.at[ + rand_neighbor_pos[1]//2,rand_neighbor_pos[0]//2 + ].set(True) + + # Pop current tile from stack if all neighbors have been visited + vstack_size -= all_visited_post + + # Push selected neighbor onto stack + vstack = vstack.at[vstack_size].set( + rand_neighbor_pos//2 + ) + vstack_size += ~all_visited + + return (wall_map, visited_map, vstack, vstack_size), None + + # for i in range(3*_h*_w): + max_n_steps = 2*_w*_h + key, *subkeys = jax.random.split(key, max_n_steps+1) + (wall_map, visited_map, vstack, vstack_size), _ = jax.lax.scan( + _scan_step, + (wall_map, visited_map, vstack, vstack_size), + jnp.array(subkeys), + length=max_n_steps + ) + + # Randomize goal position + all_pos_idx = jnp.arange(height*width) + + key, subkey = jax.random.split(key) + goal_mask = ~wall_map.flatten() + goal_pos_idx = jax.random.choice(subkey, all_pos_idx, p=goal_mask) + goal_pos = jnp.array([goal_pos_idx%width, goal_pos_idx//width]) + + # Randomize agent position + key, subkey = jax.random.split(key) + agent_mask = goal_mask.at[goal_pos_idx].set(False) + agent_pos_idx = jax.random.choice(subkey, all_pos_idx, p=agent_mask) + agent_pos = jnp.array([agent_pos_idx%width, agent_pos_idx//width], dtype=jnp.uint32) + + # Randomize agent dir + key, subkey = jax.random.split(key) + agent_dir_idx = jax.random.choice(subkey, 4) + + maze_map = make_maze_map( + self.params, + wall_map, + goal_pos, + agent_pos, + agent_dir_idx, + pad_obs=True) + + state = EnvState( + agent_pos=agent_pos, + agent_dir=DIR_TO_VEC[agent_dir_idx], + agent_dir_idx=agent_dir_idx, + goal_pos=goal_pos, + wall_map=wall_map, + maze_map=maze_map, + time=0, + terminal=False, + ) + + return self.get_obs(state), state + + +class PerfectMazeMedium(PerfectMaze): + def __init__(self, *args, **kwargs): + super().__init__(height=19, width=19, *args, **kwargs) + + +class PerfectMazeExtraLarge(PerfectMaze): + def __init__(self, *args, **kwargs): + super().__init__(height=101, width=101, *args, **kwargs) + + +class Memory(MazeSingleton): + def __init__( + self, + height=17, + width=17, + agent_view_size=7, + see_through_walls=True, + see_agent=False, + normalize_obs=False, + obs_agent_pos=False, + max_episode_steps=250, + singleton_seed=-1): + + # Generate walls + wall_map = [ + "0 0 0 0 0 0 0 0 1 0 1 0 0 0 0", + "0 0 0 0 0 0 0 0 1 0 1 0 0 0 0", + "0 0 0 0 0 0 0 0 1 0 1 0 0 0 0", + "0 0 0 0 0 0 0 0 1 0 1 0 0 0 0", + "0 0 0 0 0 0 0 0 1 0 1 0 0 0 0", + "1 1 1 1 0 0 0 0 1 0 1 0 0 0 0", + "0 0 0 1 1 1 1 1 1 0 1 0 0 0 0", + "0 0 0 0 0 0 0 0 0 0 1 0 0 0 0", + "0 0 0 1 1 1 1 1 1 0 1 0 0 0 0", + "1 1 1 1 0 0 0 0 1 0 1 0 0 0 0", + "0 0 0 0 0 0 0 0 1 0 1 0 0 0 0", + "0 0 0 0 0 0 0 0 1 0 1 0 0 0 0", + "0 0 0 0 0 0 0 0 1 0 1 0 0 0 0", + "0 0 0 0 0 0 0 0 1 0 1 0 0 0 0", + "0 0 0 0 0 0 0 0 1 0 1 0 0 0 0" + ] + + super().__init__( + wall_map=wall_map, + goal_pos=(9,5), + agent_pos=(0,7), + agent_dir_idx=0, + see_agent=see_agent, + normalize_obs=normalize_obs, + obs_agent_pos=obs_agent_pos, + max_episode_steps=max_episode_steps + ) + + self.top_pos = jnp.array([9,5], dtype=jnp.uint32) + self.bottom_pos = jnp.array([9,9], dtype=jnp.uint32) + + def reset_env( + self, + key: chex.PRNGKey, + ) -> Tuple[chex.Array, EnvState]: + params = self.params + height, width = params.height, params.width + + agent_pos = jnp.array([0,7], dtype=jnp.uint32) + agent_dir_idx = 0 + + # Randomly generate a memory location + is_top_goal = jax.random.randint(key, minval=0, maxval=2, shape=(1,), dtype=jnp.uint8) + + clue_pos = jnp.array((0,6), dtype=jnp.uint32) + self.goal_pos = is_top_goal*self.top_pos + (1-is_top_goal)*self.bottom_pos + self.distractor_pos = is_top_goal*self.bottom_pos + (1-is_top_goal)*self.top_pos + + goal_color = is_top_goal*COLOR_TO_INDEX['red'] + (1-is_top_goal)*COLOR_TO_INDEX['green'] + + wall_map = self.wall_map + maze_map = make_maze_map( + self.params, + jnp.array(wall_map, dtype=jnp.bool_), + self.goal_pos, + agent_pos, + agent_dir_idx, + pad_obs=True) + + red_goal = jnp.array([OBJECT_TO_INDEX['goal'], COLOR_TO_INDEX['red'], 0], dtype=jnp.uint8) + green_goal = jnp.array([OBJECT_TO_INDEX['goal'], COLOR_TO_INDEX['green'], 0], dtype=jnp.uint8) + clue = is_top_goal*red_goal + (1-is_top_goal)*green_goal + + padding = params.agent_view_size-1 + wall_map = wall_map.at[clue_pos[1], clue_pos[0]].set(True) + maze_map = maze_map.at[padding+clue_pos[1], padding+clue_pos[0]].set(clue) + + wall_map = wall_map.at[self.top_pos[1], self.top_pos[0]].set(True) + maze_map = maze_map.at[padding+self.top_pos[1], padding+self.top_pos[0]].set(red_goal) + + wall_map = wall_map.at[self.bottom_pos[1], self.bottom_pos[0]].set(True) + maze_map = maze_map.at[padding+self.bottom_pos[1], padding+self.bottom_pos[0]].set(green_goal) + + state = EnvState( + agent_pos=agent_pos, + agent_dir=DIR_TO_VEC[agent_dir_idx], + agent_dir_idx=agent_dir_idx, + goal_pos=self.goal_pos, + wall_map=wall_map, + maze_map=maze_map, + time=0, + terminal=False, + ) + + return self.get_obs(state), state + + def get_distractor_pos(self, state): + goal_x, goal_y = state.goal_pos + is_top_goal = jnp.logical_and( + goal_x == self.top_pos[0], + goal_y == self.top_pos[1] + ) + + return is_top_goal*self.bottom_pos + (1-is_top_goal)*self.top_pos + + + def step_agent(self, key: chex.PRNGKey, state: EnvState, action: int) -> Tuple[EnvState, float]: + next_state, reward = super().step_agent( + key=key, + state=state, + action=action + ) + + fwd_pos = jnp.minimum( + jnp.maximum(state.agent_pos + (action == Actions.forward)*state.agent_dir, 0), + jnp.array(( + self.params.width-1, self.params.height-1), + dtype=jnp.uint32 + )) + + distractor_pos = self.get_distractor_pos(state) + fwd_pos_has_distractor = jnp.logical_and( + fwd_pos[0] == distractor_pos[0], + fwd_pos[1] == distractor_pos[1] + ) + + next_state = next_state.replace( + terminal=jnp.logical_or( + next_state.terminal, + fwd_pos_has_distractor + ) + ) + + return ( + next_state, + reward + ) + + +# ======== Registration ======== +if hasattr(__loader__, 'name'): + module_path = __loader__.name +elif hasattr(__loader__, 'fullname'): + module_path = __loader__.fullname + +register(env_id='Maze-SixteenRooms', entry_point=module_path + ':SixteenRooms') +register(env_id='Maze-SixteenRooms2', entry_point=module_path + ':SixteenRooms2') +register(env_id='Maze-Labyrinth', entry_point=module_path + ':Labyrinth') +register(env_id='Maze-Labyrinth2', entry_point=module_path + ':Labyrinth2') +register(env_id='Maze-LabyrinthFlipped', entry_point=module_path + ':LabyrinthFlipped') +register(env_id='Maze-StandardMaze', entry_point=module_path + ':StandardMaze') +register(env_id='Maze-StandardMaze2', entry_point=module_path + ':StandardMaze2') +register(env_id='Maze-StandardMaze3', entry_point=module_path + ':StandardMaze3') +register(env_id='Maze-SmallCorridor', entry_point=module_path + ':SmallCorridor') +register(env_id='Maze-LargeCorridor', entry_point=module_path + ':LargeCorridor') + +register(env_id='Maze-FourRooms', entry_point=module_path + ':FourRooms') +register(env_id='Maze-Crossing', entry_point=module_path + ':Crossing') +register(env_id='Maze-PerfectMaze', entry_point=module_path + ':PerfectMaze') +register(env_id='Maze-PerfectMazeMedium', entry_point=module_path + ':PerfectMazeMedium') +register(env_id='Maze-PerfectMazeXL', entry_point=module_path + ':PerfectMazeExtraLarge') + +register(env_id='Maze-Memory', entry_point=module_path + ':Memory') \ No newline at end of file diff --git a/src/minimax/envs/maze/maze_ued.py b/src/minimax/envs/maze/maze_ued.py new file mode 100644 index 0000000..1d80e86 --- /dev/null +++ b/src/minimax/envs/maze/maze_ued.py @@ -0,0 +1,425 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from dataclasses import dataclass +from collections import namedtuple, OrderedDict +from functools import partial +from enum import IntEnum + +import numpy as np +import jax +import jax.numpy as jnp +from jax import lax +from typing import Tuple, Optional +import chex +from flax import struct +from flax.core.frozen_dict import FrozenDict + +from .common import EnvInstance, make_maze_map +from minimax.envs import environment, spaces +from minimax.envs.registration import register_ued + + +class SequentialActions(IntEnum): + skip = 0 + wall = 1 + goal = 2 + agent = 3 + + +@struct.dataclass +class EnvState: + encoding: chex.Array + time: int + terminal: bool + + +@struct.dataclass +class EnvParams: + height: int = 15 + width: int = 15 + n_walls: int = 25 + noise_dim: int = 50 + replace_wall_pos: bool = False + fixed_n_wall_steps: bool = False + first_wall_pos_sets_budget: bool = False + use_seq_actions: bool = False + set_agent_dir: bool = False + normalize_obs: bool = False + singleton_seed: int = -1 + + +class UEDMaze(environment.Environment): + def __init__( + self, + height=13, + width=13, + n_walls=25, + noise_dim=16, + replace_wall_pos=False, + fixed_n_wall_steps=False, + first_wall_pos_sets_budget=False, + use_seq_actions=False, + set_agent_dir=False, + normalize_obs=False, + ): + """ + Using the original action space requires ensuring proper handling + of a sequence with trailing dones, e.g. dones: 0 0 0 0 1 1 1 1 1 ... 1. + Advantages and value losses should only be computed where ~dones[0]. + """ + assert not (first_wall_pos_sets_budget and fixed_n_wall_steps), \ + 'Setting first_wall_pos_sets_budget=True requires fixed_n_wall_steps=False.' + + super().__init__() + + self.n_tiles = height*width + # go straight, turn left, turn right, take action + self.action_set = jnp.array(jnp.arange(self.n_tiles)) + + self.params = EnvParams( + height=height, + width=width, + n_walls=n_walls, + noise_dim=noise_dim, + replace_wall_pos=replace_wall_pos, + fixed_n_wall_steps=fixed_n_wall_steps, + first_wall_pos_sets_budget=first_wall_pos_sets_budget, + use_seq_actions=False, + set_agent_dir=set_agent_dir, + normalize_obs=normalize_obs, + ) + + @staticmethod + def align_kwargs(kwargs, other_kwargs): + kwargs.update(dict( + height=other_kwargs['height'], + width=other_kwargs['width'], + )) + + return kwargs + + def _add_noise_to_obs(self, rng, obs): + if self.params.noise_dim > 0: + noise = jax.random.uniform(rng, (self.params.noise_dim,)) + obs.update(dict(noise=noise)) + + return obs + + def reset_env( + self, + key: chex.PRNGKey): + """ + Prepares the environment state for a new design + from a blank slate. + """ + params = self.params + noise_rng, dir_rng = jax.random.split(key) + encoding = jnp.zeros((self._get_encoding_dim(),), dtype=jnp.uint32) + + if not params.set_agent_dir: + rand_dir = jax.random.randint( + dir_rng, (), minval=0, maxval=4) # deterministic + tile_scale_dir = jnp.ceil( + (rand_dir/4)*self.n_tiles).astype(jnp.uint32) + encoding = encoding.at[-1].set(tile_scale_dir) + + state = EnvState( + encoding=encoding, + time=0, + terminal=False, + ) + + obs = self._add_noise_to_obs( + noise_rng, + self.get_obs(state) + ) + + return obs, state + + def step_env( + self, + key: chex.PRNGKey, + state: EnvState, + action: int, + ) -> Tuple[chex.Array, EnvState, float, bool, dict]: + """ + Take a design step. + action: A pos as an int from 0 to (height*width)-1 + """ + params = self.params + + collision_rng, noise_rng = jax.random.split(key) + + # Sample a random free tile in case of a collision + dist_values = jnp.logical_and( # True if position taken + jnp.ones(params.n_walls + 2), + jnp.arange(params.n_walls + 2)+1 > state.time + ) + + # Get zero-indexed last wall time step + if params.fixed_n_wall_steps: + max_n_walls = params.n_walls + encoding_pos = state.encoding[:params.n_walls+2] + last_wall_step_idx = max_n_walls - 1 + else: + max_n_walls = jnp.round( + params.n_walls*state.encoding[0]/self.n_tiles).astype(jnp.uint32) + + if self.params.first_wall_pos_sets_budget: + encoding_pos = state.encoding[:params.n_walls+2] + last_wall_step_idx = jnp.maximum(max_n_walls, 1) - 1 + else: + encoding_pos = state.encoding[1:params.n_walls+3] + last_wall_step_idx = max_n_walls + + pos_dist = jnp.ones(self.n_tiles).at[ + jnp.flip(encoding_pos)].set(jnp.flip(dist_values)) + all_pos = jnp.arange(self.n_tiles, dtype=jnp.uint32) + + # Only mark collision if replace_wall_pos=False OR the agent is placed over the goal + goal_step_idx = last_wall_step_idx + 1 + agent_step_idx = last_wall_step_idx + 2 + + # Track whether it is the last time step + next_state = state.replace(time=state.time + 1) + done = self.is_terminal(next_state) + + # Always place agent idx in last enc position. + is_agent_dir_step = jnp.logical_and( + params.set_agent_dir, + done + ) + + collision = jnp.logical_and( + pos_dist[action] < 1, + jnp.logical_or( + not params.replace_wall_pos, + jnp.logical_and( # agent pos cannot override goal + jnp.equal(state.time, agent_step_idx), + jnp.equal(state.encoding[goal_step_idx], action) + ) + ) + ) + collision = (collision * (1-is_agent_dir_step)).astype(jnp.uint32) + + action = (1-collision)*action + \ + collision*jax.random.choice(collision_rng, + all_pos, replace=False, p=pos_dist) + + enc_idx = (1-is_agent_dir_step)*state.time + is_agent_dir_step*(-1) + encoding = state.encoding.at[enc_idx].set(action) + + next_state = next_state.replace( + encoding=encoding, + terminal=done + ) + reward = 0 + + obs = self._add_noise_to_obs(noise_rng, self.get_obs(next_state)) + + return ( + lax.stop_gradient(obs), + lax.stop_gradient(next_state), + reward, + done, + {}, + ) + + def get_env_instance( + self, + key: chex.PRNGKey, + state: EnvState + ) -> chex.Array: + """ + Converts internal encoding to an instance encoding that + can be interpreted by the `set_to_instance` method + the paired Environment class. + """ + params = self.params + h = params.height + w = params.width + enc = state.encoding + + # === Extract agent_dir, agent_pos, and goal_pos === + # Num walls placed currently + if params.fixed_n_wall_steps: + n_walls = params.n_walls + enc_len = self._get_encoding_dim() + wall_pos_idx = jnp.flip(enc[:params.n_walls]) + agent_pos_idx = enc_len-2 # Enc is full length + goal_pos_idx = enc_len-3 + else: + n_walls = jnp.round( + params.n_walls*enc[0]/self.n_tiles + ).astype(jnp.uint32) + if params.first_wall_pos_sets_budget: + # So 0-padding does not override pos=0 + wall_pos_idx = jnp.flip(enc[:params.n_walls]) + enc_len = n_walls + 2 # [wall_pos] + len((goal, agent)) + else: + wall_pos_idx = jnp.flip(enc[1:params.n_walls+1]) + # [wall_pos] + len((n_walls, goal, agent)) + enc_len = n_walls + 3 + # Positions are relative to n_walls when n_walls is variable. + agent_pos_idx = enc_len-1 + goal_pos_idx = enc_len-2 + + # Get agent + goal info (set agent/goal pos 1-step out of range if they are not yet placed) + goal_placed = state.time > jnp.array([goal_pos_idx], dtype=jnp.uint32) + goal_pos = \ + goal_placed*jnp.array([enc[goal_pos_idx] % w, enc[goal_pos_idx]//w], dtype=jnp.uint32) \ + + (~goal_placed)*jnp.array([w, h], dtype=jnp.uint32) + + agent_placed = state.time > jnp.array( + [agent_pos_idx], dtype=jnp.uint32) + agent_pos = \ + agent_placed*jnp.array([enc[agent_pos_idx] % w, enc[agent_pos_idx]//w], dtype=jnp.uint32) \ + + (~agent_placed)*jnp.array([w, h], dtype=jnp.uint32) + + agent_dir_idx = jnp.floor((4*enc[-1]/self.n_tiles)).astype(jnp.uint8) + + # Make wall map + wall_start_time = jnp.logical_and( # 1 if explicitly predict # blocks, else 0 + not params.fixed_n_wall_steps, + not params.first_wall_pos_sets_budget + ).astype(jnp.uint32) + wall_map = jnp.zeros(h*w, dtype=jnp.bool_) + wall_values = jnp.arange( + params.n_walls) + wall_start_time < jnp.minimum(state.time, n_walls + wall_start_time) + wall_values = jnp.flip(wall_values) + wall_map = wall_map.at[wall_pos_idx].set(wall_values) + + # Zero out walls where agent and goal reside + agent_mask = agent_placed * \ + (~(jnp.arange(h*w) == + state.encoding[agent_pos_idx])) + ~agent_placed*wall_map + goal_mask = goal_placed * \ + (~(jnp.arange(h*w) == + state.encoding[goal_pos_idx])) + ~goal_placed*wall_map + wall_map = wall_map*agent_mask*goal_mask + wall_map = wall_map.reshape(h, w) + + return EnvInstance( + agent_pos=agent_pos, + agent_dir_idx=agent_dir_idx, + goal_pos=goal_pos, + wall_map=wall_map + ) + + def is_terminal(self, state: EnvState) -> bool: + done_steps = state.time >= self.max_episode_steps() + return jnp.logical_or(done_steps, state.terminal) + + def _get_post_terminal_obs(self, state: EnvState): + dtype = jnp.float32 if self.params.normalize_obs else jnp.uint8 + image = jnp.zeros(( + self.params.height+2, self.params.width+2, 3), dtype=dtype + ) + + return OrderedDict(dict( + image=image, + time=state.time, + noise=jnp.zeros(self.params.noise_dim, dtype=jnp.float32), + )) + + def get_obs(self, state: EnvState): + instance = self.get_env_instance(jax.random.PRNGKey(0), state) + + image = make_maze_map( + self.params, + instance.wall_map, + instance.goal_pos, + instance.agent_pos, + instance.agent_dir_idx, + pad_obs=False + ) + + if self.params.normalize_obs: + image = image/10.0 + + return OrderedDict(dict( + image=image, + time=state.time, + )) + + @property + def default_params(self): + return EnvParams() + + @property + def name(self) -> str: + """Environment name.""" + return "UEDMaze" + + @property + def num_actions(self) -> int: + """Number of actions possible in environment.""" + return len(self.action_set) + + def action_space(self) -> spaces.Discrete: + """Action space of the environment.""" + params = self.params + return spaces.Discrete( + params.height*params.width, + dtype=jnp.uint32 + ) + + def observation_space(self) -> spaces.Dict: + """Observation space of the environment.""" + params = self.params + max_episode_steps = self.max_episode_steps() + spaces_dict = { + 'image': spaces.Box(0, 255, (params.height+2, params.width+2, 3)), + 'time': spaces.Discrete(max_episode_steps), + } + if self.params.noise_dim > 0: + spaces_dict.update({ + 'noise': spaces.Box(0, 1, (self.params.noise_dim,)) + }) + return spaces.Dict(spaces_dict) + + def state_space(self) -> spaces.Dict: + """State space of the environment.""" + params = self.params + encoding_dim = self._get_encoding_dim() + max_episode_steps = self.max_episode_steps() + h = params.height + w = params.width + return spaces.Dict({ + 'encoding': spaces.Box(0, 255, (encoding_dim,)), + 'time': spaces.Discrete(max_episode_steps), + "terminal": spaces.Discrete(2), + }) + + def _get_encoding_dim(self) -> int: + encoding_dim = self.max_episode_steps() + if not self.params.set_agent_dir: + encoding_dim += 1 # max steps is 1 less than full encoding dim + + return encoding_dim + + def max_episode_steps(self) -> int: + if self.params.fixed_n_wall_steps \ + or self.params.first_wall_pos_sets_budget: + max_episode_steps = self.params.n_walls + 2 + else: + max_episode_steps = self.params.n_walls + 3 + + if self.params.set_agent_dir: + max_episode_steps += 1 + + return max_episode_steps + + +if hasattr(__loader__, 'name'): + module_path = __loader__.name +elif hasattr(__loader__, 'fullname'): + module_path = __loader__.fullname + +register_ued(env_id='Maze', entry_point=module_path + ':UEDMaze') diff --git a/src/minimax/envs/overcooked_proc/__init__.py b/src/minimax/envs/overcooked_proc/__init__.py new file mode 100644 index 0000000..43e30c5 --- /dev/null +++ b/src/minimax/envs/overcooked_proc/__init__.py @@ -0,0 +1,14 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from .overcooked import Overcooked +from .overcooked_ued import UEDOvercooked +from .overcooked_ood import * + +from .overcooked_comparators import * +from .overcooked_mutators import * \ No newline at end of file diff --git a/src/minimax/envs/overcooked_proc/common.py b/src/minimax/envs/overcooked_proc/common.py new file mode 100644 index 0000000..973dcb0 --- /dev/null +++ b/src/minimax/envs/overcooked_proc/common.py @@ -0,0 +1,207 @@ +# Edited from JaxMarl: https://github.com/FLAIROx/JaxMARL/tree/main/jaxmarl/environments/overcooked + +from flax import struct +import chex +import numpy as np +import jax.numpy as jnp +import jax + + +OBJECT_TO_INDEX = { + "unseen": 0, + "empty": 1, + "wall": 2, + "onion": 3, + "onion_pile": 4, + "plate": 5, + "plate_pile": 6, + "goal": 7, + "pot": 8, + "dish": 9, + "agent": 10, +} + + +COLORS = { + 'red': np.array([255, 0, 0]), + 'green': np.array([0, 255, 0]), + 'blue': np.array([0, 0, 255]), + 'purple': np.array([112, 39, 195]), + 'yellow': np.array([255, 255, 0]), + 'grey': np.array([100, 100, 100]), + 'white': np.array([255, 255, 255]), + 'black': np.array([25, 25, 25]), + 'orange': np.array([230, 180, 0]), +} + + +COLOR_TO_INDEX = { + 'red': 0, + 'green': 1, + 'blue': 2, + 'purple': 3, + 'yellow': 4, + 'grey': 5, + 'white': 6, + 'black': 7, + 'orange': 8, +} + + +OBJECT_INDEX_TO_VEC = jnp.array([ + jnp.array([OBJECT_TO_INDEX['unseen'], 0, 0], dtype=jnp.uint8), + jnp.array([OBJECT_TO_INDEX['empty'], 0, 0], dtype=jnp.uint8), + jnp.array([OBJECT_TO_INDEX['wall'], COLOR_TO_INDEX['grey'], 0], + dtype=jnp.uint8), + jnp.array([OBJECT_TO_INDEX['onion'], + COLOR_TO_INDEX["yellow"], 0], dtype=jnp.uint8), + jnp.array([OBJECT_TO_INDEX['onion_pile'], + COLOR_TO_INDEX["yellow"], 0], dtype=jnp.uint8), + jnp.array([OBJECT_TO_INDEX['plate'], + COLOR_TO_INDEX["white"], 0], dtype=jnp.uint8), + jnp.array([OBJECT_TO_INDEX['plate_pile'], + COLOR_TO_INDEX["white"], 0], dtype=jnp.uint8), + jnp.array([OBJECT_TO_INDEX['goal'], COLOR_TO_INDEX['green'], 0], + dtype=jnp.uint8), + jnp.array([OBJECT_TO_INDEX['pot'], COLOR_TO_INDEX['black'], 0], + dtype=jnp.uint8), + jnp.array([OBJECT_TO_INDEX['dish'], COLOR_TO_INDEX["white"], 0], + dtype=jnp.uint8), + jnp.array([OBJECT_TO_INDEX['agent'], COLOR_TO_INDEX['red'], 0], + dtype=jnp.uint8), # Default color and direction +]) + + +# Map of agent direction indices to vectors +DIR_TO_VEC = jnp.array([ + # Pointing right (positive X) + # (1, 0), # right + # (0, 1), # down + # (-1, 0), # left + # (0, -1), # up + (0, -1), # NORTH + (0, 1), # SOUTH + (1, 0), # EAST + (-1, 0), # WEST +], dtype=jnp.int8) + + +@struct.dataclass +class EnvInstance: + agent_pos: chex.Array + agent_dir_idx: chex.Array + agent_inv: chex.Array + goal_pos: chex.Array + pot_pos: chex.Array + onion_pile_pos: chex.Array + plate_pile_pos: chex.Array + wall_map: chex.Array + + +def make_overcooked_map( + wall_map, + goal_pos, + agent_pos, + agent_dir_idx, + plate_pile_pos, + onion_pile_pos, + pot_pos, + pot_status, + onion_pos, + plate_pos, + dish_pos, + pad_obs=True, + num_agents=2, + agent_view_size=5): + # Expand maze map to H x W x C + empty = jnp.array([OBJECT_TO_INDEX['empty'], 0, 0], dtype=jnp.uint8) + wall = jnp.array( + [OBJECT_TO_INDEX['wall'], COLOR_TO_INDEX['grey'], 0], dtype=jnp.uint8) + maze_map = jnp.array(jnp.expand_dims(wall_map, -1), dtype=jnp.uint8) + maze_map = jnp.where(maze_map > 0, wall, empty) + + # Add agents + def _get_agent_updates(agent_dir_idx, agent_pos, agent_idx): + agent = jnp.array([OBJECT_TO_INDEX['agent'], COLOR_TO_INDEX['red'] + + agent_idx*2, agent_dir_idx], dtype=jnp.uint8) + agent_x, agent_y = agent_pos + return agent_x, agent_y, agent + + agent_x_vec, agent_y_vec, agent_vec = jax.vmap(_get_agent_updates, in_axes=( + 0, 0, 0))(agent_dir_idx, agent_pos, jnp.arange(num_agents)) + maze_map = maze_map.at[agent_y_vec, agent_x_vec, :].set(agent_vec) + + # Add goals + goal = jnp.array( + [OBJECT_TO_INDEX['goal'], COLOR_TO_INDEX['green'], 0], dtype=jnp.uint8) + + def set_based_on_position_mask(maze_map, pos_mask, obj): + pos_expanded = jnp.repeat( + jnp.expand_dims(pos_mask, axis=-1), 3, axis=-1) + obj_maze_map = pos_expanded * jnp.tile(obj, (*pos_mask.shape, 1)) + maze_map = maze_map * \ + jnp.logical_not(pos_expanded) + obj_maze_map * pos_expanded + return maze_map + + maze_map = set_based_on_position_mask(maze_map, goal_pos, goal) + + # Add onions + onion_pile = jnp.array( + [OBJECT_TO_INDEX['onion_pile'], COLOR_TO_INDEX["yellow"], 0], dtype=jnp.uint8) + maze_map = set_based_on_position_mask(maze_map, onion_pile_pos, onion_pile) + + # Add plates + plate_pile = jnp.array( + [OBJECT_TO_INDEX['plate_pile'], COLOR_TO_INDEX["white"], 0], dtype=jnp.uint8) + maze_map = set_based_on_position_mask(maze_map, plate_pile_pos, plate_pile) + + pot_obj = jnp.array( + [OBJECT_TO_INDEX['pot'], COLOR_TO_INDEX["black"], 0], dtype=jnp.uint8) + pot_status = pot_status.reshape(pot_pos.shape) + pot_status = jnp.concatenate((jnp.zeros( + (*pot_status.shape, 2), dtype=jnp.uint8), pot_status[:, :, jnp.newaxis]), axis=-1) + pos_expanded = jnp.repeat(jnp.expand_dims(pot_pos, axis=-1), 3, axis=-1) + obj_maze_map = pos_expanded * \ + jnp.tile(pot_obj, (*pot_pos.shape, 1)) + pot_status + maze_map = maze_map * \ + jnp.logical_not(pos_expanded) + obj_maze_map * pos_expanded + + onion = jnp.array( + [OBJECT_TO_INDEX['onion'], COLOR_TO_INDEX["yellow"], 0], dtype=jnp.uint8) + maze_map = set_based_on_position_mask(maze_map, onion_pos, onion) + + plate = jnp.array( + [OBJECT_TO_INDEX['plate'], COLOR_TO_INDEX["white"], 0], dtype=jnp.uint8) + maze_map = set_based_on_position_mask(maze_map, plate_pos, plate) + + dish = jnp.array( + [OBJECT_TO_INDEX['dish'], COLOR_TO_INDEX["white"], 0], dtype=jnp.uint8) + maze_map = set_based_on_position_mask(maze_map, dish_pos, dish) + + # Add observation padding + if pad_obs: + padding = agent_view_size-1 + else: + padding = 1 + + maze_map_padded = jnp.tile(wall.reshape( + (1, 1, *empty.shape)), (maze_map.shape[0]+2*padding, maze_map.shape[1]+2*padding, 1)) + maze_map_padded = maze_map_padded.at[ + padding:-padding, padding:-padding, :].set(maze_map) + + # Add surrounding walls + wall_start = padding-1 # start index for walls + wall_end_y = maze_map_padded.shape[0] - wall_start - 1 + wall_end_x = maze_map_padded.shape[1] - wall_start - 1 + maze_map_padded = maze_map_padded.at[wall_start, + wall_start:wall_end_x+1, :].set(wall) # top + maze_map_padded = maze_map_padded.at[wall_end_y, + wall_start:wall_end_x+1, :].set(wall) # bottom + # left + maze_map_padded = maze_map_padded.at[wall_start:wall_end_y + + 1, wall_start, :].set(wall) + # right + maze_map_padded = maze_map_padded.at[wall_start:wall_end_y + + 1, wall_end_x, :].set(wall) + + return maze_map_padded diff --git a/src/minimax/envs/overcooked_proc/interactive.py b/src/minimax/envs/overcooked_proc/interactive.py new file mode 100644 index 0000000..1094f90 --- /dev/null +++ b/src/minimax/envs/overcooked_proc/interactive.py @@ -0,0 +1,290 @@ +# Edited from JaxMarl: https://github.com/FLAIROx/JaxMARL/tree/main/jaxmarl/environments/overcooked + +import argparse +from functools import partial + +import jax +import jax.numpy as jnp + + +from jaxmarl.environments.overcooked.overcooked import Overcooked +from jaxmarl.environments.overcooked.layouts import overcooked_layouts as layouts + + +def redraw(state, obs, extras): + extras['viz'].render(extras['agent_view_size'], state, highlight=False) + + +def reset(key, env, extras): + key, subkey = jax.random.split(extras['rng']) + obs, state = extras['jit_reset'](subkey) + + extras['rng'] = key + extras['obs'] = obs + extras['state'] = state + + redraw(state, obs, extras) + + +def step(env, action, extras): + key, subkey = jax.random.split(extras['rng']) + + actions = {"agent_0": jnp.array(action), "agent_1": jnp.array(action)} + print("Actions : ", actions) + obs, state, reward, done, info = jax.jit( + env.step_env)(subkey, extras['state'], actions) + extras['obs'] = obs + extras['state'] = state + print( + f"t={state.time}: reward={reward['agent_0']}, agent_dir={state.agent_dir_idx}, agent_inv={state.agent_inv}, done = {done['__all__']}") + + if extras["debug"]: + layers = [f"player_{i}_loc" for i in range(2)] + layers.extend( + [f"player_{i // 4}_orientation_{i % 4}" for i in range(8)]) + layers.extend([ + "pot_loc", + "counter_loc", + "onion_disp_loc", + "tomato_disp_loc", + "plate_disp_loc", + "serve_loc", + "onions_in_pot", + "tomatoes_in_pot", + "onions_in_soup", + "tomatoes_in_soup", + "soup_cook_time_remaining", + "soup_done", + "plates", + "onions", + "tomatoes", + "urgency" + ]) + print("obs_shape: ", obs["agent_0"].shape) + print("OBS: \n", obs["agent_0"]) + debug_obs = jnp.transpose(obs["agent_0"], (2, 0, 1)) + for i, layer in enumerate(layers): + print(layer) + print(debug_obs[i]) + # print(f"agent obs =\n {obs}") + + if done["__all__"] or (jnp.array([action, action]) == Actions.done).any(): + key, subkey = jax.random.split(subkey) + reset(subkey, env, extras) + else: + redraw(state, obs, extras) + + extras['rng'] = key + + +def key_handler(env, extras, event): + print('pressed', event.key) + + if event.key == 'escape': + window.close() + return + + if event.key == 'backspace': + extras['jit_reset']((env, extras)) + return + + if event.key == 'left': + step(env, Actions.left, extras) + return + if event.key == 'right': + step(env, Actions.right, extras) + return + if event.key == 'up': + step(env, Actions.forward, extras) + return + + # Spacebar + if event.key == ' ': + step(env, Actions.toggle, extras) + return + if event.key == '[': + step(env, Actions.pickup, extras) + return + if event.key == ']': + step(env, Actions.drop, extras) + return + + if event.key == 'enter': + step(env, Actions.done, extras) + return + + +def key_handler_overcooked(env, extras, event): + print('pressed', event.key) + + if event.key == 'escape': + window.close() + return + if event.key == 'backspace': + extras['jit_reset']((env, extras)) + return + + if event.key == 'left': + step(env, Actions.left, extras) + return + if event.key == 'right': + step(env, Actions.right, extras) + return + if event.key == 'up': + # step(env, Actions.forward, extras) + step(env, Actions.up, extras) + return + if event.key == 'down': + step(env, Actions.down, extras) + return + + # Spacebar + if event.key == ' ': + step(env, Actions.interact, extras) + return + if event.key == 'tab': + step(env, Actions.stay, extras) + return + if event.key == 'enter': + step(env, Actions.done, extras) + return + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + "--env", + type=str, + help="Environment name", + default="Overcooked" + ) + parser.add_argument( + "--layout", + type=str, + help="Overcooked layout", + default="cramped_room" + ) + parser.add_argument( + '--random_reset', + default=False, + help="Reset to random state", + action='store_true' + ) + parser.add_argument( + "--seed", + type=int, + help="random seed to generate the environment with", + default=0 + ) + parser.add_argument( + '--render_agent_view', + default=False, + help="draw the agent sees (partially observable view)", + action='store_true' + ) + # parser.add_argument( + # '--height', + # default=13, + # type=int, + # help="height", + # ) + # parser.add_argument( + # '--width', + # default=13, + # type=int, + # help="width", + # ) + # parser.add_argument( + # '--n_walls', + # default=50, + # type=int, + # help="Number of walls", + # ) + # parser.add_argument( + # '--agent_view_size', + # default=5, + # type=int, + # help="Number of walls", + # ) + parser.add_argument( + '--debug', + default=False, + help="Debug mode", + action='store_true' + ) + args = parser.parse_args() + + # if args.env == "Maze": + # env = Maze( + # height=13, + # width=13, + # n_walls=25, + # see_agent=True, + # ) + # from jaxmarl.gridworld.grid_viz import GridVisualizer as Visualizer + # from jaxmarl.gridworld.maze import Actions + # + # params = env.params + # + # elif args.env == "MAMaze": + # env = MAMaze( + # height=13, + # width=13, + # n_walls=25, + # see_agent=True, + # n_agents=2 + # ) + # from jaxmarl.gridworld.grid_viz import GridVisualizer as Visualizer + # from jaxmarl.gridworld.maze import Actions + # + # params = env.params + + if args.env == "Overcooked": + if len(args.layout) > 0: + layout = layouts[args.layout] + env = Overcooked( + layout=layout, + random_reset=args.random_reset + ) + else: + print("You must provide a layout.") + from jaxmarl.viz.overcooked_visualizer import OvercookedVisualizer as Visualizer + from jaxmarl.environments.overcooked.overcooked import Actions + + viz = Visualizer() + obs_viz = None + obs_viz2 = None + if args.render_agent_view: + obs_viz = Visualizer() + if args.env == "MAMaze" or "Overcooked": + obs_viz2 = Visualizer() + + with jax.disable_jit(False): + jit_reset = jax.jit(env.reset) + # jit_reset = env.reset_env + key = jax.random.PRNGKey(args.seed) + key, subkey = jax.random.split(key) + o0, s0 = jit_reset(subkey) + viz.render(env.agent_view_size, s0, highlight=False) + + key, subkey = jax.random.split(key) + extras = { + 'rng': subkey, + 'state': s0, + 'obs': o0, + 'viz': viz, + 'obs_viz': obs_viz, + 'obs_viz2': obs_viz2, + 'jit_reset': jit_reset, + 'agent_view_size': env.agent_view_size, + 'env': args.env, + 'debug': args.debug + } + + if args.env == "Overcooked": + viz.window.reg_key_handler( + partial(key_handler_overcooked, env, extras)) + viz.show(block=True) + else: + viz.window.reg_key_handler(partial(key_handler, env, extras)) + viz.show(block=True) diff --git a/src/minimax/envs/overcooked_proc/layouts.py b/src/minimax/envs/overcooked_proc/layouts.py new file mode 100644 index 0000000..78ef5b0 --- /dev/null +++ b/src/minimax/envs/overcooked_proc/layouts.py @@ -0,0 +1,556 @@ +# Edited from JaxMarl: https://github.com/FLAIROx/JaxMARL/tree/main/jaxmarl/environments/overcooked + + +import json +import jax.numpy as jnp +from flax.core.frozen_dict import FrozenDict + +cramped_room = { + "height": 4, + "width": 5, + "wall_idx": jnp.array([0, 1, 2, 3, 4, + 5, 9, + 10, 14, + 15, 16, 17, 18, 19]), + "agent_idx": jnp.array([6, 8]), + "goal_idx": jnp.array([18]), + "plate_pile_idx": jnp.array([16]), + "onion_pile_idx": jnp.array([5, 9]), + "pot_idx": jnp.array([2]) +} +asymm_advantages = { + "height": 5, + "width": 9, + "wall_idx": jnp.array([0, 1, 2, 3, 4, 5, 6, 7, 8, + 9, 11, 12, 13, 14, 15, 17, + 18, 22, 26, + 27, 31, 35, + 36, 37, 38, 39, 40, 41, 42, 43, 44]), + "agent_idx": jnp.array([29, 32]), + "goal_idx": jnp.array([12, 17]), + "plate_pile_idx": jnp.array([39, 41]), + "onion_pile_idx": jnp.array([9, 14]), + "pot_idx": jnp.array([22, 31]) +} +coord_ring = { + "height": 5, + "width": 5, + "wall_idx": jnp.array([0, 1, 2, 3, 4, + 5, 9, + 10, 12, 14, + 15, 19, + 20, 21, 22, 23, 24]), + "agent_idx": jnp.array([7, 11]), + "goal_idx": jnp.array([22]), + "plate_pile_idx": jnp.array([10]), + "onion_pile_idx": jnp.array([15, 21]), + "pot_idx": jnp.array([3, 9]) +} +forced_coord = { + "height": 5, + "width": 5, + "wall_idx": jnp.array([0, 1, 2, 3, 4, + 5, 7, 9, + 10, 12, 14, + 15, 17, 19, + 20, 21, 22, 23, 24]), + "agent_idx": jnp.array([11, 8]), + "goal_idx": jnp.array([23]), + "onion_pile_idx": jnp.array([5, 10]), + "plate_pile_idx": jnp.array([15]), + "pot_idx": jnp.array([3, 9]) +} + +# Example of layout provided as a grid +counter_circuit_grid = """ +WWWPPWWW +W A W +B WWWW X +W AW +WWWOOWWW +""" + +asymm_advantages_6_9 = """ +WWWWWWWWW +O WXWOW X +W P A W +WA P W +WWWBWBWWW +WWWWWWWWW +""" + +counter_circuit_6_9 = """ +WWWPPWWWW +W A WW +B WWWW XW +W AWW +WWWOOWWWW +WWWWWWWWW +""" + +forced_coord_6_9 = """ +WWWPWWWWW +OAWAPWWWW +O W WWWWW +B W WWWWW +WWWXWWWWW +WWWWWWWWW +""" + +cramped_room_6_9 = """ +WWPWWWWWW +OAA OWWWW +W WWWWW +WBWXWWWWW +WWWWWWWWW +WWWWWWWWW +""" + +coord_ring_6_9 = """ +WWWPWWWWW +WA APWWWW +B W WWWWW +O WWWWW +WOXWWWWWW +WWWWWWWWW +""" + +quad_6_9 = """ +WWWWWWWWW +WWPA WWW +WWB AWWW +WWWOXOWWW +WWWWWWWWW +WWWWWWWWW +""" + +quad_6_9_1 = """ +WWWPWWWWW +WWBA WWW +WWO AWWW +WWWXOWWWW +WWWWWWWWW +WWWWWWWWW +""" + +quad_6_9_2 = """ +WWWBPWWWW +WWOA WWW +WWX AWWW +WWWOWWWWW +WWWWWWWWW +WWWWWWWWW +""" + +quad_6_9_3 = """ +WWWOBPWWW +WWXA WWW +WWO AWWW +WWWWWWWWW +WWWWWWWWW +WWWWWWWWW +""" + +quad_6_9_4 = """ +WWWXOBWWW +WWOA PWW +WWW AWWW +WWWWWWWWW +WWWWWWWWW +WWWWWWWWW +""" + +quad_6_9_5 = """ +WWWOXOWWW +WWWA BWW +WWW APWW +WWWWWWWWW +WWWWWWWWW +WWWWWWWWW +""" + +quad_6_9_6 = """ +WWWWOXWWW +WWWA OWW +WWW ABWW +WWWWWPWWW +WWWWWWWWW +WWWWWWWWW +""" + +quad_6_9_7 = """ +WWWWWOWWW +WWWA XWW +WWW AOWW +WWWWPBWWW +WWWWWWWWW +WWWWWWWWW +""" + +quad_6_9_8 = """ +WWWWWWWWW +WWWA OWW +WWW AXWW +WWWPBOWWW +WWWWWWWWW +WWWWWWWWW +""" + +quad_6_9_9 = """ +WWWWWWWWW +WWWA WWW +WWP AOWW +WWWBOXWWW +WWWWWWWWW +WWWWWWWWW +""" + +quad_6_9T = """ +WWWOXOWWW +WWW APWW +WWWA BWW +WWWWWWWWW +WWWWWWWWW +WWWWWWWWW +""" + +quad_6_9M = """ +WWWOXOWWW +WWB APWW +WWPA BWW +WWWOXOWWW +WWWWWWWWW +WWWWWWWWW +""" + +asymm_advantages_10_15 = """ +WWWWWWWWWWWWWWW +O WXWOW XWWWWWW +W P A WWWWWWW +WA P WWWWWWW +WWWBWBWWWWWWWWW +WWWWWWWWWWWWWWW +WWWWWWWWWWWWWWW +WWWWWWWWWWWWWWW +WWWWWWWWWWWWWWW +WWWWWWWWWWWWWWW +""" + +asymm_advantages_B_10_15 = """ +WWWWWWWWWWWWWWW +O WXWOW XWWWWWW +W P A WWWWWWW +W W WWWWWWW +WA P WWWWWWW +WWWBWBWWWWWWWWW +WWWWWWWWWWWWWWW +WWWWWWWWWWWWWWW +WWWWWWWWWWWWWWW +WWWWWWWWWWWWWWW +""" + +asymm_advantages_M_10_15 = """ +WWWWWWWWWWWWWWW +O WXWOW XWWWW +W P A WWWWW +WA W WWWWW +W P WWWWW +WWWBWWWBWWWWWWW +WWWWWWWWWWWWWWW +WWWWWWWWWWWWWWW +WWWWWWWWWWWWWWW +WWWWWWWWWWWWWWW +""" + +asymm_advantages_L_10_15 = """ +WWWWWWWWWWWWWWW +O WXWOW XWW +W P A WWW +WA W WWW +W W WWW +W P WWW +WWWBWWWWWBWWWWW +WWWWWWWWWWWWWWW +WWWWWWWWWWWWWWW +WWWWWWWWWWWWWWW +""" + +asymm_advantages_m = """ +WWWWWWWWW +O WOWXW X +W P A W +WA P W +WWWBWBWWW +""" + +asymm_advantages_m1 = """ +WWWWWWWWW +X WXWOW O +W P A W +WA P W +WWWBWBWWW +""" + +asymm_advantages_m2 = """ +WWWWWWWWW +O WXWOW X +WA P A W +W P W +WWWBWBWWW +""" + +asymm_advantages_m3 = """ +WWWWWWWWW +O WXWOW X +W P AW +WA P W +WWWBWBWWW +""" + +asymm_advantages_m4 = """ +WWWWWWWWW +O WXWOW X +W P A W +WA B W +WWWPWBWWW +""" + +asymm_advantages_m5 = """ +WWWWWWWWW +O WXWOW X +W B A W +WA P W +WWWPWBWWW +""" + +asymm_advantages_m6 = """ +WWWWWWWWW +O WXWOW X +W P A W +WA P W +WWBWWBWWW +""" + +asymm_advantages_m7 = """ +WWWWWWWWW +O WXWOW X +W P A W +WA P W +WWBWWWBWW +""" + +asymm_advantages_m8 = """ +WWWWWWWWW +O WXWOW X +W P A W +WA P W +WBWWWWBWW +""" + + +def layout_grid_to_onehot_dict(grid): + """Assumes `grid` is string representation of the layout, with 1 line per row, and the following symbols: + W: wall + A: agent + X: goal + B: plate (bowl) pile + O: onion pile + P: pot location + ' ' (space) : empty cell + """ + + rows = grid.split('\n') + + if len(rows[0]) == 0: + rows = rows[1:] + if len(rows[-1]) == 0: + rows = rows[:-1] + + keys = ["wall_idx", "agent_idx", "goal_idx", + "plate_pile_idx", "onion_pile_idx", "pot_idx", "empty_table_idx"] + symbol_to_key = {"W": "wall_idx", + "A": "agent_idx", + "X": "goal_idx", + "B": "plate_pile_idx", + "O": "onion_pile_idx", + "P": "pot_idx"} + + layout_dict = {key: [] for key in keys} + layout_dict["height"] = len(rows) + layout_dict["width"] = len(rows[0]) + width = len(rows[0]) + + for i, row in enumerate(rows): + for j, obj in enumerate(row): + idx = width * i + j + # if obj in symbol_to_key.keys(): + # # Add object + # layout_dict[symbol_to_key[obj]].append(idx) + + if obj == "A": + # Agent + layout_dict["agent_idx"].append(idx) + + if obj == "X": + # Goal + layout_dict["goal_idx"].append(1) + else: + layout_dict["goal_idx"].append(0) + + if obj == "B": + # Plate pile + layout_dict["plate_pile_idx"].append(1) + else: + layout_dict["plate_pile_idx"].append(0) + + if obj == "O": + # Onion pile + layout_dict["onion_pile_idx"].append(1) + else: + layout_dict["onion_pile_idx"].append(0) + + if obj == "P": + # Pot location + layout_dict["pot_idx"].append(1) + else: + layout_dict["pot_idx"].append(0) + + if obj in ["X", "B", "O", "P", "W"]: + # These objects are also walls technically + layout_dict["wall_idx"].append(1) + else: + layout_dict["wall_idx"].append(0) + + if obj == "W": + # Goal + layout_dict["empty_table_idx"].append(1) + else: + layout_dict["empty_table_idx"].append(0) + # elif obj == " ": + # # Empty cell + # continue + + for key in layout_dict.keys(): + # Transform lists to arrays + layout_dict[key] = jnp.array(layout_dict[key], dtype=jnp.uint8) + + return FrozenDict(layout_dict) + + +def layout_grid_to_dict(grid): + """Assumes `grid` is string representation of the layout, with 1 line per row, and the following symbols: + W: wall + A: agent + X: goal + B: plate (bowl) pile + O: onion pile + P: pot location + ' ' (space) : empty cell + """ + + rows = grid.split('\n') + + if len(rows[0]) == 0: + rows = rows[1:] + if len(rows[-1]) == 0: + rows = rows[:-1] + + keys = ["wall_idx", "agent_idx", "goal_idx", + "plate_pile_idx", "onion_pile_idx", "pot_idx"] + symbol_to_key = {"W": "wall_idx", + "A": "agent_idx", + "X": "goal_idx", + "B": "plate_pile_idx", + "O": "onion_pile_idx", + "P": "pot_idx"} + + layout_dict = {key: [] for key in keys} + layout_dict["height"] = len(rows) + layout_dict["width"] = len(rows[0]) + width = len(rows[0]) + + for i, row in enumerate(rows): + for j, obj in enumerate(row): + idx = width * i + j + if obj in symbol_to_key.keys(): + # Add object + layout_dict[symbol_to_key[obj]].append(idx) + if obj in ["X", "B", "O", "P"]: + # These objects are also walls technically + layout_dict["wall_idx"].append(idx) + elif obj == " ": + # Empty cell + continue + + for key in symbol_to_key.values(): + # Transform lists to arrays + layout_dict[key] = jnp.array(layout_dict[key]) + + return FrozenDict(layout_dict) + + +# load all_lvl_strs.json +# all_lvls_strs = json.load( +# open("jaxmarl/environments/overcooked/10x15_all_lvl_strs.json", "r")) +# gan_mlp_layouts = all_lvls_strs["gan_milp"] + +# automatic_overcooked_layouts_10_15 = { +# str(k): layout_grid_to_dict(v) for k, v in enumerate(gan_mlp_layouts) +# } + +# all_lvls_strs = json.load( +# open("jaxmarl/environments/overcooked/6x9_all_lvl_strs.json", "r")) +# gan_mlp_layouts = all_lvls_strs["gan_milp"] + +# automatic_overcooked_layouts_6_9 = { +# str(k): layout_grid_to_dict(v) for k, v in enumerate(gan_mlp_layouts) +# } + +# all_lvls_strs = json.load( +# open("jaxmarl/environments/overcooked/6x9_all_lvl_strs_simple.json", "r")) + +# gan_mlp_layouts = all_lvls_strs["gan_milp"] + +# automatic_overcooked_layouts_6_9_simple = { +# str(k): layout_grid_to_dict(v) for k, v in enumerate(gan_mlp_layouts) +# } + + +overcooked_layouts = { + "cramped_room": FrozenDict(cramped_room), + "asymm_advantages": FrozenDict(asymm_advantages), + "asymm_advantages_m": FrozenDict(asymm_advantages), + "coord_ring": FrozenDict(coord_ring), + "forced_coord": FrozenDict(forced_coord), + "counter_circuit": layout_grid_to_dict(counter_circuit_grid), + "asymm_advantages_6_9": layout_grid_to_dict(asymm_advantages_6_9), + "counter_circuit_6_9": layout_grid_to_dict(counter_circuit_6_9), + "forced_coord_6_9": layout_grid_to_dict(forced_coord_6_9), + "cramped_room_6_9": layout_grid_to_dict(cramped_room_6_9), + "coord_ring_6_9": layout_grid_to_dict(coord_ring_6_9), + "quad_6_9": layout_grid_to_dict(quad_6_9), + "quad_6_9_1": layout_grid_to_dict(quad_6_9_1), + "quad_6_9_2": layout_grid_to_dict(quad_6_9_2), + "quad_6_9_3": layout_grid_to_dict(quad_6_9_3), + "quad_6_9_4": layout_grid_to_dict(quad_6_9_4), + "quad_6_9_5": layout_grid_to_dict(quad_6_9_5), + "quad_6_9_6": layout_grid_to_dict(quad_6_9_6), + "quad_6_9_7": layout_grid_to_dict(quad_6_9_7), + "quad_6_9_8": layout_grid_to_dict(quad_6_9_8), + "quad_6_9_9": layout_grid_to_dict(quad_6_9_9), + "quad_6_9T": layout_grid_to_dict(quad_6_9T), + "quad_6_9M": layout_grid_to_dict(quad_6_9M), + "asymm_advantages_m": layout_grid_to_dict(asymm_advantages_m), + "asymm_advantages_m1": layout_grid_to_dict(asymm_advantages_m1), + "asymm_advantages_m2": layout_grid_to_dict(asymm_advantages_m2), + "asymm_advantages_m3": layout_grid_to_dict(asymm_advantages_m3), + "asymm_advantages_m4": layout_grid_to_dict(asymm_advantages_m4), + "asymm_advantages_m5": layout_grid_to_dict(asymm_advantages_m5), + "asymm_advantages_m6": layout_grid_to_dict(asymm_advantages_m6), + "asymm_advantages_m7": layout_grid_to_dict(asymm_advantages_m7), + "asymm_advantages_m8": layout_grid_to_dict(asymm_advantages_m8), + "asymm_advantages_10_15": layout_grid_to_dict(asymm_advantages_10_15), + "asymm_advantages_B_10_15": layout_grid_to_dict(asymm_advantages_B_10_15), + "asymm_advantages_M_10_15": layout_grid_to_dict(asymm_advantages_M_10_15), + "asymm_advantages_L_10_15": layout_grid_to_dict(asymm_advantages_L_10_15), +} diff --git a/src/minimax/envs/overcooked_proc/overcooked.py b/src/minimax/envs/overcooked_proc/overcooked.py new file mode 100644 index 0000000..316dfb9 --- /dev/null +++ b/src/minimax/envs/overcooked_proc/overcooked.py @@ -0,0 +1,1389 @@ +# Edited from JaxMarl: https://github.com/FLAIROx/JaxMARL/tree/main/jaxmarl/environments/overcooked + +from enum import IntEnum +from hmac import new +import os +import random +import time + +import imageio +import numpy as np +import jax +import jax.numpy as jnp +from jax import lax +from typing import Tuple, Dict +import chex +from flax import struct +from sympy import jn + +from minimax.envs import environment, spaces +from minimax.envs.registration import register +from minimax.envs.overcooked_proc.layouts import layout_grid_to_onehot_dict +import minimax.util.graph as _graph_util +from minimax.envs.viz.overcooked_visualizer import OvercookedVisualizer + +from .common import EnvInstance, make_overcooked_map + +asymm_advantages_6_9 = """ +WWWWWWWWW +O WXWOW X +W P A W +WA P W +WWWBWBWWW +WWWWWWWWW +""" + +counter_circuit_6_9 = """ +WWWPPWWWW +W A WW +B WWWW XW +W AWW +WWWOOWWWW +WWWWWWWWW +""" + +forced_coord_6_9 = """ +WWWPWWWWW +OAWAPWWWW +O W WWWWW +B W WWWWW +WWWXWWWWW +WWWWWWWWW +""" + +cramped_room_6_9 = """ +WWPWWWWWW +OAA OWWWW +W WWWWW +WBWXWWWWW +WWWWWWWWW +WWWWWWWWW +""" + +coord_ring_6_9 = """ +WWWPWWWWW +WA APWWWW +B W WWWWW +O WWWWW +WOXWWWWWW +WWWWWWWWW +""" + +forced_coord_5_5 = """ +WWWPW +OAWAP +O W W +B W W +WWWXW +""" + +cramped_room_5_5 = """ +WWPWW +OAA O +W W +WBWXW +WWWWW +""" + +coord_ring_5_5 = """ +WWWPW +WA AP +B W W +O W +WOXWW +""" + + +class Actions(IntEnum): + # Turn left, turn right, move forward + right = 0 + down = 1 + left = 2 + up = 3 + stay = 4 + interact = 5 + done = 6 + + +@struct.dataclass +class EnvState: + agent_pos: chex.Array + agent_dir: chex.Array + agent_dir_idx: chex.Array + agent_inv: chex.Array + goal_pos: chex.Array + pot_pos: chex.Array + wall_map: chex.Array + maze_map: chex.Array + bowl_pile_pos: chex.Array + onion_pile_pos: chex.Array + time: int + terminal: bool + +@struct.dataclass +class EnvParams: + height: int = 6 + width: int = 9 + h_min: int = 4 + w_min: int = 4 + n_walls: int = 5 + agent_view_size: int = 5 + replace_wall_pos: bool = False + normalize_obs: bool = False + sample_n_walls: bool = False # Sample n_walls uniformly in [0, n_walls] + max_steps: int = 400 + singleton_seed: int = -1 + max_episode_steps: int = 400 + + +# Pot status indicated by an integer, which ranges from 23 to 0 +POT_EMPTY_STATUS = 23 # 22 = 1 onion in pot; 21 = 2 onions in pot; 20 = 3 onions in pot +# 3 onions. Below this status, pot is cooking, and status acts like a countdown timer. +POT_FULL_STATUS = 20 +POT_READY_STATUS = 0 +# A pot has at most 3 onions. A soup contains exactly 3 onions. +MAX_ONIONS_IN_POT = 3 + +URGENCY_CUTOFF = 40 # When this many time steps remain, the urgency layer is flipped on +DELIVERY_REWARD = 20 + + +SHAPED_REWARD = { + "PLACEMENT_IN_POT_REW": 0, + "DISH_PICKUP_REWARD": 3, + "SOUP_PICKUP_REWARD": 5, + "PICKUP_TOMATO_REWARD": 0, + "DISH_DISP_DISTANCE_REW": 0, + "POT_DISTANCE_REW": 0, + "SOUP_DISTANCE_REW": 0, +} + +OBJECT_TO_INDEX = { + "unseen": 0, + "empty": 1, + "wall": 2, + "onion": 3, + "onion_pile": 4, + "plate": 5, + "plate_pile": 6, + "goal": 7, + "pot": 8, + "dish": 9, + "agent": 10, +} + + +COLORS = { + 'red': np.array([255, 0, 0]), + 'green': np.array([0, 255, 0]), + 'blue': np.array([0, 0, 255]), + 'purple': np.array([112, 39, 195]), + 'yellow': np.array([255, 255, 0]), + 'grey': np.array([100, 100, 100]), + 'white': np.array([255, 255, 255]), + 'black': np.array([25, 25, 25]), + 'orange': np.array([230, 180, 0]), +} + + +COLOR_TO_INDEX = { + 'red': 0, + 'green': 1, + 'blue': 2, + 'purple': 3, + 'yellow': 4, + 'grey': 5, + 'white': 6, + 'black': 7, + 'orange': 8, +} + +LAYOUT_STR_TO_LAYOUT = { + "asymm_advantages_6_9": asymm_advantages_6_9, + "counter_circuit_6_9": counter_circuit_6_9, + "forced_coord_6_9": forced_coord_6_9, + "cramped_room_6_9": cramped_room_6_9, + "coord_ring_6_9": coord_ring_6_9, + "coord_ring_5_5": coord_ring_5_5, + "forced_coord_5_5": forced_coord_5_5, + "cramped_room_5_5": cramped_room_5_5, +} + + +OBJECT_INDEX_TO_VEC = jnp.array([ + jnp.array([OBJECT_TO_INDEX['unseen'], 0, 0], dtype=jnp.uint8), + jnp.array([OBJECT_TO_INDEX['empty'], 0, 0], dtype=jnp.uint8), + jnp.array([OBJECT_TO_INDEX['wall'], COLOR_TO_INDEX['grey'], 0], + dtype=jnp.uint8), + jnp.array([OBJECT_TO_INDEX['onion'], + COLOR_TO_INDEX["yellow"], 0], dtype=jnp.uint8), + jnp.array([OBJECT_TO_INDEX['onion_pile'], + COLOR_TO_INDEX["yellow"], 0], dtype=jnp.uint8), + jnp.array([OBJECT_TO_INDEX['plate'], + COLOR_TO_INDEX["white"], 0], dtype=jnp.uint8), + jnp.array([OBJECT_TO_INDEX['plate_pile'], + COLOR_TO_INDEX["white"], 0], dtype=jnp.uint8), + jnp.array([OBJECT_TO_INDEX['goal'], COLOR_TO_INDEX['green'], 0], + dtype=jnp.uint8), + jnp.array([OBJECT_TO_INDEX['pot'], COLOR_TO_INDEX['black'], 0], + dtype=jnp.uint8), + jnp.array([OBJECT_TO_INDEX['dish'], COLOR_TO_INDEX["white"], 0], + dtype=jnp.uint8), + jnp.array([OBJECT_TO_INDEX['agent'], COLOR_TO_INDEX['red'], 0], + dtype=jnp.uint8), # Default color and direction +]) + + +# Map of agent direction indices to vectors +DIR_TO_VEC = jnp.array([ + # Pointing right (positive X) + # (1, 0), # right + # (0, 1), # down + # (-1, 0), # left + # (0, -1), # up + (0, -1), # NORTH + (0, 1), # SOUTH + (1, 0), # EAST + (-1, 0), # WEST +], dtype=jnp.int8) + + +def _obtain_from_layout(key, layout, h, w, random_reset, num_agents): + all_pos = np.arange(np.prod([h, w]), dtype=jnp.uint32) + occupied_mask = layout.get("wall_idx") + # occupied_mask = jnp.zeros_like(all_pos) + # occupied_mask = occupied_mask.at[wall_idx].set(1) + wall_map = occupied_mask.reshape(h, w).astype(jnp.bool_) + + # Reset agent position + dir + key, subkey = jax.random.split(key) + agent_idx = jax.random.choice(subkey, all_pos, shape=(num_agents,), + p=(~occupied_mask.astype(jnp.bool_)).astype(jnp.uint8), replace=False) + # agent_idx = jnp.zeros_like(occupied_mask).at[agent_idx].set(1) + + # Replace with fixed layout if applicable. Also randomize if agent position not provided + # agent_idx = random_reset*agent_idx + \ # (1-random_reset)* + agent_idx = layout.get("agent_idx", agent_idx) + agent_pos = jnp.array([agent_idx % w, agent_idx // w], + dtype=jnp.uint32).transpose() # dim = n_agents x 2 + # agent_pos = agent_idx.reshape(h,w) + occupied_mask = occupied_mask.at[agent_idx].set(1) + + key, subkey = jax.random.split(key) + agent_dir_idx = jax.random.choice(subkey, jnp.arange( + len(DIR_TO_VEC), dtype=jnp.int32), shape=(num_agents,)) + agent_dir = DIR_TO_VEC.at[agent_dir_idx].get() # dim = n_agents x 2 + + empty_table_mask = jnp.zeros_like(all_pos) + empty_table_mask = jnp.array(layout.get("empty_table_idx")).reshape(h, w) + + goal_idx = layout.get("goal_idx") + goal_pos = goal_idx.reshape(h, w) + empty_table_mask = empty_table_mask.at[goal_idx].set(0) + + onion_pile_idx = layout.get("onion_pile_idx") + onion_pile_pos = onion_pile_idx.reshape(h, w) + empty_table_mask = empty_table_mask.at[onion_pile_idx].set(0) + + plate_pile_idx = layout.get("plate_pile_idx") + plate_pile_pos = plate_pile_idx.reshape(h, w) + empty_table_mask = empty_table_mask.at[plate_pile_idx].set(0) + + pot_idx = layout.get("pot_idx") + pot_pos = pot_idx.reshape(h, w) + empty_table_mask = empty_table_mask.at[pot_idx].set(0) + + key, subkey = jax.random.split(key) + pot_status = pot_idx * \ + jax.random.randint(subkey, (pot_idx.shape[0],), 0, 24, dtype=jnp.uint8) + pot_status = pot_status * random_reset + \ + (1-random_reset) * jnp.ones((pot_idx.shape[0]), dtype=jnp.uint8) * 23 + return wall_map, goal_pos, agent_pos, agent_dir, agent_dir_idx, plate_pile_pos, onion_pile_pos, pot_pos, pot_status + + +class Overcooked(environment.Environment): + """Overcooked Procedural Multi-Agent""" + + def __init__( + self, + height: int, + width: int, + random_reset: bool = False, + n_walls=25, + agent_view_size=5, + replace_wall_pos=False, + max_steps=400, + normalize_obs=False, + sample_n_walls=False, + fix_to_single_layout=None, + dense_obs=False, + singleton_seed=-1 + ): + # Sets self.num_agents to 2 + super().__init__() + + self.num_agents = 2 + self.default_shaped_reward_coeff = 0.0 + # self.obs_shape = (agent_view_size, agent_view_size, 3) + # Observations given by 26 channels, most of which are boolean masks + # The idea is that we never create levels biger that this for zero padding. + self.width = width + self.height = height + self.num_features = 62 # Akin to the original Overcooked-AI + + # Hard coded. Only affects map padding -- not observations. + self.agent_view_size = 5 + self.agents = ["agent_0", "agent_1"] + # Fixes Resets to this layout instead to a random one. + # Mostly used for debugging. + # Example: "asymm_advantages_6_9" -> asymm_advantages_6_9 + self.fix_to_single_layout = fix_to_single_layout + self.dense_obs = dense_obs + + # Define the observation function + if dense_obs: # (62,) + self.get_obs = self.get_obs_dense + self.obs_shape = (self.num_features,) + else: # (h, w, 26,) + self.get_obs = self.get_obs_sparse + self.obs_shape = (self.width, self.height, 26) + + self.action_set = jnp.array([ + Actions.right, + Actions.down, + Actions.left, + Actions.up, + Actions.stay, + Actions.interact, + ]) + + self.random_reset = random_reset + self.max_steps = max_steps + + self.params = EnvParams( + height=height, + width=width, + n_walls=n_walls, + agent_view_size=agent_view_size, + replace_wall_pos=replace_wall_pos and not sample_n_walls, + max_steps=max_steps, + normalize_obs=normalize_obs, + sample_n_walls=sample_n_walls, + singleton_seed=-1, + max_episode_steps=max_steps, + ) + + def step_env( + self, + key: chex.PRNGKey, + state: EnvState, + actions: Dict[str, chex.Array], + ) -> Tuple[Dict[str, chex.Array], EnvState, Dict[str, float], Dict[str, bool], Dict]: + """Perform single timestep state transition.""" + + acts = self.action_set.take(indices=jnp.array( + [actions["agent_0"], actions["agent_1"]])) + + state, reward, shaped_reward_alice, shaped_reward_bob = self.step_agents( + key, state, acts) + + state = state.replace(time=state.time + 1) + + done = self.is_terminal(state) + state = state.replace(terminal=done) + + obs = self.get_obs(state) + rewards = { + "agent_0": reward, + "agent_1": reward + } + dones = {"agent_0": done, "agent_1": done, "__all__": done} + + return ( + lax.stop_gradient(obs), + lax.stop_gradient(state), + rewards, + dones, + { + "sparse_reward": jnp.array([reward, reward]), + "shaped_reward": jnp.array([shaped_reward_alice, shaped_reward_bob]), + }, + ) + + def sample_random_layout(self, key: chex.PRNGKey, h, w) -> Dict[str, chex.Array]: + """Samples a random layout that might or might not be playable. + """ + params = self.params + + all_pos = np.arange(np.prod([h, w]), dtype=jnp.uint8) + + key, walls_key, nwalls_key, goal_key, plate_pile_key, onion_pile_key, pot_key, agpos_key = jax.random.split( + key, 8) + wall_idx = jax.random.choice( + walls_key, all_pos, + shape=(params.n_walls,), + replace=params.replace_wall_pos) + + if params.sample_n_walls: + sampled_n_walls = jax.random.randint( + nwalls_key, (), minval=0, maxval=params.n_walls) + sample_wall_mask = jnp.arange(params.n_walls) < sampled_n_walls + dummy_wall_idx = wall_idx.at[0].get().repeat(params.n_walls) + wall_idx = jax.lax.select( + sample_wall_mask, + wall_idx, + dummy_wall_idx + ) + + walls = jnp.zeros_like(all_pos, dtype=jnp.uint8) + walls = walls.at[wall_idx].set(1) + walls = walls.reshape(h, w) + walls = walls.at[:, 0].set(1) + walls = walls.at[0, :].set(1) + walls = walls.at[:, -1].set(1) + walls = walls.at[-1, :].set(1).reshape(-1) + + occupied_obj_mask = jnp.zeros_like(all_pos, dtype=jnp.uint8) + wall_mask = occupied_obj_mask + walls + + # Do not want corners to have objects + occupied_obj_mask = occupied_obj_mask.reshape(h, w) + occupied_obj_mask = occupied_obj_mask.at[0, 0].set(1) + occupied_obj_mask = occupied_obj_mask.at[-1, -1].set(1) + occupied_obj_mask = occupied_obj_mask.at[0, -1].set(1) + occupied_obj_mask = occupied_obj_mask.at[-1, 0].set(1) + occupied_obj_mask = occupied_obj_mask.reshape(-1) + + def add_1_or_2_items(key, all_pos, wall_mask, occupied_obj_mask): + # occupied_obj_mask is only objects on tables so we can do: + possible_positions = wall_mask - occupied_obj_mask + obj_mask = jnp.zeros_like(all_pos, dtype=jnp.uint8) + key, subkey1, subkey2, subkey3 = jax.random.split(key, 4) + item_idx_1 = jax.random.choice(subkey1, all_pos, shape=( + 1,), p=(possible_positions.astype(jnp.bool_)).astype(jnp.uint8)) + + and_2 = jax.random.bernoulli(subkey2, 0.5) + + item_idx_2 = jax.random.choice(subkey3, all_pos, shape=( + 1,), p=(possible_positions.astype(jnp.bool_)).astype(jnp.uint8)) + + obj_mask = obj_mask.at[item_idx_1].set(1) + + update_2 = jnp.logical_or( + obj_mask.at[item_idx_2].get(), and_2.astype(jnp.uint8)) + obj_mask = obj_mask.at[item_idx_2].set(update_2) + return obj_mask + + goal_pos = add_1_or_2_items( + goal_key, all_pos, wall_mask, occupied_obj_mask) + occupied_obj_mask = occupied_obj_mask + goal_pos + + plate_pile_pos = add_1_or_2_items( + plate_pile_key, all_pos, wall_mask, occupied_obj_mask) + occupied_obj_mask = occupied_obj_mask + plate_pile_pos + + onion_pile_pos = add_1_or_2_items( + onion_pile_key, all_pos, wall_mask, occupied_obj_mask) + occupied_obj_mask = occupied_obj_mask + onion_pile_pos + + pot_pos = add_1_or_2_items( + pot_key, all_pos, wall_mask, occupied_obj_mask) + occupied_obj_mask = occupied_obj_mask + pot_pos + + agent_idx = jax.random.choice(agpos_key, all_pos, shape=(2,), replace=False, p=( + ~wall_mask.astype(jnp.bool_)).astype(jnp.uint8)) + # occupied_mask = occupied_mask.at[agent_idx].set(2) + + layout = { + "height": self.height, + "width": self.width, + "wall_idx": walls, + "empty_table_idx": walls - occupied_obj_mask, + "agent_idx": agent_idx, + "goal_idx": goal_pos, + "plate_pile_idx": plate_pile_pos, + "onion_pile_idx": onion_pile_pos, + "pot_idx": pot_pos + } + return layout + + def reset_env( # NOTE: Has been renamed to fit minimax + self, + key: chex.PRNGKey, + ) -> Tuple[Dict[str, chex.Array], EnvState]: + """Reset environment state based on `self.random_reset` + + If True, everything is randomized, including agent inventories and positions, pot states and items on counters + If False, only resample agent orientations + + In both cases, the environment layout is determined by `self.layout` + """ + # Whether to fully randomize the start state + random_reset = self.random_reset + + h = self.height + w = self.width + num_agents = self.num_agents + + if self.fix_to_single_layout is None: + layout = self.sample_random_layout(key, h, w) + else: + layout = layout_grid_to_onehot_dict( + LAYOUT_STR_TO_LAYOUT[self.fix_to_single_layout]) + + wall_map, goal_pos, agent_pos, agent_dir, agent_dir_idx, plate_pile_pos, onion_pile_pos, pot_pos, pot_status\ + = _obtain_from_layout(key, layout, h, w, random_reset, num_agents) + + onion_pos = jnp.zeros((h, w), dtype=jnp.uint8) + plate_pos = jnp.zeros((h, w), dtype=jnp.uint8) + dish_pos = jnp.zeros((h, w), dtype=jnp.uint8) + + maze_map = make_overcooked_map( + wall_map, + goal_pos, + agent_pos, + agent_dir_idx, + plate_pile_pos, + onion_pile_pos, + pot_pos, + pot_status, + onion_pos, + plate_pos, + dish_pos, + pad_obs=True, + num_agents=self.num_agents, + agent_view_size=self.agent_view_size + ) + # Its to make padding static with respect to the jitted code later. + # Its static since we compute it in advance now. + padding = (maze_map.shape[0]-h) // 2 + + # agent inventory (empty by default, can be randomized) + key, subkey = jax.random.split(key) + possible_items = jnp.array([OBJECT_TO_INDEX['empty'], OBJECT_TO_INDEX['onion'], + OBJECT_TO_INDEX['plate'], OBJECT_TO_INDEX['dish']]) + random_agent_inv = jax.random.choice( + subkey, possible_items, shape=(num_agents,), replace=True) + agent_inv = random_reset * random_agent_inv + \ + (1-random_reset) * \ + jnp.array([OBJECT_TO_INDEX['empty'], OBJECT_TO_INDEX['empty']]) + + state = EnvState( + agent_pos=agent_pos, + agent_dir=agent_dir, + agent_dir_idx=agent_dir_idx, + agent_inv=agent_inv, + goal_pos=goal_pos, + pot_pos=pot_pos, + onion_pile_pos=onion_pile_pos, + bowl_pile_pos=plate_pile_pos, + wall_map=wall_map.astype(jnp.bool_), + maze_map=maze_map, + time=0, + terminal=False, + ) + + self.padding = padding + obs = self.get_obs(state) + + return lax.stop_gradient(obs), lax.stop_gradient(state) + + def get_obs_dense(self, state: EnvState) -> Dict[str, chex.Array]: + """ + Inspired by the original Overcooked-AI we also add a dense observation to the environment. + We use this to built the OvercookedUED light challange as it is significantly less sparse then the original observation. + + From their doc (https://github.com/HumanCompatibleAI/overcooked_ai/blob/cff884ccf5709658ee4cd489e63367200b4c86d6/src/overcooked_ai_py/mdp/overcooked_mdp.py#L2579): + Returns: + ordered_features (list[np.Array]): The ith element contains a player-centric featurized view for the ith player + + The encoding for player i is as follows: + + [player_i_features, other_player_features player_i_dist_to_other_players, player_i_position] + + player_{i}_features (length num_pots*10 + 24): + pi_orientation: length 4 one-hot-encoding of direction currently facing + pi_obj: length 4 one-hot-encoding of object currently being held (all 0s if no object held) + pi_wall_{j}: {0, 1} boolean value of whether player i has wall immediately in direction j + pi_closest_{onion|tomato|dish|soup|serving|empty_counter}: (dx, dy) where dx = x dist to item, dy = y dist to item. (0, 0) if item is currently held + pi_cloest_soup_n_{onions|tomatoes}: int value for number of this ingredient in closest soup + pi_closest_pot_{j}_exists: {0, 1} depending on whether jth closest pot found. If 0, then all other pot features are 0. Note: can + be 0 even if there are more than j pots on layout, if the pot is not reachable by player i + pi_closest_pot_{j}_{is_empty|is_full|is_cooking|is_ready}: {0, 1} depending on boolean value for jth closest pot + pi_closest_pot_{j}_{num_onions|num_tomatoes}: int value for number of this ingredient in jth closest pot + pi_closest_pot_{j}_cook_time: int value for seconds remaining on soup. -1 if no soup is cooking + pi_closest_pot_{j}: (dx, dy) to jth closest pot from player i location + + other_player_features (length (num_players - 1)*(num_pots*10 + 24)): + ordered concatenation of player_{j}_features for j != i + + player_i_dist_to_other_players (length (num_players - 1)*2): + [player_j.pos - player_i.pos for j != i] + + player_i_position (length 2) + """ + agent_dir = state.agent_dir + agent_inv = state.agent_inv + maze_map = state.maze_map + + w = self.width + h = self.height + + padding = 4 + maze_map = maze_map[padding:-padding, padding:-padding, :] + + def get_player_rep(player_idx: int, state): + + agent_pos = state.agent_pos[player_idx] + + # pi_orientation: length 4 one-hot-encoding of direction currently facing + pi_orientation = jnp.zeros((4)).at[state.agent_dir_idx].set(1) + + # pi_obj: length 3 one-hot-encoding of object currently being held (all 0s if no object held) + pi_obj = OBJECT_TO_INDEX["empty"] + (agent_inv[player_idx] == OBJECT_TO_INDEX["onion"]) * jnp.array([1, 0, 0], dtype=jnp.uint8)\ + + (agent_inv[player_idx] == OBJECT_TO_INDEX["plate"]) * jnp.array([0, 1, 0], dtype=jnp.uint8)\ + + (agent_inv[player_idx] == OBJECT_TO_INDEX["dish"] + ) * jnp.array([0, 0, 1], dtype=jnp.uint8) + + # pi_wall_{j}: {0, 1} boolean value of whether player i has wall immediately in direction j + fwd_pos_0 = agent_pos + DIR_TO_VEC[0] + is_wall_0 = state.wall_map.at[fwd_pos_0[1], fwd_pos_0[0]].get() + + fwd_pos_1 = agent_pos + DIR_TO_VEC[1] + is_wall_1 = state.wall_map.at[fwd_pos_1[1], fwd_pos_1[0]].get() + + fwd_pos_2 = agent_pos + DIR_TO_VEC[2] + is_wall_2 = state.wall_map.at[fwd_pos_2[1], fwd_pos_2[0]].get() + + fwd_pos_3 = agent_pos + DIR_TO_VEC[3] + is_wall_3 = state.wall_map.at[fwd_pos_3[1], fwd_pos_3[0]].get() + + pi_wall_j = jnp.array([is_wall_0, is_wall_1, is_wall_2, is_wall_3]) + + # pi_closest_{onion|dish|soup|serving|empty_counter}: (dx, dy) where dx = x dist to item, dy = y dist to item. (0, 0) if item is currently held + def find_closest_between_masks(agent_pos, object_map, name): + obj_idx = OBJECT_TO_INDEX[name] + padded_pos = jnp.argwhere( + object_map.T == obj_idx, size=2, # w*h, + fill_value=jnp.inf) + dist = padded_pos-agent_pos + abs_dist = jnp.abs(dist) + manhatten = abs_dist.sum(-1) + closest_idx = jnp.argmin(manhatten) + clostest_obj_pos = padded_pos[closest_idx] + dxdy_obj_ag_inf = dist[closest_idx] + dxdy_obj_ag = jnp.nan_to_num(dxdy_obj_ag_inf, nan=0, posinf=0) + return clostest_obj_pos.astype(jnp.uint8), dxdy_obj_ag + + object_map = maze_map[:, :, 0] + pos_closest_pot, pi_closest_pot = find_closest_between_masks( + agent_pos, object_map, "pot") + _, pi_closest_onion = find_closest_between_masks( + agent_pos, object_map, "onion") + _, pi_closest_plate = find_closest_between_masks( + agent_pos, object_map, "plate") + _, pi_closest_dish = find_closest_between_masks( + agent_pos, object_map, "dish") + _, pi_closest_goal = find_closest_between_masks( + agent_pos, object_map, "goal") + # If it has something on it its type is not wall -> i.e. walls are always empty + # empty_wall_map = (maze_map[:,:,0] == OBJECT_TO_INDEX["wall"]).astype(jnp.uint8) + _, pi_closest_wall = find_closest_between_masks( + agent_pos, object_map, "wall") + + # pi_cloest_soup_n_{onions}: int value for number of this ingredient in closest soup + # Not apllicable: We only have 3 onion soups + # pi_closest_pot_{j}_exists: {0, 1} depending on whether jth closest pot found. If 0, then all other pot features are 0. Note: can + # be 0 even if there are more than j pots on layout, if the pot is not reachable by player i + # pi_closest_pot_{j}_{is_empty|is_full|is_cooking|is_ready}: {0, 1} depending on boolean value for jth closest pot + # pi_closest_pot_{j}_{num_onions|num_tomatoes}: int value for number of this ingredient in jth closest pot + # pi_closest_pot_{j}_cook_time: int value for seconds remaining on soup. -1 if no soup is cooking + # pi_closest_pot_{j}: (dx, dy) to jth closest pot from player i location + closest_pot = maze_map.at[pos_closest_pot[1], + pos_closest_pot[0]].get() + + # agent_obj = maze_map.at[agent_pos[1], agent_pos[0]].get() + + path_len = _graph_util.shortest_path_len( + state.wall_map, agent_pos, pos_closest_pot) + + # pi_closest_pot_{j}_exists + pi_closest_pot_exists = path_len > 0 + pi_closest_pot_is_empty = ( + closest_pot[2] == 23) * pi_closest_pot_exists + pi_closest_pot_is_full = ( + jnp.logical_and(closest_pot[2] <= 20, closest_pot[2] > 0)) * pi_closest_pot_exists + pi_closest_pot_is_cooking = ( + jnp.logical_and(closest_pot[2] <= 19, closest_pot[2] > 0)) * pi_closest_pot_exists + pi_closest_pot_is_ready = ( + closest_pot[2] == 0) * pi_closest_pot_exists + pi_closest_pot_num_onions = ( + (closest_pot[2] <= 20)*3 + (closest_pot[2] == 21)*2 + (closest_pot[2] == 22)*1) * pi_closest_pot_exists + pi_closest_pot_cook_time = pi_closest_pot_is_cooking * \ + closest_pot[2] + + return jnp.hstack([ + pi_orientation, pi_obj, pi_wall_j, pi_closest_onion, pi_closest_plate, pi_closest_dish, + pi_closest_goal, pi_closest_wall, pi_closest_pot_exists, pi_closest_pot_is_empty, pi_closest_pot_is_full, + pi_closest_pot_is_cooking, pi_closest_pot_is_ready, pi_closest_pot_num_onions, pi_closest_pot_cook_time, pi_closest_pot + ]) + + agent_vec_0 = get_player_rep(0, state) + agent_vec_1 = get_player_rep(1, state) + + obs = { + 'agent_0': jnp.hstack([agent_vec_0, agent_vec_1, state.agent_pos[0, 1], state.agent_pos[0, 0]]), + 'agent_1': jnp.hstack([agent_vec_1, agent_vec_0, state.agent_pos[1, 1], state.agent_pos[1, 0]]) + } + return obs + + def get_obs_sparse(self, state: EnvState) -> Dict[str, chex.Array]: + """Return a full observation, of size(height x width x n_layers), where n_layers = 26. + Layers are of shape(height x width) and are binary(0/1) except where indicated otherwise. + The obs is very sparse(most elements are 0), which prob. contributes to generalization problems in Overcooked. + A v2 of this environment should have much more efficient observations, e.g. using item embeddings + + The list of channels is below. Agent-specific layers are ordered so that an agent perceives its layers first. + Env layers are the same (and in same order) for both agents. + + Agent positions: + 0. position of agent i(1 at agent loc, 0 otherwise) + 1. position of agent(1-i) + + Agent orientations: + 2-5. agent_{i}_orientation_0 to agent_{i}_orientation_3(layers are entirely zero except for the one orientation + layer that matches the agent orientation. That orientation has a single 1 at the agent coordinates.) + 6-9. agent_{i-1}_orientation_{dir} + + Static env positions(1 where object of type X is located, 0 otherwise.): + 10. pot locations + 11. counter locations(table) + 12. onion pile locations + 13. tomato pile locations(tomato layers are included for consistency, but this env does not support tomatoes) + 14. plate pile locations + 15. delivery locations(goal) + + Pot and soup specific layers. These are non-binary layers: + 16. number of onions in pot(0, 1, 2, 3) for elements corresponding to pot locations. Nonzero only for pots that + have NOT started cooking yet. When a pot starts cooking (or is ready), the corresponding element is set to 0 + 17. number of tomatoes in pot. + 18. number of onions in soup(0, 3) for elements corresponding to either a cooking/done pot or to a soup(dish) + ready to be served. This is a useless feature since all soups have exactly 3 onions, but it made sense in the + full Overcooked where recipes can be a mix of tomatoes and onions + 19. number of tomatoes in soup + 20. pot cooking time remaining. [19 -> 1] for pots that are cooking. 0 for pots that are not cooking or done + 21. soup done. (Binary) 1 for pots done cooking and for locations containing a soup(dish). O otherwise. + + Variable env layers(binary): + 22. plate locations + 23. onion locations + 24. tomato locations + + Urgency: + 25. Urgency. The entire layer is 1 there are 40 or fewer remaining time steps. 0 otherwise + """ + width = self.obs_shape[0] + height = self.obs_shape[1] + n_channels = self.obs_shape[2] + # NOTE: Original code here was: padding = (state.maze_map.shape[0]-height) // 2 + padding = 4 + # padding = state.padding # Must be somehow static + + maze_map = state.maze_map[padding:-padding, padding:-padding, 0] + soup_loc = jnp.array( + maze_map == OBJECT_TO_INDEX["dish"], dtype=jnp.uint8) + + pot_loc_layer = jnp.array( + maze_map == OBJECT_TO_INDEX["pot"], dtype=jnp.uint8) + pot_status = state.maze_map[padding:-padding, + padding: -padding, 2] * pot_loc_layer + onions_in_pot_layer = jnp.minimum(POT_EMPTY_STATUS - pot_status, MAX_ONIONS_IN_POT) * ( + pot_status >= POT_FULL_STATUS) # 0/1/2/3, as long as not cooking or not done + onions_in_soup_layer = jnp.minimum(POT_EMPTY_STATUS - pot_status, MAX_ONIONS_IN_POT) * (pot_status < POT_FULL_STATUS) \ + * pot_loc_layer + MAX_ONIONS_IN_POT * soup_loc # 0/3, as long as cooking or done + pot_cooking_time_layer = pot_status * \ + (pot_status < POT_FULL_STATUS) # Timer: 19 to 0 + # Ready soups, plated or not + soup_ready_layer = pot_loc_layer * \ + (pot_status == POT_READY_STATUS) + soup_loc + urgency_layer = jnp.ones(maze_map.shape, dtype=jnp.uint8) * \ + ((self.max_steps - state.time) < URGENCY_CUTOFF) + + agent_pos_layers = jnp.zeros((2, height, width), dtype=jnp.uint8) + agent_pos_layers = agent_pos_layers.at[0, + state.agent_pos[0, 1], state.agent_pos[0, 0]].set(1) + agent_pos_layers = agent_pos_layers.at[1, + state.agent_pos[1, 1], state.agent_pos[1, 0]].set(1) + + # Add agent inv: This works because loose items and agent cannot overlap + agent_inv_items = jnp.expand_dims( + state.agent_inv, (1, 2)) * agent_pos_layers + maze_map = jnp.where(jnp.sum(agent_pos_layers, 0), + agent_inv_items.sum(0), maze_map) + soup_ready_layer = soup_ready_layer + + (jnp.sum(agent_inv_items, 0) == + OBJECT_TO_INDEX["dish"]) * jnp.sum(agent_pos_layers, 0) + onions_in_soup_layer = onions_in_soup_layer \ + + (jnp.sum(agent_inv_items, 0) == + OBJECT_TO_INDEX["dish"]) * 3 * jnp.sum(agent_pos_layers, 0) + + env_layers = [ + # Channel 10 + jnp.array(maze_map == OBJECT_TO_INDEX["pot"], dtype=jnp.uint8), + jnp.array(maze_map == OBJECT_TO_INDEX["wall"], dtype=jnp.uint8), + jnp.array( + maze_map == OBJECT_TO_INDEX["onion_pile"], dtype=jnp.uint8), + # tomato pile + jnp.zeros(maze_map.shape, dtype=jnp.uint8), + jnp.array( + maze_map == OBJECT_TO_INDEX["plate_pile"], dtype=jnp.uint8), + # 15 + jnp.array(maze_map == OBJECT_TO_INDEX["goal"], dtype=jnp.uint8), + jnp.array(onions_in_pot_layer, dtype=jnp.uint8), + # tomatoes in pot + jnp.zeros(maze_map.shape, dtype=jnp.uint8), + jnp.array(onions_in_soup_layer, dtype=jnp.uint8), + # tomatoes in soup + jnp.zeros(maze_map.shape, dtype=jnp.uint8), + jnp.array(pot_cooking_time_layer, + dtype=jnp.uint8), # 20 + jnp.array(soup_ready_layer, dtype=jnp.uint8), + jnp.array(maze_map == OBJECT_TO_INDEX["plate"], dtype=jnp.uint8), + jnp.array(maze_map == OBJECT_TO_INDEX["onion"], dtype=jnp.uint8), + # tomatoes + jnp.zeros(maze_map.shape, dtype=jnp.uint8), + urgency_layer, # 25 + ] + + # Agent related layers + agent_direction_layers = jnp.zeros((8, height, width), dtype=jnp.uint8) + dir_layer_idx = state.agent_dir_idx+jnp.array([0, 4]) + agent_direction_layers = agent_direction_layers.at[dir_layer_idx, :, :].set( + agent_pos_layers) + + # Both agent see their layers first, then the other layer + alice_obs = jnp.zeros((n_channels, height, width), dtype=jnp.uint8) + alice_obs = alice_obs.at[0:2].set(agent_pos_layers) + + alice_obs = alice_obs.at[2:10].set(agent_direction_layers) + alice_obs = alice_obs.at[10:].set(jnp.stack(env_layers)) + + bob_obs = jnp.zeros((n_channels, height, width), dtype=jnp.uint8) + bob_obs = bob_obs.at[0].set( + agent_pos_layers[1]).at[1].set(agent_pos_layers[0]) + bob_obs = bob_obs.at[2:6].set(agent_direction_layers[4:]).at[6:10].set( + agent_direction_layers[0:4]) + bob_obs = bob_obs.at[10:].set(jnp.stack(env_layers)) + + # NOTE: Changed, was not inline with self.obs_shape: [self.width, self.height, 26] + alice_obs = jnp.transpose(alice_obs, (2, 1, 0)) + bob_obs = jnp.transpose(bob_obs, (2, 1, 0)) + return {"agent_0": alice_obs, "agent_1": bob_obs} + + def step_agents( + self, key: chex.PRNGKey, state: EnvState, action: chex.Array + ) -> Tuple[EnvState, float]: + + # Update agent position (forward action) + is_move_action = jnp.logical_and( + action != Actions.stay, action != Actions.interact) + is_move_action_transposed = jnp.expand_dims( + is_move_action, 0).transpose() # Necessary to broadcast correctly + + fwd_pos = jnp.minimum( + jnp.maximum(state.agent_pos + is_move_action_transposed * DIR_TO_VEC[jnp.minimum(action, 3)] + + ~is_move_action_transposed * state.agent_dir, 0), + jnp.array((self.width - 1, self.height - 1), dtype=jnp.uint32) + ) + + # Can't go past wall or goal + def _wall_or_goal(fwd_position, wall_map, goal_pos): + fwd_wall = wall_map.at[fwd_position[1], fwd_position[0]].get() + def goal_collision(pos, goal): return jnp.logical_and( + pos[0] == goal[0], pos[1] == goal[1]) + fwd_goal = jax.vmap(goal_collision, in_axes=( + None, 0))(fwd_position, goal_pos) + # fwd_goal = jnp.logical_and(fwd_position[0] == goal_pos[0], fwd_position[1] == goal_pos[1]) + fwd_goal = jnp.any(fwd_goal) + return fwd_wall, fwd_goal + + fwd_pos_has_wall, fwd_pos_has_goal = jax.vmap(_wall_or_goal, in_axes=( + 0, None, None))(fwd_pos, state.wall_map, state.goal_pos) + + fwd_pos_blocked = jnp.logical_or( + fwd_pos_has_wall, fwd_pos_has_goal).reshape((self.num_agents, 1)) + + bounced = jnp.logical_or(fwd_pos_blocked, ~is_move_action_transposed) + + # Agents can't overlap + # Hardcoded for 2 agents (call them Alice and Bob) + agent_pos_prev = jnp.array(state.agent_pos) + fwd_pos = (bounced * state.agent_pos + (~bounced) + * fwd_pos).astype(jnp.uint32) + collision = jnp.all(fwd_pos[0] == fwd_pos[1]) + + # No collision = No movement. This matches original Overcooked env. + alice_pos = jnp.where( + collision, + state.agent_pos[0], # collision and Bob bounced + fwd_pos[0], + ) + bob_pos = jnp.where( + collision, + # collision and Alice bounced + state.agent_pos[1], + fwd_pos[1], + ) + + # Prevent swapping places (i.e. passing through each other) + swap_places = jnp.logical_and( + jnp.all(fwd_pos[0] == state.agent_pos[1]), + jnp.all(fwd_pos[1] == state.agent_pos[0]), + ) + alice_pos = jnp.where( + ~collision * swap_places, + state.agent_pos[0], + alice_pos + ) + bob_pos = jnp.where( + ~collision * swap_places, + state.agent_pos[1], + bob_pos + ) + + fwd_pos = fwd_pos.at[0].set(alice_pos) + fwd_pos = fwd_pos.at[1].set(bob_pos) + agent_pos = fwd_pos.astype(jnp.uint32) + + # Update agent direction + agent_dir_idx = ~is_move_action * state.agent_dir_idx + is_move_action * action + agent_dir = DIR_TO_VEC[agent_dir_idx] + + # Handle interacts. Agent 1 first, agent 2 second, no collision handling. + # This matches the original Overcooked + fwd_pos = state.agent_pos + state.agent_dir + maze_map = state.maze_map + is_interact_action = (action == Actions.interact) + + # Compute the effect of interact first, then apply it if needed + candidate_maze_map, alice_inv, alice_reward, alice_shaped_reward = self.process_interact( + maze_map, state, fwd_pos[0], state.agent_inv[0], state.agent_inv[1]) + alice_interact = is_interact_action[0] + bob_interact = is_interact_action[1] + + maze_map = jax.lax.select(alice_interact, + candidate_maze_map, + maze_map) + alice_inv = jax.lax.select(alice_interact, + alice_inv, + state.agent_inv[0]) + alice_reward = jax.lax.select(alice_interact, alice_reward, 0.) + alice_shaped_reward = jax.lax.select( + alice_interact, alice_shaped_reward, 0.) + + candidate_maze_map, bob_inv, bob_reward, bob_shaped_reward = self.process_interact( + maze_map, state, fwd_pos[1], state.agent_inv[1], state.agent_inv[0]) + maze_map = jax.lax.select(bob_interact, + candidate_maze_map, + maze_map) + bob_inv = jax.lax.select(bob_interact, + bob_inv, + state.agent_inv[1]) + bob_reward = jax.lax.select(bob_interact, bob_reward, 0.) + bob_shaped_reward = jax.lax.select(bob_interact, bob_shaped_reward, 0.) + + agent_inv = jnp.array([alice_inv, bob_inv]) + + # Update agent component in maze_map + def _get_agent_updates(agent_dir_idx, agent_pos, agent_pos_prev, agent_idx): + agent = jnp.array([OBJECT_TO_INDEX['agent'], COLOR_TO_INDEX['red'] + + agent_idx*2, agent_dir_idx], dtype=jnp.uint8) + agent_x_prev, agent_y_prev = agent_pos_prev + agent_x, agent_y = agent_pos + return agent_x, agent_y, agent_x_prev, agent_y_prev, agent + + vec_update = jax.vmap(_get_agent_updates, in_axes=(0, 0, 0, 0)) + agent_x, agent_y, agent_x_prev, agent_y_prev, agent_vec = vec_update( + agent_dir_idx, agent_pos, agent_pos_prev, jnp.arange(self.num_agents)) + empty = jnp.array([OBJECT_TO_INDEX['empty'], 0, 0], dtype=jnp.uint8) + + # Compute padding, added automatically by map maker function + # height = self.obs_shape[1] + padding = 4 # (state.maze_map.shape[0] - height) // 2 + + maze_map = maze_map.at[padding + agent_y_prev, + padding + agent_x_prev, :].set(empty) + maze_map = maze_map.at[padding + agent_y, + padding + agent_x, :].set(agent_vec) + + # Update pot cooking status + def _cook_pots(maze_map, pot_pos): + pot_pos_padded = jnp.zeros( + (maze_map.shape[0], maze_map.shape[1]), dtype=jnp.uint8 + ) + pot_pos_padded = pot_pos_padded.at[ + padding:-padding, padding:-padding].set(pot_pos) + is_cooking = jnp.array( + maze_map[:, :, -1] * pot_pos_padded <= POT_FULL_STATUS, dtype=jnp.uint8) * pot_pos_padded + not_done = jnp.array( + maze_map[:, :, -1] * pot_pos_padded > POT_READY_STATUS, dtype=jnp.uint8) * pot_pos_padded + pot_status_is_cooking_not_done = is_cooking * \ + not_done * (maze_map[:, :, -1] - 1) * pot_pos_padded + pot_status_is_not_cooking = jnp.logical_not( + is_cooking) * (maze_map[:, :, -1]) * pot_pos_padded # defaults to zero if done pot_status + pot_status = pot_status_is_cooking_not_done + pot_status_is_not_cooking + + pot_status_map = pot_pos_padded * pot_status + \ + jnp.logical_not(pot_pos_padded) * maze_map[:, :, -1] + pot_status_map = jnp.concatenate( + (jnp.zeros((*pot_status_map.shape, 2), dtype=jnp.uint8), pot_status_map[:, :, jnp.newaxis]), axis=-1) + + pot_pos_3 = jnp.concatenate( + (jnp.zeros((pot_status_map.shape[0], pot_status_map.shape[1], 2), dtype=jnp.uint8), pot_pos_padded[:, :, jnp.newaxis]), axis=-1) + + maze_map = maze_map * (1-pot_pos_3) + pot_status_map * pot_pos_3 + + return maze_map # pot.at[-1].set(pot_status) + + maze_map = _cook_pots(maze_map, state.pot_pos) + + reward = alice_reward + bob_reward + # shaped_reward = alice_shaped_reward + bob_shaped_reward + + return ( + state.replace( + agent_pos=agent_pos, + agent_dir_idx=agent_dir_idx, + agent_dir=agent_dir, + agent_inv=agent_inv, + maze_map=maze_map, + terminal=False), + reward, + alice_shaped_reward, + bob_shaped_reward, + ) + + def process_interact( + self, + maze_map: chex.Array, + state: EnvState, + fwd_pos: chex.Array, + inventory: chex.Array, + other_inventory: chex.Array): + """Assume agent took interact actions. Result depends on what agent is facing and what it is holding.""" + + wall_map = state.wall_map + height = self.height # self.obs_shape[1] + # padding = (maze_map.shape[0] - height) // 2 + padding = 4 + + # Get object in front of agent (on the "table") + maze_object_on_table = maze_map.at[padding + + fwd_pos[1], padding + fwd_pos[0]].get() + object_on_table = maze_object_on_table[0] # Simple index + + # Booleans depending on what the object is + object_is_pile = jnp.logical_or( + object_on_table == OBJECT_TO_INDEX["plate_pile"], object_on_table == OBJECT_TO_INDEX["onion_pile"]) + object_is_pot = jnp.array(object_on_table == OBJECT_TO_INDEX["pot"]) + object_is_goal = jnp.array(object_on_table == OBJECT_TO_INDEX["goal"]) + object_is_agent = jnp.array( + object_on_table == OBJECT_TO_INDEX["agent"]) + object_is_pickable = jnp.logical_or( + jnp.logical_or( + object_on_table == OBJECT_TO_INDEX["plate"], object_on_table == OBJECT_TO_INDEX["onion"]), + object_on_table == OBJECT_TO_INDEX["dish"] + ) + # Whether the object in front is counter space that the agent can drop on. + is_table = jnp.logical_and( + wall_map.at[fwd_pos[1], fwd_pos[0]].get(), ~object_is_pot) + + table_is_empty = jnp.logical_or( + object_on_table == OBJECT_TO_INDEX["wall"], object_on_table == OBJECT_TO_INDEX["empty"]) + + # Pot status (used if the object is a pot) + pot_status = maze_object_on_table[-1] + + # Get inventory object, and related booleans + inv_is_empty = jnp.array(inventory == OBJECT_TO_INDEX["empty"]) + object_in_inv = inventory + holding_onion = jnp.array(object_in_inv == OBJECT_TO_INDEX["onion"]) + holding_plate = jnp.array(object_in_inv == OBJECT_TO_INDEX["plate"]) + holding_dish = jnp.array(object_in_inv == OBJECT_TO_INDEX["dish"]) + + # Interactions with pot. 3 cases: add onion if missing, collect soup if ready, do nothing otherwise + case_1 = (pot_status > POT_FULL_STATUS) * holding_onion * object_is_pot + case_2 = (pot_status == POT_READY_STATUS) * \ + holding_plate * object_is_pot + case_3 = (pot_status > POT_READY_STATUS) * \ + (pot_status <= POT_FULL_STATUS) * object_is_pot + else_case = ~case_1 * ~case_2 * ~case_3 + + # Update pot status and object in inventory + new_pot_status = \ + case_1 * (pot_status - 1) \ + + case_2 * POT_EMPTY_STATUS \ + + case_3 * pot_status \ + + else_case * pot_status + new_object_in_inv = \ + case_1 * OBJECT_TO_INDEX["empty"] \ + + case_2 * OBJECT_TO_INDEX["dish"] \ + + case_3 * object_in_inv \ + + else_case * object_in_inv + + # Interactions with onion/plate piles and objects on counter + # Pickup if: table, not empty, room in inv & object is not something unpickable (e.g. pot or goal) + successful_pickup = is_table * ~table_is_empty * inv_is_empty * \ + jnp.logical_or(object_is_pile, object_is_pickable) + successful_drop = is_table * table_is_empty * ~inv_is_empty + successful_delivery = is_table * object_is_goal * holding_dish + no_effect = jnp.logical_and(jnp.logical_and( + ~successful_pickup, ~successful_drop), ~successful_delivery) + + # Update object on table + new_object_on_table = \ + no_effect * object_on_table \ + + successful_delivery * object_on_table \ + + successful_pickup * object_is_pile * object_on_table \ + + successful_pickup * object_is_pickable * OBJECT_TO_INDEX["wall"] \ + + successful_drop * object_in_inv + + # Update object in inventory + new_object_in_inv = \ + no_effect * new_object_in_inv \ + + successful_delivery * OBJECT_TO_INDEX["empty"] \ + + successful_pickup * object_is_pickable * object_on_table \ + + successful_pickup * (object_on_table == OBJECT_TO_INDEX["plate_pile"]) * OBJECT_TO_INDEX["plate"] \ + + successful_pickup * (object_on_table == OBJECT_TO_INDEX["onion_pile"]) * OBJECT_TO_INDEX["onion"] \ + + successful_drop * OBJECT_TO_INDEX["empty"] + + # Apply inventory update + inventory = new_object_in_inv + + # Apply changes to maze + new_maze_object_on_table = \ + object_is_pot * OBJECT_INDEX_TO_VEC[new_object_on_table].at[-1].set(new_pot_status) \ + + ~object_is_pot * ~object_is_agent * OBJECT_INDEX_TO_VEC[new_object_on_table] \ + + object_is_agent * maze_object_on_table + + maze_map = maze_map.at[padding + fwd_pos[1], + padding + fwd_pos[0], :].set(new_maze_object_on_table) + + # Reward of 20 for a soup delivery + reward = jnp.array(successful_delivery, dtype=float)*DELIVERY_REWARD + + no_plate_on_counter = ( + (maze_map[padding:-padding, padding:-padding, 0] * wall_map) == OBJECT_TO_INDEX["plate"]).sum() == 0 + num_pots = state.pot_pos.sum() + # (maze_map[padding:-padding, padding:-padding, -1].at[state.pot_pos].get() <= POT_FULL_STATUS).sum() + num_pots_cooking = ( + (maze_map[padding:-padding, padding:-padding, -1] <= POT_FULL_STATUS) * state.pot_pos).sum() + # (maze_map[padding:-padding, padding:-padding, -1].at[state.pot_pos].get() > POT_FULL_STATUS).sum() + num_pots_not_started = ( + (maze_map[padding:-padding, padding:-padding, -1] > POT_FULL_STATUS) * state.pot_pos).sum() + num_pots_ready = num_pots - num_pots_cooking - num_pots_not_started + pot_left_over_for_plate = (num_pots_cooking + num_pots_ready - + 1 * (other_inventory == OBJECT_TO_INDEX["dish"])) > 0 + # As in orignal work: adding onion 3, getting a bowl while cooking 5, pickung up a soup 5 + shaped_reward_c1 = (new_object_in_inv == OBJECT_TO_INDEX["empty"]) * ( + object_in_inv == OBJECT_TO_INDEX["onion"]) * case_1 * 3.0 + shaped_reward_c2 = (new_object_in_inv == OBJECT_TO_INDEX["plate"]) * (object_on_table == OBJECT_TO_INDEX["plate_pile"]) * \ + successful_pickup * no_plate_on_counter * pot_left_over_for_plate * 5.0 + shaped_reward_c3 = (new_object_in_inv == OBJECT_TO_INDEX["dish"]) * ( + object_in_inv == OBJECT_TO_INDEX["plate"]) * case_2 * 5.0 + + # jax.debug.print("no_plate {a}: {s}", a=no_plate_on_counter, s=shaped_reward_c2) + shaped_reward = shaped_reward_c1 + shaped_reward_c2 + shaped_reward_c3 + return maze_map, inventory, reward, shaped_reward + + def is_terminal(self, state: EnvState) -> bool: + """Check whether state is terminal.""" + done_steps = state.time >= self.max_steps + return done_steps | state.terminal + + def get_eval_solved_rate_fn(self): + def _fn(ep_stats): + return ep_stats['return'] > 20 # More than one soup delivered + + return _fn + + @property + def name(self) -> str: + """Environment name.""" + return "Overcooked" + + @property + def num_actions(self) -> int: + """Number of actions possible in environment.""" + return len(self.action_set) + + def action_space(self, agent_id="") -> spaces.Discrete: + """Action space of the environment. Agent_id not used since action_space is uniform for all agents""" + return spaces.Discrete( + len(self.action_set), + dtype=jnp.uint8 + ) + + def observation_space(self) -> spaces.Box: + """Observation space of the environment.""" + return spaces.Box(0, 255, self.obs_shape) + + def max_episode_steps(self) -> int: + return self.params.max_episode_steps + + def set_env_instance( + self, + encoding: EnvInstance): + """ + Instance is encoded as a PyTree containing the following fields: + agent_pos, agent_dir, goal_pos, wall_map + """ + params = self.params + agent_pos = encoding.agent_pos + agent_dir_idx = encoding.agent_dir_idx + h, w = encoding.wall_map.shape + agent_dir = DIR_TO_VEC.at[agent_dir_idx].get() + goal_pos = encoding.goal_pos + wall_map = encoding.wall_map + agent_inv = encoding.agent_inv + pot_pos = encoding.pot_pos + + onion_pile_pos = encoding.onion_pile_pos + plate_pile_pos = encoding.plate_pile_pos + + onion_pos = jnp.zeros((h, w), dtype=jnp.uint8) + plate_pos = jnp.zeros((h, w), dtype=jnp.uint8) + dish_pos = jnp.zeros((h, w), dtype=jnp.uint8) + + pot_status = jnp.ones( + (encoding.wall_map.reshape(-1).shape), dtype=jnp.uint8) * 23 + + maze_map = make_overcooked_map( + wall_map, + goal_pos, + agent_pos, + agent_dir_idx, + plate_pile_pos, + onion_pile_pos, + pot_pos, + pot_status, + onion_pos, + plate_pos, + dish_pos, + pad_obs=True, + num_agents=2, + agent_view_size=5) + + state = EnvState( + agent_pos=agent_pos, + agent_dir=agent_dir, + agent_dir_idx=agent_dir_idx, + goal_pos=goal_pos, + wall_map=wall_map, + maze_map=maze_map, + bowl_pile_pos=plate_pile_pos, + onion_pile_pos=onion_pile_pos, + agent_inv=agent_inv, + pot_pos=pot_pos, + time=0, + terminal=False + ) + + return self.get_obs(state), state + + def get_env_metrics(self, state: EnvState) -> dict: + n_walls = state.wall_map.sum() + return dict( + n_walls=n_walls, + ) + + def state_space(self) -> spaces.Dict: + """EnvState space of the environment.""" + h = self.height + w = self.width + agent_view_size = self.agent_view_size + return spaces.Dict({ + "agent_pos": spaces.Box(0, max(w, h), (2,), dtype=jnp.uint32), + "agent_dir": spaces.Discrete(4), + "goal_pos": spaces.Box(0, max(w, h), (2,), dtype=jnp.uint32), + "maze_map": spaces.Box(0, 255, (w + agent_view_size, h + agent_view_size, 3), dtype=jnp.uint32), + "time": spaces.Discrete(self.max_steps), + "terminal": spaces.Discrete(2), + }) + + def max_steps(self) -> int: + return self.max_steps + + def get_monitored_metrics(self): + return ('reward', 'shaped_reward', 'shaped_reward_scaled_by_shaped_reward_coeff', 'reward_p_shaped_reward_scaled') + + @property + def default_params(self) -> EnvParams: + # Default environment parameters + return EnvParams() + + +if hasattr(__loader__, 'name'): + module_path = __loader__.name +elif hasattr(__loader__, 'fullname'): + module_path = __loader__.fullname + +register(env_id='Overcooked', entry_point=module_path + ':Overcooked') + +if __name__ == '__main__': + from minimax.envs.wrappers import MonitorReturnWrapper + + render = False + n_envs = 1 + + kwargs = dict( + # max_episode_steps=400, + height=6, + width=9, + n_walls=15, + agent_view_size=5, + fix_to_single_layout="coord_ring_6_9" + ) + env = MonitorReturnWrapper(Overcooked(**kwargs)) + params = env.params + extra = env.reset_extra() + + jit_reset_env = env.reset + jit_step_env = env.step + + key = jax.random.PRNGKey(0) + key, subkey = jax.random.split(key) + obs, state, extra = jit_reset_env(subkey) + + all_sps = [] + + import time + for ac in [0, 0, 5, 0, 0]: # [1, 1, 3, 1, 5]: + key, subkey = jax.random.split(key) + # vrngs = jax.random.split(subkey) + start = time.time() + jax.debug.print('obs:\n{a}', a=(obs['agent_0'][:, :, 0] + * 1 + obs['agent_0'][:, :, 1]*2+obs['agent_0'][:, :, 11]*3).T) + obs, state, reward, done, info, extra = jit_step_env( + subkey, + state, + action={ + 'agent_0': ac, + 'agent_1': ac + }, + extra=extra + ) + jax.debug.print("reward r {r} {ir} {isr}", r=reward, + ir=info["sparse_reward"], isr=info["shaped_reward"]) + + state = state.replace(agent_inv=jnp.array( + [OBJECT_TO_INDEX['onion'], OBJECT_TO_INDEX['onion']])) + + obs['agent_0'].block_until_ready() + end = time.time() + # print(f"sps: {1/(end-start) * n_envs}") + # print('return:', info['return']) + all_sps.append(1/(end-start) * n_envs) + + print('mean sps:', np.mean(all_sps)) + print('std sps:', np.std(all_sps)) diff --git a/src/minimax/envs/overcooked_proc/overcooked_comparators.py b/src/minimax/envs/overcooked_proc/overcooked_comparators.py new file mode 100644 index 0000000..c2444bf --- /dev/null +++ b/src/minimax/envs/overcooked_proc/overcooked_comparators.py @@ -0,0 +1,40 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +import jax +import jax.numpy as jnp + +from minimax.envs.registration import register_comparator + + +@jax.jit +def is_equal_map(a, b): + agent_pos_eq = jnp.equal(a.agent_pos, b.agent_pos).all() + goal_pos_eq = jnp.equal(a.goal_pos, b.goal_pos).all() + wall_map_eq = jnp.equal(a.wall_map, b.wall_map).all() + pot_pos_eq = jnp.equal(a.pot_pos, b.pot_pos).all() + onion_pos_eq = jnp.equal(a.onion_pile_pos, b.onion_pile_pos).all() + bowl_pos_eq = jnp.equal(a.bowl_pile_pos, b.bowl_pile_pos).all() + + _eq = jnp.logical_and(agent_pos_eq, goal_pos_eq) + _eq = jnp.logical_and(_eq, pot_pos_eq) + _eq = jnp.logical_and(_eq, onion_pos_eq) + _eq = jnp.logical_and(_eq, bowl_pos_eq) + _eq = jnp.logical_and(_eq, wall_map_eq) + + return _eq + + +# Register the mutators +if hasattr(__loader__, 'name'): + module_path = __loader__.name +elif hasattr(__loader__, 'fullname'): + module_path = __loader__.fullname + +register_comparator(env_id='Overcooked', comparator_id=None, + entry_point=module_path + ':is_equal_map') diff --git a/src/minimax/envs/overcooked_proc/overcooked_mutators.py b/src/minimax/envs/overcooked_proc/overcooked_mutators.py new file mode 100644 index 0000000..9612304 --- /dev/null +++ b/src/minimax/envs/overcooked_proc/overcooked_mutators.py @@ -0,0 +1,253 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial +from enum import IntEnum + +import numpy as np +import jax +import jax.numpy as jnp + +from .common import make_overcooked_map +from minimax.envs.registration import register_mutator + + +class Mutations(IntEnum): + # Turn left, turn right, move forward + NO_OP = 0 + FLIP_WALL = 1 + MOVE_GOAL = 2 + MOVE_POT = 3 + MOVE_ONION_PILE = 4 + MOVE_PLATE_PILE = 5 + + +def add_1_or_2_items(key, all_pos, legal_pick_mask): + obj_mask = jnp.zeros_like(all_pos, dtype=jnp.uint8) + key, subkey = jax.random.split(key) + item_idx_1 = jax.random.choice(subkey, all_pos, shape=( + 1,), p=(legal_pick_mask.astype(jnp.bool_)).astype(jnp.uint8)) + + key, subkey = jax.random.split(key) + and_2 = jax.random.bernoulli(subkey, 0.5) + + key, subkey = jax.random.split(key) + item_idx_2 = jax.random.choice(subkey, all_pos, shape=( + 1,), p=(legal_pick_mask.astype(jnp.bool_)).astype(jnp.uint8)) + + obj_mask = obj_mask.at[item_idx_1].set(1) + + update_2 = jnp.logical_or( + obj_mask.at[item_idx_2].get(), and_2.astype(jnp.uint8)) + obj_mask = obj_mask.at[item_idx_2].set(update_2) + return obj_mask + + +def flip_wall(rng, state): + wall_map = state.wall_map + h, w = wall_map.shape + wall_mask = jnp.ones((h*w,), dtype=jnp.bool_) + + goal_idx = state.goal_pos.flatten() + agent_idx = state.agent_pos.flatten() + pot_pos_idx = state.pot_pos.flatten() + onion_pile_pos_idx = state.onion_pile_pos.flatten() + plate_pile_pos_idx = state.bowl_pile_pos.flatten() + + # Do not flip wall below an object or agent + wall_mask = wall_mask.at[goal_idx].set(False) + wall_mask = wall_mask.at[agent_idx].set(False) + wall_mask = wall_mask.at[pot_pos_idx].set(False) + wall_mask = wall_mask.at[onion_pile_pos_idx].set(False) + wall_mask = wall_mask.at[plate_pile_pos_idx].set(False) + + # Never allowed to flip a edge in overcooked + wall_mask = wall_mask.reshape(h, w) + wall_mask = wall_mask.at[:, 0].set(False) + wall_mask = wall_mask.at[:, -1].set(False) + wall_mask = wall_mask.at[0, :].set(False) + wall_mask = wall_mask.at[-1, :].set(False) + wall_mask = wall_mask.flatten() + + flip_idx = jax.random.choice(rng, np.arange(h*w), shape=(), p=wall_mask) + + wall_map = wall_map.flatten() + flip_val = ~wall_map.at[flip_idx].get() + + wall_map = wall_map.at[flip_idx].set(flip_val) + next_wall_map = wall_map.reshape(state.wall_map.shape) + return state.replace(wall_map=next_wall_map) + + +def move_goal(rng, state): + wall_map = state.wall_map + h, w = wall_map.shape + wall_mask = wall_map.flatten() + + onion_pile_pos_idx = state.onion_pile_pos.flatten() + bowl_pile_pos_idx = state.bowl_pile_pos.flatten() + goal_idx = state.goal_pos.flatten() + pot_idx = state.pot_pos.flatten() + + # No previous position and other objects + wall_mask = wall_mask.at[goal_idx].set(False) + wall_mask = wall_mask.at[pot_idx].set(False) + wall_mask = wall_mask.at[bowl_pile_pos_idx].set(False) + wall_mask = wall_mask.at[onion_pile_pos_idx].set(False) + + # Move around the wall + all_pos = jnp.zeros((h*w,), dtype=jnp.uint8) + next_goal_pos = add_1_or_2_items(rng, all_pos, wall_mask) + return state.replace(goal_pos=next_goal_pos.reshape(state.goal_pos.shape)) + + +def move_pot(rng, state): + wall_map = state.wall_map + h, w = wall_map.shape + wall_mask = wall_map.flatten() + + onion_pile_pos_idx = state.onion_pile_pos.flatten() + bowl_pile_pos_idx = state.bowl_pile_pos.flatten() + goal_idx = state.goal_pos.flatten() + pot_idx = state.pot_pos.flatten() + + # No previous position and other objects + wall_mask = wall_mask.at[goal_idx].set(False) + wall_mask = wall_mask.at[pot_idx].set(False) + wall_mask = wall_mask.at[bowl_pile_pos_idx].set(False) + wall_mask = wall_mask.at[onion_pile_pos_idx].set(False) + + # Move around the wall + all_pos = jnp.zeros((h*w,), dtype=jnp.uint8) + next_pot_pos = add_1_or_2_items(rng, all_pos, wall_mask) + return state.replace(pot_pos=next_pot_pos.reshape(state.pot_pos.shape)) + + +def move_onion_pile(rng, state): + wall_map = state.wall_map + h, w = wall_map.shape + wall_mask = wall_map.flatten() + + onion_pile_pos_idx = state.onion_pile_pos.flatten() + bowl_pile_pos_idx = state.bowl_pile_pos.flatten() + goal_idx = state.goal_pos.flatten() + pot_idx = state.pot_pos.flatten() + + # No previous position and other objects + wall_mask = wall_mask.at[goal_idx].set(False) + wall_mask = wall_mask.at[pot_idx].set(False) + wall_mask = wall_mask.at[bowl_pile_pos_idx].set(False) + wall_mask = wall_mask.at[onion_pile_pos_idx].set(False) + + # Move around the wall + all_pos = jnp.zeros((h*w,), dtype=jnp.uint8) + next_onion_pile_pos = add_1_or_2_items(rng, all_pos, wall_mask) + return state.replace(onion_pile_pos=next_onion_pile_pos.reshape(state.onion_pile_pos.shape)) + + +def move_bowl_pile(rng, state): + wall_map = state.wall_map + h, w = wall_map.shape + wall_mask = wall_map.flatten() + + onion_pile_pos_idx = state.onion_pile_pos.flatten() + bowl_pile_pos_idx = state.bowl_pile_pos.flatten() + goal_idx = state.goal_pos.flatten() + pot_idx = state.pot_pos.flatten() + + # No previous position and other objects + wall_mask = wall_mask.at[goal_idx].set(False) + wall_mask = wall_mask.at[pot_idx].set(False) + wall_mask = wall_mask.at[bowl_pile_pos_idx].set(False) + wall_mask = wall_mask.at[onion_pile_pos_idx].set(False) + + # Move around the wall + all_pos = jnp.zeros((h*w,), dtype=jnp.uint8) + next_plate_pile_pos = add_1_or_2_items(rng, all_pos, wall_mask) + return state.replace(bowl_pile_pos=next_plate_pile_pos.reshape(state.bowl_pile_pos.shape)) + + +@partial(jax.jit, static_argnums=(1, 3)) +def move_goal_flip_walls(rng, params, state, n=1): + if n == 0: + return state + + def _mutate(carry, step): + state = carry + rng, mutation = step + + rng, arng, brng, crng, drng, erng = jax.random.split(rng, 6) + + is_flip_wall = jnp.equal(mutation, Mutations.FLIP_WALL.value) + mutated_state = flip_wall(arng, state) + next_state = jax.tree_map(lambda x, y: jax.lax.select( + is_flip_wall, x, y), mutated_state, state) + + is_move_goal = jnp.equal(mutation, Mutations.MOVE_GOAL.value) + mutated_state = move_goal(brng, state) + next_state = jax.tree_map(lambda x, y: jax.lax.select( + is_move_goal, x, y), mutated_state, next_state) + + is_move_pot = jnp.equal(mutation, Mutations.MOVE_POT.value) + mutated_state = move_pot(crng, state) + next_state = jax.tree_map(lambda x, y: jax.lax.select( + is_move_pot, x, y), mutated_state, next_state) + + is_move_onion_pile = jnp.equal( + mutation, Mutations.MOVE_ONION_PILE.value) + mutated_state = move_onion_pile(drng, state) + next_state = jax.tree_map(lambda x, y: jax.lax.select( + is_move_onion_pile, x, y), mutated_state, next_state) + + is_move_plate_pile = jnp.equal( + mutation, Mutations.MOVE_PLATE_PILE.value) + mutated_state = move_bowl_pile(erng, state) + next_state = jax.tree_map(lambda x, y: jax.lax.select( + is_move_plate_pile, x, y), mutated_state, next_state) + + return next_state, None + + rng, nrng, *mrngs = jax.random.split(rng, n+2) + mutations = jax.random.choice(nrng, np.arange(len(Mutations)), (n,)) + + state, _ = jax.lax.scan(_mutate, state, (jnp.array(mrngs), mutations)) + + onion_pos = jnp.zeros(state.wall_map.shape, dtype=jnp.uint8) + plate_pos = jnp.zeros(state.wall_map.shape, dtype=jnp.uint8) + dish_pos = jnp.zeros(state.wall_map.shape, dtype=jnp.uint8) + + pot_status = jnp.ones((state.wall_map.reshape(-1).shape), dtype=jnp.uint8) * 23 + + next_maze_map = make_overcooked_map( + state.wall_map, + state.goal_pos, + state.agent_pos, + state.agent_dir_idx, + state.bowl_pile_pos, + state.onion_pile_pos, + state.pot_pos, + pot_status, + onion_pos, + plate_pos, + dish_pos, + pad_obs=True, + num_agents=2, + agent_view_size=5 + ) + + return state.replace(maze_map=next_maze_map) + + +# Register the mutators +if hasattr(__loader__, 'name'): + module_path = __loader__.name +elif hasattr(__loader__, 'fullname'): + module_path = __loader__.fullname + +register_mutator(env_id='Overcooked', mutator_id=None, + entry_point=module_path + ':move_goal_flip_walls') diff --git a/src/minimax/envs/overcooked_proc/overcooked_ood.py b/src/minimax/envs/overcooked_proc/overcooked_ood.py new file mode 100644 index 0000000..e600899 --- /dev/null +++ b/src/minimax/envs/overcooked_proc/overcooked_ood.py @@ -0,0 +1,405 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from typing import Tuple + +import jax +import jax.numpy as jnp +import chex + +from flax.core.frozen_dict import FrozenDict + +from minimax.envs.registration import register +from minimax.envs.overcooked_proc.layouts import layout_grid_to_onehot_dict +from .common import ( + OBJECT_TO_INDEX, + make_overcooked_map, +) +from .overcooked import ( + Overcooked, + EnvParams, + EnvState, + _obtain_from_layout, + +) + +cramped_room = { + "height": 4, + "width": 5, + "wall_idx": jnp.array([0, 1, 2, 3, 4, + 5, 9, + 10, 14, + 15, 16, 17, 18, 19]), + "agent_idx": jnp.array([6, 8]), + "goal_idx": jnp.array([18]), + "plate_pile_idx": jnp.array([16]), + "onion_pile_idx": jnp.array([5, 9]), + "pot_idx": jnp.array([2]) +} +asymm_advantages = { + "height": 5, + "width": 9, + "wall_idx": jnp.array([0, 1, 2, 3, 4, 5, 6, 7, 8, + 9, 11, 12, 13, 14, 15, 17, + 18, 22, 26, + 27, 31, 35, + 36, 37, 38, 39, 40, 41, 42, 43, 44]), + "agent_idx": jnp.array([29, 32]), + "goal_idx": jnp.array([12, 17]), + "plate_pile_idx": jnp.array([39, 41]), + "onion_pile_idx": jnp.array([9, 14]), + "pot_idx": jnp.array([22, 31]) +} +coord_ring = { + "height": 5, + "width": 5, + "wall_idx": jnp.array([0, 1, 2, 3, 4, + 5, 9, + 10, 12, 14, + 15, 19, + 20, 21, 22, 23, 24]), + "agent_idx": jnp.array([7, 11]), + "goal_idx": jnp.array([22]), + "plate_pile_idx": jnp.array([10]), + "onion_pile_idx": jnp.array([15, 21]), + "pot_idx": jnp.array([3, 9]) +} +forced_coord = { + "height": 5, + "width": 5, + "wall_idx": jnp.array([0, 1, 2, 3, 4, + 5, 7, 9, + 10, 12, 14, + 15, 17, 19, + 20, 21, 22, 23, 24]), + "agent_idx": jnp.array([11, 8]), + "goal_idx": jnp.array([23]), + "onion_pile_idx": jnp.array([5, 10]), + "plate_pile_idx": jnp.array([15]), + "pot_idx": jnp.array([3, 9]) +} + +# Example of layout provided as a grid +counter_circuit_grid = """ +WWWPPWWW +W A W +B WWWW X +W AW +WWWOOWWW +""" + +asymm_advantages_6_9 = """ +WWWWWWWWW +O WXWOW X +W P A W +WA P W +WWWBWBWWW +WWWWWWWWW +""" + +counter_circuit_6_9 = """ +WWWPPWWWW +W A WW +B WWWW XW +W AWW +WWWOOWWWW +WWWWWWWWW +""" + +forced_coord_6_9 = """ +WWWPWWWWW +OAWAPWWWW +O W WWWWW +B W WWWWW +WWWXWWWWW +WWWWWWWWW +""" + +cramped_room_6_9 = """ +WWPWWWWWW +OAA OWWWW +W WWWWW +WBWXWWWWW +WWWWWWWWW +WWWWWWWWW +""" + +coord_ring_6_9 = """ +WWWPWWWWW +WA APWWWW +B W WWWWW +O WWWWW +WOXWWWWWW +WWWWWWWWW +""" + +forced_coord_5_5 = """ +WWWPW +OAWAP +O W W +B W W +WWWXW +""" + +cramped_room_5_5 = """ +WWPWW +OAA O +W W +WBWXW +WWWWW +""" + +coord_ring_5_5 = """ +WWWPW +WA AP +B W W +O W +WOXWW +""" + + +# ======== Singleton mazes ======== +class OvercookedSingleton(Overcooked): + def __init__( + self, + grid, + agent_view_size=5, + replace_wall_pos=False, + max_steps=400, + normalize_obs=False, + sample_n_walls=False, + singleton_seed=-1 + ): + height = grid["height"] + width = grid["width"] + super().__init__( + height=height, + width=width, + agent_view_size=agent_view_size, + replace_wall_pos=replace_wall_pos and not sample_n_walls, + max_steps=max_steps, + normalize_obs=normalize_obs, + sample_n_walls=sample_n_walls, + singleton_seed=singleton_seed, + ) + + self.params = EnvParams( + height=height, + width=width, + agent_view_size=agent_view_size, + normalize_obs=normalize_obs, + max_steps=max_steps, + singleton_seed=singleton_seed, + ) + + h = self.height + w = self.width + + # NOTE: that since the layout is fixed, the random_reset is set to False + # and this is why jax.random.PRNGKey(0) is used too (not needed if no random_reset). + wall_map, goal_pos, agent_pos, agent_dir, agent_dir_idx, plate_pile_pos, onion_pile_pos, pot_pos, pot_status\ + = _obtain_from_layout(jax.random.PRNGKey(0), grid, h, w, random_reset=False, num_agents=2) + + onion_pos = jnp.zeros((h, w), dtype=jnp.uint8) + plate_pos = jnp.zeros((h, w), dtype=jnp.uint8) + dish_pos = jnp.zeros((h, w), dtype=jnp.uint8) + + agent_inv = jnp.array( + [OBJECT_TO_INDEX['empty'], OBJECT_TO_INDEX['empty']]) + + self.overcooked_map = make_overcooked_map( + wall_map, + goal_pos, + agent_pos, + agent_dir_idx, + plate_pile_pos, + onion_pile_pos, + pot_pos, + pot_status, + onion_pos, + plate_pos, + dish_pos, + pad_obs=True, + num_agents=self.num_agents, + agent_view_size=self.agent_view_size) + + self.agent_pos = agent_pos + self.agent_dir = agent_dir + self.agent_dir_idx = agent_dir_idx + self.agent_inv = agent_inv + self.goal_pos = goal_pos + self.pot_pos = pot_pos + self.bowl_pile_pos = plate_pile_pos + self.onion_pile_pos = onion_pile_pos + self.wall_map = wall_map + + @property + def default_params(self) -> EnvParams: + # Default environment parameters + return EnvParams() + + def reset_env( + self, + key: chex.PRNGKey, + ) -> Tuple[chex.Array, EnvState]: + + state = EnvState( + agent_pos=self.agent_pos, + agent_dir=self.agent_dir, + agent_dir_idx=self.agent_dir_idx, + agent_inv=self.agent_inv, + goal_pos=self.goal_pos, + pot_pos=self.pot_pos, + wall_map=self.wall_map.astype(jnp.bool_), + maze_map=self.overcooked_map, + bowl_pile_pos=self.bowl_pile_pos, + onion_pile_pos=self.onion_pile_pos, + time=0, + terminal=False, + ) + + return self.get_obs(state), state + + +# ======== Specific mazes ======== +class CoordRing6_9(OvercookedSingleton): + def __init__( + self, + normalize_obs=False): + self.layout_name = "coord_ring_6_9" + + grid = layout_grid_to_onehot_dict(coord_ring_6_9) + + super().__init__( + grid=grid, + normalize_obs=normalize_obs, + ) + + +class ForcedCoord6_9(OvercookedSingleton): + def __init__( + self, + normalize_obs=False): + self.layout_name = "forced_coord_6_9" + + grid = layout_grid_to_onehot_dict(forced_coord_6_9) + + super().__init__( + grid=grid, + normalize_obs=normalize_obs, + ) + + +class CounterCircuit6_9(OvercookedSingleton): + def __init__( + self, + normalize_obs=False): + self.layout_name = "counter_circuit_6_9" + + grid = layout_grid_to_onehot_dict(counter_circuit_6_9) + + super().__init__( + grid=grid, + normalize_obs=normalize_obs, + ) + + +class AsymmAdvantages6_9(OvercookedSingleton): + def __init__( + self, + normalize_obs=False): + self.layout_name = "asymm_advantages_6_9" + + grid = layout_grid_to_onehot_dict(asymm_advantages_6_9) + + super().__init__( + grid=grid, + normalize_obs=normalize_obs, + ) + + +class CrampedRoom6_9(OvercookedSingleton): + def __init__( + self, + normalize_obs=False): + self.layout_name = "cramped_room_6_9" + + grid = layout_grid_to_onehot_dict(cramped_room_6_9) + + super().__init__( + grid=grid, + normalize_obs=normalize_obs, + ) + + +class CoordRing5_5(OvercookedSingleton): + def __init__( + self, + normalize_obs=False): + self.layout_name = "coord_ring_5_5" + + grid = layout_grid_to_onehot_dict(coord_ring_5_5) + + super().__init__( + grid=grid, + normalize_obs=normalize_obs, + ) + + +class ForcedCoord5_5(OvercookedSingleton): + def __init__( + self, + normalize_obs=False): + self.layout_name = "forced_coord_5_5" + + grid = layout_grid_to_onehot_dict(forced_coord_5_5) + + super().__init__( + grid=grid, + normalize_obs=normalize_obs, + ) + + +class CrampedRoom5_5(OvercookedSingleton): + def __init__( + self, + normalize_obs=False): + self.layout_name = "cramped_room_5_5" + + grid = layout_grid_to_onehot_dict(cramped_room_5_5) + + super().__init__( + grid=grid, + normalize_obs=normalize_obs, + ) + + +# ======== Registration ======== +if hasattr(__loader__, 'name'): + module_path = __loader__.name +elif hasattr(__loader__, 'fullname'): + module_path = __loader__.fullname + +# register(env_id='Overcooked', entry_point=module_path + ':') +register(env_id='Overcooked-CoordRing6_9', + entry_point=module_path + ':CoordRing6_9') +register(env_id='Overcooked-ForcedCoord6_9', + entry_point=module_path + ':ForcedCoord6_9') +register(env_id='Overcooked-CounterCircuit6_9', + entry_point=module_path + ':CounterCircuit6_9') +register(env_id='Overcooked-AsymmAdvantages6_9', + entry_point=module_path + ':AsymmAdvantages6_9') +register(env_id='Overcooked-CrampedRoom6_9', + entry_point=module_path + ':CrampedRoom6_9') + +register(env_id='Overcooked-CoordRing5_5', + entry_point=module_path + ':CoordRing5_5') +register(env_id='Overcooked-ForcedCoord5_5', + entry_point=module_path + ':ForcedCoord5_5') +register(env_id='Overcooked-CrampedRoom5_5', + entry_point=module_path + ':CrampedRoom5_5') diff --git a/src/minimax/envs/overcooked_proc/overcooked_ued.py b/src/minimax/envs/overcooked_proc/overcooked_ued.py new file mode 100644 index 0000000..f33da24 --- /dev/null +++ b/src/minimax/envs/overcooked_proc/overcooked_ued.py @@ -0,0 +1,541 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from collections import OrderedDict +from enum import IntEnum + +import jax +import jax.numpy as jnp +from jax import lax +from typing import Dict, Tuple +import chex +from flax import struct + +from minimax.envs.overcooked_proc.overcooked import DIR_TO_VEC, EnvState + +from .common import OBJECT_TO_INDEX, EnvInstance, make_overcooked_map +from minimax.envs import environment, spaces +from minimax.envs.registration import register_ued + + +class SequentialActions(IntEnum): + skip = 0 + wall = 1 + goal = 2 + agent = 3 + onion = 4 + soup = 5 + bowls = 6 + + +@struct.dataclass +class UEDEnvState: + encoding: chex.Array + time: int + terminal: bool + + +@struct.dataclass +class EnvParams: + height: int = 6 + width: int = 9 + n_walls: int = 25 + noise_dim: int = 50 + agent_view_size: int = 5 + replace_wall_pos: bool = False + fixed_n_wall_steps: bool = False + first_wall_pos_sets_budget: bool = False + use_seq_actions: bool = False + normalize_obs: bool = False + sample_n_walls: bool = False # Sample n_walls uniformly in [0, n_walls] + max_steps: int = 400 + singleton_seed: int = -1 + max_episode_steps: int = 400 + + +class UEDOvercooked(environment.Environment): + def __init__( + self, + height=6, + width=9, + n_walls=25, + noise_dim=16, + replace_wall_pos=False, + fixed_n_wall_steps=False, + first_wall_pos_sets_budget=False, + use_seq_actions=False, + normalize_obs=False, + ): + """ + Using the original action space requires ensuring proper handling + of a sequence with trailing dones, e.g. dones: 0 0 0 0 1 1 1 1 1 ... 1. + Advantages and value losses should only be computed where ~dones[0]. + """ + assert not (first_wall_pos_sets_budget and fixed_n_wall_steps), \ + 'Setting first_wall_pos_sets_budget=True requires fixed_n_wall_steps=False.' + + super().__init__() + + self.n_tiles = height*width + # go straight, turn left, turn right, take action + self.action_set = jnp.array(jnp.arange(self.n_tiles)) + + self.agents = ["agent_0", "agent_1"] + + self.params = EnvParams( + height=height, + width=width, + n_walls=n_walls, + noise_dim=noise_dim, + replace_wall_pos=replace_wall_pos, + fixed_n_wall_steps=fixed_n_wall_steps, + first_wall_pos_sets_budget=first_wall_pos_sets_budget, + use_seq_actions=False, + normalize_obs=normalize_obs, + ) + + @staticmethod + def align_kwargs(kwargs, other_kwargs): + kwargs.update(dict( + height=other_kwargs['height'], + width=other_kwargs['width'], + )) + + return kwargs + + def _add_noise_to_obs(self, rng, obs): + if self.params.noise_dim > 0: + noise = jax.random.uniform(rng, (self.params.noise_dim,)) + obs.update(dict(noise=noise)) + + return obs + + def reset_env( + self, + key: chex.PRNGKey): + """ + Prepares the environment state for a new design + from a blank slate. + """ + params = self.params + noise_rng, dir_rng = jax.random.split(key) + encoding = jnp.zeros((self._get_encoding_dim(),), dtype=jnp.uint8) + + state = UEDEnvState( + encoding=encoding, + time=0, + terminal=False, + ) + + obs = self._add_noise_to_obs( + noise_rng, + self.get_obs(state) + ) + + return obs, state + + def get_monitored_metrics(self): + return () + + def step_env( + self, + key: chex.PRNGKey, + state: UEDEnvState, + action: int, + ) -> Tuple[chex.Array, UEDEnvState, float, bool, dict]: + """ + Take a design step. + action: A pos as an int from 0 to (height*width)-1 + """ + params = self.params + + collision_rng, noise_rng = jax.random.split(key) + + # Sample a random free tile in case of a collision + dist_values = jnp.logical_and( # True if position taken + jnp.ones(params.n_walls + 10), + jnp.arange(params.n_walls + 10)+1 > state.time + ) + + # Get zero-indexed last wall time step + if params.fixed_n_wall_steps: + max_n_walls = params.n_walls + encoding_pos = state.encoding[:params.n_walls+10] + last_wall_step_idx = max_n_walls - 1 + else: + max_n_walls = jnp.round( + params.n_walls*state.encoding[0]/self.n_tiles).astype(jnp.uint32) + + if self.params.first_wall_pos_sets_budget: + encoding_pos = state.encoding[:params.n_walls+10] + last_wall_step_idx = jnp.maximum(max_n_walls, 1) - 1 + else: + encoding_pos = state.encoding[1:params.n_walls+11] + last_wall_step_idx = max_n_walls + + pos_dist = jnp.ones(self.n_tiles).at[ + jnp.flip(encoding_pos)].set(jnp.flip(dist_values)) + all_pos = jnp.arange(self.n_tiles, dtype=jnp.uint8) + + agent_step_1_idx = last_wall_step_idx+1 # Enc is full length + agent_step_2_idx = last_wall_step_idx+2 + + # Track whether it is the last time step + next_state = state.replace(time=state.time + 1) + done = self.is_terminal(next_state) + + collision = jnp.logical_and( + pos_dist[action] < 1, + jnp.logical_or( + not params.replace_wall_pos, + jnp.logical_and( # agent pos cannot be overriden + # jnp.logical_or(), + # jnp.equal(state.time, goal_step_1_idx), + jnp.equal(state.encoding[agent_step_1_idx], action), + jnp.equal(state.encoding[agent_step_2_idx], action) + ) + ) + ) + # collision = (collision * (1-is_agent_dir_step)).astype(jnp.uint32) + + action = (1-collision)*action + \ + collision*jax.random.choice(collision_rng, + all_pos, replace=False, p=pos_dist) + + # (1-is_agent_dir_step)* # + is_agent_dir_step*(-1) + enc_idx = state.time + encoding = state.encoding.at[enc_idx].set(action) + + next_state = next_state.replace( + encoding=encoding, + terminal=done + ) + reward = 0 + + obs = self._add_noise_to_obs(noise_rng, self.get_obs(next_state)) + + # jax.debug.breakpoint() + return ( + lax.stop_gradient(obs), + lax.stop_gradient(next_state), + reward, + done, + {}, + ) + + def get_env_instance( + self, + key: chex.PRNGKey, + state: UEDEnvState + ) -> chex.Array: + """ + Converts internal encoding to an instance encoding that + can be interpreted by the `set_to_instance` method + the paired Environment class. + """ + params = self.params + h = params.height + w = params.width + enc = state.encoding + + # === Extract agent_dir, agent_pos, and goal_pos === + # Num walls placed currently + if params.fixed_n_wall_steps: + n_walls = params.n_walls + enc_len = self._get_encoding_dim() + wall_pos_idx = jnp.flip(enc[:params.n_walls]) + agent_pos_1_idx = enc_len-2 # Enc is full length + agent_pos_2_idx = enc_len-3 + goal_pos_1_idx = enc_len-4 + onion_pos_1_idx = enc_len-6 + pot_pos_1_idx = enc_len-8 + bowl_pos_1_idx = enc_len-10 + else: + n_walls = jnp.round( + params.n_walls*enc[0]/self.n_tiles + ).astype(jnp.uint32) + if params.first_wall_pos_sets_budget: + # So 0-padding does not override pos=0 + wall_pos_idx = jnp.flip(enc[:params.n_walls]) + enc_len = n_walls + 10 # [wall_pos] + len((goal, agent)) + else: + wall_pos_idx = jnp.flip(enc[1:params.n_walls+1]) + # [wall_pos] + len((n_walls, goal, agent)) + enc_len = n_walls + 11 + agent_pos_1_idx = enc_len-1 # Enc is full length + agent_pos_2_idx = enc_len-2 + goal_pos_1_idx = enc_len-3 + onion_pos_1_idx = enc_len-5 + pot_pos_1_idx = enc_len-7 + bowl_pos_1_idx = enc_len-9 + + # Make wall map + wall_start_time = jnp.logical_and( # 1 if explicitly predict # blocks, else 0 + not params.fixed_n_wall_steps, + not params.first_wall_pos_sets_budget + ).astype(jnp.uint32) + + wall_map = jnp.zeros((h * w), dtype=jnp.bool_) + wall_values = jnp.arange( + params.n_walls) + wall_start_time < jnp.minimum(state.time, n_walls + wall_start_time) + wall_values = jnp.flip(wall_values) + wall_map = wall_map.at[wall_pos_idx].set(wall_values) + wall_map = wall_map.reshape((h, w)) + wall_map = wall_map.at[0, :].set(True) + wall_map = wall_map.at[:, 0].set(True) + wall_map = wall_map.at[-1, :].set(True) + wall_map = wall_map.at[:, -1].set(True) + wall_map = wall_map.reshape(-1) + + occupied_mask = wall_map + + """Agents should always end up on an empty square. If they are placed on a wall pick randomly.""" + is_occupied = occupied_mask[enc[agent_pos_1_idx]] == 1 + agent_pos_1_idx_enc = is_occupied*jax.random.choice(key, jnp.arange(h*w), shape=( + ), p=jnp.logical_not(occupied_mask)) + jnp.logical_not(is_occupied)*enc[agent_pos_1_idx] + agent_1_placed = state.time > jnp.array( + [agent_pos_1_idx], dtype=jnp.uint8) + agent_1_pos = \ + agent_1_placed*jnp.array([agent_pos_1_idx_enc % w, agent_pos_1_idx_enc // w], dtype=jnp.uint8) \ + + (~agent_1_placed)*jnp.array([h, w], dtype=jnp.uint8) + occupied_mask = occupied_mask.at[agent_pos_1_idx_enc].set(True) + + is_occupied = occupied_mask[enc[agent_pos_2_idx]] == 1 + agent_pos_2_idx_enc = is_occupied*jax.random.choice(key, jnp.arange( + h*w), shape=(), p=jnp.logical_not(occupied_mask)) + jnp.logical_not(is_occupied)*enc[agent_pos_2_idx] + agent_2_placed = state.time > jnp.array( + [agent_pos_2_idx], dtype=jnp.uint8) + agent_2_pos = \ + agent_2_placed*jnp.array([agent_pos_2_idx_enc % w, agent_pos_2_idx_enc // w], dtype=jnp.uint8) \ + + (~agent_2_placed)*jnp.array([h, w], dtype=jnp.uint8) + occupied_mask = occupied_mask.at[agent_pos_2_idx_enc].set(True) + + agents_obj_occupied_mask = jnp.zeros_like(occupied_mask) + agents_obj_occupied_mask = agents_obj_occupied_mask.reshape((h, w)) + # Exlude corners, will never be actually reachable + agents_obj_occupied_mask = agents_obj_occupied_mask.at[0, 0].set(True) + agents_obj_occupied_mask = agents_obj_occupied_mask.at[0, -1].set(True) + agents_obj_occupied_mask = agents_obj_occupied_mask.at[-1, 0].set(True) + agents_obj_occupied_mask = agents_obj_occupied_mask.at[-1, -1].set( + True) + agents_obj_occupied_mask = agents_obj_occupied_mask.reshape(-1) + agents_obj_occupied_mask = agents_obj_occupied_mask.at[ + agent_pos_1_idx_enc].set(True) + agents_obj_occupied_mask = agents_obj_occupied_mask.at[ + agent_pos_2_idx_enc].set(True) + + """Objects can end up on a wall but never on a agent or another agent.""" + is_occupied = agents_obj_occupied_mask[enc[goal_pos_1_idx]] == 1 + goal_pos_1_idx_enc = is_occupied*jax.random.choice(key, jnp.arange( + h*w), shape=(), p=jnp.logical_not(agents_obj_occupied_mask)) + jnp.logical_not(is_occupied)*enc[goal_pos_1_idx] + goal_1_placed = state.time > jnp.array( + [goal_pos_1_idx], dtype=jnp.uint8) + goal_1_pos = \ + goal_1_placed*jnp.zeros((h*w), dtype=jnp.uint8).at[goal_pos_1_idx_enc].set(1) \ + + (~goal_1_placed)*jnp.zeros((h*w), dtype=jnp.uint8) + goal_1_pos = goal_1_pos.reshape((h, w)) + agents_obj_occupied_mask = agents_obj_occupied_mask.at[ + goal_pos_1_idx_enc].set(True) + wall_map = wall_map.at[goal_pos_1_idx_enc].set(True) + + is_occupied = agents_obj_occupied_mask[enc[onion_pos_1_idx]] == 1 + onion_pos_1_idx_enc = is_occupied*jax.random.choice(key, jnp.arange( + h*w), shape=(), p=jnp.logical_not(agents_obj_occupied_mask)) + jnp.logical_not(is_occupied)*enc[onion_pos_1_idx] + onion_1_placed = state.time > jnp.array( + [onion_pos_1_idx], dtype=jnp.uint8) + onion_1_pos = \ + onion_1_placed*jnp.zeros((h*w), dtype=jnp.uint8).at[onion_pos_1_idx_enc].set(1) \ + + (~onion_1_placed)*jnp.zeros((h*w), dtype=jnp.uint8) + onion_1_pos = onion_1_pos.reshape((h, w)) + agents_obj_occupied_mask = agents_obj_occupied_mask.at[ + onion_pos_1_idx_enc].set(True) + wall_map = wall_map.at[onion_pos_1_idx_enc].set(True) + + is_occupied = agents_obj_occupied_mask[enc[pot_pos_1_idx]] == 1 + pot_pos_1_idx_enc = is_occupied*jax.random.choice(key, jnp.arange( + h*w), shape=(), p=jnp.logical_not(agents_obj_occupied_mask)) + jnp.logical_not(is_occupied)*enc[pot_pos_1_idx] + pot_1_placed = state.time > jnp.array( + [pot_pos_1_idx], dtype=jnp.uint8) + pot_1_pos = \ + pot_1_placed*jnp.zeros((h*w), dtype=jnp.uint8).at[pot_pos_1_idx_enc].set(1) \ + + (~pot_1_placed)*jnp.zeros((h*w), dtype=jnp.uint8) + pot_1_pos = pot_1_pos.reshape((h, w)) + agents_obj_occupied_mask = agents_obj_occupied_mask.at[ + pot_pos_1_idx_enc].set(True) + wall_map = wall_map.at[pot_pos_1_idx_enc].set(True) + + is_occupied = agents_obj_occupied_mask[enc[bowl_pos_1_idx]] == 1 + bowl_pos_1_idx_enc = is_occupied*jax.random.choice(key, jnp.arange( + h*w), shape=(), p=jnp.logical_not(agents_obj_occupied_mask)) + jnp.logical_not(is_occupied)*enc[bowl_pos_1_idx] + bowl_1_placed = state.time > jnp.array( + [bowl_pos_1_idx], dtype=jnp.uint8) + bowl_1_pos = \ + bowl_1_placed*jnp.zeros((h*w), dtype=jnp.uint8).at[bowl_pos_1_idx_enc].set(1) \ + + (~bowl_1_placed)*jnp.zeros((h*w), dtype=jnp.uint8) + bowl_1_pos = bowl_1_pos.reshape((h, w)) + agents_obj_occupied_mask = agents_obj_occupied_mask.at[ + bowl_pos_1_idx_enc].set(True) + wall_map = wall_map.at[bowl_pos_1_idx_enc].set(True) + + # agent_dir_idx = jnp.floor((4*enc[-1]/self.n_tiles)).astype(jnp.uint8) + key, subkey = jax.random.split(key) + agent_dir_idx = jax.random.choice(subkey, jnp.arange( + len(DIR_TO_VEC), dtype=jnp.int32), shape=(2,)) + + # Zero out walls where agent and goal reside + # Should not be the case but just in case + agent_1_mask = agent_1_placed * \ + (~(jnp.arange(h*w) == agent_pos_1_idx_enc)) + ~agent_1_placed*wall_map + agent_2_mask = agent_2_placed * \ + (~(jnp.arange(h*w) == agent_pos_2_idx_enc)) + ~agent_2_placed*wall_map + goal_mask = goal_1_placed * \ + (~(jnp.arange(h*w) == goal_pos_1_idx_enc)) + ~goal_1_placed*wall_map + wall_map = wall_map*agent_1_mask*agent_2_mask + wall_map = wall_map.reshape(h, w) + + possible_items = jnp.array([OBJECT_TO_INDEX['empty'], OBJECT_TO_INDEX['onion'], + OBJECT_TO_INDEX['plate'], OBJECT_TO_INDEX['dish']]) + key, subkey = jax.random.split(key) + random_agent_inv = jax.random.choice( + subkey, possible_items, shape=(2,), replace=True) + + return EnvInstance( + agent_pos=jnp.array([agent_1_pos, agent_2_pos], dtype=jnp.uint32), + agent_dir_idx=agent_dir_idx, + goal_pos=goal_1_pos, + wall_map=wall_map, + onion_pile_pos=onion_1_pos, + pot_pos=pot_1_pos, + plate_pile_pos=bowl_1_pos, + agent_inv=random_agent_inv + ) + + def is_terminal(self, state: UEDEnvState) -> bool: + done_steps = state.time >= self.max_episode_steps() + return jnp.logical_or(done_steps, state.terminal) + + def _get_post_terminal_obs(self, state: UEDEnvState): + dtype = jnp.float32 if self.params.normalize_obs else jnp.uint8 + image = jnp.zeros(( + self.params.height+2, self.params.width+2, 3), dtype=dtype + ) + + return OrderedDict(dict( + image=image, + time=state.time, + noise=jnp.zeros(self.params.noise_dim, dtype=jnp.float32), + )) + + def get_obs(self, state: UEDEnvState): + instance = self.get_env_instance(jax.random.PRNGKey(0), state) + h = self.params.height + w = self.params.width + onion_pos = jnp.zeros((h, w), dtype=jnp.uint8) + plate_pos = jnp.zeros((h, w), dtype=jnp.uint8) + dish_pos = jnp.zeros((h, w), dtype=jnp.uint8) + + pot_status = jnp.ones( + (instance.wall_map.reshape(-1).shape), dtype=jnp.uint8) * 23 + + agent_dir = DIR_TO_VEC.at[instance.agent_dir_idx].get() + + maze_map = make_overcooked_map( + wall_map=instance.wall_map, + goal_pos=instance.goal_pos, + agent_pos=instance.agent_pos, + agent_dir_idx=instance.agent_dir_idx, + plate_pile_pos=instance.plate_pile_pos, + onion_pile_pos=instance.onion_pile_pos, + pot_pos=instance.pot_pos, + pot_status=pot_status, + onion_pos=onion_pos, + plate_pos=plate_pos, + dish_pos=dish_pos, + pad_obs=True, + num_agents=2, + agent_view_size=5 + ) + + padding = 4 + return OrderedDict(dict( + image=maze_map[padding:-padding, padding:-padding, :], + time=state.time, + )) + + @property + def default_params(self): + return EnvParams() + + @property + def name(self) -> str: + """Environment name.""" + return "UEDOvercooked" + + @property + def num_actions(self) -> int: + """Number of actions possible in environment.""" + return len(self.action_set) + + def action_space(self) -> spaces.Discrete: + """Action space of the environment.""" + params = self.params + return spaces.Discrete( + params.height*params.width, + dtype=jnp.uint8 + ) + + def observation_space(self) -> spaces.Dict: + """Observation space of the environment.""" + params = self.params + max_episode_steps = self.max_episode_steps() + spaces_dict = { + 'image': spaces.Box(0, 255, (params.height, params.width, 3)), + 'time': spaces.Discrete(max_episode_steps), + } + if self.params.noise_dim > 0: + spaces_dict.update({ + 'noise': spaces.Box(0, 1, (self.params.noise_dim,)) + }) + return spaces.Dict(spaces_dict) + + def state_space(self) -> spaces.Dict: + """State space of the environment.""" + params = self.params + encoding_dim = self._get_encoding_dim() + max_episode_steps = self.max_episode_steps() + h = params.height + w = params.width + return spaces.Dict({ + 'encoding': spaces.Box(0, 255, (encoding_dim,)), + 'time': spaces.Discrete(max_episode_steps), + "terminal": spaces.Discrete(2), + }) + + def _get_encoding_dim(self) -> int: + encoding_dim = self.max_episode_steps() + # if not self.params.set_agent_dir: + # encoding_dim += 1 # max steps is 1 less than full encoding dim + + return encoding_dim + + def max_episode_steps(self) -> int: + if self.params.fixed_n_wall_steps \ + or self.params.first_wall_pos_sets_budget: + max_episode_steps = self.params.n_walls + 10 + else: + max_episode_steps = self.params.n_walls + 11 + + return max_episode_steps + + +if hasattr(__loader__, 'name'): + module_path = __loader__.name +elif hasattr(__loader__, 'fullname'): + module_path = __loader__.fullname + +register_ued(env_id='Overcooked', entry_point=module_path + ':UEDOvercooked') diff --git a/src/minimax/envs/registration.py b/src/minimax/envs/registration.py new file mode 100644 index 0000000..be4460d --- /dev/null +++ b/src/minimax/envs/registration.py @@ -0,0 +1,149 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +import importlib + +from .wrappers import * +from minimax.envs.environment_ued import UEDEnvironment + + +# Global registry +registered_envs = [] + +env2entry = {} + +env2ued_entry = {} + +env2comparator_entry = {} + +env2mutator_entry = {} + +name2wrapper = { + 'env_wrapper': EnvWrapper, # for testing, + 'world_state_wrapper': WorldStateWrapper, + 'ued_env_wrapper': UEDEnvWrapper, + 'monitor_return': MonitorReturnWrapper, + 'monitor_ep_metrics': MonitorEpisodicMetricsWrapper, +} + + +def _load(name): + mod_name, attr_name = name.split(":") + mod = importlib.import_module(mod_name) + fn = getattr(mod, attr_name) + return fn + + +def _fn_for_entry(entry): + if callable(entry): + return entry + else: + return _load(entry) + + +def cls_for_env_id(env_id): + if env_id not in env2entry.keys(): + raise ValueError(f"{env_id} is not registered.") + else: + entry = env2entry[env_id] + return _fn_for_entry(entry) + + +def _make_env(entry, **env_kwargs): + return _fn_for_entry(entry)(**env_kwargs) + + +def _apply_wrappers(env, wrappers): + base_env = env + if wrappers is not None and len(wrappers) > 0: + for name in wrappers: + wrapper_cls = name2wrapper[name] + if wrapper_cls.is_compatible(base_env): + env = wrapper_cls(env) + + return env + + +def make( + env_id: str, + env_kwargs={}, + ued_env_kwargs={}, + wrappers=None, + ued_wrappers=None +): + """The minimax equivalent of OpenAI's env.make(env_name)""" + if env_id not in env2entry.keys(): + raise ValueError(f"{env_id} is not registered.") + else: + entry = env2entry[env_id] + env = _make_env(entry, **env_kwargs) + + if len(ued_env_kwargs) > 0: + if env_id not in env2ued_entry.keys(): + raise ValueError(f"{env_id} has no UED counterpart registered.") + + _env_kwargs = env_kwargs + env_kwargs = env.default_params.__dict__ + env_kwargs.update(_env_kwargs) + + ued_entry = env2ued_entry[env_id] + ued_env_kwargs = _fn_for_entry(ued_entry).align_kwargs(ued_env_kwargs, env_kwargs) + + ued_env = _make_env(ued_entry, **ued_env_kwargs) + + env = UEDEnvironment(env=env, ued_env=ued_env) + + base_env = env + + env = _apply_wrappers(env, wrappers) + + if isinstance(base_env, UEDEnvironment): + env = _apply_wrappers(env, ued_wrappers) + return env, env.env.params, env.ued_env.params + + return env, env.params + + +def get_comparator(env_id: str, comparator_id: str = 'default'): + entry_point = env2comparator_entry[env_id].get(comparator_id, None) + assert entry_point is not None, f'Undefined comparator {comparator_id} for environment {env_id}.' + + return _fn_for_entry(entry_point) + + +def get_mutator(env_id: str, mutator_id: str = 'default'): + entry_point = env2mutator_entry[env_id].get(mutator_id, None) + assert entry_point is not None, f'Undefined mutator {mutator_id} for environment {env_id}.' + + return _fn_for_entry(entry_point) + + +def register(env_id: str, entry_point: str): + env2entry[env_id] = entry_point + + +def register_ued(env_id: str, entry_point: str): + env2ued_entry[env_id] = entry_point + + +def register_comparator(env_id: str, entry_point: str, comparator_id: str = None): + if comparator_id is None: + comparator_id = 'default' + + if env_id not in env2comparator_entry: + env2comparator_entry[env_id] = {} + env2comparator_entry[env_id][comparator_id] = entry_point + + +def register_mutator(env_id: str, entry_point: str, mutator_id: str = None): + if mutator_id is None: + mutator_id = 'default' + + if env_id not in env2mutator_entry: + env2mutator_entry[env_id] = {} + env2mutator_entry[env_id][mutator_id] = entry_point diff --git a/src/minimax/envs/spaces.py b/src/minimax/envs/spaces.py new file mode 100644 index 0000000..90f289f --- /dev/null +++ b/src/minimax/envs/spaces.py @@ -0,0 +1,154 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This file is modified from +https://github.com/RobertTLange/gymnax/ + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from typing import Tuple, Union +from collections import OrderedDict +import chex +import jax +import jax.numpy as jnp + + +class Discrete(object): + """ + Minimal jittable class for discrete gymnax spaces. + TODO: For now this is a 1d space. Make composable for multi-discrete. + """ + + def __init__(self, num_categories: int, shape: Tuple[int] = (), dtype=jnp.int32): + assert num_categories >= 0 + self.n = num_categories + self.shape = shape + self.dtype = dtype + + def sample(self, rng: chex.PRNGKey) -> chex.Array: + """Sample random action uniformly from set of categorical choices.""" + return jax.random.randint( + rng, shape=self.shape, minval=0, maxval=self.n + ).astype(self.dtype) + + def contains(self, x: jnp.int_) -> bool: + """Check whether specific object is within space.""" + # type_cond = isinstance(x, self.dtype) + # shape_cond = (x.shape == self.shape) + range_cond = jnp.logical_and(x >= 0, x < self.n) + return range_cond + + +class Box(object): + """ + Minimal jittable class for array-shaped gymnax spaces. + """ + + def __init__( + self, + low: float, + high: float, + shape: Tuple[int], + dtype: jnp.dtype = jnp.float32, + ): + self.low = low + self.high = high + self.shape = shape + self.dtype = dtype + + def sample(self, rng: chex.PRNGKey) -> chex.Array: + """Sample random action uniformly from 1D continuous range.""" + return jax.random.uniform( + rng, shape=self.shape, minval=self.low, maxval=self.high + ).astype(self.dtype) + + def contains(self, x: jnp.int_) -> bool: + """Check whether specific object is within space.""" + # type_cond = isinstance(x, self.dtype) + # shape_cond = (x.shape == self.shape) + range_cond = jnp.logical_and( + jnp.all(x >= self.low), jnp.all(x <= self.high) + ) + return range_cond + + +class Dict(object): + """Minimal jittable class for dictionary of simpler jittable spaces.""" + + def __init__(self, spaces: dict): + self.spaces = spaces + self.num_spaces = len(spaces) + + def sample(self, rng: chex.PRNGKey) -> dict: + """Sample random action from all subspaces.""" + key_split = jax.random.split(rng, self.num_spaces) + return OrderedDict( + [ + (k, self.spaces[k].sample(key_split[i])) + for i, k in enumerate(self.spaces) + ] + ) + + def contains(self, x: jnp.int_) -> bool: + """Check whether dimensions of object are within subspace.""" + # type_cond = isinstance(x, dict) + # num_space_cond = len(x) != len(self.spaces) + # Check for each space individually + out_of_space = 0 + for k, space in self.spaces.items(): + out_of_space += 1 - space.contains(getattr(x, k)) + return out_of_space == 0 + + +class Tuple(object): + """Minimal jittable class for tuple (product) of jittable spaces.""" + + def __init__(self, spaces: Union[tuple, list]): + self.spaces = spaces + self.num_spaces = len(spaces) + + def sample(self, rng: chex.PRNGKey) -> Tuple[chex.Array]: + """Sample random action from all subspaces.""" + key_split = jax.random.split(rng, self.num_spaces) + return tuple( + [ + space.sample(key_split[i]) + for i, space in enumerate(self.spaces) + ] + ) + + def contains(self, x: jnp.int_) -> bool: + """Check whether dimensions of object are within subspace.""" + # type_cond = isinstance(x, tuple) + # num_space_cond = len(x) != len(self.spaces) + # Check for each space individually + out_of_space = 0 + for space in self.spaces: + out_of_space += 1 - space.contains(x) + return out_of_space == 0 + + +class Dummy(object): + def __init__(self, default_value=None): + self._default_value = default_value + self.dtype = jnp.uint32 + if self._default_value is None: + self.n = 0 + else: + self.n = 1 + + def sample(self, rng: chex.PRNGKey): + if self._default_value is None: + return None + else: + return jnp.array(0, dtype=self.dtype) + + def contains(self, x: jnp.int_) -> bool: + return False diff --git a/src/minimax/envs/viz/__init__.py b/src/minimax/envs/viz/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/minimax/envs/viz/grid_rendering.py b/src/minimax/envs/viz/grid_rendering.py new file mode 100644 index 0000000..9f7947f --- /dev/null +++ b/src/minimax/envs/viz/grid_rendering.py @@ -0,0 +1,133 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This file is modified from +https://github.com/Farama-Foundation/Minigrid + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +""" + +import math +import numpy as np + + +def downsample(img, factor): + """ + Downsample an image along both dimensions by some factor + """ + + assert img.shape[0] % factor == 0 + assert img.shape[1] % factor == 0 + + img = img.reshape([img.shape[0]//factor, factor, img.shape[1]//factor, factor, 3]) + img = img.mean(axis=3) + img = img.mean(axis=1) + + return img + +def fill_coords(img, fn, color): + """ + Fill pixels of an image with coordinates matching a filter function + """ + + for y in range(img.shape[0]): + for x in range(img.shape[1]): + yf = (y + 0.5) / img.shape[0] + xf = (x + 0.5) / img.shape[1] + if fn(xf, yf): + img[y, x] = color + + return img + +def rotate_fn(fin, cx, cy, theta): + def fout(x, y): + x = x - cx + y = y - cy + + x2 = cx + x * math.cos(-theta) - y * math.sin(-theta) + y2 = cy + y * math.cos(-theta) + x * math.sin(-theta) + + return fin(x2, y2) + + return fout + +def point_in_line(x0, y0, x1, y1, r): + p0 = np.array([x0, y0]) + p1 = np.array([x1, y1]) + dir = p1 - p0 + dist = np.linalg.norm(dir) + dir = dir / dist + + xmin = min(x0, x1) - r + xmax = max(x0, x1) + r + ymin = min(y0, y1) - r + ymax = max(y0, y1) + r + + def fn(x, y): + # Fast, early escape test + if x < xmin or x > xmax or y < ymin or y > ymax: + return False + + q = np.array([x, y]) + pq = q - p0 + + # Closest point on line + a = np.dot(pq, dir) + a = np.clip(a, 0, dist) + p = p0 + a * dir + + dist_to_line = np.linalg.norm(q - p) + return dist_to_line <= r + + return fn + +def point_in_circle(cx, cy, r): + def fn(x, y): + return (x-cx)*(x-cx) + (y-cy)*(y-cy) <= r * r + return fn + +def point_in_rect(xmin, xmax, ymin, ymax): + def fn(x, y): + return x >= xmin and x <= xmax and y >= ymin and y <= ymax + return fn + +def point_in_triangle(a, b, c): + a = np.array(a) + b = np.array(b) + c = np.array(c) + + def fn(x, y): + v0 = c - a + v1 = b - a + v2 = np.array((x, y)) - a + + # Compute dot products + dot00 = np.dot(v0, v0) + dot01 = np.dot(v0, v1) + dot02 = np.dot(v0, v2) + dot11 = np.dot(v1, v1) + dot12 = np.dot(v1, v2) + + # Compute barycentric coordinates + inv_denom = 1 / (dot00 * dot11 - dot01 * dot01) + u = (dot11 * dot02 - dot01 * dot12) * inv_denom + v = (dot00 * dot12 - dot01 * dot02) * inv_denom + + # Check if point is in triangle + return (u >= 0) and (v >= 0) and (u + v) < 1 + + return fn + +def highlight_img(img, color=(255, 255, 255), alpha=0.30): + """ + Add highlighting to an image + """ + # color = [60, 182, 234] + blend_img = img + alpha * (np.array(color, dtype=np.uint8) - img) + blend_img = blend_img.clip(0, 255).astype(np.uint8) + img[:, :, :] = blend_img diff --git a/src/minimax/envs/viz/grid_viz.py b/src/minimax/envs/viz/grid_viz.py new file mode 100644 index 0000000..d792aa0 --- /dev/null +++ b/src/minimax/envs/viz/grid_viz.py @@ -0,0 +1,272 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This file is modified from +https://github.com/Farama-Foundation/Minigrid + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +""" + +import math + +import numpy as np + +from minimax.envs.viz.window import Window +import minimax.envs.viz.grid_rendering as rendering +from minimax.envs.overcooked_proc.overcooked import OBJECT_TO_INDEX, COLOR_TO_INDEX, COLORS + + +INDEX_TO_COLOR = [k for k, v in COLOR_TO_INDEX.items()] +TILE_PIXELS = 32 + + +class GridVisualizer: + """ + Manages a window and renders contents of EnvState instances to it. + """ + tile_cache = {} + + def __init__(self): + self.window = None + + def _lazy_init_window(self): + if self.window is None: + self.window = Window('minimax') + + def show(self, block=False): + self._lazy_init_window() + self.window.show(block=block) + + def screenshot(self, path): + self._lazy_init_window() + self.window.save_img(path) + + def render(self, params, state, highlight=True, tile_size=TILE_PIXELS, maze_map=None): + return self._render_state(params, state, highlight, tile_size, maze_map) + + def render_grid(self, grid, tile_size=TILE_PIXELS, k_rot90=0, agent_dir_idx=None): + self._lazy_init_window() + + img = GridVisualizer._render_grid( + grid, + tile_size, + highlight_mask=None, + agent_dir_idx=agent_dir_idx, + ) + + if k_rot90 > 0: + img = np.rot90(img, k=k_rot90) + + self.window.show_img(img) + + def _render_state(self, params, state, highlight=True, tile_size=TILE_PIXELS, maze_map=None): + """ + Render the state + """ + self._lazy_init_window() + + if hasattr(params, 'agent_view_size'): + agent_view_size = params.agent_view_size + padding = agent_view_size-2 # show + # padding = 4 + grid = np.asarray( + state.maze_map[padding:-padding, padding:-padding, :]) + else: + assert maze_map is not None, 'Either params contains agent_view_size or explicit maze map is passed in.' + grid = np.asarray(maze_map) + + grid_offset = np.array([1, 1]) + h, w = grid.shape[:2] + + # === Compute highlight mask + if highlight: + highlight_mask = np.zeros(shape=(h, w), dtype=bool) + + f_vec = state.agent_dir + r_vec = np.array([-f_vec[1], f_vec[0]]) + + fwd_bound1 = state.agent_pos + fwd_bound2 = state.agent_pos + state.agent_dir*(agent_view_size-1) + side_bound1 = state.agent_pos - r_vec*(agent_view_size//2) + side_bound2 = state.agent_pos + r_vec*(agent_view_size//2) + + min_bound = np.min(np.stack([ + fwd_bound1, + fwd_bound2, + side_bound1, + side_bound2]) + grid_offset, 0) + + min_y = min(max(min_bound[1], 0), highlight_mask.shape[0]-1) + min_x = min(max(min_bound[0], 0), highlight_mask.shape[1]-1) + + max_y = min( + max(min_bound[1]+agent_view_size, 0), highlight_mask.shape[0]-1) + max_x = min( + max(min_bound[0]+agent_view_size, 0), highlight_mask.shape[1]-1) + + highlight_mask[min_y:max_y, min_x:max_x] = True + + # Render the whole grid + img = GridVisualizer._render_grid( + grid, + tile_size, + highlight_mask=highlight_mask if highlight else None, + agent_dir_idx=state.agent_dir_idx if hasattr( + state, 'agent_dir_idx') else 0 + ) + + self.window.show_img(img) + + @classmethod + def _render_obj( + cls, + obj, + img): + # Render each kind of object + obj_type = obj[0] + color = INDEX_TO_COLOR[obj[1]] + + if obj_type == OBJECT_TO_INDEX['wall']: + rendering.fill_coords( + img, rendering.point_in_rect(0, 1, 0, 1), COLORS[color]) + elif obj_type == OBJECT_TO_INDEX['goal']: + rendering.fill_coords( + img, rendering.point_in_rect(0, 1, 0, 1), COLORS[color]) + elif obj_type == OBJECT_TO_INDEX['agent']: + agent_dir_idx = obj[2] + tri_fn = rendering.point_in_triangle( + (0.12, 0.19), + (0.87, 0.50), + (0.12, 0.81), + ) + tri_fn = rendering.rotate_fn( + tri_fn, cx=0.5, cy=0.5, theta=0.5*math.pi*agent_dir_idx) + rendering.fill_coords(img, tri_fn, (255, 0, 0)) + # rendering.fill_coords(img, tri_fn, (60, 182, 234)) + + elif obj_type == OBJECT_TO_INDEX['empty']: + rendering.fill_coords( + img, rendering.point_in_rect(0, 1, 0, 1), COLORS[color]) + elif obj_type == OBJECT_TO_INDEX['lava']: + c = (255, 128, 0) + + # Background color + rendering.fill_coords(img, rendering.point_in_rect(0, 1, 0, 1), c) + + # Little waves + for i in range(3): + ylo = 0.3 + 0.2 * i + yhi = 0.4 + 0.2 * i + rendering.fill_coords(img, rendering.point_in_line( + 0.1, ylo, 0.3, yhi, r=0.03), (0, 0, 0)) + rendering.fill_coords(img, rendering.point_in_line( + 0.3, yhi, 0.5, ylo, r=0.03), (0, 0, 0)) + rendering.fill_coords(img, rendering.point_in_line( + 0.5, ylo, 0.7, yhi, r=0.03), (0, 0, 0)) + rendering.fill_coords(img, rendering.point_in_line( + 0.7, yhi, 0.9, ylo, r=0.03), (0, 0, 0)) + else: + raise ValueError( + f'Rendering object at index {obj_type} is currently unsupported.') + + @classmethod + def _render_tile( + cls, + obj, + highlight=False, + agent_dir_idx=None, + tile_size=TILE_PIXELS, + subdivs=3 + ): + """ + Render a tile and cache the result + """ + # Hash map lookup key for the cache + if obj is not None and \ + obj[0] == OBJECT_TO_INDEX['agent'] and \ + agent_dir_idx is not None: + obj = np.array(obj) + obj[-1] = agent_dir_idx + + no_object = obj is None or ( + obj[0] in [OBJECT_TO_INDEX['empty'], OBJECT_TO_INDEX['unseen']] + and obj[2] == 0 + ) + + if not no_object: + key = (*obj, highlight, tile_size) + else: + key = (obj, highlight, tile_size) + + if key in cls.tile_cache: + return cls.tile_cache[key] + + img = np.zeros(shape=(tile_size * subdivs, + tile_size * subdivs, 3), dtype=np.uint8) + + # Draw the grid lines (top and left edges) + rendering.fill_coords(img, rendering.point_in_rect( + 0, 0.031, 0, 1), (100, 100, 100)) + rendering.fill_coords(img, rendering.point_in_rect( + 0, 1, 0, 0.031), (100, 100, 100)) + + if not no_object: + GridVisualizer._render_obj(obj, img) + + if highlight: + rendering.highlight_img(img) + + # Downsample the image to perform supersampling/anti-aliasing + img = rendering.downsample(img, subdivs) + + # Cache the rendered tile + cls.tile_cache[key] = img + + return img + + @classmethod + def _render_grid( + cls, + grid, + tile_size=TILE_PIXELS, + highlight_mask=None, + agent_dir_idx=None): + if highlight_mask is None: + highlight_mask = np.zeros(shape=grid.shape[:2], dtype=np.bool_) + + # Compute the total grid size in pixels + width_px = grid.shape[1]*tile_size + height_px = grid.shape[0]*tile_size + + img = np.zeros(shape=(height_px, width_px, 3), dtype=np.uint8) + + # Render the grid + for y in range(grid.shape[0]): + for x in range(grid.shape[1]): + obj = grid[y, x, :] + if obj[0] in [OBJECT_TO_INDEX['empty'], OBJECT_TO_INDEX['unseen']] \ + and obj[2] == 0: + obj = None + + tile_img = GridVisualizer._render_tile( + obj, + highlight=highlight_mask[y, x], + tile_size=tile_size, + agent_dir_idx=agent_dir_idx, + ) + + ymin = y*tile_size + ymax = (y+1)*tile_size + xmin = x*tile_size + xmax = (x+1)*tile_size + img[ymin:ymax, xmin:xmax, :] = tile_img + + return img + + def close(self): + self.window.close() diff --git a/src/minimax/envs/viz/overcooked_visualizer.py b/src/minimax/envs/viz/overcooked_visualizer.py new file mode 100644 index 0000000..022a6b8 --- /dev/null +++ b/src/minimax/envs/viz/overcooked_visualizer.py @@ -0,0 +1,378 @@ +import math + +import numpy as np + +from minimax.envs.viz.window import Window +import minimax.envs.viz.grid_rendering as rendering +from minimax.envs.overcooked_proc.common import OBJECT_TO_INDEX, COLOR_TO_INDEX, COLORS + + +INDEX_TO_COLOR = [k for k, v in COLOR_TO_INDEX.items()] +TILE_PIXELS = 32 + +COLOR_TO_AGENT_INDEX = {0: 0, 2: 1} # Hardcoded. Red is first, blue is second + + +class OvercookedVisualizer: + """ + Manages a window and renders contents of EnvState instances to it. + """ + tile_cache = {} + + def __init__(self): + self.window = None + + def _lazy_init_window(self): + if self.window is None: + self.window = Window('minimax') + + def show(self, block=False): + self._lazy_init_window() + self.window.show(block=block) + + def render(self, agent_view_size, state, highlight=True, tile_size=TILE_PIXELS): + """Method for rendering the state in a window. Esp. useful for interactive mode.""" + return self._render_state(agent_view_size, state, highlight, tile_size) + + def animate(self, state_seq, agent_view_size, filename="animation.gif"): + """Animate a gif give a state sequence and save if to file.""" + import imageio + + padding = agent_view_size - 2 # show + + def get_frame(state): + grid = np.asarray( + state.maze_map[padding:-padding, padding:-padding, :]) + # Render the state + frame = OvercookedVisualizer._render_grid( + grid, + tile_size=TILE_PIXELS, + highlight_mask=None, + agent_dir_idx=state.agent_dir_idx, + agent_inv=state.agent_inv + ) + return frame + + frame_seq = [get_frame(state) for state in state_seq] + + imageio.mimsave(filename, frame_seq, 'GIF', duration=0.5) + + def render_grid(self, grid, tile_size=TILE_PIXELS, k_rot90=0, agent_dir_idx=None): + self._lazy_init_window() + + img = OvercookedVisualizer._render_grid( + grid, + tile_size, + highlight_mask=None, + agent_dir_idx=agent_dir_idx, + ) + # img = np.transpose(img, axes=(1,0,2)) + if k_rot90 > 0: + img = np.rot90(img, k=k_rot90) + + self.window.show_img(img) + + def _render_state(self, agent_view_size, state, highlight=True, tile_size=TILE_PIXELS): + """ + Render the state + """ + self._lazy_init_window() + + padding = agent_view_size-2 # show + grid = np.asarray( + state.maze_map[padding:-padding, padding:-padding, :]) + grid_offset = np.array([1, 1]) + h, w = grid.shape[:2] + # === Compute highlight mask + highlight_mask = np.zeros(shape=(h, w), dtype=bool) + + if highlight: + f_vec = state.agent_dir + r_vec = np.array([-f_vec[1], f_vec[0]]) + + fwd_bound1 = state.agent_pos + fwd_bound2 = state.agent_pos + state.agent_dir*(agent_view_size-1) + side_bound1 = state.agent_pos - r_vec*(agent_view_size//2) + side_bound2 = state.agent_pos + r_vec*(agent_view_size//2) + + min_bound = np.min(np.stack([ + fwd_bound1, + fwd_bound2, + side_bound1, + side_bound2]) + grid_offset, 0) + + min_y = min(max(min_bound[1], 0), highlight_mask.shape[0]-1) + min_x = min(max(min_bound[0], 0), highlight_mask.shape[1]-1) + + max_y = min( + max(min_bound[1]+agent_view_size, 0), highlight_mask.shape[0]-1) + max_x = min( + max(min_bound[0]+agent_view_size, 0), highlight_mask.shape[1]-1) + + highlight_mask[min_y:max_y, min_x:max_x] = True + + # Render the whole grid + img = OvercookedVisualizer._render_grid( + grid, + tile_size, + highlight_mask=highlight_mask if highlight else None, + agent_dir_idx=state.agent_dir_idx, + agent_inv=state.agent_inv + ) + self.window.show_img(img) + + @classmethod + def _render_obj( + cls, + obj, + img): + # Render each kind of object + obj_type = obj[0] + color = INDEX_TO_COLOR[obj[1]] + + if obj_type == OBJECT_TO_INDEX['wall']: + rendering.fill_coords( + img, rendering.point_in_rect(0, 1, 0, 1), COLORS[color]) + elif obj_type == OBJECT_TO_INDEX['goal']: + rendering.fill_coords( + img, rendering.point_in_rect(0, 1, 0, 1), COLORS["grey"]) + rendering.fill_coords(img, rendering.point_in_rect( + 0.1, 0.9, 0.1, 0.9), COLORS[color]) + elif obj_type == OBJECT_TO_INDEX['agent']: + agent_dir_idx = obj[2] + tri_fn = rendering.point_in_triangle( + (0.12, 0.19), + (0.87, 0.50), + (0.12, 0.81), + ) + tri_fn = rendering.rotate_fn( + tri_fn, cx=0.5, cy=0.5, theta=0.5*math.pi*agent_dir_idx) + rendering.fill_coords(img, tri_fn, COLORS[color]) + elif obj_type == OBJECT_TO_INDEX['empty']: + rendering.fill_coords( + img, rendering.point_in_rect(0, 1, 0, 1), COLORS[color]) + elif obj_type == OBJECT_TO_INDEX['onion_pile']: + rendering.fill_coords( + img, rendering.point_in_rect(0, 1, 0, 1), COLORS["grey"]) + onion_fns = [rendering.point_in_circle(*coord, 0.15) for coord in [(0.5, 0.15), (0.3, 0.4), (0.8, 0.35), + (0.4, 0.8), (0.75, 0.75)]] + [rendering.fill_coords(img, onion_fn, COLORS[color]) + for onion_fn in onion_fns] + elif obj_type == OBJECT_TO_INDEX['onion']: + rendering.fill_coords( + img, rendering.point_in_rect(0, 1, 0, 1), COLORS["grey"]) + onion_fn = rendering.point_in_circle(0.5, 0.5, 0.15) + rendering.fill_coords(img, onion_fn, COLORS[color]) + elif obj_type == OBJECT_TO_INDEX['plate_pile']: + rendering.fill_coords( + img, rendering.point_in_rect(0, 1, 0, 1), COLORS["grey"]) + plate_fns = [rendering.point_in_circle(*coord, 0.2) for coord in [(0.3, 0.3), (0.75, 0.42), + (0.4, 0.75)]] + [rendering.fill_coords(img, plate_fn, COLORS[color]) + for plate_fn in plate_fns] + elif obj_type == OBJECT_TO_INDEX['plate']: + rendering.fill_coords( + img, rendering.point_in_rect(0, 1, 0, 1), COLORS["grey"]) + plate_fn = rendering.point_in_circle(0.5, 0.5, 0.2) + rendering.fill_coords(img, plate_fn, COLORS[color]) + elif obj_type == OBJECT_TO_INDEX['dish']: + rendering.fill_coords( + img, rendering.point_in_rect(0, 1, 0, 1), COLORS["grey"]) + plate_fn = rendering.point_in_circle(0.5, 0.5, 0.2) + rendering.fill_coords(img, plate_fn, COLORS[color]) + onion_fn = rendering.point_in_circle(0.5, 0.5, 0.13) + rendering.fill_coords(img, onion_fn, COLORS["orange"]) + elif obj_type == OBJECT_TO_INDEX['pot']: + OvercookedVisualizer._render_pot(obj, img) + # rendering.fill_coords(img, rendering.point_in_rect(0, 1, 0, 1), COLORS["grey"]) + # pot_fns = [rendering.point_in_rect(0.1, 0.9, 0.3, 0.9), + # rendering.point_in_rect(0.1, 0.9, 0.20, 0.23), + # rendering.point_in_rect(0.4, 0.6, 0.15, 0.20),] + # [rendering.fill_coords(img, pot_fn, COLORS[color]) for pot_fn in pot_fns] + else: + raise ValueError( + f'Rendering object at index {obj_type} is currently unsupported.') + + @classmethod + def _render_pot( + cls, + obj, + img): + pot_status = obj[-1] + num_onions = np.max([23-pot_status, 0]) + is_cooking = np.array((pot_status < 20) * (pot_status > 0)) + is_done = np.array(pot_status == 0) + + pot_fn = rendering.point_in_rect(0.1, 0.9, 0.33, 0.9) + lid_fn = rendering.point_in_rect(0.1, 0.9, 0.21, 0.25) + handle_fn = rendering.point_in_rect(0.4, 0.6, 0.16, 0.21) + + rendering.fill_coords( + img, rendering.point_in_rect(0, 1, 0, 1), COLORS["grey"]) + + # Render onions in pot + if num_onions > 0 and not is_done: + onion_fns = [rendering.point_in_circle( + *coord, 0.13) for coord in [(0.23, 0.33), (0.77, 0.33), (0.50, 0.33)]] + onion_fns = onion_fns[:num_onions] + [rendering.fill_coords(img, onion_fn, COLORS["yellow"]) + for onion_fn in onion_fns] + if not is_cooking: + lid_fn = rendering.rotate_fn( + lid_fn, cx=0.1, cy=0.25, theta=-0.1 * math.pi) + handle_fn = rendering.rotate_fn( + handle_fn, cx=0.1, cy=0.25, theta=-0.1 * math.pi) + + # Render done soup + if is_done: + soup_fn = rendering.point_in_rect(0.12, 0.88, 0.23, 0.35) + rendering.fill_coords(img, soup_fn, COLORS["orange"]) + + # Render the pot itself + pot_fns = [pot_fn, lid_fn, handle_fn] + [rendering.fill_coords(img, pot_fn, COLORS["black"]) + for pot_fn in pot_fns] + + # Render progress bar + if is_cooking: + progress_fn = rendering.point_in_rect( + 0.1, 0.9-(0.9-0.1)/20*pot_status, 0.83, 0.88) + rendering.fill_coords(img, progress_fn, COLORS["green"]) + + @classmethod + def _render_inv( + cls, + obj, + img): + # Render each kind of object + obj_type = obj[0] + if obj_type == OBJECT_TO_INDEX['empty']: + pass + elif obj_type == OBJECT_TO_INDEX['onion']: + onion_fn = rendering.point_in_circle(0.75, 0.75, 0.15) + rendering.fill_coords(img, onion_fn, COLORS["yellow"]) + elif obj_type == OBJECT_TO_INDEX['plate']: + plate_fn = rendering.point_in_circle(0.75, 0.75, 0.2) + rendering.fill_coords(img, plate_fn, COLORS["white"]) + elif obj_type == OBJECT_TO_INDEX['dish']: + plate_fn = rendering.point_in_circle(0.75, 0.75, 0.2) + rendering.fill_coords(img, plate_fn, COLORS["white"]) + onion_fn = rendering.point_in_circle(0.75, 0.75, 0.13) + rendering.fill_coords(img, onion_fn, COLORS["orange"]) + else: + raise ValueError( + f'Rendering object at index {obj_type} is currently unsupported.') + + @classmethod + def _render_tile( + cls, + obj, + highlight=False, + agent_dir_idx=None, + agent_inv=None, + tile_size=TILE_PIXELS, + subdivs=3 + ): + """ + Render a tile and cache the result + """ + # Hash map lookup key for the cache + if obj is not None and obj[0] == OBJECT_TO_INDEX['agent']: + # Get inventory of this specific agent + if agent_inv is not None: + color_idx = obj[1] + agent_inv = agent_inv[COLOR_TO_AGENT_INDEX[color_idx]] + agent_inv = np.array([agent_inv, -1, 0]) + + if agent_dir_idx is not None: + obj = np.array(obj) + + if len(agent_dir_idx) == 1: + # Hacky way of making agent views orientations consistent with global view + obj[-1] = agent_dir_idx[0] + + no_object = obj is None or ( + obj[0] in [OBJECT_TO_INDEX['empty'], OBJECT_TO_INDEX['unseen']] + and obj[2] == 0 + ) + + if not no_object: + if agent_inv is not None and obj[0] == OBJECT_TO_INDEX['agent']: + key = (*obj, *agent_inv, highlight, tile_size) + else: + key = (*obj, highlight, tile_size) + else: + key = (obj, highlight, tile_size) + + if key in cls.tile_cache: + return cls.tile_cache[key] + + img = np.zeros(shape=(tile_size * subdivs, + tile_size * subdivs, 3), dtype=np.uint8) + + # Draw the grid lines (top and left edges) + rendering.fill_coords(img, rendering.point_in_rect( + 0, 0.031, 0, 1), (100, 100, 100)) + rendering.fill_coords(img, rendering.point_in_rect( + 0, 1, 0, 0.031), (100, 100, 100)) + + if not no_object: + OvercookedVisualizer._render_obj(obj, img) + # render inventory + if agent_inv is not None and obj[0] == OBJECT_TO_INDEX['agent']: + OvercookedVisualizer._render_inv(agent_inv, img) + + if highlight: + rendering.highlight_img(img) + + # Downsample the image to perform supersampling/anti-aliasing + img = rendering.downsample(img, subdivs) + + # Cache the rendered tile + cls.tile_cache[key] = img + + return img + + @classmethod + def _render_grid( + cls, + grid, + tile_size=TILE_PIXELS, + highlight_mask=None, + agent_dir_idx=None, + agent_inv=None): + if highlight_mask is None: + highlight_mask = np.zeros(shape=grid.shape[:2], dtype=bool) + + # Compute the total grid size in pixels + width_px = grid.shape[1]*tile_size + height_px = grid.shape[0]*tile_size + + img = np.zeros(shape=(height_px, width_px, 3), dtype=np.uint8) + + # Render the grid + for y in range(grid.shape[0]): + for x in range(grid.shape[1]): + obj = grid[y, x, :] + if obj[0] in [OBJECT_TO_INDEX['empty'], OBJECT_TO_INDEX['unseen']] \ + and obj[2] == 0: + obj = None + + tile_img = OvercookedVisualizer._render_tile( + obj, + highlight=highlight_mask[y, x], + tile_size=tile_size, + agent_dir_idx=agent_dir_idx, + agent_inv=agent_inv, + ) + + ymin = y*tile_size + ymax = (y+1)*tile_size + xmin = x*tile_size + xmax = (x+1)*tile_size + img[ymin:ymax, xmin:xmax, :] = tile_img + + return img + + def close(self): + self.window.close() diff --git a/src/minimax/envs/viz/window.py b/src/minimax/envs/viz/window.py new file mode 100644 index 0000000..d97665b --- /dev/null +++ b/src/minimax/envs/viz/window.py @@ -0,0 +1,107 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This file is modified from +https://github.com/Farama-Foundation/Minigrid + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +""" + +import sys +import numpy as np + +# Only ask users to install matplotlib if they actually need it +try: + import matplotlib.pyplot as plt +except: + print('To display the environment in a window, please install matplotlib, eg:') + print('pip3 install --user matplotlib') + sys.exit(-1) + +class Window: + """ + Window to draw a gridworld instance using Matplotlib + """ + + def __init__(self, title): + self.fig = None + + self.imshow_obj = None + + # Create the figure and axes + self.fig, self.ax = plt.subplots() + + # Show the env name in the window title + self.fig.canvas.manager.set_window_title(title) + + # Turn off x/y axis numbering/ticks + self.ax.set_xticks([], []) + self.ax.set_yticks([], []) + + # Flag indicating the window was closed + self.closed = False + + def close_handler(evt): + self.closed = True + + self.fig.canvas.mpl_connect('close_event', close_handler) + + def show_img(self, img): + """ + Show an image or update the image being shown + """ + + # Show the first image of the environment + if self.imshow_obj is None: + self.imshow_obj = self.ax.imshow(img, interpolation='bilinear') + + self.imshow_obj.set_data(img) + self.fig.canvas.draw() + + # Let matplotlib process UI events + # This is needed for interactive mode to work properly + plt.pause(0.001) + + def save_img(self, path): + plt.savefig(path, bbox_inches='tight', pad_inches=0) + + def set_caption(self, text): + """ + Set/update the caption text below the image + """ + + plt.xlabel(text) + + def reg_key_handler(self, key_handler): + """ + Register a keyboard event handler + """ + + # Keyboard handler + self.fig.canvas.mpl_connect('key_press_event', key_handler) + + def show(self, block=True): + """ + Show the window, and start an event loop + """ + + # If not blocking, trigger interactive mode + if not block: + plt.ion() + + # Show the plot + # In non-interative mode, this enters the matplotlib event loop + # In interactive mode, this call does not block + plt.show() + + def close(self): + """ + Close the window + """ + + plt.close() diff --git a/src/minimax/envs/wrappers/__init__.py b/src/minimax/envs/wrappers/__init__.py new file mode 100644 index 0000000..d53f4e9 --- /dev/null +++ b/src/minimax/envs/wrappers/__init__.py @@ -0,0 +1,14 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from .env_wrapper import EnvWrapper +from .monitor_return import MonitorReturnWrapper +from .monitor_ep_metrics import MonitorEpisodicMetricsWrapper +from .world_state_wrapper import WorldStateWrapper + +from .ued_env_wrapper import UEDEnvWrapper \ No newline at end of file diff --git a/src/minimax/envs/wrappers/env_wrapper.py b/src/minimax/envs/wrappers/env_wrapper.py new file mode 100644 index 0000000..37e47c7 --- /dev/null +++ b/src/minimax/envs/wrappers/env_wrapper.py @@ -0,0 +1,114 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial + +import jax +import chex +from typing import Tuple, Union, Optional + +from minimax.envs.environment import EnvState + + +class EnvWrapper: + """ + Abstract base class for an env wrapper. + """ + + def __init__(self, env): + self._env = env + + self._wrap_level = 1 + while hasattr(env, '_env'): + if isinstance(env, EnvWrapper): + self._wrap_level += 1 + + env = env._env + + @classmethod + def is_compatible(cls, env): + return True + + @property + def base_env(self): + env = self + for i in range(self._wrap_level): + env = env._env + + return env + + def reset_extra(self): + return {} + + def get_monitored_metrics(self): + # breakpoint() + # if self._wrap_level > 1: + return self._env.get_monitored_metrics() + # return () + + def _append_extra_to_tuple(self, _tuple, extra=None): + if extra is None: + extra = self.reset_extra() + + if self._wrap_level > 1 and len(_tuple) > 2: + _tuple[-1].update(extra) + else: + _tuple = _tuple + (extra,) + + return _tuple + + def step( + self, + key: chex.PRNGKey, + state: EnvState, + action: Union[int, float], + reset_state: Optional[chex.ArrayTree] = None, + extra: dict = None, + ) -> Tuple[chex.Array, EnvState, float, bool]: + if self._wrap_level > 1: + return self._env.step(key, state, action, reset_state, extra) + else: + _tuple = self._env.step( + key, state, action, reset_state=reset_state + ) + return self._append_extra_to_tuple(_tuple, extra) + + def reset( + self, + key: chex.PRNGKey, + ) -> Tuple[chex.Array, EnvState, chex.ArrayTree]: + _tuple = self._env.reset(key) + return self._append_extra_to_tuple(_tuple) + + def set_state( + self, + state: EnvState, + ) -> Tuple[chex.ArrayTree, EnvState, chex.ArrayTree]: + _tuple = self._env.set_state(state) + + return self._append_extra_to_tuple(_tuple) + + def set_env_instance( + self, + encoding: chex.ArrayTree, + ) -> Tuple[chex.ArrayTree, EnvState, chex.ArrayTree]: + _tuple = self._env.set_env_instance(encoding) + + return self._append_extra_to_tuple(_tuple) + + def reset_student( + self, + key: chex.PRNGKey, + state: chex.ArrayTree, + ) -> Tuple[chex.ArrayTree, EnvState, chex.ArrayTree]: + _tuple = self._env.reset_student(key, state) + + return self._append_extra_to_tuple(_tuple) + + def __getattr__(self, attr): + return getattr(self._env, attr) diff --git a/src/minimax/envs/wrappers/monitor_ep_metrics.py b/src/minimax/envs/wrappers/monitor_ep_metrics.py new file mode 100644 index 0000000..a7714f3 --- /dev/null +++ b/src/minimax/envs/wrappers/monitor_ep_metrics.py @@ -0,0 +1,84 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial + +import jax +import jax.numpy as jnp +import chex +from typing import Tuple, Union, Optional + +from .env_wrapper import EnvWrapper +from minimax.envs.environment import EnvState + + +class MonitorEpisodicMetricsWrapper(EnvWrapper): + """ + Tracks episodic metrics about environment instances. + """ + def __init__(self, env): + super().__init__(env) + + base_env = env.base_env if hasattr(env, 'base_env') else env + _env = base_env.env if hasattr(base_env, 'env') else base_env + + self.metrics = () + if hasattr(_env, 'get_episodic_metrics'): + reset_tuple = _env.reset(jax.random.PRNGKey(0)) + dummy_state = reset_tuple[1] + + self.metrics = tuple({ + k: jnp.zeros_like(v) \ + for k,v in _env.get_episodic_metrics(dummy_state).items() + }.keys()) + + @classmethod + def is_compatible(cls, env): + _env = env.env if hasattr(env, 'env') else env + return hasattr(_env, 'get_episodic_metrics') + + def get_monitored_metrics(self): + metrics = tuple(f'ep/{m}' for m in self.metrics) + if self._wrap_level > 1: + return self._env.get_monitored_metrics() + metrics + else: + return self.metrics + + def step( + self, + key: chex.PRNGKey, + state: EnvState, + action: Union[int, float], + reset_state: Optional[chex.ArrayTree] = None, + extra: dict = None, + ) -> Tuple[chex.Array, EnvState, float, bool]: + step_kwargs = dict( + reset_state=reset_state + ) + if self._wrap_level > 1: + step_kwargs.update(dict( + extra=extra + )) + + step = self._env.step( + key, + state, + action, + **step_kwargs) + + if len(step) == 5: + obs, state, reward, done, info = step + else: + obs, state, reward, done, info, extra = step + + if len(self.metrics) > 0: + for m in self.metrics: + info[f'ep/{m}'] = info[m] + del info[m] + + return obs, state, reward, done, info, extra diff --git a/src/minimax/envs/wrappers/monitor_return.py b/src/minimax/envs/wrappers/monitor_return.py new file mode 100644 index 0000000..206eb62 --- /dev/null +++ b/src/minimax/envs/wrappers/monitor_return.py @@ -0,0 +1,76 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial + +import jax +import chex +from typing import Tuple, Union, Optional + +from .env_wrapper import EnvWrapper +from minimax.envs.environment import EnvState + + +class MonitorReturnWrapper(EnvWrapper): + """ + Tracks episodic returns and, optionally, environment metrics. + """ + + def reset_extra(self): + if self._wrap_level > 1: + extra = self._env.reset_extra() + else: + extra = {} + + extra.update({ + 'ep_return': 0., + }) + + return extra + + def get_monitored_metrics(self): + return super().get_monitored_metrics() + ('return',) + + def step( + self, + key: chex.PRNGKey, + state: EnvState, + action: Union[int, float], + reset_state: Optional[chex.ArrayTree] = None, + extra: dict = None, + ) -> Tuple[chex.Array, EnvState, float, bool]: + step_kwargs = dict( + reset_state=reset_state + ) + if self._wrap_level > 1: + step_kwargs.update(dict( + extra=extra + )) + + step = self._env.step( + key, + state, + action, + **step_kwargs) + + if len(step) == 5: + obs, state, reward, done, info = step + else: + obs, state, reward, done, info, extra = step + + if type(reward) == dict: + reward = reward['agent_0'] # NOTE: Fully Cooperative taks + + if type(done) == dict: + done = done['__all__'] + # Track returns + extra['ep_return'] += reward + info['return'] = done*extra['ep_return'] + extra['ep_return'] *= (1-done) + + return obs, state, reward, done, info, extra diff --git a/src/minimax/envs/wrappers/ued_env_wrapper.py b/src/minimax/envs/wrappers/ued_env_wrapper.py new file mode 100644 index 0000000..9f96924 --- /dev/null +++ b/src/minimax/envs/wrappers/ued_env_wrapper.py @@ -0,0 +1,85 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial + +import jax +import chex +from typing import Tuple, Union, Optional + +from minimax.envs.environment import Environment, EnvState + + +class UEDEnvWrapper: + """ + Abstract base class for an env wrapper. + """ + def __init__(self, env): + self._env = env + + self._wrap_level = 1 + while hasattr(env, '_env'): + if not isinstance(env, Environment): + self._wrap_level += 1 + + env = env._env + + @classmethod + def is_compatible(cls, env): + return True + + @property + def base_env(self): + env = self + for i in range(self._wrap_level): + env = env._env + + return env + + def reset_extra(self): + return {} + + def get_monitored_metrics(self): + if self._wrap_level > 1: + return self._env.get_monitored_metrics() + return () + + def _append_extra_to_tuple(self, _tuple, extra=None): + if extra is None: + extra = self.reset_extra() + + if self._wrap_level > 1: + _tuple[-1].update(extra) + else: + _tuple = _tuple + (extra,) + + return _tuple + + def reset_teacher( + self, + rng: chex.PRNGKey + ) -> Tuple[chex.ArrayTree, EnvState]: + _tuple = self._env.reset_teacher(rng) + + return self._append_extra_to_tuple(_tuple) + + def step_teacher( + self, + rng: chex.PRNGKey, + ued_state: EnvState, + action: Union[int, float], + extra: dict = None, + ) -> Tuple[chex.ArrayTree, EnvState, float, bool, dict]: + if self._wrap_level > 1: + return self._env.step_teacher(rng, ued_state, action, extra) + else: + _tuple = self._env.step_teacher(rng, ued_state, action) + return self._append_extra_to_tuple(_tuple) + + def __getattr__(self, attr): + return getattr(self._env, attr) diff --git a/src/minimax/envs/wrappers/world_state_wrapper.py b/src/minimax/envs/wrappers/world_state_wrapper.py new file mode 100644 index 0000000..06ea805 --- /dev/null +++ b/src/minimax/envs/wrappers/world_state_wrapper.py @@ -0,0 +1,160 @@ +from functools import partial +import jax +import jax.numpy as jnp + +import chex +from typing import Union, Optional + +from minimax.envs.environment import EnvState + +from minimax.envs import environment +from minimax.envs.wrappers.env_wrapper import EnvWrapper + + +class JaxMARLWrapper(object): + """Base class for all jaxmarl wrappers. + Copied from the JaxMARL project: https://github.com/FLAIROx/JaxMARL + """ + + def __init__(self, env: environment.Environment): + self._env = env + + def __getattr__(self, name: str): + return getattr(self._env, name) + + def _batchify_floats(self, x: dict): + return jnp.stack([x[a] for a in self._env.agents]) + + +class WorldStateWrapper(EnvWrapper): + + def __init__(self, env): + self._env = env + + self._wrap_level = 1 + while hasattr(env, '_env'): + if isinstance(env, EnvWrapper): + self._wrap_level += 1 + + env = env._env + + def __getattr__(self, name: str): + return getattr(self._env, name) + + def _batchify_floats(self, x: dict): + return jnp.stack([x[a] for a in self._env.agents]) + + @partial(jax.jit, static_argnums=0) + def world_state(self, obs): + """ + For each agent: [agent obs, all other agent obs] + + NOTE: This assumes two agents! + """ + # This is consistent with the OvercookedEnv implementation. + world_state_0 = jnp.concatenate( + [obs['agent_0'], obs['agent_1']], axis=-1) + world_state_1 = jnp.concatenate( + [obs['agent_1'], obs['agent_0']], axis=-1) + + return { + 'agent_0': world_state_0, + 'agent_1': world_state_1 + } + + @partial(jax.jit, static_argnums=0) + def reset(self, key): + res = self._env.reset(key) + obs = res[0] + world_state = self.world_state(obs) + obs["world_state"] = world_state + _tuple = (obs, *res[1:]) + return self._append_extra_to_tuple(_tuple) + + @partial(jax.jit, static_argnums=0) + def step(self, + key: chex.PRNGKey, + state: EnvState, + action: Union[int, float], + reset_state: Optional[chex.ArrayTree] = None, + extra: dict = None, + **kwargs): + if self._wrap_level > 1: + obs, env_state, reward, done, info = self._env.step( + key, state, action, **kwargs + ) + world_state = self.world_state(obs) + obs["world_state"] = world_state + return obs, env_state, reward, done, info + else: + obs, env_state, reward, done, info = self._env.step( + key, state, action, **kwargs + ) + world_state = self.world_state(obs) + obs["world_state"] = world_state + _tuple = (obs, env_state, reward, done, info) + return self._append_extra_to_tuple(_tuple, extra) + + @partial(jax.jit, static_argnums=0) + def set_state(self, state): + if self._wrap_level > 1: + obs, state = self._env.set_state(state) + world_state = self.world_state(obs) + obs["world_state"] = world_state + return obs, state + else: + obs, state = self._env.set_state(state) + world_state = self.world_state(obs) + obs["world_state"] = world_state + _tuple = (obs, state) + return self._append_extra_to_tuple(_tuple) + + @partial(jax.jit, static_argnums=0) + def reset_student( + self, + key, + state + ): + res = self._env.reset_student(key, state) + obs = res[0] + world_state = self.world_state(obs) + obs["world_state"] = world_state + return obs, *res[1:] + + def world_state_size(self): + spaces = [ + jnp.zeros(self._env.observation_space().shape) for _ in self._env.agents] + y = jnp.concatenate(spaces, axis=-1).shape + return y + + def reset_extra(self): + if self._wrap_level > 1: + extra = self._env.reset_extra() + else: + extra = {} + return extra + + def reset_teacher( + self, + rng + ): + _tuple = self._env.reset_teacher(rng) + + return self._append_extra_to_tuple(_tuple) + + def step_teacher( + self, + rng, + ued_state, + action, + extra: dict = None, + ): + if self._wrap_level > 1: + return self._env.step_teacher(rng, ued_state, action, extra) + else: + _tuple = self._env.step_teacher(rng, ued_state, action) + return self._append_extra_to_tuple(_tuple) + + @classmethod + def is_compatible(cls, env): + return env.name == "Overcooked" diff --git a/src/minimax/evaluate.py b/src/minimax/evaluate.py new file mode 100644 index 0000000..2de0b33 --- /dev/null +++ b/src/minimax/evaluate.py @@ -0,0 +1,244 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +import os +import json +import re +import fnmatch +import sys +from collections import defaultdict + +import numpy as np +import pandas as pd +import scipy.stats as spstats +import jax +import jax.numpy as jnp +from tqdm import tqdm + +from minimax.util.parsnip import Parsnip +from minimax.util.checkpoint import load_pkl_object, load_config +from minimax.util.loggers import HumanOutputFormat +from minimax.util.rl import AgentPop +import minimax.models as models +import minimax.agents as agents + + +parser = Parsnip() + +# ==== Define top-level arguments +parser.add_argument( + '--seed', + type=int, + default=1, + help='Random seed.') +parser.add_argument( + '--log_dir', + type=str, + default='~/logs/minimax', + help='Log directory containing experiment dirs.') +parser.add_argument( + '--xpid', + type=str, + default='latest', + help='Experiment ID dir name for model.') +parser.add_argument( + '--xpid_prefix', + type=str, + default=None, + help='Experiment ID dir name for model.') +parser.add_argument( + '--checkpoint_name', + type=str, + default='checkpoint', + help='Name of checkpoint .pkl.') +parser.add_argument( + '--env_names', + type=str, + help='csv of evaluation environments.') +parser.add_argument( + '--n_episodes', + type=int, + default=1, + help='Number of evaluation episodes.') +parser.add_argument( + '--agent_idxs', + type=str, + default='*', + help="Indices of agents to evaluate. '*' indicates all.") +parser.add_argument( + '--render_mode', + type=str, + nargs='?', + const=True, + default=None, + help='Visualize episodes.') +parser.add_argument( + '--results_path', + type=str, + default='results/', + help='Results dir.') +parser.add_argument( + '--results_fname', + type=str, + default=None, + help='Results filename (without .csv).') + + +if __name__ == '__main__': + """ + Usage: + python -m eval \ + --xpid= \ + --env_names="Maze-SixteenRooms" \ + --n_episodes=100 \ + --agent_idxs=0 + """ + args = parser.parse_args() + + log_dir_path = os.path.expandvars(os.path.expanduser(args.log_dir)) + + xpids = [] + if args.xpid_prefix is not None: + # Get all matching xpid directories + all_xpids = fnmatch.filter(os.listdir( + log_dir_path), f"{args.xpid_prefix}*") + filter_re = re.compile('.*_[0-9]*$') + xpids = [x for x in all_xpids if filter_re.match(x)] + else: + xpids = [args.xpid] + + pbar = tqdm(total=len(xpids)) + + all_eval_stats = defaultdict(list) + for xpid in xpids: + xpid_dir_path = os.path.join(log_dir_path, xpid) + checkpoint_path = os.path.join( + xpid_dir_path, f'{args.checkpoint_name}.pkl') + meta_path = os.path.join(xpid_dir_path, f'meta.json') + + # Load checkpoint info + if not os.path.exists(meta_path): + print(f'Configuration at {meta_path} does not exist. Skipping...') + continue + + if not os.path.exists(checkpoint_path): + print( + f'Checkpoint path {checkpoint_path} does not exist. Skipping...') + continue + + xp_args = load_config(meta_path) + + agent_idxs = args.agent_idxs + if agent_idxs == '*': + agent_idxs = np.arange(xp_args.train_runner_args.n_students) + else: + agent_idxs = \ + np.array([int(x) for x in agent_idxs.split(',')]) + assert np.max(agent_idxs) <= xp_args.train_runner_args.n_students, \ + 'Agent index is out of bounds.' + + runner_state = load_pkl_object(checkpoint_path) + if "params" in runner_state[1].keys(): + params = runner_state[1]['params'] + elif "actor_params" in runner_state[1].keys(): + params = runner_state[1]['actor_params'] + else: + raise ValueError("No params found in checkpoint.") + + params = jax.tree_util.tree_map( + lambda x: jnp.take(x, indices=agent_idxs, axis=0), + params + ) + + with jax.disable_jit(args.render_mode is not None): + student_model = models.make( + env_name=xp_args.env_name, + model_name=xp_args.student_model_name, + **xp_args.student_model_args + ) + + # We force EvalRunner to select all params, since we've already + # extracted the relevant agent indices. + if "Overcooked" in args.env_names: + from minimax.runners_ma import EvalRunner + + pop = AgentPop( + agent=agents.MAPPOAgent(actor=student_model, critic=None), + n_agents=len(agent_idxs) + ) + elif "Maze" in args.env_names: + from minimax.runners import EvalRunner + + pop = AgentPop( + agent=agents.PPOAgent(model=student_model), + n_agents=len(agent_idxs) + ) + else: + raise ValueError("Unknown environment.") + + runner = EvalRunner( + pop=pop, + env_names=args.env_names, + env_kwargs=xp_args.eval_env_args, + n_episodes=args.n_episodes, + render_mode=args.render_mode, + agent_idxs='0' + ) + + rng = jax.random.PRNGKey(args.seed) + _eval_stats = runner.run(rng, params) + + eval_stats = {} + for k, v in _eval_stats.items(): + prefix_match = re.match(r'^eval/(a[0-9]+):.*', k) + if prefix_match is not None: + prefix = prefix_match.groups()[0] + _idx = int(prefix.lstrip('a').rstrip(':')) + idx = agent_idxs[_idx] + new_prefix = f'a{idx}' + new_k = k.replace(prefix, new_prefix) + eval_stats[new_k] = v + else: + eval_stats[k] = v + + for k, v in eval_stats.items(): + all_eval_stats[k].append(float(v)) + + pbar.update(1) + + pbar.close() + + aggregate_eval_stats = {} + for k, v in all_eval_stats.items(): + mean = np.mean(all_eval_stats[k]) + if len(all_eval_stats[k]) > 1: + sem = spstats.sem(all_eval_stats[k]) + else: + sem = 0.0 + aggregate_eval_stats[k] = f'{mean: 0.4}+/-{sem: 0.4}' + + _min = np.min(all_eval_stats[k]) + aggregate_eval_stats[f'min:{k}'] = f'{_min: 0.4}' + + logger = HumanOutputFormat(sys.stdout) + logger.writekvs(aggregate_eval_stats) + + if args.results_fname is not None: + if args.results_fname.strip('"') == '*': + results_fname = args.xpid_prefix or args.xpid + else: + results_fname = args.results_fname + + df = pd.DataFrame.from_dict(all_eval_stats) + results_path = args.results_path + if not os.path.isabs(results_path): + results_path = os.path.join( + os.path.abspath(__file__), results_path) + results_path = os.path.join(results_path, f'{results_fname}.csv') + df.to_csv(results_path, index=False) + print(f'Saved results to {results_path}') diff --git a/src/minimax/evaluate_against_baseline.py b/src/minimax/evaluate_against_baseline.py new file mode 100644 index 0000000..a79573c --- /dev/null +++ b/src/minimax/evaluate_against_baseline.py @@ -0,0 +1,276 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +import os +import json +import re +import fnmatch +import sys +from collections import defaultdict + +import numpy as np +import pandas as pd +import scipy.stats as spstats +import jax +import jax.numpy as jnp +from tqdm import tqdm + +from minimax.util.parsnip import Parsnip +from minimax.util.checkpoint import load_pkl_object, load_config +from minimax.util.loggers import HumanOutputFormat +from minimax.util.rl import AgentPop +import minimax.models as models +import minimax.agents as agents + + +class FixedModel(nn.Module): + """Useful as a model that acts randomly or always takes a certain action. + We use it to establish a baseline for cooperation. + """ + is_random: bool = False + always_pick_action: int = None + num_actions: int = 6 + + def setup(self): + super().__init__() + + def __call__(self, x, carry=None, reset=None): + if self.is_random: + # Same logits for all actions + logits = jnp.ones((x.shape[0], self.num_actions)) + return logits, carry + # Logits one for one action always + logits = jnp.zeros((x.shape[0], self.num_actions)) + return logits.at[:, self.always_pick_action].set(jnp.inf), carry + + def initialize_carry( + self, + rng: chex.PRNGKey, + batch_dims: Tuple[int] = ()) -> Tuple[chex.ArrayTree, chex.ArrayTree]: + """Initialize hidden state of LSTM.""" + return None + + @property + def is_recurrent(self): + return False + + +parser = Parsnip() + +# ==== Define top-level arguments +parser.add_argument( + '--seed', + type=int, + default=1, + help='Random seed.') +parser.add_argument( + '--log_dir', + type=str, + default='~/logs/minimax', + help='Log directory containing experiment dirs.') +parser.add_argument( + '--xpid', + type=str, + default='latest', + help='Experiment ID dir name for model.') +parser.add_argument( + '--xpid_prefix', + type=str, + default=None, + help='Experiment ID dir name for model.') +parser.add_argument( + '--checkpoint_name', + type=str, + default='checkpoint', + help='Name of checkpoint .pkl.') +parser.add_argument( + '--env_names', + type=str, + help='csv of evaluation environments.') +parser.add_argument( + '--n_episodes', + type=int, + default=1, + help='Number of evaluation episodes.') +parser.add_argument( + '--agent_idxs', + type=str, + default='*', + help="Indices of agents to evaluate. '*' indicates all.") +parser.add_argument( + '--render_mode', + type=str, + nargs='?', + const=True, + default=None, + help='Visualize episodes.') +parser.add_argument( + '--results_path', + type=str, + default='results/', + help='Results dir.') +parser.add_argument( + '--results_fname', + type=str, + default=None, + help='Results filename (without .csv).') + + +if __name__ == '__main__': + """ + Usage: + python -m eval \ + --xpid= \ + --env_names="Maze-SixteenRooms" \ + --n_episodes=100 \ + --agent_idxs=0 + """ + args = parser.parse_args() + + log_dir_path = os.path.expandvars(os.path.expanduser(args.log_dir)) + + xpids = [] + if args.xpid_prefix is not None: + # Get all matching xpid directories + all_xpids = fnmatch.filter(os.listdir( + log_dir_path), f"{args.xpid_prefix}*") + filter_re = re.compile('.*_[0-9]*$') + xpids = [x for x in all_xpids if filter_re.match(x)] + else: + xpids = [args.xpid] + + pbar = tqdm(total=len(xpids)) + + all_eval_stats = defaultdict(list) + for xpid in xpids: + xpid_dir_path = os.path.join(log_dir_path, xpid) + checkpoint_path = os.path.join( + xpid_dir_path, f'{args.checkpoint_name}.pkl') + meta_path = os.path.join(xpid_dir_path, f'meta.json') + + # Load checkpoint info + if not os.path.exists(meta_path): + print(f'Configuration at {meta_path} does not exist. Skipping...') + continue + + if not os.path.exists(checkpoint_path): + print( + f'Checkpoint path {checkpoint_path} does not exist. Skipping...') + continue + + xp_args = load_config(meta_path) + + agent_idxs = args.agent_idxs + if agent_idxs == '*': + agent_idxs = np.arange(xp_args.train_runner_args.n_students) + else: + agent_idxs = \ + np.array([int(x) for x in agent_idxs.split(',')]) + assert np.max(agent_idxs) <= xp_args.train_runner_args.n_students, \ + 'Agent index is out of bounds.' + + runner_state = load_pkl_object(checkpoint_path) + if "params" in runner_state[1].keys(): + params = runner_state[1]['params'] + elif "actor_params" in runner_state[1].keys(): + params = runner_state[1]['actor_params'] + else: + raise ValueError("No params found in checkpoint.") + + params = jax.tree_util.tree_map( + lambda x: jnp.take(x, indices=agent_idxs, axis=0), + params + ) + + with jax.disable_jit(args.render_mode is not None): + student_model = models.make( + env_name=xp_args.env_name, + model_name=xp_args.student_model_name, + **xp_args.student_model_args + ) + + # We force EvalRunner to select all params, since we've already + # extracted the relevant agent indices. + if "Overcooked" in args.env_names: + from minimax.runners_ma import EvalRunner + + pop = AgentPop( + agent=agents.MAPPOAgent(actor=student_model, critic=None), + n_agents=len(agent_idxs) + ) + elif "Maze" in args.env_names: + from minimax.runners import EvalRunner + + pop = AgentPop( + agent=agents.PPOAgent(model=student_model), + n_agents=len(agent_idxs) + ) + else: + raise ValueError("Unknown environment.") + + runner = EvalRunner( + pop=pop, + env_names=args.env_names, + env_kwargs=xp_args.eval_env_args, + n_episodes=args.n_episodes, + render_mode=args.render_mode, + agent_idxs='*' + ) + + rng = jax.random.PRNGKey(args.seed) + _eval_stats = runner.run(rng, params) + + eval_stats = {} + for k, v in _eval_stats.items(): + prefix_match = re.match(r'^eval/(a[0-9]+):.*', k) + if prefix_match is not None: + prefix = prefix_match.groups()[0] + _idx = int(prefix.lstrip('a').rstrip(':')) + idx = agent_idxs[_idx] + new_prefix = f'a{idx}' + new_k = k.replace(prefix, new_prefix) + eval_stats[new_k] = v + else: + eval_stats[k] = v + + for k, v in eval_stats.items(): + all_eval_stats[k].append(float(v)) + + pbar.update(1) + + pbar.close() + + aggregate_eval_stats = {} + for k, v in all_eval_stats.items(): + mean = np.mean(all_eval_stats[k]) + if len(all_eval_stats[k]) > 1: + sem = spstats.sem(all_eval_stats[k]) + else: + sem = 0.0 + aggregate_eval_stats[k] = f'{mean: 0.4}+/-{sem: 0.4}' + + _min = np.min(all_eval_stats[k]) + aggregate_eval_stats[f'min:{k}'] = f'{_min: 0.4}' + + logger = HumanOutputFormat(sys.stdout) + logger.writekvs(aggregate_eval_stats) + + if args.results_fname is not None: + if args.results_fname.strip('"') == '*': + results_fname = args.xpid_prefix or args.xpid + else: + results_fname = args.results_fname + + df = pd.DataFrame.from_dict(all_eval_stats) + results_path = args.results_path + if not os.path.isabs(results_path): + results_path = os.path.join( + os.path.abspath(__file__), results_path) + results_path = os.path.join(results_path, f'{results_fname}.csv') + df.to_csv(results_path, index=False) + print(f'Saved results to {results_path}') diff --git a/src/minimax/evaluate_against_population.py b/src/minimax/evaluate_against_population.py new file mode 100644 index 0000000..649323f --- /dev/null +++ b/src/minimax/evaluate_against_population.py @@ -0,0 +1,281 @@ +import os +import json +import re +import fnmatch +import sys +from collections import defaultdict + +import numpy as np +import pandas as pd +import scipy.stats as spstats +import jax +import jax.numpy as jnp +from tqdm import tqdm + +from minimax.util.parsnip import Parsnip +from minimax.util.checkpoint import load_pkl_object, load_config +from minimax.util.loggers import HumanOutputFormat +from minimax.util.rl import AgentPopHeterogenous +import minimax.models as models +import minimax.agents as agents + + +parser = Parsnip() + +# ==== Define top-level arguments +parser.add_argument( + '--seed', + type=int, + default=1, + help='Random seed.') +parser.add_argument( + '--population_json', + type=str, + default=None, + help='Path to population json file.') +parser.add_argument( + '--log_dir', + type=str, + default='~/logs/minimax', + help='Log directory containing experiment dirs.') +parser.add_argument( + '--xpid', + type=str, + default='latest', + help='Experiment ID dir name for model.') +parser.add_argument( + '--xpid_prefix', + type=str, + default=None, + help='Experiment ID dir name for model.') +parser.add_argument( + '--checkpoint_name', + type=str, + default='checkpoint', + help='Name of checkpoint .pkl.') +parser.add_argument( + '--env_names', + type=str, + help='csv of evaluation environments.') +parser.add_argument( + '--n_episodes', + type=int, + default=1, + help='Number of evaluation episodes.') +parser.add_argument( + '--agent_idxs', + type=str, + default='*', + help="Indices of agents to evaluate. '*' indicates all.") +parser.add_argument( + '--render_mode', + type=str, + nargs='?', + const=True, + default=None, + help='Visualize episodes.') +parser.add_argument( + '--results_path', + type=str, + default='results/', + help='Results dir.') +parser.add_argument( + '--results_fname', + type=str, + default=None, + help='Results filename (without .csv).') + + +if __name__ == '__main__': + """ + Usage: + python -m eval \ + --xpid= \ + --env_names="Maze-SixteenRooms" \ + --n_episodes=100 \ + --agent_idxs=0 + """ + args = parser.parse_args() + + log_dir_path = os.path.expandvars(os.path.expanduser(args.log_dir)) + + xpid = args.xpid + + population_json_path = args.population_json + + with open(population_json_path, 'r') as f: + population = json.load(f) + + population_size = int(population["population_size"]) + + pbar = tqdm(total=population_size*2) + + all_eval_stats = defaultdict(list) + for agent_id in range(1, population_size+1): + xpid_dir_path = os.path.join(log_dir_path, xpid) + checkpoint_path = os.path.join( + xpid_dir_path, f'{args.checkpoint_name}.pkl') + meta_path = os.path.join(xpid_dir_path, f'meta.json') + + other_agent_checkpoint_path = f"{os.getcwd()}/{population[str(agent_id)]}" + other_agent_meta_path = f"{os.getcwd()}/{population[f'{agent_id}_meta']}" + + # Load checkpoint info + if not os.path.exists(meta_path): + print(f'Configuration at {meta_path} does not exist. Skipping...') + continue + + if not os.path.exists(other_agent_meta_path): + raise ValueError(f"Did not find: {other_agent_meta_path}") + + if not os.path.exists(checkpoint_path): + print( + f'Checkpoint path {checkpoint_path} does not exist. Skipping...') + continue + + if not os.path.exists(other_agent_checkpoint_path): + raise ValueError(f"Did not find: {other_agent_checkpoint_path}") + + xp_args = load_config(meta_path) + + xp_population_args = load_config(other_agent_meta_path) + + agent_idxs = args.agent_idxs + if agent_idxs == '*': + agent_idxs = np.arange(xp_args.train_runner_args.n_students) + else: + agent_idxs = \ + np.array([int(x) for x in agent_idxs.split(',')]) + assert np.max(agent_idxs) <= xp_args.train_runner_args.n_students, \ + 'Agent index is out of bounds.' + + runner_state_0 = load_pkl_object(checkpoint_path) + if "params" in runner_state_0[1].keys(): + params_0 = runner_state_0[1]['params'] + elif "actor_params" in runner_state_0[1].keys(): + params_0 = runner_state_0[1]['actor_params'] + else: + raise ValueError("No params found in checkpoint.") + + params_0 = jax.tree_util.tree_map( + lambda x: jnp.take(x, indices=agent_idxs, axis=0), + params_0 + ) + + xp_args_other = load_config(other_agent_meta_path) + + runner_state_1 = load_pkl_object(other_agent_checkpoint_path) + if "params" in runner_state_1[1].keys(): + params_1 = runner_state_1[1]['params'] + elif "actor_params" in runner_state_1[1].keys(): + params_1 = runner_state_1[1]['actor_params'] + else: + raise ValueError("No params found in checkpoint.") + + params_1 = jax.tree_util.tree_map( + lambda x: jnp.take(x, indices=agent_idxs, axis=0), + params_1 + ) + + for i in range(2): + # Swap params and runner states + # Bit finicky be careful here + if i == 1: + params_0, params_1 = params_1, params_0 + xp_args, xp_population_args = xp_population_args, xp_args + runner_state_0, runner_state_1 = runner_state_1, runner_state_0 + + with jax.disable_jit(args.render_mode is not None): + student_model = models.make( + env_name=xp_args.env_name, + model_name=xp_args.student_model_name, + **xp_args.student_model_args + ) + + population_model = models.make( + env_name=xp_args.env_name, + model_name=xp_population_args.student_model_name, + **xp_population_args.student_model_args + ) + + # We force EvalRunner to select all params, since we've already + # extracted the relevant agent indices. + if "Overcooked" in args.env_names: + from minimax.runners_ma import EvalRunnerHeterogenous + + pop = AgentPopHeterogenous( + agent_0=agents.MAPPOAgent( + actor=student_model, critic=None), + agent_1=agents.MAPPOAgent( + actor=population_model, critic=None), + n_agents=len(agent_idxs) + ) + else: + raise ValueError("Unknown environment.") + + runner = EvalRunnerHeterogenous( + pop=pop, + env_names=args.env_names, + env_kwargs=xp_args.eval_env_args, + n_episodes=args.n_episodes, + render_mode=args.render_mode, + agent_idxs='*' + ) + + rng = jax.random.PRNGKey(args.seed) + _eval_stats = runner.run(rng, params_0, params_1) + + eval_stats = {} + for k, v in _eval_stats.items(): + prefix_match = re.match(r'^eval/(a[0-9]+):.*', k) + if prefix_match is not None: + prefix = prefix_match.groups()[0] + _idx = int(prefix.lstrip('a').rstrip(':')) + idx = agent_idxs[_idx] + new_prefix = f'a{idx}' + new_k = k.replace(prefix, new_prefix) + eval_stats[new_k] = v + else: + eval_stats[k] = v + + for k, v in eval_stats.items(): + all_eval_stats[k].append(float(v)) + all_eval_stats[k+":"+population[str(agent_id)].split( + "/")[-1][:-4]].append(float(v)) + + pbar.update(1) + + pbar.close() + + aggregate_eval_stats = {} + for k, v in all_eval_stats.items(): + max = np.max(all_eval_stats[k]) + mean = np.mean(all_eval_stats[k]) + if ":test_return:" in k: + print(f"k {k}, v {v}") + if len(all_eval_stats[k]) > 1: + sem = spstats.sem(all_eval_stats[k]) + else: + sem = 0.0 + aggregate_eval_stats[k] = f'{mean: 0.4}+/-{sem: 0.4} (max: {max: 0.4})' + + _min = np.min(all_eval_stats[k]) + aggregate_eval_stats[f'min:{k}'] = f'{_min: 0.4}' + + logger = HumanOutputFormat(sys.stdout) + logger.writekvs(aggregate_eval_stats) + + if args.results_fname is not None: + if args.results_fname.strip('"') == '*': + results_fname = args.xpid_prefix or args.xpid + else: + results_fname = args.results_fname + + df = pd.DataFrame.from_dict(all_eval_stats) + results_path = args.results_path + if not os.path.isabs(results_path): + results_path = os.path.join( + os.path.abspath(__file__), results_path) + results_path = os.path.join(results_path, f'{results_fname}.csv') + df.to_csv(results_path, index=False) + print(f'Saved results to {results_path}') diff --git a/src/minimax/evaluate_baseline_against_population.py b/src/minimax/evaluate_baseline_against_population.py new file mode 100644 index 0000000..f26f7d5 --- /dev/null +++ b/src/minimax/evaluate_baseline_against_population.py @@ -0,0 +1,319 @@ +import os +import json +import re +import sys +from collections import defaultdict +from typing import Tuple + +import chex +import numpy as np +import pandas as pd +import scipy.stats as spstats +import jax +import jax.numpy as jnp +import flax.linen as nn +from tqdm import tqdm + +from minimax.util.parsnip import Parsnip +from minimax.util.checkpoint import load_pkl_object, load_config +from minimax.util.loggers import HumanOutputFormat +from minimax.util.rl import AgentPopHeterogenous +import minimax.models as models +import minimax.agents as agents + + +class FixedModel(nn.Module): + """Useful as a model that acts randomly or always takes a certain action. + We use it to establish a baseline for cooperation. + """ + is_random: bool = False + always_pick_action: int = None + num_actions: int = 6 + + def setup(self): + super().__init__() + + def __call__(self, x, carry=None, reset=None): + if self.is_random: + # Same logits for all actions + logits = jnp.ones((x.shape[0], self.num_actions)) + return logits, carry + # Logits one for one action always + logits = jnp.zeros((x.shape[0], self.num_actions)) + return logits.at[:, self.always_pick_action].set(jnp.inf), carry + + def initialize_carry( + self, + rng: chex.PRNGKey, + batch_dims: Tuple[int] = ()) -> Tuple[chex.ArrayTree, chex.ArrayTree]: + """Initialize hidden state of LSTM.""" + return None + + @property + def is_recurrent(self): + return False + + +parser = Parsnip() + +# ==== Define top-level arguments +parser.add_argument( + '--seed', + type=int, + default=1, + help='Random seed.') +parser.add_argument( + '--population_json', + type=str, + default=None, + help='Path to population json file.') +parser.add_argument( + '--log_dir', + type=str, + default='~/logs/minimax', + help='Log directory containing experiment dirs.') +parser.add_argument( + '--env_names', + type=str, + help='csv of evaluation environments.') +parser.add_argument( + '--n_episodes', + type=int, + default=1, + help='Number of evaluation episodes.') +parser.add_argument( + '--agent_idxs', + type=str, + default='*', + help="Indices of agents to evaluate. '*' indicates all.") +parser.add_argument( + '--render_mode', + type=str, + nargs='?', + const=True, + default=None, + help='Visualize episodes.') +parser.add_argument( + '--results_path', + type=str, + default='results/', + help='Results dir.') +parser.add_argument( + '--results_fname', + type=str, + default=None, + help='Results filename (without .csv).') +parser.add_argument( + '--is_random', + type=str, + nargs='?', + const=True, + default=None, + help='Random fixed agent.') + +if __name__ == '__main__': + """ + Usage: + python -m eval \ + --xpid= \ + --env_names="Maze-SixteenRooms" \ + --n_episodes=100 \ + --agent_idxs=0 + """ + args = parser.parse_args() + + log_dir_path = os.path.expandvars(os.path.expanduser(args.log_dir)) + + xpid = args.xpid + + population_json_path = args.population_json + + with open(population_json_path, 'r') as f: + population = json.load(f) + + population_size = int(population["population_size"]) + + pbar = tqdm(total=population_size*2) + + all_eval_stats = defaultdict(list) + for agent_id in range(1, population_size+1): + # xpid_dir_path = os.path.join(log_dir_path, xpid) + # checkpoint_path = os.path.join( + # xpid_dir_path, f'{args.checkpoint_name}.pkl') + # meta_path = os.path.join(xpid_dir_path, f'meta.json') + + other_agent_checkpoint_path = f"{os.getcwd()}/{population[str(agent_id)]}" + other_agent_meta_path = f"{os.getcwd()}/{population[f'{agent_id}_meta']}" + + # Load checkpoint info + # if not os.path.exists(meta_path): + # print(f'Configuration at {meta_path} does not exist. Skipping...') + # continue + + if not os.path.exists(other_agent_meta_path): + raise ValueError(f"Did not find: {other_agent_meta_path}") + + # if not os.path.exists(checkpoint_path): + # print( + # f'Checkpoint path {checkpoint_path} does not exist. Skipping...') + # continue + + if not os.path.exists(other_agent_checkpoint_path): + raise ValueError(f"Did not find: {other_agent_checkpoint_path}") + + # xp_args = load_config(meta_path) + + xp_population_args = load_config(other_agent_meta_path) + + agent_idxs = args.agent_idxs + if agent_idxs == '*': + agent_idxs = np.arange( + xp_population_args.train_runner_args.n_students) + else: + agent_idxs = \ + np.array([int(x) for x in agent_idxs.split(',')]) + assert np.max(agent_idxs) <= xp_population_args.train_runner_args.n_students, \ + 'Agent index is out of bounds.' + + # runner_state_0 = load_pkl_object(checkpoint_path) + # if "params" in runner_state_0[1].keys(): + # params_0 = runner_state_0[1]['params'] + # elif "actor_params" in runner_state_0[1].keys(): + # params_0 = runner_state_0[1]['actor_params'] + # else: + # raise ValueError("No params found in checkpoint.") + + # params_0 = jax.tree_util.tree_map( + # lambda x: jnp.take(x, indices=agent_idxs, axis=0), + # params_0 + # ) + + xp_args_other = load_config(other_agent_meta_path) + + runner_state_1 = load_pkl_object(other_agent_checkpoint_path) + if "params" in runner_state_1[1].keys(): + params_1 = runner_state_1[1]['params'] + elif "actor_params" in runner_state_1[1].keys(): + params_1 = runner_state_1[1]['actor_params'] + else: + raise ValueError("No params found in checkpoint.") + + params_1 = jax.tree_util.tree_map( + lambda x: jnp.take(x, indices=agent_idxs, axis=0), + params_1 + ) + + # We use models without parameters + # {'params': {'place_holder': jnp.zeros(1,)}} + params_0 = params_1.copy() + + for i in range(2): + # Swap params and runner states + # Bit finicky be careful here + + with jax.disable_jit(args.render_mode is not None): + if args.is_random: + student_model = FixedModel( + is_random=True, + num_actions=6 + ) + else: + student_model = FixedModel( + is_random=False, + always_pick_action=4, # Stay = 4 + num_actions=6 + ) + + population_model = models.make( + env_name=xp_population_args.env_name, + model_name=xp_population_args.student_model_name, + **xp_population_args.student_model_args + ) + + if i == 1: + params_0, params_1 = params_1, params_0 + student_model, population_model = population_model, student_model + # xp_args, xp_population_args = xp_population_args, xp_args + # runner_state_0, runner_state_1 = runner_state_1, runner_state_0 + + # We force EvalRunner to select all params, since we've already + # extracted the relevant agent indices. + if "Overcooked" in args.env_names: + from minimax.runners_ma import EvalRunnerHeterogenous + + pop = AgentPopHeterogenous( + agent_0=agents.MAPPOAgent( + actor=student_model, critic=None), + agent_1=agents.MAPPOAgent( + actor=population_model, critic=None), + n_agents=len(agent_idxs) + ) + else: + raise ValueError("Unknown environment.") + + runner = EvalRunnerHeterogenous( + pop=pop, + env_names=args.env_names, + env_kwargs=xp_population_args.eval_env_args, + n_episodes=args.n_episodes, + render_mode=args.render_mode, + agent_idxs='*' + ) + + rng = jax.random.PRNGKey(args.seed) + _eval_stats = runner.run(rng, params_0, params_1) + + eval_stats = {} + for k, v in _eval_stats.items(): + prefix_match = re.match(r'^eval/(a[0-9]+):.*', k) + if prefix_match is not None: + prefix = prefix_match.groups()[0] + _idx = int(prefix.lstrip('a').rstrip(':')) + idx = agent_idxs[_idx] + new_prefix = f'a{idx}' + new_k = k.replace(prefix, new_prefix) + eval_stats[new_k] = v + else: + eval_stats[k] = v + + for k, v in eval_stats.items(): + all_eval_stats[k].append(float(v)) + + pbar.update(1) + + pbar.close() + + aggregate_eval_stats = {} + for k, v in all_eval_stats.items(): + max = np.max(all_eval_stats[k]) + mean = np.mean(all_eval_stats[k]) + if ":test_return:" in k: + print(f"k {k}, v {v}") + if ":test_solved_rate:" in k: + print(f"k {k}, v {v}") + if len(all_eval_stats[k]) > 1: + sem = spstats.sem(all_eval_stats[k]) + else: + sem = 0.0 + aggregate_eval_stats[k] = f'{mean: 0.4}+/-{sem: 0.4} (max: {max: 0.4})' + + _min = np.min(all_eval_stats[k]) + aggregate_eval_stats[f'min:{k}'] = f'{_min: 0.4}' + + logger = HumanOutputFormat(sys.stdout) + logger.writekvs(aggregate_eval_stats) + + if args.results_fname is not None: + if args.results_fname.strip('"') == '*': + results_fname = args.xpid_prefix or args.xpid + else: + results_fname = args.results_fname + + df = pd.DataFrame.from_dict(all_eval_stats) + results_path = args.results_path + if not os.path.isabs(results_path): + results_path = os.path.join( + os.path.abspath(__file__), results_path) + results_path = os.path.join(results_path, f'{results_fname}.csv') + df.to_csv(results_path, index=False) + print(f'Saved results to {results_path}') diff --git a/src/minimax/evaluate_from_pckl.py b/src/minimax/evaluate_from_pckl.py new file mode 100644 index 0000000..6dd51a7 --- /dev/null +++ b/src/minimax/evaluate_from_pckl.py @@ -0,0 +1,264 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +import os +import re +import sys +from collections import defaultdict + +import numpy as np +import pandas as pd +import scipy.stats as spstats +import jax +import jax.numpy as jnp +from tqdm import tqdm + +from minimax.util.parsnip import Parsnip +from minimax.util.checkpoint import load_pkl_object, load_config +from minimax.util.loggers import HumanOutputFormat +from minimax.util.rl import AgentPopHeterogenous +import minimax.models as models +import minimax.agents as agents + + +parser = Parsnip() + +# ==== Define top-level arguments +parser.add_argument( + '--seed', + type=int, + default=1, + help='Random seed.') +parser.add_argument( + '--pckl_path', + type=str, + default=None, + help='Path to population json file.') +parser.add_argument( + '--meta_path', + type=str, + default=None, + help='Path to population json file.') +parser.add_argument( + '--log_dir', + type=str, + default='~/logs/minimax', + help='Log directory containing experiment dirs.') +parser.add_argument( + '--other_pckl_path', + type=str, + default=None, + help='Path to population json file.') +parser.add_argument( + '--other_meta_path', + type=str, + default=None, + help='Path to population json file.') +parser.add_argument( + '--env_names', + type=str, + help='csv of evaluation environments.') +parser.add_argument( + '--n_episodes', + type=int, + default=1, + help='Number of evaluation episodes.') +parser.add_argument( + '--agent_idxs', + type=str, + default='*', + help="Indices of agents to evaluate. '*' indicates all.") +parser.add_argument( + '--render_mode', + type=str, + nargs='?', + const=True, + default=None, + help='Visualize episodes.') +parser.add_argument( + '--results_path', + type=str, + default='results/', + help='Results dir.') +parser.add_argument( + '--results_fname', + type=str, + default=None, + help='Results filename (without .csv).') + + +if __name__ == '__main__': + """ + Usage: + python -m eval \ + --xpid= \ + --env_names="Maze-SixteenRooms" \ + --n_episodes=100 \ + --agent_idxs=0 + """ + args = parser.parse_args() + + # log_dir_path = os.path.expandvars(os.path.expanduser(args.log_dir)) + + all_eval_stats = defaultdict(list) + # xpid_dir_path = os.path.join(log_dir_path, xpid) + checkpoint_path = args.pckl_path + meta_path = args.meta_path + + other_agent_checkpoint_path = args.other_pckl_path + other_agent_meta_path = args.other_meta_path + + # Load checkpoint info + if not os.path.exists(meta_path): + print(f'Configuration at {meta_path} does not exist. Skipping...') + + if not os.path.exists(other_agent_meta_path): + raise ValueError(f"Did not find: {other_agent_meta_path}") + + if not os.path.exists(checkpoint_path): + print( + f'Checkpoint path {checkpoint_path} does not exist. Skipping...') + + if not os.path.exists(other_agent_checkpoint_path): + raise ValueError(f"Did not find: {other_agent_checkpoint_path}") + + xp_args = load_config(meta_path) + + xp_other_args = load_config(other_agent_meta_path) + + agent_idxs = args.agent_idxs + if agent_idxs == '*': + agent_idxs = np.arange(xp_args.train_runner_args.n_students) + else: + agent_idxs = \ + np.array([int(x) for x in agent_idxs.split(',')]) + assert np.max(agent_idxs) <= xp_args.train_runner_args.n_students, \ + 'Agent index is out of bounds.' + + runner_state_0 = load_pkl_object(checkpoint_path) + if "params" in runner_state_0[1].keys(): + params_0 = runner_state_0[1]['params'] + elif "actor_params" in runner_state_0[1].keys(): + params_0 = runner_state_0[1]['actor_params'] + else: + raise ValueError("No params found in checkpoint.") + + params_0 = jax.tree_util.tree_map( + lambda x: jnp.take(x, indices=agent_idxs, axis=0), + params_0 + ) + + xp_args_other = load_config(other_agent_meta_path) + + runner_state_1 = load_pkl_object(other_agent_checkpoint_path) + if "params" in runner_state_1[1].keys(): + params_1 = runner_state_1[1]['params'] + elif "actor_params" in runner_state_1[1].keys(): + params_1 = runner_state_1[1]['actor_params'] + else: + raise ValueError("No params found in checkpoint.") + + params_1 = jax.tree_util.tree_map( + lambda x: jnp.take(x, indices=agent_idxs, axis=0), + params_1 + ) + + for i in range(2): + # Swap params and runner states + # Bit finicky be careful here + if i == 1: + params_0, params_1 = params_1, params_0 + xp_args, xp_other_args = xp_other_args, xp_args + runner_state_0, runner_state_1 = runner_state_1, runner_state_0 + + with jax.disable_jit(args.render_mode is not None): + student_model = models.make( + env_name=xp_args.env_name, + model_name=xp_args.student_model_name, + **xp_args.student_model_args + ) + + population_model = models.make( + env_name=xp_args.env_name, + model_name=xp_other_args.student_model_name, + **xp_other_args.student_model_args + ) + + # We force EvalRunner to select all params, since we've already + # extracted the relevant agent indices. + if "Overcooked" in args.env_names: + from minimax.runners_ma import EvalRunnerHeterogenous + + pop = AgentPopHeterogenous( + agent_0=agents.MAPPOAgent( + actor=student_model, critic=None), + agent_1=agents.MAPPOAgent( + actor=population_model, critic=None), + n_agents=len(agent_idxs) + ) + else: + raise ValueError("Unknown environment.") + + runner = EvalRunnerHeterogenous( + pop=pop, + env_names=args.env_names, + env_kwargs=xp_args.eval_env_args, + n_episodes=args.n_episodes, + render_mode=args.render_mode, + agent_idxs='*' + ) + + rng = jax.random.PRNGKey(args.seed) + _eval_stats = runner.run(rng, params_0, params_1) + + eval_stats = {} + for k, v in _eval_stats.items(): + prefix_match = re.match(r'^eval/(a[0-9]+):.*', k) + if prefix_match is not None: + prefix = prefix_match.groups()[0] + _idx = int(prefix.lstrip('a').rstrip(':')) + idx = agent_idxs[_idx] + new_prefix = f'a{idx}' + new_k = k.replace(prefix, new_prefix) + eval_stats[new_k] = v + else: + eval_stats[k] = v + + for k, v in eval_stats.items(): + all_eval_stats[k].append(float(v)) + + aggregate_eval_stats = {} + for k, v in all_eval_stats.items(): + max = np.max(all_eval_stats[k]) + mean = np.mean(all_eval_stats[k]) + if len(all_eval_stats[k]) > 1: + sem = spstats.sem(all_eval_stats[k]) + else: + sem = 0.0 + aggregate_eval_stats[k] = f'{mean: 0.4}+/-{sem: 0.4} (max: {max: 0.4})' + + _min = np.min(all_eval_stats[k]) + aggregate_eval_stats[f'min:{k}'] = f'{_min: 0.4}' + + logger = HumanOutputFormat(sys.stdout) + logger.writekvs(aggregate_eval_stats) + + if args.results_fname is not None: + if args.results_fname.strip('"') == '*': + results_fname = args.xpid_prefix or args.xpid + else: + results_fname = args.results_fname + + df = pd.DataFrame.from_dict(all_eval_stats) + results_path = args.results_path + if not os.path.isabs(results_path): + results_path = os.path.join( + os.path.abspath(__file__), results_path) + results_path = os.path.join(results_path, f'{results_fname}.csv') + df.to_csv(results_path, index=False) + print(f'Saved results to {results_path}') diff --git a/src/minimax/extract_fcp.py b/src/minimax/extract_fcp.py new file mode 100644 index 0000000..0efad9c --- /dev/null +++ b/src/minimax/extract_fcp.py @@ -0,0 +1,276 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +import os +import json +import glob +import re +import fnmatch +import sys +from collections import defaultdict + +import numpy as np +import pandas as pd +import scipy.stats as spstats +import jax +import jax.numpy as jnp +from tqdm import tqdm + +from minimax.util.parsnip import Parsnip +from minimax.util.checkpoint import load_pkl_object, load_config +from minimax.util.loggers import HumanOutputFormat +from minimax.util.rl import AgentPop +import minimax.models as models +import minimax.agents as agents + + +parser = Parsnip() + +# ==== Define top-level arguments +parser.add_argument( + '--seed', + type=int, + default=1, + help='Random seed.') +parser.add_argument( + '--log_dir', + type=str, + default='~/logs/minimax', + help='Log directory containing experiment dirs.') +parser.add_argument( + '--xpid', + type=str, + default='latest', + help='Experiment ID dir name for model.') +parser.add_argument( + '--xpid_prefix', + type=str, + default=None, + help='Experiment ID dir name for model.') +parser.add_argument( + '--checkpoint_name', + type=str, + default='checkpoint', + help='Name of checkpoint .pkl.') +parser.add_argument( + '--env_names', + type=str, + help='csv of evaluation environments.') +parser.add_argument( + '--n_episodes', + type=int, + default=1, + help='Number of evaluation episodes.') +parser.add_argument( + '--agent_idxs', + type=str, + default='*', + help="Indices of agents to evaluate. '*' indicates all.") +parser.add_argument( + '--render_mode', + type=str, + nargs='?', + const=True, + default=None, + help='Visualize episodes.') +parser.add_argument( + '--results_path', + type=str, + default='results/', + help='Results dir.') +parser.add_argument( + '--results_fname', + type=str, + default=None, + help='Results filename (without .csv).') + +parser.add_argument( + '--trained_seed', + type=int, + default=None, + help='Seed that the model was trained with') + +if __name__ == '__main__': + args = parser.parse_args() + + log_dir_path = os.path.expandvars(os.path.expanduser(args.log_dir)) + + xpids = [] + if args.xpid_prefix is not None: + # Get all matching xpid directories + all_xpids = fnmatch.filter(os.listdir( + log_dir_path), f"{args.xpid_prefix}*") + filter_re = re.compile('.*_[0-9]*$') + xpids = [x for x in all_xpids if filter_re.match(x)] + else: + xpids = [args.xpid] + + pbar = tqdm(total=len(xpids)) + + all_eval_stats = defaultdict(list) + for xpid in xpids: + xpid_dir_path = os.path.join(log_dir_path, xpid) + checkpoint_path = os.path.join( + xpid_dir_path, f'{args.checkpoint_name}.pkl') + meta_path = os.path.join(xpid_dir_path, f'meta.json') + + # Load checkpoint info + if not os.path.exists(meta_path): + print(f'Configuration at {meta_path} does not exist. Skipping...') + continue + + if not os.path.exists(checkpoint_path): + print( + f'Checkpoint path {checkpoint_path} does not exist. Skipping...') + continue + + xp_args = load_config(meta_path) + + agent_idxs = args.agent_idxs + if agent_idxs == '*': + agent_idxs = np.arange(xp_args.train_runner_args.n_students) + else: + agent_idxs = \ + np.array([int(x) for x in agent_idxs.split(',')]) + assert np.max(agent_idxs) <= xp_args.train_runner_args.n_students, \ + 'Agent index is out of bounds.' + + sub_checkpoint_paths = glob.glob(f"{checkpoint_path[:-4]}*.pkl") + sub_checkpoint_paths = sorted(list(sub_checkpoint_paths)) + + map_name_path = {} + map_name_params = {} + for sub_checkpoint_path in sub_checkpoint_paths: + desc = sub_checkpoint_path[len(checkpoint_path[:-4])+1:-4] + if desc == '': + desc = 'final' + runner_state = load_pkl_object(sub_checkpoint_path) + if "params" in runner_state[1].keys(): + params = runner_state[1]['params'] + elif "actor_params" in runner_state[1].keys(): + params = runner_state[1]['actor_params'] + else: + raise ValueError("No params found in checkpoint.") + + params = jax.tree_util.tree_map( + lambda x: jnp.take(x, indices=agent_idxs, axis=0), + params + ) + map_name_path[desc] = sub_checkpoint_path + map_name_params[desc] = params + + map_name_eval_stas = {} + for desc, params in map_name_params.items(): + with jax.disable_jit(args.render_mode is not None): + student_model = models.make( + env_name=xp_args.env_name, + model_name=xp_args.student_model_name, + **xp_args.student_model_args + ) + + # We force EvalRunner to select all params, since we've already + # extracted the relevant agent indices. + if "Overcooked" in args.env_names: + from minimax.runners_ma import EvalRunner + + pop = AgentPop( + agent=agents.MAPPOAgent( + actor=student_model, critic=None), + n_agents=len(agent_idxs) + ) + elif "Maze" in args.env_names: + from minimax.runners import EvalRunner + + pop = AgentPop( + agent=agents.PPOAgent(model=student_model), + n_agents=len(agent_idxs) + ) + else: + raise ValueError("Unknown environment.") + + runner = EvalRunner( + pop=pop, + env_names=args.env_names, + env_kwargs=xp_args.eval_env_args, + n_episodes=args.n_episodes, + render_mode=args.render_mode, + agent_idxs='*' + ) + + rng = jax.random.PRNGKey(args.seed) + _eval_stats = runner.run(rng, params) + + eval_stats = {} + for k, v in _eval_stats.items(): + prefix_match = re.match(r'^eval/(a[0-9]+):.*', k) + if prefix_match is not None: + prefix = prefix_match.groups()[0] + _idx = int(prefix.lstrip('a').rstrip(':')) + idx = agent_idxs[_idx] + new_prefix = f'a{idx}' + new_k = k.replace(prefix, new_prefix) + eval_stats[new_k] = v + else: + eval_stats[k] = v + + for k, v in eval_stats.items(): + all_eval_stats[k].append(float(v)) + + pbar.update(1) + + assert len( + runner.ext_env_names) == 1, "Only one at a time to avoid confusion!" + + map_name_eval_stas[desc] = eval_stats[ + f"eval/a0:test_return:{runner.ext_env_names[0]}"] + + best = max(map_name_eval_stas.items(), key=lambda x: x[1]) + mid = min(map_name_eval_stas.items(), + key=lambda x: abs(x[1]-int(best[1])/2)) + low = min(map_name_eval_stas.items(), + key=lambda x: abs(x[1]-int(best[1])/5)) + + print("\n\n------------------------------------\n\n") + print("Best: ", best) + print("Mid: ", mid) + print("Low: ", low) + + high_id = best[0] + mid_id = mid[0] + low_id = low[0] + + # Take paths from ids and cp files to /populations/fcp/seed/ + high_path = map_name_path[high_id] + mid_path = map_name_path[mid_id] + low_path = map_name_path[low_id] + + print("High path: ", high_path) + print("Mid path: ", mid_path) + print("Low path: ", low_path) + + # find seed string with SEED_*_ in xpid + seed = args.trained_seed + + target_dir = f"{os.getcwd()}/populations/fcp/{args.env_names}/{seed}/" + + print("Target dir: ", target_dir) + + if not os.path.exists(target_dir): + os.makedirs(target_dir) + + os.system(f"cp {high_path} {target_dir}high.pkl") + os.system(f"cp {mid_path} {target_dir}mid.pkl") + os.system(f"cp {low_path} {target_dir}low.pkl") + # also copy meta + os.system(f"cp {meta_path} {target_dir}meta.json") + + # make a txt file there and copy the xpid + with open(f"{target_dir}xpid.txt", "w") as f: + f.write(xpid) + + pbar.close() diff --git a/src/minimax/models/__init__.py b/src/minimax/models/__init__.py new file mode 100644 index 0000000..8579102 --- /dev/null +++ b/src/minimax/models/__init__.py @@ -0,0 +1,33 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from .registration import register, make + + +from .maze import ( + GridWorldACStudentModel, + GridWorldACTeacherModel, +) + +from .overcooked import ( + ACStudentModel, + ACStudentCriticModel, + ACStudentActorModel, + ACTeacherModel, +) + +__all__ = [ + register, + make, + GridWorldACStudentModel, + GridWorldACTeacherModel, + ACStudentModel, + ACStudentCriticModel, + ACStudentActorModel, + ACTeacherModel, +] diff --git a/src/minimax/models/common.py b/src/minimax/models/common.py new file mode 100644 index 0000000..d72d1c5 --- /dev/null +++ b/src/minimax/models/common.py @@ -0,0 +1,383 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial +from typing import Tuple, Callable + +import numpy as np +import jax +import jax.numpy as jnp +from flax import linen as nn +import chex + +import minimax.envs as envs +from .rnn import CustomOptimizedLSTMCell + +from flax.linen.initializers import constant, orthogonal + + +def calc_gain(kind): + if kind == 'linear': + return 1.0 + elif kind == 'conv': + return 1.0 + elif kind == 'sigmoid': + return 1.0 + elif kind == 'tanh': + return np.sqrt(2) + elif kind == 'relu': + return np.sqrt(2) + elif kind == 'leaky_relu': + return np.sqrt(2/(1+0.01)) + elif kind == 'selu': + return 0.75 + elif kind == 'gelu': + return 0.75 + elif kind == 'crelu': + return np.sqrt(2) + + +def crelu(x): + return jnp.concatenate((nn.relu(x), nn.relu(-x)), axis=-1) + + +def get_activation(name): + if name == 'crelu': + return crelu + else: + return getattr(nn, name) + + +def default_bias_init(scale=1.0): + return nn.initializers.zeros + + +def init_orth(scale=1.0): + return nn.initializers.orthogonal(scale) + + +def make_fc_layers( + name=None, + n_layers=1, + hidden_dim=32, + activation=None, + kernel_init=None, + bias_init=nn.initializers.zeros_init(), + use_layernorm=False): + if kernel_init is None: + kernel_init = init_orth( + scale=calc_gain('linear') + ) + + layers = [] + for i in range(n_layers): + layer_name = None + if name: + layer_name = f'{name}_{i+1}' + + layers.append( + nn.Dense( + hidden_dim, + kernel_init=kernel_init, + bias_init=bias_init, + name=layer_name, + ) + ) + + if activation is not None: + layers.append(activation) + + if use_layernorm: + layers.append(nn.LayerNorm()) + + return nn.Sequential(layers) + + +def make_rnn( + arch='lstm', + kernel_init=init_orth(), + recurrent_kernel_init=init_orth(), + name=None): + if arch == 'lstm': + rnn = CustomOptimizedLSTMCell( + kernel_init=init_orth(), + recurrent_kernel_init=init_orth(), + name=name + ) + elif arch == 'gru': + rnn = nn.GRUCell( + kernel_init=init_orth(), + recurrent_kernel_init=init_orth(), + name=name + ) + else: + rnn = None + + return rnn + + +class RecurrentModuleBase(nn.Module): + def initialize_carry( + self, + rng: chex.PRNGKey, + batch_dims: Tuple[int] = ()) -> Tuple[chex.ArrayTree, chex.ArrayTree]: + """Initialize hidden state of LSTM.""" + if self.recurrent_arch == 'lstm': + return nn.OptimizedLSTMCell.initialize_carry( + rng, batch_dims, self.recurrent_hidden_dim + ) + elif self.recurrent_arch == 'gru': + return nn.GRUCell.initialize_carry( + rng, batch_dims, self.recurrent_hidden_dim + ) + else: + raise ValueError('Model is not recurrent.') + + @property + def is_recurrent(self): + return self.recurrent_arch is not None + + +class ScannedRNN(nn.Module): + """ + Scanned RNN. + Inputs: + carry: time-major input hidden states, LxBxH and optional + resets: Reset flags of shape LxB, where 1 indicates reset (equivalent to done==True). + """ + recurrent_arch: str = 'lstm' + recurrent_hidden_dim: int = 256 + kernel_init: Callable = init_orth() + recurrent_kernel_init: Callable = init_orth() + + @partial( + nn.scan, + variable_broadcast="params", + in_axes=0, + out_axes=0, + split_rngs={"params": False}, + ) + @nn.compact + def __call__(self, carry, step): + x, reset = step + rnn_state = carry + + # zero_carry = ScannedRNN.initialize_carry(jax.random.PRNGKey( + # 0), (*x.shape[:-1],), self.recurrent_hidden_dim, self.recurrent_arch) + # rnn_state = jax.tree_map( + # lambda x, y: jax.vmap(jax.lax.select)(reset, x, y), + # zero_carry, + # rnn_state + # ) + + rnn_state = jax.tree_map( + lambda x, y: jax.vmap(jax.lax.select)(reset, x, y), + ScannedRNN.initialize_carry( + jax.random.PRNGKey(0), (x.shape[0],), self.recurrent_hidden_dim, self.recurrent_arch), + rnn_state + ) + + rnn_kwargs = dict( + features=self.recurrent_hidden_dim, + kernel_init=self.kernel_init, + recurrent_kernel_init=self.recurrent_kernel_init, + ) + if self.recurrent_arch == 'lstm': + rnn_cell = nn.OptimizedLSTMCell( + **rnn_kwargs) # defaults to orth init + elif self.recurrent_arch == 'gru': + rnn_cell = nn.GRUCell(**rnn_kwargs) + else: + raise ValueError( + f'Unsupported recurrent_arch={self.recurrent_arch}') + + new_rnn_state, y = rnn_cell(rnn_state, x) + return new_rnn_state, y + + @staticmethod + def initialize_carry(rng, batch_dims, recurrent_hidden_dim, recurrent_arch): + init_args = (rng, (*batch_dims, recurrent_hidden_dim)) + if recurrent_arch == 'lstm': + # defaults to orth init + return nn.OptimizedLSTMCell(recurrent_hidden_dim, parent=None).initialize_carry(*init_args) + elif recurrent_arch == 'gru': + return nn.GRUCell(recurrent_hidden_dim, parent=None).initialize_carry(*init_args) + else: + raise ValueError(f'Unsupported recurrent_arch={recurrent_arch}') + + +class StateEncoderFF(nn.Module): + activation: str = "tanh" + + @nn.compact + def __call__(self, x): + if self.activation == "relu": + activation = nn.relu + elif self.activation == "tanh": + activation = nn.tanh + else: + raise ValueError('Activation not recognized.') + + x = x.reshape((*x.shape[:-3], -1)) + x = nn.Dense( + 64, kernel_init=orthogonal(np.sqrt(2)), bias_init=constant(0.0) + )(x) + x = activation(x) + x = nn.LayerNorm()(x) + + x = nn.Dense( + 64, kernel_init=orthogonal(np.sqrt(2)), bias_init=constant(0.0) + )(x) + x = activation(x) + x = nn.LayerNorm()(x) + + x = nn.Dense( + 64, kernel_init=orthogonal(np.sqrt(2)), bias_init=constant(0.0) + )(x) + x = activation(x) + x = nn.LayerNorm()(x) + return x + + +class StateCNNBase(nn.Module): + activation: str = "tanh" + out_features: int = 32 + + @nn.compact + def __call__(self, x): + if self.activation == "relu": + activation = nn.relu + elif self.activation == "tanh": + activation = nn.tanh + else: + raise ValueError('Activation not recognized.') + + x = nn.Conv(features=32, kernel_size=(3, 3))(x) + x = nn.max_pool(x, window_shape=(2, 2), strides=(1, 1), padding="SAME") + x = activation(x) + x = nn.Conv(features=64, kernel_size=(3, 3))(x) + x = nn.max_pool(x, window_shape=(2, 2), strides=(1, 1), padding="SAME") + x = activation(x) + x = nn.Conv(features=self.out_features, kernel_size=(3, 3))(x) + x = nn.max_pool(x, window_shape=(2, 2), strides=(1, 1), padding="SAME") + x = activation(x) + return x + + +class StateEncoderCNN(nn.Module): + activation: str = "tanh" + + @nn.compact + def __call__(self, x): + if self.activation == "relu": + activation = nn.relu + elif self.activation == "tanh": + activation = nn.tanh + else: + raise ValueError('Activation not recognized.') + + x = StateCNNBase(activation=activation)(x) + x = x.reshape((*x.shape[:-3], -1)) # Flatten + + x = nn.Dense( + 64, kernel_init=orthogonal(np.sqrt(2)), bias_init=constant(0.0) + )(x) + x = activation(x) + x = nn.LayerNorm()(x) + + # x = nn.Dense( + # 64, kernel_init=orthogonal(np.sqrt(2)), bias_init=constant(0.0) + # )(x) + # x = activation(x) + # x = nn.LayerNorm()(x) + + return x + + +class ValueHead(nn.Module): + n_hidden_layers: int = 1 + hidden_dim: int = 256 + activation: Callable = nn.tanh + kernel_init: Callable = init_orth(calc_gain('tanh')) + last_layer_kernel_init: Callable = init_orth(calc_gain('linear')) + use_layernorm: bool = False + + @nn.compact + def __call__(self, x): + + if self.n_hidden_layers > 1: + nn.Sequential([ + make_fc_layers( + n_layers=self.n_hidden_layers, + hidden_dim=self.hidden_dim, + activation=self.activation, + kernel_init=self.kernel_init, + use_layernorm=self.use_layernorm + ), + nn.Dense( + 1, + kernel_init=self.last_layer_kernel_init, + name='fc_value_final' + ) + ])(x) + return nn.Sequential([ + nn.Dense( + 1, + kernel_init=self.last_layer_kernel_init, + name='fc_value_final' + ) + ])(x) + + +class EnsembleValueHead(nn.Module): + n: int = 2 + + n_hidden_layers: int = 1 + hidden_dim: int = 256 + activation: Callable = nn.tanh + kernel_init: Callable = init_orth(calc_gain('tanh')) + last_layer_kernel_init: Callable = init_orth(calc_gain('linear')) + + @nn.compact + def __call__(self, x): + """ + Assumes x is batch + """ + VmapValue = nn.vmap( + ValueHead, + variable_axes={"params": 0}, + split_rngs={"params": True}, + in_axes=None, + out_axes=1, + axis_size=self.n, + ) + vs = VmapValue( + n_hidden_layers=self.n_hidden_layers, + hidden_dim=self.hidden_dim, + activation=self.activation, + kernel_init=self.kernel_init, + last_layer_kernel_init=self.last_layer_kernel_init + )(x) + + return vs + + +def clean_init_kwargs_prefix(prefix): + def class_decorator(cls): + old_init = cls.__init__ + + def new_init(self, *args, **kwargs): + kwargs = { + k.removeprefix(prefix): v for k, v in kwargs.items() + } + old_init(self, *args, **kwargs) + + cls.__init__ = new_init + return cls + + return class_decorator diff --git a/src/minimax/models/fast_attention.py b/src/minimax/models/fast_attention.py new file mode 100644 index 0000000..3da27e8 --- /dev/null +++ b/src/minimax/models/fast_attention.py @@ -0,0 +1,711 @@ +# coding=utf-8 +# Copyright 2023 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Core Fast Attention Module for Flax. + +Implementation of the approximate fast softmax and generalized +attention mechanism leveraging structured random feature maps [RFM] techniques +and low rank decomposition of the attention matrix. +""" +# pylint: disable=invalid-name, missing-function-docstring, line-too-long + +import abc +from collections.abc import Iterable # pylint: disable=g-importing-member +import functools +from absl import logging +import jax +from jax import lax +from jax import random +import jax.numpy as jnp + +import numpy as onp + + +def nonnegative_softmax_kernel_feature_creator(data, + projection_matrix, + attention_dims_t, + batch_dims_t, + precision, + is_query, + normalize_data=True, + eps=0.0001): + """Constructs nonnegative kernel features for fast softmax attention. + + + Args: + data: input for which features are computes + projection_matrix: random matrix used to compute features + attention_dims_t: tuple of attention dimensions + batch_dims_t: tuple of batch dimensions + precision: precision parameter + is_query: predicate indicating whether input data corresponds to queries or + keys + normalize_data: predicate indicating whether data should be normalized, + eps: numerical stabilizer. + + Returns: + Random features for fast softmax attention. + """ + + if normalize_data: + # We have e^{qk^T/sqrt{d}} = e^{q_norm k_norm^T}, where + # w_norm = w * data_normalizer for w in {q,k}. + data_normalizer = 1.0 / (jnp.sqrt(jnp.sqrt(data.shape[-1]))) + else: + data_normalizer = 1.0 + ratio = 1.0 / jnp.sqrt(projection_matrix.shape[0]) + data_mod_shape = data.shape[0:len(batch_dims_t)] + projection_matrix.shape + data_thick_random_matrix = jnp.zeros(data_mod_shape) + projection_matrix + + data_dash = lax.dot_general( + data_normalizer * data, + data_thick_random_matrix, + (((data.ndim - 1,), (data_thick_random_matrix.ndim - 1,)), + (batch_dims_t, batch_dims_t)), + precision=precision) + + diag_data = jnp.square(data) + diag_data = jnp.sum(diag_data, axis=data.ndim - 1) + diag_data = (diag_data / 2.0) * data_normalizer * data_normalizer + diag_data = jnp.expand_dims(diag_data, axis=data.ndim - 1) + + last_dims_t = (len(data_dash.shape) - 1,) + if is_query: + data_dash = ratio * ( + jnp.exp(data_dash - diag_data - + jnp.max(data_dash, axis=last_dims_t, keepdims=True)) + eps) + else: + data_dash = ratio * ( + jnp.exp(data_dash - diag_data - jnp.max( + data_dash, axis=last_dims_t + attention_dims_t, keepdims=True)) + + eps) + + return data_dash + + +def sincos_softmax_kernel_feature_creator(data, + projection_matrix, + attention_dims_t, + batch_dims_t, + precision, + normalize_data=True): + """Constructs kernel sin-cos features for fast softmax attention. + + + Args: + data: input for which features are computes + projection_matrix: random matrix used to compute features + attention_dims_t: tuple of attention dimensions + batch_dims_t: tuple of batch dimensions + precision: precision parameter + normalize_data: predicate indicating whether data should be normalized. + + Returns: + Random features for fast softmax attention. + """ + if normalize_data: + # We have: exp(qk^T/sqrt{d}) = exp(|q|^2/2sqrt{d}) * exp(|k|^2/2sqrt{d}) * + # exp(-(|q*c-k*c|^2)/2), where c = 1.0 / sqrt{sqrt{d}}. + data_normalizer = 1.0 / (jnp.sqrt(jnp.sqrt(data.shape[-1]))) + else: + data_normalizer = 1.0 + ratio = 1.0 / jnp.sqrt(projection_matrix.shape[0]) + data_mod_shape = data.shape[0:len(batch_dims_t)] + projection_matrix.shape + data_thick_random_matrix = jnp.zeros(data_mod_shape) + projection_matrix + + data_dash = lax.dot_general( + data_normalizer * data, + data_thick_random_matrix, + (((data.ndim - 1,), (data_thick_random_matrix.ndim - 1,)), + (batch_dims_t, batch_dims_t)), + precision=precision) + data_dash_cos = ratio * jnp.cos(data_dash) + data_dash_sin = ratio * jnp.sin(data_dash) + data_dash = jnp.concatenate((data_dash_cos, data_dash_sin), axis=-1) + + # Constructing D_data and data^{'} + diag_data = jnp.square(data) + diag_data = jnp.sum(diag_data, axis=data.ndim - 1) + diag_data = (diag_data / 2.0) * data_normalizer * data_normalizer + diag_data = jnp.expand_dims(diag_data, axis=data.ndim - 1) + # Additional renormalization for numerical stability + data_renormalizer = jnp.max(diag_data, attention_dims_t, keepdims=True) + diag_data -= data_renormalizer + diag_data = jnp.exp(diag_data) + data_prime = data_dash * diag_data + return data_prime + + +def generalized_kernel_feature_creator(data, projection_matrix, batch_dims_t, + precision, kernel_fn, kernel_epsilon, + normalize_data): + """Constructs kernel features for fast generalized attention. + + + Args: + data: input for which features are computes + projection_matrix: matrix used to compute features + batch_dims_t: tuple of batch dimensions + precision: precision parameter + kernel_fn: kernel function used + kernel_epsilon: additive positive term added to every feature for numerical + stability + normalize_data: predicate indicating whether data should be normalized. + + Returns: + Random features for fast generalized attention. + """ + if normalize_data: + data_normalizer = 1.0 / (jnp.sqrt(jnp.sqrt(data.shape[-1]))) + else: + data_normalizer = 1.0 + if projection_matrix is None: + return kernel_fn(data_normalizer * data) + kernel_epsilon + else: + data_mod_shape = data.shape[0:len( + batch_dims_t)] + projection_matrix.shape + data_thick_random_matrix = jnp.zeros( + data_mod_shape) + projection_matrix + data_dash = lax.dot_general( + data_normalizer * data, + data_thick_random_matrix, + (((data.ndim - 1,), (data_thick_random_matrix.ndim - 1,)), + (batch_dims_t, batch_dims_t)), + precision=precision) + data_prime = kernel_fn(data_dash) + kernel_epsilon + return data_prime + + +def make_fast_softmax_attention(qkv_dim, + renormalize_attention=True, + numerical_stabilizer=0.000001, + nb_features=256, + ortho_features=True, + ortho_scaling=0.0, + redraw_features=True, + unidirectional=False, + nonnegative_features=True, + lax_scan_unroll=1): + """Construct a fast softmax attention method.""" + logging.info( + 'Fast softmax attention: %s features and orthogonal=%s, renormalize=%s', + nb_features, ortho_features, renormalize_attention) + if ortho_features: + matrix_creator = functools.partial( + GaussianOrthogonalRandomMatrix, + nb_features, + qkv_dim, + scaling=ortho_scaling) + else: + matrix_creator = functools.partial(GaussianUnstructuredRandomMatrix, + nb_features, qkv_dim) + if nonnegative_features: + + def kernel_feature_creator(data, + projection_matrix, + attention_dims_t, + batch_dims_t, + precision, + is_query, + normalize_data=True): + return nonnegative_softmax_kernel_feature_creator( + data, projection_matrix, attention_dims_t, batch_dims_t, precision, + is_query, normalize_data, numerical_stabilizer) + else: + + def kernel_feature_creator(data, + projection_matrix, + attention_dims_t, + batch_dims_t, + precision, + is_query, + normalize_data=True): + del is_query + return sincos_softmax_kernel_feature_creator(data, projection_matrix, + attention_dims_t, + batch_dims_t, precision, + normalize_data) + + attention_fn = FastAttentionviaLowRankDecomposition( + matrix_creator, + kernel_feature_creator, + renormalize_attention=renormalize_attention, + numerical_stabilizer=numerical_stabilizer, + redraw_features=redraw_features, + unidirectional=unidirectional, + lax_scan_unroll=lax_scan_unroll).dot_product_attention + return attention_fn + + +def make_fast_generalized_attention(qkv_dim, + renormalize_attention=True, + numerical_stabilizer=0.0, + nb_features=256, + features_type='deterministic', + kernel_fn=jax.nn.relu, + kernel_epsilon=0.001, + redraw_features=False, + unidirectional=False, + lax_scan_unroll=1): + """Construct a fast generalized attention menthod.""" + logging.info('Fast generalized attention.: %s features and renormalize=%s', + nb_features, renormalize_attention) + if features_type == 'ortho': + matrix_creator = functools.partial( + GaussianOrthogonalRandomMatrix, nb_features, qkv_dim, scaling=False) + elif features_type == 'iid': + matrix_creator = functools.partial(GaussianUnstructuredRandomMatrix, + nb_features, qkv_dim) + elif features_type == 'deterministic': + matrix_creator = None + else: + raise ValueError('Unknown feature value type') + + def kernel_feature_creator(data, + projection_matrix, + attention_dims_t, + batch_dims_t, + precision, + is_query, + normalize_data=False): + del attention_dims_t + del is_query + return generalized_kernel_feature_creator(data, projection_matrix, + batch_dims_t, precision, + kernel_fn, kernel_epsilon, + normalize_data) + + attention_fn = FastAttentionviaLowRankDecomposition( + matrix_creator, + kernel_feature_creator, + renormalize_attention=renormalize_attention, + numerical_stabilizer=numerical_stabilizer, + redraw_features=redraw_features, + unidirectional=unidirectional, + lax_scan_unroll=lax_scan_unroll).dot_product_attention + return attention_fn + + +class RandomMatrix(object): + r"""Abstract class providing a method for constructing 2D random arrays. + + Class is responsible for constructing 2D random arrays. + """ + + __metaclass__ = abc.ABCMeta + + @abc.abstractmethod + def get_2d_array(self): + raise NotImplementedError('Abstract method') + + +class GaussianUnstructuredRandomMatrix(RandomMatrix): + + def __init__(self, nb_rows, nb_columns, key): + self.nb_rows = nb_rows + self.nb_columns = nb_columns + self.key = key + + def get_2d_array(self): + return random.normal(self.key, (self.nb_rows, self.nb_columns)) + + +class GaussianOrthogonalRandomMatrix(RandomMatrix): + r"""Class providing a method to create Gaussian orthogonal matrix. + + Class is responsible for constructing 2D Gaussian orthogonal arrays. + """ + + def __init__(self, nb_rows, nb_columns, key, scaling=0): + self.nb_rows = nb_rows + self.nb_columns = nb_columns + self.key = key + self.scaling = scaling + + def get_2d_array(self): + nb_full_blocks = int(self.nb_rows / self.nb_columns) + block_list = [] + rng = self.key + for _ in range(nb_full_blocks): + rng, rng_input = jax.random.split(rng) + unstructured_block = random.normal(rng_input, + (self.nb_columns, self.nb_columns)) + q, _ = jnp.linalg.qr(unstructured_block) + q = jnp.transpose(q) + block_list.append(q) + remaining_rows = self.nb_rows - nb_full_blocks * self.nb_columns + if remaining_rows > 0: + rng, rng_input = jax.random.split(rng) + unstructured_block = random.normal(rng_input, + (self.nb_columns, self.nb_columns)) + q, _ = jnp.linalg.qr(unstructured_block) + q = jnp.transpose(q) + block_list.append(q[0:remaining_rows]) + final_matrix = jnp.vstack(block_list) + + if self.scaling == 0: + multiplier = jnp.linalg.norm( + random.normal(self.key, (self.nb_rows, self.nb_columns)), axis=1) + elif self.scaling == 1: + multiplier = jnp.sqrt(float(self.nb_columns)) * \ + jnp.ones((self.nb_rows)) + else: + raise ValueError( + 'Scaling must be one of {0, 1}. Was %s' % self._scaling) + + return jnp.matmul(jnp.diag(multiplier), final_matrix) + + +class FastAttention(object): + r"""Abstract class providing a method for fast attention. + + Class is responsible for providing a method for fast + approximate attention. + """ + + __metaclass__ = abc.ABCMeta + + @abc.abstractmethod + def dot_product_attention(self, + query, + key, + value, + dtype=jnp.float32, + bias=None, + mask=None, + axis=None, + broadcast_dropout=True, + dropout_rng=None, + dropout_rate=0., + deterministic=False, + precision=None): + """Computes dot-product attention given query, key, and value. + + This is the core function for applying fast approximate dot-product + attention. It calculates the attention weights given query and key and + combines the values using the attention weights. This function supports + multi-dimensional inputs. + + + Args: + query: queries for calculating attention with shape of [batch_size, dim1, + dim2, ..., dimN, num_heads, mem_channels]. + key: keys for calculating attention with shape of [batch_size, dim1, dim2, + ..., dimN, num_heads, mem_channels]. + value: values to be used in attention with shape of [batch_size, dim1, + dim2,..., dimN, num_heads, value_channels]. + dtype: the dtype of the computation (default: float32) + bias: bias for the attention weights. This can be used for incorporating + autoregressive mask, padding mask, proximity bias. + mask: mask for the attention weights. This can be used for incorporating + autoregressive masks. + axis: axises over which the attention is applied. + broadcast_dropout: bool: use a broadcasted dropout along batch dims. + dropout_rng: JAX PRNGKey: to be used for dropout. + dropout_rate: dropout rate. + deterministic: bool, deterministic or not (to apply dropout). + precision: numerical precision of the computation see `jax.lax.Precision` + for details. + + Returns: + Output of shape [bs, dim1, dim2, ..., dimN,, num_heads, value_channels]. + """ + raise NotImplementedError('Abstract method') + + +def _numerator(z_slice_shape, precision, unroll=1): + + def fwd(qs, ks, vs): + + def body(p, qkv): + (q, k, v) = qkv + p += jnp.einsum('...m,...d->...md', k, v, precision=precision) + X_slice = jnp.einsum('...m,...md->...d', q, p, precision=precision) + return p, X_slice + + init_value = jnp.zeros(z_slice_shape) + p, W = lax.scan(body, init_value, (qs, ks, vs), unroll=unroll) + return W, (p, qs, ks, vs) + + def bwd(pqkv, W_ct): + + def body(carry, qkv_xct): + p, p_ct = carry + q, k, v, x_ct = qkv_xct + q_ct = jnp.einsum('...d,...md->...m', x_ct, p, precision=precision) + p_ct += jnp.einsum('...d,...m->...md', x_ct, + q, precision=precision) + k_ct = jnp.einsum('...md,...d->...m', p_ct, v, precision=precision) + v_ct = jnp.einsum('...md,...m->...d', p_ct, k, precision=precision) + p -= jnp.einsum('...m,...d->...md', k, v, precision=precision) + return (p, p_ct), (q_ct, k_ct, v_ct) + + p, qs, ks, vs = pqkv + _, (qs_ct, ks_ct, vs_ct) = lax.scan( + body, (p, jnp.zeros_like(p)), (qs, ks, vs, W_ct), + reverse=True, + unroll=unroll) + return qs_ct, ks_ct, vs_ct + + @jax.custom_vjp + def _numerator_impl(qs, ks, vs): + W, _ = fwd(qs, ks, vs) + return W + + _numerator_impl.defvjp(fwd, bwd) + + return _numerator_impl + + +def _denominator(t_slice_shape, precision, unroll=1): + + def fwd(qs, ks): + + def body(p, qk): + q, k = qk + p += k + x = jnp.einsum('...m,...m->...', q, p, precision=precision) + return p, x + + p = jnp.zeros(t_slice_shape) + p, R = lax.scan(body, p, (qs, ks), unroll=unroll) + return R, (qs, ks, p) + + def bwd(qkp, R_ct): + + def body(carry, qkx): + p, p_ct = carry + q, k, x_ct = qkx + q_ct = jnp.einsum('...,...m->...m', x_ct, p, precision=precision) + p_ct += jnp.einsum('...,...m->...m', x_ct, q, precision=precision) + k_ct = p_ct + p -= k + return (p, p_ct), (q_ct, k_ct) + + qs, ks, p = qkp + _, (qs_ct, ks_ct) = lax.scan( + body, (p, jnp.zeros_like(p)), (qs, ks, R_ct), + reverse=True, + unroll=unroll) + return (qs_ct, ks_ct) + + @jax.custom_vjp + def _denominator_impl(qs, ks): + R, _ = fwd(qs, ks) + return R + + _denominator_impl.defvjp(fwd, bwd) + + return _denominator_impl + + +class FastAttentionviaLowRankDecomposition(FastAttention): + r"""Class providing a method for fast attention via low rank decomposition. + + Class is responsible for providing a method for fast + dot-product attention with the use of low rank decomposition (e.g. with + random feature maps). + """ + + def __init__(self, + matrix_creator, + kernel_feature_creator, + renormalize_attention, + numerical_stabilizer, + redraw_features, + unidirectional, + lax_scan_unroll=1): # For optimal GPU performance, set to 16. + rng = random.PRNGKey(0) + self.matrix_creator = matrix_creator + self.projection_matrix = self.draw_weights(rng) + self.kernel_feature_creator = kernel_feature_creator + self.renormalize_attention = renormalize_attention + self.numerical_stabilizer = numerical_stabilizer + self.redraw_features = redraw_features + self.unidirectional = unidirectional + self.lax_scan_unroll = lax_scan_unroll + + def draw_weights(self, key): + if self.matrix_creator is None: + return None + matrixrng, _ = random.split(key) + projection_matrix = self.matrix_creator(key=matrixrng).get_2d_array() + return projection_matrix + + def dot_product_attention(self, + query, + key, + value, + dtype=jnp.float32, + bias=None, + mask=None, + axis=None, + broadcast_dropout=True, + dropout_rng=None, + dropout_rate=0., + deterministic=False, + precision=None): + + assert key.shape[:-1] == value.shape[:-1] + assert (query.shape[0:1] == key.shape[0:1] and + query.shape[-1] == key.shape[-1]) + if axis is None: + axis = tuple(range(1, key.ndim - 2)) + if not isinstance(axis, Iterable): + axis = (axis,) + assert key.ndim == query.ndim + assert key.ndim == value.ndim + for ax in axis: + if not (query.ndim >= 3 and 1 <= ax < query.ndim - 2): + raise ValueError('Attention axis must be between the batch ' + 'axis and the last-two axes.') + n = key.ndim + + # Constructing projection tensor. + if self.redraw_features: + query_seed = lax.convert_element_type( + jnp.ceil(jnp.sum(query) * 10000000.0), jnp.int32) + rng = random.PRNGKey(query_seed) + self.projection_matrix = self.draw_weights(rng) + + # batch_dims is , num_heads> + batch_dims = tuple(onp.delete(range(n), axis + (n - 1,))) + # q & k -> (bs, , num_heads, , channels) + qk_perm = batch_dims + axis + (n - 1,) + k_extra_perm = axis + batch_dims + (n - 1,) + key_extra = key.transpose(k_extra_perm) + key = key.transpose(qk_perm) + query = query.transpose(qk_perm) + # v -> (bs, , num_heads, , channels) + v_perm = batch_dims + axis + (n - 1,) + value = value.transpose(v_perm) + batch_dims_t = tuple(range(len(batch_dims))) + attention_dims_t = tuple( + range(len(batch_dims), + len(batch_dims) + len(axis))) + + # Constructing tensors Q^{'} and K^{'}. + query_prime = self.kernel_feature_creator(query, self.projection_matrix, + attention_dims_t, batch_dims_t, + precision, True) + key_prime = self.kernel_feature_creator(key, self.projection_matrix, + attention_dims_t, batch_dims_t, + precision, False) + + if self.unidirectional: + index = attention_dims_t[0] + z_slice_shape = key_prime.shape[0:len(batch_dims_t)] + ( + key_prime.shape[-1],) + (value.shape[-1],) + + numerator_fn = _numerator( + z_slice_shape, precision, self.lax_scan_unroll) + W = numerator_fn( + jnp.moveaxis(query_prime, index, 0), + jnp.moveaxis(key_prime, index, 0), jnp.moveaxis(value, index, 0)) + + # Constructing W = (Q^{'}(K^{'})^{T})_{masked}V + W = jnp.moveaxis(W, 0, index) + + if not self.renormalize_attention: + # Unidirectional, not-normalized attention. + perm_inv = _invert_perm(qk_perm) + result = W.transpose(perm_inv) + return result + else: + # Unidirectional, normalized attention. + thick_all_ones = jnp.zeros(key.shape[0:-1]) + jnp.ones( + key_extra.shape[0:len(axis)]) + + index = attention_dims_t[0] + t_slice_shape = key_prime.shape[0:len(batch_dims_t)] + ( + key_prime.shape[-1],) + denominator_fn = _denominator(t_slice_shape, precision, + self.lax_scan_unroll) + R = denominator_fn( + jnp.moveaxis(query_prime, index, 0), + jnp.moveaxis(key_prime, index, 0)) + + R = jnp.moveaxis(R, 0, index) + else: + contract_query = tuple( + range(len(batch_dims) + len(axis), + len(batch_dims) + len(axis) + 1)) + contract_z = tuple(range(len(batch_dims), len(batch_dims) + 1)) + # Constructing Z = (K^{'})^{T}V + # Z (bs, , num_heads, channels_m, channels_v) + Z = lax.dot_general( + key_prime, + value, + ((attention_dims_t, attention_dims_t), + (batch_dims_t, batch_dims_t)), + precision=precision) + # Constructing W = Q^{'}Z = Q^{'}(K^{'})^{T}V + # q (bs, , num_heads, , channels_m) + # Z (bs, , num_heads, channels_m, channels_v) + # W (bs, , num_heads, , channels_v) + W = lax.dot_general( + query_prime, + Z, ((contract_query, contract_z), (batch_dims_t, batch_dims_t)), + precision=precision) + if not self.renormalize_attention: + # Bidirectional, not-normalized attention. + perm_inv = _invert_perm(qk_perm) + result = W.transpose(perm_inv) + return result + else: + # Bidirectional, normalized attention. + thick_all_ones = jnp.zeros(key.shape[0:-1]) + jnp.ones( + key_extra.shape[0:len(axis)]) + contract_key = tuple( + range(len(batch_dims), + len(batch_dims) + len(axis))) + contract_thick_all_ones = tuple( + range(thick_all_ones.ndim - len(axis), thick_all_ones.ndim)) + # Construct T = (K^{'})^{T} 1_L + # k (bs, , num_heads, , channels) + T = lax.dot_general( + key_prime, + thick_all_ones, ((contract_key, contract_thick_all_ones), + (batch_dims_t, batch_dims_t)), + precision=precision) + + # Construct partition function: R = Q^{'} T = Q^{'}(K^{'})^{T} 1_L + # q_p (bs, , num_heads, , channs_m) + # T (bs, , num_heads, channels_m) + R = lax.dot_general( + query_prime, + T, (((query_prime.ndim - 1,), (T.ndim - 1,)), + (batch_dims_t, range(0, + len(T.shape) - 1))), + precision=precision) + + R = R + 2 * self.numerical_stabilizer * ( + jnp.abs(R) <= self.numerical_stabilizer) + R = jnp.reciprocal(R) + R = jnp.expand_dims(R, len(R.shape)) + # W (bs, , num_heads, , channels_v) + # R (bs, , num_heads, , extra_channel) + result = W * R + # back to (bs, dim1, dim2, ..., dimN, num_heads, channels) + perm_inv = _invert_perm(qk_perm) + result = result.transpose(perm_inv) + return result + + +def _invert_perm(perm): + perm_inv = [0] * len(perm) + for i, j in enumerate(perm): + perm_inv[j] = i + return tuple(perm_inv) diff --git a/src/minimax/models/maze/__init__.py b/src/minimax/models/maze/__init__.py new file mode 100644 index 0000000..2ddb707 --- /dev/null +++ b/src/minimax/models/maze/__init__.py @@ -0,0 +1,12 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from .gridworld_models import ( + GridWorldACStudentModel, + GridWorldACTeacherModel, +) \ No newline at end of file diff --git a/src/minimax/models/maze/gridworld_models.py b/src/minimax/models/maze/gridworld_models.py new file mode 100644 index 0000000..36d8e6d --- /dev/null +++ b/src/minimax/models/maze/gridworld_models.py @@ -0,0 +1,277 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from typing import Tuple + +import numpy as np +import jax +import jax.numpy as jnp +import flax.linen as nn +import chex +from tensorflow_probability.substrates import jax as tfp + +from minimax.models import common +from minimax.models import s5 +from minimax.models.registration import register + + +class GridWorldBasicModel(nn.Module): + """Split Actor-Critic Architecture for PPO.""" + output_dim: int = 7 + n_hidden_layers: int = 1 + hidden_dim: int = 32 + n_conv_layers: int = 1 + n_conv_filters: int = 16 + conv_kernel_size: int = 3 + n_scalar_embeddings: int = 4 + max_scalar: int = 4 + scalar_embed_dim: int = 5 + recurrent_arch: str = None + recurrent_hidden_dim: int = 256 + base_activation: str = 'relu' + head_activation: str = 'tanh' + + s5_n_blocks: int = 2 + s5_n_layers: int = 4 + s5_layernorm_pos: str = None + s5_activation: str = "half_glu1" + + value_ensemble_size: int = 1 + + def setup(self): + self.conv = nn.Sequential([ + nn.Conv( + features=self.n_conv_filters, + kernel_size=[self.conv_kernel_size,]*2, + strides=1, + kernel_init=common.init_orth( + scale=common.calc_gain(self.base_activation) + ), + padding='VALID', + name='cnn'), + common.get_activation(self.base_activation) + ]) + + if self.n_scalar_embeddings > 0: + self.fc_scalar = nn.Embed( + num_embeddings=self.n_scalar_embeddings, + features=self.scalar_embed_dim, + embedding_init=common.init_orth( + common.calc_gain('linear') + ), + name=f'fc_scalar' + ) + elif self.scalar_embed_dim > 0: + self.fc_scalar = nn.Dense( + self.scalar_embed_dim, + kernel_init=common.init_orth( + common.calc_gain('linear') + ), + name=f'fc_scalar' + ) + else: + self.fc_scalar = None + + if self.recurrent_arch is not None: + if self.recurrent_arch == 's5': + self.embed_pre_s5 = nn.Sequential([ + nn.Dense( + self.recurrent_hidden_dim, + kernel_init=common.init_orth( + common.calc_gain('linear') + ), + name=f'fc_pre_s5' + ) + ]) + self.rnn = s5.make_s5_encoder_stack( + input_dim=self.recurrent_hidden_dim, + ssm_state_dim=self.recurrent_hidden_dim, + n_blocks=self.s5_n_blocks, + n_layers=self.s5_n_layers, + activation=self.s5_activation, + layernorm_pos=self.s5_layernorm_pos + ) + else: + self.rnn = common.ScannedRNN( + recurrent_arch=self.recurrent_arch, + recurrent_hidden_dim=self.recurrent_hidden_dim, + kernel_init=common.init_orth(), + recurrent_kernel_init=common.init_orth() + ) + else: + self.rnn = None + + self.pi_head = nn.Sequential([ + common.make_fc_layers( + 'fc_pi', + n_layers=self.n_hidden_layers, + hidden_dim=self.hidden_dim, + activation=common.get_activation(self.head_activation), + kernel_init=common.init_orth( + common.calc_gain(self.head_activation) + ) + ), + nn.Dense( + self.output_dim, + kernel_init=nn.initializers.constant(0.01), + name=f'fc_pi_final' + ) + ]) + + value_head_kwargs = dict( + n_hidden_layers=self.n_hidden_layers, + hidden_dim=self.hidden_dim, + activation=nn.tanh, + kernel_init=common.init_orth( + common.calc_gain('tanh') + ), + last_layer_kernel_init=common.init_orth( + common.calc_gain('linear') + ) + ) + + if self.value_ensemble_size > 1: + self.v_head = common.EnsembleValueHead( + n=self.value_ensemble_size, **value_head_kwargs) + else: + self.v_head = common.ValueHead(**value_head_kwargs) + + def __call__(self, x, carry=None): + raise NotImplementedError + + def initialize_carry( + self, + rng: chex.PRNGKey, + batch_dims: Tuple[int] = ()) -> Tuple[chex.ArrayTree, chex.ArrayTree]: + """Initialize hidden state of LSTM.""" + if self.recurrent_arch is not None: + if self.recurrent_arch == 's5': + return s5.S5EncoderStack.initialize_carry( # Since conj_sym=True + rng, batch_dims, self.recurrent_hidden_dim//2, self.s5_n_layers + ) + else: + return common.ScannedRNN.initialize_carry( + rng, batch_dims, self.recurrent_hidden_dim, self.recurrent_arch) + else: + raise ValueError('Model is not recurrent.') + + @property + def is_recurrent(self): + return self.recurrent_arch is not None + + +class GridWorldACStudentModel(GridWorldBasicModel): + def __call__(self, x, carry=None, reset=None): + """ + Inputs: + x: B x h x w observations + hxs: B x hx_dim hidden states + masks: B length vector of done masks + """ + old_x = x + img = x['image'] + agent_dir = x['agent_dir'] + aux = x.get('aux') + + if self.rnn is not None: + batch_dims = img.shape[:2] + x = self.conv(img).reshape(*batch_dims, -1) + else: + batch_dims = img.shape[:1] + x = self.conv(img).reshape(*batch_dims, -1) + + if self.fc_scalar is not None: + if self.n_scalar_embeddings == 0: + agent_dir /= self.max_scalar + + scalar_emb = self.fc_scalar(agent_dir).reshape(*batch_dims, -1) + x = jnp.concatenate([x, scalar_emb], axis=-1) + + if aux is not None: + x = jnp.concatenate([x, aux], axis=-1) + + if self.rnn is not None: + if self.recurrent_arch == 's5': + x = self.embed_pre_s5(x) + carry, x = self.rnn(carry, x, reset) + else: + carry, x = self.rnn(carry, (x, reset)) + + logits = self.pi_head(x) + + v = self.v_head(x) + + return v, logits, carry + + +class GridWorldACTeacherModel(GridWorldBasicModel): + """ + Original teacher model from Dennis et al, 2020. It is identical ins + high-level spec to the student model, but with the additional fwd logic + of concatenating a noise vector. + """ + def __call__(self, x, carry=None, reset=None): + """ + Inputs: + x: B x h x w observations + hxs: B x hx_dim hidden states + masks: B length vector of done masks + """ + img = x['image'] + time = x['time'] + noise = x.get('noise') + aux = x.get('aux') + + if self.rnn is not None: + batch_dims = img.shape[:2] + x = self.conv(img).reshape(*batch_dims, -1) + else: + batch_dims = img.shape[:1] + x = self.conv(img).reshape(*batch_dims, -1) + + if self.fc_scalar is not None: + if self.n_scalar_embeddings == 0: + time /= self.max_scalar + + scalar_emb = self.fc_scalar(time).reshape(*batch_dims, -1) + x = jnp.concatenate([x, scalar_emb], axis=-1) + + if noise is not None: + noise = noise.reshape(*batch_dims, -1) + x = jnp.concatenate([x, noise], axis=-1) + + if aux is not None: + x = jnp.concatenate([x, aux], axis=-1) + + if self.rnn is not None: + if self.recurrent_arch == 's5': + x = self.embed_pre_s5(x) + carry, x = self.rnn(carry, x, reset) + else: + carry, x = self.rnn(carry, (x, reset)) + + logits = self.pi_head(x) + + v = self.v_head(x) + + return v, logits, carry + + +# Register models +if hasattr(__loader__, 'name'): + module_path = __loader__.name +elif hasattr(__loader__, 'fullname'): + module_path = __loader__.fullname + +register( + env_group_id='Maze', model_id='default_student_cnn', + entry_point=module_path + ':GridWorldACStudentModel') + +register( + env_group_id='Maze', model_id='default_teacher_cnn', + entry_point=module_path + ':GridWorldACTeacherModel') diff --git a/src/minimax/models/moe.py b/src/minimax/models/moe.py new file mode 100644 index 0000000..06eb461 --- /dev/null +++ b/src/minimax/models/moe.py @@ -0,0 +1,162 @@ +from typing import Any + +import einops +import jax +import flax.linen as nn +import jax.numpy as jnp + +from flax.linen.initializers import constant, orthogonal + +from minimax.models.common import StateCNNBase + + + +class MultiExpertLayer(nn.Module): + + in_features: int + out_features: int + num_experts: int + slots_per_expert: int + + def setup(self) -> None: + + self.weight = self.param( + "weight", + nn.initializers.xavier_uniform(), + (self.num_experts, self.in_features, self.out_features), + ) + + self.bias = self.param( + "bias", + nn.initializers.xavier_uniform(), + (self.num_experts, self.out_features), + ) + + def __call__(self, x) -> Any: + x = einops.einsum(x, self.weight, "b n ... d1, n d1 d2 -> b n ... d2") + + if self.bias is not None: + # NOTE: When used with 'SoftMoE' the inputs to 'MultiExpertLayer' will + # always be 4-dimensional. But it's easy enough to generalize for 3D + # inputs as well, so I decided to include that here. + # if x.ndim == 3: + # bias = einops.rearrange(self.bias, "n d -> () n d") + if x.ndim == 4: + bias = einops.rearrange(self.bias, "n d -> () n () d") + else: + raise ValueError( + f"Expected input to have 3 or 4 dimensions, but got {x.ndim}" + ) + x = x + bias + + return x + + +class SoftMoE(nn.Module): + + in_features: int + out_features: int + num_experts: int + slots_per_expert: int + + def setup(self) -> None: + self.experts = MultiExpertLayer( + in_features=self.in_features, + out_features=self.out_features, + num_experts=self.num_experts, + slots_per_expert=self.slots_per_expert, + + ) + self.phi = self.param( + 'phi', + nn.initializers.xavier_uniform(), + (self.in_features, self.num_experts, self.slots_per_expert), + ) + + def __call__(self, x) -> Any: + logits = einops.einsum(x, self.phi, "b m d, d n p -> b m n p") + dispatch_weights = nn.softmax(logits, axis=1) + # dispatch_weights = logits.softmax(dim=1) # denoted 'D' in the paper + # NOTE: The 'torch.softmax' function does not support multiple values for the + # 'dim' argument (unlike jax), so we are forced to flatten the last two dimensions. + # Then, we rearrange the Tensor into its original shape. + combine_weights = nn.softmax(logits, axis=(-2,-1)) + # combine_weights = einops.rearrange( + # nn.softmax(logits.reshape((*logits.shape[:-2], -1)), axis=-1), + # # logits.flatten(start_dim=2).softmax(dim=-1), + # "b m (n p) -> b m n p", + # n=self.num_experts, + # ) + + # NOTE: To save memory, I don't rename the intermediate tensors Y, Ys, Xs. + # Instead, I just overwrite the 'x' variable. The names from the paper are + # included in a comment for each line below. + x = einops.einsum( + x, dispatch_weights, "b m d, b m n p -> b n p d") # Xs + x = self.experts(x) # Ys + x = einops.einsum(x, combine_weights, "b n p d, b m n p -> b m d") # Y + + return x + + +class MoE(nn.Module): + activation: str = "tanh" + state_encoder_module: nn.Module = StateCNNBase + hiddem_dim: int = 64 + recurrent_arch: str = None + + def setup(self) -> None: + self.state_encoder = self.state_encoder_module( + activation=self.activation) + + self.moe = SoftMoE( + in_features=self.state_encoder.out_features, + out_features=self.hiddem_dim, + num_experts=4, + slots_per_expert=32, + ) + + self.proj = nn.Dense( + self.hiddem_dim, kernel_init=orthogonal(0.01), bias_init=constant(0.0) + ) + self.proj_layer_norm = nn.LayerNorm() + + + + def __call__(self, x) -> Any: + if self.activation == "relu": + activation = nn.relu + elif self.activation == "tanh": + activation = nn.tanh + else: + raise ValueError('Activation not recognized.') + + input_shape = x.shape + state_embedding = self.state_encoder(x) + + if len(input_shape) == 5: + a, n = state_embedding.shape[:2] + state_embedding = einops.rearrange(state_embedding, "a n ... -> (a n) ...", a=a, n=n) + + state_embedding = einops.rearrange(state_embedding, "... w h c -> ... (w h) c") + state_embedding = self.moe(state_embedding) + + state_embedding = x.reshape((*state_embedding.shape[:-2], -1)) + + if len(input_shape) == 5: + state_embedding = einops.rearrange(state_embedding, "(a n) ... -> a n ...", a=a, n=n) + + state_embedding = self.proj(state_embedding) + state_embedding = self.proj_layer_norm(state_embedding) + state_embedding = activation(state_embedding) + + return state_embedding + + +if __name__ == '__main__': + rng = jax.random.PRNGKey(30) + obs = jnp.zeros((200,6,9,26)) + moe = MoE(action_dim=6) + params = moe.init(rng, obs) + logits, _ = moe.apply(params, obs) + jax.debug.breakpoint() \ No newline at end of file diff --git a/src/minimax/models/overcooked/__init__.py b/src/minimax/models/overcooked/__init__.py new file mode 100644 index 0000000..5d61337 --- /dev/null +++ b/src/minimax/models/overcooked/__init__.py @@ -0,0 +1,14 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from .models import ( + ACStudentModel, + ACStudentActorModel, + ACStudentCriticModel, + ACTeacherModel, +) diff --git a/src/minimax/models/overcooked/models.py b/src/minimax/models/overcooked/models.py new file mode 100644 index 0000000..7bbc8b0 --- /dev/null +++ b/src/minimax/models/overcooked/models.py @@ -0,0 +1,536 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from typing import Tuple, Sequence + +import einops +import numpy as np +import jax +import jax.numpy as jnp +import flax.linen as nn +import chex +from tensorflow_probability.substrates import jax as tfp + +from minimax.models import common +from minimax.models import s5 +from minimax.models import transformer +from minimax.models.registration import register +from minimax.models.moe import SoftMoE + +from flax.linen.initializers import constant, orthogonal + + +class BasicModel(nn.Module): + """Split Actor-Critic Architecture for PPO.""" + output_dim: int = 6 + n_hidden_layers: int = 1 + hidden_dim: int = 32 + n_conv_layers: int = 1 + n_conv_filters: int = 16 + conv_encoder: bool = True + conv_kernel_size: int = 3 + n_scalar_embeddings: int = 4 + max_scalar: int = 4 + scalar_embed_dim: int = 5 + recurrent_arch: str = None + recurrent_hidden_dim: int = 256 + base_activation: str = 'relu' + head_activation: str = 'tanh' + + s5_n_blocks: int = 2 + s5_n_layers: int = 4 + s5_layernorm_pos: str = None + s5_activation: str = "half_glu1" + + transf_init_scale: float = 0.1 + transf_num_layers: int = 2 + transf_num_heads: int = 4 + transf_dropout_prob: float = 0.0 + transf_deterministic: bool = True + transf_return_embeddings: bool = False + transf_use_fast_attention: bool = False + transf_gated: bool = True + + is_soft_moe: bool = False + soft_moe_num_experts: int = 4 + soft_moe_num_slots: int = 32 + + value_ensemble_size: int = 1 + + def setup(self): + + if self.conv_encoder: + conv_list = [] + for i, feat in enumerate([32, 64, 32]): + # padding = "SAME" if i < self.n_conv_layers - 2 else "VALID" + conv_list.append( + nn.Conv( + features=feat, + kernel_size=[self.conv_kernel_size,]*2, + strides=1, + kernel_init=common.init_orth( + scale=common.calc_gain(self.base_activation) + ), + bias_init=common.default_bias_init(), + padding=((1, 1), (1, 1)), # padding, # 'SAME', + name=f'cnn_{i}' + ) + ) + conv_list.append( + common.get_activation(self.base_activation) + ) + + self.conv = nn.Sequential(conv_list) + self.after_conv = common.make_fc_layers( + n_layers=self.n_hidden_layers, + hidden_dim=self.hidden_dim, + activation=common.get_activation(self.base_activation), + kernel_init=common.init_orth( + scale=common.calc_gain(self.base_activation) + ), + bias_init=common.default_bias_init(), + use_layernorm=True, + ) + self.linear_encoder = None + else: + self.conv = None + self.linear_encoder = nn.Sequential([ + nn.Dense( + self.hidden_dim, + kernel_init=common.init_orth( + common.calc_gain('linear') + ), + name=f'fc_linear' + ), + common.get_activation(self.base_activation), + nn.LayerNorm(name='ln_linear'), + ]) + + if self.is_soft_moe: + self.moe = SoftMoE( + in_features=self.n_conv_filters, + out_features=self.hidden_dim, + num_experts=self.soft_moe_num_experts, # 4, + slots_per_expert=self.soft_moe_num_slots, # 32, + ) + + if self.n_scalar_embeddings > 0: + self.fc_scalar = nn.Embed( + num_embeddings=self.n_scalar_embeddings, + features=self.scalar_embed_dim, + embedding_init=common.init_orth( + common.calc_gain('linear') + ), + name=f'fc_scalar' + ) + elif self.scalar_embed_dim > 0: + self.fc_scalar = nn.Dense( + self.scalar_embed_dim, + kernel_init=common.init_orth( + common.calc_gain('linear') + ), + name=f'fc_scalar' + ) + else: + self.fc_scalar = None + + if self.recurrent_arch is not None: + if self.recurrent_arch == 's5': + self.embed_pre_s5 = nn.Sequential([ + nn.Dense( + self.recurrent_hidden_dim, + kernel_init=common.init_orth( + common.calc_gain('linear') + ), + name=f'fc_pre_s5' + ) + ]) + self.rnn = s5.make_s5_encoder_stack( + input_dim=self.recurrent_hidden_dim, + ssm_state_dim=self.recurrent_hidden_dim, + n_blocks=self.s5_n_blocks, + n_layers=self.s5_n_layers, + activation=self.s5_activation, + layernorm_pos=self.s5_layernorm_pos + ) + elif self.recurrent_arch == 'transformer': + self.rnn = transformer.ScannedTransformer( + hidden_dim=self.recurrent_hidden_dim, + init_scale=self.transf_init_scale, + transf_num_layers=self.transf_num_layers, + transf_num_heads=self.transf_num_heads, + transf_dim_feedforward=self.recurrent_hidden_dim, + transf_dropout_prob=self.transf_transf_dropout_prob, + deterministic=self.transf_deterministic, + return_embeddings=self.transf_return_embeddings, + use_fast_attention=self.transf_use_fast_attention, + gated=self.transf_gated, + ) + else: + self.rnn = common.ScannedRNN( + recurrent_arch=self.recurrent_arch, + recurrent_hidden_dim=self.recurrent_hidden_dim, + kernel_init=common.init_orth(), + recurrent_kernel_init=common.init_orth() + ) + else: + self.rnn = None + + self.pi_head = nn.Sequential([ + # common.make_fc_layers( + # 'fc_pi', + # n_layers=self.n_hidden_layers, + # hidden_dim=self.hidden_dim, + # activation=common.get_activation(self.head_activation), + # kernel_init=common.init_orth( + # common.calc_gain(self.head_activation) + # ) + # ), + nn.Dense( + self.output_dim, + kernel_init=nn.initializers.constant(0.01), + name=f'fc_pi_final' + ) + ]) + + value_head_kwargs = dict( + n_hidden_layers=0, + hidden_dim=self.hidden_dim, + activation=nn.tanh, + kernel_init=common.init_orth( + common.calc_gain('tanh') + ), + last_layer_kernel_init=common.init_orth( + common.calc_gain('linear') + ) + ) + + if self.value_ensemble_size > 1: + self.v_head = common.EnsembleValueHead( + n=self.value_ensemble_size, **value_head_kwargs) + else: + self.v_head = common.ValueHead(**value_head_kwargs) + + def __call__(self, x, carry=None): + raise NotImplementedError + + def initialize_carry( + self, + rng: chex.PRNGKey, + batch_dims: Tuple[int] = ()) -> Tuple[chex.ArrayTree, chex.ArrayTree]: + """Initialize hidden state of LSTM.""" + if self.recurrent_arch is not None: + if self.recurrent_arch == 's5': + return s5.S5EncoderStack.initialize_carry( # Since conj_sym=True + rng, batch_dims, self.recurrent_hidden_dim//2, self.s5_n_layers + ) + elif self.recurrent_arch == 'transformer': + return transformer.ScannedTransformer.initialize_carry( + self.recurrent_hidden_dim, batch_dims) + else: + return common.ScannedRNN.initialize_carry( + rng, batch_dims, self.recurrent_hidden_dim, self.recurrent_arch) + else: + raise ValueError('Model is not recurrent.') + + @property + def is_recurrent(self): + return self.recurrent_arch is not None + + +class ACStudentActorModel(BasicModel): + def __call__(self, x, carry=None, reset=None): + """ + Inputs: + x: B x h x w observations + hxs: B x hx_dim hidden states + masks: B length vector of done masks + """ + img = x + + if self.rnn is not None: + batch_dims = img.shape[:-3] + x = self.conv(img) + else: + batch_dims = img.shape[:-3] + x = self.conv(img) + + if self.is_soft_moe: + initial_shape = x.shape + if len(initial_shape) == 5: + a, n, h, w, f = x.shape + x = einops.rearrange(x, "a n ... -> (a n) ...", a=a, n=n) + + x = einops.rearrange(x, "... w h c -> ... (w h) c") + + x = self.moe(x) + + if len(initial_shape) == 5: + x = einops.rearrange(x, "(a n) ... -> a n ...", a=a, n=n) + + x = x.reshape(*batch_dims, -1) + x = self.after_conv(x) + + if self.rnn is not None: + if self.recurrent_arch == 's5': + x = self.embed_pre_s5(x) + carry, x = self.rnn(carry, x, reset) + elif self.recurrent_arch == 'transformer': + x = self.rnn(carry, (x, mask, reset)) + else: + carry, x = self.rnn(carry, (x, reset)) + + logits = self.pi_head(x) + return logits, carry + + +class ACStudentActorModelMlp(BasicModel): + + conv_encoder: bool = False + + def __call__(self, x, carry=None, reset=None): + """ + Inputs: + x: B x h x w observations + hxs: B x hx_dim hidden states + masks: B length vector of done masks + """ + img = x + + if self.rnn is not None: + batch_dims = img.shape[:-1] + x = self.linear_encoder(img) + x = x.reshape(*batch_dims, -1) + else: + batch_dims = img.shape[:-1] + x = self.linear_encoder(img) + x = x.reshape(*batch_dims, -1) + + if self.rnn is not None: + if self.recurrent_arch == 's5': + x = self.embed_pre_s5(x) + carry, x = self.rnn(carry, x, reset) + elif self.recurrent_arch == 'transformer': + x = self.rnn(carry, (x, mask, reset)) + else: + carry, x = self.rnn(carry, (x, reset)) + + logits = self.pi_head(x) + return logits, carry + + +class ACStudentCriticModel(BasicModel): + + def __call__(self, x, carry=None, reset=None): + """ + Inputs: + x: B x h x w observations + hxs: B x hx_dim hidden states + masks: B length vector of done masks + """ + img = x + + if self.rnn is not None: + batch_dims = img.shape[:-3] + x = self.conv(img) + else: + batch_dims = img.shape[:-3] + x = self.conv(img) + + if self.is_soft_moe: + initial_shape = x.shape + if len(initial_shape) == 5: + a, n, h, w, f = x.shape + x = einops.rearrange(x, "a n ... -> (a n) ...", a=a, n=n) + + x = einops.rearrange(x, "... w h c -> ... (w h) c") + + x = self.moe(x) + + if len(initial_shape) == 5: + x = einops.rearrange(x, "(a n) ... -> a n ...", a=a, n=n) + + x = x.reshape(*batch_dims, -1) + x = self.after_conv(x) + + if self.rnn is not None: + if self.recurrent_arch == 's5': + x = self.embed_pre_s5(x) + carry, x = self.rnn(carry, x, reset) + else: + carry, x = self.rnn(carry, (x, reset)) + + v = self.v_head(x) + + return v, carry + + +class ACStudentCriticModelMlp(BasicModel): + + conv_encoder: bool = False + + def __call__(self, x, carry=None, reset=None): + """ + Inputs: + x: B x h x w observations + hxs: B x hx_dim hidden states + masks: B length vector of done masks + """ + img = x + + if self.rnn is not None: + batch_dims = img.shape[:-1] + x = self.linear_encoder(img) + x = x.reshape(*batch_dims, -1) + else: + batch_dims = img.shape[:-1] + x = self.linear_encoder(img) + x = x.reshape(*batch_dims, -1) + + # NOTE: Continue here tomorrow + # Is x reshape of shape zero?? + if self.rnn is not None: + if self.recurrent_arch == 's5': + x = self.embed_pre_s5(x) + carry, x = self.rnn(carry, x, reset) + elif self.recurrent_arch == 'transformer': + x = self.rnn(carry, (x, mask, reset)) + else: + carry, x = self.rnn(carry, (x, reset)) + + v = self.v_head(x) + + return v, carry + + +class ACStudentModel(BasicModel): + def __call__(self, x, carry=None, reset=None): + """ + Inputs: + x: B x h x w observations + hxs: B x hx_dim hidden states + masks: B length vector of done masks + """ + img = x + + if self.rnn is not None: + batch_dims = img.shape[:-3] + x = self.conv(img) + x = x.reshape(*batch_dims, -1) + else: + batch_dims = img.shape[:-3] + x = self.conv(img) + x = x.reshape(*batch_dims, -1) + + if self.rnn is not None: + if self.recurrent_arch == 's5': + x = self.embed_pre_s5(x) + carry, x = self.rnn(carry, x, reset) + elif self.recurrent_arch == 'transformer': + x = self.rnn(carry, (x, mask, reset)) + else: + carry, x = self.rnn(carry, (x, reset)) + + v = self.v_head(x) + + logits = self.pi_head(x) + + return v, logits, carry + + +class ACTeacherModel(BasicModel): + """ + Original teacher model from Dennis et al, 2020. It is identical ins + high-level spec to the student model, but with the additional fwd logic + of concatenating a noise vector. + """ + + def __call__(self, x, carry=None, reset=None): + """ + Inputs: + x: B x h x w observations + hxs: B x hx_dim hidden states + masks: B length vector of done masks + """ + img = x['image'] + time = x['time'] + noise = x.get('noise') + aux = x.get('aux') + + if self.rnn is not None: + batch_dims = img.shape[:2] + x = self.conv(img).reshape(*batch_dims, -1) + else: + batch_dims = img.shape[:1] + x = self.conv(img).reshape(*batch_dims, -1) + + if self.fc_scalar is not None: + if self.n_scalar_embeddings == 0: + time /= self.max_scalar + + scalar_emb = self.fc_scalar(time).reshape(*batch_dims, -1) + x = jnp.concatenate([x, scalar_emb], axis=-1) + + if noise is not None: + noise = noise.reshape(*batch_dims, -1) + x = jnp.concatenate([x, noise], axis=-1) + + if aux is not None: + x = jnp.concatenate([x, aux], axis=-1) + + if self.rnn is not None: + if self.recurrent_arch == 's5': + x = self.embed_pre_s5(x) + carry, x = self.rnn(carry, x, reset) + else: + carry, x = self.rnn(carry, (x, reset)) + + logits = self.pi_head(x) + + v = self.v_head(x) + + return v, logits, carry + + +# Register models +if hasattr(__loader__, 'name'): + module_path = __loader__.name +elif hasattr(__loader__, 'fullname'): + module_path = __loader__.fullname + +register( + env_group_id='Overcooked', model_id='default_student_actor_moe', + entry_point=module_path + ':ACStudentActorModelSoftMoE') + +register( + env_group_id='Overcooked', model_id='default_student_critic_moe', + entry_point=module_path + ':ACStudentCriticModelMoE') + +register( + env_group_id='Overcooked', model_id='default_student_actor_cnn', + entry_point=module_path + ':ACStudentActorModel') + +register( + env_group_id='Overcooked', model_id='default_student_critic_cnn', + entry_point=module_path + ':ACStudentCriticModel') + +register( + env_group_id='Overcooked', model_id='default_student_actor_mlp', + entry_point=module_path + ':ACStudentActorModelMlp') + +register( + env_group_id='Overcooked', model_id='default_student_critic_mlp', + entry_point=module_path + ':ACStudentCriticModelMlp') + +register( + env_group_id='Overcooked', model_id='default_student_cnn', + entry_point=module_path + ':ACStudentModel') + +register( + env_group_id='Overcooked', model_id='default_teacher_cnn', + entry_point=module_path + ':ACTeacherModel') diff --git a/src/minimax/models/registration.py b/src/minimax/models/registration.py new file mode 100644 index 0000000..afc84e5 --- /dev/null +++ b/src/minimax/models/registration.py @@ -0,0 +1,51 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +import importlib +import copy + +# Global registry +registered_models = {} + + +def _load(name): + mod_name, attr_name = name.split(":") + mod = importlib.import_module(mod_name) + fn = getattr(mod, attr_name) + return fn + + +def _get_register_id(env_group_id, model_id): + return f"{env_group_id.lower()}-{model_id}" + + +def register(env_group_id, model_id, entry_point): + register_id = _get_register_id(env_group_id, model_id) + if register_id in registered_models: + raise ValueError(f'A model has already been registered as {register_id}.') + else: + registered_models[register_id] = entry_point + + +def make( + env_name, model_name=None, **model_kwargs): + env_group_id = env_name.split('-')[0].lstrip('UED') + model_id = model_name + + register_id = _get_register_id(env_group_id, model_id) + if register_id not in registered_models: + raise ValueError(f'No model for {register_id} found.') + else: + entry = registered_models[register_id] + + if callable(entry): + model = entry(**model_kwargs) + else: + model = _load(entry)(**model_kwargs) + + return model \ No newline at end of file diff --git a/src/minimax/models/rnn.py b/src/minimax/models/rnn.py new file mode 100644 index 0000000..01e8b93 --- /dev/null +++ b/src/minimax/models/rnn.py @@ -0,0 +1,98 @@ +""" +Copyright 2018 The JAX Authors. + +This file is based on the OptimizedLSTMCell class from +https://github.com/google/jax + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from functools import partial +from typing import Any, Tuple, Mapping + +import numpy as np +import jax.numpy as jnp +import flax.linen as nn +from flax.linen.dtypes import promote_dtype +from flax.linen.module import compact +from flax.linen.recurrent import DenseParams + + +Array = Any + + +class CustomOptimizedLSTMCell(nn.OptimizedLSTMCell): + @compact + def __call__(self, carry: Tuple[Array, Array], + inputs: Array) -> Tuple[Tuple[Array, Array], Array]: + r"""An optimized long short-term memory (LSTM) cell. + + Args: + carry: the hidden state of the LSTM cell, initialized using + `LSTMCell.initialize_carry`. + inputs: an ndarray with the input for the current time step. All + dimensions except the final are considered batch dimensions. + + Returns: + A tuple with the new carry and the output. + """ + c, h = carry + hidden_features = h.shape[-1] + + def _concat_dense(inputs: Array, + params: Mapping[str, Tuple[Array, Array]], + use_bias: bool = True) -> Array: + # Concatenates the individual kernels and biases, given in params, into a + # single kernel and single bias for efficiency before applying them using + # dot_general. + kernels, biases = zip(*params.values()) + kernel = jnp.concatenate(kernels, axis=-1) + if use_bias: + bias = jnp.concatenate(biases, axis=-1) + else: + bias = None + inputs, kernel, bias = promote_dtype(inputs, kernel, bias, dtype=self.dtype) + y = jnp.dot(inputs, kernel) + if use_bias: + y += jnp.reshape(bias, (1,) * (y.ndim - 1) + (-1,)) + + # Split the result back into individual (i, f, g, o) outputs. + split_indices = np.cumsum([kernel.shape[-1] for kernel in kernels[:-1]]) + ys = jnp.split(y, split_indices, axis=-1) + return dict(zip(params.keys(), ys)) + + # Create params with the same names/shapes as `LSTMCell` for compatibility. + dense_params_h = {} + dense_params_i = {} + for component in ['i', 'f', 'g', 'o']: + dense_params_i[component] = DenseParams( + features=hidden_features, use_bias=True, + param_dtype=self.param_dtype, + kernel_init=self.kernel_init, bias_init=self.bias_init, + name=f'i{component}')(inputs) + dense_params_h[component] = DenseParams( + features=hidden_features, use_bias=True, + param_dtype=self.param_dtype, + kernel_init=self.recurrent_kernel_init, bias_init=self.bias_init, + name=f'h{component}')(h) + dense_h = _concat_dense(h, dense_params_h, use_bias=True) + dense_i = _concat_dense(inputs, dense_params_i, use_bias=True) + + i = self.gate_fn(dense_h['i'] + dense_i['i']) + f = self.gate_fn(dense_h['f'] + dense_i['f']) + g = self.activation_fn(dense_h['g'] + dense_i['g']) + o = self.gate_fn(dense_h['o'] + dense_i['o']) + + new_c = f * c + i * g + new_h = o * self.activation_fn(new_c) + return (new_c, new_h), new_h diff --git a/src/minimax/models/s5.py b/src/minimax/models/s5.py new file mode 100644 index 0000000..bacab12 --- /dev/null +++ b/src/minimax/models/s5.py @@ -0,0 +1,706 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This file is modified from +https://github.com/luchris429/purejaxrl + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +""" + +from functools import partial +import jax +import jax.numpy as np +import jax.numpy as jnp +from flax import linen as nn +from jax.nn.initializers import lecun_normal, normal +from jax import random +from jax.numpy.linalg import eigh +from jax.scipy.linalg import block_diag + + +def log_step_initializer(dt_min=0.001, dt_max=0.1): + """ Initialize the learnable timescale Delta by sampling + uniformly between dt_min and dt_max. + Args: + dt_min (float32): minimum value + dt_max (float32): maximum value + Returns: + init function + """ + def init(key, shape): + """ Init function + Args: + key: jax random key + shape tuple: desired shape + Returns: + sampled log_step (float32) + """ + return random.uniform(key, shape) * ( + np.log(dt_max) - np.log(dt_min) + ) + np.log(dt_min) + + return init + + +def init_log_steps(key, input): + """ Initialize an array of learnable timescale parameters + Args: + key: jax random key + input: tuple containing the array shape H and + dt_min and dt_max + Returns: + initialized array of timescales (float32): (H,) + """ + H, dt_min, dt_max = input + log_steps = [] + for i in range(H): + key, skey = random.split(key) + log_step = log_step_initializer( + dt_min=dt_min, dt_max=dt_max)(skey, shape=(1,)) + log_steps.append(log_step) + + return np.array(log_steps) + + +def init_VinvB(init_fun, rng, shape, Vinv): + """ Initialize B_tilde=V^{-1}B. First samples B. Then compute V^{-1}B. + Note we will parameterize this with two different matrices for complex + numbers. + Args: + init_fun: the initialization function to use, e.g. lecun_normal() + rng: jax random key to be used with init function. + shape (tuple): desired shape (P,H) + Vinv: (complex64) the inverse eigenvectors used for initialization + Returns: + B_tilde (complex64) of shape (P,H,2) + """ + B = init_fun(rng, shape) + VinvB = Vinv @ B + VinvB_real = VinvB.real + VinvB_imag = VinvB.imag + return np.concatenate((VinvB_real[..., None], VinvB_imag[..., None]), axis=-1) + + +def trunc_standard_normal(key, shape): + """ Sample C with a truncated normal distribution with standard deviation 1. + Args: + key: jax random key + shape (tuple): desired shape, of length 3, (H,P,_) + Returns: + sampled C matrix (float32) of shape (H,P,2) (for complex parameterization) + """ + H, P, _ = shape + Cs = [] + for i in range(H): + key, skey = random.split(key) + C = lecun_normal()(skey, shape=(1, P, 2)) + Cs.append(C) + return np.array(Cs)[:, 0] + + +def init_CV(init_fun, rng, shape, V): + """ Initialize C_tilde=CV. First sample C. Then compute CV. + Note we will parameterize this with two different matrices for complex + numbers. + Args: + init_fun: the initialization function to use, e.g. lecun_normal() + rng: jax random key to be used with init function. + shape (tuple): desired shape (H,P) + V: (complex64) the eigenvectors used for initialization + Returns: + C_tilde (complex64) of shape (H,P,2) + """ + C_ = init_fun(rng, shape) + C = C_[..., 0] + 1j * C_[..., 1] + CV = C @ V + CV_real = CV.real + CV_imag = CV.imag + return np.concatenate((CV_real[..., None], CV_imag[..., None]), axis=-1) + + +# Discretization functions +def discretize_bilinear(Lambda, B_tilde, Delta): + """ Discretize a diagonalized, continuous-time linear SSM + using bilinear transform method. + Args: + Lambda (complex64): diagonal state matrix (P,) + B_tilde (complex64): input matrix (P, H) + Delta (float32): discretization step sizes (P,) + Returns: + discretized Lambda_bar (complex64), B_bar (complex64) (P,), (P,H) + """ + Identity = np.ones(Lambda.shape[0]) + + BL = 1 / (Identity - (Delta / 2.0) * Lambda) + Lambda_bar = BL * (Identity + (Delta / 2.0) * Lambda) + B_bar = (BL * Delta)[..., None] * B_tilde + return Lambda_bar, B_bar + + +def discretize_zoh(Lambda, B_tilde, Delta): + """ Discretize a diagonalized, continuous-time linear SSM + using zero-order hold method. + Args: + Lambda (complex64): diagonal state matrix (P,) + B_tilde (complex64): input matrix (P, H) + Delta (float32): discretization step sizes (P,) + Returns: + discretized Lambda_bar (complex64), B_bar (complex64) (P,), (P,H) + """ + Identity = np.ones(Lambda.shape[0]) + Lambda_bar = np.exp(Lambda * Delta) + B_bar = (1/Lambda * (Lambda_bar-Identity))[..., None] * B_tilde + return Lambda_bar, B_bar + + +# Parallel scan operations +@jax.vmap +def binary_operator(q_i, q_j): + """ Binary operator for parallel scan of linear recurrence. Assumes a diagonal matrix A. + Args: + q_i: tuple containing A_i and Bu_i at position i (P,), (P,) + q_j: tuple containing A_j and Bu_j at position j (P,), (P,) + Returns: + new element ( A_out, Bu_out ) + """ + A_i, b_i = q_i + A_j, b_j = q_j + return A_j * A_i, A_j * b_i + b_j + +# Parallel scan operations + + +@jax.vmap +def binary_operator_reset(q_i, q_j): + """ Binary operator for parallel scan of linear recurrence. Assumes a diagonal matrix A. + Args: + q_i: tuple containing A_i and Bu_i at position i (P,), (P,) + q_j: tuple containing A_j and Bu_j at position j (P,), (P,) + Returns: + new element ( A_out, Bu_out ) + """ + A_i, b_i, c_i = q_i + A_j, b_j, c_j = q_j + return ( + (A_j * A_i)*(1 - c_j) + A_j * c_j, + (A_j * b_i + b_j)*(1 - c_j) + b_j * c_j, + c_i * (1 - c_j) + c_j, + ) + + +def apply_ssm(Lambda_bar, B_bar, C_tilde, hidden, input_sequence, resets, conj_sym, bidirectional): + """ Compute the LxH output of discretized SSM given an LxH input. + Args: + Lambda_bar (complex64): discretized diagonal state matrix (P,) + B_bar (complex64): discretized input matrix (P, H) + C_tilde (complex64): output matrix (H, P) + input_sequence (float32): input sequence of features (L, H) + reset (bool): input sequence of features (L,) + conj_sym (bool): whether conjugate symmetry is enforced + bidirectional (bool): whether bidirectional setup is used, + Note for this case C_tilde will have 2P cols + Returns: + ys (float32): the SSM outputs (S5 layer preactivations) (L, H) + """ + Lambda_elements = Lambda_bar * jnp.ones((input_sequence.shape[0], + Lambda_bar.shape[0])) + Bu_elements = jax.vmap(lambda u: B_bar @ u)(input_sequence) + + Lambda_elements = jnp.concatenate([ + jnp.ones((1, Lambda_bar.shape[0])), + Lambda_elements, + ]) + + Bu_elements = jnp.concatenate([ + hidden, + Bu_elements, + ]) + + resets = jnp.concatenate([ + jnp.zeros(1), + resets, + ]) + + _, xs, _ = jax.lax.associative_scan( + binary_operator_reset, (Lambda_elements, Bu_elements, resets)) + xs = xs[1:] + + if conj_sym: + return xs[np.newaxis, -1], jax.vmap(lambda x: 2*(C_tilde @ x).real)(xs) + else: + return xs[np.newaxis, -1], jax.vmap(lambda x: (C_tilde @ x).real)(xs) + + +class S5SSM(nn.Module): + Lambda_re_init: jax.Array + Lambda_im_init: jax.Array + V: jax.Array + Vinv: jax.Array + + H: int + P: int + C_init: str + discretization: str + dt_min: float + dt_max: float + conj_sym: bool = True + clip_eigs: bool = False + bidirectional: bool = False + step_rescale: float = 1.0 + + """ The S5 SSM + Args: + Lambda_re_init (complex64): Real part of init diag state matrix (P,) + Lambda_im_init (complex64): Imag part of init diag state matrix (P,) + V (complex64): Eigenvectors used for init (P,P) + Vinv (complex64): Inverse eigenvectors used for init (P,P) + H (int32): Number of features of input seq + P (int32): state size + C_init (string): Specifies How C is initialized + Options: [trunc_standard_normal: sample from truncated standard normal + and then multiply by V, i.e. C_tilde=CV. + lecun_normal: sample from Lecun_normal and then multiply by V. + complex_normal: directly sample a complex valued output matrix + from standard normal, does not multiply by V] + conj_sym (bool): Whether conjugate symmetry is enforced + clip_eigs (bool): Whether to enforce left-half plane condition, i.e. + constrain real part of eigenvalues to be negative. + True recommended for autoregressive task/unbounded sequence lengths + Discussed in https://arxiv.org/pdf/2206.11893.pdf. + bidirectional (bool): Whether model is bidirectional, if True, uses two C matrices + discretization: (string) Specifies discretization method + options: [zoh: zero-order hold method, + bilinear: bilinear transform] + dt_min: (float32): minimum value to draw timescale values from when + initializing log_step + dt_max: (float32): maximum value to draw timescale values from when + initializing log_step + step_rescale: (float32): allows for uniformly changing the timescale parameter, e.g. after training + on a different resolution for the speech commands benchmark + """ + + def setup(self): + """Initializes parameters once and performs discretization each time + the SSM is applied to a sequence + """ + if self.conj_sym: + # Need to account for case where we actually sample real B and C, and then multiply + # by the half sized Vinv and possibly V + local_P = 2*self.P + else: + local_P = self.P + + # Initialize diagonal state to state matrix Lambda (eigenvalues) + self.Lambda_re = self.param( + "Lambda_re", lambda rng, shape: self.Lambda_re_init, (None,)) + self.Lambda_im = self.param( + "Lambda_im", lambda rng, shape: self.Lambda_im_init, (None,)) + if self.clip_eigs: + self.Lambda = np.clip(self.Lambda_re, None, - + 1e-4) + 1j * self.Lambda_im + else: + self.Lambda = self.Lambda_re + 1j * self.Lambda_im + + # Initialize input to state (B) matrix + B_init = lecun_normal() + B_shape = (local_P, self.H) + self.B = self.param("B", + lambda rng, shape: init_VinvB(B_init, + rng, + shape, + self.Vinv), + B_shape) + B_tilde = self.B[..., 0] + 1j * self.B[..., 1] + + # Initialize state to output (C) matrix + if self.C_init in ["trunc_standard_normal"]: + C_init = trunc_standard_normal + C_shape = (self.H, local_P, 2) + elif self.C_init in ["lecun_normal"]: + C_init = lecun_normal() + C_shape = (self.H, local_P, 2) + elif self.C_init in ["complex_normal"]: + C_init = normal(stddev=0.5 ** 0.5) + else: + raise NotImplementedError( + "C_init method {} not implemented".format(self.C_init)) + + if self.C_init in ["complex_normal"]: + if self.bidirectional: + C = self.param("C", C_init, (self.H, 2 * self.P, 2)) + self.C_tilde = C[..., 0] + 1j * C[..., 1] + + else: + C = self.param("C", C_init, (self.H, self.P, 2)) + self.C_tilde = C[..., 0] + 1j * C[..., 1] + + else: + if self.bidirectional: + self.C1 = self.param("C1", + lambda rng, shape: init_CV( + C_init, rng, shape, self.V), + C_shape) + self.C2 = self.param("C2", + lambda rng, shape: init_CV( + C_init, rng, shape, self.V), + C_shape) + + C1 = self.C1[..., 0] + 1j * self.C1[..., 1] + C2 = self.C2[..., 0] + 1j * self.C2[..., 1] + self.C_tilde = np.concatenate((C1, C2), axis=-1) + + else: + self.C = self.param("C", + lambda rng, shape: init_CV( + C_init, rng, shape, self.V), + C_shape) + + self.C_tilde = self.C[..., 0] + 1j * self.C[..., 1] + + # Initialize feedthrough (D) matrix + self.D = self.param("D", normal(stddev=1.0), (self.H,)) + + # Initialize learnable discretization timescale value + self.log_step = self.param("log_step", + init_log_steps, + (self.P, self.dt_min, self.dt_max)) + step = self.step_rescale * np.exp(self.log_step[:, 0]) + + # Discretize + if self.discretization in ["zoh"]: + self.Lambda_bar, self.B_bar = discretize_zoh( + self.Lambda, B_tilde, step) + elif self.discretization in ["bilinear"]: + self.Lambda_bar, self.B_bar = discretize_bilinear( + self.Lambda, B_tilde, step) + else: + raise NotImplementedError( + "Discretization method {} not implemented".format(self.discretization)) + + def __call__(self, hidden, input_sequence, resets): + """ + Compute the LxH output of the S5 SSM given an LxH input sequence + using a parallel scan. + Args: + input_sequence (float32): input sequence (L, H) + resets (bool): input sequence (L,) + Returns: + output sequence (float32): (L, H) + """ + hidden, ys = apply_ssm( + self.Lambda_bar, + self.B_bar, + self.C_tilde, + hidden, + input_sequence, + resets, + self.conj_sym, + self.bidirectional) + # Add feedthrough matrix output Du; + Du = jax.vmap(lambda u: self.D * u)(input_sequence) + return hidden, ys + Du + + +class SequenceLayer(nn.Module): + """ Defines a single S5 layer, with S5 SSM, nonlinearity, + dropout, batch/layer norm, etc. + Args: + ssm (nn.Module): the SSM to be used (i.e. S5 ssm) + dropout (float32): dropout rate + d_model (int32): this is the feature size of the layer inputs and outputs + we usually refer to this size as H + activation (string): Type of activation function to use + training (bool): whether in training mode or not + prenorm (bool): apply prenorm if true or postnorm if false + batchnorm (bool): apply batchnorm if true or layernorm if false + bn_momentum (float32): the batchnorm momentum if batchnorm is used + step_rescale (float32): allows for uniformly changing the timescale parameter, + e.g. after training on a different resolution for + the speech commands benchmark + """ + ssm: nn.Module + d_model: int + activation: str = "gelu" + layernorm_pos: str = None # ['pre', 'post', None] + step_rescale: float = 1.0 + + def setup(self): + """Initializes the ssm, batch/layer norm and dropout + """ + self.seq = self.ssm(step_rescale=self.step_rescale) + + if self.activation in ["full_glu"]: + self.out1 = nn.Dense(self.d_model) + self.out2 = nn.Dense(self.d_model) + elif self.activation in ["half_glu1", "half_glu2"]: + self.out2 = nn.Dense(self.d_model) + + self.norm = nn.LayerNorm() + self.drop = lambda x: x + + def __call__(self, hidden, x, d): + """ + Compute the LxH output of S5 layer given an LxH input. + Args: + x (float32): input sequence (L, B, d_model) + d (bool): reset signal (L,B) + Returns: + output sequence (float32): (L, B, d_model) + """ + is_one_step = len(hidden.shape) == 2 + if is_one_step: # Add time axis + hidden = hidden[jnp.newaxis, :] + + skip = x + if self.layernorm_pos == 'pre': + x = self.norm(x) + hidden, x = jax.vmap(self.seq, in_axes=1, out_axes=1)(hidden, x, d) + + if self.activation in ["full_glu"]: + x = self.drop(nn.gelu(x)) + x = self.out1(x) * jax.nn.sigmoid(self.out2(x)) + x = self.drop(x) + elif self.activation in ["half_glu1"]: + x = self.drop(nn.gelu(x)) + x = x * jax.nn.sigmoid(self.out2(x)) + x = self.drop(x) + elif self.activation in ["half_glu2"]: + # Only apply GELU to the gate input + x1 = self.drop(nn.gelu(x)) + x = x * jax.nn.sigmoid(self.out2(x1)) + x = self.drop(x) + elif self.activation in ["gelu"]: + x = self.drop(nn.gelu(x)) + else: + raise NotImplementedError( + "Activation: {} not implemented".format(self.activation)) + + x = skip + x + if self.layernorm_pos == 'post': + x = self.norm(x) + if is_one_step: + hidden = hidden.squeeze(0) + + return hidden, x + + @staticmethod + def initialize_carry(batch_size, hidden_size): + return jnp.zeros((batch_size, hidden_size), dtype=jnp.complex64) + + +def init_S5SSM( + H, + P, + Lambda_re_init, + Lambda_im_init, + V, + Vinv, + C_init, + discretization, + dt_min, + dt_max, + conj_sym, + clip_eigs, + bidirectional): + """Convenience function that will be used to initialize the SSM. + Same arguments as defined in S5SSM above.""" + return partial(S5SSM, + H=H, + P=P, + Lambda_re_init=Lambda_re_init, + Lambda_im_init=Lambda_im_init, + V=V, + Vinv=Vinv, + C_init=C_init, + discretization=discretization, + dt_min=dt_min, + dt_max=dt_max, + conj_sym=conj_sym, + clip_eigs=clip_eigs, + bidirectional=bidirectional) + + +def make_HiPPO(N): + """ Create a HiPPO-LegS matrix. + From https://github.com/srush/annotated-s4/blob/main/s4/s4.py + Args: + N (int32): state size + Returns: + N x N HiPPO LegS matrix + """ + P = np.sqrt(1 + 2 * np.arange(N)) + A = P[:, np.newaxis] * P[np.newaxis, :] + A = np.tril(A) - np.diag(np.arange(N)) + return -A + + +def make_NPLR_HiPPO(N): + """ + Makes components needed for NPLR representation of HiPPO-LegS + From https://github.com/srush/annotated-s4/blob/main/s4/s4.py + Args: + N (int32): state size + Returns: + N x N HiPPO LegS matrix, low-rank factor P, HiPPO input matrix B + """ + # Make -HiPPO + hippo = make_HiPPO(N) + + # Add in a rank 1 term. Makes it Normal. + P = np.sqrt(np.arange(N) + 0.5) + + # HiPPO also specifies the B matrix + B = np.sqrt(2 * np.arange(N) + 1.0) + return hippo, P, B + + +def make_DPLR_HiPPO(N): + """ + Makes components needed for DPLR representation of HiPPO-LegS + From https://github.com/srush/annotated-s4/blob/main/s4/s4.py + Note, we will only use the diagonal part + Args: + N: + Returns: + eigenvalues Lambda, low-rank term P, conjugated HiPPO input matrix B, + eigenvectors V, HiPPO B pre-conjugation + """ + A, P, B = make_NPLR_HiPPO(N) + + S = A + P[:, np.newaxis] * P[np.newaxis, :] + + S_diag = np.diagonal(S) + Lambda_real = np.mean(S_diag) * np.ones_like(S_diag) + + # Diagonalize S to V \Lambda V^* + Lambda_imag, V = eigh(S * -1j) + + P = V.conj().T @ P + B_orig = B + B = V.conj().T @ B + return Lambda_real + 1j * Lambda_imag, P, B, V, B_orig + + +class S5EncoderStack(nn.Module): + """ Defines a stack of S5 layers to be used as an encoder. + Args: + ssm (nn.Module): the SSM to be used (i.e. S5 ssm) + d_model (int32): this is the feature size of the layer inputs and outputs + we usually refer to this size as H + n_layers (int32): the number of S5 layers to stack + activation (string): Type of activation function to use + dropout (float32): dropout rate + training (bool): whether in training mode or not + prenorm (bool): apply prenorm if true or postnorm if false + batchnorm (bool): apply batchnorm if true or layernorm if false + bn_momentum (float32): the batchnorm momentum if batchnorm is used + step_rescale (float32): allows for uniformly changing the timescale parameter, + e.g. after training on a different resolution for + the speech commands benchmark + """ + ssm: nn.Module + d_model: int + n_layers: int + layernorm_pos: str = None + activation: str = "gelu" + + def setup(self): + """ + Initializes a linear encoder and the stack of S5 layers. + """ + self.layers = [ + SequenceLayer( + ssm=self.ssm, + d_model=self.d_model, + activation=self.activation, + layernorm_pos=self.layernorm_pos, + ) + for _ in range(self.n_layers) + ] + + def __call__(self, hidden, x, reset): + """ + Compute the BxLxH output of the stacked encoder given an Lxd_input + input sequence. + Args: + x (float32): input sequence (L, d_input) + Returns: + output sequence (float32): (L, d_model) + """ + new_hiddens = [] + for i, layer in enumerate(self.layers): + new_h, x = layer(hidden[i], x, reset) + new_hiddens.append(new_h) + + return new_hiddens, x + + @staticmethod + def initialize_carry(rng, batch_dims, hidden_dim, n_layers): + # Use a dummy key since the default state init fn is just zeros. + return [jnp.zeros((*batch_dims, hidden_dim), dtype=jnp.complex64) for _ in range(n_layers)] + + +BatchS5EncoderStack = nn.vmap( + S5EncoderStack, + in_axes=(1, 1, 1), + out_axes=1, + variable_axes={"params": None}, + split_rngs={"params": False}, axis_name='batch') + + +def make_s5_encoder_stack( + input_dim, + ssm_state_dim, + n_blocks=1, + n_layers=4, + discretization='zoh', + dt_min=0.001, + dt_max=0.1, + conj_sym=True, + clip_eigs=False, + bidirectional=False, + activation="half_glu1", + layernorm_pos=None): + block_size = int(ssm_state_dim / n_blocks) + + Lambda, _, B, V, B_orig = make_DPLR_HiPPO(block_size) + + if conj_sym: + block_size = block_size // 2 + ssm_state_dim = ssm_state_dim // 2 + + Lambda = Lambda[:block_size] + V = V[:, :block_size] + Vc = V.conj().T + + Lambda = (Lambda*np.ones((n_blocks, block_size))).ravel() + V = block_diag(*([V]*n_blocks)) + Vinv = block_diag(*([Vc]*n_blocks)) + + ssm_init_fn = init_S5SSM( + H=input_dim, + P=ssm_state_dim, + Lambda_re_init=Lambda.real, + Lambda_im_init=Lambda.imag, + V=V, + Vinv=Vinv, + C_init="lecun_normal", + discretization=discretization, + dt_min=dt_min, + dt_max=dt_max, + conj_sym=conj_sym, + clip_eigs=clip_eigs, + bidirectional=bidirectional) + + return S5EncoderStack( + ssm=ssm_init_fn, + d_model=input_dim, + n_layers=n_layers, + activation=activation, + layernorm_pos=layernorm_pos + ) diff --git a/src/minimax/models/transformer.py b/src/minimax/models/transformer.py new file mode 100644 index 0000000..ca74cb4 --- /dev/null +++ b/src/minimax/models/transformer.py @@ -0,0 +1,264 @@ +from functools import partial +import flax.linen as nn +import jax.numpy as jnp + +import einops + + +import numpy as np + +from flax.linen.initializers import constant, orthogonal + + +class GRUGating(nn.Module): + + dim: int + scale_residual: bool = False + + def setup(self): + super().__init__() + self.gru = nn.GRUCell(self.dim, self.dim) + self.residual_scale = nn.Parameter( + jnp.ones(self.dim)) if self.scale_residual else None + + def __call__(self, x, residual): + if self.residual_scale is not None: + residual = residual * self.residual_scale + + gated_output = self.gru( + einops.rearrange(x, 'b n d -> (b n) d'), + einops.rearrange(residual, 'b n d -> (b n) d') + ) + + return gated_output.reshape_as(x) + + +class GatedEncoderBlock(nn.Module): + # Input dimension is needed here since it is equal to the output dimension (residual connection) + hidden_dim: int + num_heads: int + dim_feedforward: int + init_scale: float + use_fast_attention: bool + dropout_prob: float = 0. + + def setup(self): + # Attention layer + if self.use_fast_attention: + from fast_attention import make_fast_generalized_attention + raw_attention_fn = make_fast_generalized_attention( + self.hidden_dim // self.num_heads, + renormalize_attention=True, + nb_features=self.hidden_dim, + unidirectional=False + ) + self.self_attn = nn.MultiHeadDotProductAttention( + num_heads=self.num_heads, + dropout_rate=self.dropout_prob, + attention_fn=raw_attention_fn, + kernel_init=nn.initializers.xavier_uniform(), + use_bias=False, + ) + else: + self.self_attn = nn.MultiHeadDotProductAttention( + num_heads=self.num_heads, + dropout_rate=self.dropout_prob, + kernel_init=nn.initializers.xavier_uniform(), + use_bias=False, + ) + # Two-layer MLP + self.linear = [ + nn.Dense(self.dim_feedforward, kernel_init=nn.initializers.xavier_uniform( + ), bias_init=constant(0.0)), + nn.Dense(self.hidden_dim, kernel_init=nn.initializers.xavier_uniform( + ), bias_init=constant(0.0)) + ] + # Layers to apply in between the main layers + self.gate1 = GRUGating(dim=self.hidden_dim) + self.norm1 = nn.LayerNorm() + self.gate2 = GRUGating(dim=self.hidden_dim) + self.norm2 = nn.LayerNorm() + self.dropout = nn.Dropout(self.dropout_prob) + + def __call__(self, x, mask=None, deterministic=True): + + # Attention part + # masking is not compatible with fast self attention + x_norm1 = self.norm1(x) + if mask is not None and not self.use_fast_attention: + mask = jnp.repeat(nn.make_attention_mask( + mask, mask), self.num_heads, axis=-3) + attended = self.self_attn( + inputs_q=x_norm1, inputs_kv=x_norm1, mask=mask, deterministic=deterministic) + + # GRU gate + x = self.gate1(attended, x_norm1) + x = self.dropout(x, deterministic=deterministic) + + x_res = x + + # MLP part + x = self.norm2(x) + feedforward = self.linear[0](x) + feedforward = nn.relu(feedforward) + feedforward = self.linear[1](feedforward) + + # GRU Gate + x = self.gate2(x, x_res) + x = self.dropout(x, deterministic=deterministic) + return x + + +class EncoderBlock(nn.Module): + # Input dimension is needed here since it is equal to the output dimension (residual connection) + hidden_dim: int + num_heads: int + dim_feedforward: int + init_scale: float + use_fast_attention: bool + dropout_prob: float = 0. + + def setup(self): + # Attention layer + if self.use_fast_attention: + from fast_attention import make_fast_generalized_attention + raw_attention_fn = make_fast_generalized_attention( + self.hidden_dim // self.num_heads, + renormalize_attention=True, + nb_features=self.hidden_dim, + unidirectional=False + ) + self.self_attn = nn.MultiHeadDotProductAttention( + num_heads=self.num_heads, + dropout_rate=self.dropout_prob, + attention_fn=raw_attention_fn, + kernel_init=nn.initializers.xavier_uniform(), + use_bias=False, + ) + else: + self.self_attn = nn.MultiHeadDotProductAttention( + num_heads=self.num_heads, + dropout_rate=self.dropout_prob, + kernel_init=nn.initializers.xavier_uniform(), + use_bias=False, + ) + # Two-layer MLP + self.linear = [ + nn.Dense(self.dim_feedforward, kernel_init=nn.initializers.xavier_uniform( + ), bias_init=constant(0.0)), + nn.Dense(self.hidden_dim, kernel_init=nn.initializers.xavier_uniform( + ), bias_init=constant(0.0)) + ] + # Layers to apply in between the main layers + self.norm1 = nn.LayerNorm() + self.norm2 = nn.LayerNorm() + self.dropout = nn.Dropout(self.dropout_prob) + + def __call__(self, x, mask=None, deterministic=True): + + # Attention part + # masking is not compatible with fast self attention + if mask is not None and not self.use_fast_attention: + mask = jnp.repeat(nn.make_attention_mask( + mask, mask), self.num_heads, axis=-3) + attended = self.self_attn( + inputs_q=x, inputs_kv=x, mask=mask, deterministic=deterministic) + + x = self.norm1(attended + x) + x = x + self.dropout(x, deterministic=deterministic) + + # MLP part + feedforward = self.linear[0](x) + feedforward = nn.relu(feedforward) + feedforward = self.linear[1](feedforward) + + x = self.norm2(feedforward+x) + x = x + self.dropout(x, deterministic=deterministic) + + return x + + +class Embedder(nn.Module): + hidden_dim: int + init_scale: float + scale_inputs: bool = True + activation: bool = False + + @nn.compact + def __call__(self, x, train: bool): + if self.scale_inputs: + x = nn.BatchNorm(use_running_average=not train)(x) + x = nn.Dense(self.hidden_dim, kernel_init=orthogonal( + self.init_scale), bias_init=constant(0.0))(x) + if self.activation: + x = nn.relu(x) + x = nn.BatchNorm(use_running_average=not train)(x) + return x + + +class ScannedTransformer(nn.Module): + + hidden_dim: int + init_scale: float + transf_num_layers: int + transf_num_heads: int + transf_dim_feedforward: int + transf_dropout_prob: float = 0 + deterministic: bool = True + return_embeddings: bool = False + use_fast_attention: bool = False + gated: bool = True + + def setup(self): + self.encoders = [ + GatedEncoderBlock( + self.hidden_dim, + self.transf_num_heads, + self.transf_dim_feedforward, + self.init_scale, + self.use_fast_attention, + self.transf_dropout_prob, + ) if self.gated else EncoderBlock( + self.hidden_dim, + self.transf_num_heads, + self.transf_dim_feedforward, + self.init_scale, + self.use_fast_attention, + self.transf_dropout_prob, + ) for _ in range(self.transf_num_layers) + ] + + @partial( + nn.scan, + variable_broadcast="params", + in_axes=0, + out_axes=0, + split_rngs={"params": False}, + ) + def __call__(self, carry, x): + hs = carry + embeddings, mask, done = x + + hs = jnp.where( + done[:, np.newaxis, np.newaxis], + self.initialize_carry(self.hidden_dim, *done.shape, 1), + hs + ) + embeddings = jnp.concatenate(( + hs, + embeddings, + ), axis=-2) + for layer in self.encoders: + embeddings = layer(embeddings, mask=mask, + deterministic=self.deterministic) + hs = embeddings[..., 0:1, :] + + # as y return the entire embeddings if required (i.e. transformer mixer), otherwise only agents' hs embeddings + if self.return_embeddings: + return hs, embeddings + else: + return hs, hs + + @staticmethod + def initialize_carry(hidden_size, *batch_size): + return jnp.zeros((*batch_size, hidden_size)) diff --git a/src/minimax/runners/__init__.py b/src/minimax/runners/__init__.py new file mode 100644 index 0000000..74cd61a --- /dev/null +++ b/src/minimax/runners/__init__.py @@ -0,0 +1,22 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from .xp_runner import ExperimentRunner +from .eval_runner import EvalRunner +from .dr_runner import DRRunner +from .plr_runner import PLRRunner +from .paired_runner import PAIREDRunner + + +__all__ = [ + ExperimentRunner, + EvalRunner, + DRRunner, + PLRRunner, + PAIREDRunner +] \ No newline at end of file diff --git a/src/minimax/runners/dr_runner.py b/src/minimax/runners/dr_runner.py new file mode 100644 index 0000000..ecb0ef1 --- /dev/null +++ b/src/minimax/runners/dr_runner.py @@ -0,0 +1,458 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial +from typing import Tuple, Optional +import inspect + +import numpy as np +import jax +import jax.numpy as jnp +from jax.sharding import PartitionSpec as P +import optax +import flax +import flax.linen as nn +from flax.core.frozen_dict import FrozenDict + +import minimax.envs as envs +from minimax.util import pytree as _tree_util +from minimax.util.rl import ( + AgentPop, + VmapTrainState, + RolloutStorage, + RollingStats +) + + +class DRRunner: + """ + Orchestrates rollouts across one or more students. + The main components at play: + - AgentPop: Manages train state and batched inference logic + for a population of agents. + - BatchEnv: Manages environment step and reset logic, using a + populaton of agents. + - RolloutStorage: Manages the storing and sampling of collected txns. + - PPO: Handles PPO updates, which take a train state + batch of txns. + """ + + def __init__( + self, + env_name, + env_kwargs, + student_agents, + n_students=1, + n_parallel=1, + n_eval=1, + n_rollout_steps=256, + lr=1e-4, + lr_final=None, + lr_anneal_steps=0, + max_grad_norm=0.5, + discount=0.99, + gae_lambda=0.95, + adam_eps=1e-5, + normalize_return=False, + track_env_metrics=False, + n_unroll_rollout=1, + n_devices=1, + render=False): + + assert len(student_agents) == 1, 'Only one type of student supported.' + assert n_parallel % n_devices == 0, 'Num envs must be divisible by num devices.' + + self.n_students = n_students + self.n_parallel = n_parallel // n_devices + self.n_eval = n_eval + self.n_devices = n_devices + self.step_batch_size = n_students*n_eval*n_parallel + self.n_rollout_steps = n_rollout_steps + self.n_updates = 0 + self.lr = lr + self.lr_final = lr if lr_final is None else lr_final + self.lr_anneal_steps = lr_anneal_steps + self.max_grad_norm = max_grad_norm + self.adam_eps = adam_eps + self.normalize_return = normalize_return + self.track_env_metrics = track_env_metrics + self.n_unroll_rollout = n_unroll_rollout + self.render = render + + self.student_pop = AgentPop(student_agents[0], n_agents=n_students) + + self.env, self.env_params = envs.make( + env_name, + env_kwargs=env_kwargs + ) + self._action_shape = self.env.action_space().shape + + self.benv = envs.BatchEnv( + env_name=env_name, + n_parallel=self.n_parallel, + n_eval=self.n_eval, + env_kwargs=env_kwargs, + wrappers=['monitor_return', 'monitor_ep_metrics'] + ) + self.action_dtype = self.benv.env.action_space().dtype + + self.student_rollout = RolloutStorage( + discount=discount, + gae_lambda=gae_lambda, + n_steps=n_rollout_steps, + n_agents=n_students, + n_envs=self.n_parallel, + n_eval=self.n_eval, + action_space=self.env.action_space(), + obs_space=self.env.observation_space(), + agent=self.student_pop.agent, + ) + + monitored_metrics = self.benv.env.get_monitored_metrics() + self.rolling_stats = RollingStats( + names=monitored_metrics, + window=10, + ) + self._update_ep_stats = jax.vmap( + jax.vmap(self.rolling_stats.update_stats)) + + if self.render: + from envs.viz.grid_viz import GridVisualizer + self.viz = GridVisualizer() + self.viz.show() + + def reset(self, rng): + self.n_updates = 0 + + n_parallel = self.n_parallel*self.n_devices + + rngs, *vrngs = jax.random.split(rng, self.n_students+1) + obs, state, extra = self.benv.reset( + jnp.array(vrngs), n_parallel=n_parallel) + dummy_obs = jax.tree_util.tree_map( + lambda x: x[0], obs) # for one agent only + + rng, subrng = jax.random.split(rng) + if self.student_pop.agent.is_recurrent: + carry = self.student_pop.init_carry(subrng, obs) + self.zero_carry = jax.tree_map( + lambda x: x.at[:, :self.n_parallel].get(), carry) + else: + carry = None + + rng, subrng = jax.random.split(rng) + params = self.student_pop.init_params(subrng, dummy_obs) + + schedule_fn = optax.linear_schedule( + init_value=-float(self.lr), + end_value=-float(self.lr_final), + transition_steps=self.lr_anneal_steps, + ) + + tx = optax.chain( + optax.clip_by_global_norm(self.max_grad_norm), + optax.adam(learning_rate=float(self.lr), eps=self.adam_eps) + ) + + train_state = VmapTrainState.create( + apply_fn=self.student_pop.agent.evaluate, + params=params, + tx=tx + ) + + ep_stats = self.rolling_stats.reset_stats( + batch_shape=(self.n_students, n_parallel*self.n_eval)) + + start_state = state + + return ( + rng, + train_state, + state, + start_state, # Used to track metrics from starting state + obs, + carry, + extra, + ep_stats + ) + + def get_checkpoint_state(self, state): + _state = list(state) + _state[1] = state[1].state_dict + + return _state + + def load_checkpoint_state(self, runner_state, state): + runner_state = list(runner_state) + runner_state[1] = runner_state[1].load_state_dict(state[1]) + + return tuple(runner_state) + + @partial(jax.jit, static_argnums=(0, 2)) + def _get_transition( + self, + rng, + pop, + params, + rollout, + state, + start_state, + obs, + carry, + done, + extra=None): + # Sample action + value, pi_params, next_carry = pop.act(params, obs, carry, done) + + pi = pop.get_action_dist(pi_params, dtype=self.action_dtype) + rng, subrng = jax.random.split(rng) + action = pi.sample(seed=subrng) + log_pi = pi.log_prob(action) + + rng, *vrngs = jax.random.split(rng, self.n_students+1) + (next_obs, + next_state, + reward, + done, + info, + extra) = self.benv.step(jnp.array(vrngs), state, action, extra) + + next_start_state = jax.vmap(_tree_util.pytree_select)( + done, next_state, start_state + ) + + # Add transition to storage + step = (obs, action, reward, done, log_pi, value) + if carry is not None: + step += (carry,) + + rollout = self.student_rollout.append(rollout, *step) + + if self.render: + self.viz.render( + self.benv.env.params, + jax.tree_util.tree_map(lambda x: x[0][0], state)) + + return ( + rollout, + next_state, + next_start_state, + next_obs, + next_carry, + done, + info, + extra + ) + + @partial(jax.jit, static_argnums=(0,)) + def _rollout_students( + self, + rng, + train_state, + state, + start_state, + obs, + carry, + done, + extra=None, + ep_stats=None): + rollout = self.student_rollout.reset() + + rngs = jax.random.split(rng, self.n_rollout_steps) + + def _scan_rollout(scan_carry, rng): + rollout, state, start_state, obs, carry, done, extra, ep_stats, train_state = scan_carry + + next_scan_carry = \ + self._get_transition( + rng, + self.student_pop, + jax.lax.stop_gradient(train_state.params), + rollout, + state, + start_state, + obs, + carry, + done, + extra) + (rollout, + next_state, + next_start_state, + next_obs, + next_carry, + done, + info, + extra) = next_scan_carry + + ep_stats = self._update_ep_stats(ep_stats, done, info) + + return ( + rollout, + next_state, + next_start_state, + next_obs, + next_carry, + done, + extra, + ep_stats, + train_state), None + + (rollout, + state, + start_state, + obs, + carry, + done, + extra, + ep_stats, + train_state), _ = jax.lax.scan( + _scan_rollout, + (rollout, + state, + start_state, + obs, + carry, + done, + extra, + ep_stats, + train_state), + rngs, + length=self.n_rollout_steps, + ) + + return rollout, state, start_state, obs, carry, extra, ep_stats, train_state + + @partial(jax.jit, static_argnums=(0,)) + def _compile_stats(self, update_stats, ep_stats, env_metrics=None): + stats = jax.vmap(lambda info: jax.tree_map(lambda x: x.mean(), info))( + {k: ep_stats[k] for k in self.rolling_stats.names} + ) + stats.update(update_stats) + + if self.n_students > 1: + _stats = {} + for i in range(self.n_students): + _student_stats = jax.tree_util.tree_map( + lambda x: x[i], stats) # for agent0 + _stats.update( + {f'a{i}/{k}': v for k, v in _student_stats.items()}) + stats = _stats + + if self.track_env_metrics: + mean_env_metrics = jax.vmap(lambda info: jax.tree_map( + lambda x: x.mean(), info))(env_metrics) + mean_env_metrics = {f'env/{k}': v for k, + v in mean_env_metrics.items()} + + if self.n_students > 1: + _env_metrics = {} + for i in range(self.n_students): + _student_env_metrics = jax.tree_util.tree_map( + lambda x: x[i], mean_env_metrics) # for agent0 + _env_metrics.update( + {f'{k}_a{i}': v for k, v in _student_env_metrics.items()}) + mean_env_metrics = _env_metrics + + stats.update(mean_env_metrics) + + if self.n_students == 1: + stats = jax.tree_map(lambda x: x[0], stats) + + if self.n_devices > 1: + stats = jax.tree_map(lambda x: jax.lax.pmean(x, 'device'), stats) + + return stats + + def get_shmap_spec(self): + runner_state_size = len(inspect.signature(self.run).parameters) + in_spec = [P(None, 'device'),]*(runner_state_size) + out_spec = [P(None, 'device'),]*(runner_state_size) + + in_spec[:2] = [P(None),]*2 + in_spec = tuple(in_spec) + out_spec = (P(None),) + in_spec + + return in_spec, out_spec + + @partial(jax.jit, static_argnums=(0,)) + def run( + self, + rng, + train_state, + state, + start_state, + obs, + carry=None, + extra=None, + ep_stats=None): + """ + Perform one update step: rollout all students and teachers + update with PPO + """ + if self.n_devices > 1: + rng = jax.random.fold_in(rng, jax.lax.axis_index('device')) + + rng, *vrngs = jax.random.split(rng, self.n_students+1) + rollout_batch_shape = (self.n_students, self.n_parallel*self.n_eval) + + obs, state, extra = self.benv.reset(jnp.array(vrngs)) + ep_stats = self.rolling_stats.reset_stats( + batch_shape=rollout_batch_shape) + + rollout_start_state = state + + done = jnp.zeros(rollout_batch_shape, dtype=jnp.bool_) + rng, subrng = jax.random.split(rng) + rollout, state, start_state, obs, carry, extra, ep_stats, train_state = \ + self._rollout_students( + subrng, + train_state, + state, + start_state, + obs, + carry, + done, + extra, + ep_stats + ) + + train_batch = self.student_rollout.get_batch( + rollout, + self.student_pop.get_value( + jax.lax.stop_gradient(train_state.params), + obs, + carry, + ) + ) + + # PPOAgent vmaps over the train state and batch. Batch must be N x EM + rng, subrng = jax.random.split(rng) + train_state, update_stats = self.student_pop.update( + subrng, train_state, train_batch) + + # Collect env metrics + if self.track_env_metrics: + env_metrics = self.benv.get_env_metrics(rollout_start_state) + else: + env_metrics = None + + stats = self._compile_stats(update_stats, ep_stats, env_metrics) + stats.update(dict(n_updates=train_state.n_updates[0])) + + train_state = train_state.increment() + self.n_updates += 1 + + return ( + stats, + rng, + train_state, + state, + start_state, + obs, + carry, + extra, + ep_stats + ) diff --git a/src/minimax/runners/eval_runner.py b/src/minimax/runners/eval_runner.py new file mode 100644 index 0000000..7b53ded --- /dev/null +++ b/src/minimax/runners/eval_runner.py @@ -0,0 +1,325 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial +from typing import Tuple, Optional + +import numpy as np +import jax +import jax.numpy as jnp + +import minimax.envs as envs +from minimax.util.rl import ( + AgentPop, + RollingStats +) +import minimax.util.pytree as _tree_util + + +def generate_all_kwargs_combos(arg_choices): + def update_kwargs_with_choices(prev_combos, k, choices): + updated_combos = [] + for v in choices: + for p in prev_combos: + updated = p.copy() + updated[k] = v + updated_combos.append(updated) + + return updated_combos + + all_combos = [{}] + for k, choices in arg_choices.items(): + all_combos = update_kwargs_with_choices(all_combos, k, choices) + + return all_combos + + +def create_envs_for_kwargs(env_names, kwargs): + # Check for csv kwargs + arg_choices = {} + varied_args = [] + for k, v in kwargs.items(): + if isinstance(v, str) and ',' in v: + vs = eval(v) + arg_choices[k] = vs + varied_args.append(k) + elif isinstance(v, str): + arg_choices[k] = [eval(v)] + else: + arg_choices[k] = [v] + + # List of kwargs + kwargs_combos = generate_all_kwargs_combos(arg_choices) + + env_infos = [] + incl_ext = len(varied_args) > 0 + for name in env_names: + for kwargs in kwargs_combos: + if incl_ext and len(kwargs) > 0: + ext = ':'.join([f'{k}={kwargs[k]}' for k in varied_args]) + ext_name = f'{name}:{ext}' + else: + ext_name = name + env_infos.append( + (name, ext_name, kwargs) + ) + + return env_infos + + +class EvalRunner: + def __init__( + self, + pop, + env_names, + env_kwargs=None, + n_episodes=10, + agent_idxs='*', + render_mode=None): + + self.pop = pop + + if isinstance(agent_idxs, str): + if "*" in agent_idxs: + self.agent_idxs = np.arange(pop.n_agents) + else: + self.agent_idxs = \ + np.array([int(x) for x in agent_idxs.split(',')]) + else: + self.agent_idxs = agent_idxs # assume array + + assert np.max(self.agent_idxs) < pop.n_agents, \ + 'Agent index is out of bounds.' + + if isinstance(env_names, str): + env_names = [ + x.strip() for x in env_names.split(',') + ] + + self.n_episodes = n_episodes + env_infos = create_envs_for_kwargs(env_names, env_kwargs) + env_names = [] + self.ext_env_names = [] + env_kwargs = [] + for (name, ext_name, kwargs) in env_infos: + env_names.append(name) + self.ext_env_names.append(ext_name) + env_kwargs.append(kwargs) + self.n_envs = len(env_names) + + self.benvs = [] + self.env_params = [] + self.env_has_solved_rate = [] + for env_name, kwargs in zip(env_names, env_kwargs): + benv = envs.BatchEnv( + env_name=env_name, + n_parallel=n_episodes, + n_eval=1, + env_kwargs=kwargs, + wrappers=['monitor_return', 'monitor_ep_metrics'] + ) + self.benvs.append(benv) + self.env_params.append(benv.env.params) + self.env_has_solved_rate.append( + benv.env.eval_solved_rate is not None) + + self.action_dtype = self.benvs[0].env.action_space().dtype + + monitored_metrics = self.benvs[0].env.get_monitored_metrics() + self.rolling_stats = RollingStats(names=monitored_metrics, window=1) + self._update_ep_stats = jax.vmap( + jax.vmap( + self.rolling_stats.update_stats, in_axes=(0, 0, 0, None)), + in_axes=(0, 0, 0, None)) + + self.test_return_pre = 'test_return' + self.test_solved_rate_pre = 'test_solved_rate' + + self.render_mode = render_mode + if render_mode: + from minimax.envs.viz.grid_viz import GridVisualizer + self.viz = GridVisualizer() + self.viz.show() + + if render_mode == 'ipython': + from IPython import display + self.ipython_display = display + + def load_checkpoint_state(self, runner_state, state): + runner_state = list(runner_state) + runner_state[1] = runner_state[1].load_state_dict(state[1]) + + return tuple(runner_state) + + @partial(jax.jit, static_argnums=(0, 2)) + def _get_transition( + self, + rng, + benv, + params, + state, + obs, + carry, + zero_carry, + extra): + value, pi_params, next_carry = self.pop.act(params, obs, carry) + pi = self.pop.get_action_dist(pi_params, dtype=self.action_dtype) + rng, subrng = jax.random.split(rng) + action = pi.sample(seed=subrng) + log_pi = pi.log_prob(action) + + rng, *vrngs = jax.random.split(rng, self.pop.n_agents+1) + + step_args = (jnp.array(vrngs), state, action, extra) + (next_obs, + next_state, + reward, + done, + info, + extra) = benv.step(*step_args) + + # Add transition to storage + step = (obs, action, reward, done, log_pi, value) + if carry is not None: + step += (carry,) + + # Zero carry if needed + if carry is not None: + next_carry = jax.vmap(_tree_util.pytree_select)( + done, zero_carry, next_carry) + + if self.render_mode: + self.viz.render( + benv.env.params, + jax.tree_util.tree_map(lambda x: x[0][0], state)) + if self.render_mode == 'ipython': + self.ipython_display.display(self.viz.window.fig) + self.ipython_display.clear_output(wait=True) + + return next_state, next_obs, next_carry, done, info, extra + + @partial(jax.jit, static_argnums=(0, 2)) + def _rollout_benv( + self, + rng, + benv, + params, + env_params, + state, + obs, + carry, + zero_carry, + extra, + ep_stats): + + def _scan_rollout(scan_carry, rng): + (state, + obs, + carry, + extra, + ep_stats) = scan_carry + + step = \ + self._get_transition( + rng, + benv, + params, + state, + obs, + carry, + zero_carry, + extra) + + (next_state, + next_obs, + next_carry, + done, + info, + extra) = step + + ep_stats = self._update_ep_stats(ep_stats, done, info, 1) + + return (next_state, next_obs, next_carry, extra, ep_stats), None + + n_steps = benv.env.max_episode_steps() + rngs = jax.random.split(rng, n_steps) + (state, + obs, + carry, + extra, + ep_stats), _ = jax.lax.scan( + _scan_rollout, + (state, obs, carry, extra, ep_stats), + rngs, + length=n_steps) + + return ep_stats + + @partial(jax.jit, static_argnums=(0,)) + def run(self, rng, params): + """ + Rollout agents on each env. + + For each env, run n_eval episodes in parallel, + where each is indexed to return in order. + """ + eval_stats = self.fake_run(rng, params) + rng, *rollout_rngs = jax.random.split(rng, self.n_envs+1) + for i, (benv, env_param) in enumerate(zip(self.benvs, self.env_params)): + rng, *reset_rngs = jax.random.split(rng, self.pop.n_agents+1) + obs, state, extra = benv.reset(jnp.array(reset_rngs)) + + if self.pop.agent.is_recurrent: + rng, subrng = jax.random.split(rng) + zero_carry = self.pop.init_carry(subrng, obs) + else: + zero_carry = None + + # Reset episodic stats + ep_stats = self.rolling_stats.reset_stats( + batch_shape=(self.pop.n_agents, self.n_episodes)) + + ep_stats = self._rollout_benv( + rollout_rngs[i], + benv, + jax.lax.stop_gradient(params), + env_param, + state, + obs, + zero_carry, + zero_carry, + extra, + ep_stats) + + env_name = self.ext_env_names[i] + mean_return = ep_stats['return'].mean(1) + + if self.env_has_solved_rate[i]: + mean_solved_rate = jax.vmap( + jax.vmap(benv.env.eval_solved_rate))(ep_stats).mean(1) + + for idx in self.agent_idxs: + eval_stats[f'eval/a{idx}:{self.test_return_pre}:{env_name}'] = mean_return[idx].squeeze() + if self.env_has_solved_rate[i]: + eval_stats[f'eval/a{idx}:{self.test_solved_rate_pre}:{env_name}'] = mean_solved_rate[idx].squeeze() + + return eval_stats + + def fake_run(self, rng, params): + eval_stats = {} + for i, env_name in enumerate(self.ext_env_names): + for idx in self.agent_idxs: + eval_stats.update({ + f'eval/a{idx}:{self.test_return_pre}:{env_name}': 0. + }) + if self.env_has_solved_rate[i]: + eval_stats.update({ + f'eval/a{idx}:{self.test_solved_rate_pre}:{env_name}': 0., + }) + + return eval_stats diff --git a/src/minimax/runners/paired_runner.py b/src/minimax/runners/paired_runner.py new file mode 100644 index 0000000..5b73fd3 --- /dev/null +++ b/src/minimax/runners/paired_runner.py @@ -0,0 +1,604 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from enum import Enum +from functools import partial +from typing import Tuple, Optional +import inspect + +import numpy as np +import jax +import jax.numpy as jnp +from jax.sharding import PartitionSpec as P +import optax +import flax +import flax.linen as nn +from flax.core.frozen_dict import FrozenDict + +import minimax.envs as envs +from minimax.util import pytree as _tree_util +from minimax.util.rl import ( + AgentPop, + VmapTrainState, + RolloutStorage, + RollingStats, + UEDScore, + compute_ued_scores +) + + +class PAIREDRunner: + """ + Orchestrates rollouts across one or more students and teachers. + The main components at play: + - AgentPop: Manages train state and batched inference logic + for a population of agents. + - BatchUEDEnv: Manages environment step and reset logic for a + population of agents batched over a pair of student and + teacher MDPs. + - RolloutStorage: Manages the storing and sampling of collected txns. + - PPO: Handles PPO updates, which take a train state + batch of txns. + """ + + def __init__( + self, + env_name, + env_kwargs, + ued_env_kwargs, + student_agents, + n_students=2, + n_parallel=1, + n_eval=1, + n_rollout_steps=250, + lr=1e-4, + lr_final=None, + lr_anneal_steps=0, + max_grad_norm=0.5, + discount=0.99, + gae_lambda=0.95, + adam_eps=1e-5, + teacher_lr=None, + teacher_lr_final=None, + teacher_lr_anneal_steps=None, + teacher_discount=0.99, + teacher_gae_lambda=0.95, + teacher_agents=None, + ued_score='relative_regret', + track_env_metrics=False, + n_unroll_rollout=1, + render=False, + n_devices=1): + assert n_parallel % n_devices == 0, 'Num envs must be divisible by num devices.' + + ued_score = UEDScore[ued_score.upper()] + + assert len(student_agents) == 1, \ + 'Only one type of student supported.' + assert not (n_students > 2 and ued_score in [UEDScore.RELATIVE_REGRET, UEDScore.MEAN_RELATIVE_REGRET]), \ + 'Standard PAIRED uses only 2 students.' + assert teacher_agents is None or len(teacher_agents) == 1, \ + 'Only one type of teacher supported.' + + self.n_students = n_students + self.n_parallel = n_parallel // n_devices + self.n_eval = n_eval + self.n_devices = n_devices + self.step_batch_size = n_students*n_eval*n_parallel + self.n_rollout_steps = n_rollout_steps + self.n_updates = 0 + self.lr = lr + self.lr_final = lr if lr_final is None else lr_final + self.lr_anneal_steps = lr_anneal_steps + self.teacher_lr = \ + lr if teacher_lr is None else lr + self.teacher_lr_final = \ + self.lr_final if teacher_lr_final is None else teacher_lr_final + self.teacher_lr_anneal_steps = \ + lr_anneal_steps if teacher_lr_anneal_steps is None else teacher_lr_anneal_steps + self.max_grad_norm = max_grad_norm + self.adam_eps = adam_eps + self.ued_score = ued_score + self.track_env_metrics = track_env_metrics + + self.n_unroll_rollout = n_unroll_rollout + self.render = render + + self.student_pop = AgentPop(student_agents[0], n_agents=n_students) + + if teacher_agents is not None: + self.teacher_pop = AgentPop(teacher_agents[0], n_agents=1) + + # This ensures correct partial-episodic bootstrapping by avoiding + # any termination purely due to timeouts. + # env_kwargs.max_episode_steps = self.n_rollout_steps + 1 + self.benv = envs.BatchUEDEnv( + env_name=env_name, + n_parallel=self.n_parallel, + n_eval=n_eval, + env_kwargs=env_kwargs, + ued_env_kwargs=ued_env_kwargs, + wrappers=['monitor_return', 'monitor_ep_metrics'], + ued_wrappers=[] + ) + self.teacher_n_rollout_steps = \ + self.benv.env.ued_max_episode_steps() + + self.student_rollout = RolloutStorage( + discount=discount, + gae_lambda=gae_lambda, + n_steps=n_rollout_steps, + n_agents=n_students, + n_envs=self.n_parallel, + n_eval=self.n_eval, + action_space=self.benv.env.action_space(), + obs_space=self.benv.env.observation_space(), + agent=self.student_pop.agent + ) + + self.teacher_rollout = RolloutStorage( + discount=teacher_discount, + gae_lambda=teacher_gae_lambda, + n_steps=self.teacher_n_rollout_steps, + n_agents=1, + n_envs=self.n_parallel, + n_eval=1, + action_space=self.benv.env.ued_action_space(), + obs_space=self.benv.env.ued_observation_space(), + agent=self.teacher_pop.agent, + ) + + ued_monitored_metrics = ('return',) + self.ued_rolling_stats = RollingStats( + names=ued_monitored_metrics, + window=10, + ) + + monitored_metrics = self.benv.env.get_monitored_metrics() + self.rolling_stats = RollingStats( + names=monitored_metrics, + window=10, + ) + + self._update_ep_stats = jax.vmap( + jax.vmap(self.rolling_stats.update_stats)) + self._update_ued_ep_stats = jax.vmap( + jax.vmap(self.ued_rolling_stats.update_stats)) + + if self.render: + from envs.viz.grid_viz import GridVisualizer + self.viz = GridVisualizer() + self.viz.show() + + def reset(self, rng): + self.n_updates = 0 + + n_parallel = self.n_parallel*self.n_devices + + rng, student_rng, teacher_rng = jax.random.split(rng, 3) + student_info = self._reset_pop( + student_rng, + self.student_pop, + partial(self.benv.reset, sub_batch_size=n_parallel*self.n_eval), + n_parallel_ep=n_parallel*self.n_eval, + lr_init=self.lr, + lr_final=self.lr_final, + lr_anneal_steps=self.lr_anneal_steps) + + teacher_info = self._reset_pop( + teacher_rng, + self.teacher_pop, + partial(self.benv.reset_teacher, n_parallel=n_parallel), + n_parallel_ep=n_parallel, + lr_init=self.teacher_lr, + lr_final=self.teacher_lr_final, + lr_anneal_steps=self.teacher_lr_anneal_steps) + + return ( + rng, + *student_info, + *teacher_info + ) + + def _reset_pop( + self, + rng, + pop, + env_reset_fn, + n_parallel_ep=1, + lr_init=3e-4, + lr_final=3e-4, + lr_anneal_steps=0): + rng, *vrngs = jax.random.split(rng, pop.n_agents+1) + reset_out = env_reset_fn(jnp.array(vrngs)) + if len(reset_out) == 2: + obs, state = reset_out + else: + obs, state, extra = reset_out + dummy_obs = jax.tree_util.tree_map( + lambda x: x[0], obs) # for one agent only + + rng, subrng = jax.random.split(rng) + if pop.agent.is_recurrent: + carry = pop.init_carry(subrng, obs) + else: + carry = None + + rng, subrng = jax.random.split(rng) + params = pop.init_params(subrng, dummy_obs) + + schedule_fn = optax.linear_schedule( + init_value=-float(lr_init), + end_value=-float(lr_final), + transition_steps=lr_anneal_steps, + ) + + tx = optax.chain( + optax.clip_by_global_norm(self.max_grad_norm), + optax.scale_by_adam(eps=self.adam_eps), + optax.scale_by_schedule(schedule_fn), + ) + + train_state = VmapTrainState.create( + apply_fn=pop.agent.evaluate, + params=params, + tx=tx + ) + + ep_stats = self.rolling_stats.reset_stats( + batch_shape=(pop.n_agents, n_parallel_ep)) + + return train_state, state, obs, carry, ep_stats + + def get_checkpoint_state(self, state): + _state = list(state) + _state[1] = state[1].state_dict + _state[6] = state[6].state_dict + + return _state + + def load_checkpoint_state(self, runner_state, state): + runner_state = list(runner_state) + runner_state[1] = runner_state[1].load_state_dict(state[1]) + runner_state[6] = runner_state[6].load_state_dict(state[6]) + + return tuple(runner_state) + + @partial(jax.jit, static_argnums=(0, 2, 3)) + def _get_transition( + self, + rng, + pop, + rollout_mgr, + rollout, + params, + state, + obs, + carry, + done, + reset_state=None, + extra=None): + # Sample action + value, pi_params, next_carry = pop.act(params, obs, carry, done) + pi = pop.get_action_dist(pi_params) + rng, subrng = jax.random.split(rng) + action = pi.sample(seed=subrng) + log_pi = pi.log_prob(action) + + rng, *vrngs = jax.random.split(rng, pop.n_agents+1) + + if pop is self.student_pop: + step_fn = self.benv.step_student + else: + step_fn = self.benv.step_teacher + step_args = (jnp.array(vrngs), state, action) + + if reset_state is not None: # Needed for student to reset to same instance + step_args += (reset_state,) + + if extra is not None: + step_args += (extra,) + next_obs, next_state, reward, done, info, extra = step_fn( + *step_args) + else: + next_obs, next_state, reward, done, info = step_fn(*step_args) + + # Add transition to storage + step = (obs, action, reward, done, log_pi, value) + if carry is not None: + step += (carry,) + + rollout = rollout_mgr.append(rollout, *step) + + if self.render and pop is self.student_pop: + self.viz.render( + self.benv.env.env.params, + jax.tree_util.tree_map(lambda x: x[0][0], state)) + + return rollout, next_state, next_obs, next_carry, done, info, extra + + @partial(jax.jit, static_argnums=(0, 2, 3, 4)) + def _rollout( + self, + rng, + pop, + rollout_mgr, + n_steps, + params, + state, + obs, + carry, + done, + reset_state=None, + extra=None, + ep_stats=None): + rngs = jax.random.split(rng, n_steps) + + rollout = rollout_mgr.reset() + + def _scan_rollout(scan_carry, rng): + (rollout, + state, + obs, + carry, + done, + extra, + ep_stats) = scan_carry + + next_scan_carry = \ + self._get_transition( + rng, + pop, + rollout_mgr, + rollout, + params, + state, + obs, + carry, + done, + reset_state, + extra) + + (rollout, + next_state, + next_obs, + next_carry, + done, + info, + extra) = next_scan_carry + + if ep_stats is not None: + _ep_stats_update_fn = self._update_ep_stats \ + if pop is self.student_pop else self._update_ued_ep_stats + + ep_stats = _ep_stats_update_fn(ep_stats, done, info) + + return (rollout, next_state, next_obs, next_carry, done, extra, ep_stats), None + + (rollout, state, obs, carry, done, extra, ep_stats), _ = jax.lax.scan( + _scan_rollout, + (rollout, state, obs, carry, done, extra, ep_stats), + rngs, + length=n_steps, + unroll=self.n_unroll_rollout + ) + + return rollout, state, obs, carry, extra, ep_stats + + @partial(jax.jit, static_argnums=(0,)) + def _compile_stats(self, + update_stats, ep_stats, + ued_update_stats, ued_ep_stats, + env_metrics=None, + grad_stats=None, ued_grad_stats=None): + mean_returns_by_student = jax.vmap( + lambda x: x.mean())(ep_stats['return']) + mean_returns_by_teacher = jax.vmap( + lambda x: x.mean())(ued_ep_stats['return']) + + mean_ep_stats = jax.vmap(lambda info: jax.tree_map(lambda x: x.mean(), info))( + {k: ep_stats[k] for k in self.rolling_stats.names} + ) + ued_mean_ep_stats = jax.vmap(lambda info: jax.tree_map(lambda x: x.mean(), info))( + {k: ued_ep_stats[k] for k in self.ued_rolling_stats.names} + ) + + student_stats = { + f'mean_{k}': v for k, v in mean_ep_stats.items() + } + student_stats.update(update_stats) + + stats = {} + for i in range(self.n_students): + _student_stats = jax.tree_util.tree_map( + lambda x: x[i], student_stats) # for agent0 + stats.update({f'{k}_a{i}': v for k, v in _student_stats.items()}) + + teacher_stats = { + f'mean_{k}_tch': v for k, v in ued_mean_ep_stats.items() + } + teacher_stats.update({ + f'{k}_tch': v[0] for k, v in ued_update_stats.items() + }) + stats.update(teacher_stats) + + if self.track_env_metrics: + passable_mask = env_metrics.pop('passable') + mean_env_metrics = jax.tree_util.tree_map( + lambda x: (x*passable_mask).sum()/passable_mask.sum(), + env_metrics + ) + mean_env_metrics.update({'passable_ratio': passable_mask.mean()}) + stats.update({ + f'env/{k}': v for k, v in mean_env_metrics.items() + }) + + if self.n_devices > 1: + stats = jax.tree_map(lambda x: jax.lax.pmean(x, 'device'), stats) + + return stats + + def get_shmap_spec(self): + runner_state_size = len(inspect.signature(self.run).parameters) + in_spec = [P(None, 'device'),]*(runner_state_size) + out_spec = [P(None, 'device'),]*(runner_state_size) + + in_spec[:2] = [P(None),]*2 + in_spec[6] = P(None) + in_spec = tuple(in_spec) + out_spec = (P(None),) + in_spec + + return in_spec, out_spec + + @partial(jax.jit, static_argnums=(0,)) + def run( + self, + rng, + train_state, + state, + obs, + carry, + ep_stats, + ued_train_state, + ued_state, + ued_obs, + ued_carry, + ued_ep_stats): + """ + Perform one update step: rollout teacher + students + """ + if self.n_devices > 1: + rng = jax.random.fold_in(rng, jax.lax.axis_index('device')) + + # === Reset teacher env + rollout teacher + rng, *vrngs = jax.random.split(rng, self.teacher_pop.n_agents+1) + ued_reset_out = self.benv.reset_teacher(jnp.array(vrngs)) + if len(ued_reset_out) > 2: + ued_obs, ued_state, ued_extra = ued_reset_out + else: + ued_obs, ued_state = ued_reset_out + ued_extra = None + + # Reset UED ep_stats + if self.ued_rolling_stats is not None: + ued_ep_stats = self.ued_rolling_stats.reset_stats( + batch_shape=(1, self.n_parallel)) + else: + ued_ep_stats = None + + tch_rollout_batch_shape = (1, self.n_parallel*self.n_eval) + done = jnp.zeros(tch_rollout_batch_shape, dtype=jnp.bool_) + rng, subrng = jax.random.split(rng) + ued_rollout, ued_state, ued_obs, ued_carry, _, ued_ep_stats = \ + self._rollout( + subrng, + self.teacher_pop, + self.teacher_rollout, + self.teacher_n_rollout_steps, + jax.lax.stop_gradient(ued_train_state.params), + ued_state, + ued_obs, + ued_carry, + done, + extra=ued_extra, + ep_stats=ued_ep_stats + ) + + # === Reset student to new envs + rollout students + rng, *vrngs = jax.random.split(rng, self.teacher_pop.n_agents+1) + obs, state, extra = jax.tree_util.tree_map( + lambda x: x.squeeze(0), self.benv.reset_student( + jnp.array(vrngs), + ued_state, + self.student_pop.n_agents)) + reset_state = state + + # Reset student ep_stats + st_rollout_batch_shape = (self.n_students, self.n_parallel*self.n_eval) + ep_stats = self.rolling_stats.reset_stats( + batch_shape=st_rollout_batch_shape) + + done = jnp.zeros(st_rollout_batch_shape, dtype=jnp.bool_) + rng, subrng = jax.random.split(rng) + rollout, state, obs, carry, extra, ep_stats = \ + self._rollout( + subrng, + self.student_pop, + self.student_rollout, + self.n_rollout_steps, + jax.lax.stop_gradient(train_state.params), + state, + obs, + carry, + done, + reset_state=reset_state, + extra=extra, + ep_stats=ep_stats) + + # === Update student with PPO + # PPOAgent vmaps over the train state and batch. Batch must be N x EM + student_rollout_last_value = self.student_pop.get_value( + jax.lax.stop_gradient(train_state.params), obs, carry + ) + train_batch = self.student_rollout.get_batch( + rollout, + student_rollout_last_value + ) + + rng, subrng = jax.random.split(rng) + train_state, update_stats = self.student_pop.update( + subrng, train_state, train_batch) + + # === Update teacher with PPO + # - Compute returns per env per agent + # - Compute batched returns based on returns per env per agent + ued_score, _ = compute_ued_scores( + self.ued_score, train_batch, self.n_eval) + ued_rollout = self.teacher_rollout.set_final_reward( + ued_rollout, ued_score) + ued_train_batch = self.teacher_rollout.get_batch( + ued_rollout, + jnp.zeros((1, self.n_parallel)) # Last step terminates episode + ) + + ued_ep_stats = self._update_ued_ep_stats( + ued_ep_stats, + jnp.ones((1, len(ued_score), 1), dtype=jnp.bool_), + {'return': jnp.expand_dims(ued_score, (0, -1))} + ) + + # Update teacher, batch must be 1 x Ex1 + rng, subrng = jax.random.split(rng) + ued_train_state, ued_update_stats = self.teacher_pop.update( + subrng, ued_train_state, ued_train_batch) + + # -------------------------------------------------- + # Collect metrics + if self.track_env_metrics: + env_metrics = self.benv.get_env_metrics(reset_state) + else: + env_metrics = None + + grad_stats, ued_grad_stats = None, None + + stats = self._compile_stats( + update_stats, ep_stats, + ued_update_stats, ued_ep_stats, + env_metrics, + grad_stats, ued_grad_stats) + stats.update(dict(n_updates=train_state.n_updates[0])) + + train_state = train_state.increment() + ued_train_state = ued_train_state.increment() + self.n_updates += 1 + + return ( + stats, + rng, + train_state, state, obs, carry, ep_stats, + ued_train_state, ued_state, ued_obs, ued_carry, ued_ep_stats + ) diff --git a/src/minimax/runners/plr_runner.py b/src/minimax/runners/plr_runner.py new file mode 100644 index 0000000..14a98f2 --- /dev/null +++ b/src/minimax/runners/plr_runner.py @@ -0,0 +1,549 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial +from enum import Enum +from typing import Tuple, Optional + +import numpy as np +import jax +import jax.numpy as jnp +from jax.sharding import PartitionSpec as P +import optax +import flax +import flax.linen as nn +from flax.core.frozen_dict import FrozenDict + +import minimax.envs as envs +from minimax.runners.dr_runner import DRRunner +from minimax.util import pytree as _tree_util +from minimax.util.rl import ( + AgentPop, + VmapTrainState, + RolloutStorage, + RollingStats, + UEDScore, + compute_ued_scores, + PopPLRManager +) + + +class MutationCriterion(Enum): + BATCH = 0 + EASY = 1 + HARD = 2 + + +class PLRRunner(DRRunner): + def __init__( + self, + *, + replay_prob=0.5, + buffer_size=100, + staleness_coef=0.3, + use_score_ranks=True, + temp=1.0, + min_fill_ratio=0.5, + use_robust_plr=False, + use_parallel_eval=False, + ued_score='l1_value_loss', + force_unique=False, # Slower if True + mutation_fn=None, + n_mutations=0, + mutation_criterion='batch', + mutation_subsample_size=1, + **kwargs): + use_mutations = mutation_fn is not None + if use_parallel_eval: + replay_prob = 1.0 # Replay every rollout cycle + # Force batch mutations (no UED scores) + mutation_criterion = 'batch' + self._n_parallel_batches = 3 if use_mutations else 2 + kwargs['n_parallel'] *= self._n_parallel_batches + + super().__init__(**kwargs) + + self.replay_prob = replay_prob + self.buffer_size = buffer_size + self.staleness_coef = staleness_coef + self.temp = temp + self.use_score_ranks = use_score_ranks + self.min_fill_ratio = min_fill_ratio + self.use_robust_plr = use_robust_plr + self.use_parallel_eval = use_parallel_eval + self.ued_score = UEDScore[ued_score.upper()] + + self.use_mutations = use_mutations + if self.use_mutations: + self.mutation_fn = envs.get_mutator( + self.benv.env_name, mutation_fn) + else: + self.mutation_fn = None + self.n_mutations = n_mutations + self.mutation_criterion = MutationCriterion[mutation_criterion.upper()] + self.mutation_subsample_size = mutation_subsample_size + + self.force_unique = force_unique + if force_unique: + self.comparator_fn = envs.get_comparator(self.benv.env_name) + else: + self.comparator_fn = None + + if mutation_fn is not None and mutation_criterion != 'batch': + assert self.n_parallel % self.mutation_subsample_size == 0, \ + 'Number of parallel envs must be divisible by mutation subsample size.' + + def reset(self, rng): + runner_state = list(super().reset(rng)) + rng = runner_state[0] + runner_state[0], subrng = jax.random.split(rng) + example_state = self.benv.env.reset(rng)[1] + + self.plr_mgr = PopPLRManager( + n_agents=self.n_students, + example_level=example_state, + ued_score=self.ued_score, + replay_prob=self.replay_prob, + buffer_size=self.buffer_size, + staleness_coef=self.staleness_coef, + temp=self.temp, + use_score_ranks=self.use_score_ranks, + min_fill_ratio=self.min_fill_ratio, + use_robust_plr=self.use_robust_plr, + use_parallel_eval=self.use_parallel_eval, + comparator_fn=self.comparator_fn, + n_devices=self.n_devices + ) + plr_buffer = self.plr_mgr.reset(self.n_students) + + train_state = runner_state[1] + train_state = train_state.replace(plr_buffer=plr_buffer) + if self.n_devices == 1: + runner_state[1] = train_state + else: + plr_buffer = jax.tree_map(lambda x: x.repeat( + self.n_devices, 1), plr_buffer) # replicate plr buffer + # Return PLR buffer directly to make shmap easier + runner_state += (plr_buffer,) + + self.dummy_eval_output = self._create_dummy_eval_output(train_state) + + return tuple(runner_state) + + def _create_dummy_eval_output(self, train_state): + rng, * \ + vrngs = jax.random.split(jax.random.PRNGKey(0), self.n_students+1) + obs, state, extra = self.benv.reset(jnp.array(vrngs)) + + ep_stats = self.rolling_stats.reset_stats( + batch_shape=(self.n_students, self.n_parallel*self.n_eval)) + + ued_scores = jnp.zeros((self.n_students, self.n_parallel)) + + if self.student_pop.agent.is_recurrent: + carry = self.zero_carry + else: + carry = None + rollout = self.student_rollout.reset() + + value, _ = self.student_pop.get_value( + jax.lax.stop_gradient(train_state.params), + obs, + carry, + ) + train_batch = self.student_rollout.get_batch( + rollout, value + ) + + return ( + rng, + train_state, + state, + state, + obs, + carry, + extra, + ep_stats, + state, + train_batch, + ued_scores + ) + + @partial(jax.jit, static_argnums=(0, 8)) + def _eval_and_update_plr( + self, + rng, + levels, + level_idxs, + train_state, + update_plr, + parent_idxs=None, + dupe_mask=None, + fake=False): + # Collect rollout and optionally update plr buffer + # Returns train_batch and ued_scores + if fake: + dummy_eval_output = list(self.dummy_eval_output) + dummy_eval_output[1] = train_state + return tuple(dummy_eval_output) + + rollout_batch_shape = (self.n_students, self.n_parallel*self.n_eval) + obs, state, extra = self.benv.set_state(levels) + ep_stats = self.rolling_stats.reset_stats( + batch_shape=rollout_batch_shape) + + rollout_start_state = state + + done = jnp.zeros(rollout_batch_shape, dtype=jnp.bool_) + if self.student_pop.agent.is_recurrent: + carry = self.zero_carry + else: + carry = None + + rng, subrng = jax.random.split(rng) + start_state = state + rollout, state, start_state, obs, carry, extra, ep_stats, train_state = \ + self._rollout_students( + subrng, + train_state, + state, + start_state, + obs, + carry, + done, + extra, + ep_stats + ) + + value, _ = self.student_pop.get_value( + jax.lax.stop_gradient(train_state.params), + obs, + carry + ) + train_batch = self.student_rollout.get_batch(rollout, value) + + # Update PLR buffer + if self.ued_score == UEDScore.MAX_MC: + max_returns = jax.vmap(lambda x, y: x.at[y].get())( + train_state.plr_buffer.max_returns, level_idxs) + max_returns = jnp.where( + jnp.greater_equal(level_idxs, 0), + max_returns, + jnp.full_like(max_returns, -jnp.inf) + ) + ued_info = {'max_returns': max_returns} + else: + ued_info = None + ued_scores, ued_score_info = compute_ued_scores( + self.ued_score, train_batch, self.n_eval, info=ued_info, ignore_val=-jnp.inf, per_agent=True) + next_plr_buffer = self.plr_mgr.update( + train_state.plr_buffer, + levels=levels, + level_idxs=level_idxs, + ued_scores=ued_scores, + dupe_mask=dupe_mask, + info=ued_score_info, + ignore_val=-jnp.inf, + parent_idxs=parent_idxs) + + next_plr_buffer = jax.vmap( + lambda update, new, prev: jax.tree_map( + lambda x, y: jax.lax.select(update, x, y), new, prev) + )(update_plr, next_plr_buffer, train_state.plr_buffer) + + train_state = train_state.replace(plr_buffer=next_plr_buffer) + + return ( + rng, + train_state, + state, + start_state, + obs, + carry, + extra, + ep_stats, + rollout_start_state, + train_batch, + ued_scores, + ) + + @partial(jax.jit, static_argnums=(0,)) + def _mutate_levels(self, rng, levels, level_idxs, ued_scores=None): + if not self.use_mutations: + return levels, level_idxs, jnp.full_like(level_idxs, -1) + + def upsample_levels(levels, level_idxs, subsample_idxs): + subsample_idxs = subsample_idxs.repeat( + self.n_parallel//self.mutation_subsample_size, -1) + parent_idxs = level_idxs.take(subsample_idxs) + levels = jax.vmap( + lambda x, y: jax.tree_map( + lambda _x: jnp.array(_x).take(y, 0), x) + )(levels, parent_idxs) + + return levels, parent_idxs + + if self.mutation_criterion == MutationCriterion.BATCH: + parent_idxs = level_idxs + + if self.mutation_criterion == MutationCriterion.EASY: + _, top_level_idxs = jax.lax.approx_min_k( + ued_scores, self.mutation_subsample_size) + levels, parent_idxs = upsample_levels( + levels, level_idxs, top_level_idxs) + + elif self.mutation_criterion == MutationCriterion.HARD: + _, top_level_idxs = jax.lax.approx_max_k( + ued_scores, self.mutation_subsample_size) + levels, parent_idxs = upsample_levels( + levels, level_idxs, top_level_idxs) + + n_parallel = level_idxs.shape[-1] + vrngs = jax.vmap(lambda subrng: jax.random.split(subrng, n_parallel))( + jax.random.split(rng, self.n_students) + ) + + mutated_levels = jax.vmap( + lambda *args: jax.vmap(self.mutation_fn, + in_axes=(0, None, 0, None))(*args), + in_axes=(0, None, 0, None) + )(vrngs, self.benv.env_params, levels, self.n_mutations) + + # Mutated levels do not have existing idxs in the PLR buffer. + mutated_level_idxs = jnp.full((self.n_students, n_parallel), -1) + + return mutated_levels, mutated_level_idxs, parent_idxs + + def _efficient_grad_update(self, rng, train_state, train_batch, is_replay): + # PPOAgent vmaps over the train state and batch. Batch must be N x EM + skip_grad_update = jnp.logical_and(self.use_robust_plr, ~is_replay) + + if self.n_students == 1: + train_state, stats = jax.lax.cond( + skip_grad_update[0], + partial(self.student_pop.update, fake=True), + self.student_pop.update, + *(rng, train_state, train_batch) + ) + elif self.n_students > 1: # Have to vmap all students + take only students that need updates + _, dummy_stats = jax.vmap( + lambda *_: self.student_pop.agent.get_empty_update_stats())(np.arange(self.n_students)) + _train_state, stats = self.student.update( + rng, train_state, train_batch) + train_state, stats = jax.vmap(lambda cond, x, y: + jax.tree_map(lambda _cond, _x, _y: jax.lax.select(_cond, _x, _y), cond, x, y))( + is_replay, (train_state, stats), (_train_state, dummy_stats) + ) + + return train_state, stats + + @partial(jax.jit, static_argnums=(0,)) + def _compile_stats(self, update_stats, ep_stats, env_metrics=None, plr_stats=None): + stats = super()._compile_stats(update_stats, ep_stats, env_metrics) + + if plr_stats is not None: + plr_stats = jax.vmap(lambda info: jax.tree_map( + lambda x: x.mean(), info))(plr_stats) + + if self.n_students > 1: + _plr_stats = {} + for i in range(self.n_students): + _student_plr_stats = jax.tree_util.tree_map( + lambda x: x[i], plr_stats) # for agent0 + _plr_stats.update( + {f'{k}_a{i}': v for k, v in _student_plr_stats.items()}) + plr_stats = _plr_stats + else: + plr_stats = jax.tree_map(lambda x: x[0], plr_stats) + + stats.update({f'plr_{k}': v for k, v in plr_stats.items()}) + + if self.n_devices > 1: + stats = jax.tree_map(lambda x: jax.lax.pmean(x, 'device'), stats) + + return stats + + @partial(jax.jit, static_argnums=(0,)) + def run( + self, + rng, + train_state, + state, + start_state, + obs, + carry=None, + extra=None, + ep_stats=None, + plr_buffer=None): + # If device sharded, load sharded PLR buffer into train state + if self.n_devices > 1: + rng = jax.random.fold_in(rng, jax.lax.axis_index('device')) + train_state = train_state.replace(plr_buffer=plr_buffer) + + # Sample next training levels via PLR + rng, *vrngs = jax.random.split(rng, self.n_students+1) + obs, state, extra = self.benv.reset( + jnp.array(vrngs), self.n_parallel, 1) + + if self.use_parallel_eval: + n_level_samples = self.n_parallel//self._n_parallel_batches + new_levels = jax.tree_map( + lambda x: x.at[:, n_level_samples:2*n_level_samples].get(), state) + else: + n_level_samples = self.n_parallel + new_levels = state + + rng, subrng = jax.random.split(rng) + levels, level_idxs, is_replay, next_plr_buffer = \ + self.plr_mgr.sample(subrng, train_state.plr_buffer, + new_levels, n_level_samples) + train_state = train_state.replace(plr_buffer=next_plr_buffer) + + # If use_parallel_eval=True, need to combine replay and non-replay levels together + # Need to mutate levels as well + parent_idxs = jnp.full((self.n_students, self.n_parallel), -1) + if self.use_parallel_eval: # Parallel ACCEL + new_level_idxs = jnp.full_like(parent_idxs, -1) + + _all_levels = jax.vmap( + lambda x, y: _tree_util.pytree_merge( + x, y, start_idx=n_level_samples, src_len=n_level_samples), + )(state, levels) + _all_level_idxs = jax.vmap( + lambda x, y: _tree_util.pytree_merge( + x, y, start_idx=n_level_samples, src_len=n_level_samples), + )(new_level_idxs, level_idxs) + + if self.use_mutations: + rng, subrng = jax.random.split(rng) + mutated_levels, mutated_level_idxs, _parent_idxs = self._mutate_levels( + subrng, levels, level_idxs) + + fallback_levels = jax.tree_map( + lambda x: x.at[:, -n_level_samples:].get(), state) + fallback_level_idxs = jnp.full_like(mutated_level_idxs, -1) + + mutated_levels = jax.vmap( + lambda cond, x, y: jax.tree_map( + lambda _x, _y: jax.lax.select(cond, _x, _y), x, y + ))(is_replay, mutated_levels, fallback_levels) + + mutated_level_idxs = jax.vmap( + lambda cond, x, y: jax.tree_map( + lambda _x, _y: jax.lax.select(cond, _x, _y), x, y + ))(is_replay, mutated_level_idxs, fallback_level_idxs) + + _parent_idxs = jax.vmap( + lambda cond, x, y: jax.tree_map( + lambda _x, _y: jax.lax.select(cond, _x, _y), x, y + ))(is_replay, _parent_idxs, fallback_level_idxs) + + mutated_levels_start_idx = 2*n_level_samples + _all_levels = jax.vmap( + lambda x, y: _tree_util.pytree_merge( + x, y, start_idx=mutated_levels_start_idx, src_len=n_level_samples), + )(_all_levels, mutated_levels) + _all_level_idxs = jax.vmap( + lambda x, y: _tree_util.pytree_merge( + x, y, start_idx=mutated_levels_start_idx, src_len=n_level_samples), + )(_all_level_idxs, mutated_level_idxs) + parent_idxs = jax.vmap( + lambda x, y: _tree_util.pytree_merge( + x, y, start_idx=mutated_levels_start_idx, src_len=n_level_samples), + )(parent_idxs, _parent_idxs) + + levels = _all_levels + level_idxs = _all_level_idxs + + # dedupe levels, move into PLR buffer logic + if self.force_unique: + level_idxs, dupe_mask = self.plr_mgr.dedupe_levels( + next_plr_buffer, levels, level_idxs) + else: + dupe_mask = None + + # Evaluate levels + update PLR + result = self._eval_and_update_plr( + rng, levels, level_idxs, train_state, update_plr=jnp.array([True]*self.n_students), parent_idxs=parent_idxs, dupe_mask=dupe_mask) + rng, train_state, state, start_state, obs, carry, extra, ep_stats, \ + rollout_start_state, train_batch, ued_scores = result + + if self.use_parallel_eval: + replay_start_idx = self.n_eval*n_level_samples + replay_end_idx = 2*replay_start_idx + train_batch = jax.vmap( + lambda x: jax.tree_map( + lambda _x: _x.at[:, replay_start_idx:replay_end_idx].get(), x) + )(train_batch) + + # Gradient update + rng, subrng = jax.random.split(rng) + train_state, update_stats = self._efficient_grad_update( + subrng, train_state, train_batch, is_replay) + + # Mutation step + use_mutations = jnp.logical_and(self.use_mutations, is_replay) + # Already mutated above in parallel + use_mutations = jnp.logical_and( + use_mutations, not self.use_parallel_eval) + rng, arng, brng = jax.random.split(rng, 3) + + mutated_levels, mutated_level_idxs, parent_idxs = jax.lax.cond( + use_mutations.any(), + self._mutate_levels, + lambda *_: (levels, level_idxs, jnp.full_like(level_idxs, -1)), + *(arng, levels, level_idxs, ued_scores) + ) + + mutated_dupe_mask = jnp.zeros_like(mutated_level_idxs, dtype=jnp.bool_) + if self.force_unique: # Should move into update plr logic + mutated_level_idxs, mutated_dupe_mask = jax.lax.cond( + use_mutations.any(), + self.plr_mgr.dedupe_levels, + lambda *_: (mutated_level_idxs, mutated_dupe_mask), + *(next_plr_buffer, mutated_levels, mutated_level_idxs) + ) + + mutation_eval_result = jax.lax.cond( + use_mutations.any(), + self._eval_and_update_plr, + partial(self._eval_and_update_plr, fake=True), + *(brng, mutated_levels, mutated_level_idxs, train_state, use_mutations, parent_idxs, mutated_dupe_mask) + ) + train_state = mutation_eval_result[1] + + # Collect training env metrics + if self.track_env_metrics: + env_metrics = self.benv.get_env_metrics(levels) + else: + env_metrics = None + + plr_stats = self.plr_mgr.get_metrics(train_state.plr_buffer) + + stats = self._compile_stats( + update_stats, ep_stats, env_metrics, plr_stats) + + if self.n_devices > 1: + plr_buffer = train_state.plr_buffer + train_state = train_state.replace(plr_buffer=None) + + train_state = train_state.increment() + stats.update(dict(n_updates=train_state.n_updates[0])) + + return ( + stats, + rng, + train_state, + state, + start_state, + obs, + carry, + extra, + ep_stats, + plr_buffer + ) diff --git a/src/minimax/runners/xp_runner.py b/src/minimax/runners/xp_runner.py new file mode 100644 index 0000000..55c23d7 --- /dev/null +++ b/src/minimax/runners/xp_runner.py @@ -0,0 +1,310 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +import copy +from functools import partial +from collections import defaultdict +import time + +import numpy as np +import jax +from jax.sharding import Mesh, PartitionSpec as P +from jax.experimental import mesh_utils +from jax.experimental.shard_map import shard_map + +from .eval_runner import EvalRunner +from .dr_runner import DRRunner +from .paired_runner import PAIREDRunner +from .plr_runner import PLRRunner +from minimax.util.rl import UEDScore, PopPLRManager +import minimax.envs as envs +import minimax.models as models +import minimax.agents as agents + + +class RunnerInfo: + def __init__( + self, + runner_cls, + is_ued=False): + self.runner_cls = runner_cls + self.is_ued = is_ued + + +RUNNER_INFO = { + 'dr': RunnerInfo( + runner_cls=DRRunner, + ), + 'plr': RunnerInfo( + runner_cls=PLRRunner, + ), + 'paired': RunnerInfo( + runner_cls=PAIREDRunner, + is_ued=True + ) +} + + +class ExperimentRunner: + def __init__( + self, + train_runner, + env_name, + agent_rl_algo, + student_model_name, + student_critic_model_name=None, + student_agent_kind="ppo", + teacher_model_name=None, + train_runner_kwargs={}, + env_kwargs={}, + ued_env_kwargs={}, + student_rl_kwargs={}, + teacher_rl_kwargs={}, + student_model_kwargs={}, + teacher_model_kwargs={}, + eval_kwargs={}, + eval_env_kwargs={}, + shaped_reward_steps=None, + n_devices=1, + xpid=None, + ): + self.env_name = env_name + self.agent_rl_algo = agent_rl_algo + self.is_ued = RUNNER_INFO[train_runner].is_ued + self.xpid = xpid + + dummy_env = envs.make( + env_name, + env_kwargs, + ued_env_kwargs)[0] + + # ---- Make agent ---- + student_model_kwargs['output_dim'] = dummy_env.action_space().n + student_model = models.make( + env_name=env_name, + model_name=student_model_name, + **student_model_kwargs + ) + + if student_agent_kind == "ppo": + student_agent = agents.PPOAgent( + model=student_model, + n_devices=n_devices, + **student_rl_kwargs + ) + else: + raise ValueError( + "Unknown student_agent_kind: {}".format(student_agent_kind)) + + # ---- Handle UED-related settings ---- + if self.is_ued: + max_teacher_steps = dummy_env.ued_max_episode_steps() + teacher_model_kwargs['n_scalar_embeddings'] = max_teacher_steps + teacher_model_kwargs['max_scalar'] = max_teacher_steps + teacher_model_kwargs['output_dim'] = dummy_env.ued_action_space().n + + teacher_model = models.make( + env_name=env_name, + model_name=teacher_model_name, + **teacher_model_kwargs + ) + + teacher_agent = agents.PPOAgent( + model=teacher_model, + n_devices=n_devices, + **teacher_rl_kwargs + ) + + train_runner_kwargs.update(dict( + teacher_agents=[teacher_agent] + )) + train_runner_kwargs.update(dict( + ued_env_kwargs=ued_env_kwargs + )) + + # Debug, tabulate student and teacher model + # import jax.numpy as jnp + # dummy_rng = jax.random.PRNGKey(0) + # obs, _ = dummy_env.reset(dummy_rng) + # hx = student_model.initialize_carry(dummy_rng, (1,)) + # ued_obs, _ = dummy_env.reset_teacher(dummy_rng) + # ued_hx = teacher_model.initialize_carry(dummy_rng, (1,)) + + # obs['image'] = jnp.expand_dims(obs['image'], 0) + # ued_obs['image'] = jnp.expand_dims(ued_obs['image'], 0) + + # print(student_model.tabulate(dummy_rng, obs, hx)) + # print(teacher_model.tabulate(dummy_rng, ued_obs, hx)) + + # import pdb; pdb.set_trace() + + # ---- Set up train runner ---- + runner_cls = RUNNER_INFO[train_runner].runner_cls + + # Set up learning rate annealing parameters + lr_init = train_runner_kwargs.lr + lr_final = train_runner_kwargs.lr_final + lr_anneal_steps = train_runner_kwargs.lr_anneal_steps + + if lr_final is None: + train_runner_kwargs.lr_final = lr_init + if train_runner_kwargs.lr_final == train_runner_kwargs.lr: + train_runner_kwargs.lr_anneal_steps = 0 + + self.runner = runner_cls( + env_name=env_name, + env_kwargs=env_kwargs, + student_agents=[student_agent], + n_devices=n_devices, + **train_runner_kwargs) + + # ---- Make eval runner ---- + if eval_kwargs.get('env_names') is None: + self.eval_runner = None + else: + self.eval_runner = EvalRunner( + pop=self.runner.student_pop, + env_kwargs=eval_env_kwargs, + **eval_kwargs) + + self._start_tick = 0 + + # ---- Set up device parallelism ---- + self.n_devices = n_devices + if n_devices > 1: + dummy_runner_state = self.reset_train_runner(jax.random.PRNGKey(0)) + self._shmap_run = self._make_shmap_run(dummy_runner_state) + else: + self._shmap_run = None + + @partial(jax.jit, static_argnums=(0,)) + def step(self, runner_state, evaluate=False): + if self.n_devices > 1: + run_fn = self._shmap_run + else: + run_fn = self.runner.run + + stats, *runner_state = run_fn(*runner_state) + + rng = runner_state[0] + rng, subrng = jax.random.split(rng) + + if self.eval_runner is not None: + params = runner_state[1].params + eval_stats = jax.lax.cond( + evaluate, + self.eval_runner.run, + self.eval_runner.fake_run, + *(subrng, params) + ) + else: + eval_stats = {} + + return stats, eval_stats, rng, *runner_state[1:] + + def _make_shmap_run(self, runner_state): + devices = mesh_utils.create_device_mesh((self.n_devices,)) + mesh = Mesh(devices, axis_names=('device')) + + in_specs, out_specs = self.runner.get_shmap_spec() + + return partial(shard_map, + mesh=mesh, + in_specs=in_specs, + out_specs=out_specs, + check_rep=False + )(self.runner.run) + + def train( + self, + rng, + agent_algo='ppo', + algo_runner='dr', + n_total_updates=1000, + logger=None, + log_interval=1, + test_interval=1, + checkpoint_interval=0, + archive_interval=0, + archive_init_checkpoint=False, + from_last_checkpoint=False + ): + """ + Entry-point for training + """ + # Load checkpoint if any + runner_state = self.runner.reset(rng) + + if from_last_checkpoint: + last_checkpoint_state = logger.load_last_checkpoint_state() + if last_checkpoint_state is not None: + runner_state = self.runner.load_checkpoint_state( + runner_state, + last_checkpoint_state + ) + self._start_tick = runner_state[1].n_iters[0] + + # Archive initialization weights if necessary + if archive_init_checkpoint: + logger.checkpoint( + self.runner.get_checkpoint_state(runner_state), + index=0, + archive_interval=1) + + # Train loop + log_on = logger is not None and log_interval > 0 + checkpoint_on = checkpoint_interval > 0 or archive_interval > 0 + train_state = runner_state[1] + + tick = self._start_tick + train_steps = tick*self.runner.step_batch_size * \ + self.runner.n_rollout_steps*self.n_devices + real_train_steps = train_steps//self.runner.n_students + + while (train_state.n_updates < n_total_updates).any(): + evaluate = test_interval > 0 and (tick+1) % test_interval == 0 + + start = time.time() + stats, eval_stats, *runner_state = \ + self.step(runner_state, evaluate) + end = time.time() + + if evaluate: + stats.update(eval_stats) + else: + stats.update({k: None for k in eval_stats.keys()}) + + train_state = runner_state[1] + + dsteps = self.runner.step_batch_size*self.runner.n_rollout_steps*self.n_devices + real_dsteps = dsteps//self.runner.n_students + train_steps += dsteps + real_train_steps += real_dsteps + sps = int(dsteps/(end-start)) + real_sps = int(real_dsteps/(end-start)) + stats.update(dict( + steps=train_steps, + sps=sps, + real_steps=real_train_steps, + real_sps=real_sps + )) + + tick += 1 + + if log_on and tick % log_interval == 0: + logger.log(stats, tick, ignore_val=-np.inf) + + if checkpoint_on and tick > 0: + if tick % checkpoint_interval == 0 \ + or (archive_interval > 0 and tick % archive_interval == 0): + checkpoint_state = \ + self.runner.get_checkpoint_state(runner_state) + logger.checkpoint( + checkpoint_state, + index=tick, + archive_interval=archive_interval) diff --git a/src/minimax/runners_ma/__init__.py b/src/minimax/runners_ma/__init__.py new file mode 100644 index 0000000..e5058e6 --- /dev/null +++ b/src/minimax/runners_ma/__init__.py @@ -0,0 +1,24 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from .xp_runner import ExperimentRunner +from .eval_runner import EvalRunner +from .eval_runner_heterogenous import EvalRunnerHeterogenous +from .dr_runner import DRRunner +from .plr_runner import PLRRunner +from .paired_runner import PAIREDRunner + + +__all__ = [ + ExperimentRunner, + EvalRunner, + EvalRunnerHeterogenous, + DRRunner, + PLRRunner, + PAIREDRunner +] diff --git a/src/minimax/runners_ma/dr_runner.py b/src/minimax/runners_ma/dr_runner.py new file mode 100644 index 0000000..f3784f3 --- /dev/null +++ b/src/minimax/runners_ma/dr_runner.py @@ -0,0 +1,569 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial +from typing import Dict, Tuple, Optional +import inspect + +import chex +import einops +import numpy as np +import jax +import jax.numpy as jnp +from jax.sharding import PartitionSpec as P +import optax +import flax +import flax.linen as nn +from flax.core.frozen_dict import FrozenDict +from torch import NoneType + +import minimax.envs as envs +from minimax.util import pytree as _tree_util +from minimax.util.rl import ( + AgentPop, + VmapMAPPOTrainState, + RolloutStorageSeperate, + RollingStats +) + + +class DRRunner: + """ + Orchestrates rollouts across one or more students. + The main components at play: + - AgentPop: Manages train state and batched inference logic + for a population of agents. + - BatchEnv: Manages environment step and reset logic, using a + populaton of agents. + - RolloutStorage: Manages the storing and sampling of collected txns. + - PPO: Handles PPO updates, which take a train state + batch of txns. + """ + + def __init__( + self, + env_name, + env_kwargs, + student_agents, + student_agent_kind, + n_students=1, + n_parallel=1, + n_eval=1, + n_rollout_steps=256, + lr=1e-4, + lr_final=None, + lr_anneal_steps=0, + max_grad_norm=0.5, + discount=0.99, + gae_lambda=0.95, + adam_eps=1e-5, + normalize_return=False, + track_env_metrics=False, + n_unroll_rollout=1, + n_devices=1, + render=False, + shaped_reward=False, + ): + + assert len(student_agents) == 1, 'Only one type of student supported.' + assert n_parallel % n_devices == 0, 'Num envs must be divisible by num devices.' + + self.n_students = n_students + self.n_parallel = n_parallel // n_devices + self.n_eval = n_eval + self.n_devices = n_devices + self.step_batch_size = n_students*n_eval*n_parallel + self.n_rollout_steps = n_rollout_steps + self.n_updates = 0 + self.lr = lr + self.lr_final = lr if lr_final is None else lr_final + self.lr_anneal_steps = lr_anneal_steps + self.max_grad_norm = max_grad_norm + self.adam_eps = adam_eps + self.normalize_return = normalize_return + self.track_env_metrics = track_env_metrics + self.n_unroll_rollout = n_unroll_rollout + self.render = render + + self.shaped_reward = shaped_reward + + self.student_agent_kind = student_agent_kind + self.student_pop = AgentPop(student_agents[0], n_agents=n_students) + + self.env, self.env_params = envs.make( + env_name, + env_kwargs=env_kwargs + ) + self._action_shape = self.env.action_space().shape + + wrappers_lst = ['monitor_return', 'monitor_ep_metrics'] + if self.student_agent_kind == "mappo": + wrappers_lst = ['world_state_wrapper'] + wrappers_lst + + self.benv = envs.BatchEnv( + env_name=env_name, + n_parallel=self.n_parallel, + n_eval=self.n_eval, + env_kwargs=env_kwargs, + wrappers=wrappers_lst, + ) + self.action_dtype = self.benv.env.action_space().dtype + + self.student_rollout = RolloutStorageSeperate( + discount=discount, + gae_lambda=gae_lambda, + n_steps=n_rollout_steps, + n_agents=n_students, + n_envs=self.n_parallel, + n_eval=self.n_eval, + action_space=self.env.action_space(), + obs_space=self.env.observation_space(), + obs_space_shared_shape=self.benv.env.world_state_size(), + agent=self.student_pop.agent, + ) + + monitored_metrics = self.benv.env.get_monitored_metrics() + self.rolling_stats = RollingStats( + names=monitored_metrics, + window=10, + ) + self._update_ep_stats = jax.vmap( + jax.vmap(self.rolling_stats.update_stats)) + + if self.render: + from minimax.envs.viz.grid_viz import GridVisualizer + self.viz = GridVisualizer() + self.viz.show() + + def reset(self, rng): + self.n_updates = 0 + + n_parallel = self.n_parallel*self.n_devices + + rngs, *vrngs = jax.random.split(rng, self.n_students+1) + obs, state, extra = self.benv.reset( + jnp.array(vrngs), n_parallel=n_parallel) + + # dummy_obs = jax.tree_util.tree_map(lambda x: x[0], obs) # for one agent only + dummy_obs = self._concat_multi_agent_obs(obs) + dummy_shared_obs = self._concat_multi_agent_obs(obs['world_state']) + + rng, subrng = jax.random.split(rng) + if self.student_pop.agent.is_recurrent: + actor_carry, critic_carry = self.student_pop.init_carry( + subrng, dummy_obs) + self.zero_carry = jax.tree_map( + lambda x: x.at[:, :self.n_parallel].get(), actor_carry) + else: + actor_carry, critic_carry = None, None + + rng, subrng = jax.random.split(rng) + actor_params, critic_params = self.student_pop.init_params( + subrng, (dummy_obs[0], dummy_shared_obs[0])) + + schedule_fn = optax.linear_schedule( + init_value=-float(self.lr), + end_value=-float(self.lr_final), + transition_steps=self.lr_anneal_steps, + ) + + tx_actor = optax.chain( + optax.clip_by_global_norm(self.max_grad_norm), + optax.adam(learning_rate=float(self.lr), eps=self.adam_eps) + ) + + tx_critic = optax.chain( + optax.clip_by_global_norm(self.max_grad_norm), + optax.adam(learning_rate=float(self.lr), eps=self.adam_eps) + ) + + shaped_reward_coeff_value = 1.0 if self.shaped_reward else 0.0 + shaped_reward_coeff = jnp.full( + (self.n_students, 1), fill_value=shaped_reward_coeff_value) + train_state = VmapMAPPOTrainState.create( + actor_apply_fn=self.student_pop.agent.evaluate_action, + actor_params=actor_params, + actor_tx=tx_actor, + critic_apply_fn=self.student_pop.agent.get_value, + critic_params=critic_params, + critic_tx=tx_critic, + shaped_reward_coeff=shaped_reward_coeff, + ) + + ep_stats = self.rolling_stats.reset_stats( + batch_shape=(self.n_students, n_parallel*self.n_eval)) + + start_state = state + + return ( + rng, + train_state, + state, + start_state, # Used to track metrics from starting state + obs, + actor_carry, + critic_carry, + extra, + ep_stats + ) + + def get_checkpoint_state(self, state): + _state = list(state) + _state[1] = state[1].state_dict + + return _state + + def load_checkpoint_state(self, runner_state, state): + runner_state = list(runner_state) + runner_state[1] = runner_state[1].load_state_dict(state[1]) + + return tuple(runner_state) + + @partial(jax.jit, static_argnums=(0, 2)) + def _get_transition( + self, + rng, + pop, + actor_params, + critic_params, + rollout, + state, + start_state, + obs, + actor_carry, + critic_carry, + done, + extra=None): + # Sample action + + ma_obs = self._concat_multi_agent_obs(obs) + + # PRINT THE CURRENT STATE + + _, pi_params, next_actor_carry = jax.vmap(pop.act, in_axes=(None, 2, 2, None))( + actor_params, ma_obs, actor_carry, done) + next_actor_carry = jax.tree_map(lambda x: einops.rearrange( + x, 't n a d -> a t n d'), next_actor_carry) + shared_obs = self._concat_multi_agent_obs(obs['world_state']) + value, next_critic_carry = jax.vmap(pop.get_value, in_axes=(None, 2, 2, None))( + critic_params, shared_obs, critic_carry, done) + next_critic_carry = jax.tree_map(lambda x: einops.rearrange( + x, 't n a d -> a t n d'), next_critic_carry) + + pi = pop.get_action_dist(pi_params, dtype=self.action_dtype) + rng, subrng = jax.random.split(rng) + action = pi.sample(seed=subrng) + log_pi = pi.log_prob(action) + + env_action = { + 'agent_0': action[0], + 'agent_1': action[1] + } + + rng, *vrngs = jax.random.split(rng, self.n_students+1) + (next_obs, + next_state, + reward, + done, + info, + extra) = self.benv.step(jnp.array(vrngs), state, env_action, extra) + + # jax.debug.print("Current state (r: {r}, sparse: {spa}, shaped: {sha}) =\n{a}", spa=info["sparse_reward"][0, 0].mean(), sha=info["shaped_reward"][0, 0].mean(), r=reward[0, 0], a=ma_obs[0, 0, 0, :, :, 0] + # * 1 + ma_obs[0, 0, 0, :, :, 1]*2+ma_obs[0, 0, 0, :, :, 11]*3) + + next_start_state = jax.vmap(_tree_util.pytree_select)( + done, next_state, start_state + ) + + # Add transition to storage + log_pi = einops.rearrange(log_pi, 'a s n -> s n a') + value = einops.rearrange(value, 'a s n -> s n a') + + action = einops.rearrange(action, 'a s n -> s n a') + + done_ = jnp.concatenate( + [done[:, :, jnp.newaxis], done[:, :, jnp.newaxis]], axis=2) + + # jax.debug.print("sparse reward = {b}, reward = {c}", # a=info["shaped_reward"].mean(), + # b=info["sparse_reward"].mean(), c=reward.mean()) + step = (ma_obs, shared_obs, action, info["sparse_reward"], + info["shaped_reward"], done_, log_pi, value) + if actor_carry is not None and critic_carry is not None: + step += (actor_carry, critic_carry) + + rollout = self.student_rollout.append(rollout, *step) + + if self.render: + self.viz.render( + self.benv.env.params, + jax.tree_util.tree_map(lambda x: x[0][0], state)) + + return ( + rollout, + next_state, + next_start_state, + next_obs, + jax.tree_map(lambda x: einops.rearrange( + x, 'n a s d -> s n a d'), next_actor_carry), + jax.tree_map(lambda x: einops.rearrange( + x, 'n a s d -> s n a d'), next_critic_carry), + done, + info, + extra + ) + + @partial(jax.jit, static_argnums=(0,)) + def _rollout_students( + self, + rng, + train_state, + state, + start_state, + obs, + actor_carry, + critic_carry, + done, + extra=None, + ep_stats=None): + rollout = self.student_rollout.reset() + + rngs = jax.random.split(rng, self.n_rollout_steps) + + def _scan_rollout(scan_carry, rng): + rollout, state, start_state, obs, actor_carry, critic_carry, done, extra, ep_stats, train_state = scan_carry + + next_scan_carry = \ + self._get_transition( + rng, + self.student_pop, + jax.lax.stop_gradient(train_state.actor_params), + jax.lax.stop_gradient(train_state.critic_params), + rollout, + state, + start_state, + obs, + actor_carry, + critic_carry, + done, + extra) + (rollout, + next_state, + next_start_state, + next_obs, + next_actor_carry, + next_critic_carry, + done, + info, + extra) = next_scan_carry + + ep_stats = self._update_ep_stats(ep_stats, done, info) + + return ( + rollout, + next_state, + next_start_state, + next_obs, + next_actor_carry, + next_critic_carry, + done, + extra, + ep_stats, + train_state), None + + (rollout, + state, + start_state, + obs, + actor_carry, + critic_carry, + done, + extra, + ep_stats, + train_state), _ = jax.lax.scan( + _scan_rollout, + (rollout, + state, + start_state, + obs, + actor_carry, + critic_carry, + done, + extra, + ep_stats, + train_state), + rngs, + length=self.n_rollout_steps, + ) + + return rollout, state, start_state, obs, actor_carry, critic_carry, extra, ep_stats, train_state + + @partial(jax.jit, static_argnums=(0,)) + def _compile_stats(self, update_stats, ep_stats, env_metrics=None, shaped_reward_coeff=None): + + info = {k: ep_stats[k] for k in self.rolling_stats.names} + + stats = jax.vmap(lambda info: jax.tree_map(lambda x: x.mean(), info))( + info + ) + + if shaped_reward_coeff is not None: + update_stats.update( + {"shaped_reward_coeff": shaped_reward_coeff}) + + stats.update(update_stats) + + if self.n_students > 1: + _stats = {} + for i in range(self.n_students): + _student_stats = jax.tree_util.tree_map( + lambda x: x[i], stats) # for agent0 + _stats.update( + {f'a{i}/{k}': v for k, v in _student_stats.items()}) + stats = _stats + + if self.track_env_metrics: + mean_env_metrics = jax.vmap(lambda info: jax.tree_map( + lambda x: x.mean(), info))(env_metrics) + mean_env_metrics = {f'env/{k}': v for k, + v in mean_env_metrics.items()} + + if self.n_students > 1: + _env_metrics = {} + for i in range(self.n_students): + _student_env_metrics = jax.tree_util.tree_map( + lambda x: x[i], mean_env_metrics) # for agent0 + _env_metrics.update( + {f'{k}_a{i}': v for k, v in _student_env_metrics.items()}) + mean_env_metrics = _env_metrics + + stats.update(mean_env_metrics) + + if self.n_students == 1: + stats = jax.tree_map(lambda x: x[0], stats) + + if self.n_devices > 1: + stats = jax.tree_map(lambda x: jax.lax.pmean(x, 'device'), stats) + + return stats + + def get_shmap_spec(self): + runner_state_size = len(inspect.signature(self.run).parameters) + in_spec = [P(None, 'device'),]*(runner_state_size) + out_spec = [P(None, 'device'),]*(runner_state_size) + + in_spec[:2] = [P(None),]*2 + in_spec = tuple(in_spec) + out_spec = (P(None),) + in_spec + + return in_spec, out_spec + + @partial(jax.jit, static_argnums=(0,)) + def run( + self, + rng, + train_state, + state, + start_state, + obs, + actor_carry=None, + critic_carry=None, + extra=None, + ep_stats=None): + """ + Perform one update step: rollout all students and teachers + update with PPO + """ + if self.n_devices > 1: + rng = jax.random.fold_in(rng, jax.lax.axis_index('device')) + + rng, *vrngs = jax.random.split(rng, self.n_students+1) + rollout_batch_shape = (self.n_students, self.n_parallel*self.n_eval) + + obs, state, extra = self.benv.reset(jnp.array(vrngs)) + ep_stats = self.rolling_stats.reset_stats( + batch_shape=rollout_batch_shape) + + rollout_start_state = state + + done = jnp.zeros(rollout_batch_shape, dtype=jnp.bool_) + rng, subrng = jax.random.split(rng) + rollout, state, start_state, obs, actor_carry, critic_carry, extra, ep_stats, train_state = \ + self._rollout_students( + subrng, + train_state, + state, + start_state, + obs, + actor_carry, + critic_carry, + done, + extra, + ep_stats + ) + + reward = rollout["rewards"].sum(axis=1).mean(-1)[:, :, jnp.newaxis] + shaped_reward = rollout["shaped_rewards"].sum( + axis=1).mean(-1)[:, :, jnp.newaxis] + + ep_stats["reward"] = reward + ep_stats["shaped_reward"] = shaped_reward + ep_stats["shaped_reward_scaled_by_shaped_reward_coeff"] = shaped_reward * \ + train_state.shaped_reward_coeff + ep_stats["reward_p_shaped_reward_scaled"] = reward + shaped_reward * \ + train_state.shaped_reward_coeff + + shared_obs = self._concat_multi_agent_obs(obs['world_state']) + value, _ = jax.vmap(self.student_pop.get_value, in_axes=(None, 2, 2))( + jax.lax.stop_gradient(train_state.critic_params), + shared_obs, + critic_carry + ) + + value = einops.rearrange( + value, "n_env_agents n_students n_parallel -> n_students n_parallel n_env_agents") + train_batch = self.student_rollout.get_batch( + rollout, + value, + train_state.shaped_reward_coeff + ) + + # PPOAgent vmaps over the train state and batch. Batch must be N x EM + rng, subrng = jax.random.split(rng) + train_state, update_stats = self.student_pop.update( + subrng, train_state, train_batch) + + # Collect env metrics + if self.track_env_metrics: + env_metrics = self.benv.get_env_metrics(rollout_start_state) + else: + env_metrics = None + + stats = self._compile_stats( + update_stats, ep_stats, env_metrics, shaped_reward_coeff=train_state.shaped_reward_coeff) + stats.update(dict(n_updates=train_state.n_updates[0])) + + train_state = train_state.increment() + self.n_updates += 1 + + return ( + stats, + rng, + train_state, + state, + start_state, + obs, + actor_carry, + critic_carry, + extra, + ep_stats, + rollout_start_state + ) + + def _concat_multi_agent_obs(self, obs: Dict[str, chex.Array]) -> chex.Array: + """Concatenates a obs dictionary that was built for two env agents. + Doubles the number of parallel_envs, i.e. (num_students, n_parallel, ...) -> (num_students, 2*n_parallel, ...) + """ + return jnp.concatenate([obs['agent_0'][:, :, jnp.newaxis, :], obs['agent_1'][:, :, jnp.newaxis, :]], axis=2) diff --git a/src/minimax/runners_ma/eval_runner.py b/src/minimax/runners_ma/eval_runner.py new file mode 100644 index 0000000..9f7a508 --- /dev/null +++ b/src/minimax/runners_ma/eval_runner.py @@ -0,0 +1,371 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial +from typing import Dict, Tuple, Optional + +import chex +import einops +import numpy as np +import jax +import jax.numpy as jnp + +import minimax.envs as envs +from minimax.util.rl import ( + RollingStats +) +import minimax.util.pytree as _tree_util + + +def generate_all_kwargs_combos(arg_choices): + def update_kwargs_with_choices(prev_combos, k, choices): + updated_combos = [] + for v in choices: + for p in prev_combos: + updated = p.copy() + updated[k] = v + updated_combos.append(updated) + + return updated_combos + + all_combos = [{}] + for k, choices in arg_choices.items(): + all_combos = update_kwargs_with_choices(all_combos, k, choices) + + return all_combos + + +def create_envs_for_kwargs(env_names, kwargs): + # Check for csv kwargs + arg_choices = {} + varied_args = [] + for k, v in kwargs.items(): + if isinstance(v, str) and ',' in v: + vs = eval(v) + arg_choices[k] = vs + varied_args.append(k) + elif isinstance(v, str): + arg_choices[k] = [eval(v)] + else: + arg_choices[k] = [v] + + # List of kwargs + kwargs_combos = generate_all_kwargs_combos(arg_choices) + + env_infos = [] + incl_ext = len(varied_args) > 0 + for name in env_names: + for kwargs in kwargs_combos: + if incl_ext and len(kwargs) > 0: + ext = ':'.join([f'{k}={kwargs[k]}' for k in varied_args]) + ext_name = f'{name}:{ext}' + else: + ext_name = name + env_infos.append( + (name, ext_name, kwargs) + ) + + return env_infos + + +class EvalRunner: + def __init__( + self, + pop, + env_names, + env_kwargs=None, + n_episodes=10, + agent_idxs='*', + render_mode=None): + + self.pop = pop + + if isinstance(agent_idxs, str): + if "*" in agent_idxs: + self.agent_idxs = np.arange(pop.n_agents) + else: + self.agent_idxs = \ + np.array([int(x) for x in agent_idxs.split(',')]) + else: + self.agent_idxs = agent_idxs # assume array + + assert np.max(self.agent_idxs) < pop.n_agents, \ + 'Agent index is out of bounds.' + + if isinstance(env_names, str): + env_names = [ + x.strip() for x in env_names.split(',') + ] + + self.n_episodes = n_episodes + env_infos = create_envs_for_kwargs(env_names, env_kwargs) + env_names = [] + self.ext_env_names = [] + env_kwargs = [] + for (name, ext_name, kwargs) in env_infos: + env_names.append(name) + self.ext_env_names.append(ext_name) + env_kwargs.append(kwargs) + self.n_envs = len(env_names) + + self.benvs = [] + self.env_params = [] + self.env_has_solved_rate = [] + for env_name, kwargs in zip(env_names, env_kwargs): + benv = envs.BatchEnv( + env_name=env_name, + n_parallel=n_episodes, + n_eval=1, + env_kwargs=kwargs, + wrappers=['monitor_return', 'monitor_ep_metrics'] + ) + self.benvs.append(benv) + self.env_params.append(benv.env.params) + self.env_has_solved_rate.append( + benv.env.eval_solved_rate is not None) + + self.action_dtype = self.benvs[0].env.action_space().dtype + + monitored_metrics = self.benvs[0].env.get_monitored_metrics() + self.rolling_stats = RollingStats(names=monitored_metrics, window=1) + self._update_ep_stats = jax.vmap( + jax.vmap( + self.rolling_stats.update_stats, in_axes=(0, 0, 0, None)), + in_axes=(0, 0, 0, None)) + + self.test_return_pre = 'test_return' + self.test_solved_rate_pre = 'test_solved_rate' + + self.render_mode = render_mode + if render_mode: + from minimax.envs.viz.grid_viz import GridVisualizer + self.viz = GridVisualizer() + self.viz.show() + + if render_mode == 'ipython': + from IPython import display + self.ipython_display = display + + def load_checkpoint_state(self, runner_state, state): + runner_state = list(runner_state) + runner_state[1] = runner_state[1].load_state_dict(state[1]) + + return tuple(runner_state) + + def _concat_multi_agent_obs(self, obs: Dict[str, chex.Array]) -> chex.Array: + """Concatenates a obs dictionary that was built for two env agents. + Doubles the number of parallel_envs, i.e. (num_students, n_parallel, ...) -> (num_students, 2*n_parallel, ...) + """ + return jnp.concatenate([obs['agent_0'][:, :, jnp.newaxis, :], obs['agent_1'][:, :, jnp.newaxis, :]], axis=2) + + @partial(jax.jit, static_argnums=(0, 2)) + def _get_transition( + self, + rng, + benv, + actor_params, + state, + obs, + actor_carry, + zero_carry, + done, + extra): + # Sample action + ma_obs = self._concat_multi_agent_obs(obs) + _, pi_params, next_actor_carry = jax.vmap(self.pop.act, in_axes=(None, 2, 2, None))( + actor_params, ma_obs, actor_carry, done) + next_actor_carry = jax.tree_map(lambda x: einops.rearrange( + x, 't n a d -> a t n d'), next_actor_carry) + + pi = self.pop.get_action_dist(pi_params, dtype=self.action_dtype) + rng, subrng = jax.random.split(rng) + action = pi.sample(seed=subrng) + log_pi = pi.log_prob(action) + + env_action = { + 'agent_0': action[0], + 'agent_1': action[1] + } + + rng, *vrngs = jax.random.split(rng, self.pop.n_agents+1) + (next_obs, + next_state, + reward, + done, + info, + extra) = benv.step(jnp.array(vrngs), state, env_action, extra) + + log_pi = einops.rearrange(log_pi, 'a s n -> s n a') + + action = einops.rearrange(action, 'a s n -> s n a') + + done_ = jnp.concatenate( + [done[:, :, jnp.newaxis], done[:, :, jnp.newaxis]], axis=2) + + next_actor_carry = jax.tree_map(lambda x: einops.rearrange( + x, 'n a s d -> s n a d'), next_actor_carry) + step = (ma_obs, action, info["sparse_reward"], + info["shaped_reward"], done_, log_pi) + if actor_carry is not None: + step += (actor_carry,) + + if actor_carry is not None: + next_actor_carry = jax.vmap(_tree_util.pytree_select)( + done, zero_carry, next_actor_carry) + + if self.render_mode: + self.viz.render( + benv.env.params, + jax.tree_util.tree_map(lambda x: x[0][0], state), + highlight=False) + if self.render_mode == 'ipython': + self.ipython_display.display(self.viz.window.fig) + self.ipython_display.clear_output(wait=True) + return ( + next_state, + next_obs, + next_actor_carry, + done, + info, + extra + ) + + @partial(jax.jit, static_argnums=(0, 2)) + def _rollout_benv( + self, + rng, + benv, + params, + env_params, + state, + obs, + carry, + zero_carry, + extra, + done, + ep_stats): + + def _scan_rollout(scan_carry, rng): + (state, + obs, + carry, + extra, + done, + ep_stats) = scan_carry + + step = \ + self._get_transition( + rng, + benv, + params, + state, + obs, + carry, + zero_carry, + done, + extra) + + (next_state, + next_obs, + next_carry, + done, + info, + extra) = step + + ep_stats = self._update_ep_stats(ep_stats, done, info, 1) + + return (next_state, next_obs, next_carry, extra, done, ep_stats), None + + n_steps = benv.env.max_episode_steps() + rngs = jax.random.split(rng, n_steps) + (state, + obs, + carry, + extra, + done, + ep_stats), _ = jax.lax.scan( + _scan_rollout, + (state, obs, carry, extra, done, ep_stats), + rngs, + length=n_steps) + + return ep_stats + + @partial(jax.jit, static_argnums=(0,)) + def run(self, rng, params): + """ + Rollout agents on each env. + + For each env, run n_eval episodes in parallel, + where each is indexed to return in order. + """ + eval_stats = self.fake_run(rng, params) + rng, *rollout_rngs = jax.random.split(rng, self.n_envs+1) + for i, (benv, env_param) in enumerate(zip(self.benvs, self.env_params)): + rng, *reset_rngs = jax.random.split(rng, self.pop.n_agents+1) + obs, state, extra = benv.reset(jnp.array(reset_rngs)) + + if self.pop.agent.is_recurrent: + rng, subrng = jax.random.split(rng) + dummy_obs = self._concat_multi_agent_obs(obs) + actor_zero_carry, _ = self.pop.init_carry(subrng, dummy_obs) + else: + actor_zero_carry = None + + # Reset episodic stats + ep_stats = self.rolling_stats.reset_stats( + batch_shape=(self.pop.n_agents, self.n_episodes)) + + done = jnp.zeros( + (self.pop.n_agents, self.n_episodes), dtype=jnp.bool_) + + ep_stats = self._rollout_benv( + rollout_rngs[i], + benv, + jax.lax.stop_gradient(params), + env_param, + state, + obs, + actor_zero_carry, + actor_zero_carry, + extra, + done, + ep_stats) + + env_name = self.ext_env_names[i] + mean_return = ep_stats['return'].mean(1) + std_return = ep_stats['return'].std(1) + + if self.env_has_solved_rate[i]: + mean_solved_rate = jax.vmap( + jax.vmap(benv.env.eval_solved_rate))(ep_stats).mean(1) + + for idx in self.agent_idxs: + eval_stats[f'eval/a{idx}:{self.test_return_pre}:{env_name}'] = mean_return[idx].squeeze() + eval_stats[f'eval/a{idx}:{self.test_return_pre}_std:{env_name}'] = std_return[idx].squeeze() + if self.env_has_solved_rate[i]: + eval_stats[f'eval/a{idx}:{self.test_solved_rate_pre}:{env_name}'] = mean_solved_rate[idx].squeeze() + + return eval_stats + + def fake_run(self, rng, params): + eval_stats = {} + for i, env_name in enumerate(self.ext_env_names): + for idx in self.agent_idxs: + eval_stats.update({ + f'eval/a{idx}:{self.test_return_pre}:{env_name}': 0. + }) + eval_stats.update({ + f'eval/a{idx}:{self.test_return_pre}_std:{env_name}': 0. + }) + if self.env_has_solved_rate[i]: + eval_stats.update({ + f'eval/a{idx}:{self.test_solved_rate_pre}:{env_name}': 0., + }) + + return eval_stats diff --git a/src/minimax/runners_ma/eval_runner_heterogenous.py b/src/minimax/runners_ma/eval_runner_heterogenous.py new file mode 100644 index 0000000..08b735d --- /dev/null +++ b/src/minimax/runners_ma/eval_runner_heterogenous.py @@ -0,0 +1,388 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial +from typing import Dict + +import chex +import numpy as np +import jax +import jax.numpy as jnp + +import minimax.envs as envs +from minimax.util.rl import ( + AgentPop, + RollingStats +) +import minimax.util.pytree as _tree_util + + +def generate_all_kwargs_combos(arg_choices): + def update_kwargs_with_choices(prev_combos, k, choices): + updated_combos = [] + for v in choices: + for p in prev_combos: + updated = p.copy() + updated[k] = v + updated_combos.append(updated) + + return updated_combos + + all_combos = [{}] + for k, choices in arg_choices.items(): + all_combos = update_kwargs_with_choices(all_combos, k, choices) + + return all_combos + + +def create_envs_for_kwargs(env_names, kwargs): + # Check for csv kwargs + arg_choices = {} + varied_args = [] + for k, v in kwargs.items(): + if isinstance(v, str) and ',' in v: + vs = eval(v) + arg_choices[k] = vs + varied_args.append(k) + elif isinstance(v, str): + arg_choices[k] = [eval(v)] + else: + arg_choices[k] = [v] + + # List of kwargs + kwargs_combos = generate_all_kwargs_combos(arg_choices) + + env_infos = [] + incl_ext = len(varied_args) > 0 + for name in env_names: + for kwargs in kwargs_combos: + if incl_ext and len(kwargs) > 0: + ext = ':'.join([f'{k}={kwargs[k]}' for k in varied_args]) + ext_name = f'{name}:{ext}' + else: + ext_name = name + env_infos.append( + (name, ext_name, kwargs) + ) + + return env_infos + + +class EvalRunnerHeterogenous: + def __init__( + self, + pop, + env_names, + env_kwargs=None, + n_episodes=10, + agent_idxs='*', + render_mode=None): + + self.pop = pop + + if isinstance(agent_idxs, str): + if "*" in agent_idxs: + self.agent_idxs = np.arange(pop.n_agents) + else: + self.agent_idxs = \ + np.array([int(x) for x in agent_idxs.split(',')]) + else: + self.agent_idxs = agent_idxs # assume array + + assert np.max(self.agent_idxs) < pop.n_agents, \ + 'Agent index is out of bounds.' + + if isinstance(env_names, str): + env_names = [ + x.strip() for x in env_names.split(',') + ] + + self.n_episodes = n_episodes + env_infos = create_envs_for_kwargs(env_names, env_kwargs) + env_names = [] + self.ext_env_names = [] + env_kwargs = [] + for (name, ext_name, kwargs) in env_infos: + env_names.append(name) + self.ext_env_names.append(ext_name) + env_kwargs.append(kwargs) + self.n_envs = len(env_names) + + self.benvs = [] + self.env_params = [] + self.env_has_solved_rate = [] + for env_name, kwargs in zip(env_names, env_kwargs): + benv = envs.BatchEnv( + env_name=env_name, + n_parallel=n_episodes, + n_eval=1, + env_kwargs=kwargs, + wrappers=['monitor_return', 'monitor_ep_metrics'] + ) + self.benvs.append(benv) + self.env_params.append(benv.env.params) + self.env_has_solved_rate.append( + benv.env.eval_solved_rate is not None) + + self.action_dtype = self.benvs[0].env.action_space().dtype + + monitored_metrics = self.benvs[0].env.get_monitored_metrics() + self.rolling_stats = RollingStats(names=monitored_metrics, window=1) + self._update_ep_stats = jax.vmap( + jax.vmap( + self.rolling_stats.update_stats, in_axes=(0, 0, 0, None)), + in_axes=(0, 0, 0, None)) + + self.test_return_pre = 'test_return' + self.test_solved_rate_pre = 'test_solved_rate' + + self.render_mode = render_mode + if render_mode: + from minimax.envs.viz.grid_viz import GridVisualizer + self.viz = GridVisualizer() + self.viz.show() + + if render_mode == 'ipython': + from IPython import display + self.ipython_display = display + + def load_checkpoint_state(self, runner_state, state): + runner_state = list(runner_state) + runner_state[1] = runner_state[1].load_state_dict(state[1]) + + return tuple(runner_state) + + def _concat_multi_agent_obs(self, obs: Dict[str, chex.Array]) -> chex.Array: + """Concatenates a obs dictionary that was built for two env agents. + Doubles the number of parallel_envs, i.e. (num_students, n_parallel, ...) -> (num_students, 2*n_parallel, ...) + """ + return jnp.concatenate([obs['agent_0'][:, :, jnp.newaxis, :], obs['agent_1'][:, :, jnp.newaxis, :]], axis=2) + + @partial(jax.jit, static_argnums=(0, 2)) + def _get_transition( + self, + rng, + benv, + actor_0_params, + actor_1_params, + state, + obs, + actor_0_carry, + actor_1_carry, + zero_0_carry, + zero_1_carry, + done, + extra): + _, _, pi_0_params, pi_1_params, next_actor_0_carry, next_actor_1_carry = self.pop.act( + (actor_0_params, actor_1_params), obs, (actor_0_carry, actor_1_carry), done) + + pi_0 = self.pop.get_action_0_dist(pi_0_params, dtype=self.action_dtype) + pi_1 = self.pop.get_action_1_dist(pi_1_params, dtype=self.action_dtype) + rng, subrng = jax.random.split(rng) + action_0 = pi_0.sample(seed=subrng) + log_pi_0 = pi_0.log_prob(action_0) + + rng, subrng = jax.random.split(rng) + action_1 = pi_1.sample(seed=subrng) + log_pi_1 = pi_1.log_prob(action_1) + + env_action = { + 'agent_0': action_0, + 'agent_1': action_1 + } + + rng, *vrngs = jax.random.split(rng, self.pop.n_agents+1) + (next_obs, + next_state, + reward, + done, + info, + extra) = benv.step(jnp.array(vrngs), state, env_action, extra) + + done_ = jnp.concatenate( + [done[:, :, jnp.newaxis], done[:, :, jnp.newaxis]], axis=2) + + if actor_0_carry is not None: + next_actor_0_carry = jax.vmap(_tree_util.pytree_select)( + done, zero_0_carry, next_actor_0_carry) + + if actor_1_carry is not None: + next_actor_1_carry = jax.vmap(_tree_util.pytree_select)( + done, zero_1_carry, next_actor_1_carry) + + if self.render_mode: + self.viz.render( + benv.env.params, + jax.tree_util.tree_map(lambda x: x[0][0], state)) + if self.render_mode == 'ipython': + self.ipython_display.display(self.viz.window.fig) + self.ipython_display.clear_output(wait=True) + return ( + next_state, + next_obs, + next_actor_0_carry, + next_actor_1_carry, + done, + info, + extra + ) + + @partial(jax.jit, static_argnums=(0, 2)) + def _rollout_benv( + self, + rng, + benv, + params_0, + params_1, + env_params, + state, + obs, + carry_0, + carry_1, + zero_0_carry, + zero_1_carry, + extra, + done, + ep_stats): + + def _scan_rollout(scan_carry, rng): + (state, + obs, + carry_0, + carry_1, + extra, + done, + ep_stats) = scan_carry + + step = \ + self._get_transition( + rng, + benv, + params_0, + params_1, + state, + obs, + carry_0, + carry_1, + zero_0_carry, + zero_1_carry, + done, + extra) + + (next_state, + next_obs, + next_0_carry, + next_1_carry, + done, + info, + extra) = step + + ep_stats = self._update_ep_stats(ep_stats, done, info, 1) + + return (next_state, next_obs, next_0_carry, next_1_carry, extra, done, ep_stats), None + + n_steps = benv.env.max_episode_steps() + rngs = jax.random.split(rng, n_steps) + (state, + obs, + carry_0, + carry_1, + extra, + done, + ep_stats), _ = jax.lax.scan( + _scan_rollout, + (state, obs, carry_0, carry_1, extra, done, ep_stats), + rngs, + length=n_steps) + + return ep_stats + + @partial(jax.jit, static_argnums=(0,)) + def run(self, rng, params_0, params_1): + """ + Rollout agents on each env. + + For each env, run n_eval episodes in parallel, + where each is indexed to return in order. + """ + eval_stats = self.fake_run( + rng, params_0) # Params do not matter for the fake run + rng, *rollout_rngs = jax.random.split(rng, self.n_envs+1) + for i, (benv, env_param) in enumerate(zip(self.benvs, self.env_params)): + rng, *reset_rngs = jax.random.split(rng, self.pop.n_agents+1) + obs, state, extra = benv.reset(jnp.array(reset_rngs)) + + if self.pop.agent_0.is_recurrent: + rng, subrng = jax.random.split(rng) + actor_0_zero_carry, _ = self.pop.init_carry_agent_0( + subrng, obs['agent_0']) + else: + actor_0_zero_carry = None + + if self.pop.agent_1.is_recurrent: + rng, subrng = jax.random.split(rng) + actor_1_zero_carry, _ = self.pop.init_carry_agent_1( + subrng, obs['agent_1']) + else: + actor_1_zero_carry = None + + # Reset episodic stats + ep_stats = self.rolling_stats.reset_stats( + batch_shape=(self.pop.n_agents, self.n_episodes)) + + done = jnp.zeros( + (self.pop.n_agents, self.n_episodes), dtype=jnp.bool_) + + ep_stats = self._rollout_benv( + rollout_rngs[i], + benv, + jax.lax.stop_gradient(params_0), + jax.lax.stop_gradient(params_1), + env_param, + state, + obs, + actor_0_zero_carry, + actor_1_zero_carry, + actor_0_zero_carry, + actor_1_zero_carry, + extra, + done, + ep_stats) + + env_name = self.ext_env_names[i] + mean_return = ep_stats['return'].mean(1) + std_return = ep_stats['return'].std(1) + + if self.env_has_solved_rate[i]: + mean_solved_rate = jax.vmap( + jax.vmap(benv.env.eval_solved_rate))(ep_stats).mean(1) + + for idx in self.agent_idxs: + eval_stats[f'eval/a{idx}:{self.test_return_pre}:{env_name}'] = mean_return[idx].squeeze() + eval_stats[f'eval/a{idx}:{self.test_return_pre}_std:{env_name}'] = std_return[idx].squeeze() + if self.env_has_solved_rate[i]: + eval_stats[f'eval/a{idx}:{self.test_solved_rate_pre}:{env_name}'] = mean_solved_rate[idx].squeeze() + + return eval_stats + + def fake_run(self, rng, params): + eval_stats = {} + for i, env_name in enumerate(self.ext_env_names): + for idx in self.agent_idxs: + eval_stats.update({ + f'eval/a{idx}:{self.test_return_pre}:{env_name}': 0. + }) + eval_stats.update({ + f'eval/a{idx}:{self.test_return_pre}_std:{env_name}': 0. + }) + if self.env_has_solved_rate[i]: + eval_stats.update({ + f'eval/a{idx}:{self.test_solved_rate_pre}:{env_name}': 0., + }) + + return eval_stats diff --git a/src/minimax/runners_ma/paired_runner.py b/src/minimax/runners_ma/paired_runner.py new file mode 100644 index 0000000..79d1ff5 --- /dev/null +++ b/src/minimax/runners_ma/paired_runner.py @@ -0,0 +1,818 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from enum import Enum +from functools import partial +from typing import Dict, Tuple, Optional +import inspect + +import chex +import einops +import numpy as np +import jax +import jax.numpy as jnp +from jax.sharding import PartitionSpec as P +import optax +import flax +import flax.linen as nn +from flax.core.frozen_dict import FrozenDict + +import minimax.envs as envs +from minimax.util import pytree as _tree_util +from minimax.util.rl import ( + AgentPop, + VmapTrainState, + VmapMAPPOTrainState, + RolloutStorage, + RolloutStorageSeperate, + RollingStats, + UEDScore, + compute_ued_scores +) + + +class PAIREDRunner: + """ + Orchestrates rollouts across one or more students and teachers. + The main components at play: + - AgentPop: Manages train state and batched inference logic + for a population of agents. + - BatchUEDEnv: Manages environment step and reset logic for a + population of agents batched over a pair of student and + teacher MDPs. + - RolloutStorage: Manages the storing and sampling of collected txns. + - PPO: Handles PPO updates, which take a train state + batch of txns. + """ + + def __init__( + self, + env_name, + env_kwargs, + ued_env_kwargs, + student_agents, + student_agent_kind, + n_students=2, + n_parallel=1, + n_eval=1, + n_rollout_steps=250, + lr=1e-4, + lr_final=None, + lr_anneal_steps=0, + max_grad_norm=0.5, + discount=0.99, + gae_lambda=0.95, + adam_eps=1e-5, + teacher_lr=None, + teacher_lr_final=None, + teacher_lr_anneal_steps=None, + teacher_discount=0.99, + teacher_gae_lambda=0.95, + teacher_agents=None, + ued_score='relative_regret', + track_env_metrics=False, + n_unroll_rollout=1, + render=False, + n_devices=1, + shaped_reward=False, + ): + assert n_parallel % n_devices == 0, 'Num envs must be divisible by num devices.' + + ued_score = UEDScore[ued_score.upper()] + + assert len(student_agents) == 1, \ + 'Only one type of student supported.' + assert not (n_students > 2 and ued_score in [UEDScore.RELATIVE_REGRET, UEDScore.MEAN_RELATIVE_REGRET]), \ + 'Standard PAIRED uses only 2 students.' + assert teacher_agents is None or len(teacher_agents) == 1, \ + 'Only one type of teacher supported.' + + self.student_agent_kind = student_agent_kind + self.n_students = n_students + self.n_parallel = n_parallel // n_devices + self.n_eval = n_eval + self.n_devices = n_devices + self.step_batch_size = n_students*n_eval*n_parallel + self.n_rollout_steps = n_rollout_steps + self.n_updates = 0 + self.lr = lr + self.lr_final = lr if lr_final is None else lr_final + self.lr_anneal_steps = lr_anneal_steps + self.teacher_lr = \ + lr if teacher_lr is None else lr + self.teacher_lr_final = \ + self.lr_final if teacher_lr_final is None else teacher_lr_final + self.teacher_lr_anneal_steps = \ + lr_anneal_steps if teacher_lr_anneal_steps is None else teacher_lr_anneal_steps + self.max_grad_norm = max_grad_norm + self.adam_eps = adam_eps + self.ued_score = ued_score + self.track_env_metrics = track_env_metrics + + self.shaped_reward = shaped_reward + + self.n_unroll_rollout = n_unroll_rollout + self.render = render + + self.student_pop = AgentPop(student_agents[0], n_agents=n_students) + + if teacher_agents is not None: + self.teacher_pop = AgentPop(teacher_agents[0], n_agents=1) + + # This ensures correct partial-episodic bootstrapping by avoiding + # any termination purely due to timeouts. + # env_kwargs.max_episode_steps = self.n_rollout_steps + 1 + + wrappers_lst = ['monitor_return', 'monitor_ep_metrics'] + if self.student_agent_kind == "mappo": + wrappers_lst = ['world_state_wrapper'] + wrappers_lst + + self.benv = envs.BatchUEDEnv( + env_name=env_name, + n_parallel=self.n_parallel, + n_eval=n_eval, + env_kwargs=env_kwargs, + ued_env_kwargs=ued_env_kwargs, + wrappers=wrappers_lst, + ued_wrappers=[] + ) + self.action_dtype = self.benv.env.action_space().dtype + + self.teacher_n_rollout_steps = \ + self.benv.env.ued_max_episode_steps() + + self.student_rollout = RolloutStorageSeperate( + discount=discount, + gae_lambda=gae_lambda, + n_steps=n_rollout_steps, + n_agents=n_students, + n_envs=self.n_parallel, + n_eval=self.n_eval, + action_space=self.benv.env.action_space(), + obs_space=self.benv.env.observation_space(), + obs_space_shared_shape=self.benv.env.world_state_size(), + agent=self.student_pop.agent + ) + + self.teacher_rollout = RolloutStorage( + discount=teacher_discount, + gae_lambda=teacher_gae_lambda, + n_steps=self.teacher_n_rollout_steps, + n_agents=1, + n_envs=self.n_parallel, + n_eval=1, + action_space=self.benv.env.ued_action_space(), + obs_space=self.benv.env.ued_observation_space(), + agent=self.teacher_pop.agent, + ) + + ued_monitored_metrics = ('return',) + self.ued_rolling_stats = RollingStats( + names=ued_monitored_metrics, + window=10, + ) + + monitored_metrics = self.benv.env.get_monitored_metrics() + self.rolling_stats = RollingStats( + names=monitored_metrics, + window=10, + ) + + self._update_ep_stats = jax.vmap( + jax.vmap(self.rolling_stats.update_stats)) + self._update_ued_ep_stats = jax.vmap( + jax.vmap(self.ued_rolling_stats.update_stats)) + + if self.render: + from envs.viz.grid_viz import GridVisualizer + self.viz = GridVisualizer() + self.viz.show() + + def reset(self, rng): + self.n_updates = 0 + + n_parallel = self.n_parallel*self.n_devices + + rng, student_rng, teacher_rng = jax.random.split(rng, 3) + student_info = self._reset_pop( + student_rng, + self.student_pop, + partial(self.benv.reset, sub_batch_size=n_parallel*self.n_eval), + n_parallel_ep=n_parallel*self.n_eval, + lr_init=self.lr, + lr_final=self.lr_final, + lr_anneal_steps=self.lr_anneal_steps) + + teacher_info = self._reset_teacher_pop( + teacher_rng, + self.teacher_pop, + partial(self.benv.reset_teacher, n_parallel=n_parallel), + n_parallel_ep=n_parallel, + lr_init=self.teacher_lr, + lr_final=self.teacher_lr_final, + lr_anneal_steps=self.teacher_lr_anneal_steps) + + return ( + rng, + *student_info, + *teacher_info + ) + + def _reset_pop( + self, + rng, + pop, + env_reset_fn, + n_parallel_ep=1, + lr_init=3e-4, + lr_final=3e-4, + lr_anneal_steps=0): + rng, *vrngs = jax.random.split(rng, pop.n_agents+1) + reset_out = env_reset_fn(jnp.array(vrngs)) + if len(reset_out) == 2: + obs, state = reset_out + else: + obs, state, extra = reset_out + + n_parallel = self.n_parallel*self.n_devices + + # dummy_obs = jax.tree_util.tree_map(lambda x: x[0], obs) # for one agent only + dummy_obs = self._concat_multi_agent_obs(obs) + dummy_shared_obs = self._concat_multi_agent_obs(obs['world_state']) + + rng, subrng = jax.random.split(rng) + if self.student_pop.agent.is_recurrent: + actor_carry, critic_carry = self.student_pop.init_carry( + subrng, dummy_obs) + # Technically returns actor and critic carry but we only need one + self.zero_carry = jax.tree_map( + lambda x: x.at[:, :self.n_parallel].get(), actor_carry) + else: + actor_carry, critic_carry = None, None + + rng, subrng = jax.random.split(rng) + actor_params, critic_params = self.student_pop.init_params( + subrng, (dummy_obs[0], dummy_shared_obs[0])) + + schedule_fn = optax.linear_schedule( + init_value=-float(lr_init), + end_value=-float(lr_final), + transition_steps=lr_anneal_steps, + ) + + tx_actor = optax.chain( + optax.clip_by_global_norm(self.max_grad_norm), + optax.scale_by_adam(eps=self.adam_eps), + optax.scale_by_schedule(schedule_fn), + ) + + tx_critic = optax.chain( + optax.clip_by_global_norm(self.max_grad_norm), + optax.scale_by_adam(eps=self.adam_eps), + optax.scale_by_schedule(schedule_fn), + ) + + shaped_reward_coeff_value = 1.0 if self.shaped_reward else 0.0 + shaped_reward_coeff = jnp.full( + (self.n_students, 1), fill_value=shaped_reward_coeff_value) + train_state = VmapMAPPOTrainState.create( + actor_apply_fn=self.student_pop.agent.evaluate_action, + actor_params=actor_params, + actor_tx=tx_actor, + critic_apply_fn=self.student_pop.agent.get_value, + critic_params=critic_params, + critic_tx=tx_critic, + shaped_reward_coeff=shaped_reward_coeff, + ) + + ep_stats = self.rolling_stats.reset_stats( + batch_shape=(self.n_students, n_parallel*self.n_eval)) + + start_state = state + + ep_stats = self.rolling_stats.reset_stats( + batch_shape=(pop.n_agents, n_parallel_ep)) + + return train_state, state, obs, actor_carry, critic_carry, ep_stats + + def _reset_teacher_pop( + self, + rng, + pop, + env_reset_fn, + n_parallel_ep=1, + lr_init=3e-4, + lr_final=3e-4, + lr_anneal_steps=0): + rng, *vrngs = jax.random.split(rng, pop.n_agents+1) + reset_out = env_reset_fn(jnp.array(vrngs)) + if len(reset_out) == 2: + obs, state = reset_out + else: + obs, state, extra = reset_out + dummy_obs = jax.tree_util.tree_map( + lambda x: x[0], obs) # for one agent only + + rng, subrng = jax.random.split(rng) + if pop.agent.is_recurrent: + carry = pop.init_carry(subrng, obs) + else: + carry = None + + rng, subrng = jax.random.split(rng) + params = pop.init_params(subrng, dummy_obs) + + schedule_fn = optax.linear_schedule( + init_value=-float(lr_init), + end_value=-float(lr_final), + transition_steps=lr_anneal_steps, + ) + + tx = optax.chain( + optax.clip_by_global_norm(self.max_grad_norm), + optax.scale_by_adam(eps=self.adam_eps), + optax.scale_by_schedule(schedule_fn), + ) + + train_state = VmapTrainState.create( + apply_fn=pop.agent.evaluate, + params=params, + tx=tx + ) + + ep_stats = self.rolling_stats.reset_stats( + batch_shape=(pop.n_agents, n_parallel_ep)) + + return train_state, state, obs, carry, ep_stats + + def get_checkpoint_state(self, state): + _state = list(state) + _state[1] = state[1].state_dict + _state[7] = state[7].state_dict + + return _state + + def load_checkpoint_state(self, runner_state, state): + runner_state = list(runner_state) + runner_state[1] = runner_state[1].load_state_dict(state[1]) + runner_state[7] = runner_state[7].load_state_dict(state[7]) + + return tuple(runner_state) + + @partial(jax.jit, static_argnums=(0, 2,)) + def _get_ma_transition( + self, + rng, + pop, + actor_params, + critic_params, + obs, + actor_carry, + critic_carry, + done + ): + ma_obs = self._concat_multi_agent_obs(obs) + _, pi_params, next_actor_carry = jax.vmap(pop.act, in_axes=(None, 2, 2, None))( + actor_params, ma_obs, actor_carry, done) + next_actor_carry = jax.tree_map(lambda x: einops.rearrange( + x, 't n a d -> a t n d'), next_actor_carry) + shared_obs = self._concat_multi_agent_obs(obs['world_state']) + value, next_critic_carry = jax.vmap(pop.get_value, in_axes=(None, 2, 2, None))( + critic_params, shared_obs, critic_carry, done) + next_critic_carry = jax.tree_map(lambda x: einops.rearrange( + x, 't n a d -> a t n d'), next_critic_carry) + + pi = pop.get_action_dist(pi_params, dtype=self.action_dtype) + rng, subrng = jax.random.split(rng) + action = pi.sample(seed=subrng) + log_pi = pi.log_prob(action) + + env_action = { + 'agent_0': action[0], + 'agent_1': action[1] + } + + # # Add transition to storage + log_pi = einops.rearrange(log_pi, 'a s n -> s n a') + value = einops.rearrange(value, 'a s n -> s n a') + action = einops.rearrange(action, 'a s n -> s n a') + + return ( + shared_obs, + value, + log_pi, + env_action, + action, + (jax.tree_map(lambda x: einops.rearrange( + x, 'n a s d -> s n a d'), next_actor_carry), + jax.tree_map(lambda x: einops.rearrange( + x, 'n a s d -> s n a d'), next_critic_carry)), + ) + + @partial(jax.jit, static_argnums=(0, 2, 3)) + def _get_transition( + self, + rng, + pop, + rollout_mgr, + rollout, + params, + state, + obs, + carry, + done, + reset_state=None, + extra=None): + # Sample action + if type(params) == tuple and len(params) == 2: + actor_carry, critic_carry = carry + actor_params, critic_params = params + shared_obs, value, log_pi, env_action, action, next_carry = self._get_ma_transition( + rng, + pop, + actor_params, + critic_params, + obs, + actor_carry, + critic_carry, + done + ) + is_multi_agent = True + else: + value, pi_params, next_carry = pop.act(params, obs, carry, done) + pi = pop.get_action_dist(pi_params, dtype=self.action_dtype) + rng, subrng = jax.random.split(rng) + env_action = pi.sample(seed=subrng) + action = env_action # Is the same in single agent case but a dict in multi_agent + log_pi = pi.log_prob(action) + is_multi_agent = False + shared_obs = None + + rng, *vrngs = jax.random.split(rng, pop.n_agents+1) + + if pop is self.student_pop: + step_fn = self.benv.step_student + else: + step_fn = self.benv.step_teacher + step_args = (jnp.array(vrngs), state, env_action) + + if reset_state is not None: # Needed for student to reset to same instance + step_args += (reset_state,) + + if extra is not None: + step_args += (extra,) + next_obs, next_state, reward, done, info, extra = step_fn( + *step_args) + else: + next_obs, next_state, reward, done, info = step_fn(*step_args) + + if is_multi_agent: + obs = self._concat_multi_agent_obs(obs) + + # Add transition to storage + if shared_obs is not None: + done_ = jnp.concatenate( + [done[:, :, jnp.newaxis], done[:, :, jnp.newaxis]], axis=2) + # jax.debug.print("info r {i}, info sr {s}, r {r}", i=jnp.sum( + # info["sparse_reward"]), s=jnp.sum(info["shaped_reward"]), r=jnp.sum(reward)) + + step = (obs, shared_obs, action, + info["sparse_reward"], info["shaped_reward"], done_, log_pi, value) + else: + step = (obs, action, reward, done, log_pi, value) + + if is_multi_agent: + if carry[0] is not None: + step += (carry[0], carry[1]) # Actor and Critic + else: + if carry is not None: + step += (carry,) + + rollout = rollout_mgr.append(rollout, *step) + + if self.render and pop is self.student_pop: + self.viz.render( + self.benv.env.env.params, + jax.tree_util.tree_map(lambda x: x[0][0], state)) + + return rollout, next_state, next_obs, next_carry, done, info, extra + + @partial(jax.jit, static_argnums=(0, 2, 3, 4)) + def _rollout( + self, + rng, + pop, + rollout_mgr, + n_steps, + params, + state, + obs, + carry, + done, + reset_state=None, + extra=None, + ep_stats=None): + rngs = jax.random.split(rng, n_steps) + + rollout = rollout_mgr.reset() + + def _scan_rollout(scan_carry, rng): + (rollout, + state, + obs, + carry, + done, + extra, + ep_stats) = scan_carry + + next_scan_carry = \ + self._get_transition( + rng, + pop, + rollout_mgr, + rollout, + params, + state, + obs, + carry, + done, + reset_state, + extra) + + (rollout, + next_state, + next_obs, + next_carry, + done, + info, + extra) = next_scan_carry + + if ep_stats is not None: + _ep_stats_update_fn = self._update_ep_stats \ + if pop is self.student_pop else self._update_ued_ep_stats + + ep_stats = _ep_stats_update_fn(ep_stats, done, info) + + return (rollout, next_state, next_obs, next_carry, done, extra, ep_stats), None + + (rollout, state, obs, carry, done, extra, ep_stats), _ = jax.lax.scan( + _scan_rollout, + (rollout, state, obs, carry, done, extra, ep_stats), + rngs, + length=n_steps, + unroll=self.n_unroll_rollout + ) + + return rollout, state, obs, carry, extra, ep_stats + + @partial(jax.jit, static_argnums=(0,)) + def _compile_stats(self, + update_stats, ep_stats, + ued_update_stats, ued_ep_stats, + env_metrics=None, + grad_stats=None, + ued_grad_stats=None, + shaped_reward_coeff=None + ): + mean_returns_by_student = jax.vmap( + lambda x: x.mean())(ep_stats['return']) + mean_returns_by_teacher = jax.vmap( + lambda x: x.mean())(ued_ep_stats['return']) + + mean_ep_stats = jax.vmap(lambda info: jax.tree_map(lambda x: x.mean(), info))( + {k: ep_stats[k] for k in self.rolling_stats.names} + ) + ued_mean_ep_stats = jax.vmap(lambda info: jax.tree_map(lambda x: x.mean(), info))( + {k: ued_ep_stats[k] for k in self.ued_rolling_stats.names} + ) + + student_stats = { + f'mean_{k}': v for k, v in mean_ep_stats.items() + } + student_stats.update(update_stats) + + stats = {} + + if shaped_reward_coeff is not None: + stats.update( + {"shaped_reward_coeff": shaped_reward_coeff}) + + for i in range(self.n_students): + _student_stats = jax.tree_util.tree_map( + lambda x: x[i], student_stats) # for agent0 + stats.update({f'{k}_a{i}': v for k, v in _student_stats.items()}) + + teacher_stats = { + f'mean_{k}_tch': v for k, v in ued_mean_ep_stats.items() + } + teacher_stats.update({ + f'{k}_tch': v[0] for k, v in ued_update_stats.items() + }) + stats.update(teacher_stats) + + if self.n_devices > 1: + stats = jax.tree_map(lambda x: jax.lax.pmean(x, 'device'), stats) + + return stats + + def get_shmap_spec(self): + runner_state_size = len(inspect.signature(self.run).parameters) + in_spec = [P(None, 'device'),]*(runner_state_size) + out_spec = [P(None, 'device'),]*(runner_state_size) + + in_spec[:2] = [P(None),]*2 + in_spec[6] = P(None) + in_spec = tuple(in_spec) + out_spec = (P(None),) + in_spec + + return in_spec, out_spec + + @partial(jax.jit, static_argnums=(0,)) + def run( + self, + rng, + train_state, + state, + obs, + actor_carry, + critic_carry, + ep_stats, + ued_train_state, + ued_state, + ued_obs, + ued_carry, + ued_ep_stats): + """ + Perform one update step: rollout teacher + students + """ + if self.n_devices > 1: + rng = jax.random.fold_in(rng, jax.lax.axis_index('device')) + + # === Reset teacher env + rollout teacher + rng, *vrngs = jax.random.split(rng, self.teacher_pop.n_agents+1) + ued_reset_out = self.benv.reset_teacher(jnp.array(vrngs)) + if len(ued_reset_out) > 2: + ued_obs, ued_state, ued_extra = ued_reset_out + else: + ued_obs, ued_state = ued_reset_out + ued_extra = None + + # Reset UED ep_stats + if self.ued_rolling_stats is not None: + ued_ep_stats = self.ued_rolling_stats.reset_stats( + batch_shape=(1, self.n_parallel)) + else: + ued_ep_stats = None + + tch_rollout_batch_shape = (1, self.n_parallel*self.n_eval) + done = jnp.zeros(tch_rollout_batch_shape, dtype=jnp.bool_) + rng, subrng = jax.random.split(rng) + ued_rollout, ued_state, ued_obs, ued_carry, _, ued_ep_stats = \ + self._rollout( + subrng, + self.teacher_pop, + self.teacher_rollout, + self.teacher_n_rollout_steps, + jax.lax.stop_gradient(ued_train_state.params), + ued_state, + ued_obs, + ued_carry, + done, + extra=ued_extra, + ep_stats=ued_ep_stats + ) + + # === Reset student to new envs + rollout students + rng, *vrngs = jax.random.split(rng, self.teacher_pop.n_agents+1) + obs, state, extra = jax.tree_util.tree_map( + lambda x: x.squeeze(0), self.benv.reset_student( + jnp.array(vrngs), + ued_state, + self.student_pop.n_agents)) + reset_state = state + + # Reset student ep_stats + st_rollout_batch_shape = (self.n_students, self.n_parallel*self.n_eval) + ep_stats = self.rolling_stats.reset_stats( + batch_shape=st_rollout_batch_shape) + + done = jnp.zeros(st_rollout_batch_shape, dtype=jnp.bool_) + rng, subrng = jax.random.split(rng) + rollout, state, obs, carry, extra, ep_stats = \ + self._rollout( + subrng, + self.student_pop, + self.student_rollout, + self.n_rollout_steps, + (jax.lax.stop_gradient(train_state.actor_params), + jax.lax.stop_gradient(train_state.critic_params)), + state, + obs, + (actor_carry, critic_carry), + done, + reset_state=reset_state, + extra=extra, + ep_stats=ep_stats) + + reward = rollout["rewards"].sum(axis=1).mean(-1)[:, :, jnp.newaxis] + shaped_reward = rollout["shaped_rewards"].sum( + axis=1).mean(-1)[:, :, jnp.newaxis] + + ep_stats["reward"] = reward + ep_stats["shaped_reward"] = shaped_reward + ep_stats["shaped_reward_scaled_by_shaped_reward_coeff"] = shaped_reward * \ + train_state.shaped_reward_coeff.mean() + ep_stats["reward_p_shaped_reward_scaled"] = reward + shaped_reward * \ + train_state.shaped_reward_coeff.mean() + + # === Update student with PPO + # PPOAgent vmaps over the train state and batch. Batch must be N x EM + _, critic_carry = carry + shared_obs = self._concat_multi_agent_obs(obs['world_state']) + value, _ = jax.vmap(self.student_pop.get_value, in_axes=(None, 2, 2))( + jax.lax.stop_gradient(train_state.critic_params), + shared_obs, + critic_carry + ) + + jax.debug.print( + "train_state.shaped_reward_coeff {s}", s=train_state.shaped_reward_coeff) + + value = einops.rearrange( + value, "n_env_agents n_students n_parallel -> n_students n_parallel n_env_agents") + train_batch = self.student_rollout.get_batch( + rollout, + value, + train_state.shaped_reward_coeff + ) + + rng, subrng = jax.random.split(rng) + train_state, update_stats = self.student_pop.update( + subrng, train_state, train_batch) + + # === Update teacher with PPO + # - Compute returns per env per agent + # - Compute batched returns based on returns per env per agent + ued_score, _ = compute_ued_scores( + self.ued_score, train_batch, self.n_eval) + ued_rollout = self.teacher_rollout.set_final_reward( + ued_rollout, ued_score) + ued_train_batch = self.teacher_rollout.get_batch( + ued_rollout, + jnp.zeros((1, self.n_parallel)) # Last step terminates episode + ) + + ued_ep_stats = self._update_ued_ep_stats( + ued_ep_stats, + jnp.ones((1, len(ued_score), 1), dtype=jnp.bool_), + {'return': jnp.expand_dims(ued_score, (0, -1))} + ) + + # Update teacher, batch must be 1 x Ex1 + rng, subrng = jax.random.split(rng) + ued_train_state, ued_update_stats = self.teacher_pop.update( + subrng, ued_train_state, ued_train_batch) + + # -------------------------------------------------- + # Collect metrics + if self.track_env_metrics: + env_metrics = self.benv.get_env_metrics(reset_state) + else: + env_metrics = None + + grad_stats, ued_grad_stats = None, None + + stats = self._compile_stats( + update_stats, ep_stats, + ued_update_stats, ued_ep_stats, + env_metrics, + grad_stats, ued_grad_stats, + shaped_reward_coeff=train_state.shaped_reward_coeff[0]) + stats.update(dict(n_updates=train_state.n_updates[0])) + + train_state = train_state.increment() + ued_train_state = ued_train_state.increment() + self.n_updates += 1 + + return ( + stats, + rng, + train_state, state, obs, actor_carry, critic_carry, ep_stats, + ued_train_state, ued_state, ued_obs, ued_carry, ued_ep_stats, reset_state + ) + + def _concat_multi_agent_obs(self, obs: Dict[str, chex.Array]) -> chex.Array: + """Concatenates a obs dictionary that was built for two env agents. + Doubles the number of parallel_envs, i.e. (num_students, n_parallel, ...) -> (num_students, 2*n_parallel, ...) + """ + return jnp.concatenate([obs['agent_0'][:, :, jnp.newaxis, :], obs['agent_1'][:, :, jnp.newaxis, :]], axis=2) + + # def _double_world_state(self, world_state: chex.Array) -> chex.Array: + # """Doubles the world state to simulate two agents. + # Doubles the number of parallel_envs, i.e. (num_students, n_parallel, ...) -> (num_students, 2*n_parallel, ...) + # """ + # return jnp.concatenate([world_state[:, :, jnp.newaxis, :], world_state[:, :, jnp.newaxis, :]], axis=2) diff --git a/src/minimax/runners_ma/plr_runner.py b/src/minimax/runners_ma/plr_runner.py new file mode 100644 index 0000000..e6cf373 --- /dev/null +++ b/src/minimax/runners_ma/plr_runner.py @@ -0,0 +1,578 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial +from enum import Enum + +import einops +import numpy as np +import jax +import jax.numpy as jnp + +import minimax.envs as envs +from minimax.runners_ma.dr_runner import DRRunner +from minimax.util import pytree as _tree_util +from minimax.util.rl import ( + UEDScore, + compute_ued_scores, + PopPLRManager +) + + +class MutationCriterion(Enum): + BATCH = 0 + EASY = 1 + HARD = 2 + + +class PLRRunner(DRRunner): + def __init__( + self, + *, + replay_prob=0.5, + buffer_size=100, + staleness_coef=0.3, + use_score_ranks=True, + temp=1.0, + min_fill_ratio=0.5, + use_robust_plr=False, + use_parallel_eval=False, + ued_score='l1_value_loss', + force_unique=False, # Slower if True + mutation_fn=None, + n_mutations=0, + mutation_criterion='batch', + mutation_subsample_size=1, + **kwargs): + use_mutations = mutation_fn is not None + if use_parallel_eval: + replay_prob = 1.0 # Replay every rollout cycle + # Force batch mutations (no UED scores) + mutation_criterion = 'batch' + self._n_parallel_batches = 3 if use_mutations else 2 + kwargs['n_parallel'] *= self._n_parallel_batches + + super().__init__(**kwargs) + + self.replay_prob = replay_prob + self.buffer_size = buffer_size + self.staleness_coef = staleness_coef + self.temp = temp + self.use_score_ranks = use_score_ranks + self.min_fill_ratio = min_fill_ratio + self.use_robust_plr = use_robust_plr + self.use_parallel_eval = use_parallel_eval + self.ued_score = UEDScore[ued_score.upper()] + + self.use_mutations = use_mutations + if self.use_mutations: + self.mutation_fn = envs.get_mutator( + self.benv.env_name, mutation_fn) + else: + self.mutation_fn = None + self.n_mutations = n_mutations + self.mutation_criterion = MutationCriterion[mutation_criterion.upper()] + self.mutation_subsample_size = mutation_subsample_size + + self.force_unique = force_unique + if force_unique: + self.comparator_fn = envs.get_comparator(self.benv.env_name) + else: + self.comparator_fn = None + + if mutation_fn is not None and mutation_criterion != 'batch': + assert self.n_parallel % self.mutation_subsample_size == 0, \ + 'Number of parallel envs must be divisible by mutation subsample size.' + + def reset(self, rng): + runner_state = list(super().reset(rng)) + rng = runner_state[0] + runner_state[0], subrng = jax.random.split(rng) + example_state = self.benv.env.reset(rng)[1] + + self.plr_mgr = PopPLRManager( + n_agents=self.n_students, + example_level=example_state, + ued_score=self.ued_score, + replay_prob=self.replay_prob, + buffer_size=self.buffer_size, + staleness_coef=self.staleness_coef, + temp=self.temp, + use_score_ranks=self.use_score_ranks, + min_fill_ratio=self.min_fill_ratio, + use_robust_plr=self.use_robust_plr, + use_parallel_eval=self.use_parallel_eval, + comparator_fn=self.comparator_fn, + n_devices=self.n_devices + ) + plr_buffer = self.plr_mgr.reset(self.n_students) + + train_state = runner_state[1] + train_state = train_state.replace(plr_buffer=plr_buffer) + if self.n_devices == 1: + runner_state[1] = train_state + else: + plr_buffer = jax.tree_map(lambda x: x.repeat( + self.n_devices, 1), plr_buffer) # replicate plr buffer + # Return PLR buffer directly to make shmap easier + runner_state += (plr_buffer,) + + self.dummy_eval_output = self._create_dummy_eval_output(train_state) + + return tuple(runner_state) + + def _create_dummy_eval_output(self, train_state): + rng, * \ + vrngs = jax.random.split(jax.random.PRNGKey(0), self.n_students+1) + obs, state, extra = self.benv.reset(jnp.array(vrngs)) + + ep_stats = self.rolling_stats.reset_stats( + batch_shape=(self.n_students, self.n_parallel*self.n_eval)) + + ued_scores = jnp.zeros((self.n_students, self.n_parallel)) + + if self.student_pop.agent.is_recurrent: + actor_carry, critic_carry = self.zero_carry, self.zero_carry + else: + actor_carry, critic_carry = None, None + rollout = self.student_rollout.reset() + + # Map over the n_env_agents dimension in this multi agent rl setting. + # Dimensions are (n_students, n_parallel, n_env_agents, ...) + value, _ = jax.vmap(self.student_pop.get_value, in_axes=(None, 2, 2))( + jax.lax.stop_gradient(train_state.critic_params), + self._concat_multi_agent_obs(obs["world_state"]), + critic_carry, + ) + + value = einops.rearrange( + value, + "n_env_agents n_students n_parallel -> n_students n_parallel n_env_agents") + + jax.debug.print( + "train_state.shaped_reward_coeff {s}", s=train_state.shaped_reward_coeff) + + train_batch = self.student_rollout.get_batch( + rollout, value, train_state.shaped_reward_coeff + ) + + return ( + rng, + train_state, + state, + state, + obs, + actor_carry, + critic_carry, + extra, + ep_stats, + state, + train_batch, + ued_scores + ) + + @partial(jax.jit, static_argnums=(0, 8)) + def _eval_and_update_plr( + self, + rng, + levels, + level_idxs, + train_state, + update_plr, + parent_idxs=None, + dupe_mask=None, + fake=False): + # Collect rollout and optionally update plr buffer + # Returns train_batch and ued_scores + if fake: + dummy_eval_output = list(self.dummy_eval_output) + dummy_eval_output[1] = train_state + return tuple(dummy_eval_output) + + rollout_batch_shape = (self.n_students, self.n_parallel*self.n_eval) + obs, state, extra = self.benv.set_state(levels) + ep_stats = self.rolling_stats.reset_stats( + batch_shape=rollout_batch_shape) + + rollout_start_state = state + + done = jnp.zeros(rollout_batch_shape, dtype=jnp.bool_) + if self.student_pop.agent.is_recurrent: + actor_carry = self.zero_carry + critic_carry = self.zero_carry + else: + actor_carry, critic_carry = None, None + + rng, subrng = jax.random.split(rng) + start_state = state + rollout, state, start_state, obs, actor_carry, critic_carry, extra, ep_stats, train_state = \ + self._rollout_students( + subrng, + train_state, + state, + start_state, + obs, + actor_carry, + critic_carry, + done, + extra, + ep_stats + ) + + reward = rollout["rewards"].sum(axis=1).mean(-1)[:, :, jnp.newaxis] + shaped_reward = rollout["shaped_rewards"].sum( + axis=1).mean(-1)[:, :, jnp.newaxis] + + ep_stats["reward"] = reward + ep_stats["shaped_reward"] = shaped_reward + ep_stats["shaped_reward_scaled_by_shaped_reward_coeff"] = shaped_reward * \ + train_state.shaped_reward_coeff + ep_stats["reward_p_shaped_reward_scaled"] = reward + shaped_reward * \ + train_state.shaped_reward_coeff + + shared_obs = self._concat_multi_agent_obs(obs['world_state']) + value, _ = jax.vmap(self.student_pop.get_value, in_axes=(None, 2, 2))( + jax.lax.stop_gradient(train_state.critic_params), + shared_obs, + critic_carry + ) + + value = einops.rearrange( + value, "n_env_agents n_students n_parallel -> n_students n_parallel n_env_agents") + train_batch = self.student_rollout.get_batch( + rollout, + value, + train_state.shaped_reward_coeff + ) + + # Update PLR buffer + if self.ued_score == UEDScore.MAX_MC: + max_returns = jax.vmap(lambda x, y: x.at[y].get())( + train_state.plr_buffer.max_returns, level_idxs) + max_returns = jnp.where( + jnp.greater_equal(level_idxs, 0), + max_returns, + jnp.full_like(max_returns, -jnp.inf) + ) + ued_info = {'max_returns': max_returns} + else: + ued_info = None + ued_scores, ued_score_info = compute_ued_scores( + self.ued_score, train_batch, self.n_eval, info=ued_info, ignore_val=-jnp.inf, per_agent=True) + next_plr_buffer = self.plr_mgr.update( + train_state.plr_buffer, + levels=levels, + level_idxs=level_idxs, + ued_scores=ued_scores, + dupe_mask=dupe_mask, + info=ued_score_info, + ignore_val=-jnp.inf, + parent_idxs=parent_idxs) + + next_plr_buffer = jax.vmap( + lambda update, new, prev: jax.tree_map( + lambda x, y: jax.lax.select(update, x, y), new, prev) + )(update_plr, next_plr_buffer, train_state.plr_buffer) + + train_state = train_state.replace(plr_buffer=next_plr_buffer) + + return ( + rng, + train_state, + state, + start_state, + obs, + actor_carry, + critic_carry, + extra, + ep_stats, + rollout_start_state, + train_batch, + ued_scores, + ) + + @partial(jax.jit, static_argnums=(0,)) + def _mutate_levels(self, rng, levels, level_idxs, ued_scores=None): + if not self.use_mutations: + return levels, level_idxs, jnp.full_like(level_idxs, -1) + + def upsample_levels(levels, level_idxs, subsample_idxs): + subsample_idxs = subsample_idxs.repeat( + self.n_parallel//self.mutation_subsample_size, -1) + parent_idxs = level_idxs.take(subsample_idxs) + levels = jax.vmap( + lambda x, y: jax.tree_map( + lambda _x: jnp.array(_x).take(y, 0), x) + )(levels, parent_idxs) + + return levels, parent_idxs + + if self.mutation_criterion == MutationCriterion.BATCH: + parent_idxs = level_idxs + + if self.mutation_criterion == MutationCriterion.EASY: + _, top_level_idxs = jax.lax.approx_min_k( + ued_scores, self.mutation_subsample_size) + levels, parent_idxs = upsample_levels( + levels, level_idxs, top_level_idxs) + + elif self.mutation_criterion == MutationCriterion.HARD: + _, top_level_idxs = jax.lax.approx_max_k( + ued_scores, self.mutation_subsample_size) + levels, parent_idxs = upsample_levels( + levels, level_idxs, top_level_idxs) + + n_parallel = level_idxs.shape[-1] + vrngs = jax.vmap(lambda subrng: jax.random.split(subrng, n_parallel))( + jax.random.split(rng, self.n_students) + ) + + mutated_levels = jax.vmap( + lambda *args: jax.vmap(self.mutation_fn, + in_axes=(0, None, 0, None))(*args), + in_axes=(0, None, 0, None) + )(vrngs, self.benv.env_params, levels, self.n_mutations) + + # Mutated levels do not have existing idxs in the PLR buffer. + mutated_level_idxs = jnp.full((self.n_students, n_parallel), -1) + + return mutated_levels, mutated_level_idxs, parent_idxs + + def _efficient_grad_update(self, rng, train_state, train_batch, is_replay): + # PPOAgent vmaps over the train state and batch. Batch must be N x EM + skip_grad_update = jnp.logical_and(self.use_robust_plr, ~is_replay) + + if self.n_students == 1: + train_state, stats = jax.lax.cond( + skip_grad_update[0], + partial(self.student_pop.update, fake=True), + self.student_pop.update, + *(rng, train_state, train_batch) + ) + elif self.n_students > 1: # Have to vmap all students + take only students that need updates + _, dummy_stats = jax.vmap( + lambda *_: self.student_pop.agent.get_empty_update_stats())(np.arange(self.n_students)) + _train_state, stats = self.student.update( + rng, train_state, train_batch) + train_state, stats = jax.vmap(lambda cond, x, y: + jax.tree_map(lambda _cond, _x, _y: jax.lax.select(_cond, _x, _y), cond, x, y))( + is_replay, (train_state, + stats), (_train_state, dummy_stats) + ) + + return train_state, stats + + @partial(jax.jit, static_argnums=(0,)) + def _compile_stats(self, update_stats, ep_stats, env_metrics=None, plr_stats=None, shaped_reward_coeff=None): + stats = super()._compile_stats(update_stats, ep_stats, env_metrics) + + if plr_stats is not None: + plr_stats = jax.vmap(lambda info: jax.tree_map( + lambda x: x.mean(), info))(plr_stats) + + if shaped_reward_coeff is not None: + stats['shaped_reward_coeff'] = shaped_reward_coeff + + if self.n_students > 1: + _plr_stats = {} + for i in range(self.n_students): + _student_plr_stats = jax.tree_util.tree_map( + lambda x: x[i], plr_stats) # for agent0 + _plr_stats.update( + {f'{k}_a{i}': v for k, v in _student_plr_stats.items()}) + plr_stats = _plr_stats + else: + plr_stats = jax.tree_map(lambda x: x[0], plr_stats) + + stats.update({f'plr_{k}': v for k, v in plr_stats.items()}) + + if self.n_devices > 1: + stats = jax.tree_map(lambda x: jax.lax.pmean(x, 'device'), stats) + + return stats + + @partial(jax.jit, static_argnums=(0,)) + def run( + self, + rng, + train_state, + state, + start_state, + obs, + carry=None, + extra=None, + ep_stats=None, + plr_buffer=None): + # If device sharded, load sharded PLR buffer into train state + if self.n_devices > 1: + rng = jax.random.fold_in(rng, jax.lax.axis_index('device')) + train_state = train_state.replace(plr_buffer=plr_buffer) + + # Sample next training levels via PLR + rng, *vrngs = jax.random.split(rng, self.n_students+1) + obs, state, extra = self.benv.reset( + jnp.array(vrngs), self.n_parallel, 1) + + if self.use_parallel_eval: + n_level_samples = self.n_parallel//self._n_parallel_batches + new_levels = jax.tree_map( + lambda x: x.at[:, n_level_samples:2*n_level_samples].get(), state) + else: + n_level_samples = self.n_parallel + new_levels = state + + rng, subrng = jax.random.split(rng) + levels, level_idxs, is_replay, next_plr_buffer = \ + self.plr_mgr.sample(subrng, train_state.plr_buffer, + new_levels, n_level_samples) + train_state = train_state.replace(plr_buffer=next_plr_buffer) + + # If use_parallel_eval=True, need to combine replay and non-replay levels together + # Need to mutate levels as well + parent_idxs = jnp.full((self.n_students, self.n_parallel), -1) + if self.use_parallel_eval: # Parallel ACCEL + new_level_idxs = jnp.full_like(parent_idxs, -1) + + _all_levels = jax.vmap( + lambda x, y: _tree_util.pytree_merge( + x, y, start_idx=n_level_samples, src_len=n_level_samples), + )(state, levels) + _all_level_idxs = jax.vmap( + lambda x, y: _tree_util.pytree_merge( + x, y, start_idx=n_level_samples, src_len=n_level_samples), + )(new_level_idxs, level_idxs) + + if self.use_mutations: + rng, subrng = jax.random.split(rng) + mutated_levels, mutated_level_idxs, _parent_idxs = self._mutate_levels( + subrng, levels, level_idxs) + + fallback_levels = jax.tree_map( + lambda x: x.at[:, -n_level_samples:].get(), state) + fallback_level_idxs = jnp.full_like(mutated_level_idxs, -1) + + mutated_levels = jax.vmap( + lambda cond, x, y: jax.tree_map( + lambda _x, _y: jax.lax.select(cond, _x, _y), x, y + ))(is_replay, mutated_levels, fallback_levels) + + mutated_level_idxs = jax.vmap( + lambda cond, x, y: jax.tree_map( + lambda _x, _y: jax.lax.select(cond, _x, _y), x, y + ))(is_replay, mutated_level_idxs, fallback_level_idxs) + + _parent_idxs = jax.vmap( + lambda cond, x, y: jax.tree_map( + lambda _x, _y: jax.lax.select(cond, _x, _y), x, y + ))(is_replay, _parent_idxs, fallback_level_idxs) + + mutated_levels_start_idx = 2*n_level_samples + _all_levels = jax.vmap( + lambda x, y: _tree_util.pytree_merge( + x, y, start_idx=mutated_levels_start_idx, src_len=n_level_samples), + )(_all_levels, mutated_levels) + _all_level_idxs = jax.vmap( + lambda x, y: _tree_util.pytree_merge( + x, y, start_idx=mutated_levels_start_idx, src_len=n_level_samples), + )(_all_level_idxs, mutated_level_idxs) + parent_idxs = jax.vmap( + lambda x, y: _tree_util.pytree_merge( + x, y, start_idx=mutated_levels_start_idx, src_len=n_level_samples), + )(parent_idxs, _parent_idxs) + + levels = _all_levels + level_idxs = _all_level_idxs + + # dedupe levels, move into PLR buffer logic + if self.force_unique: + level_idxs, dupe_mask = self.plr_mgr.dedupe_levels( + next_plr_buffer, levels, level_idxs) + else: + dupe_mask = None + + # Evaluate levels + update PLR + result = self._eval_and_update_plr( + rng, levels, level_idxs, train_state, update_plr=jnp.array([True]*self.n_students), parent_idxs=parent_idxs, dupe_mask=dupe_mask) + rng, train_state, state, start_state, obs, actor_carry, critic_carry, extra, ep_stats, \ + rollout_start_state, train_batch, ued_scores = result + + if self.use_parallel_eval: + replay_start_idx = self.n_eval*n_level_samples + replay_end_idx = 2*replay_start_idx + train_batch = jax.vmap( + lambda x: jax.tree_map( + lambda _x: _x.at[:, replay_start_idx:replay_end_idx].get(), x) + )(train_batch) + + # Gradient update + rng, subrng = jax.random.split(rng) + train_state, update_stats = self._efficient_grad_update( + subrng, train_state, train_batch, is_replay) + + # Mutation step + use_mutations = jnp.logical_and(self.use_mutations, is_replay) + # Already mutated above in parallel + use_mutations = jnp.logical_and( + use_mutations, not self.use_parallel_eval) + rng, arng, brng = jax.random.split(rng, 3) + + mutated_levels, mutated_level_idxs, parent_idxs = jax.lax.cond( + use_mutations.any(), + self._mutate_levels, + lambda *_: (levels, level_idxs, jnp.full_like(level_idxs, -1)), + *(arng, levels, level_idxs, ued_scores) + ) + + mutated_dupe_mask = jnp.zeros_like(mutated_level_idxs, dtype=jnp.bool_) + if self.force_unique: # Should move into update plr logic + mutated_level_idxs, mutated_dupe_mask = jax.lax.cond( + use_mutations.any(), + self.plr_mgr.dedupe_levels, + lambda *_: (mutated_level_idxs, mutated_dupe_mask), + *(next_plr_buffer, mutated_levels, mutated_level_idxs) + ) + + mutation_eval_result = jax.lax.cond( + use_mutations.any(), + self._eval_and_update_plr, + partial(self._eval_and_update_plr, fake=True), + *(brng, mutated_levels, mutated_level_idxs, train_state, use_mutations, parent_idxs, mutated_dupe_mask) + ) + train_state = mutation_eval_result[1] + + # Collect training env metrics + if self.track_env_metrics: + env_metrics = self.benv.get_env_metrics(levels) + else: + env_metrics = None + + plr_stats = self.plr_mgr.get_metrics(train_state.plr_buffer) + + stats = self._compile_stats( + update_stats, ep_stats, env_metrics, plr_stats, shaped_reward_coeff=train_state.shaped_reward_coeff) + + if self.n_devices > 1: + plr_buffer = train_state.plr_buffer + train_state = train_state.replace(plr_buffer=None) + + train_state = train_state.increment() + stats.update(dict(n_updates=train_state.n_updates[0])) + + return ( + stats, + rng, + train_state, + state, + start_state, + obs, + carry, + extra, + ep_stats, + plr_buffer, + rollout_start_state, + ) diff --git a/src/minimax/runners_ma/xp_runner.py b/src/minimax/runners_ma/xp_runner.py new file mode 100644 index 0000000..9600280 --- /dev/null +++ b/src/minimax/runners_ma/xp_runner.py @@ -0,0 +1,377 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial +import os +import time + +import imageio +import numpy as np +import jax +import jax.numpy as jnp +from jax.experimental import mesh_utils +from jax.experimental.shard_map import shard_map + + +from .eval_runner import EvalRunner +from .dr_runner import DRRunner +from .paired_runner import PAIREDRunner +from .plr_runner import PLRRunner +import minimax.envs as envs +import minimax.models as models +import minimax.agents as agents +from minimax.envs.viz.overcooked_visualizer import OvercookedVisualizer + + +class RunnerInfo: + def __init__( + self, + runner_cls, + is_ued=False): + self.runner_cls = runner_cls + self.is_ued = is_ued + + +RUNNER_INFO = { + 'dr': RunnerInfo( + runner_cls=DRRunner, + ), + 'plr': RunnerInfo( + runner_cls=PLRRunner, + ), + 'paired': RunnerInfo( + runner_cls=PAIREDRunner, + is_ued=True + ) +} + + +class ExperimentRunner: + def __init__( + self, + train_runner, + env_name, + agent_rl_algo, + student_model_name, + student_critic_model_name, + student_agent_kind="mappo", + teacher_model_name=None, + train_runner_kwargs={}, + env_kwargs={}, + ued_env_kwargs={}, + student_rl_kwargs={}, + teacher_rl_kwargs={}, + student_model_kwargs={}, + teacher_model_kwargs={}, + eval_kwargs={}, + eval_env_kwargs={}, + n_devices=1, + shaped_reward_steps=0, + shaped_reward_n_updates=0, + xpid=None + ): + self.env_name = env_name + self.agent_rl_algo = agent_rl_algo + self.is_ued = RUNNER_INFO[train_runner].is_ued + self.xpid = xpid + self.shaped_reward_steps = shaped_reward_steps + self.shaped_reward_n_updates = shaped_reward_n_updates + + dummy_env = envs.make( + env_name, + env_kwargs, + ued_env_kwargs)[0] + + # ---- Make agent ---- + if student_agent_kind == 'mappo': + student_model_kwargs['output_dim'] = dummy_env.action_space().n + student_actor = models.make( + env_name=env_name, + model_name=student_model_name, + **student_model_kwargs + ) + + student_model_kwargs['output_dim'] = 1 + student_critic = models.make( + env_name=env_name, + model_name=student_critic_model_name, + **student_model_kwargs + ) + + student_agent = agents.MAPPOAgent( + actor=student_actor, + critic=student_critic, + n_devices=n_devices, + **student_rl_kwargs + ) + else: + raise ValueError( + f"Unknown student agent kind: {student_agent_kind}") + + # ---- Handle UED-related settings ---- + if self.is_ued: + max_teacher_steps = dummy_env.ued_max_episode_steps() + teacher_model_kwargs['n_scalar_embeddings'] = max_teacher_steps + teacher_model_kwargs['max_scalar'] = max_teacher_steps + teacher_model_kwargs['output_dim'] = dummy_env.ued_action_space().n + + teacher_model = models.make( + env_name=env_name, + model_name=teacher_model_name, + **teacher_model_kwargs + ) + + teacher_agent = agents.PPOAgent( + model=teacher_model, + n_devices=n_devices, + **teacher_rl_kwargs + ) + + train_runner_kwargs.update(dict( + teacher_agents=[teacher_agent] + )) + train_runner_kwargs.update(dict( + ued_env_kwargs=ued_env_kwargs + )) + + # Debug, tabulate student and teacher model + # import jax.numpy as jnp + # dummy_rng = jax.random.PRNGKey(0) + # obs, _ = dummy_env.reset(dummy_rng) + # # hx = student_actor.initialize_carry(dummy_rng, (1,)) + # ued_obs, _ = dummy_env.reset_teacher(dummy_rng) + # # ued_hx = teacher_model.initialize_carry(dummy_rng, (1,)) + + # obs['image'] = jnp.expand_dims(obs['image'], 0) + # ued_obs['image'] = jnp.expand_dims(ued_obs['image'], 0) + + # print(student_actor.tabulate(dummy_rng, obs, None)) + # print(teacher_model.tabulate(dummy_rng, ued_obs, None)) + + # import pdb + # pdb.set_trace() + + # ---- Set up train runner ---- + runner_cls = RUNNER_INFO[train_runner].runner_cls + + # Set up learning rate annealing parameters + lr_init = train_runner_kwargs.lr + lr_final = train_runner_kwargs.lr_final + lr_anneal_steps = train_runner_kwargs.lr_anneal_steps + + if lr_final is None: + train_runner_kwargs.lr_final = lr_init + if train_runner_kwargs.lr_final == train_runner_kwargs.lr: + train_runner_kwargs.lr_anneal_steps = 0 + + use_shaped_reward = (shaped_reward_steps is not None and shaped_reward_steps > 0) or ( + shaped_reward_n_updates is not None and shaped_reward_n_updates > 0) + + self.runner = runner_cls( + env_name=env_name, + env_kwargs=env_kwargs, + student_agents=[student_agent], + student_agent_kind=student_agent_kind, + n_devices=n_devices, + shaped_reward=use_shaped_reward, + **train_runner_kwargs) + + # ---- Make eval runner ---- + if eval_kwargs.get('env_names') is None: + self.eval_runner = None + else: + self.eval_runner = EvalRunner( + pop=self.runner.student_pop, + env_kwargs=eval_env_kwargs, + **eval_kwargs) + + self._start_tick = 0 + + # ---- Set up device parallelism ---- + self.n_devices = n_devices + if n_devices > 1: + dummy_runner_state = self.reset_train_runner(jax.random.PRNGKey(0)) + self._shmap_run = self._make_shmap_run(dummy_runner_state) + else: + self._shmap_run = None + + @partial(jax.jit, static_argnums=(0,)) + def step(self, runner_state, evaluate=False): + if self.n_devices > 1: + run_fn = self._shmap_run + else: + run_fn = self.runner.run + + stats, *runner_state = run_fn(*runner_state) + + rng = runner_state[0] + rng, subrng = jax.random.split(rng) + + if self.eval_runner is not None: + params = runner_state[1].actor_params + eval_stats = jax.lax.cond( + evaluate, + self.eval_runner.run, + self.eval_runner.fake_run, + *(subrng, params) + ) + else: + eval_stats = {} + + return stats, eval_stats, rng, *runner_state[1:] + + def _make_shmap_run(self, runner_state): + devices = mesh_utils.create_device_mesh((self.n_devices,)) + mesh = Mesh(devices, axis_names=('device')) + + in_specs, out_specs = self.runner.get_shmap_spec() + + return partial(shard_map, + mesh=mesh, + in_specs=in_specs, + out_specs=out_specs, + check_rep=False + )(self.runner.run) + + def train( + self, + rng, + agent_algo='ppo', + algo_runner='dr', + n_total_updates=1000, + logger=None, + log_interval=1, + test_interval=1, + checkpoint_interval=0, + archive_interval=0, + archive_init_checkpoint=False, + from_last_checkpoint=False + ): + """ + Entry-point for training + """ + # Load checkpoint if any + runner_state = self.runner.reset(rng) + + if from_last_checkpoint: + last_checkpoint_state = logger.load_last_checkpoint_state() + if last_checkpoint_state is not None: + runner_state = self.runner.load_checkpoint_state( + runner_state, + last_checkpoint_state + ) + self._start_tick = runner_state[1].n_iters[0] + + # Archive initialization weights if necessary + if archive_init_checkpoint: + logger.checkpoint( + self.runner.get_checkpoint_state(runner_state), + index=0, + archive_interval=1) + + # Train loop + log_on = logger is not None and log_interval > 0 + checkpoint_on = checkpoint_interval > 0 or archive_interval > 0 + train_state = runner_state[1] + + tick = self._start_tick + train_steps = tick*self.runner.step_batch_size * \ + self.runner.n_rollout_steps*self.n_devices + real_train_steps = train_steps//self.runner.n_students + + while (train_state.n_updates < n_total_updates).any(): + evaluate = test_interval > 0 and (tick+1) % test_interval == 0 + + start = time.time() + stats, eval_stats, *runner_state = \ + self.step(runner_state, evaluate) + end = time.time() + + start_state = runner_state[-1] + runner_state = runner_state[:-1] + + if evaluate: + stats.update(eval_stats) + else: + stats.update({k: None for k in eval_stats.keys()}) + + train_state = runner_state[1] + + dsteps = self.runner.step_batch_size*self.runner.n_rollout_steps*self.n_devices + real_dsteps = dsteps//self.runner.n_students + train_steps += dsteps + real_train_steps += real_dsteps + + if (self.shaped_reward_steps is not None and self.shaped_reward_steps > 0) or (self.shaped_reward_n_updates is not None and self.shaped_reward_n_updates > 0): + if self.shaped_reward_n_updates: # Meassure based on n_updates + new_shaped_reward_coeff_value = max( + 0.0, 1.0 - (train_state.n_updates[0]/self.shaped_reward_n_updates)) + else: # Meassure based on steps in the env + new_shaped_reward_coeff_value = max( + 0.0, 1.0 - (real_train_steps/self.shaped_reward_steps)) + + new_shaped_reward_coeff = jnp.full( + runner_state[1].shaped_reward_coeff.shape, fill_value=new_shaped_reward_coeff_value) + jax.debug.print("Shaped reward coeff: {a}, real_dsteps: {b}, shaped_reward_steps: {c}", + a=new_shaped_reward_coeff, b=real_dsteps, c=self.shaped_reward_steps) + # runner_state[1] is the training state object where the shaped reward coefficient is stored + runner_state[1] = runner_state[1].set_new_shaped_reward_coeff( + new_shaped_reward_coeff) + + sps = int(dsteps/(end-start)) + real_sps = int(real_dsteps/(end-start)) + time_per_iter = float(end-start) + stats.update(dict( + steps=train_steps, + sps=sps, + real_steps=real_train_steps, + real_sps=real_sps, + time_per_iter=time_per_iter, + )) + + tick += 1 + + jax.debug.print("-----\n{stats}", stats=stats) + if log_on and tick % log_interval == 0: + logger.log(stats, tick, ignore_val=-np.inf) + + if checkpoint_on and tick > 0: + if tick % checkpoint_interval == 0 \ + or (archive_interval > 0 and tick % archive_interval == 0): + + maze_map = start_state.maze_map + agent_dir_idx = start_state.agent_dir_idx + agent_inv = start_state.agent_inv + for i in range(1): # self.runner.n_parallel + + padding = 4 # Fixed + grid = np.asarray( + maze_map[0, i, padding:-padding, padding:-padding, :]) + # Render the state + frame = OvercookedVisualizer._render_grid( + grid, + tile_size=32, + highlight_mask=None, + agent_dir_idx=agent_dir_idx[0][i], + agent_inv=agent_inv[0][i] + ) + + # Save the numpy frame as image + dir = f"{os.getcwd()}/overcooked_teacher_layout_imgs/{self.xpid}/" + + os.makedirs(os.path.dirname(dir), exist_ok=True) + imageio.imwrite( + dir + f"{tick}_{i}.png", frame) + + # Also produce an image of the teachers env output currently + checkpoint_state = \ + self.runner.get_checkpoint_state(runner_state) + logger.checkpoint( + checkpoint_state, + index=tick, + archive_interval=archive_interval) diff --git a/src/minimax/tests/__init__.py b/src/minimax/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/minimax/tests/base_req_rollout_storage.py b/src/minimax/tests/base_req_rollout_storage.py new file mode 100644 index 0000000..f4f1f31 --- /dev/null +++ b/src/minimax/tests/base_req_rollout_storage.py @@ -0,0 +1,116 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +import pytest +import numpy as np +import jax +import jax.numpy as jnp + +from util.rl import RolloutStorage +import envs +import models +import agents + + +class RequiresRolloutStorageTestClass: + def setup_class(self): + # Create maze object + env_name = 'Maze' + self.env, self.env_params = envs.make(env_name) + + self.n_steps = 32 + self.n_agents = 2 + self.n_envs = 3 + self.n_eval = 5 + self.rnn_dim = 4 + self.discount = 0.995 + self.gae_lambda = 0.95 + + self.batch_shape = ( + self.n_agents, + self.n_steps, + self.n_envs*self.n_eval, + ) + + self.t_batch_shape = ( + self.n_agents, + self.n_envs*self.n_eval, + ) + + # Create dummy agent + self.agent_model = models.make( + env_name=env_name, + model_name='default_student_cnn', + recurrent_arch='lstm', + recurrent_hidden_dim=self.rnn_dim + ) + + self.agent = agents.PPOAgent( + model=self.agent_model, + ) + + dummy_rng = jax.random.PRNGKey(0) + self.zero_carry_t = \ + self.agent.init_carry( + dummy_rng, + batch_dims=( + self.n_agents, + self.n_envs*self.n_eval) + ) + + # Initialize RolloutStorage obj + self.rollout_mgr = RolloutStorage( + discount=self.discount, + gae_lambda=self.gae_lambda, + n_envs=self.n_envs, + n_eval=self.n_eval, + n_steps=self.n_steps, + action_space=self.env.action_space(), + obs_space=self.env.observation_space(), + agent=self.agent, + n_agents=self.n_agents + ) + + def setup_method(self): + pass + + def _get_dummy_step_components(self): + t_batch_shape = self.t_batch_shape + + dummy_rng = jax.random.PRNGKey(0) + obs, state = self.env.reset(dummy_rng) + obs = jax.tree_util.tree_map(lambda x: x.repeat( + np.prod(t_batch_shape) + ).reshape( + *t_batch_shape, *x.shape + ), obs + ) + action = jnp.ones( + (*t_batch_shape, *self.env.action_space().shape), + dtype=self.env.action_space().dtype) + + done = jnp.ones( + (*t_batch_shape, *self.env.action_space().shape), dtype=jnp.uint8) + + reward = jnp.ones(t_batch_shape, dtype=jnp.float32) + + log_pis_old = jnp.ones(t_batch_shape, dtype=jnp.float32) + + values_old = jnp.ones(t_batch_shape, dtype=jnp.float32) + + carry = self.zero_carry_t + + return ( + obs, + action, + reward, + done, + log_pis_old, + values_old, + carry + ) \ No newline at end of file diff --git a/src/minimax/tests/dummy_test_envs.py b/src/minimax/tests/dummy_test_envs.py new file mode 100644 index 0000000..5c0f32d --- /dev/null +++ b/src/minimax/tests/dummy_test_envs.py @@ -0,0 +1,130 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from collections import OrderedDict +from typing import Tuple, Optional + +import jax +from flax import struct +import chex + +from envs import environment +from envs import spaces +from envs.registration import register, register_ued + + +@struct.dataclass +class EnvState: + time: int = 0 + terminal: bool = False + +@struct.dataclass +class EnvParams: + reward_per_step: int = 1.0 + max_episode_steps: int = 250 + + +class DummyRewardEnv(environment.Environment): + def __init__(self, reward_per_step=1.0, max_episode_steps=250): + self.reward_per_step = 1.0 + + self.params = EnvParams( + reward_per_step=reward_per_step, + max_episode_steps=max_episode_steps + ) + + @staticmethod + def align_kwargs(kwargs, other_kwargs): + return kwargs + + def reset_env( + self, + key: chex.PRNGKey, + ) -> Tuple[chex.ArrayTree, EnvState]: + state = EnvState( + time=0, + terminal=False + ) + + return self.get_obs(state), state + + def step_env( + self, + key: chex.PRNGKey, + state: EnvState, + action: int, + ) -> Tuple[chex.Array, EnvState, float, bool, dict]: + next_time = state.time + 1 + done = next_time >= self.params.max_episode_steps + + state = state.replace( + time=next_time, + terminal=done + ) + + return ( + jax.lax.stop_gradient(self.get_obs(state)), + jax.lax.stop_gradient(state), + self.params.reward_per_step, + done, + {}, + ) + + def get_obs(self, state: EnvState) -> chex.ArrayTree: + return OrderedDict(dict(time=state.time)) + + @property + def default_params(self) -> EnvParams: + return EnvParams() + + @property + def name(self) -> str: + return "DummyRewardEnv" + + @property + def num_actions(self) -> int: + return len(self.action_set) + + def action_space(self) -> spaces.Discrete: + return spaces.Discrete(1) + + def observation_space(self) -> spaces.Dict: + return spaces.Dict({ + "time": spaces.Discrete(self.params.max_episode_steps), + }) + + def state_space(self) -> spaces.Dict: + return spaces.Dict({ + "time": spaces.Discrete(self.params.max_episode_steps), + "terminal": spaces.Discrete(2), + }) + + def max_episode_steps(self) -> int: + return self.params.max_episode_steps + + # UED-specific + def get_env_instance( + self, + key: chex.PRNGKey, + state: EnvState + ) -> chex.ArrayTree: + return state + + def set_env_instance(self, encoding: chex.ArrayTree): + state = encoding + return self.get_obs(state), state + + +# Register the env as its own UED env +if hasattr(__loader__, 'name'): + module_path = __loader__.name +elif hasattr(__loader__, 'fullname'): + module_path = __loader__.fullname + +register(env_id='DummyRewardEnv', entry_point=module_path + ':DummyRewardEnv') +register_ued(env_id='DummyRewardEnv', entry_point=module_path + ':DummyRewardEnv') diff --git a/src/minimax/tests/test_rollout_storage.py b/src/minimax/tests/test_rollout_storage.py new file mode 100644 index 0000000..3fdd125 --- /dev/null +++ b/src/minimax/tests/test_rollout_storage.py @@ -0,0 +1,163 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +import pytest +import numpy as np +import jax +import jax.numpy as jnp + +from tests.base_req_rollout_storage import RequiresRolloutStorageTestClass + + +class TestRolloutStorage(RequiresRolloutStorageTestClass): + def test_reset(self): + rollout = self.rollout_mgr.reset() + + batch_shape = self.batch_shape + obs_space = self.env.observation_space() + for k,v in rollout['obs'].items(): + assert rollout['obs'][k].shape == ( + *batch_shape, + *obs_space.spaces[k].shape + ) + assert (rollout['obs'][k] > 0).sum() == 0 + + assert rollout['actions'].shape == ( + *batch_shape, + *self.env.action_space().shape + ) + + assert rollout['rewards'].shape == batch_shape + assert (rollout['rewards'] > 0).sum() == 0 + + assert rollout['dones'].shape == batch_shape + assert (rollout['dones'] > 0).sum() == 0 + + assert rollout['log_pis_old'].shape == batch_shape + assert (rollout['log_pis_old'] > 0).sum() == 0 + + assert rollout['values_old'].shape == batch_shape + assert (rollout['values_old'] > 0).sum() == 0 + + assert rollout['_t'].shape == (self.n_agents,) + assert (rollout['_t'] > 0).sum() == 0 + + assert rollout['carry'][0].shape == (*batch_shape, self.rnn_dim) + assert rollout['carry'][1].shape == (*batch_shape, self.rnn_dim) + assert jax.tree_util.tree_structure(rollout['carry'][0].at[0,0,0].get()) \ + == jax.tree_util.tree_structure(self.zero_carry_t[0]) + assert jax.tree_util.tree_structure(rollout['carry'][1].at[0,0,0].get()) \ + == jax.tree_util.tree_structure(self.zero_carry_t[1]) + assert (rollout['carry'][0] > 0).sum() == 0 + assert (rollout['carry'][1] > 0).sum() == 0 + + def test_append(self): + # Make sure appending the full rollout length looks right + n_appends = 10 + + t_batch_shape = self.t_batch_shape + + rollout = self.rollout_mgr.reset() + + step = self._get_dummy_step_components() + + for i in range(n_appends): + rollout = self.rollout_mgr.append(rollout, *step) + + t_batch_size = np.prod(t_batch_shape) + + assert rollout['actions'].sum() == t_batch_size*n_appends + assert rollout['rewards'].sum() == t_batch_size*n_appends + assert rollout['dones'].sum() == t_batch_size*n_appends + assert rollout['log_pis_old'].sum() == t_batch_size*n_appends + assert rollout['values_old'].sum() == t_batch_size*n_appends + assert rollout['_t'].mean() == n_appends + + t_overshoot = 2 + for t in range(self.n_steps - n_appends + t_overshoot): + rollout = self.rollout_mgr.append(rollout, *step) + + assert rollout['_t'].mean() == t_overshoot + + def test_compute_gae(self): + # Set up placeholder values + (obs, + action, + _, + _, + log_pi, + _, + carry) = self._get_dummy_step_components() + + # Mark episode done every 8 steps + batch_shape = self.batch_shape + done = jnp.zeros(batch_shape, dtype=jnp.uint8) + done = done.at[:,jnp.arange(4,self.n_steps,4),:].set(1) + + # Reward of 10 at the end of every episode + reward = done*10 + + # Predict 0.1 at every time step + value = jnp.ones(batch_shape)*0.1 + + # Last value is 1 + last_value = jnp.ones(self.t_batch_shape) + + advantages, targets = jax.vmap(self.rollout_mgr.compute_gae)( + value, reward, done, last_value + ) + + _advantages = jnp.zeros(batch_shape) + + next_value = last_value + next_advantage = np.zeros_like(advantages.at[:,0,:].get()) + for t in np.arange(self.n_steps)[::-1]: + _done = done.at[:,t,:].get() + cur_value = value.at[:,t,:].get() + td = reward.at[:,t,:].get() + self.discount*next_value*(1-_done) - cur_value + _advantages = \ + _advantages.at[:,t,:].set(td + self.discount*self.gae_lambda*(1-_done)*next_advantage) + next_advantage = _advantages.at[:,t,:].get() + next_value = cur_value + + _targets = _advantages + value + + assert (_advantages != advantages).sum() == 0 + assert (_targets != targets).sum() == 0 + + def test_get_batch(self): + rollout = self.rollout_mgr.reset() + step = self._get_dummy_step_components() + for i in range(self.n_steps): + rollout = self.rollout_mgr.append(rollout, *step) + + last_value = jnp.ones(self.t_batch_shape) + + batch = self.rollout_mgr.get_batch(rollout, last_value) + + for k,v in batch.obs.items(): + assert (batch.obs[k] != rollout['obs'][k]).sum() == 0 + + assert (batch.actions != rollout['actions']).sum() == 0 + assert (batch.dones != rollout['dones']).sum() == 0 + assert (batch.rewards != rollout['rewards']).sum() == 0 + assert (batch.log_pis != rollout['log_pis_old']).sum() == 0 + assert (batch.values != rollout['values_old']).sum() == 0 + + assert ((batch.advantages + batch.values) != batch.targets).sum() == 0 + + assert (batch.carry[0] != rollout['carry'][0]).sum() == 0 + assert (batch.carry[1] != rollout['carry'][1]).sum() == 0 + + def test_set_final_reward(self): + rollout = self.rollout_mgr.reset() + + final_reward = jnp.ones(self.t_batch_shape)*3 + rollout = self.rollout_mgr.set_final_reward(rollout, final_reward) + + assert (rollout['rewards'].at[:,-1,:].get() != final_reward).sum() == 0 diff --git a/src/minimax/tests/test_ued_scores.py b/src/minimax/tests/test_ued_scores.py new file mode 100644 index 0000000..1323a31 --- /dev/null +++ b/src/minimax/tests/test_ued_scores.py @@ -0,0 +1,74 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +import pytest +import numpy as np +import jax +import jax.numpy as jnp + +import util.rl.ued_scores as _ued_scores + +from tests.base_req_rollout_storage import RequiresRolloutStorageTestClass + + +class TestUEDScores(RequiresRolloutStorageTestClass): + def test_compute_ued_scores_returns(self): + (obs, + action, + done, + reward, + log_pi, + value, + carry) = self._get_dummy_step_components() + + # Mark episode done every 8 steps + batch_shape = self.batch_shape + dones = jnp.zeros(batch_shape, dtype=jnp.uint8) + dones = dones.at[:,jnp.arange(4,self.n_steps,4),:].set(1) + + # Reward of 10 at the end of every episode + rewards = jnp.zeros_like(dones, dtype=jnp.float32) + rewards = rewards.at[:,jnp.arange(4,self.n_steps,4),:].set( + jnp.arange(1,8,dtype=jnp.float32).reshape(1,7,1) + ) + + rollout = self.rollout_mgr.reset() + for t in range(self.n_steps): + rollout = self.rollout_mgr.append( + rollout, + obs, + action, + rewards.at[:,t,:].get(), + dones.at[:,t,:].get(), + log_pi, + value, + carry + ) + + score_name = _ued_scores.UEDScore.RETURN + last_value = jnp.zeros(self.t_batch_shape) + batch = self.rollout_mgr.get_batch(rollout, last_value) + ued_score, _ = _ued_scores.compute_ued_scores( + score_name, batch, n_eval=self.n_eval) + + n_agents, n_steps, flat_batch_size = batch.dones.shape + pop_batch_shape = (n_agents, n_steps, flat_batch_size//self.n_eval, self.n_eval) + batch = jax.tree_util.tree_map(lambda x: x.reshape(*pop_batch_shape, *x.shape[3:]), batch) + mean_env_returns_per_agent, max_env_returns_per_agent, _ = \ + jax.vmap(_ued_scores._compute_ued_scores, in_axes=(None, 0))( + score_name, batch + ) + + mean_return = mean_env_returns_per_agent.mean(0) + max_return = max_env_returns_per_agent.max(0) + + assert (mean_return != ued_score).sum() == 0 + + batch_size = self.n_agents*self.n_envs + assert mean_env_returns_per_agent.sum(0).sum(0) == jnp.arange(1,8).mean()*batch_size + assert max_env_returns_per_agent.sum(0).sum(0) == 7*batch_size diff --git a/src/minimax/tests/test_wrappers.py b/src/minimax/tests/test_wrappers.py new file mode 100644 index 0000000..19ee4de --- /dev/null +++ b/src/minimax/tests/test_wrappers.py @@ -0,0 +1,146 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +import pytest +import numpy as np +import jax +import jax.numpy as jnp + +import tests.dummy_test_envs as dummy_test_envs +import envs +from envs.environment_ued import UEDEnvironment +from envs.wrappers import EnvWrapper +from envs.wrappers import UEDEnvWrapper + + +class TestEnvWrapper: + def setup_class(self): + # Set up environment + wrapper + env_kwargs = dict( + reward_per_step=1, + max_episode_steps=3 + ) + env_kwargs = env_kwargs + ued_env_kwargs = env_kwargs + self.env_kwargs = env_kwargs + + self.env, self.env_params, self.ued_params = \ + envs.make( + 'DummyRewardEnv', + env_kwargs=env_kwargs, + ued_env_kwargs=ued_env_kwargs, + **self._get_wrappers() + ) + + self.dummy_rng = jax.random.PRNGKey(0) + + @staticmethod + def _get_wrappers(): + return { + 'wrappers':['env_wrapper'], + 'ued_wrappers':['ued_env_wrapper'] + } + + +class TestDefaultEnvWrapper(TestEnvWrapper): + def setup_class(self): + # Set up environment + wrapper + env_kwargs = dict( + reward_per_step=1, + max_episode_steps=2 + ) + env_kwargs = env_kwargs + ued_env_kwargs = env_kwargs + + self.env, self.env_params, self.ued_params = \ + envs.make( + 'DummyRewardEnv', + env_kwargs=env_kwargs, + ued_env_kwargs=ued_env_kwargs, + **self._get_wrappers() + ) + + self.dummy_rng = jax.random.PRNGKey(0) + + def test_base_env(self): + assert isinstance(self.env.base_env, UEDEnvironment) + + def test_reset_extra(self): + extra = self.env.reset_extra() + assert len(extra) == 0 + + def test_step(self): + obs, state, extra = self.env.reset(self.dummy_rng) + extra = self.env.step(self.dummy_rng, state, 0)[-1] + assert len(extra) == 0 + + def test_reset(self): + obs, state, extra = self.env.reset(self.dummy_rng) + assert len(extra) == 0 + + def test_reset_env_instance(self): + _, ued_state = self.env.ued_env.reset(self.dummy_rng) + instance = self.env.ued_env.get_env_instance(self.dummy_rng, ued_state) + extra = self.env.set_env_instance(instance)[-1] + assert len(extra) == 0 + + def reset_teacher(self): + out = self.env.reset_teacher(self.dummy_rng) + assert len(out) == 2 + + def step_teacher(self): + _, ued_state = self.env.reset_teacher(self.dummy_rng) + out = self.env.step_teacher(self.dummy_rng, ued_state, 0) + assert len(out) == 5 + + def reset_student(self): + _, ued_state = self.env.reset_teacher(self.dummy_rng) + _, state, extra = self.env.reset_student(self.dummy_rng, ued_state) + assert len(extra) == 0 + + +class TestMonitorReturnWrapper(TestEnvWrapper): + @staticmethod + def _get_wrappers(): + return { + 'wrappers':['monitor_return'] + } + + def test_wrap_level(self): + assert self.env._wrap_level == 1 + + def test_reset_extra(self): + extra = self.env.reset_extra() + assert len(extra) == 1 and extra['ep_return'] == 0 + + def test_get_monitored_metrics(self): + metrics = self.env.get_monitored_metrics() + assert len(metrics) == 1 and 'return' in metrics + + def test_reset(self): + _, _, extra = self.env.reset(self.dummy_rng) + assert len(extra) == 1 and extra['ep_return'] == 0 + + def test_step(self): + obs, state, extra = self.env.reset(self.dummy_rng) + + n_steps = 2 + return_ = 0 + for i in range(n_steps): + _, state, r, _, _, extra = self.env.step( + self.dummy_rng, state, 0, extra=extra) + return_ += r + + assert extra['ep_return'] == self.env_kwargs['reward_per_step']*n_steps + + # Finish the episode + _, state, r, _, info, extra = self.env.step( + self.dummy_rng, state, 0, extra=extra) + + assert extra['ep_return'] == 0 + assert info['return'] == self.env_kwargs['reward_per_step']*(n_steps+1) \ No newline at end of file diff --git a/src/minimax/train.py b/src/minimax/train.py new file mode 100644 index 0000000..99beadd --- /dev/null +++ b/src/minimax/train.py @@ -0,0 +1,97 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +import os +import copy + +# os.environ['XLA_FLAGS'] = '--xla_force_host_platform_device_count=2' +# os.environ['JAX_TRACEBACK_FILTERING'] = 'off' +# os.environ['XLA_PYTHON_CLIENT_MEM_FRACTION'] = '.70' + +import jax +import wandb + + +from minimax.util.loggers import Logger +from .arguments import parser + + +if __name__ == '__main__': + with jax.disable_jit(False): + args = parser.parse_args(preview=True) + + # === Setup the main runner === + _args = copy.deepcopy(args) # Mutable record of args + if _args.is_multi_agent: + from minimax.runners_ma import ExperimentRunner + else: + from minimax.runners import ExperimentRunner + + xp_runner = ExperimentRunner( + train_runner=_args.train_runner, + env_name=_args.env_name, + agent_rl_algo=_args.agent_rl_algo, + student_model_name=_args.student_model_name, + student_critic_model_name=_args.student_critic_model_name, + student_agent_kind=_args.student_agent_kind, + teacher_model_name=_args.teacher_model_name, + train_runner_kwargs=_args.train_runner_args, + env_kwargs=_args.env_args, + ued_env_kwargs=_args.ued_env_args, + student_rl_kwargs=_args.student_rl_args, + teacher_rl_kwargs=_args.teacher_rl_args, + student_model_kwargs=_args.student_model_args, + teacher_model_kwargs=_args.teacher_model_args, + eval_kwargs=_args.eval_args, + eval_env_kwargs=_args.eval_env_args, + n_devices=_args.n_devices, + shaped_reward_steps=_args.n_shaped_reward_steps, + shaped_reward_n_updates=_args.n_shaped_reward_updates, + xpid=args.xpid + ) + + # === Configure logging === + # Set up wandb + wandb_args = args.wandb_args + if wandb_args.base_url: + os.environ["WANDB_BASE_URL"] = wandb_args.base_url + # if wandb_args.api_key: + # os.environ["WANDB_API_KEY"] = wandb_args.api_key + if wandb_args.base_url: # and wandb_args.api_key: + os.environ["WANDB_CACHE_DIR"] = '~/.cache/wandb' + wandb.init(project=wandb_args.project, + entity=wandb_args.entity, + config=args, + name=args.xpid, + group=wandb_args.group, + mode=wandb_args.mode + ) + callback = wandb.log + else: + callback = None + + logger = Logger( + log_dir=args.log_dir, + xpid=args.xpid, + xp_args=args, + callback=callback, + from_last_checkpoint=args.from_last_checkpoint, + verbose=args.verbose) + + # === Start training === + rng = jax.random.PRNGKey(args.seed) + xp_runner.train( + rng=rng, + n_total_updates=args.n_total_updates, + logger=logger, + log_interval=args.log_interval, + test_interval=args.test_interval, + checkpoint_interval=args.checkpoint_interval, + archive_interval=args.archive_interval, + archive_init_checkpoint=args.archive_init_checkpoint, + from_last_checkpoint=args.from_last_checkpoint) diff --git a/src/minimax/util/__init__.py b/src/minimax/util/__init__.py new file mode 100644 index 0000000..915c09b --- /dev/null +++ b/src/minimax/util/__init__.py @@ -0,0 +1,9 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from .dotdict import * \ No newline at end of file diff --git a/src/minimax/util/args.py b/src/minimax/util/args.py new file mode 100644 index 0000000..a4ad4c2 --- /dev/null +++ b/src/minimax/util/args.py @@ -0,0 +1,20 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +import argparse + + +def str2bool(v): + if isinstance(v, bool): + return v + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') \ No newline at end of file diff --git a/src/minimax/util/checkpoint.py b/src/minimax/util/checkpoint.py new file mode 100644 index 0000000..717d459 --- /dev/null +++ b/src/minimax/util/checkpoint.py @@ -0,0 +1,74 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +import os +import shutil +import pickle +import json +from pathlib import Path + +from .dotdict import DefaultDotDict + + +def save_pkl_object(obj, path): + """Helper to store pickle objects.""" + output_file = Path(path) + output_file.parent.mkdir(exist_ok=True, parents=True) + + with open(path, "wb") as output: + # Overwrites any existing file. + pickle.dump(obj, output, pickle.HIGHEST_PROTOCOL) + + print(f"Stored checkpoint at {path}.") + + +def load_pkl_object(path: str): + """Helper to reload pickle objects.""" + with open(path, "rb") as input: + obj = pickle.load(input) + print(f"Loaded checkpoint from {path}.") + return obj + + +def safe_checkpoint( + state, + dir_path, + name, + index=None, + archive_interval=None): + savename = f'{name}.pkl' + tmp_savepath = f'{name}_tmp.pkl' + + save_path = os.path.join(dir_path, savename) + tmp_savepath = os.path.join(dir_path, tmp_savepath) + + save_pkl_object(state, tmp_savepath) + + # Rename + os.replace(tmp_savepath, save_path) + + # Archive if needed + if index is not None and archive_interval is not None and archive_interval > 0: + if index % archive_interval == 0: + archive_path = os.path.join(dir_path, f'{name}_{index}.pkl') + shutil.copy(save_path, archive_path) + + +def load_config(path: str): + with open(path) as meta_file: + _config = json.load(meta_file)['config'] + + config = {} + for k,v in _config.items(): + if isinstance(v, dict): + v = DefaultDotDict(v) + config[k] = v + + return DefaultDotDict(config) + + diff --git a/src/minimax/util/dotdict.py b/src/minimax/util/dotdict.py new file mode 100644 index 0000000..11cea5c --- /dev/null +++ b/src/minimax/util/dotdict.py @@ -0,0 +1,68 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +import copy + + +class DotDict(dict): + __getattr__ = dict.__getitem__ + __setattr__ = dict.__setitem__ + __delattr__ = dict.__delitem__ + + def __init__(self, dct): + for key, value in dct.items(): + if hasattr(value, 'keys'): + value = DotDict(value) + self[key] = value + + def __getstate__(self): + return self + + def __setstate__(self, state): + self.update(state) + self.__dict__ = self + + def __deepcopy__(self, memo): + return DotDict(copy.deepcopy(dict(self))) + + +class DefaultDotDict(dict): + __delattr__ = dict.__delitem__ + + def __init__(self, dct, default=None): + super().__init__(dct) + self._default = default + + def __getstate__(self): + return (self, self._default) + + def __setstate__(self, state): + self.update(state[0]) + self.__dict__ = self + self._default = state[1] + + def __setattr__(self, name, value): + if name == '_default': + super().__setattr__('_default', value) + else: + self.__setitem__(name, value) + + def __deepcopy__(self, memo): + return DefaultDotDict( + copy.deepcopy(dict(self)), + default=self._default + ) + + def __getattr__(self, name): + if name == '_default': + return self._default + else: + try: + return self.__getitem__(name) + except: + return self._default \ No newline at end of file diff --git a/src/minimax/util/graph.py b/src/minimax/util/graph.py new file mode 100644 index 0000000..34fcb35 --- /dev/null +++ b/src/minimax/util/graph.py @@ -0,0 +1,263 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial + +import jax +import jax.numpy as jnp +import numpy as np + + +@partial(jax.jit, static_argnums=(1,)) +def apsp(A, n=None): + """ + Compute APSP for adjacency matrix A + using Seidel's algorithm. + """ + if n is None: + n = A.shape[0] + assert(n == A.shape[0]), 'n must equal dim of A.' + + n_steps = int(np.ceil(np.log(n)/np.log(2))) + A_cache = jnp.zeros((n_steps, n, n), dtype=jnp.uint32) + steps_to_reduce = jnp.array(1, dtype=jnp.int32) + + def _scan_fwd_step(carry, step): + i = step + A, A_cache, steps_to_reduce = carry + A_cache = A_cache.at[i].set(A) + + Z = A@A + B = jnp.logical_or( + A == 1, + Z > 0 + ).astype(jnp.uint32) \ + .at[jnp.diag_indices(n)].set(0) + A = B + + complete = B.sum() - jnp.diagonal(B).sum() == n*(n-1) + steps_to_reduce += ~complete + + return (A, A_cache, steps_to_reduce), None + + (B, A_cache, steps_to_reduce), _ = jax.lax.scan( + _scan_fwd_step, + (A, A_cache, 1), + jnp.arange(n_steps), + length=n_steps + ) + + D = 2*B - A_cache[steps_to_reduce-1] + + def _scan_bkwd_step(carry, step): + i = step + (T, A_cache,steps_to_reduce) = carry + + A = A_cache[steps_to_reduce - i - 1] + X = T@A + + thresh = T*(jnp.tile(A.sum(0, keepdims=True), (n, 1))) + D = 2*T*(X >= thresh) + (2*T - 1)*(X < thresh) + T = D*(i < steps_to_reduce) + T*(i >= steps_to_reduce) + + return (T, A_cache, steps_to_reduce), None + + (D, _, _), _ = jax.lax.scan( + _scan_bkwd_step, + (D, A_cache, steps_to_reduce), + jnp.arange(1, n_steps), + length=n_steps-1 + ) + + return D + + +@jax.jit +def grid_to_graph(grid): + """ + Transform a binary grid (True == wall) into a + graph, where walls are all connected to a default node. + """ + h, w = grid.shape + nodes = grid.flatten() + n = len(nodes) + A = jnp.zeros((n,n), dtype=jnp.uint32) + + all_idx = jnp.arange(n) + dum_neighbor_idx = jnp.argmax(~nodes) + dum_neighbor_mask = jnp.zeros(n, dtype=jnp.bool_) + dum_neighbor_mask = \ + dum_neighbor_mask.at[dum_neighbor_idx].set(True) + + def _get_neigbors(idx): + # Return length n boolean mask of neighbors + # We then vmap this function over all n + r = idx + 1 + l = idx - 1 + t = idx - w + b = idx + w + + l_border = jnp.logical_or( + idx % w == 0, + nodes[l] + ) + r_border = jnp.logical_or( + r % w == 0, + nodes[r] + ) + t_border = jnp.logical_or( + idx // w == 0, + nodes[t], + ) + b_border = jnp.logical_or( + idx // w == h - 1, + nodes[b] + ) + + l_ignore = jnp.logical_or( + l_border, + nodes[idx] + ) + r_ignore = jnp.logical_or( + r_border, + nodes[idx] + ) + t_ignore = jnp.logical_or( + t_border, + nodes[idx] + ) + b_ignore = jnp.logical_or( + b_border, + nodes[idx] + ) + + left = l*(1-l_ignore) + idx*(l_ignore) + right = r*(1-r_ignore) + idx*(r_ignore) + top = t*(1-t_ignore) + idx*(t_ignore) + bottom = b*(1-b_ignore) + idx*(b_ignore) + + neighbor_mask = jnp.zeros(n, dtype=jnp.bool_) + neighbor_mask = neighbor_mask.at[jnp.array([left, right, top, bottom])].set(True) + + neighbor_mask = (1-nodes[idx])*neighbor_mask + nodes[idx]*dum_neighbor_mask + + neighbor_mask = neighbor_mask.at[idx].set(False) + + return neighbor_mask + + A = jax.vmap(_get_neigbors)(all_idx).astype(dtype=jnp.uint32) + A = jnp.maximum(A, A.transpose()) + + return A + + +NEIGHBOR_OFFSETS = jnp.array([ + [1,0], # right + [0,1], # bottom + [-1,0], # left + [0,-1], # top + [0,0] # self +], dtype=jnp.int32) + + +@jax.jit +def component_mask_with_pos(grid, pos_a): + """ + Return a mask set to True in all cells that are + a part of the connected component containing pos_a. + """ + h,w = grid.shape + visited_mask = grid + + pos = pos_a + visited_mask = visited_mask.at[ + pos[1],pos[0] + ].set(True) + vstack = jnp.zeros((h*w, 2), dtype=jnp.uint32) + vstack = vstack.at[:2].set(pos) + vstack_size = 2 + + def _scan_dfs(carry, step): + (visited_mask, vstack, vstack_size) = carry + + pos = vstack[vstack_size-1] + + neighbors = \ + jnp.minimum( + jnp.maximum( + pos + NEIGHBOR_OFFSETS, 0 + ), jnp.array([[h,w]]) + ).astype(jnp.uint32) + + neighbors_mask = visited_mask.at[ + neighbors[:,1],neighbors[:,0] + ].get() + n_neighbor_visited = neighbors_mask[:4].sum() + all_visited = n_neighbor_visited == 4 + all_visited_post = n_neighbor_visited >= 3 + neighbors_mask = neighbors_mask.at[-1].set(~all_visited) + + next_neighbor_idx = jnp.argmax(~neighbors_mask) + next_neighbor = neighbors[next_neighbor_idx] + + visited_mask = visited_mask.at[ + next_neighbor[1],next_neighbor[0] + ].set(True) + + vstack_size -= all_visited_post + + vstack = vstack.at[vstack_size].set(next_neighbor) + vstack_size += ~all_visited + + pos = next_neighbor + + return (visited_mask, vstack, vstack_size), None + + max_n_steps = 2*h*w + (visited_mask, vstack, vstack_size), _ = jax.lax.scan( + _scan_dfs, + (visited_mask, vstack, vstack_size), + jnp.arange(max_n_steps), + length=max_n_steps + ) + + visited_mask = visited_mask ^ grid + return visited_mask + + +@jax.jit +def shortest_path_len(grid, pos_a, pos_b,ignore_value=-1): + grid = ~component_mask_with_pos(grid, pos_a) + + A = grid_to_graph(grid) + D = apsp(A, n=A.shape[0]) + + if len(pos_b.shape) == 2: # batch eval + return jax.vmap(_shortest_path_len, in_axes=(None, None, 0, None))( + grid, pos_a, pos_b, D, ignore_value + ) + else: + return _shortest_path_len(grid, pos_a, pos_b, D, ignore_value) + + +@jax.jit +def _shortest_path_len(grid, pos_a, pos_b, D, ignore_value): + h,w = grid.shape + + a_idx = pos_a[1]*w + pos_a[0] + b_idx = pos_b[1]*w + pos_b[0] + d = D[a_idx][b_idx] + + mhttn_d = jnp.sum(jnp.abs(jnp.maximum(pos_a,pos_b)- jnp.minimum(pos_a,pos_b))) + + impossible = jnp.logical_and( + d == 1, + mhttn_d > 1 + ) + + return d*(1-impossible) diff --git a/src/minimax/util/loggers.py b/src/minimax/util/loggers.py new file mode 100644 index 0000000..8f5d646 --- /dev/null +++ b/src/minimax/util/loggers.py @@ -0,0 +1,291 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This file is modified from +https://github.com/openai/baselines + +Licensed under the MIT License; +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://opensource.org/license/mit/ +""" + + +import os +import sys +import json +import csv +import time +import datetime +import copy +import logging +import git +from typing import Dict + +import minimax.util.checkpoint as _checkpoint_util + + +class KVWriter(object): + def writekvs(self, kvs): + raise NotImplementedError + + +class SeqWriter(object): + def writeseq(self, seq): + raise NotImplementedError + + +class HumanOutputFormat(KVWriter, SeqWriter): + def __init__(self, filename_or_file): + if isinstance(filename_or_file, str): + self.file = open(filename_or_file, 'wt') + self.own_file = True + else: + assert hasattr( + filename_or_file, 'read'), 'expected file or str, got %s' % filename_or_file + self.file = filename_or_file + self.own_file = False + + def writekvs(self, kvs): + # Create strings for printing + key2str = {} + for (key, val) in sorted(kvs.items()): + if hasattr(val, '__float__'): + valstr = '%-8.3g' % val + else: + valstr = str(val) + key2str[self._truncate(key)] = self._truncate(valstr) + + # Find max widths + if len(key2str) == 0: + print('WARNING: tried to write empty key-value dict') + return + else: + keywidth = max(map(len, key2str.keys())) + valwidth = max(map(len, key2str.values())) + + # Write out the data + dashes = '-' * (keywidth + valwidth + 7) + lines = [dashes] + for (key, val) in sorted(key2str.items(), key=lambda kv: kv[0].lower()): + lines.append('| %s%s | %s%s |' % ( + key, + ' ' * (keywidth - len(key)), + val, + ' ' * (valwidth - len(val)), + )) + lines.append(dashes) + self.file.write('\n'.join(lines) + '\n') + + # Flush the output to the file + self.file.flush() + + def _truncate(self, s): + maxlen = 64 + return s[:maxlen-3] + '...' if len(s) > maxlen else s + + def writeseq(self, seq): + seq = list(seq) + for (i, elem) in enumerate(seq): + self.file.write(elem) + if i < len(seq) - 1: # add space unless this is the last one + self.file.write(' ') + self.file.write('\n') + self.file.flush() + + def close(self): + if self.own_file: + self.file.close() + + +def gather_metadata() -> Dict: + date_start = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f") + # Gathering git metadata. + try: + import git + + try: + repo = git.Repo(search_parent_directories=True) + git_sha = repo.commit().hexsha + git_data = dict( + commit=git_sha, + branch=None if repo.head.is_detached else repo.active_branch.name, + is_dirty=repo.is_dirty(), + path=repo.git_dir, + ) + except git.InvalidGitRepositoryError: + git_data = None + except ImportError: + git_data = None + + # Gathering slurm metadata. + if "SLURM_JOB_ID" in os.environ: + slurm_env_keys = [k for k in os.environ if k.startswith("SLURM")] + slurm_data = {} + for k in slurm_env_keys: + d_key = k.replace("SLURM_", "").replace("SLURMD_", "").lower() + slurm_data[d_key] = os.environ[k] + else: + slurm_data = None + return dict( + date_start=date_start, + date_end=None, + successful=False, + git=git_data, + slurm=slurm_data, + env=os.environ.copy(), + ) + + +class Logger: + def __init__( + self, + log_dir='~/logs/minimax', + xpid=None, + xp_args=None, + callback=None, + from_last_checkpoint=False, + verbose=False): + # Set up checkpoint meta + self.verbose = verbose + if self.verbose: + self._stdout = HumanOutputFormat(sys.stdout) + + self._callback = callback + + formatter = logging.Formatter("%(message)s") + self._logger = logging.getLogger("logs/out") + shandle = logging.StreamHandler() + shandle.setFormatter(formatter) + self._logger.addHandler(shandle) + self._logger.setLevel(logging.INFO) + + # Set up main paths for logs and checkpoints + self.paths = {} + log_dir_path = os.path.expandvars(os.path.expanduser(log_dir)) + xpid_dir_path = os.path.join(log_dir_path, xpid) + if not xpid: + xpid = "{proc}_{unixtime}".format( + proc=os.getpid(), unixtime=int(time.time()) + ) + if not os.path.exists(xpid_dir_path): + self._logger.info("Creating log directory: %s", xpid_dir_path) + os.makedirs(xpid_dir_path, exist_ok=True) + self.paths['log_dir'] = log_dir_path + self.paths['xpid_dir'] = xpid_dir_path + self.paths['checkpoint'] = os.path.join( + xpid_dir_path, 'checkpoint.pkl') + + # Create logs.csv file + logs_csv_path = os.path.join(xpid_dir_path, 'logs.csv') + self.paths['log_csv'] = logs_csv_path + self._last_n_logged_lines = 0 + self._last_logged_tick = self._get_last_logged_tick() + + self.append_to_existing_logs = \ + self._last_logged_tick >= 0 \ + and from_last_checkpoint \ + and os.path.exists(self.paths['checkpoint']) + log_mode = "a" if self.append_to_existing_logs else "w+" + self._logfile = open(logs_csv_path, log_mode) + self._logwriter = None + + # Create meta file + if xp_args is not None: + meta_path = os.path.join(xpid_dir_path, 'meta.json') + + meta = gather_metadata() + meta['config'] = dict(xp_args) + meta['xpid'] = xpid + + self._save_metadata(meta_path, meta) + + def _save_metadata(self, meta_path, meta): + with open(meta_path, "w") as jsonfile: + json.dump(meta, jsonfile, indent=4, sort_keys=True) + + def _get_last_logged_tick(self): + last_tick = -1 + logs_csv_path = self.paths['log_csv'] + if os.path.exists(logs_csv_path): + with open(logs_csv_path, "r") as csvfile: + reader = csv.reader(csvfile) + try: + lines = list(reader) + except: + return -1 + # Need at least two lines in order to read the last tick: + # the first is the csv header and the second is the first line + # of data. + if len(lines) > 1: + self._last_n_logged_lines = len(lines) + try: + last_tick = int(lines[-1][0]) + except: + last_tick = -1 + + return last_tick + + def log(self, stats, _tick, ignore_val=None): + if ignore_val is not None: + stats = {k: v if v != ignore_val else None for k, v in stats.items()} + + _stats = { + '_tick': _tick, + '_time': time.time() + } + _stats.update(stats) + stats = _stats + + if self._logwriter is None: + fieldnames = list(stats.keys()) + self._logwriter = csv.DictWriter( + self._logfile, fieldnames=fieldnames) + + if _tick > self._last_logged_tick \ + or not self.append_to_existing_logs: + if self._last_n_logged_lines == 0: + fieldnames = list(stats.keys()) + self._logfile.write("# %s\n" % ",".join(fieldnames)) + self._logfile.flush() + self._last_n_logged_lines = 1 + + self._logwriter.writerow(stats) + self._logfile.flush() + + if self._callback is not None: + self._callback(stats) + + if self.verbose: + self._stdout.writekvs(stats) + + @property + def checkpoint_path(self): + return self.paths['checkpoint'] + + def checkpoint( + self, + runner_state, + name='checkpoint', + index=None, + archive_interval=None): + _checkpoint_util.safe_checkpoint( + runner_state, + self.paths['xpid_dir'], + name, + index, + archive_interval + ) + + def load_last_checkpoint_state(self): + checkpoint_path = \ + os.path.join(self.paths['xpid_dir'], f'checkpoint.pkl') + + if os.path.exists(checkpoint_path): + self._logger.info( + f'Loading previous checkpoint from {checkpoint_path}...') + return _checkpoint_util.load_pkl_object(checkpoint_path) + else: + return None diff --git a/src/minimax/util/parsnip.py b/src/minimax/util/parsnip.py new file mode 100644 index 0000000..8c1f330 --- /dev/null +++ b/src/minimax/util/parsnip.py @@ -0,0 +1,329 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from collections import defaultdict +import argparse +import sys +import re +import pprint + +from minimax.util import DefaultDotDict, DotDict + + +def append_subparser_prefix(prefix, func): + def prefixed_add_argument(*args, **kwargs): + if len(args) > 0: + name = args[0] + if name.startswith('--'): + name = f'--{prefix}_{name[2:]}' + args = (name,) + args[1:] + + return func(*args, **kwargs) + + return prefixed_add_argument + + +def ensure_args_suffix(name): + if not name.endswith('args'): + name = f'{name}_args' + + return name + + +def get_all_cmd_arg_names(): + cmd_args = [s.removeprefix('--') for s in sys.argv if s.startswith('--')] + arg_names = [x.split('=')[0] for x in cmd_args] + + return set(arg_names) + + +def get_argument_kwargs(subparser): + kwargs = [] + skip_list = ['-h', '--help'] + for k, info in subparser.__dict__['_option_string_actions'].items(): + if k in skip_list: + continue + + info_dict = info.__dict__ + kwargs_dict = dict( + # option_strings=info_dict['option_strings'], + name=info_dict['dest'], + const=info_dict['const'], + default=info_dict['default'], + type=info_dict['type'], + choices=info_dict['choices'], + required=info_dict['required'] + ) + + nargs = info_dict['nargs'] + if nargs == '?' or (nargs is not None and int(nargs) > 0): + kwargs_dict.update(dict(nargs=info_dict['nargs'])) + + kwargs.append(kwargs_dict) + + return kwargs + + +class Parsnip: + """ + Wraps a collection of argparse instances + to enable convenient grouping of arguments and + access via a DotDict-style interface. + """ + def __init__(self, description=None): + self._base_parser = \ + argparse.ArgumentParser(description=description) + self._subparsers = {} + self._prefixes = [] + self._dependencies = defaultdict() + self._dests = defaultdict() + self._dependent_args = set() + + def add_subparser( + self, + name, + prefix=None, + dest=None, + depends_on=None, + dependency=None, + is_individual_arg=False, + description=None + ): + if not is_individual_arg: + name = ensure_args_suffix(name) + + assert name not in self._subparsers, \ + f'Multiple subparsers named {name} detected.' + + if dependency is not None: + if depends_on is not None: + depends_on = ensure_args_suffix(depends_on) + + assert depends_on in self._subparsers, \ + f'Missing subparse {depends_on} must be added before dependent {name}.' + + assert isinstance(dependency, dict), \ + f'Subparser dependencies must be specified as dicts.' + + self._dependencies[name] = (depends_on, dependency) + + if dest is not None: + dest = ensure_args_suffix(dest) + assert dest in self._subparsers, \ + f"Missing dest {dest} must be specified before source {name}." + + subparser = argparse.ArgumentParser( + description=description, + allow_abbrev=False) + if prefix is not None: + subparser.add_argument = append_subparser_prefix( + prefix, subparser.add_argument, + ) + + self._subparsers[name] = subparser + self._prefixes.append(prefix) + self._dests[name] = dest + + return subparser + + def add_dependent_argument( + self, + *args, + **kwargs,): + + assert 'dependency' in kwargs, \ + 'Must specify dependency in kwargs.' + dependency = kwargs.pop('dependency') + + name = args[0].removeprefix('--') + + prefix = kwargs.pop('prefix', None) + dest = kwargs.pop('dest', None) + + subparser = self.add_subparser( + name, + prefix=prefix, + dependency=dependency, + dest=dest, + is_individual_arg=True, + description=kwargs.pop('description', '') + ) + subparser.add_argument(*args, **kwargs) + + self._dependent_args.add(name) + + def copy_arguments( + self, + src, + dest, + arg_prefix=None + ): + all_subparsers = [k for k in self._subparsers.keys()] + src_name = f'{src}_args' + src_subparser = self._subparsers[src_name] + src_idx = all_subparsers.index(src_name) + src_prefix = self._prefixes[src_idx] + + dest_name = f'{dest}_args' + dest_subparser = self._subparsers[dest_name] + dest_idx = all_subparsers.index(dest_name) + + arg_prefix = f'{arg_prefix}_' if arg_prefix else '' + + argument_kwargs = get_argument_kwargs(src_subparser) + for kwargs in argument_kwargs: + name = kwargs.pop('name') + flag = f"--{arg_prefix}{name.removeprefix(f'{src_prefix}_')}" + + dest_subparser.add_argument( + flag, + **kwargs + ) + + def parse_args(self, preview=False): + cmd_arg_names, args, arg_data, _ = self._parse_cmd_line_flags() + + for name in cmd_arg_names: + if name not in arg_data: + raise ValueError(f'Unknown argument {name}.') + + if preview: + print('🥕 Parsnip harvested the following arguments:') + pp = pprint.PrettyPrinter(indent=4) + pp.pprint(args) + + return args + + def parse_cmd_line_flags(self, as_grid_json=False): + _, _, arg_data, _ = self._parse_cmd_line_flags() + + if as_grid_json: + arg_data = {k:[v] for k,v in arg_data.items() if v is not None} + + return arg_data + + def _parse_cmd_line_flags(self, keep_structure=False): + cmd_arg_names = get_all_cmd_arg_names() + + arg_data = {} + argname2keypath = {} + + args = DefaultDotDict(vars(self._base_parser.parse_known_args()[0])) + for k in args: + arg_data[k] = args[k] + argname2keypath[k] = [k] + + for i, (name, subparser) in enumerate(self._subparsers.items()): + if name in self._dependencies: + depends_on, dependencies = self._dependencies[name] + if depends_on is None: + dsubargs = arg_data + else: + dsubargs = arg_data[depends_on] + skip_subparser = False + for dk, dv in dependencies.items(): + if not isinstance(dv, (tuple, list)): + dv = [dv] + + for v in dv: + if isinstance(v, str): + match_regex = f"^{v.replace('*', '.*')}$" + match = re.match(match_regex, dsubargs[dk]) is not None + else: + match = dsubargs[dk] == v + if match: + break + + if not match: + skip_subparser = True + break + + if skip_subparser: + continue + + prefix = self._prefixes[i] + subargs, _ = subparser.parse_known_args() + + subargs = vars(subargs) + for k in subargs: + arg_data[k] = subargs[k] + + subargs = DotDict( + {k.removeprefix(f'{prefix}_'):v for k,v in subargs.items()} + ) + + # Check for flatten and merge conditions + dest = self._dests.get(name) + if name in self._dependent_args: + if dest is None: + args[name] = subargs[name] + else: + args[dest].update({name: subargs[name]}) + else: + if dest is None: + args[name] = subargs + else: + args[dest].update(subargs) + + for k in subargs: + if prefix is not None: + argname = f'{prefix}_{k}' + else: + argname = k + if dest is None: + sp_name = name + else: + sp_name = dest + if sp_name == k: + argname2keypath[argname] = [k] + else: + argname2keypath[argname] = [sp_name, k] + + return cmd_arg_names, args, arg_data, argname2keypath + + @property + def argname2keypath(self): + args = DefaultDotDict(vars(self._base_parser.parse_known_args()[0])) + argname2keypath = {} + for k in args: + argname2keypath[k] = args[k] + + for i, (name, subparser) in enumerate(self._subparsers.items()): + if name in self._dependencies: + depends_on, dependencies = self._dependencies[name] + if depends_on is None: + dsubargs = arg_data + else: + dsubargs = arg_data[depends_on] + skip_subparser = False + for dk, dv in dependencies.items(): + if not isinstance(dv, (tuple, list)): + dv = [dv] + + for v in dv: + if isinstance(v, str): + match_regex = f"^{v.replace('*', '.*')}$" + match = re.match(match_regex, dsubargs[dk]) is not None + else: + match = dsubargs[dk] == v + if match: + break + + if not match: + skip_subparser = True + break + + if skip_subparser: + continue + + prefix = self._prefixes[i] + subargs, _ = subparser.parse_known_args() + + def __getattr__(self, attr): + # Default missing attr to _base_argparse instance + return getattr(self._base_parser, attr) \ No newline at end of file diff --git a/src/minimax/util/pytree.py b/src/minimax/util/pytree.py new file mode 100644 index 0000000..f25f572 --- /dev/null +++ b/src/minimax/util/pytree.py @@ -0,0 +1,37 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +import collections + +import jax + + +def pytree_set_array_at(pytree, i, value): + return jax.tree_util.tree_map(lambda x,y: x.at[i].set(y), pytree, value) + +def pytree_set_struct_at(pytree, i, value): + return jax.tree_util.tree_map(lambda x,y: x.at[i].set(y), pytree, value) + +def pytree_at(pytree, start, end=None): + return jax.tree_util.tree_map(lambda x: x.at[start:end].get(), pytree) + +def pytree_select(pred, on_true, on_false): + vselect = jax.vmap(jax.lax.select, in_axes=(0, 0, 0)) + return jax.tree_util.tree_map(lambda x,y: vselect(pred, x, y), on_true, on_false) + +def pytree_expand_batch_dim(pytree, batch_shape, n_batch_axes=2): + """ + Expands a single batch dimension into a multi-dim batch shape + """ + return jax.tree_util.tree_map(lambda x: x.reshape(*batch_shape, *x.shape[n_batch_axes:]), pytree) + +def pytree_transform(pytree, transform): + return jax.tree_util.tree_map(lambda x: transform(x), pytree) + +def pytree_merge(dst, src, start_idx, src_len): + return jax.tree_map(lambda x,y: x.at[start_idx:start_idx+src_len].set(y.at[:src_len].get()), dst, src) \ No newline at end of file diff --git a/src/minimax/util/rl/__init__.py b/src/minimax/util/rl/__init__.py new file mode 100644 index 0000000..11b8e73 --- /dev/null +++ b/src/minimax/util/rl/__init__.py @@ -0,0 +1,16 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from .training import VmapTrainState, VmapMAPPOTrainState +from .agent_pop import AgentPop +from .agent_pop_heterogenous import AgentPopHeterogenous +from .rolling_stats import RollingStats +from .rollout_storage import RolloutStorage +from .rollout_storage_seperate import RolloutStorageSeperate +from .ued_scores import * +from .plr import PLRManager, PopPLRManager diff --git a/src/minimax/util/rl/agent_pop.py b/src/minimax/util/rl/agent_pop.py new file mode 100644 index 0000000..0a8ee1a --- /dev/null +++ b/src/minimax/util/rl/agent_pop.py @@ -0,0 +1,119 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial + +import numpy as np +import jax +import jax.numpy as jnp + + +class AgentPop: + """ + Manages multiple agents of the same architecture + """ + + def __init__( + self, + agent, + n_agents): + """ + Maintains a set of model parameters. + """ + self.agent = agent + self.n_agents = n_agents + + def _reshape_to_pop(self, x): + return jax.tree_map( + lambda x: jnp.reshape(x, newshape=(self.n_agents, x.shape[0]//self.n_agents, *x.shape[1:])), x) + + def _flatten(self, x): + return jax.tree_map(lambda x: jnp.reshape(x, newshape=(self.n_agents*x.shape[1], -1)).squeeze(), x) + + def init_params(self, rng, obs): + if self.agent.is_recurrent: + # Make time first dim + obs = jax.tree_map(lambda x: x[jnp.newaxis, :], obs) + + vrngs = jax.random.split(rng, self.n_agents) + return jax.vmap( + self.agent.init_params, + in_axes=(0, None) + )(vrngs, obs) + + @partial(jax.jit, static_argnums=(0,)) + def init_carry(self, rng, obs): + if hasattr(self.agent, "actor") and self.agent.actor.conv_encoder: + agent_batch_dim = jax.tree_util.tree_leaves(obs)[0].shape[:-3] + elif not hasattr(self.agent, "actor"): + agent_batch_dim = jax.tree_util.tree_leaves(obs)[0].shape[:-3] + else: # Linear obs + agent_batch_dim = jax.tree_util.tree_leaves(obs)[0].shape[:-1] + return self.agent.init_carry(rng=rng, batch_dims=agent_batch_dim) + + @partial(jax.jit, static_argnums=(0,)) + def act(self, params, obs, carry, reset=None): + # If recurrent, add time axis to support scanned rollouts + if self.agent.is_recurrent: + # Add time dim after agent dim + obs = jax.tree_map(lambda x: x[:, jnp.newaxis, :], obs) + + if reset is None: + agent_batch_dim = jax.tree_util.tree_leaves(obs)[0].shape[2] + reset = jnp.zeros( + (self.n_agents, 1, agent_batch_dim), dtype=jnp.bool_) + else: + reset = reset[:, jnp.newaxis, :] + + value, pi_params, next_carry = jax.vmap( + self.agent.act)(params, obs, carry, reset) + + if self.agent.is_recurrent: # Remove time dim + if value is not None: + value = value.squeeze(1) + pi_params = jax.tree_map(lambda x: x.squeeze(1), pi_params) + + return value, pi_params, next_carry + + def get_action_dist(self, dist_params, dtype=jnp.uint8): + return self.agent.get_action_dist(dist_params, dtype=dtype) + + @partial(jax.jit, static_argnums=(0,)) + def get_value(self, params, obs, carry, reset=None): + if self.agent.is_recurrent: + # Add time dim after agent dim + obs = jax.tree_map(lambda x: x[:, jnp.newaxis, :], obs) + + if reset is None: + agent_batch_dim = jax.tree_util.tree_leaves(obs)[0].shape[2] + reset = jnp.zeros( + (self.n_agents, 1, agent_batch_dim), dtype=jnp.bool_) + else: + reset = reset[:, jnp.newaxis, :] + + value, next_carry = jax.vmap( + self.agent.get_value)(params, obs, carry, reset) + + if self.agent.is_recurrent: # Remove time dim + value = value.squeeze(1) + + if value.shape[-1] == 1: + value = value.squeeze(-1) + return value, next_carry + + @partial(jax.jit, static_argnums=(0, 4, 5)) + def update(self, rng, train_state, batch, prefix_steps=0, fake=False): + if fake: + return train_state, jax.vmap(lambda *_: self.agent.get_empty_update_stats())(np.arange(self.n_agents)) + + rng, *vrngs = jax.random.split(rng, self.n_agents+1) + vrngs = jnp.array(vrngs) + + new_train_state, stats = jax.vmap( + self.agent.update)(vrngs, train_state, batch) + return new_train_state, stats diff --git a/src/minimax/util/rl/agent_pop_heterogenous.py b/src/minimax/util/rl/agent_pop_heterogenous.py new file mode 100644 index 0000000..39381ca --- /dev/null +++ b/src/minimax/util/rl/agent_pop_heterogenous.py @@ -0,0 +1,166 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial + +import numpy as np +import jax +import jax.numpy as jnp + + +class AgentPopHeterogenous: + """ + Manages multiple agents with no assumption regarding the architecture + """ + + def __init__( + self, + agent_0, + agent_1, + n_agents): + """ + Maintains a set of model parameters. + """ + self.agent_0 = agent_0 + self.agent_1 = agent_1 + self.n_agents = n_agents + + def _reshape_to_pop(self, x): + return jax.tree_map( + lambda x: jnp.reshape(x, newshape=(self.n_agents, x.shape[0]//self.n_agents, *x.shape[1:])), x) + + def _flatten(self, x): + return jax.tree_map(lambda x: jnp.reshape(x, newshape=(self.n_agents*x.shape[1], -1)).squeeze(), x) + + def init_params(self, rng, obs): + if self.agent.is_recurrent: + # Make time first dim + obs = jax.tree_map(lambda x: x[jnp.newaxis, :], obs) + + vrngs = jax.random.split(rng, self.n_agents) + return jax.vmap( + self.agent.init_params, + in_axes=(0, None) + )(vrngs, obs) + + # @partial(jax.jit, static_argnums=(0,)) + # def init_carry(self, rng, obs): + # if self.agent.actor.conv_encoder: + # agent_batch_dim = jax.tree_util.tree_leaves(obs)[0].shape[:-3] + # else: # Linear obs + # agent_batch_dim = jax.tree_util.tree_leaves(obs)[0].shape[:-1] + # return self.agent.init_carry(rng=rng, batch_dims=agent_batch_dim) + + @partial(jax.jit, static_argnums=(0,)) + def init_carry_agent_0(self, rng, obs): + if self.agent_0.actor.conv_encoder: + agent_batch_dim = jax.tree_util.tree_leaves(obs)[0].shape[:-3] + else: # Linear obs + agent_batch_dim = jax.tree_util.tree_leaves(obs)[0].shape[:-1] + return self.agent_0.init_carry(rng=rng, batch_dims=agent_batch_dim) + + @partial(jax.jit, static_argnums=(0,)) + def init_carry_agent_1(self, rng, obs): + if self.agent_1.actor.conv_encoder: + agent_batch_dim = jax.tree_util.tree_leaves(obs)[0].shape[:-3] + else: # Linear obs + agent_batch_dim = jax.tree_util.tree_leaves(obs)[0].shape[:-1] + return self.agent_1.init_carry(rng=rng, batch_dims=agent_batch_dim) + + @partial(jax.jit, static_argnums=(0,)) + def act(self, params, obs, carry, reset=None): + # If recurrent, add time axis to support scanned rollouts + actor_0_params, actor_1_params = params + actor_0_carry, actor_1_carry = carry + + if self.agent_0.is_recurrent: + # Add time dim after agent dim + obs_0 = jax.tree_map( + lambda x: x[:, jnp.newaxis, :], obs['agent_0']) + + if reset is None: + agent_batch_dim = jax.tree_util.tree_leaves(obs_0)[0].shape[2] + reset = jnp.zeros( + (self.n_agents, 1, agent_batch_dim), dtype=jnp.bool_) + else: + reset = reset[:, jnp.newaxis, :] + else: + obs_0 = obs['agent_0'] + + if self.agent_1.is_recurrent: + # Add time dim after agent dim + obs_1 = jax.tree_map( + lambda x: x[:, jnp.newaxis, :], obs['agent_1']) + + if reset is None: + agent_batch_dim = jax.tree_util.tree_leaves(obs_1)[0].shape[2] + reset = jnp.zeros( + (self.n_agents, 1, agent_batch_dim), dtype=jnp.bool_) + else: + reset = reset[:, jnp.newaxis, :] + else: + obs_1 = obs['agent_1'] + + value_0, pi_params_0, next_0_carry = jax.vmap( + self.agent_0.act)(actor_0_params, obs_0, actor_0_carry, reset) + + value_1, pi_param_1, next_1_carry = jax.vmap( + self.agent_1.act)(actor_1_params, obs_1, actor_1_carry, reset) + + if self.agent_0.is_recurrent: # Remove time dim + if value_0 is not None: + value_0 = value_0.squeeze(1) + pi_params_0 = jax.tree_map(lambda x: x.squeeze(1), pi_params_0) + + if self.agent_1.is_recurrent: # Remove time dim + if value_1 is not None: + value_1 = value_1.squeeze(1) + pi_param_1 = jax.tree_map(lambda x: x.squeeze(1), pi_param_1) + + return value_0, value_1, pi_params_0, pi_param_1, next_0_carry, next_1_carry + + def get_action_0_dist(self, dist_params, dtype=jnp.uint8): + return self.agent_0.get_action_dist(dist_params, dtype=dtype) + + def get_action_1_dist(self, dist_params, dtype=jnp.uint8): + return self.agent_1.get_action_dist(dist_params, dtype=dtype) + + @partial(jax.jit, static_argnums=(0,)) + def get_value(self, params, obs, carry, reset=None): + if self.agent.is_recurrent: + # Add time dim after agent dim + obs = jax.tree_map(lambda x: x[:, jnp.newaxis, :], obs) + + if reset is None: + agent_batch_dim = jax.tree_util.tree_leaves(obs)[0].shape[2] + reset = jnp.zeros( + (self.n_agents, 1, agent_batch_dim), dtype=jnp.bool_) + else: + reset = reset[:, jnp.newaxis, :] + + value, next_carry = jax.vmap( + self.agent.get_value)(params, obs, carry, reset) + + if self.agent.is_recurrent: # Remove time dim + value = value.squeeze(1) + + if value.shape[-1] == 1: + value = value.squeeze(-1) + return value, next_carry + + @partial(jax.jit, static_argnums=(0, 4, 5)) + def update(self, rng, train_state, batch, prefix_steps=0, fake=False): + if fake: + return train_state, jax.vmap(lambda *_: self.agent.get_empty_update_stats())(np.arange(self.n_agents)) + + rng, *vrngs = jax.random.split(rng, self.n_agents+1) + vrngs = jnp.array(vrngs) + + new_train_state, stats = jax.vmap( + self.agent.update)(vrngs, train_state, batch) + return new_train_state, stats diff --git a/src/minimax/util/rl/hl_gauss_transform.py b/src/minimax/util/rl/hl_gauss_transform.py new file mode 100644 index 0000000..a44f57c --- /dev/null +++ b/src/minimax/util/rl/hl_gauss_transform.py @@ -0,0 +1,39 @@ +import chex +import jax +import jax.numpy as jnp +import jax.scipy.special + + +def hl_gauss_transform( + min_value: float, + max_value: float, + num_bins: int, + sigma: float, +): + support = jnp.linspace(min_value, max_value, num_bins+1, dtype=jnp.float32) + + def transform_to_probs(target: chex.Array) -> chex.Array: + cdf_evals = jax.scipy.special.erf((support-target)/(jnp.sqrt(2)*sigma)) + z = cdf_evals[-1] - cdf_evals[0] + bin_probs = cdf_evals[1:] - cdf_evals[:-1] + return bin_probs / z + + def transform_from_probs(probs: chex.Array) -> chex.Array: + centers = (support[:-1] + support[1:]) / 2 + return jnp.sum(probs * centers) + + return transform_to_probs, transform_from_probs + + +if __name__ == '__main__': + transform_to_probs, transform_from_probs = hl_gauss_transform( + min_value=0, + max_value=20.0, + num_bins=10, + sigma=0.1, + ) + + for r in [0, 3, 20]: + probs = transform_to_probs(jnp.array(r)) + print(f'Probs for {r}: {probs}') + print(f'Reconstructed from probs: {transform_from_probs(probs)}') diff --git a/src/minimax/util/rl/plr.py b/src/minimax/util/rl/plr.py new file mode 100644 index 0000000..ffbd9fc --- /dev/null +++ b/src/minimax/util/rl/plr.py @@ -0,0 +1,466 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial + +import jax +import jax.numpy as jnp +from flax import struct +import chex +import numpy as np + +from .ued_scores import UEDScore + + +class PLRBuffer(struct.PyTreeNode): + levels: chex.Array + scores: chex.Array + ages: chex.Array + max_returns: chex.Array # for MaxMC + filled: chex.Array + filled_count: chex.Array + n_mutations: chex.Array + + ued_score: int = struct.field( + pytree_node=False, default=UEDScore.L1_VALUE_LOSS.value) + replay_prob: float = struct.field(pytree_node=False, default=0.5) + buffer_size: int = struct.field(pytree_node=False, default=100) + staleness_coef: float = struct.field(pytree_node=False, default=0.3) + temp: float = struct.field(pytree_node=False, default=1.0) + use_score_ranks: bool = struct.field(pytree_node=False, default=True) + min_fill_ratio: float = struct.field(pytree_node=False, default=0.5) + use_robust_plr: bool = struct.field(pytree_node=False, default=False) + use_parallel_eval: bool = struct.field(pytree_node=False, default=False) + + +class PLRManager: + def __init__( + self, + example_level, # Example env instance + ued_score, + replay_prob=0.5, + buffer_size=100, + staleness_coef=0.3, + temp=1.0, + min_fill_ratio=0.5, + use_score_ranks=True, + use_robust_plr=False, + use_parallel_eval=False, + comparator_fn=None, + n_devices=1): + + assert not (ued_score == UEDScore.MAX_MC and not use_score_ranks), \ + 'Cannot use proportional normalization with MaxMC, which can produce negative scores.' + + self.ued_score = ued_score + self.replay_prob = replay_prob + self.buffer_size = buffer_size + self.staleness_coef = staleness_coef + self.temp = temp + self.min_fill_ratio = min_fill_ratio + self.use_score_ranks = use_score_ranks + self.use_robust_plr = use_robust_plr + self.use_parallel_eval = use_parallel_eval + self.comparator_fn = comparator_fn + + self.n_devices = n_devices + + example_level = jax.tree_map(lambda x: jnp.array(x), example_level) + self.levels = jax.tree_map( + lambda x: ( + jnp.tile(jnp.zeros_like(x), (buffer_size,) + (1,)*(len(x.shape)-1))).reshape(buffer_size, *x.shape), + example_level) + + self.scores = jnp.full(buffer_size, -jnp.inf) + self.max_returns = jnp.full(buffer_size, -jnp.inf) + self.ages = jnp.zeros(buffer_size, dtype=jnp.uint32) + self.filled = jnp.zeros(buffer_size, dtype=jnp.bool_) + self.filled_count = jnp.zeros((1,), dtype=jnp.int32) + self.n_mutations = jnp.zeros(buffer_size, dtype=jnp.uint32) + + @partial(jax.jit, static_argnums=(0,)) + def reset(self): + return PLRBuffer( + ued_score=self.ued_score.value, + replay_prob=self.replay_prob, + buffer_size=self.buffer_size, + staleness_coef=self.staleness_coef, + temp=self.temp, + min_fill_ratio=self.min_fill_ratio, + use_robust_plr=self.use_robust_plr, + use_parallel_eval=self.use_parallel_eval, + levels=self.levels, + scores=self.scores, + max_returns=self.max_returns, + ages=self.ages, + filled=self.filled, + filled_count=self.filled_count, + n_mutations=self.n_mutations) + + partial(jax.jit, static_argnums=(0,)) + + def _get_replay_dist(self, scores, ages, filled): + # Score dist + if self.use_score_ranks: + sorted_idx = jnp.argsort(-scores) # Top first + scores = jnp.zeros(self.buffer_size, dtype=jnp.int32)\ + .at[sorted_idx]\ + .set(1/jnp.arange(self.buffer_size)) + + scores = scores*filled + # Scores in this implementation might contain NaN + # scores = jnp.nan_to_num( + # scores, nan=0.0, posinf=+jnp.inf, neginf=-jnp.inf) + score_dist = scores/self.temp + z = score_dist.sum() + z = jnp.where(jnp.equal(z, 0), 1, z) + score_dist = jax.lax.select( + jnp.greater(z, 0), + score_dist/z, + filled*1. # Assign equal weight to all present levels + ) + + # Staleness dist + staleness_scores = ages*filled + # staleness_scores = jnp.nan_to_num( + # staleness_scores, nan=0.0, posinf=+jnp.inf, neginf=-jnp.inf) + _z = staleness_scores.sum() + z = jnp.where(jnp.equal(_z, 0), 1, _z) + staleness_dist = jax.lax.select( + jnp.greater(_z, 0), + staleness_scores/z, + score_dist # If no solutions are stale, do not sample from staleness dist + ) + + # Replay dist + replay_dist = (1-self.staleness_coef)*score_dist \ + + self.staleness_coef*staleness_dist + + return replay_dist + + partial(jax.jit, static_argnums=(0,)) + + def _get_next_insert_idx(self, plr_buffer): + return jax.lax.cond( + jnp.greater(plr_buffer.buffer_size, plr_buffer.filled_count[0]), + lambda *_: plr_buffer.filled_count[0], + lambda *_: jnp.argmin(self._get_replay_dist(plr_buffer.scores, + plr_buffer.ages, plr_buffer.filled)) + ) + + @partial(jax.jit, static_argnums=(0, 3)) + def _sample_replay_levels(self, rng, plr_buffer, n): + def _sample_replay_level(carry, step): + ages = carry + subrng = step + replay_dist = self._get_replay_dist( + plr_buffer.scores, ages, plr_buffer.filled) + replay_idx = jax.random.choice(subrng, np.arange( + self.buffer_size), shape=(), p=replay_dist) + replay_level = jax.tree_map(lambda x: x.take( + replay_idx, axis=0), plr_buffer.levels) + + ages = ((ages + 1)*(plr_buffer.filled)).at[replay_idx].set(0) + + return ages, (replay_level, replay_idx) + + rng, *subrngs = jax.random.split(rng, n+1) + next_ages, (replay_levels, replay_idxs) = jax.lax.scan( + _sample_replay_level, + plr_buffer.ages, + jnp.array(subrngs) + ) + + next_plr_buffer = plr_buffer.replace( + ages=next_ages + ) + + return replay_levels, replay_idxs, next_plr_buffer + + def _sample_buffer_uniform(self, rng, plr_buffer, n): + rand_idxs = jax.random.choice(rng, np.arange( + self.buffer_size), shape=(n,), p=plr_buffer.filled) + levels = jax.tree_map(lambda x: x.take( + replay_idx, axis=0), plr_buffer.levels) + + return levels, rand_idxs, plr_buffer + + # Levels must be sampled sequentially, to account for staleness + @partial(jax.jit, static_argnums=(0, 4, 5)) + def sample(self, rng, plr_buffer, new_levels, n, random=False): + rng, replay_rng, sample_rng = jax.random.split(rng, 3) + + is_replay = jnp.greater( + self.replay_prob, jax.random.uniform(replay_rng)) + is_warm = jnp.greater_equal( + plr_buffer.filled.sum()/self.buffer_size, self.min_fill_ratio) + + if self.n_devices > 1: # Synchronize replay + is_replay = jax.lax.all_gather(is_replay, axis_name='device')[0] + is_warm = jnp.all(jax.lax.all_gather(is_warm, axis_name='device')) + + is_replay = jnp.logical_and(is_replay, is_warm) + + if random: + sample_fn = self._sample_buffer_uniform + else: + sample_fn = self._sample_replay_levels + + levels, level_idxs, next_plr_buffer = jax.lax.cond( + is_replay, + partial(sample_fn, n=n), + lambda *_: (new_levels, np.full(n, -1), plr_buffer), + *(sample_rng, plr_buffer) + ) + + # Update ages when not sampling replay + next_plr_buffer = jax.lax.cond( + is_replay, + lambda *_: next_plr_buffer, + lambda *_: next_plr_buffer.replace( + ages=(plr_buffer.ages+n)*(plr_buffer.filled)) + ) + + return levels, level_idxs, is_replay, next_plr_buffer + + @partial(jax.jit, static_argnums=(0,)) + def dedupe_levels(self, plr_buffer, levels, level_idxs): + if self.comparator_fn is not None and level_idxs.shape[-1] > 2: + def _check_equal(carry, step): + match_idxs, other_levels, is_self = carry + batch_idx, level = step + + matches = jax.vmap(self.comparator_fn, in_axes=( + 0, None))(other_levels, level) + + top2match, top2match_idxs = jax.lax.top_k(matches, 2) + + is_self_dupe = jnp.logical_and( + is_self, top2match[1]) # More than 1 match + is_dedupe_idx = jnp.logical_and( + is_self_dupe, jnp.greater(batch_idx, top2match_idxs[0])) + self_match_idx = top2match_idxs[0] * \ + is_dedupe_idx - (~is_dedupe_idx) + + _match_idx = jnp.where( + is_self, + self_match_idx, # only first + top2match_idxs[0], # use first matching index in buffer + ) + + match_idxs = jnp.where( + matches.any(), + match_idxs.at[batch_idx].set(_match_idx), + match_idxs + ) + + return (match_idxs, other_levels, is_self), None + + # dedupe among batch levels + batch_dupe_idxs = jnp.full_like(level_idxs, -1) + (batch_dupe_idxs, _, _), _ = jax.lax.scan( + _check_equal, + (batch_dupe_idxs, levels, True), + (np.arange(level_idxs.shape[-1]), levels) + ) + batch_dupe_mask = jnp.greater(batch_dupe_idxs, -1) + + # dedupe against PLR buffer levels + (level_idxs, _, _), _ = jax.lax.scan( + _check_equal, + (level_idxs, plr_buffer.levels, False), + (np.arange(level_idxs.shape[-1]), levels) + ) + + return level_idxs, batch_dupe_mask + else: + return level_idxs, jnp.zeros_like(level_idxs, dtype=jnp.bool_) + + @partial(jax.jit, static_argnums=(0, 7)) + def update(self, plr_buffer, levels, level_idxs, ued_scores, dupe_mask=None, info=None, ignore_val=-jnp.inf, parent_idxs=None): + # Note: parent_idxs are only used for mutated levels + done_masks = (ued_scores != ignore_val) + if dupe_mask is not None: + # Ignore duplicate levels in batch by treating them as not done + done_masks = jnp.logical_and(done_masks, ~dupe_mask) + + cur_n_mutations = plr_buffer.n_mutations + insert_mask = jnp.zeros((self.buffer_size,), dtype=jnp.bool_) + + def update_level_info(carry, step): + plr_buffer, insert_mask = carry + levels = plr_buffer.levels + scores = plr_buffer.scores + filled = plr_buffer.filled + + score, level, level_idx, done_mask, parent_idx, max_return = step + + next_insert_idx = self._get_next_insert_idx(plr_buffer) + is_new_level = jnp.greater(0, level_idx) + insert_idx = jnp.where( + is_new_level, + next_insert_idx, # new level + level_idx, + ) + + should_insert = jnp.greater_equal( + score, scores.at[insert_idx].get()) + should_insert = jnp.logical_and(should_insert, done_mask) + + is_existing_level = jnp.logical_and(~is_new_level, done_mask) + should_update = jnp.logical_and( + is_existing_level, ~insert_mask.at[insert_idx].get()) + should_insert = jnp.logical_and(should_insert, ~should_update) + next_insert_mask = jnp.where( + should_insert, + insert_mask.at[insert_idx].set(True), + insert_mask + ) + should_insert_or_update = jnp.logical_or( + should_insert, should_update) + + # Update max return if needed + next_max_returns = jnp.where( + should_insert_or_update, + plr_buffer.max_returns.at[insert_idx].set(max_return), + plr_buffer.max_returns + ) + + updated_level = jax.tree_map( + lambda x, y: jax.lax.select(should_insert, x, y), + level, + jax.tree_map(lambda x: x.at[insert_idx].get(), levels) + ) + next_levels = jax.tree_map( + lambda x, y: x.at[insert_idx].set(y), levels, updated_level) + + next_scores = jnp.where( + should_insert_or_update, + scores.at[insert_idx].set(score), + scores + ) + next_filled = jnp.where( + should_insert, + filled.at[insert_idx].set(True), + filled + ) + + plr_replace_kwargs = dict( + levels=next_levels, + scores=next_scores, + filled=next_filled, + filled_count=jnp.array([next_filled.sum()]), + max_returns=next_max_returns + ) + + # Update mutation count + n_mutations = plr_buffer.n_mutations + should_incr_n_mutations = jnp.logical_and( + jnp.not_equal(parent_idx, -1), should_insert) + should_reset_n_mutations = jnp.logical_and( + jnp.equal(parent_idx, -1), should_insert_or_update) + reset_n_mutations = jnp.where( + is_existing_level, + cur_n_mutations.at[insert_idx].get(), + 0 + ) + next_n_mutations = jnp.where( + should_incr_n_mutations, + n_mutations.at[insert_idx].set( + cur_n_mutations.at[parent_idx].get() + 1), + n_mutations + ) + next_n_mutations = jnp.where( + should_reset_n_mutations, + n_mutations.at[insert_idx].set(reset_n_mutations), + next_n_mutations + ) + + plr_replace_kwargs['n_mutations'] = next_n_mutations + + next_plr_buffer = plr_buffer.replace(**plr_replace_kwargs) + + return (next_plr_buffer, next_insert_mask), None + + if parent_idxs is None: + parent_idxs = jnp.full_like(level_idxs, -1) + + if plr_buffer.ued_score == UEDScore.MAX_MC.value: + max_returns = info['max_returns'] + else: + max_returns = jnp.full_like(level_idxs, -1) + carry = (ued_scores, levels, level_idxs, + done_masks, parent_idxs, max_returns) + + (next_plr_buffer, _), _ = jax.lax.scan( + update_level_info, + (plr_buffer, insert_mask), + carry + ) + + return next_plr_buffer + + @partial(jax.jit, static_argnums=(0,)) + def get_metrics(self, plr_buffer): + replay_dist = self._get_replay_dist( + plr_buffer.scores, + plr_buffer.ages, + plr_buffer.filled) + weighted_n_mutations = (plr_buffer.n_mutations*replay_dist).sum() + scores = jnp.where(plr_buffer.filled, plr_buffer.scores, 0) + weighted_ued_score = (scores*replay_dist).sum() + + weighted_age = (plr_buffer.ages*replay_dist).sum() + + return dict( + weighted_n_mutations=weighted_n_mutations, + weighted_ued_score=weighted_ued_score, + weighted_age=weighted_age + ) + + +class PopPLRManager(PLRManager): + def __init__(self, *, n_agents, **kwargs): + super().__init__(**kwargs) + + self.n_agents = n_agents + + @partial(jax.jit, static_argnums=(0, 1)) + def reset(self, n): + sup = super() + return jax.vmap(lambda *_: sup.reset())(np.arange(n)) + + partial(jax.jit, static_argnums=(0, 4, 5)) + + def sample(self, rng, plr_buffer, new_levels, n, random=False): + sup = super() + + rng, *vrngs = jax.random.split(rng, self.n_agents+1) + + return jax.vmap(sup.sample, in_axes=(0, 0, 0, None, None))( + jnp.array(vrngs), plr_buffer, new_levels, n, random + ) + + @partial(jax.jit, static_argnums=(0,)) + def dedupe_levels(self, plr_buffer, levels, level_idxs): + sup = super() + return jax.vmap(sup.dedupe_levels)(plr_buffer, levels, level_idxs) + + partial(jax.jit, static_argnums=(0, 7)) + + def update(self, plr_buffer, levels, level_idxs, ued_scores, dupe_mask=None, info=None, ignore_val=-jnp.inf, parent_idxs=None): + sup = super() + return jax.vmap(sup.update, in_axes=(0, 0, 0, 0, 0, 0, None, 0))( + plr_buffer, levels, level_idxs, ued_scores, dupe_mask, info, ignore_val, parent_idxs + ) + + partial(jax.jit, static_argnums=(0,)) + + def get_metrics(self, plr_buffer): + sup = super() + return jax.vmap(sup.get_metrics)(plr_buffer) diff --git a/src/minimax/util/rl/rolling_stats.py b/src/minimax/util/rl/rolling_stats.py new file mode 100644 index 0000000..7d992fd --- /dev/null +++ b/src/minimax/util/rl/rolling_stats.py @@ -0,0 +1,116 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial + +import jax +import jax.numpy as jnp + + +class RollingStats: + """ + This class tracks episodic stats, such as final returns + and env complexity metrics. Works on a per-env basis. + """ + + def __init__(self, names, step_metrics_names=[], window=None): + self.names = names + self.step_metric_names = step_metrics_names + self.window = window + + @partial(jax.jit, static_argnums=(0, 1)) + def reset_stats(self, batch_shape=(1,)): + stats = { + 'n_episodes': jnp.zeros((*(batch_shape), 1), dtype=jnp.uint32), + 'n_steps': jnp.zeros((*(batch_shape), 1), dtype=jnp.uint32), + 'reward': jnp.zeros((*(batch_shape), 1), dtype=jnp.float32), + } + stats.update({ + name: jnp.zeros((*(batch_shape), 1)) for name in self.names + }) + + if self.window is not None: + # Average over window + stats.update({ + f'{name}_buffer': jnp.zeros((*(batch_shape), self.window)) + for name in self.names + }) + + return stats + + @partial(jax.jit, static_argnums=(0,)) + def update_stats(self, stats, done, info, max_episodes=jnp.inf): + n_eps = stats['n_episodes'] + n_steps = stats['n_steps'] + + + for name in self.names: + # Update stat + if name not in info: + continue + + new_val = info[name] + + # NOTE: in MA settings sparse and dense rewards are per agent. + new_val = new_val.sum() + + # Only record first max_episode episodes + done = done*(n_eps < max_episodes) + if name in self.step_metric_names: + n_incr_prev = n_steps + n_incr_total = n_steps + 1 + _metric_done = True + else: + n_incr_prev = n_eps + n_incr_total = n_eps + done + _metric_done = done + + if self.window is None: + mean = stats[name] + new_mean = self._update_stat_mean( + new_val, mean, n_incr_total, _metric_done + ) + else: + buffer_key = f'{name}_buffer' + buffer = stats[buffer_key] + new_mean, buffer = self._update_stat_window( + new_val, buffer, n_incr_total, n_incr_prev, _metric_done + ) + stats.update({buffer_key: buffer}) + + stats.update({ + name: new_mean, + }) + + # Only update n_episodes based on real episodes + if name in self.step_metric_names: + stats.update({ + 'n_steps': n_incr_total + }) + else: + stats.update({ + 'n_episodes': n_incr_total + }) + + return stats + + @partial(jax.jit, static_argnums=(0,)) + def _update_stat_mean(self, new_val, mean, n_eps_total, done): + z = 1/jnp.maximum(1, n_eps_total) + new_mean = done*(mean*(1-z) + new_val*z) + (1-done)*mean + + return new_mean + + @partial(jax.jit, static_argnums=(0,)) + def _update_stat_window(self, new_val, buffer, n_eps_total, n_eps_prev, done): + cur_val = buffer[n_eps_prev % self.window] + new_val = done*new_val + (1-done)*cur_val + buffer = buffer.at[n_eps_prev % self.window].set(new_val) + new_mean = buffer.sum()/jnp.maximum(jnp.minimum(self.window, n_eps_total), 1) + + return new_mean, buffer diff --git a/src/minimax/util/rl/rollout_storage.py b/src/minimax/util/rl/rollout_storage.py new file mode 100644 index 0000000..59a032a --- /dev/null +++ b/src/minimax/util/rl/rollout_storage.py @@ -0,0 +1,227 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial +from collections import namedtuple + +import numpy as np +import jax +import jax.numpy as jnp + +import minimax.util.pytree as _tree_util +from .ued_scores import compute_episodic_stats + + +RolloutBatch = namedtuple( + 'RolloutBatch', ( + 'obs', + 'actions', + 'rewards', + 'dones', + 'log_pis', + 'values', + 'targets', + 'advantages', + 'carry' + )) + + +class RolloutStorage: + def __init__( + self, + discount, + gae_lambda, + n_envs, + n_eval, + n_steps, + action_space, + obs_space, + agent, + n_agents=1): + self.discount = discount + self.gae_lambda = gae_lambda + self.n_agents = n_agents + self.n_steps = n_steps + self.n_envs = n_envs + self.n_evals = n_eval + self.flat_batch_size = n_envs*self.n_evals + self.action_space = action_space + self.value_ensemble_size = agent.model.value_ensemble_size + + dummy_rng = jax.random.PRNGKey(0) + self.empty_obs = \ + jax.jax.tree_util.tree_map( + lambda x: jnp.empty((n_agents, n_steps, self.flat_batch_size) + + x.shape, dtype=x.dtype), + obs_space.sample(dummy_rng)) + + self.empty_action = \ + jax.jax.tree_util.tree_map( + lambda x: jnp.empty((n_agents, n_steps, self.flat_batch_size) + + x.shape, dtype=x.dtype), + action_space.sample(dummy_rng)) + + if agent.is_recurrent: + self.empty_carry = \ + agent.init_carry( + dummy_rng, batch_dims=(n_agents, self.n_steps, self.flat_batch_size)) + else: + self.empty_carry = None + + if agent.is_recurrent: + self.append = jax.vmap(self._append_with_carry, in_axes=0) + else: + self.append = jax.vmap(self._append_without_carry, in_axes=0) + self.get_batch = jax.vmap(self._get_batch) + self.get_return_stats = jax.vmap( + self._get_return_stats, in_axes=(0, None)) + + @partial(jax.jit, static_argnums=0) + def reset(self): + """ + Maintains a pytree of rollout transitions and metadata + """ + if self.empty_carry is None: + carry_buffer = None + else: + carry_buffer = self.empty_carry + + value_batch_size = (self.flat_batch_size,) + if self.value_ensemble_size > 1: + value_batch_size += (self.value_ensemble_size,) + + return { + "obs": self.empty_obs, + "actions": self.empty_action, + "rewards": jnp.empty( + (self.n_agents, self.n_steps, + self.flat_batch_size), dtype=jnp.float32 + ), + "dones": jnp.empty((self.n_agents, self.n_steps, self.flat_batch_size), dtype=jnp.uint8), + "log_pis_old": jnp.empty( + (self.n_agents, self.n_steps, + self.flat_batch_size), dtype=jnp.float32 + ), + "values_old": jnp.empty( + (self.n_agents, self.n_steps, * + value_batch_size), dtype=jnp.float32 + ), + "carry": carry_buffer, + "_t": jnp.zeros((self.n_agents,), dtype=jnp.uint32) # for vmap + } + + @partial(jax.jit, static_argnums=0) + def _append(self, buffer, obs, action, reward, done, log_pi, value, carry): + if carry is not None: + carry_buffer = _tree_util.pytree_set_array_at( + buffer["carry"], buffer["_t"], carry) + else: + carry_buffer = None + + return { + "obs": _tree_util.pytree_set_struct_at(buffer["obs"], buffer["_t"], obs), + "actions": _tree_util.pytree_set_struct_at(buffer["actions"], buffer["_t"], action), + "rewards": buffer["rewards"].at[buffer["_t"]].set(reward.squeeze()), + "dones": buffer["dones"].at[buffer["_t"]].set(done.squeeze()), + "log_pis_old": buffer["log_pis_old"].at[buffer["_t"]].set(log_pi), + "values_old": buffer["values_old"].at[buffer["_t"]].set(value.squeeze()), + "carry": carry_buffer, + "_t": (buffer["_t"] + 1) % self.n_steps, + } + + @partial(jax.jit, static_argnums=0) + def _append_with_carry(self, buffer, obs, action, reward, done, log_pi, value, carry): + return self._append(buffer, obs, action, reward, done, log_pi, value, carry) + + @partial(jax.jit, static_argnums=0) + def _append_without_carry(self, buffer, obs, action, reward, done, log_pi, value): + return self._append(buffer, obs, action, reward, done, log_pi, value, None) + + @partial(jax.jit, static_argnums=(0,)) + def _get_batch(self, buffer, last_value): + _dones = buffer["dones"] + rewards = buffer["rewards"] + + gae, target = self.compute_gae( + value=buffer["values_old"], + reward=rewards, + done=_dones, + last_value=last_value + ) + + # T x N x E x M --> N x T x EM if recurrent or N x TEM if not + if self.empty_carry is not None: + carry = buffer["carry"] + else: + carry = None + + batch_kwargs = dict( + obs=buffer["obs"], + actions=buffer["actions"], + rewards=rewards, + dones=_dones, + log_pis=buffer["log_pis_old"], + values=buffer["values_old"], + targets=target, + advantages=gae, + carry=carry, + ) + + return RolloutBatch(**batch_kwargs) + + def compute_gae(self, value, reward, done, last_value): + def _compute_gae(carry, step): + (discount, gae_lambda, gae, value_next) = carry + value, reward, done = step + + value_diff = discount*value_next*(1-done) - value + delta = reward + value_diff + + gae = delta + discount*gae_lambda*(1-done) * gae + + return (discount, gae_lambda, gae, value), gae + + value, reward, done = jnp.flip(value, 0), jnp.flip( + reward, 0), jnp.flip(done, 0) + + # Handle ensemble values, which have an extra ensemble dim at index -1 + if value.shape != done.shape: + reward = jnp.expand_dims(reward, -1) + done = jnp.expand_dims(done, -1) + + gae = jnp.zeros(value.shape[1:]) + _, advantages = jax.lax.scan( + _compute_gae, + (self.discount, self.gae_lambda, gae, last_value), + (value, reward, done), + length=len(reward) + ) + advantages = jnp.flip(advantages, 0) + targets = advantages + jnp.flip(value, 0) + + return advantages, targets + + def _get_return_stats(self, rollout, control_idxs=None): + if control_idxs is not None: + positive_signs = (control_idxs == 0) + reward_signs = -1*(positive_signs.astype(jnp.float32) - + (~positive_signs).astype(jnp.float32)) + rewards = rollout["rewards"]*reward_signs + else: + rewards = rollout["rewards"] + + pop_batch_shape = (self.n_steps, self.n_envs, self.n_evals) + rewards = jnp.flip(rewards.reshape(*pop_batch_shape), 0) + dones = jnp.flip(rollout["dones"].reshape(*pop_batch_shape), 0) + + return compute_episodic_stats(rewards, dones) + + def set_final_reward(self, rollout, reward): + rollout["rewards"] = rollout["rewards"].at[:, -1, :].set(reward) + + return rollout diff --git a/src/minimax/util/rl/rollout_storage_seperate.py b/src/minimax/util/rl/rollout_storage_seperate.py new file mode 100644 index 0000000..51f4d52 --- /dev/null +++ b/src/minimax/util/rl/rollout_storage_seperate.py @@ -0,0 +1,274 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial +from collections import namedtuple + +import jax +import jax.numpy as jnp + +import minimax.util.pytree as _tree_util +from .ued_scores import compute_episodic_stats + + +RolloutBatch = namedtuple( + 'RolloutBatch', ( + 'obs', + 'obs_shared', + 'actions', + 'rewards', + 'dones', + 'log_pis', + 'values', + 'targets', + 'advantages', + 'actor_carry', + 'critic_carry' + )) + + +class RolloutStorageSeperate: + def __init__( + self, + discount, + gae_lambda, + n_envs, + n_eval, + n_steps, + action_space, + obs_space, + obs_space_shared_shape, + agent, + n_agents=1): + self.discount = discount + self.gae_lambda = gae_lambda + + # NOTE: n_students refers to minimax's use of n_agents + # Since I added a multi agent env I need an actual n_agents + self.n_students = n_agents + self.n_env_agents = 2 + self.n_steps = n_steps + self.n_envs = n_envs + self.n_evals = n_eval + self.flat_batch_size = n_envs*self.n_evals + self.action_space = action_space + + dummy_rng = jax.random.PRNGKey(0) + self.empty_obs = \ + jax.jax.tree_util.tree_map( + lambda x: jnp.empty( + (self.n_students, n_steps, self.flat_batch_size, + self.n_env_agents) + x.shape, + dtype=x.dtype + ), + obs_space.sample(dummy_rng)) + + self.empty_obs_shared = jnp.empty( + (self.n_students, n_steps, self.flat_batch_size, + self.n_env_agents) + obs_space_shared_shape) + + self.empty_action = \ + jax.jax.tree_util.tree_map( + lambda x: jnp.empty( + (self.n_students, n_steps, self.flat_batch_size, self.n_env_agents) + x.shape, dtype=x.dtype), + action_space.sample(dummy_rng)) + + if agent.is_recurrent: + self.empty_actor_carry, self.empty_critic_carry = \ + agent.init_carry( + dummy_rng, batch_dims=( + self.n_students, n_steps, self.flat_batch_size, self.n_env_agents)) + else: + self.empty_actor_carry, self.empty_critic_carry = None, None + + if agent.is_recurrent: + self.append = jax.vmap(self._append_with_carry, in_axes=0) + else: + self.append = jax.vmap(self._append_without_carry, in_axes=0) + self.get_batch = jax.vmap(self._get_batch) + self.get_return_stats = jax.vmap( + self._get_return_stats, in_axes=(0, None)) + + @partial(jax.jit, static_argnums=0) + def reset(self): + """ + Maintains a pytree of rollout transitions and metadata + """ + if self.empty_actor_carry is None and self.empty_critic_carry is None: + actor_carry_buffer = None + critic_carry_buffer = None + else: + actor_carry_buffer = self.empty_actor_carry + critic_carry_buffer = self.empty_critic_carry + + value_batch_size = (self.flat_batch_size,) + + return { + "obs": self.empty_obs, + "obs_shared": self.empty_obs_shared, + "actions": self.empty_action, + "rewards": jnp.empty( + (self.n_students, self.n_steps, self.flat_batch_size, + self.n_env_agents), dtype=jnp.float32 + ), + "shaped_rewards": jnp.empty( + (self.n_students, self.n_steps, self.flat_batch_size, + self.n_env_agents), dtype=jnp.float32 + ), + "dones": jnp.empty((self.n_students, self.n_steps, self.flat_batch_size, + self.n_env_agents), dtype=jnp.uint8), + "log_pis_old": jnp.empty( + (self.n_students, self.n_steps, self.flat_batch_size, + self.n_env_agents), dtype=jnp.float32 + ), + "values_old": jnp.empty( + (self.n_students, self.n_steps, + *value_batch_size, self.n_env_agents), dtype=jnp.float32 + ), + "actor_carry": actor_carry_buffer, + "critic_carry": critic_carry_buffer, + "_t": jnp.zeros((self.n_students,), dtype=jnp.uint32) # for vmap + } + + @partial(jax.jit, static_argnums=0) + def _append(self, buffer, obs, obs_shared, action, reward, shaped_reward, done, log_pi, value, actor_carry, critic_carry): + if actor_carry is not None: + actor_carry_buffer = _tree_util.pytree_set_array_at( + buffer["actor_carry"], buffer["_t"], actor_carry) + else: + actor_carry_buffer = None + + if critic_carry is not None: + critic_carry_buffer = _tree_util.pytree_set_array_at( + buffer["critic_carry"], buffer["_t"], critic_carry) + else: + critic_carry_buffer = None + + obs = _tree_util.pytree_set_struct_at(buffer["obs"], buffer["_t"], obs) + obs_shared = _tree_util.pytree_set_struct_at( + buffer["obs_shared"], buffer["_t"], obs_shared) + + return { + "obs": obs, + "obs_shared": obs_shared, + "actions": _tree_util.pytree_set_struct_at(buffer["actions"], buffer["_t"], action), + "rewards": buffer["rewards"].at[buffer["_t"]].set(reward.squeeze()), + "shaped_rewards": buffer["shaped_rewards"].at[buffer["_t"]].set(shaped_reward.squeeze()), + "dones": buffer["dones"].at[buffer["_t"]].set(done.squeeze()), + "log_pis_old": buffer["log_pis_old"].at[buffer["_t"]].set(log_pi), + "values_old": buffer["values_old"].at[buffer["_t"]].set(value.squeeze()), + "actor_carry": actor_carry_buffer, + "critic_carry": critic_carry_buffer, + "_t": (buffer["_t"] + 1) % self.n_steps, + } + + @partial(jax.jit, static_argnums=0) + def _append_with_carry(self, buffer, obs, obs_shared, action, reward, shaped_reward, done, log_pi, value, actor_carry, critic_carry): + return self._append(buffer, obs, obs_shared, action, reward, shaped_reward, done, log_pi, value, actor_carry, critic_carry) + + @partial(jax.jit, static_argnums=0) + def _append_without_carry(self, buffer, obs, obs_shared, action, reward, shaped_reward, done, log_pi, value): + return self._append(buffer, obs, obs_shared, action, reward, shaped_reward, done, log_pi, value, None, None) + + @partial(jax.jit, static_argnums=(0,)) + # , intrinsic_reward_coeff=0.0): + def _get_batch(self, buffer, last_value, shaped_reward_coeff=None): + _dones = buffer["dones"] + rewards = buffer["rewards"] + + # if intrinsic_reward is not None: + # rewards = rewards + 0.0001 * intrinsic_reward_coeff * intrinsic_reward + # 0.0001 * + jax.debug.print("rewards buffer {r}", r=rewards.mean()) + + rewards = rewards + shaped_reward_coeff.mean() * \ + buffer["shaped_rewards"] + + jax.debug.print("rewards buffer {r}", r=rewards.mean()) + + gae, target = self.compute_gae( + value=buffer["values_old"], + reward=rewards, + done=_dones, + last_value=last_value + ) + + # T x N x E x M --> N x T x EM if recurrent or N x TEM if not + if self.empty_actor_carry is not None and self.empty_critic_carry is not None: + actor_carry = buffer["actor_carry"] + critic_carry = buffer["critic_carry"] + else: + actor_carry = None + critic_carry = None + + batch_kwargs = dict( + obs=buffer["obs"], + obs_shared=buffer["obs_shared"], + actions=buffer["actions"], + rewards=rewards, + dones=_dones, + log_pis=buffer["log_pis_old"], + values=buffer["values_old"], + targets=target, + advantages=gae, + actor_carry=actor_carry, + critic_carry=critic_carry + ) + return RolloutBatch(**batch_kwargs) + + def compute_gae(self, value, reward, done, last_value): + def _compute_gae(carry, step): + (discount, gae_lambda, gae, value_next) = carry + value, reward, done = step + + value_diff = discount*value_next*(1-done) - value + delta = reward + value_diff + + gae = delta + discount*gae_lambda*(1-done) * gae + + return (discount, gae_lambda, gae, value), gae + + value, reward, done = jnp.flip(value, 0), jnp.flip( + reward, 0), jnp.flip(done, 0) + + # Handle ensemble values, which have an extra ensemble dim at index -1 + if value.shape != done.shape: + reward = jnp.expand_dims(reward, -1) + done = jnp.expand_dims(done, -1) + + gae = jnp.zeros(value.shape[1:]) + _, advantages = jax.lax.scan( + _compute_gae, + (self.discount, self.gae_lambda, gae, last_value), + (value, reward, done), + length=len(reward) + ) + advantages = jnp.flip(advantages, 0) + targets = advantages + jnp.flip(value, 0) + + return advantages, targets + + def _get_return_stats(self, rollout, control_idxs=None): + if control_idxs is not None: + positive_signs = (control_idxs == 0) + reward_signs = -1*(positive_signs.astype(jnp.float32) - + (~positive_signs).astype(jnp.float32)) + rewards = rollout["rewards"]*reward_signs + else: + rewards = rollout["rewards"] + + pop_batch_shape = (self.n_steps, self.n_envs, self.n_evals) + rewards = jnp.flip(rewards.reshape(*pop_batch_shape), 0) + dones = jnp.flip(rollout["dones"].reshape(*pop_batch_shape), 0) + + return compute_episodic_stats(rewards, dones) + + def set_final_reward(self, rollout, reward): + rollout["rewards"] = rollout["rewards"].at[:, -1, :].set(reward) + + return rollout diff --git a/src/minimax/util/rl/training.py b/src/minimax/util/rl/training.py new file mode 100644 index 0000000..4e8c2db --- /dev/null +++ b/src/minimax/util/rl/training.py @@ -0,0 +1,201 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from typing import Any, Callable + +import jax +import jax.numpy as jnp +from flax import core +from flax import struct +import optax +import chex + +from .plr import PLRBuffer + + +class VmapMAPPOTrainState(struct.PyTreeNode): + n_iters: chex.Array + n_updates: chex.Array # per agent + n_grad_updates: chex.Array # per agent + actor_apply_fn: Callable = struct.field(pytree_node=False) + actor_params: core.FrozenDict[str, Any] + actor_tx: optax.GradientTransformation = struct.field(pytree_node=False) + actor_opt_state: optax.OptState + + critic_apply_fn: Callable = struct.field(pytree_node=False) + critic_params: core.FrozenDict[str, Any] + critic_tx: optax.GradientTransformation = struct.field(pytree_node=False) + critic_opt_state: optax.OptState + + shaped_reward_coeff: float = 0.0 + + plr_buffer: PLRBuffer = None + + def apply_gradients(self, *, actor_grads, critic_grads, **kwargs): + # Actor update + actor_updates, new_actor_opt_state = self.actor_tx.update( + actor_grads, self.actor_opt_state, self.actor_params) + new_actor_params = optax.apply_updates( + self.actor_params, actor_updates) + + # Critic update + critic_updates, new_critic_opt_state = self.critic_tx.update( + critic_grads, self.critic_opt_state, self.critic_params) + new_critic_params = optax.apply_updates( + self.critic_params, critic_updates) + + return self.replace( + n_grad_updates=self.n_updates + 1, + actor_params=new_actor_params, + actor_opt_state=new_actor_opt_state, + critic_params=new_critic_params, + critic_opt_state=new_critic_opt_state, + **kwargs, + ) + + @classmethod + def create( + cls, *, + actor_apply_fn, + actor_params, + actor_tx, + critic_apply_fn, + critic_params, + critic_tx, + **kwargs + ): + actor_opt_state = jax.vmap(actor_tx.init)(actor_params) + critic_opt_state = jax.vmap(critic_tx.init)(critic_params) + return cls( + n_iters=jnp.array(jax.vmap(lambda x: 0)( + actor_params), dtype=jnp.uint32), + n_updates=jnp.array(jax.vmap(lambda x: 0) + (actor_params), dtype=jnp.uint32), + n_grad_updates=jnp.array( + jax.vmap(lambda x: 0)(actor_params), dtype=jnp.uint32), + actor_apply_fn=actor_apply_fn, + actor_params=actor_params, + actor_tx=actor_tx, + actor_opt_state=actor_opt_state, + critic_apply_fn=critic_apply_fn, + critic_params=critic_params, + critic_tx=critic_tx, + critic_opt_state=critic_opt_state, + **kwargs, + ) + + def increment(self): + return self.replace( + n_iters=self.n_iters + 1, + ) + + def increment_updates(self): + return self.replace( + n_updates=self.n_updates + 1, + ) + + @property + def state_dict(self): + return dict( + n_iters=self.n_iters, + n_updates=self.n_updates, + n_grad_updates=self.n_grad_updates, + actor_params=self.actor_params, + actor_opt_state=self.actor_opt_state, + critic_params=self.critic_params, + critic_opt_state=self.critic_opt_state, + ) + + def set_new_shaped_reward_coeff(self, new_coeff): + return self.replace( + shaped_reward_coeff=new_coeff + ) + + def load_state_dict(self, state): + return self.replace( + n_iters=state['n_iters'], + n_updates=state['n_updates'], + n_grad_updates=state['n_grad_updates'], + actor_params=state['actor_params'], + actor_opt_state=state['actor_opt_state'], + critic_params=state['critic_params'], + critic_opt_state=state['critic_opt_state'], + ) + + +class VmapTrainState(struct.PyTreeNode): + n_iters: chex.Array + n_updates: chex.Array # per agent + n_grad_updates: chex.Array # per agent + apply_fn: Callable = struct.field(pytree_node=False) + params: core.FrozenDict[str, Any] + tx: optax.GradientTransformation = struct.field(pytree_node=False) + opt_state: optax.OptState + plr_buffer: PLRBuffer = None + + def apply_gradients(self, *, grads, **kwargs): + updates, new_opt_state = self.tx.update( + grads, self.opt_state, self.params) + new_params = optax.apply_updates(self.params, updates) + + return self.replace( + n_grad_updates=self.n_updates + 1, + params=new_params, + opt_state=new_opt_state, + **kwargs, + ) + + @classmethod + def create(cls, *, + apply_fn, + params, + tx, + **kwargs + ): + opt_state = jax.vmap(tx.init)(params) + return cls( + n_iters=jnp.array(jax.vmap(lambda x: 0)(params), dtype=jnp.uint32), + n_updates=jnp.array(jax.vmap(lambda x: 0) + (params), dtype=jnp.uint32), + n_grad_updates=jnp.array( + jax.vmap(lambda x: 0)(params), dtype=jnp.uint32), + apply_fn=apply_fn, + params=params, + tx=tx, + opt_state=opt_state, + **kwargs, + ) + + def increment(self): + return self.replace( + n_iters=self.n_iters + 1, + ) + + def increment_updates(self): + return self.replace( + n_updates=self.n_updates + 1, + ) + + @property + def state_dict(self): + return dict( + n_iters=self.n_iters, + n_updates=self.n_updates, + n_grad_updates=self.n_grad_updates, + params=self.params, + opt_state=self.opt_state + ) + + def load_state_dict(self, state): + return self.replace( + n_iters=state['n_iters'], + n_updates=state['n_updates'], + n_grad_updates=state['n_grad_updates'], + params=state['params'], + opt_state=state['opt_state'] + ) diff --git a/src/minimax/util/rl/ued_scores.py b/src/minimax/util/rl/ued_scores.py new file mode 100644 index 0000000..7492c48 --- /dev/null +++ b/src/minimax/util/rl/ued_scores.py @@ -0,0 +1,245 @@ +""" +Copyright (c) Meta Platforms, Inc. and affiliates. +All rights reserved. + +This source code is licensed under the license found in the +LICENSE file in the root directory of this source tree. +""" + +from functools import partial +from enum import Enum +from collections import namedtuple + +import einops +import jax +import jax.numpy as jnp + + +class UEDScore(Enum): + RELATIVE_REGRET = 1 + MEAN_RELATIVE_REGRET = 2 + POPULATION_REGRET = 3 + RETURN = 4 + NEG_RETURN = 5 + L1_VALUE_LOSS = 6 + POSITIVE_VALUE_LOSS = 7 + MAX_MC = 8 + VALUE_DISAGREEMENT = 9 + + +@partial(jax.jit, static_argnums=(2, 3)) +def compute_episodic_stats( + metrics, + dones, + time_average=False, + partial_metrics=0, + partial_steps=0, + return_partial=False): + env_batch_shape = dones.shape[1:] + n_episodes = jnp.zeros(env_batch_shape, dtype=jnp.uint32) + sum_ep_metrics = jnp.zeros(env_batch_shape, dtype=jnp.float32) + partial_metrics = jnp.zeros(env_batch_shape, dtype=jnp.float32) + max_metrics = jnp.zeros(env_batch_shape, dtype=jnp.float32) + steps = jnp.zeros(env_batch_shape, dtype=jnp.float32) + partial_steps = jnp.zeros(env_batch_shape, dtype=jnp.float32) + + def _compute_metrics(carry, step): + (n_episodes, + sum_ep_metrics, + max_metrics, + partial_metrics, + partial_steps) = carry + + _metrics, _dones = step + + partial_metrics += _metrics + partial_steps += 1 + + if time_average: + ep_metric = partial_metrics/partial_steps + else: + ep_metric = partial_metrics + + sum_ep_metrics += _dones*ep_metric + max_metrics = _dones * \ + jnp.maximum(max_metrics, ep_metric) + (1-_dones)*max_metrics + + n_episodes += _dones + + partial_metrics = (1-_dones)*partial_metrics + partial_steps = (1-_dones)*partial_steps + + return ( + n_episodes, + sum_ep_metrics, + max_metrics, + partial_metrics, + partial_steps + ), None + + (n_episodes, sum_ep_metrics, max_metrics, partial_metrics, partial_steps), _ = jax.lax.scan( + _compute_metrics, + (n_episodes, sum_ep_metrics, max_metrics, partial_metrics, partial_steps), + (metrics, dones), + length=len(metrics) + ) + + """Score per level based on two agents.""" + if len(n_episodes.shape) == 3: # n_parallel envs x n_parallel eval_envs x n_env_agents + n_episodes = n_episodes.sum(-1) + max_metrics = max_metrics.max(-1) + sum_ep_metrics = sum_ep_metrics.sum(-1) + + # Take mean over eval dimension + total_metrics_per_env = sum_ep_metrics.sum(-1) + n_episodes_per_env = n_episodes.sum(-1) + n_episodes_per_env = jnp.maximum(n_episodes_per_env, 1) + + # Take max over eval dimension + max_metrics_per_env = max_metrics.max(-1) + + return total_metrics_per_env/n_episodes_per_env, max_metrics_per_env + + +@partial(jax.jit, static_argnums=(0,)) +def _compute_ued_scores(score_name: UEDScore, batch: namedtuple, info=None): + """ + Compute UED score from a rollout batch. + Individual score functions return a tuple of mean_scores and max_scores, + where each is of dimension n_agents x n_envs. + """ + if score_name in [UEDScore.RELATIVE_REGRET, UEDScore.MEAN_RELATIVE_REGRET, UEDScore.POPULATION_REGRET]: + mean_scores, max_scores, score_info = compute_return(batch) + + elif score_name == UEDScore.RETURN: + mean_scores, max_scores, score_info = compute_return(batch) + + elif score_name == UEDScore.NEG_RETURN: + batch = batch._replace(rewards=-batch.rewards) + mean_scores, max_scores, score_info = compute_return(batch) + + elif score_name == UEDScore.MAX_MC: + mean_scores, max_scores, score_info = compute_max_mc(batch, info) + + elif score_name == UEDScore.L1_VALUE_LOSS: + mean_scores, max_scores, score_info = compute_l1_value_loss(batch) + + elif score_name == UEDScore.POSITIVE_VALUE_LOSS: + mean_scores, max_scores, score_info = compute_positive_value_loss( + batch) + + elif score_name == UEDScore.VALUE_DISAGREEMENT: + mean_scores, max_scores, score_info = compute_value_disagreement(batch) + + return mean_scores, max_scores, score_info + + +@partial(jax.jit, static_argnums=(0, 2, 4, 5)) +def compute_ued_scores(score_name: UEDScore, batch: namedtuple, n_eval: int, info: dict = None, ignore_val=None, per_agent=False): + if len(batch.dones.shape) == 3: + n_agents, n_steps, flat_batch_size = batch.dones.shape + else: + n_agents, n_steps, flat_batch_size, _ = batch.dones.shape + # pop_batch_shape = (n_agents, n_steps, flat_batch_size//n_eval, n_eval) + # batch = jax.tree_util.tree_map(lambda x: x.reshape( + # *pop_batch_shape, *x.shape[3:]), batch) + + batch = jax.tree_util.tree_map( + lambda x: einops.rearrange( + x, 'a t (s e) ... -> a t s e ...', + a=n_agents, t=n_steps, s=flat_batch_size, e=n_eval), batch) + + mean_env_returns_per_agent, max_env_returns_per_agent, score_info = \ + jax.vmap(_compute_ued_scores, in_axes=(None, 0, 0))( + score_name, batch, info + ) + + if score_name in [UEDScore.RELATIVE_REGRET, UEDScore.MEAN_RELATIVE_REGRET]: + assert len(mean_env_returns_per_agent) == 2, \ + "Standard PAIRED requires exactly 2 agents." + + if score_name == UEDScore.RELATIVE_REGRET: + scores = jnp.clip(max_env_returns_per_agent[1] + - mean_env_returns_per_agent[0], 0) + + elif score_name == UEDScore.MEAN_RELATIVE_REGRET: + scores = jnp.clip(mean_env_returns_per_agent[1] + - mean_env_returns_per_agent[0], 0) + + elif score_name == UEDScore.POPULATION_REGRET: + max_env_returns = max_env_returns_per_agent.max(0) + mean_env_returns = mean_env_returns_per_agent.mean(0) + scores = max_env_returns - mean_env_returns + else: + if per_agent: + scores = mean_env_returns_per_agent + max_scores = max_env_returns_per_agent + else: + scores = mean_env_returns_per_agent.mean(0) + max_scores = max_env_returns_per_agent.max(0) + + if ignore_val is not None: + if per_agent: + axis = (1, -1) if len(batch.dones.shape) == 3 else (1, -2, -1) + else: + axis = (0, 1, -1) if len(batch.dones.shape) == 3 else (0, 1, -2, -1) + + incomplete_idxs = batch.dones.sum(axis=axis) == 0 + + scores = jnp.where(incomplete_idxs, ignore_val, scores) + return scores, score_info + +# ======== UED score computations ======== + + +def compute_return(batch): + mean_scores, max_scores = compute_episodic_stats( + batch.rewards, batch.dones, time_average=False) + + return mean_scores, max_scores, None + + +def compute_l1_value_loss(batch): + mean_scores, max_scores = compute_episodic_stats( + jnp.abs(batch.advantages), batch.dones, time_average=True) + + return mean_scores, max_scores, None + + +def compute_positive_value_loss(batch): + mean_scores, max_scores = compute_episodic_stats( + jnp.clip(batch.advantages, 0), batch.dones, time_average=True) + + return mean_scores, max_scores, None + + +def compute_max_mc(batch, info): + _, max_env_returns_per_agent = \ + compute_episodic_stats(batch.rewards, batch.dones, time_average=False) + + max_returns = jnp.maximum(max_env_returns_per_agent, info['max_returns']) + # Multi Agent setting, we have mutlitple values. + if len(batch.dones.shape) == 4: + max_returns = jnp.concatenate( + [max_returns[jnp.newaxis, :, jnp.newaxis, jnp.newaxis], + max_returns[jnp.newaxis, :, jnp.newaxis, jnp.newaxis]], axis=-1 + ) + else: + max_returns = max_returns[jnp.newaxis, :, jnp.newaxis] + mean_scores, max_scores = compute_episodic_stats( + max_returns - batch.values, # Can be negative + batch.dones, + time_average=True + ) + + score_info = {'max_returns': max_env_returns_per_agent} + + return mean_scores, max_scores, score_info + + +def compute_value_disagreement(batch): + mean_scores, max_scores = compute_episodic_stats( + batch.values.std(-1), batch.dones, time_average=True + ) + + return mean_scores, max_scores, None diff --git a/src/run_results_txt/al_all_xpid_against_population_in_all_69_layouts_out.txt b/src/run_results_txt/al_all_xpid_against_population_in_all_69_layouts_out.txt new file mode 100644 index 0000000..5fc6345 --- /dev/null +++ b/src/run_results_txt/al_all_xpid_against_population_in_all_69_layouts_out.txt @@ -0,0 +1,54 @@ +Evaluating Overcooked-CoordRing6_9 against population for 9SEED_9_dr-overcookedNonexNonewNone_fs_FIXcoord_ring_6_9_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr3e-5g0.99cv0.5ce0.01e5mb1l0.95_pc0.2_h64cf32fc2se5ba_re_0 + +----------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 39.25+/- 4.04 (max: 127.8) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 16.08+/- 0.5815 (max: 26.51) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.5467+/- 0.05222 (max: 1.0) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 6.2 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 9.25 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +----------------------------------------------------------------------------------------- + +Evaluating Overcooked-ForcedCoord6_9 against population for 9SEED_9_dr-overcookedNonexNonewNone_fs_FIXforced_coord_6_9_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr3e-5g0.99cv0.5ce0.01e5mb1l0.95_pc0.2_h64cf32fc2se5ba_re_0 + + +------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 37.32+/- 6.509 (max: 140.4) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 12.47+/- 0.8239 (max: 28.85) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.389+/- 0.06325 (max: 1.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.2 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 1.99 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +------------------------------------------------------------------------------------------- + +Evaluating Overcooked-CounterCircuit6_9 against population for 9SEED_9_dr-overcookedNonexNonewNone_fs_FIXcounter_circuit_6_9_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr3e-5g0.99cv0.5ce0.01e5mb1l0.95_pc0.2_h64cf32fc2se5ba_re_0 + +---------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 45.24+/- 2.834 (max: 76.8) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 15.99+/- 0.572 (max: 29.8) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.7121+/- 0.04546 (max: 1.0) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 5.6 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 9.415 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.01 | +---------------------------------------------------------------------------------------------- + +Evaluating Overcooked-AsymmAdvantages6_9 against population for 9SEED_9_dr-overcookedNonexNonewNone_fs_FIXasymm_advantages_6_9_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr3e-5g0.99cv0.5ce0.01e5mb1l0.95_pc0.2_h64cf32fc2se5ba_re_0 + +----------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 152.0+/- 9.415 (max: 230.4) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 23.14+/- 1.555 (max: 45.34) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.9665+/- 0.01026 (max: 1.0) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 48.8 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 8.697 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.74 | +----------------------------------------------------------------------------------------------- +Evaluating Overcooked-CrampedRoom6_9 against population for 9SEED_9_dr-overcookedNonexNonewNone_fs_FIXcramped_room_6_9_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr3e-5g0.99cv0.5ce0.01e5mb1l0.95_pc0.2_h64cf32fc2se5ba_re_0 + +--------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 150.3+/- 5.073 (max: 203.4) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 19.68+/- 0.7425 (max: 41.5) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.9973+/- 0.0008802 (max: 1.0) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 94.6 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 12.67 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.98 | +--------------------------------------------------------------------------------------------- diff --git a/src/run_results_txt/eval_all_xpid_against_population_in_all_layouts_out.txt b/src/run_results_txt/eval_all_xpid_against_population_in_all_layouts_out.txt new file mode 100644 index 0000000..d516295 --- /dev/null +++ b/src/run_results_txt/eval_all_xpid_against_population_in_all_layouts_out.txt @@ -0,0 +1,138 @@ +Evaluating dr against population in Overcooked-CoordRing5_5 for dr-overcooked5x5w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 + +------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CoordRing5_5 | 15.35+/- 0.6965 (max: 23.2) | +| eval/a0:test_return_std:Overcooked-CoordRing5_5 | 13.78+/- 0.2437 (max: 17.16) | +| eval/a0:test_solved_rate:Overcooked-CoordRing5_5 | 0.1606+/- 0.01226 (max: 0.35) | +| min:eval/a0:test_return:Overcooked-CoordRing5_5 | 6.4 | +| min:eval/a0:test_return_std:Overcooked-CoordRing5_5 | 10.15 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing5_5 | 0.02 | +------------------------------------------------------------------------------------------ +Evaluating dr against population in Overcooked-ForcedCoord5_5 for dr-overcooked5x5w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 + +------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-ForcedCoord5_5 | 0.6208+/- 0.06281 (max: 1.8) | +| eval/a0:test_return_std:Overcooked-ForcedCoord5_5 | 3.16+/- 0.199 (max: 5.724) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord5_5 | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord5_5 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord5_5 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord5_5 | 0.0 | +------------------------------------------------------------------------------------------- +Evaluating dr against population in Overcooked-CrampedRoom5_5 for dr-overcooked5x5w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 + +-------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom5_5 | 60.29+/- 4.157 (max: 99.4) | +| eval/a0:test_return_std:Overcooked-CrampedRoom5_5 | 27.89+/- 1.176 (max: 38.66) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom5_5 | 0.7073+/- 0.04138 (max: 0.98) | +| min:eval/a0:test_return:Overcooked-CrampedRoom5_5 | 15.8 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom5_5 | 14.53 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom5_5 | 0.18 | +-------------------------------------------------------------------------------------------- +Evaluating plr against population in Overcooked-CoordRing5_5 for plr-overcooked5x5w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 + +-------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing5_5 | 1.212+/- 0.2802 (max: 8.6) | +| eval/a0:test_return_std:Overcooked-CoordRing5_5 | 3.613+/- 0.4178 (max: 11.4) | +| eval/a0:test_solved_rate:Overcooked-CoordRing5_5 | 0.002708+/- 0.00102 (max: 0.04) | +| min:eval/a0:test_return:Overcooked-CoordRing5_5 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing5_5 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing5_5 | 0.0 | +-------------------------------------------------------------------------------------------- +Evaluating plr against population in Overcooked-ForcedCoord5_5 for plr-overcooked5x5w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 + +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-ForcedCoord5_5 | 0.3167+/- 0.1021 (max: 3.2) | +| eval/a0:test_return_std:Overcooked-ForcedCoord5_5 | 1.334+/- 0.2965 (max: 7.332) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord5_5 | 0.0002083+/- 0.0002083 (max: 0.01) | +| min:eval/a0:test_return:Overcooked-ForcedCoord5_5 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord5_5 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord5_5 | 0.0 | +------------------------------------------------------------------------------------------------- +Evaluating plr against population in Overcooked-CrampedRoom5_5 for plr-overcooked5x5w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 + +------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom5_5 | 56.94+/- 3.333 (max: 89.0) | +| eval/a0:test_return_std:Overcooked-CrampedRoom5_5 | 26.26+/- 0.8617 (max: 35.83) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom5_5 | 0.7342+/- 0.03327 (max: 1.0) | +| min:eval/a0:test_return:Overcooked-CrampedRoom5_5 | 21.8 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom5_5 | 16.99 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom5_5 | 0.32 | +------------------------------------------------------------------------------------------- +Evaluating paired against population in Overcooked-CoordRing5_5 for paired-overcooked5x5w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 + +------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CoordRing5_5 | 19.15+/- 0.6706 (max: 26.4) | +| eval/a0:test_return_std:Overcooked-CoordRing5_5 | 13.84+/- 0.1905 (max: 16.12) | +| eval/a0:test_solved_rate:Overcooked-CoordRing5_5 | 0.2283+/- 0.01399 (max: 0.42) | +| eval/a1:test_return:Overcooked-CoordRing5_5 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing5_5 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing5_5 | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CoordRing5_5 | 9.6 | +| min:eval/a0:test_return_std:Overcooked-CoordRing5_5 | 11.61 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing5_5 | 0.06 | +| min:eval/a1:test_return:Overcooked-CoordRing5_5 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing5_5 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing5_5 | 0.0 | +------------------------------------------------------------------------------------------ +Evaluating paired against population in Overcooked-ForcedCoord5_5 for paired-overcooked5x5w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 + +------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-ForcedCoord5_5 | 0.375+/- 0.05446 (max: 1.6) | +| eval/a0:test_return_std:Overcooked-ForcedCoord5_5 | 2.292+/- 0.2046 (max: 5.426) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord5_5 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord5_5 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord5_5 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord5_5 | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord5_5 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord5_5 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord5_5 | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord5_5 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord5_5 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord5_5 | 0.0 | +------------------------------------------------------------------------------------------- +Evaluating paired against population in Overcooked-CrampedRoom5_5 for paired-overcooked5x5w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 + +------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom5_5 | 66.46+/- 3.787 (max: 101.2) | +| eval/a0:test_return_std:Overcooked-CrampedRoom5_5 | 27.66+/- 1.134 (max: 42.87) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom5_5 | 0.7854+/- 0.02626 (max: 1.0) | +| eval/a1:test_return:Overcooked-CrampedRoom5_5 | 3.1+/- 0.4522 (max: 6.2) | +| eval/a1:test_return_std:Overcooked-CrampedRoom5_5 | 5.039+/- 0.735 (max: 10.08) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom5_5 | 0.01+/- 0.001459 (max: 0.02) | +| min:eval/a0:test_return:Overcooked-CrampedRoom5_5 | 27.2 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom5_5 | 17.95 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom5_5 | 0.44 | +| min:eval/a1:test_return:Overcooked-CrampedRoom5_5 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom5_5 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom5_5 | 0.0 | +------------------------------------------------------------------------------------------- +Evaluating accel against population in Overcooked-CoordRing5_5 for plr-overcooked5x5w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 + +----------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing5_5 | 19.28+/- 0.7542 (max: 27.2) | +| eval/a0:test_return_std:Overcooked-CoordRing5_5 | 13.71+/- 0.2366 (max: 17.09) | +| eval/a0:test_solved_rate:Overcooked-CoordRing5_5 | 0.2304+/- 0.0163 (max: 0.45) | +| min:eval/a0:test_return:Overcooked-CoordRing5_5 | 10.2 | +| min:eval/a0:test_return_std:Overcooked-CoordRing5_5 | 10.44 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing5_5 | 0.04 | +----------------------------------------------------------------------------------------- +Evaluating accel against population in Overcooked-ForcedCoord5_5 for plr-overcooked5x5w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 + +---------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-ForcedCoord5_5 | 4.071+/- 0.6144 (max: 12.6) | +| eval/a0:test_return_std:Overcooked-ForcedCoord5_5 | 6.018+/- 0.5814 (max: 11.54) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord5_5 | 0.00625+/- 0.001826 (max: 0.05) | +| min:eval/a0:test_return:Overcooked-ForcedCoord5_5 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord5_5 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord5_5 | 0.0 | +---------------------------------------------------------------------------------------------- +Evaluating accel against population in Overcooked-CrampedRoom5_5 for plr-overcooked5x5w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 + +-------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom5_5 | 63.35+/- 4.381 (max: 97.6) | +| eval/a0:test_return_std:Overcooked-CrampedRoom5_5 | 23.11+/- 0.861 (max: 32.16) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom5_5 | 0.7387+/- 0.04596 (max: 0.99) | +| min:eval/a0:test_return:Overcooked-CrampedRoom5_5 | 10.8 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom5_5 | 12.14 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom5_5 | 0.06 | +-------------------------------------------------------------------------------------------- diff --git a/src/run_results_txt/eval_xpid_all_cnn_lstm_out.txt b/src/run_results_txt/eval_xpid_all_cnn_lstm_out.txt new file mode 100644 index 0000000..07d6024 --- /dev/null +++ b/src/run_results_txt/eval_xpid_all_cnn_lstm_out.txt @@ -0,0 +1,2381 @@ +Evaluating DR_CNN-LSTM_SEED1 against population in Overcooked-CoordRing6_9 for xpid dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [4.199999809265137, 5.400000095367432, 18.600000381469727, 19.0, 17.600000381469727, 16.399999618530273, 12.799999237060547, 12.0, 31.799999237060547, 32.599998474121094, 15.399999618530273, 15.199999809265137, 13.0, 11.0, 25.799999237060547, 22.399999618530273, 16.399999618530273, 20.19999885559082, 18.0, 15.59999942779541, 18.799999237060547, 20.399999618530273, 18.600000381469727, 19.399999618530273, 13.399999618530273, 18.0, 19.19999885559082, 18.0, 18.399999618530273, 18.799999237060547, 17.0, 13.59999942779541, 26.19999885559082, 28.799999237060547, 29.799999237060547, 31.0, 6.599999904632568, 3.799999952316284, 23.799999237060547, 25.399999618530273, 29.0, 26.799999237060547, 20.0, 14.399999618530273, 19.799999237060547, 17.19999885559082, 21.399999618530273, 21.399999618530273] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [4.199999809265137, 5.400000095367432, 12.799999237060547, 12.0, 13.0, 11.0, 18.0, 15.59999942779541, 13.399999618530273, 18.0, 17.0, 13.59999942779541, 6.599999904632568, 3.799999952316284, 20.0, 14.399999618530273] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [18.600000381469727, 19.0, 31.799999237060547, 32.599998474121094, 25.799999237060547, 22.399999618530273, 18.799999237060547, 20.399999618530273, 19.19999885559082, 18.0, 26.19999885559082, 28.799999237060547, 23.799999237060547, 25.399999618530273, 19.799999237060547, 17.19999885559082] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [17.600000381469727, 16.399999618530273, 15.399999618530273, 15.199999809265137, 16.399999618530273, 20.19999885559082, 18.600000381469727, 19.399999618530273, 18.399999618530273, 18.799999237060547, 29.799999237060547, 31.0, 29.0, 26.799999237060547, 21.399999618530273, 21.399999618530273] +----------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 18.8+/- 0.9826 (max: 32.6) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 20.99+/- 1.315 (max: 31.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 12.42+/- 1.266 (max: 20.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 22.99+/- 1.244 (max: 32.6) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 13.49+/- 0.304 (max: 17.52) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 13.64+/- 0.5608 (max: 17.52) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 12.74+/- 0.6188 (max: 15.36) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 14.09+/- 0.3293 (max: 17.37) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.226+/- 0.02245 (max: 0.6) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.2469+/- 0.04221 (max: 0.53) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.1187+/- 0.01932 (max: 0.23) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.3125+/- 0.03586 (max: 0.6) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 3.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 15.2 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 3.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 17.2 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 7.846 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 10.69 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 7.846 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 12.28 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.07 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.17 | +----------------------------------------------------------------------------------------------- +Evaluating DR_CNN-LSTM_SEED1 against population in Overcooked-ForcedCoord6_9 for xpid dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [0.3999999761581421, 0.0, 0.3999999761581421, 0.19999998807907104, 0.0, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 0.19999998807907104, 0.0, 1.0, 0.3999999761581421, 0.7999999523162842, 0.0, 0.0, 0.19999998807907104, 0.5999999642372131, 0.0, 0.3999999761581421, 0.0, 1.1999999284744263, 0.0, 0.19999998807907104, 0.19999998807907104, 0.5999999642372131, 0.19999998807907104, 0.19999998807907104, 0.3999999761581421, 0.5999999642372131, 0.0, 0.3999999761581421, 0.0, 0.0, 0.19999998807907104, 0.3999999761581421, 0.19999998807907104, 0.7999999523162842, 0.19999998807907104, 0.19999998807907104, 0.0, 0.5999999642372131, 0.19999998807907104, 0.5999999642372131, 0.19999998807907104, 1.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [0.3999999761581421, 0.0, 1.1999999284744263, 0.0, 1.0, 0.3999999761581421, 0.5999999642372131, 0.0, 0.19999998807907104, 0.19999998807907104, 0.5999999642372131, 0.0, 0.3999999761581421, 0.19999998807907104, 0.5999999642372131, 0.19999998807907104] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [0.3999999761581421, 0.19999998807907104, 1.1999999284744263, 0.0, 0.7999999523162842, 0.0, 0.3999999761581421, 0.0, 0.5999999642372131, 0.19999998807907104, 0.3999999761581421, 0.0, 0.7999999523162842, 0.19999998807907104, 0.5999999642372131, 0.19999998807907104] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [0.0, 0.0, 0.19999998807907104, 0.0, 0.0, 0.19999998807907104, 1.1999999284744263, 0.0, 0.19999998807907104, 0.3999999761581421, 0.0, 0.19999998807907104, 0.19999998807907104, 0.0, 1.0, 0.0] +----------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.325+/- 0.05144 (max: 1.2) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.225+/- 0.09106 (max: 1.2) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.375+/- 0.08921 (max: 1.2) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.375+/- 0.08732 (max: 1.2) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 1.944+/- 0.2301 (max: 4.75) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 1.366+/- 0.4049 (max: 4.75) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 2.231+/- 0.3882 (max: 4.75) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 2.236+/- 0.387 (max: 4.75) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +----------------------------------------------------------------------------------------------- +Evaluating DR_CNN-LSTM_SEED1 against population in Overcooked-CounterCircuit6_9 for xpid dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [11.0, 8.0, 18.0, 12.399999618530273, 10.800000190734863, 9.0, 5.799999713897705, 4.799999713897705, 11.0, 8.59999942779541, 1.7999999523162842, 0.0, 3.799999952316284, 2.0, 11.399999618530273, 8.0, 10.0, 5.400000095367432, 7.599999904632568, 4.199999809265137, 19.600000381469727, 17.799999237060547, 8.0, 3.3999998569488525, 8.399999618530273, 6.0, 16.399999618530273, 12.399999618530273, 4.799999713897705, 2.799999952316284, 10.59999942779541, 7.799999713897705, 14.199999809265137, 9.59999942779541, 6.599999904632568, 3.5999999046325684, 6.599999904632568, 5.0, 15.199999809265137, 11.399999618530273, 8.800000190734863, 2.3999998569488525, 10.800000190734863, 7.599999904632568, 18.0, 13.0, 7.199999809265137, 2.3999998569488525] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [11.0, 8.0, 5.799999713897705, 4.799999713897705, 3.799999952316284, 2.0, 7.599999904632568, 4.199999809265137, 8.399999618530273, 6.0, 10.59999942779541, 7.799999713897705, 6.599999904632568, 5.0, 10.800000190734863, 7.599999904632568] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [18.0, 12.399999618530273, 11.0, 8.59999942779541, 11.399999618530273, 8.0, 19.600000381469727, 17.799999237060547, 16.399999618530273, 12.399999618530273, 14.199999809265137, 9.59999942779541, 15.199999809265137, 11.399999618530273, 18.0, 13.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [10.800000190734863, 9.0, 1.7999999523162842, 0.0, 10.0, 5.400000095367432, 8.0, 3.3999998569488525, 4.799999713897705, 2.799999952316284, 6.599999904632568, 3.5999999046325684, 8.800000190734863, 2.3999998569488525, 7.199999809265137, 2.3999998569488525] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 8.625+/- 0.6853 (max: 19.6) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 5.437+/- 0.8184 (max: 10.8) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 6.875+/- 0.6519 (max: 11.0) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 13.56+/- 0.9017 (max: 19.6) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 10.74+/- 0.468 (max: 18.22) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 8.364+/- 0.7386 (max: 11.83) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 9.884+/- 0.4215 (max: 12.45) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 13.96+/- 0.4691 (max: 18.22) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.05458+/- 0.01037 (max: 0.26) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.01125+/- 0.004171 (max: 0.05) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.02062+/- 0.005588 (max: 0.07) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.1319+/- 0.019 (max: 0.26) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 2.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 8.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 6.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 10.58 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.02 | +------------------------------------------------------------------------------------------------------ +Evaluating DR_CNN-LSTM_SEED1 against population in Overcooked-AsymmAdvantages6_9 for xpid dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [2.3999998569488525, 2.200000047683716, 1.399999976158142, 8.199999809265137, 2.799999952316284, 1.5999999046325684, 2.799999952316284, 5.0, 1.399999976158142, 41.20000076293945, 2.5999999046325684, 1.1999999284744263, 2.3999998569488525, 34.599998474121094, 2.3999998569488525, 56.0, 3.3999998569488525, 12.399999618530273, 1.7999999523162842, 13.59999942779541, 2.799999952316284, 10.199999809265137, 3.5999999046325684, 4.199999809265137, 2.5999999046325684, 1.7999999523162842, 2.3999998569488525, 18.399999618530273, 0.5999999642372131, 2.200000047683716, 1.7999999523162842, 3.5999999046325684, 1.7999999523162842, 26.799999237060547, 3.1999998092651367, 1.399999976158142, 2.0, 3.3999998569488525, 1.7999999523162842, 15.799999237060547, 4.199999809265137, 4.799999713897705, 2.3999998569488525, 5.199999809265137, 2.200000047683716, 23.799999237060547, 3.799999952316284, 0.5999999642372131] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [2.3999998569488525, 2.200000047683716, 2.799999952316284, 5.0, 2.3999998569488525, 34.599998474121094, 1.7999999523162842, 13.59999942779541, 2.5999999046325684, 1.7999999523162842, 1.7999999523162842, 3.5999999046325684, 2.0, 3.3999998569488525, 2.3999998569488525, 5.199999809265137] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [1.399999976158142, 8.199999809265137, 1.399999976158142, 41.20000076293945, 2.3999998569488525, 56.0, 2.799999952316284, 10.199999809265137, 2.3999998569488525, 18.399999618530273, 1.7999999523162842, 26.799999237060547, 1.7999999523162842, 15.799999237060547, 2.200000047683716, 23.799999237060547] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [2.799999952316284, 1.5999999046325684, 2.5999999046325684, 1.1999999284744263, 3.3999998569488525, 12.399999618530273, 3.5999999046325684, 4.199999809265137, 0.5999999642372131, 2.200000047683716, 3.1999998092651367, 1.399999976158142, 4.199999809265137, 4.799999713897705, 3.799999952316284, 0.5999999642372131] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 7.433+/- 1.638 (max: 56.0) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 3.287+/- 0.6906 (max: 12.4) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 5.475+/- 2.071 (max: 34.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 13.54+/- 4.072 (max: 56.0) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 9.837+/- 0.9617 (max: 32.25) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 7.049+/- 0.5832 (max: 12.26) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 9.034+/- 1.551 (max: 30.44) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 13.43+/- 2.128 (max: 32.25) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.07521+/- 0.02527 (max: 0.81) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0075+/- 0.003476 (max: 0.05) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.04187+/- 0.03055 (max: 0.49) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.1762+/- 0.06323 (max: 0.81) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.6 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.6 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 1.8 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 1.4 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 3.412 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 3.412 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 5.724 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 5.103 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating DR_CNN-LSTM_SEED1 against population in Overcooked-CrampedRoom6_9 for xpid dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [10.399999618530273, 10.59999942779541, 69.4000015258789, 65.79999542236328, 86.79999542236328, 90.5999984741211, 16.799999237060547, 20.399999618530273, 78.0, 78.5999984741211, 99.19999694824219, 101.79999542236328, 19.0, 16.799999237060547, 99.5999984741211, 95.4000015258789, 103.5999984741211, 102.79999542236328, 21.0, 18.799999237060547, 95.0, 92.0, 80.5999984741211, 82.4000015258789, 7.599999904632568, 7.599999904632568, 80.0, 71.19999694824219, 92.5999984741211, 98.19999694824219, 17.19999885559082, 19.19999885559082, 92.0, 86.0, 102.39999389648438, 106.5999984741211, 18.600000381469727, 15.59999942779541, 85.5999984741211, 88.4000015258789, 98.79999542236328, 102.79999542236328, 22.799999237060547, 20.0, 101.0, 92.4000015258789, 93.5999984741211, 94.19999694824219] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [10.399999618530273, 10.59999942779541, 16.799999237060547, 20.399999618530273, 19.0, 16.799999237060547, 21.0, 18.799999237060547, 7.599999904632568, 7.599999904632568, 17.19999885559082, 19.19999885559082, 18.600000381469727, 15.59999942779541, 22.799999237060547, 20.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [69.4000015258789, 65.79999542236328, 78.0, 78.5999984741211, 99.5999984741211, 95.4000015258789, 95.0, 92.0, 80.0, 71.19999694824219, 92.0, 86.0, 85.5999984741211, 88.4000015258789, 101.0, 92.4000015258789] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [86.79999542236328, 90.5999984741211, 99.19999694824219, 101.79999542236328, 103.5999984741211, 102.79999542236328, 80.5999984741211, 82.4000015258789, 92.5999984741211, 98.19999694824219, 102.39999389648438, 106.5999984741211, 98.79999542236328, 102.79999542236328, 93.5999984741211, 94.19999694824219] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 66.04+/- 5.285 (max: 106.6) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 96.06+/- 1.954 (max: 106.6) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 16.4+/- 1.194 (max: 22.8) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 85.65+/- 2.703 (max: 101.0) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 24.58+/- 1.152 (max: 36.82) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 31.47+/- 0.8677 (max: 36.82) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 14.41+/- 0.4756 (max: 16.73) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 27.86+/- 0.8525 (max: 34.67) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.6948+/- 0.05278 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.955+/- 0.008266 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.1906+/- 0.02437 (max: 0.32) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.9387+/- 0.009995 (max: 1.0) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 7.6 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 80.6 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 7.6 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 65.8 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 10.87 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 26.54 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 10.87 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 22.3 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.03 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.86 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.03 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.87 | +------------------------------------------------------------------------------------------------- + + + + + + + + + +Evaluating DR_CNN-LSTM_SEED2 against population in Overcooked-CoordRing6_9 for xpid SEED_2_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [4.199999809265137, 3.1999998092651367, 15.199999809265137, 11.800000190734863, 14.399999618530273, 15.0, 11.800000190734863, 11.800000190734863, 20.399999618530273, 19.399999618530273, 13.59999942779541, 14.199999809265137, 9.399999618530273, 10.399999618530273, 19.399999618530273, 16.399999618530273, 12.399999618530273, 12.399999618530273, 13.0, 9.199999809265137, 11.800000190734863, 12.399999618530273, 10.399999618530273, 11.59999942779541, 12.0, 11.199999809265137, 14.199999809265137, 15.799999237060547, 12.199999809265137, 15.199999809265137, 11.800000190734863, 14.0, 21.0, 21.399999618530273, 21.0, 23.0, 3.1999998092651367, 2.0, 13.0, 15.799999237060547, 20.0, 21.19999885559082, 16.799999237060547, 14.59999942779541, 8.800000190734863, 10.399999618530273, 8.399999618530273, 12.399999618530273] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [4.199999809265137, 3.1999998092651367, 11.800000190734863, 11.800000190734863, 9.399999618530273, 10.399999618530273, 13.0, 9.199999809265137, 12.0, 11.199999809265137, 11.800000190734863, 14.0, 3.1999998092651367, 2.0, 16.799999237060547, 14.59999942779541] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [15.199999809265137, 11.800000190734863, 20.399999618530273, 19.399999618530273, 19.399999618530273, 16.399999618530273, 11.800000190734863, 12.399999618530273, 14.199999809265137, 15.799999237060547, 21.0, 21.399999618530273, 13.0, 15.799999237060547, 8.800000190734863, 10.399999618530273] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [14.399999618530273, 15.0, 13.59999942779541, 14.199999809265137, 12.399999618530273, 12.399999618530273, 10.399999618530273, 11.59999942779541, 12.199999809265137, 15.199999809265137, 21.0, 23.0, 20.0, 21.19999885559082, 8.399999618530273, 12.399999618530273] +------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CoordRing6_9 | 13.4+/- 0.6984 (max: 23.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 14.84+/- 1.061 (max: 23.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 9.912+/- 1.116 (max: 16.8) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 15.45+/- 0.9924 (max: 21.4) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 11.96+/- 0.2779 (max: 16.06) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 11.84+/- 0.3689 (max: 15.07) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 11.33+/- 0.6342 (max: 13.96) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 12.72+/- 0.3488 (max: 16.06) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.09854+/- 0.01253 (max: 0.34) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.09937+/- 0.02589 (max: 0.31) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.06625+/- 0.0119 (max: 0.13) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.13+/- 0.02297 (max: 0.34) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 2.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 8.4 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 2.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 8.8 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 6.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 9.992 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 6.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 11.42 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.04 | +------------------------------------------------------------------------------------------------ +Evaluating DR_CNN-LSTM_SEED2 against population in Overcooked-ForcedCoord6_9 for xpid SEED_2_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 + +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [2.3999998569488525, 0.0, 3.5999999046325684, 0.19999998807907104, 1.1999999284744263, 0.0, 1.399999976158142, 0.7999999523162842, 2.200000047683716, 0.19999998807907104, 1.1999999284744263, 0.0, 2.3999998569488525, 1.1999999284744263, 1.1999999284744263, 0.7999999523162842, 1.5999999046325684, 1.0, 1.7999999523162842, 0.3999999761581421, 1.5999999046325684, 0.0, 1.7999999523162842, 0.19999998807907104, 2.0, 0.3999999761581421, 2.799999952316284, 0.7999999523162842, 3.0, 0.19999998807907104, 2.799999952316284, 0.3999999761581421, 1.5999999046325684, 0.19999998807907104, 0.3999999761581421, 0.19999998807907104, 1.399999976158142, 0.0, 1.1999999284744263, 0.0, 0.7999999523162842, 0.0, 2.200000047683716, 0.0, 3.1999998092651367, 0.0, 3.0, 0.19999998807907104] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [2.3999998569488525, 0.0, 1.399999976158142, 0.7999999523162842, 2.3999998569488525, 1.1999999284744263, 1.7999999523162842, 0.3999999761581421, 2.0, 0.3999999761581421, 2.799999952316284, 0.3999999761581421, 1.399999976158142, 0.0, 2.200000047683716, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [3.5999999046325684, 0.19999998807907104, 2.200000047683716, 0.19999998807907104, 1.1999999284744263, 0.7999999523162842, 1.5999999046325684, 0.0, 2.799999952316284, 0.7999999523162842, 1.5999999046325684, 0.19999998807907104, 1.1999999284744263, 0.0, 3.1999998092651367, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.5999999046325684, 1.0, 1.7999999523162842, 0.19999998807907104, 3.0, 0.19999998807907104, 0.3999999761581421, 0.19999998807907104, 0.7999999523162842, 0.0, 3.0, 0.19999998807907104] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 1.125+/- 0.1505 (max: 3.6) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.925+/- 0.2496 (max: 3.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 1.225+/- 0.2407 (max: 2.8) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 1.225+/- 0.2977 (max: 3.6) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 3.793+/- 0.3557 (max: 7.684) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 3.407+/- 0.6112 (max: 7.681) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 4.075+/- 0.607 (max: 6.94) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 3.898+/- 0.6569 (max: 7.684) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0002083+/- 0.0002083 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.000625+/- 0.000625 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating DR_CNN-LSTM_SEED2 against population in Overcooked-CounterCircuit6_9 for xpid SEED_2_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 + +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [9.199999809265137, 9.399999618530273, 16.399999618530273, 10.399999618530273, 12.799999237060547, 8.59999942779541, 5.199999809265137, 6.199999809265137, 6.199999809265137, 4.799999713897705, 1.5999999046325684, 0.5999999642372131, 1.7999999523162842, 1.1999999284744263, 7.399999618530273, 5.599999904632568, 6.399999618530273, 4.0, 7.799999713897705, 6.0, 18.399999618530273, 14.0, 9.800000190734863, 5.199999809265137, 7.599999904632568, 5.400000095367432, 16.19999885559082, 11.800000190734863, 3.3999998569488525, 2.5999999046325684, 9.0, 7.399999618530273, 18.399999618530273, 11.0, 6.199999809265137, 2.799999952316284, 6.399999618530273, 3.799999952316284, 11.0, 7.199999809265137, 8.199999809265137, 4.199999809265137, 11.199999809265137, 9.0, 25.799999237060547, 16.600000381469727, 9.59999942779541, 3.799999952316284] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [9.199999809265137, 9.399999618530273, 5.199999809265137, 6.199999809265137, 1.7999999523162842, 1.1999999284744263, 7.799999713897705, 6.0, 7.599999904632568, 5.400000095367432, 9.0, 7.399999618530273, 6.399999618530273, 3.799999952316284, 11.199999809265137, 9.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [16.399999618530273, 10.399999618530273, 6.199999809265137, 4.799999713897705, 7.399999618530273, 5.599999904632568, 18.399999618530273, 14.0, 16.19999885559082, 11.800000190734863, 18.399999618530273, 11.0, 11.0, 7.199999809265137, 25.799999237060547, 16.600000381469727] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [12.799999237060547, 8.59999942779541, 1.5999999046325684, 0.5999999642372131, 6.399999618530273, 4.0, 9.800000190734863, 5.199999809265137, 3.3999998569488525, 2.5999999046325684, 6.199999809265137, 2.799999952316284, 8.199999809265137, 4.199999809265137, 9.59999942779541, 3.799999952316284] +----------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 8.283+/- 0.7421 (max: 25.8) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 5.612+/- 0.8486 (max: 12.8) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 6.662+/- 0.6901 (max: 11.2) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 12.57+/- 1.45 (max: 25.8) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 10.52+/- 0.4794 (max: 21.41) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 8.498+/- 0.5618 (max: 12.5) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 9.672+/- 0.5273 (max: 12.46) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 13.39+/- 0.8337 (max: 21.41) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.05+/- 0.01179 (max: 0.38) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.00875+/- 0.00499 (max: 0.08) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.02+/- 0.005845 (max: 0.07) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.1212+/- 0.02711 (max: 0.38) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.6 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.6 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 1.2 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 4.8 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 3.412 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 3.412 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 4.75 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 8.98 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +----------------------------------------------------------------------------------------------------- +Evaluating DR_CNN-LSTM_SEED2 against population in Overcooked-AsymmAdvantages6_9 for xpid SEED_2_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 + +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [1.1999999284744263, 3.1999998092651367, 0.5999999642372131, 6.799999713897705, 1.5999999046325684, 1.0, 1.399999976158142, 7.0, 1.1999999284744263, 41.39999771118164, 3.3999998569488525, 1.0, 2.0, 31.399999618530273, 1.399999976158142, 52.599998474121094, 2.3999998569488525, 8.199999809265137, 1.399999976158142, 13.0, 1.0, 6.0, 4.400000095367432, 1.399999976158142, 1.1999999284744263, 1.0, 1.399999976158142, 15.59999942779541, 0.5999999642372131, 0.7999999523162842, 1.1999999284744263, 5.0, 1.0, 27.19999885559082, 1.7999999523162842, 0.3999999761581421, 0.5999999642372131, 3.1999998092651367, 1.0, 15.0, 1.5999999046325684, 2.5999999046325684, 1.5999999046325684, 3.799999952316284, 1.0, 18.600000381469727, 3.3999998569488525, 0.5999999642372131] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [1.1999999284744263, 3.1999998092651367, 1.399999976158142, 7.0, 2.0, 31.399999618530273, 1.399999976158142, 13.0, 1.1999999284744263, 1.0, 1.1999999284744263, 5.0, 0.5999999642372131, 3.1999998092651367, 1.5999999046325684, 3.799999952316284] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.5999999642372131, 6.799999713897705, 1.1999999284744263, 41.39999771118164, 1.399999976158142, 52.599998474121094, 1.0, 6.0, 1.399999976158142, 15.59999942779541, 1.0, 27.19999885559082, 1.0, 15.0, 1.0, 18.600000381469727] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [1.5999999046325684, 1.0, 3.3999998569488525, 1.0, 2.3999998569488525, 8.199999809265137, 4.400000095367432, 1.399999976158142, 0.5999999642372131, 0.7999999523162842, 1.7999999523162842, 0.3999999761581421, 1.5999999046325684, 2.5999999046325684, 3.3999998569488525, 0.5999999642372131] +-------------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 6.358+/- 1.576 (max: 52.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 2.2+/- 0.4943 (max: 8.2) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 4.887+/- 1.934 (max: 31.4) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 11.99+/- 3.994 (max: 52.6) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 8.624+/- 0.9728 (max: 33.54) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 5.68+/- 0.4988 (max: 9.837) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 8.102+/- 1.428 (max: 25.96) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 12.09+/- 2.282 (max: 33.54) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.06833+/- 0.02479 (max: 0.72) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.000625+/- 0.000625 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.04625+/- 0.03188 (max: 0.5) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.1581+/- 0.0623 (max: 0.72) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.4 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.4 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 0.6 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.6 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 2.8 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 2.8 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 3.412 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 3.412 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +-------------------------------------------------------------------------------------------------------- +Evaluating DR_CNN-LSTM_SEED2 against population in Overcooked-CrampedRoom6_9 for xpid SEED_2_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 + +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [12.199999809265137, 10.399999618530273, 76.4000015258789, 69.5999984741211, 99.4000015258789, 98.19999694824219, 14.799999237060547, 13.199999809265137, 76.19999694824219, 72.19999694824219, 95.5999984741211, 102.39999389648438, 18.799999237060547, 12.399999618530273, 95.5999984741211, 97.4000015258789, 100.0, 97.19999694824219, 19.19999885559082, 16.0, 85.79999542236328, 84.5999984741211, 86.19999694824219, 84.5999984741211, 7.599999904632568, 10.59999942779541, 75.19999694824219, 68.79999542236328, 93.0, 97.5999984741211, 20.600000381469727, 19.19999885559082, 82.4000015258789, 85.4000015258789, 102.39999389648438, 105.79999542236328, 19.19999885559082, 15.0, 77.0, 79.79999542236328, 101.79999542236328, 104.39999389648438, 21.600000381469727, 18.0, 87.0, 84.4000015258789, 100.4000015258789, 92.79999542236328] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [12.199999809265137, 10.399999618530273, 14.799999237060547, 13.199999809265137, 18.799999237060547, 12.399999618530273, 19.19999885559082, 16.0, 7.599999904632568, 10.59999942779541, 20.600000381469727, 19.19999885559082, 19.19999885559082, 15.0, 21.600000381469727, 18.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [76.4000015258789, 69.5999984741211, 76.19999694824219, 72.19999694824219, 95.5999984741211, 97.4000015258789, 85.79999542236328, 84.5999984741211, 75.19999694824219, 68.79999542236328, 82.4000015258789, 85.4000015258789, 77.0, 79.79999542236328, 87.0, 84.4000015258789] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [99.4000015258789, 98.19999694824219, 95.5999984741211, 102.39999389648438, 100.0, 97.19999694824219, 86.19999694824219, 84.5999984741211, 93.0, 97.5999984741211, 102.39999389648438, 105.79999542236328, 101.79999542236328, 104.39999389648438, 100.4000015258789, 92.79999542236328] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 64.76+/- 5.248 (max: 105.8) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 97.61+/- 1.502 (max: 105.8) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 15.55+/- 1.042 (max: 21.6) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 81.11+/- 2.08 (max: 97.4) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 24.14+/- 1.036 (max: 36.09) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 29.95+/- 0.7254 (max: 36.09) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 14.76+/- 0.4026 (max: 17.85) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 27.72+/- 0.6638 (max: 32.16) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.6962+/- 0.05406 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.9725+/- 0.005951 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.1769+/- 0.01944 (max: 0.29) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.9394+/- 0.00704 (max: 0.99) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 7.6 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 84.6 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 7.6 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 68.8 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 11.93 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 25.23 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 11.93 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 23.09 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.06 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.92 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.06 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.9 | +------------------------------------------------------------------------------------------------- + + + + + + + + + + +Evaluating DR_CNN-LSTM_SEED3 against population in Overcooked-CoordRing6_9 for xpid SEED_3_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 + +k eval/a0:test_return:Overcooked-CoordRing6_9, v [3.0, 4.799999713897705, 18.600000381469727, 15.199999809265137, 24.399999618530273, 21.799999237060547, 14.0, 14.799999237060547, 36.20000076293945, 35.20000076293945, 26.0, 27.19999885559082, 12.0, 12.0, 26.799999237060547, 26.0, 23.799999237060547, 23.600000381469727, 15.799999237060547, 10.59999942779541, 20.19999885559082, 21.600000381469727, 22.799999237060547, 18.600000381469727, 17.19999885559082, 18.799999237060547, 27.0, 26.0, 25.799999237060547, 26.399999618530273, 15.799999237060547, 14.0, 26.599998474121094, 29.799999237060547, 34.0, 30.0, 4.599999904632568, 3.1999998092651367, 24.19999885559082, 26.599998474121094, 29.0, 29.0, 11.800000190734863, 15.0, 21.19999885559082, 21.399999618530273, 25.19999885559082, 20.799999237060547] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [3.0, 4.799999713897705, 14.0, 14.799999237060547, 12.0, 12.0, 15.799999237060547, 10.59999942779541, 17.19999885559082, 18.799999237060547, 15.799999237060547, 14.0, 4.599999904632568, 3.1999998092651367, 11.800000190734863, 15.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [18.600000381469727, 15.199999809265137, 36.20000076293945, 35.20000076293945, 26.799999237060547, 26.0, 20.19999885559082, 21.600000381469727, 27.0, 26.0, 26.599998474121094, 29.799999237060547, 24.19999885559082, 26.599998474121094, 21.19999885559082, 21.399999618530273] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [24.399999618530273, 21.799999237060547, 26.0, 27.19999885559082, 23.799999237060547, 23.600000381469727, 22.799999237060547, 18.600000381469727, 25.799999237060547, 26.399999618530273, 34.0, 30.0, 29.0, 29.0, 25.19999885559082, 20.799999237060547] +----------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 20.8+/- 1.165 (max: 36.2) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 25.52+/- 0.9564 (max: 34.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 11.71+/- 1.279 (max: 18.8) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 25.16+/- 1.394 (max: 36.2) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 13.49+/- 0.3268 (max: 17.55) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 14.33+/- 0.5036 (max: 17.55) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 12.02+/- 0.6642 (max: 15.18) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 14.11+/- 0.3018 (max: 16.98) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.2794+/- 0.0261 (max: 0.73) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.3756+/- 0.02985 (max: 0.65) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.09187+/- 0.01713 (max: 0.2) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.3706+/- 0.04128 (max: 0.73) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 3.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 18.6 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 3.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 15.2 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 7.141 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 11.14 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 7.141 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 12.98 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.21 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.15 | +----------------------------------------------------------------------------------------------- +Evaluating DR_CNN-LSTM_SEED3 against population in Overcooked-ForcedCoord6_9 for xpid SEED_3_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 + +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [1.399999976158142, 0.3999999761581421, 2.0, 0.0, 1.0, 0.0, 2.200000047683716, 0.19999998807907104, 2.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.0, 0.0, 2.200000047683716, 0.0, 1.399999976158142, 0.0, 0.7999999523162842, 0.0, 2.0, 0.0, 2.0, 0.0, 0.5999999642372131, 0.19999998807907104, 1.1999999284744263, 0.19999998807907104, 1.0, 0.19999998807907104, 0.19999998807907104, 0.0, 0.3999999761581421, 0.0, 0.7999999523162842, 0.19999998807907104, 0.3999999761581421, 0.0, 1.1999999284744263, 0.0, 2.3999998569488525, 0.0, 4.199999809265137, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [1.399999976158142, 0.3999999761581421, 2.200000047683716, 0.19999998807907104, 1.1999999284744263, 0.0, 2.200000047683716, 0.0, 2.0, 0.0, 1.1999999284744263, 0.19999998807907104, 0.3999999761581421, 0.0, 1.1999999284744263, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [2.0, 0.0, 2.0, 0.19999998807907104, 1.1999999284744263, 0.0, 1.399999976158142, 0.0, 2.0, 0.0, 1.0, 0.19999998807907104, 0.7999999523162842, 0.19999998807907104, 2.3999998569488525, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [1.0, 0.0, 1.0, 0.19999998807907104, 1.0, 0.0, 0.7999999523162842, 0.0, 0.5999999642372131, 0.19999998807907104, 0.19999998807907104, 0.0, 0.3999999761581421, 0.0, 4.199999809265137, 0.0] +------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.7417+/- 0.1311 (max: 4.2) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.6+/- 0.2595 (max: 4.2) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.7875+/- 0.2085 (max: 2.2) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.8375+/- 0.2208 (max: 2.4) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 2.779+/- 0.3499 (max: 8.146) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 2.333+/- 0.5881 (max: 8.146) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 2.965+/- 0.6155 (max: 6.258) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 3.037+/- 0.6372 (max: 6.499) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------ +Evaluating DR_CNN-LSTM_SEED3 against population in Overcooked-CounterCircuit6_9 for xpid SEED_3_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 + +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [8.0, 5.400000095367432, 14.199999809265137, 9.800000190734863, 13.399999618530273, 13.399999618530273, 5.199999809265137, 7.0, 12.0, 6.599999904632568, 3.0, 1.399999976158142, 5.400000095367432, 3.3999998569488525, 12.799999237060547, 10.399999618530273, 12.399999618530273, 8.800000190734863, 5.400000095367432, 6.799999713897705, 17.799999237060547, 17.19999885559082, 11.199999809265137, 4.0, 6.799999713897705, 7.0, 21.399999618530273, 18.600000381469727, 9.199999809265137, 7.599999904632568, 8.800000190734863, 7.799999713897705, 14.799999237060547, 10.800000190734863, 9.800000190734863, 7.399999618530273, 7.0, 5.0, 15.799999237060547, 12.199999809265137, 7.0, 3.5999999046325684, 6.399999618530273, 5.799999713897705, 12.799999237060547, 10.800000190734863, 8.399999618530273, 4.199999809265137] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [8.0, 5.400000095367432, 5.199999809265137, 7.0, 5.400000095367432, 3.3999998569488525, 5.400000095367432, 6.799999713897705, 6.799999713897705, 7.0, 8.800000190734863, 7.799999713897705, 7.0, 5.0, 6.399999618530273, 5.799999713897705] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [14.199999809265137, 9.800000190734863, 12.0, 6.599999904632568, 12.799999237060547, 10.399999618530273, 17.799999237060547, 17.19999885559082, 21.399999618530273, 18.600000381469727, 14.799999237060547, 10.800000190734863, 15.799999237060547, 12.199999809265137, 12.799999237060547, 10.800000190734863] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [13.399999618530273, 13.399999618530273, 3.0, 1.399999976158142, 12.399999618530273, 8.800000190734863, 11.199999809265137, 4.0, 9.199999809265137, 7.599999904632568, 9.800000190734863, 7.399999618530273, 7.0, 3.5999999046325684, 8.399999618530273, 4.199999809265137] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 9.25+/- 0.6438 (max: 21.4) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 7.8+/- 0.9413 (max: 13.4) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 6.325+/- 0.3376 (max: 8.8) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 13.62+/- 0.9536 (max: 21.4) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 11.14+/- 0.3932 (max: 17.55) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 9.623+/- 0.5015 (max: 13.28) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 9.849+/- 0.2929 (max: 12.11) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 13.96+/- 0.5662 (max: 17.55) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.05396+/- 0.01054 (max: 0.31) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.01812+/- 0.007704 (max: 0.11) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.01687+/- 0.004806 (max: 0.06) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.1269+/- 0.02073 (max: 0.31) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 1.4 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 1.4 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 3.4 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 6.6 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 5.103 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 5.103 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 7.513 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 9.82 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating DR_CNN-LSTM_SEED3 against population in Overcooked-AsymmAdvantages6_9 for xpid SEED_3_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 + +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [0.7999999523162842, 3.799999952316284, 0.19999998807907104, 12.399999618530273, 1.399999976158142, 6.399999618530273, 0.0, 9.399999618530273, 0.5999999642372131, 39.0, 1.0, 2.799999952316284, 0.3999999761581421, 38.79999923706055, 0.0, 58.79999923706055, 1.5999999046325684, 10.59999942779541, 0.5999999642372131, 14.0, 0.19999998807907104, 10.199999809265137, 0.7999999523162842, 2.0, 0.19999998807907104, 3.5999999046325684, 0.19999998807907104, 18.600000381469727, 0.0, 6.0, 0.5999999642372131, 4.599999904632568, 0.0, 31.19999885559082, 0.19999998807907104, 4.799999713897705, 0.7999999523162842, 4.0, 0.0, 18.600000381469727, 1.7999999523162842, 4.799999713897705, 0.3999999761581421, 5.0, 0.0, 20.799999237060547, 0.7999999523162842, 3.5999999046325684] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.7999999523162842, 3.799999952316284, 0.0, 9.399999618530273, 0.3999999761581421, 38.79999923706055, 0.5999999642372131, 14.0, 0.19999998807907104, 3.5999999046325684, 0.5999999642372131, 4.599999904632568, 0.7999999523162842, 4.0, 0.3999999761581421, 5.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.19999998807907104, 12.399999618530273, 0.5999999642372131, 39.0, 0.0, 58.79999923706055, 0.19999998807907104, 10.199999809265137, 0.19999998807907104, 18.600000381469727, 0.0, 31.19999885559082, 0.0, 18.600000381469727, 0.0, 20.799999237060547] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [1.399999976158142, 6.399999618530273, 1.0, 2.799999952316284, 1.5999999046325684, 10.59999942779541, 0.7999999523162842, 2.0, 0.0, 6.0, 0.19999998807907104, 4.799999713897705, 1.7999999523162842, 4.799999713897705, 0.7999999523162842, 3.5999999046325684] +------------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 7.217+/- 1.76 (max: 58.8) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 3.037+/- 0.7149 (max: 10.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 5.437+/- 2.423 (max: 38.8) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 13.17+/- 4.358 (max: 58.8) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 8.597+/- 1.147 (max: 31.6) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 6.771+/- 0.8763 (max: 12.48) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 7.538+/- 1.689 (max: 27.4) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 11.48+/- 2.808 (max: 31.6) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.08604+/- 0.0262 (max: 0.78) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.01562+/- 0.004279 (max: 0.05) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.05688+/- 0.03871 (max: 0.61) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.1856+/- 0.06235 (max: 0.78) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------- +Evaluating DR_CNN-LSTM_SEED3 against population in Overcooked-CrampedRoom6_9 for xpid SEED_3_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 + +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [11.59999942779541, 11.199999809265137, 78.0, 72.19999694824219, 98.79999542236328, 101.0, 18.399999618530273, 15.199999809265137, 79.4000015258789, 76.5999984741211, 100.79999542236328, 95.5999984741211, 21.0, 19.600000381469727, 100.79999542236328, 101.0, 99.0, 100.19999694824219, 17.600000381469727, 20.600000381469727, 92.0, 91.4000015258789, 86.5999984741211, 84.5999984741211, 10.199999809265137, 11.800000190734863, 78.19999694824219, 76.19999694824219, 98.5999984741211, 98.0, 21.799999237060547, 19.799999237060547, 88.4000015258789, 86.5999984741211, 101.19999694824219, 102.79999542236328, 21.399999618530273, 18.399999618530273, 87.79999542236328, 86.19999694824219, 103.79999542236328, 104.0, 21.399999618530273, 19.799999237060547, 96.4000015258789, 94.5999984741211, 96.5999984741211, 94.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [11.59999942779541, 11.199999809265137, 18.399999618530273, 15.199999809265137, 21.0, 19.600000381469727, 17.600000381469727, 20.600000381469727, 10.199999809265137, 11.800000190734863, 21.799999237060547, 19.799999237060547, 21.399999618530273, 18.399999618530273, 21.399999618530273, 19.799999237060547] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [78.0, 72.19999694824219, 79.4000015258789, 76.5999984741211, 100.79999542236328, 101.0, 92.0, 91.4000015258789, 78.19999694824219, 76.19999694824219, 88.4000015258789, 86.5999984741211, 87.79999542236328, 86.19999694824219, 96.4000015258789, 94.5999984741211] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [98.79999542236328, 101.0, 100.79999542236328, 95.5999984741211, 99.0, 100.19999694824219, 86.5999984741211, 84.5999984741211, 98.5999984741211, 98.0, 101.19999694824219, 102.79999542236328, 103.79999542236328, 104.0, 96.5999984741211, 94.0] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 67.32+/- 5.266 (max: 104.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 97.85+/- 1.384 (max: 104.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 17.49+/- 1.027 (max: 21.8) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 86.61+/- 2.271 (max: 101.0) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 24.09+/- 1.038 (max: 36.21) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 31.33+/- 0.6372 (max: 36.21) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 15.21+/- 0.3948 (max: 17.06) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 25.72+/- 0.7912 (max: 31.25) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.7142+/- 0.05204 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.9631+/- 0.004155 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.215+/- 0.022 (max: 0.34) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.9644+/- 0.006517 (max: 1.0) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 10.2 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 84.6 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 10.2 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 72.2 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 12.71 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 26.94 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 12.71 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 20.88 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.08 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.94 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.08 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.92 | +------------------------------------------------------------------------------------------------- + + + + + + + +Evaluating PLR_CNN-LSTM_SEED1 against population in Overcooked-CoordRing6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 + +k eval/a0:test_return:Overcooked-CoordRing6_9, v [4.799999713897705, 3.3999998569488525, 18.399999618530273, 14.799999237060547, 21.399999618530273, 21.799999237060547, 12.799999237060547, 11.399999618530273, 26.399999618530273, 29.799999237060547, 21.0, 23.399999618530273, 11.0, 11.800000190734863, 21.799999237060547, 22.600000381469727, 24.19999885559082, 22.399999618530273, 13.0, 10.399999618530273, 16.0, 16.799999237060547, 18.600000381469727, 18.399999618530273, 15.399999618530273, 17.19999885559082, 18.19999885559082, 21.399999618530273, 21.399999618530273, 24.399999618530273, 11.0, 13.199999809265137, 20.19999885559082, 20.600000381469727, 31.399999618530273, 34.0, 2.799999952316284, 5.400000095367432, 16.600000381469727, 21.0, 27.599998474121094, 30.799999237060547, 14.399999618530273, 16.19999885559082, 14.399999618530273, 16.0, 22.600000381469727, 26.599998474121094] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [4.799999713897705, 3.3999998569488525, 12.799999237060547, 11.399999618530273, 11.0, 11.800000190734863, 13.0, 10.399999618530273, 15.399999618530273, 17.19999885559082, 11.0, 13.199999809265137, 2.799999952316284, 5.400000095367432, 14.399999618530273, 16.19999885559082] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [18.399999618530273, 14.799999237060547, 26.399999618530273, 29.799999237060547, 21.799999237060547, 22.600000381469727, 16.0, 16.799999237060547, 18.19999885559082, 21.399999618530273, 20.19999885559082, 20.600000381469727, 16.600000381469727, 21.0, 14.399999618530273, 16.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [21.399999618530273, 21.799999237060547, 21.0, 23.399999618530273, 24.19999885559082, 22.399999618530273, 18.600000381469727, 18.399999618530273, 21.399999618530273, 24.399999618530273, 31.399999618530273, 34.0, 27.599998474121094, 30.799999237060547, 22.600000381469727, 26.599998474121094] +----------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 18.32+/- 1.028 (max: 34.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 24.37+/- 1.139 (max: 34.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 10.89+/- 1.126 (max: 17.2) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 19.69+/- 1.054 (max: 29.8) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 12.25+/- 0.3323 (max: 15.94) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 11.57+/- 0.6373 (max: 15.94) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 11.98+/- 0.6407 (max: 15.5) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 13.21+/- 0.3513 (max: 15.61) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.196+/- 0.02201 (max: 0.68) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.2975+/- 0.04513 (max: 0.68) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.08312+/- 0.0165 (max: 0.22) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.2075+/- 0.02643 (max: 0.44) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 2.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 18.4 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 2.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 14.4 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 6.94 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 8.127 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 6.94 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 10.98 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.11 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.05 | +----------------------------------------------------------------------------------------------- +Evaluating PLR_CNN-LSTM_SEED1 against population in Overcooked-ForcedCoord6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 + +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [1.0, 0.0, 1.1999999284744263, 0.0, 0.5999999642372131, 0.0, 1.1999999284744263, 0.3999999761581421, 1.7999999523162842, 0.7999999523162842, 0.3999999761581421, 0.0, 0.5999999642372131, 0.3999999761581421, 1.0, 0.5999999642372131, 0.19999998807907104, 0.19999998807907104, 0.7999999523162842, 0.0, 1.7999999523162842, 0.0, 1.0, 0.19999998807907104, 1.0, 0.0, 0.7999999523162842, 0.0, 1.7999999523162842, 0.0, 1.1999999284744263, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.3999999761581421, 0.3999999761581421, 0.0, 0.7999999523162842, 0.19999998807907104, 0.5999999642372131, 0.0, 1.5999999046325684, 0.19999998807907104, 1.5999999046325684, 0.7999999523162842, 2.5999999046325684, 0.19999998807907104] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [1.0, 0.0, 1.1999999284744263, 0.3999999761581421, 0.5999999642372131, 0.3999999761581421, 0.7999999523162842, 0.0, 1.0, 0.0, 1.1999999284744263, 0.0, 0.3999999761581421, 0.0, 1.5999999046325684, 0.19999998807907104] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [1.1999999284744263, 0.0, 1.7999999523162842, 0.7999999523162842, 1.0, 0.5999999642372131, 1.7999999523162842, 0.0, 0.7999999523162842, 0.0, 0.5999999642372131, 0.0, 0.7999999523162842, 0.19999998807907104, 1.5999999046325684, 0.7999999523162842] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [0.5999999642372131, 0.0, 0.3999999761581421, 0.0, 0.19999998807907104, 0.19999998807907104, 1.0, 0.19999998807907104, 1.7999999523162842, 0.0, 0.5999999642372131, 0.3999999761581421, 0.5999999642372131, 0.0, 2.5999999046325684, 0.19999998807907104] +------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.6167+/- 0.08949 (max: 2.6) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.55+/- 0.1794 (max: 2.6) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.55+/- 0.131 (max: 1.6) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.75+/- 0.1555 (max: 1.8) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 2.759+/- 0.2904 (max: 6.726) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 2.538+/- 0.5017 (max: 6.726) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 2.585+/- 0.5004 (max: 5.426) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 3.154+/- 0.5243 (max: 5.724) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------ +Evaluating PLR_CNN-LSTM_SEED1 against population in Overcooked-CounterCircuit6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [12.59999942779541, 10.399999618530273, 20.19999885559082, 15.399999618530273, 16.0, 16.0, 8.199999809265137, 6.0, 14.0, 13.399999618530273, 3.3999998569488525, 1.399999976158142, 5.199999809265137, 2.200000047683716, 12.799999237060547, 7.0, 13.399999618530273, 5.599999904632568, 10.800000190734863, 6.0, 24.19999885559082, 19.0, 9.800000190734863, 4.799999713897705, 11.0, 8.199999809265137, 21.19999885559082, 18.0, 10.800000190734863, 3.799999952316284, 14.0, 8.800000190734863, 23.799999237060547, 14.0, 11.0, 6.199999809265137, 8.800000190734863, 7.599999904632568, 20.19999885559082, 17.19999885559082, 9.0, 4.0, 14.0, 9.59999942779541, 25.599998474121094, 22.799999237060547, 9.199999809265137, 3.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [12.59999942779541, 10.399999618530273, 8.199999809265137, 6.0, 5.199999809265137, 2.200000047683716, 10.800000190734863, 6.0, 11.0, 8.199999809265137, 14.0, 8.800000190734863, 8.800000190734863, 7.599999904632568, 14.0, 9.59999942779541] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [20.19999885559082, 15.399999618530273, 14.0, 13.399999618530273, 12.799999237060547, 7.0, 24.19999885559082, 19.0, 21.19999885559082, 18.0, 23.799999237060547, 14.0, 20.19999885559082, 17.19999885559082, 25.599998474121094, 22.799999237060547] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [16.0, 16.0, 3.3999998569488525, 1.399999976158142, 13.399999618530273, 5.599999904632568, 9.800000190734863, 4.799999713897705, 10.800000190734863, 3.799999952316284, 11.0, 6.199999809265137, 9.0, 4.0, 9.199999809265137, 3.0] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 11.66+/- 0.905 (max: 25.6) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 7.962+/- 1.159 (max: 16.0) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 8.962+/- 0.8023 (max: 14.0) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 18.05+/- 1.262 (max: 25.6) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 11.84+/- 0.494 (max: 21.18) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 9.465+/- 0.4914 (max: 12.33) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 10.6+/- 0.4023 (max: 13.46) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 15.47+/- 0.7308 (max: 21.18) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.09479+/- 0.01766 (max: 0.44) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.02187+/- 0.007917 (max: 0.11) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.03187+/- 0.007428 (max: 0.11) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.2306+/- 0.03104 (max: 0.44) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 1.4 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 1.4 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 2.2 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 7.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 5.103 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 5.103 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 6.258 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 9.539 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating PLR_CNN-LSTM_SEED1 against population in Overcooked-AsymmAdvantages6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [0.3999999761581421, 3.1999998092651367, 0.7999999523162842, 10.0, 1.0, 2.3999998569488525, 0.7999999523162842, 6.0, 1.0, 39.20000076293945, 0.3999999761581421, 2.3999998569488525, 0.7999999523162842, 32.79999923706055, 1.399999976158142, 56.79999923706055, 1.1999999284744263, 10.59999942779541, 0.5999999642372131, 14.0, 1.0, 8.0, 1.1999999284744263, 3.3999998569488525, 0.3999999761581421, 3.0, 1.0, 16.799999237060547, 1.1999999284744263, 4.799999713897705, 0.3999999761581421, 4.799999713897705, 1.399999976158142, 22.19999885559082, 0.7999999523162842, 2.5999999046325684, 0.19999998807907104, 2.3999998569488525, 0.7999999523162842, 14.399999618530273, 1.399999976158142, 5.799999713897705, 0.19999998807907104, 4.799999713897705, 0.19999998807907104, 18.0, 1.0, 2.5999999046325684] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.3999999761581421, 3.1999998092651367, 0.7999999523162842, 6.0, 0.7999999523162842, 32.79999923706055, 0.5999999642372131, 14.0, 0.3999999761581421, 3.0, 0.3999999761581421, 4.799999713897705, 0.19999998807907104, 2.3999998569488525, 0.19999998807907104, 4.799999713897705] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.7999999523162842, 10.0, 1.0, 39.20000076293945, 1.399999976158142, 56.79999923706055, 1.0, 8.0, 1.0, 16.799999237060547, 1.399999976158142, 22.19999885559082, 0.7999999523162842, 14.399999618530273, 0.19999998807907104, 18.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [1.0, 2.3999998569488525, 0.3999999761581421, 2.3999998569488525, 1.1999999284744263, 10.59999942779541, 1.1999999284744263, 3.3999998569488525, 1.1999999284744263, 4.799999713897705, 0.7999999523162842, 2.5999999046325684, 1.399999976158142, 5.799999713897705, 1.0, 2.5999999046325684] +-------------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 6.471+/- 1.607 (max: 56.8) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 2.675+/- 0.6462 (max: 10.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 4.675+/- 2.073 (max: 32.8) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 12.06+/- 4.038 (max: 56.8) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 8.634+/- 1.013 (max: 31.46) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 6.425+/- 0.6483 (max: 12.79) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 7.42+/- 1.666 (max: 27.79) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 12.06+/- 2.292 (max: 31.46) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.07125+/- 0.02422 (max: 0.79) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.008125+/- 0.005018 (max: 0.08) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.04562+/- 0.02999 (max: 0.47) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.16+/- 0.06126 (max: 0.79) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.2 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.4 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 0.2 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.2 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 1.99 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 2.8 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 1.99 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 1.99 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +-------------------------------------------------------------------------------------------------------- +Evaluating PLR_CNN-LSTM_SEED1 against population in Overcooked-CrampedRoom6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [9.0, 9.399999618530273, 70.79999542236328, 65.19999694824219, 87.79999542236328, 91.19999694824219, 12.0, 16.600000381469727, 71.0, 66.79999542236328, 100.4000015258789, 99.4000015258789, 15.59999942779541, 15.199999809265137, 86.0, 88.4000015258789, 94.4000015258789, 93.79999542236328, 15.399999618530273, 14.399999618530273, 83.19999694824219, 80.5999984741211, 66.19999694824219, 71.0, 8.399999618530273, 8.800000190734863, 67.5999984741211, 66.19999694824219, 87.4000015258789, 96.19999694824219, 17.19999885559082, 13.799999237060547, 87.19999694824219, 81.5999984741211, 104.19999694824219, 102.39999389648438, 16.399999618530273, 14.199999809265137, 83.79999542236328, 75.79999542236328, 95.0, 95.0, 16.399999618530273, 15.59999942779541, 83.79999542236328, 79.79999542236328, 91.5999984741211, 94.19999694824219] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [9.0, 9.399999618530273, 12.0, 16.600000381469727, 15.59999942779541, 15.199999809265137, 15.399999618530273, 14.399999618530273, 8.399999618530273, 8.800000190734863, 17.19999885559082, 13.799999237060547, 16.399999618530273, 14.199999809265137, 16.399999618530273, 15.59999942779541] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [70.79999542236328, 65.19999694824219, 71.0, 66.79999542236328, 86.0, 88.4000015258789, 83.19999694824219, 80.5999984741211, 67.5999984741211, 66.19999694824219, 87.19999694824219, 81.5999984741211, 83.79999542236328, 75.79999542236328, 83.79999542236328, 79.79999542236328] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [87.79999542236328, 91.19999694824219, 100.4000015258789, 99.4000015258789, 94.4000015258789, 93.79999542236328, 66.19999694824219, 71.0, 87.4000015258789, 96.19999694824219, 104.19999694824219, 102.39999389648438, 95.0, 95.0, 91.5999984741211, 94.19999694824219] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 60.97+/- 5.077 (max: 104.2) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 91.89+/- 2.564 (max: 104.2) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 13.65+/- 0.7743 (max: 17.2) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 77.36+/- 2.051 (max: 88.4) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 23.42+/- 1.069 (max: 35.4) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 28.62+/- 0.6682 (max: 34.84) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 13.64+/- 0.3778 (max: 16.25) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 27.99+/- 0.76 (max: 35.4) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.6687+/- 0.05689 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.9581+/- 0.0113 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.1212+/- 0.01363 (max: 0.2) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.9269+/- 0.00982 (max: 0.98) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 8.4 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 66.2 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 8.4 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 65.2 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 11.02 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 23.68 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 11.02 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 23.61 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.03 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.81 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.03 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.85 | +------------------------------------------------------------------------------------------------- + + + + + + + + +Evaluating PLR_CNN-LSTM_SEED2 against population in Overcooked-CoordRing6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [1.7999999523162842, 2.5999999046325684, 5.0, 5.0, 2.799999952316284, 2.5999999046325684, 6.199999809265137, 5.799999713897705, 8.399999618530273, 10.0, 5.199999809265137, 3.799999952316284, 6.199999809265137, 7.599999904632568, 7.399999618530273, 6.199999809265137, 5.799999713897705, 5.599999904632568, 10.399999618530273, 6.799999713897705, 7.799999713897705, 7.599999904632568, 5.400000095367432, 4.199999809265137, 5.599999904632568, 4.799999713897705, 1.399999976158142, 0.7999999523162842, 3.799999952316284, 5.0, 7.0, 9.399999618530273, 12.799999237060547, 14.399999618530273, 12.799999237060547, 10.59999942779541, 0.7999999523162842, 0.5999999642372131, 6.599999904632568, 5.599999904632568, 11.199999809265137, 14.59999942779541, 9.199999809265137, 9.59999942779541, 5.599999904632568, 7.399999618530273, 5.599999904632568, 9.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [1.7999999523162842, 2.5999999046325684, 6.199999809265137, 5.799999713897705, 6.199999809265137, 7.599999904632568, 10.399999618530273, 6.799999713897705, 5.599999904632568, 4.799999713897705, 7.0, 9.399999618530273, 0.7999999523162842, 0.5999999642372131, 9.199999809265137, 9.59999942779541] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [5.0, 5.0, 8.399999618530273, 10.0, 7.399999618530273, 6.199999809265137, 7.799999713897705, 7.599999904632568, 1.399999976158142, 0.7999999523162842, 12.799999237060547, 14.399999618530273, 6.599999904632568, 5.599999904632568, 5.599999904632568, 7.399999618530273] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [2.799999952316284, 2.5999999046325684, 5.199999809265137, 3.799999952316284, 5.799999713897705, 5.599999904632568, 5.400000095367432, 4.199999809265137, 3.799999952316284, 5.0, 12.799999237060547, 10.59999942779541, 11.199999809265137, 14.59999942779541, 5.599999904632568, 9.0] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 6.55+/- 0.4913 (max: 14.6) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 6.75+/- 0.9253 (max: 14.6) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 5.9+/- 0.778 (max: 10.4) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 7.0+/- 0.8737 (max: 14.4) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 9.51+/- 0.3464 (max: 14.57) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 9.214+/- 0.3826 (max: 11.77) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 9.459+/- 0.7367 (max: 12.8) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 9.858+/- 0.6515 (max: 14.57) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0225+/- 0.004371 (max: 0.14) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.01312+/- 0.005456 (max: 0.07) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.02687+/- 0.006565 (max: 0.08) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0275+/- 0.009895 (max: 0.14) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 0.6 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 2.6 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 0.6 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 0.8 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 3.412 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 6.726 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 3.412 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 3.919 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------- +Evaluating PLR_CNN-LSTM_SEED2 against population in Overcooked-ForcedCoord6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [0.7999999523162842, 0.3999999761581421, 0.19999998807907104, 1.0, 0.5999999642372131, 1.0, 1.0, 1.0, 0.7999999523162842, 1.1999999284744263, 0.19999998807907104, 0.19999998807907104, 1.399999976158142, 0.3999999761581421, 0.7999999523162842, 1.5999999046325684, 0.19999998807907104, 2.0, 0.7999999523162842, 0.7999999523162842, 0.7999999523162842, 0.5999999642372131, 0.3999999761581421, 1.399999976158142, 1.1999999284744263, 0.7999999523162842, 1.0, 1.7999999523162842, 0.7999999523162842, 1.399999976158142, 0.5999999642372131, 0.19999998807907104, 0.7999999523162842, 0.5999999642372131, 0.19999998807907104, 0.5999999642372131, 0.19999998807907104, 0.5999999642372131, 0.3999999761581421, 0.7999999523162842, 0.0, 0.3999999761581421, 1.399999976158142, 0.0, 0.5999999642372131, 0.0, 1.399999976158142, 0.5999999642372131] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [0.7999999523162842, 0.3999999761581421, 1.0, 1.0, 1.399999976158142, 0.3999999761581421, 0.7999999523162842, 0.7999999523162842, 1.1999999284744263, 0.7999999523162842, 0.5999999642372131, 0.19999998807907104, 0.19999998807907104, 0.5999999642372131, 1.399999976158142, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [0.19999998807907104, 1.0, 0.7999999523162842, 1.1999999284744263, 0.7999999523162842, 1.5999999046325684, 0.7999999523162842, 0.5999999642372131, 1.0, 1.7999999523162842, 0.7999999523162842, 0.5999999642372131, 0.3999999761581421, 0.7999999523162842, 0.5999999642372131, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [0.5999999642372131, 1.0, 0.19999998807907104, 0.19999998807907104, 0.19999998807907104, 2.0, 0.3999999761581421, 1.399999976158142, 0.7999999523162842, 1.399999976158142, 0.19999998807907104, 0.5999999642372131, 0.0, 0.3999999761581421, 1.399999976158142, 0.5999999642372131] +------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.75+/- 0.06932 (max: 2.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.7125+/- 0.1437 (max: 2.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.725+/- 0.1047 (max: 1.4) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.8125+/- 0.1147 (max: 1.8) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 3.508+/- 0.2014 (max: 6.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 3.336+/- 0.3917 (max: 6.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 3.485+/- 0.3334 (max: 5.103) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 3.702+/- 0.3351 (max: 5.724) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------ +Evaluating PLR_CNN-LSTM_SEED2 against population in Overcooked-CounterCircuit6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [8.199999809265137, 8.199999809265137, 7.0, 5.400000095367432, 7.399999618530273, 4.599999904632568, 3.5999999046325684, 1.7999999523162842, 4.599999904632568, 4.199999809265137, 0.5999999642372131, 0.0, 0.19999998807907104, 0.19999998807907104, 2.0, 2.0, 2.0, 2.0, 4.199999809265137, 2.5999999046325684, 13.199999809265137, 9.800000190734863, 8.59999942779541, 2.799999952316284, 7.0, 5.0, 8.399999618530273, 4.599999904632568, 0.3999999761581421, 0.7999999523162842, 8.59999942779541, 6.199999809265137, 11.399999618530273, 3.799999952316284, 5.799999713897705, 2.200000047683716, 3.5999999046325684, 1.399999976158142, 6.199999809265137, 4.400000095367432, 4.0, 3.0, 8.800000190734863, 7.199999809265137, 9.399999618530273, 8.0, 10.0, 1.7999999523162842] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [8.199999809265137, 8.199999809265137, 3.5999999046325684, 1.7999999523162842, 0.19999998807907104, 0.19999998807907104, 4.199999809265137, 2.5999999046325684, 7.0, 5.0, 8.59999942779541, 6.199999809265137, 3.5999999046325684, 1.399999976158142, 8.800000190734863, 7.199999809265137] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [7.0, 5.400000095367432, 4.599999904632568, 4.199999809265137, 2.0, 2.0, 13.199999809265137, 9.800000190734863, 8.399999618530273, 4.599999904632568, 11.399999618530273, 3.799999952316284, 6.199999809265137, 4.400000095367432, 9.399999618530273, 8.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [7.399999618530273, 4.599999904632568, 0.5999999642372131, 0.0, 2.0, 2.0, 8.59999942779541, 2.799999952316284, 0.3999999761581421, 0.7999999523162842, 5.799999713897705, 2.200000047683716, 4.0, 3.0, 10.0, 1.7999999523162842] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 4.942+/- 0.4749 (max: 13.2) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 3.5+/- 0.7572 (max: 10.0) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 4.8+/- 0.7559 (max: 8.8) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 6.525+/- 0.8173 (max: 13.2) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 8.365+/- 0.4804 (max: 16.36) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 6.551+/- 0.7411 (max: 10.83) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 8.041+/- 0.763 (max: 12.03) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 10.5+/- 0.7122 (max: 16.36) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.01896+/- 0.004419 (max: 0.12) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0025+/- 0.001936 (max: 0.03) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.01125+/- 0.00407 (max: 0.06) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.04312+/- 0.01011 (max: 0.12) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 0.2 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 2.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 1.99 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 6.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating PLR_CNN-LSTM_SEED2 against population in Overcooked-AsymmAdvantages6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [9.399999618530273, 2.3999998569488525, 14.199999809265137, 10.399999618530273, 12.0, 0.3999999761581421, 11.199999809265137, 5.400000095367432, 14.199999809265137, 38.79999923706055, 12.799999237060547, 0.0, 14.59999942779541, 38.599998474121094, 14.199999809265137, 56.39999771118164, 8.199999809265137, 8.59999942779541, 13.59999942779541, 16.19999885559082, 15.59999942779541, 4.199999809265137, 13.399999618530273, 1.399999976158142, 9.800000190734863, 1.5999999046325684, 15.0, 15.199999809265137, 15.799999237060547, 2.200000047683716, 10.199999809265137, 2.200000047683716, 15.399999618530273, 23.799999237060547, 10.0, 0.19999998807907104, 9.0, 2.0, 16.600000381469727, 13.799999237060547, 11.59999942779541, 4.0, 8.59999942779541, 4.799999713897705, 17.19999885559082, 18.0, 14.399999618530273, 0.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [9.399999618530273, 2.3999998569488525, 11.199999809265137, 5.400000095367432, 14.59999942779541, 38.599998474121094, 13.59999942779541, 16.19999885559082, 9.800000190734863, 1.5999999046325684, 10.199999809265137, 2.200000047683716, 9.0, 2.0, 8.59999942779541, 4.799999713897705] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [14.199999809265137, 10.399999618530273, 14.199999809265137, 38.79999923706055, 14.199999809265137, 56.39999771118164, 15.59999942779541, 4.199999809265137, 15.0, 15.199999809265137, 15.399999618530273, 23.799999237060547, 16.600000381469727, 13.799999237060547, 17.19999885559082, 18.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [12.0, 0.3999999761581421, 12.799999237060547, 0.0, 8.199999809265137, 8.59999942779541, 13.399999618530273, 1.399999976158142, 15.799999237060547, 2.200000047683716, 10.0, 0.19999998807907104, 11.59999942779541, 4.0, 14.399999618530273, 0.0] +----------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 12.03+/- 1.516 (max: 56.4) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 7.187+/- 1.467 (max: 15.8) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 9.975+/- 2.233 (max: 38.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 18.94+/- 3.074 (max: 56.4) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 13.36+/- 0.8972 (max: 30.38) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 9.721+/- 1.534 (max: 17.27) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 12.74+/- 1.377 (max: 26.42) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 17.62+/- 1.095 (max: 30.38) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.1352+/- 0.02321 (max: 0.76) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0625+/- 0.0167 (max: 0.19) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.1044+/- 0.03654 (max: 0.59) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.2387+/- 0.04818 (max: 0.76) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 1.6 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 4.2 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 6.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 9.075 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.02 | +----------------------------------------------------------------------------------------------------- +Evaluating PLR_CNN-LSTM_SEED2 against population in Overcooked-CrampedRoom6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [7.399999618530273, 5.0, 70.0, 67.79999542236328, 82.5999984741211, 79.5999984741211, 12.399999618530273, 11.199999809265137, 67.5999984741211, 62.19999694824219, 73.4000015258789, 76.4000015258789, 10.399999618530273, 12.199999809265137, 74.4000015258789, 76.4000015258789, 59.79999923706055, 66.4000015258789, 12.199999809265137, 12.399999618530273, 72.19999694824219, 74.79999542236328, 58.19999694824219, 62.19999694824219, 6.199999809265137, 6.599999904632568, 60.19999694824219, 56.19999694824219, 69.4000015258789, 72.5999984741211, 13.399999618530273, 11.399999618530273, 68.79999542236328, 76.19999694824219, 84.0, 80.5999984741211, 12.399999618530273, 13.799999237060547, 70.19999694824219, 74.0, 70.5999984741211, 73.5999984741211, 14.399999618530273, 19.0, 80.5999984741211, 84.79999542236328, 74.79999542236328, 77.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [7.399999618530273, 5.0, 12.399999618530273, 11.199999809265137, 10.399999618530273, 12.199999809265137, 12.199999809265137, 12.399999618530273, 6.199999809265137, 6.599999904632568, 13.399999618530273, 11.399999618530273, 12.399999618530273, 13.799999237060547, 14.399999618530273, 19.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [70.0, 67.79999542236328, 67.5999984741211, 62.19999694824219, 74.4000015258789, 76.4000015258789, 72.19999694824219, 74.79999542236328, 60.19999694824219, 56.19999694824219, 68.79999542236328, 76.19999694824219, 70.19999694824219, 74.0, 80.5999984741211, 84.79999542236328] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [82.5999984741211, 79.5999984741211, 73.4000015258789, 76.4000015258789, 59.79999923706055, 66.4000015258789, 58.19999694824219, 62.19999694824219, 69.4000015258789, 72.5999984741211, 84.0, 80.5999984741211, 70.5999984741211, 73.5999984741211, 74.79999542236328, 77.0] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 51.62+/- 4.264 (max: 84.8) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 72.57+/- 1.953 (max: 84.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 11.27+/- 0.8865 (max: 19.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 71.02+/- 1.845 (max: 84.8) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 25.9+/- 1.375 (max: 37.64) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 31.05+/- 1.005 (max: 37.64) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 13.34+/- 0.4883 (max: 15.84) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 33.32+/- 0.7649 (max: 37.14) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.6121+/- 0.05251 (max: 0.96) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.8756+/- 0.01605 (max: 0.96) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.11+/- 0.01528 (max: 0.25) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.8506+/- 0.01424 (max: 0.91) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 5.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 58.2 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 5.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 56.2 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 9.539 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 24.33 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 9.539 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 27.26 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.73 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.71 | +------------------------------------------------------------------------------------------------- + + + + + + + + + +Evaluating PLR_CNN-LSTM_SEED3 against population in Overcooked-CoordRing6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [4.0, 4.599999904632568, 19.0, 14.59999942779541, 17.799999237060547, 16.799999237060547, 13.799999237060547, 15.199999809265137, 31.799999237060547, 28.399999618530273, 17.399999618530273, 18.19999885559082, 12.399999618530273, 11.199999809265137, 20.799999237060547, 22.19999885559082, 19.0, 17.600000381469727, 11.0, 7.399999618530273, 20.0, 18.0, 15.399999618530273, 16.399999618530273, 17.0, 13.399999618530273, 21.0, 19.799999237060547, 17.0, 18.0, 10.0, 9.800000190734863, 20.19999885559082, 22.19999885559082, 29.599998474121094, 31.0, 3.5999999046325684, 3.3999998569488525, 13.799999237060547, 14.399999618530273, 20.600000381469727, 18.799999237060547, 5.599999904632568, 7.399999618530273, 9.0, 9.0, 11.399999618530273, 12.59999942779541] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [4.0, 4.599999904632568, 13.799999237060547, 15.199999809265137, 12.399999618530273, 11.199999809265137, 11.0, 7.399999618530273, 17.0, 13.399999618530273, 10.0, 9.800000190734863, 3.5999999046325684, 3.3999998569488525, 5.599999904632568, 7.399999618530273] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [19.0, 14.59999942779541, 31.799999237060547, 28.399999618530273, 20.799999237060547, 22.19999885559082, 20.0, 18.0, 21.0, 19.799999237060547, 20.19999885559082, 22.19999885559082, 13.799999237060547, 14.399999618530273, 9.0, 9.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [17.799999237060547, 16.799999237060547, 17.399999618530273, 18.19999885559082, 19.0, 17.600000381469727, 15.399999618530273, 16.399999618530273, 17.0, 18.0, 29.599998474121094, 31.0, 20.600000381469727, 18.799999237060547, 11.399999618530273, 12.59999942779541] +------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CoordRing6_9 | 15.66+/- 0.9834 (max: 31.8) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 18.6+/- 1.277 (max: 31.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 9.362+/- 1.092 (max: 17.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 19.01+/- 1.516 (max: 31.8) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 11.64+/- 0.3008 (max: 16.09) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 11.33+/- 0.532 (max: 16.09) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 10.92+/- 0.5703 (max: 14.8) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 12.66+/- 0.363 (max: 15.2) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.1352+/- 0.02203 (max: 0.63) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.1475+/- 0.04408 (max: 0.59) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.05062+/- 0.01355 (max: 0.17) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.2075+/- 0.04001 (max: 0.63) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 3.4 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 11.4 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 3.4 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 9.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 7.513 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 8.66 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 7.513 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 10.34 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.01 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.01 | +------------------------------------------------------------------------------------------------ +Evaluating PLR_CNN-LSTM_SEED3 against population in Overcooked-ForcedCoord6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [4.799999713897705, 0.0, 7.199999809265137, 0.19999998807907104, 2.3999998569488525, 0.5999999642372131, 4.400000095367432, 0.0, 4.400000095367432, 0.19999998807907104, 3.1999998092651367, 0.0, 3.5999999046325684, 0.0, 4.599999904632568, 0.19999998807907104, 3.799999952316284, 0.19999998807907104, 5.199999809265137, 0.0, 3.799999952316284, 0.0, 2.0, 0.0, 4.599999904632568, 0.0, 5.799999713897705, 0.0, 4.0, 0.0, 3.3999998569488525, 0.0, 4.599999904632568, 0.19999998807907104, 0.5999999642372131, 0.0, 2.5999999046325684, 0.0, 2.799999952316284, 0.0, 2.5999999046325684, 0.0, 4.799999713897705, 0.0, 7.199999809265137, 0.19999998807907104, 7.599999904632568, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [4.799999713897705, 0.0, 4.400000095367432, 0.0, 3.5999999046325684, 0.0, 5.199999809265137, 0.0, 4.599999904632568, 0.0, 3.3999998569488525, 0.0, 2.5999999046325684, 0.0, 4.799999713897705, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [7.199999809265137, 0.19999998807907104, 4.400000095367432, 0.19999998807907104, 4.599999904632568, 0.19999998807907104, 3.799999952316284, 0.0, 5.799999713897705, 0.0, 4.599999904632568, 0.19999998807907104, 2.799999952316284, 0.0, 7.199999809265137, 0.19999998807907104] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [2.3999998569488525, 0.5999999642372131, 3.1999998092651367, 0.0, 3.799999952316284, 0.19999998807907104, 2.0, 0.0, 4.0, 0.0, 0.5999999642372131, 0.0, 2.5999999046325684, 0.0, 7.599999904632568, 0.0] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 2.121+/- 0.3435 (max: 7.6) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 1.687+/- 0.5407 (max: 7.6) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 2.087+/- 0.5598 (max: 5.2) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 2.587+/- 0.6903 (max: 7.2) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 4.279+/- 0.5598 (max: 9.708) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 3.841+/- 0.9043 (max: 9.708) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 4.058+/- 1.056 (max: 9.217) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 4.939+/- 0.9837 (max: 9.6) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0008333+/- 0.0004031 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.000625+/- 0.000625 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.000625+/- 0.000625 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.00125+/- 0.0008539 (max: 0.01) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating PLR_CNN-LSTM_SEED3 against population in Overcooked-CounterCircuit6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [1.0, 1.0, 2.799999952316284, 2.0, 1.0, 0.7999999523162842, 1.1999999284744263, 1.5999999046325684, 1.5999999046325684, 1.399999976158142, 0.0, 0.0, 3.0, 2.0, 13.799999237060547, 5.199999809265137, 7.399999618530273, 2.3999998569488525, 2.200000047683716, 0.7999999523162842, 8.800000190734863, 6.399999618530273, 1.7999999523162842, 0.5999999642372131, 2.799999952316284, 1.7999999523162842, 7.0, 7.0, 5.799999713897705, 1.1999999284744263, 2.5999999046325684, 0.19999998807907104, 4.0, 2.200000047683716, 3.0, 1.5999999046325684, 1.0, 0.3999999761581421, 1.399999976158142, 3.0, 0.19999998807907104, 0.0, 1.5999999046325684, 1.5999999046325684, 2.799999952316284, 1.0, 0.3999999761581421, 0.19999998807907104] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [1.0, 1.0, 1.1999999284744263, 1.5999999046325684, 3.0, 2.0, 2.200000047683716, 0.7999999523162842, 2.799999952316284, 1.7999999523162842, 2.5999999046325684, 0.19999998807907104, 1.0, 0.3999999761581421, 1.5999999046325684, 1.5999999046325684] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [2.799999952316284, 2.0, 1.5999999046325684, 1.399999976158142, 13.799999237060547, 5.199999809265137, 8.800000190734863, 6.399999618530273, 7.0, 7.0, 4.0, 2.200000047683716, 1.399999976158142, 3.0, 2.799999952316284, 1.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [1.0, 0.7999999523162842, 0.0, 0.0, 7.399999618530273, 2.3999998569488525, 1.7999999523162842, 0.5999999642372131, 5.799999713897705, 1.1999999284744263, 3.0, 1.5999999046325684, 0.19999998807907104, 0.0, 0.3999999761581421, 0.19999998807907104] +------------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 2.533+/- 0.3906 (max: 13.8) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 1.65+/- 0.5365 (max: 7.4) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 1.55+/- 0.2062 (max: 3.0) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 4.4+/- 0.8687 (max: 13.8) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 5.7+/- 0.3884 (max: 10.7) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 4.221+/- 0.7807 (max: 10.45) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 5.152+/- 0.3571 (max: 7.141) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 7.727+/- 0.4913 (max: 10.7) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.003125+/- 0.001038 (max: 0.03) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.00125+/- 0.00125 (max: 0.02) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.000625+/- 0.000625 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0075+/- 0.0025 (max: 0.03) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 0.2 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 1.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 1.99 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 4.359 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------- +Evaluating PLR_CNN-LSTM_SEED3 against population in Overcooked-AsymmAdvantages6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [0.0, 16.0, 0.5999999642372131, 34.39999771118164, 0.19999998807907104, 35.599998474121094, 0.0, 15.59999942779541, 0.0, 53.39999771118164, 0.19999998807907104, 47.20000076293945, 0.0, 54.79999923706055, 0.0, 75.79999542236328, 0.19999998807907104, 36.39999771118164, 0.0, 36.39999771118164, 0.0, 31.599998474121094, 0.0, 27.799999237060547, 0.0, 18.799999237060547, 0.0, 38.79999923706055, 0.19999998807907104, 26.0, 0.0, 15.199999809265137, 0.0, 46.20000076293945, 0.0, 36.79999923706055, 0.19999998807907104, 15.399999618530273, 0.19999998807907104, 44.39999771118164, 0.0, 26.399999618530273, 0.19999998807907104, 19.0, 0.0, 41.20000076293945, 0.19999998807907104, 18.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.0, 16.0, 0.0, 15.59999942779541, 0.0, 54.79999923706055, 0.0, 36.39999771118164, 0.0, 18.799999237060547, 0.0, 15.199999809265137, 0.19999998807907104, 15.399999618530273, 0.19999998807907104, 19.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.5999999642372131, 34.39999771118164, 0.0, 53.39999771118164, 0.0, 75.79999542236328, 0.0, 31.599998474121094, 0.0, 38.79999923706055, 0.0, 46.20000076293945, 0.19999998807907104, 44.39999771118164, 0.0, 41.20000076293945] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [0.19999998807907104, 35.599998474121094, 0.19999998807907104, 47.20000076293945, 0.19999998807907104, 36.39999771118164, 0.0, 27.799999237060547, 0.19999998807907104, 26.0, 0.0, 36.79999923706055, 0.0, 26.399999618530273, 0.19999998807907104, 18.0] +----------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 16.95+/- 2.899 (max: 75.8) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 15.95+/- 4.366 (max: 47.2) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 11.97+/- 3.935 (max: 54.8) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 22.91+/- 6.354 (max: 75.8) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 10.71+/- 1.505 (max: 28.33) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 11.3+/- 2.652 (max: 25.39) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 8.996+/- 2.301 (max: 26.02) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 11.83+/- 2.942 (max: 28.33) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.2496+/- 0.04398 (max: 0.93) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.2381+/- 0.06728 (max: 0.73) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.1594+/- 0.05977 (max: 0.82) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.3512+/- 0.09405 (max: 0.93) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +----------------------------------------------------------------------------------------------------- +Evaluating PLR_CNN-LSTM_SEED3 against population in Overcooked-CrampedRoom6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [8.800000190734863, 5.799999713897705, 57.19999694824219, 55.79999923706055, 77.19999694824219, 77.4000015258789, 12.399999618530273, 13.799999237060547, 65.79999542236328, 60.599998474121094, 76.4000015258789, 76.4000015258789, 18.600000381469727, 16.600000381469727, 76.5999984741211, 77.4000015258789, 74.5999984741211, 77.19999694824219, 17.0, 17.399999618530273, 72.19999694824219, 68.5999984741211, 55.79999923706055, 59.79999923706055, 6.599999904632568, 9.800000190734863, 65.4000015258789, 61.39999771118164, 71.4000015258789, 71.19999694824219, 17.19999885559082, 17.799999237060547, 66.5999984741211, 66.79999542236328, 74.19999694824219, 76.4000015258789, 17.19999885559082, 17.799999237060547, 67.0, 61.19999694824219, 75.0, 75.5999984741211, 16.19999885559082, 15.799999237060547, 69.5999984741211, 68.79999542236328, 70.5999984741211, 70.5999984741211] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [8.800000190734863, 5.799999713897705, 12.399999618530273, 13.799999237060547, 18.600000381469727, 16.600000381469727, 17.0, 17.399999618530273, 6.599999904632568, 9.800000190734863, 17.19999885559082, 17.799999237060547, 17.19999885559082, 17.799999237060547, 16.19999885559082, 15.799999237060547] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [57.19999694824219, 55.79999923706055, 65.79999542236328, 60.599998474121094, 76.5999984741211, 77.4000015258789, 72.19999694824219, 68.5999984741211, 65.4000015258789, 61.39999771118164, 66.5999984741211, 66.79999542236328, 67.0, 61.19999694824219, 69.5999984741211, 68.79999542236328] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [77.19999694824219, 77.4000015258789, 76.4000015258789, 76.4000015258789, 74.5999984741211, 77.19999694824219, 55.79999923706055, 59.79999923706055, 71.4000015258789, 71.19999694824219, 74.19999694824219, 76.4000015258789, 75.0, 75.5999984741211, 70.5999984741211, 70.5999984741211] +-------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 51.03+/- 3.889 (max: 77.4) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 72.49+/- 1.564 (max: 77.4) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 14.3+/- 1.068 (max: 18.6) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 66.31+/- 1.533 (max: 77.4) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 18.65+/- 0.6188 (max: 24.89) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 21.73+/- 0.3164 (max: 24.39) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 13.19+/- 0.3923 (max: 15.75) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 21.02+/- 0.5964 (max: 24.89) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.6787+/- 0.05712 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.965+/- 0.008317 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.1287+/- 0.01612 (max: 0.21) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.9425+/- 0.008391 (max: 0.98) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 5.8 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 55.8 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 5.8 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 55.8 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 10.31 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 19.49 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 10.31 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 16.5 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.03 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.87 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.03 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.87 | +-------------------------------------------------------------------------------------------------- + + + + + + +Evaluating PAIRED_CNN-LSTM_SEED1 against population in Overcooked-CoordRing6_9 for xpid paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [4.199999809265137, 3.799999952316284, 16.399999618530273, 15.399999618530273, 22.600000381469727, 21.799999237060547, 11.59999942779541, 14.59999942779541, 29.19999885559082, 29.799999237060547, 21.799999237060547, 22.0, 13.59999942779541, 13.59999942779541, 22.799999237060547, 24.399999618530273, 22.0, 20.600000381469727, 12.199999809265137, 10.59999942779541, 14.0, 16.799999237060547, 16.0, 15.799999237060547, 13.0, 15.0, 17.799999237060547, 21.0, 21.600000381469727, 21.0, 13.399999618530273, 17.0, 21.399999618530273, 23.0, 35.599998474121094, 34.0, 2.5999999046325684, 4.599999904632568, 16.600000381469727, 21.19999885559082, 26.0, 28.399999618530273, 13.199999809265137, 13.399999618530273, 12.399999618530273, 15.399999618530273, 20.0, 23.600000381469727] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [4.199999809265137, 3.799999952316284, 11.59999942779541, 14.59999942779541, 13.59999942779541, 13.59999942779541, 12.199999809265137, 10.59999942779541, 13.0, 15.0, 13.399999618530273, 17.0, 2.5999999046325684, 4.599999904632568, 13.199999809265137, 13.399999618530273] +k eval/a1:test_return:Overcooked-CoordRing6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CoordRing6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [16.399999618530273, 15.399999618530273, 29.19999885559082, 29.799999237060547, 22.799999237060547, 24.399999618530273, 14.0, 16.799999237060547, 17.799999237060547, 21.0, 21.399999618530273, 23.0, 16.600000381469727, 21.19999885559082, 12.399999618530273, 15.399999618530273] +k eval/a1:test_return:Overcooked-CoordRing6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [22.600000381469727, 21.799999237060547, 21.799999237060547, 22.0, 22.0, 20.600000381469727, 16.0, 15.799999237060547, 21.600000381469727, 21.0, 35.599998474121094, 34.0, 26.0, 28.399999618530273, 20.0, 23.600000381469727] +k eval/a1:test_return:Overcooked-CoordRing6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +----------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 18.06+/- 1.038 (max: 35.6) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 23.3+/- 1.359 (max: 35.6) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 11.02+/- 1.137 (max: 17.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 19.85+/- 1.285 (max: 29.8) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 12.26+/- 0.332 (max: 18.41) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 11.6+/- 0.5354 (max: 15.36) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 11.78+/- 0.6246 (max: 14.25) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 13.39+/- 0.4797 (max: 18.41) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.191+/- 0.02333 (max: 0.71) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.2656+/- 0.04847 (max: 0.71) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.08+/- 0.01393 (max: 0.17) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.2275+/- 0.03593 (max: 0.52) | +| eval/a1:test_return:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 2.6 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 15.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 2.6 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 12.4 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 6.726 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 8.66 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 6.726 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 11.09 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.07 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.06 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0 | +----------------------------------------------------------------------------------------------- +Evaluating PAIRED_CNN-LSTM_SEED1 against population in Overcooked-ForcedCoord6_9 for xpid paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [2.799999952316284, 0.0, 3.5999999046325684, 0.0, 4.0, 0.0, 5.0, 0.0, 3.799999952316284, 0.0, 2.5999999046325684, 0.0, 2.3999998569488525, 0.0, 2.200000047683716, 0.0, 1.5999999046325684, 0.0, 3.1999998092651367, 0.0, 3.3999998569488525, 0.0, 1.7999999523162842, 0.0, 2.3999998569488525, 0.0, 2.200000047683716, 0.0, 2.3999998569488525, 0.0, 3.799999952316284, 0.0, 3.3999998569488525, 0.0, 1.399999976158142, 0.0, 0.7999999523162842, 0.0, 2.200000047683716, 0.0, 1.5999999046325684, 0.19999998807907104, 5.400000095367432, 0.0, 3.5999999046325684, 0.0, 5.599999904632568, 0.19999998807907104] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [2.799999952316284, 0.0, 5.0, 0.0, 2.3999998569488525, 0.0, 3.1999998092651367, 0.0, 2.3999998569488525, 0.0, 3.799999952316284, 0.0, 0.7999999523162842, 0.0, 5.400000095367432, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [3.5999999046325684, 0.0, 3.799999952316284, 0.0, 2.200000047683716, 0.0, 3.3999998569488525, 0.0, 2.200000047683716, 0.0, 3.3999998569488525, 0.0, 2.200000047683716, 0.0, 3.5999999046325684, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [4.0, 0.0, 2.5999999046325684, 0.0, 1.5999999046325684, 0.0, 1.7999999523162842, 0.0, 2.3999998569488525, 0.0, 1.399999976158142, 0.0, 1.5999999046325684, 0.19999998807907104, 5.599999904632568, 0.19999998807907104] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 1.492+/- 0.2494 (max: 5.6) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 1.337+/- 0.4166 (max: 5.6) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 1.612+/- 0.4884 (max: 5.4) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 1.525+/- 0.4123 (max: 3.8) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 3.539+/- 0.5121 (max: 9.415) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 3.519+/- 0.8357 (max: 9.415) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 3.536+/- 0.9511 (max: 8.879) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 3.563+/- 0.9284 (max: 7.846) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0002083+/- 0.0002083 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.000625+/- 0.000625 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating PAIRED_CNN-LSTM_SEED1 against population in Overcooked-CounterCircuit6_9 for xpid paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [10.800000190734863, 7.599999904632568, 16.19999885559082, 11.800000190734863, 14.399999618530273, 13.199999809265137, 8.399999618530273, 5.0, 11.399999618530273, 9.800000190734863, 3.1999998092651367, 0.0, 3.0, 2.200000047683716, 13.199999809265137, 9.0, 11.199999809265137, 5.799999713897705, 9.800000190734863, 6.199999809265137, 18.600000381469727, 16.399999618530273, 9.399999618530273, 3.5999999046325684, 11.0, 9.0, 16.600000381469727, 14.799999237060547, 7.399999618530273, 3.0, 12.59999942779541, 9.800000190734863, 15.799999237060547, 10.199999809265137, 9.0, 5.799999713897705, 7.0, 6.0, 15.199999809265137, 12.199999809265137, 8.0, 3.1999998092651367, 10.199999809265137, 6.599999904632568, 13.799999237060547, 13.199999809265137, 10.59999942779541, 3.1999998092651367] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [10.800000190734863, 7.599999904632568, 8.399999618530273, 5.0, 3.0, 2.200000047683716, 9.800000190734863, 6.199999809265137, 11.0, 9.0, 12.59999942779541, 9.800000190734863, 7.0, 6.0, 10.199999809265137, 6.599999904632568] +k eval/a1:test_return:Overcooked-CounterCircuit6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [16.19999885559082, 11.800000190734863, 11.399999618530273, 9.800000190734863, 13.199999809265137, 9.0, 18.600000381469727, 16.399999618530273, 16.600000381469727, 14.799999237060547, 15.799999237060547, 10.199999809265137, 15.199999809265137, 12.199999809265137, 13.799999237060547, 13.199999809265137] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [14.399999618530273, 13.199999809265137, 3.1999998092651367, 0.0, 11.199999809265137, 5.799999713897705, 9.399999618530273, 3.5999999046325684, 7.399999618530273, 3.0, 9.0, 5.799999713897705, 8.0, 3.1999998092651367, 10.59999942779541, 3.1999998092651367] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 9.467+/- 0.64 (max: 18.6) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 6.937+/- 1.036 (max: 14.4) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 7.825+/- 0.7303 (max: 12.6) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 13.64+/- 0.6922 (max: 18.6) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 10.98+/- 0.4554 (max: 17.82) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 8.792+/- 0.6941 (max: 12.35) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 10.29+/- 0.4721 (max: 14.04) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 13.85+/- 0.586 (max: 17.82) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.05271+/- 0.009419 (max: 0.22) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0125+/- 0.006021 (max: 0.09) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.02625+/- 0.007296 (max: 0.11) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.1194+/- 0.01714 (max: 0.22) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 2.2 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 9.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 6.258 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 9.474 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating PAIRED_CNN-LSTM_SEED1 against population in Overcooked-AsymmAdvantages6_9 for xpid paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [0.19999998807907104, 6.799999713897705, 0.3999999761581421, 13.399999618530273, 0.5999999642372131, 6.799999713897705, 0.3999999761581421, 9.399999618530273, 0.3999999761581421, 42.39999771118164, 0.19999998807907104, 6.199999809265137, 0.3999999761581421, 32.599998474121094, 0.3999999761581421, 64.5999984741211, 0.5999999642372131, 14.59999942779541, 0.19999998807907104, 23.799999237060547, 1.0, 14.799999237060547, 0.7999999523162842, 2.799999952316284, 0.5999999642372131, 5.799999713897705, 0.5999999642372131, 22.0, 0.19999998807907104, 7.799999713897705, 0.5999999642372131, 4.799999713897705, 0.5999999642372131, 30.399999618530273, 0.3999999761581421, 5.599999904632568, 0.19999998807907104, 8.59999942779541, 0.3999999761581421, 19.19999885559082, 0.19999998807907104, 9.59999942779541, 0.5999999642372131, 9.0, 0.19999998807907104, 25.0, 1.5999999046325684, 7.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.19999998807907104, 6.799999713897705, 0.3999999761581421, 9.399999618530273, 0.3999999761581421, 32.599998474121094, 0.19999998807907104, 23.799999237060547, 0.5999999642372131, 5.799999713897705, 0.5999999642372131, 4.799999713897705, 0.19999998807907104, 8.59999942779541, 0.5999999642372131, 9.0] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.3999999761581421, 13.399999618530273, 0.3999999761581421, 42.39999771118164, 0.3999999761581421, 64.5999984741211, 1.0, 14.799999237060547, 0.5999999642372131, 22.0, 0.5999999642372131, 30.399999618530273, 0.3999999761581421, 19.19999885559082, 0.19999998807907104, 25.0] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [0.5999999642372131, 6.799999713897705, 0.19999998807907104, 6.199999809265137, 0.5999999642372131, 14.59999942779541, 0.7999999523162842, 2.799999952316284, 0.19999998807907104, 7.799999713897705, 0.3999999761581421, 5.599999904632568, 0.19999998807907104, 9.59999942779541, 1.5999999046325684, 7.0] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 8.433+/- 1.869 (max: 64.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 4.062+/- 1.079 (max: 14.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 6.5+/- 2.328 (max: 32.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 14.74+/- 4.699 (max: 64.6) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 9.568+/- 1.171 (max: 32.6) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 7.418+/- 1.205 (max: 15.96) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 8.482+/- 1.798 (max: 26.63) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 12.8+/- 2.67 (max: 32.6) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.09896+/- 0.02721 (max: 0.83) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.02562+/- 0.01004 (max: 0.15) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0675+/- 0.03574 (max: 0.51) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.2037+/- 0.06651 (max: 0.83) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.2 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.2 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 0.2 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.2 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 1.99 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 1.99 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 1.99 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 1.99 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating PAIRED_CNN-LSTM_SEED1 against population in Overcooked-CrampedRoom6_9 for xpid paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [8.800000190734863, 10.800000190734863, 65.79999542236328, 59.0, 84.0, 88.0, 13.199999809265137, 16.799999237060547, 70.5999984741211, 69.0, 88.5999984741211, 101.0, 18.0, 17.19999885559082, 88.79999542236328, 85.19999694824219, 90.4000015258789, 94.4000015258789, 18.0, 16.600000381469727, 85.4000015258789, 81.0, 74.0, 80.0, 10.800000190734863, 9.199999809265137, 68.0, 64.79999542236328, 88.4000015258789, 82.79999542236328, 14.799999237060547, 18.600000381469727, 82.4000015258789, 81.0, 88.0, 93.19999694824219, 16.0, 15.399999618530273, 78.19999694824219, 80.0, 91.19999694824219, 88.79999542236328, 20.600000381469727, 19.0, 84.4000015258789, 80.79999542236328, 89.19999694824219, 88.19999694824219] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [8.800000190734863, 10.800000190734863, 13.199999809265137, 16.799999237060547, 18.0, 17.19999885559082, 18.0, 16.600000381469727, 10.800000190734863, 9.199999809265137, 14.799999237060547, 18.600000381469727, 16.0, 15.399999618530273, 20.600000381469727, 19.0] +k eval/a1:test_return:Overcooked-CrampedRoom6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [65.79999542236328, 59.0, 70.5999984741211, 69.0, 88.79999542236328, 85.19999694824219, 85.4000015258789, 81.0, 68.0, 64.79999542236328, 82.4000015258789, 81.0, 78.19999694824219, 80.0, 84.4000015258789, 80.79999542236328] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [84.0, 88.0, 88.5999984741211, 101.0, 90.4000015258789, 94.4000015258789, 74.0, 80.0, 88.4000015258789, 82.79999542236328, 88.0, 93.19999694824219, 91.19999694824219, 88.79999542236328, 89.19999694824219, 88.19999694824219] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +-------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 59.97+/- 4.757 (max: 101.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 88.14+/- 1.525 (max: 101.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 15.24+/- 0.9125 (max: 20.6) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 76.52+/- 2.235 (max: 88.8) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 22.72+/- 0.9762 (max: 32.03) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 27.95+/- 0.5288 (max: 32.03) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 13.67+/- 0.3485 (max: 15.75) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 26.55+/- 0.5405 (max: 30.32) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.6831+/- 0.05542 (max: 0.99) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.9687+/- 0.004644 (max: 0.99) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.1506+/- 0.01847 (max: 0.26) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.93+/- 0.01021 (max: 0.98) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 8.8 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 74.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 8.8 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 59.0 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 11.11 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 23.83 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 11.11 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 22.86 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.03 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.93 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.03 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.86 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.0 | +-------------------------------------------------------------------------------------------------- + + + + + + + + + + + + +Evaluating PAIRED_CNN-LSTM_SEED2 against population in Overcooked-CoordRing6_9 for xpid SEED_2_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [3.0, 4.400000095367432, 20.799999237060547, 16.399999618530273, 21.600000381469727, 23.19999885559082, 16.600000381469727, 16.600000381469727, 31.799999237060547, 32.79999923706055, 22.399999618530273, 21.399999618530273, 12.799999237060547, 13.399999618530273, 22.19999885559082, 26.0, 21.0, 20.0, 9.800000190734863, 9.199999809265137, 18.600000381469727, 18.399999618530273, 17.799999237060547, 15.199999809265137, 15.59999942779541, 18.600000381469727, 20.600000381469727, 21.600000381469727, 23.399999618530273, 23.0, 11.199999809265137, 11.199999809265137, 24.19999885559082, 22.0, 39.39999771118164, 40.599998474121094, 2.200000047683716, 3.799999952316284, 17.19999885559082, 18.799999237060547, 25.399999618530273, 25.399999618530273, 11.199999809265137, 11.0, 15.0, 12.399999618530273, 20.19999885559082, 22.19999885559082] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [3.0, 4.400000095367432, 16.600000381469727, 16.600000381469727, 12.799999237060547, 13.399999618530273, 9.800000190734863, 9.199999809265137, 15.59999942779541, 18.600000381469727, 11.199999809265137, 11.199999809265137, 2.200000047683716, 3.799999952316284, 11.199999809265137, 11.0] +k eval/a1:test_return:Overcooked-CoordRing6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CoordRing6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [20.799999237060547, 16.399999618530273, 31.799999237060547, 32.79999923706055, 22.19999885559082, 26.0, 18.600000381469727, 18.399999618530273, 20.600000381469727, 21.600000381469727, 24.19999885559082, 22.0, 17.19999885559082, 18.799999237060547, 15.0, 12.399999618530273] +k eval/a1:test_return:Overcooked-CoordRing6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [21.600000381469727, 23.19999885559082, 22.399999618530273, 21.399999618530273, 21.0, 20.0, 17.799999237060547, 15.199999809265137, 23.399999618530273, 23.0, 39.39999771118164, 40.599998474121094, 25.399999618530273, 25.399999618530273, 20.19999885559082, 22.19999885559082] +k eval/a1:test_return:Overcooked-CoordRing6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CoordRing6_9 | 18.57+/- 1.172 (max: 40.6) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 23.89+/- 1.698 (max: 40.6) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 10.66+/- 1.276 (max: 18.6) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 21.17+/- 1.379 (max: 32.8) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 12.16+/- 0.3026 (max: 16.07) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 11.21+/- 0.2927 (max: 13.52) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 11.88+/- 0.7115 (max: 16.01) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 13.4+/- 0.3164 (max: 16.07) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.2098+/- 0.02699 (max: 0.9) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.2856+/- 0.05881 (max: 0.9) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.08687+/- 0.01724 (max: 0.23) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.2569+/- 0.03939 (max: 0.6) | +| eval/a1:test_return:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 2.2 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 15.2 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 2.2 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 12.4 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 6.258 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 9.404 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 6.258 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 11.93 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.1 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.06 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------ +Evaluating PAIRED_CNN-LSTM_SEED2 against population in Overcooked-ForcedCoord6_9 for xpid SEED_2_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [1.5999999046325684, 0.0, 2.799999952316284, 0.0, 0.5999999642372131, 0.0, 1.7999999523162842, 0.0, 2.0, 0.0, 1.0, 0.0, 1.1999999284744263, 0.0, 2.200000047683716, 0.0, 0.5999999642372131, 0.0, 2.200000047683716, 0.0, 1.0, 0.0, 1.1999999284744263, 0.0, 1.5999999046325684, 0.0, 2.3999998569488525, 0.0, 0.7999999523162842, 0.0, 1.7999999523162842, 0.0, 1.0, 0.0, 0.19999998807907104, 0.0, 1.1999999284744263, 0.0, 1.0, 0.0, 1.0, 0.0, 2.5999999046325684, 0.0, 2.5999999046325684, 0.0, 2.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [1.5999999046325684, 0.0, 1.7999999523162842, 0.0, 1.1999999284744263, 0.0, 2.200000047683716, 0.0, 1.5999999046325684, 0.0, 1.7999999523162842, 0.0, 1.1999999284744263, 0.0, 2.5999999046325684, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [2.799999952316284, 0.0, 2.0, 0.0, 2.200000047683716, 0.0, 1.0, 0.0, 2.3999998569488525, 0.0, 1.0, 0.0, 1.0, 0.0, 2.5999999046325684, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [0.5999999642372131, 0.0, 1.0, 0.0, 0.5999999642372131, 0.0, 1.1999999284744263, 0.0, 0.7999999523162842, 0.0, 0.19999998807907104, 0.0, 1.0, 0.0, 2.0, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.7583+/- 0.1323 (max: 2.8) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.4625+/- 0.1502 (max: 2.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.875+/- 0.2401 (max: 2.6) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.9375+/- 0.2749 (max: 2.8) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 2.552+/- 0.3929 (max: 6.94) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 2.013+/- 0.5566 (max: 6.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 2.799+/- 0.7319 (max: 6.726) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 2.844+/- 0.7594 (max: 6.94) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------ +Evaluating PAIRED_CNN-LSTM_SEED2 against population in Overcooked-CounterCircuit6_9 for xpid SEED_2_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [5.199999809265137, 2.3999998569488525, 7.0, 5.799999713897705, 4.799999713897705, 5.400000095367432, 5.0, 2.0, 6.599999904632568, 3.5999999046325684, 0.7999999523162842, 0.19999998807907104, 4.799999713897705, 1.0, 14.199999809265137, 4.0, 12.59999942779541, 5.0, 4.400000095367432, 2.0, 14.199999809265137, 11.0, 5.400000095367432, 1.5999999046325684, 6.399999618530273, 3.799999952316284, 17.0, 8.800000190734863, 9.800000190734863, 1.1999999284744263, 6.0, 3.3999998569488525, 10.0, 5.0, 4.199999809265137, 3.1999998092651367, 4.199999809265137, 3.1999998092651367, 6.599999904632568, 6.599999904632568, 2.200000047683716, 0.7999999523162842, 5.599999904632568, 1.1999999284744263, 10.0, 7.799999713897705, 4.599999904632568, 0.7999999523162842] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [5.199999809265137, 2.3999998569488525, 5.0, 2.0, 4.799999713897705, 1.0, 4.400000095367432, 2.0, 6.399999618530273, 3.799999952316284, 6.0, 3.3999998569488525, 4.199999809265137, 3.1999998092651367, 5.599999904632568, 1.1999999284744263] +k eval/a1:test_return:Overcooked-CounterCircuit6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [7.0, 5.799999713897705, 6.599999904632568, 3.5999999046325684, 14.199999809265137, 4.0, 14.199999809265137, 11.0, 17.0, 8.800000190734863, 10.0, 5.0, 6.599999904632568, 6.599999904632568, 10.0, 7.799999713897705] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [4.799999713897705, 5.400000095367432, 0.7999999523162842, 0.19999998807907104, 12.59999942779541, 5.0, 5.400000095367432, 1.5999999046325684, 9.800000190734863, 1.1999999284744263, 4.199999809265137, 3.1999998092651367, 2.200000047683716, 0.7999999523162842, 4.599999904632568, 0.7999999523162842] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +------------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 5.446+/- 0.5527 (max: 17.0) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 3.912+/- 0.8565 (max: 12.6) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 3.787+/- 0.4248 (max: 6.4) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 8.637+/- 0.9685 (max: 17.0) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 8.49+/- 0.3948 (max: 15.07) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 7.001+/- 0.6796 (max: 11.19) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 7.539+/- 0.3968 (max: 9.33) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 10.93+/- 0.4909 (max: 15.07) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.01646+/- 0.004501 (max: 0.16) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.005+/- 0.002582 (max: 0.04) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.000625+/- 0.000625 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.04375+/- 0.0104 (max: 0.16) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.2 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.2 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 1.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 3.6 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 1.99 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 1.99 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 4.359 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 7.684 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------- +Evaluating PAIRED_CNN-LSTM_SEED2 against population in Overcooked-AsymmAdvantages6_9 for xpid SEED_2_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [0.0, 7.599999904632568, 0.19999998807907104, 17.600000381469727, 0.0, 7.799999713897705, 0.0, 9.800000190734863, 0.0, 40.79999923706055, 0.0, 3.799999952316284, 0.0, 37.79999923706055, 0.7999999523162842, 60.599998474121094, 0.0, 15.399999618530273, 0.0, 20.399999618530273, 0.0, 14.59999942779541, 0.19999998807907104, 11.399999618530273, 0.0, 7.199999809265137, 0.0, 29.599998474121094, 0.0, 10.59999942779541, 0.0, 7.199999809265137, 0.0, 34.599998474121094, 0.0, 5.799999713897705, 0.0, 8.399999618530273, 0.0, 21.399999618530273, 0.19999998807907104, 12.199999809265137, 0.0, 10.59999942779541, 0.19999998807907104, 28.19999885559082, 0.19999998807907104, 6.199999809265137] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.0, 7.599999904632568, 0.0, 9.800000190734863, 0.0, 37.79999923706055, 0.0, 20.399999618530273, 0.0, 7.199999809265137, 0.0, 7.199999809265137, 0.0, 8.399999618530273, 0.0, 10.59999942779541] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9, v [0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.19999998807907104, 17.600000381469727, 0.0, 40.79999923706055, 0.7999999523162842, 60.599998474121094, 0.0, 14.59999942779541, 0.0, 29.599998474121094, 0.0, 34.599998474121094, 0.0, 21.399999618530273, 0.19999998807907104, 28.19999885559082] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [0.0, 7.799999713897705, 0.0, 3.799999952316284, 0.0, 15.399999618530273, 0.19999998807907104, 11.399999618530273, 0.0, 10.59999942779541, 0.0, 5.799999713897705, 0.19999998807907104, 12.199999809265137, 0.19999998807907104, 6.199999809265137] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high, v [0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 8.987+/- 1.926 (max: 60.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 4.612+/- 1.347 (max: 15.4) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 6.812+/- 2.535 (max: 37.8) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 15.54+/- 4.71 (max: 60.6) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 8.296+/- 1.283 (max: 33.64) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 6.591+/- 1.588 (max: 16.94) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 6.8+/- 1.918 (max: 23.13) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 11.5+/- 2.874 (max: 33.64) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.1081+/- 0.02877 (max: 0.77) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.03437+/- 0.01281 (max: 0.17) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.06562+/- 0.03807 (max: 0.59) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.2244+/- 0.06917 (max: 0.77) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9 | 0.1+/- 0.01459 (max: 0.2) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high | 0.1+/- 0.02582 (max: 0.2) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low | 0.1+/- 0.02582 (max: 0.2) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.1+/- 0.02582 (max: 0.2) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.995+/- 0.1451 (max: 1.99) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.995+/- 0.2569 (max: 1.99) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.995+/- 0.2569 (max: 1.99) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.995+/- 0.2569 (max: 1.99) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating PAIRED_CNN-LSTM_SEED2 against population in Overcooked-CrampedRoom6_9 for xpid SEED_2_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [11.0, 8.0, 65.19999694824219, 66.79999542236328, 87.79999542236328, 95.4000015258789, 13.799999237060547, 14.199999809265137, 72.5999984741211, 69.4000015258789, 101.5999984741211, 103.0, 16.799999237060547, 19.600000381469727, 89.79999542236328, 95.19999694824219, 98.79999542236328, 108.0, 17.0, 17.19999885559082, 86.0, 82.5999984741211, 77.79999542236328, 79.0, 8.59999942779541, 10.800000190734863, 70.79999542236328, 70.19999694824219, 90.5999984741211, 97.0, 21.0, 20.0, 87.19999694824219, 88.19999694824219, 103.5999984741211, 102.0, 17.0, 17.0, 77.5999984741211, 82.4000015258789, 105.5999984741211, 103.79999542236328, 20.600000381469727, 17.799999237060547, 89.4000015258789, 83.0, 96.19999694824219, 98.79999542236328] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [11.0, 8.0, 13.799999237060547, 14.199999809265137, 16.799999237060547, 19.600000381469727, 17.0, 17.19999885559082, 8.59999942779541, 10.800000190734863, 21.0, 20.0, 17.0, 17.0, 20.600000381469727, 17.799999237060547] +k eval/a1:test_return:Overcooked-CrampedRoom6_9, v [0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:low, v [0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [65.19999694824219, 66.79999542236328, 72.5999984741211, 69.4000015258789, 89.79999542236328, 95.19999694824219, 86.0, 82.5999984741211, 70.79999542236328, 70.19999694824219, 87.19999694824219, 88.19999694824219, 77.5999984741211, 82.4000015258789, 89.4000015258789, 83.0] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:mid, v [0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [87.79999542236328, 95.4000015258789, 101.5999984741211, 103.0, 98.79999542236328, 108.0, 77.79999542236328, 79.0, 90.5999984741211, 97.0, 103.5999984741211, 102.0, 105.5999984741211, 103.79999542236328, 96.19999694824219, 98.79999542236328] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:high, v [0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 64.08+/- 5.217 (max: 108.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 96.81+/- 2.231 (max: 108.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 15.65+/- 1.042 (max: 21.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 79.77+/- 2.361 (max: 95.2) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 22.7+/- 0.9937 (max: 32.85) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 27.32+/- 0.8808 (max: 32.85) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 13.98+/- 0.4449 (max: 17.52) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 26.82+/- 0.8264 (max: 31.29) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.6883+/- 0.05545 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.9731+/- 0.004892 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.1556+/- 0.01958 (max: 0.29) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.9362+/- 0.00875 (max: 0.98) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9 | 0.1+/- 0.01459 (max: 0.2) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:high | 0.1+/- 0.02582 (max: 0.2) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:low | 0.1+/- 0.02582 (max: 0.2) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:mid | 0.1+/- 0.02582 (max: 0.2) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9 | 0.995+/- 0.1451 (max: 1.99) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:high | 0.995+/- 0.2569 (max: 1.99) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:low | 0.995+/- 0.2569 (max: 1.99) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:mid | 0.995+/- 0.2569 (max: 1.99) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 8.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 77.8 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 8.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 65.2 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 11.05 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 21.24 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 11.05 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 20.25 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.03 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.92 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.03 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.88 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------- + + + + + + + +Evaluating PAIRED_CNN-LSTM_SEED3 against population in Overcooked-CoordRing6_9 for xpid SEED_3_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [5.199999809265137, 6.399999618530273, 23.19999885559082, 21.600000381469727, 24.0, 25.599998474121094, 16.399999618530273, 15.199999809265137, 33.39999771118164, 34.79999923706055, 27.799999237060547, 28.399999618530273, 15.0, 15.0, 28.19999885559082, 28.0, 27.0, 25.799999237060547, 16.399999618530273, 11.199999809265137, 19.600000381469727, 25.19999885559082, 24.799999237060547, 23.799999237060547, 19.0, 18.799999237060547, 26.399999618530273, 22.600000381469727, 29.0, 28.799999237060547, 14.399999618530273, 13.399999618530273, 28.399999618530273, 28.399999618530273, 34.0, 34.39999771118164, 3.0, 2.799999952316284, 19.0, 16.799999237060547, 29.0, 29.0, 10.800000190734863, 10.199999809265137, 14.399999618530273, 13.199999809265137, 17.799999237060547, 16.399999618530273] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [5.199999809265137, 6.399999618530273, 16.399999618530273, 15.199999809265137, 15.0, 15.0, 16.399999618530273, 11.199999809265137, 19.0, 18.799999237060547, 14.399999618530273, 13.399999618530273, 3.0, 2.799999952316284, 10.800000190734863, 10.199999809265137] +k eval/a1:test_return:Overcooked-CoordRing6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CoordRing6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [23.19999885559082, 21.600000381469727, 33.39999771118164, 34.79999923706055, 28.19999885559082, 28.0, 19.600000381469727, 25.19999885559082, 26.399999618530273, 22.600000381469727, 28.399999618530273, 28.399999618530273, 19.0, 16.799999237060547, 14.399999618530273, 13.199999809265137] +k eval/a1:test_return:Overcooked-CoordRing6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [24.0, 25.599998474121094, 27.799999237060547, 28.399999618530273, 27.0, 25.799999237060547, 24.799999237060547, 23.799999237060547, 29.0, 28.799999237060547, 34.0, 34.39999771118164, 29.0, 29.0, 17.799999237060547, 16.399999618530273] +k eval/a1:test_return:Overcooked-CoordRing6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +----------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 20.87+/- 1.205 (max: 34.8) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 26.6+/- 1.196 (max: 34.4) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 12.07+/- 1.32 (max: 19.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 23.95+/- 1.578 (max: 34.8) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 13.13+/- 0.354 (max: 19.7) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 13.42+/- 0.6714 (max: 19.7) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 12.13+/- 0.6485 (max: 16.34) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 13.84+/- 0.4485 (max: 17.93) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.2823+/- 0.02862 (max: 0.72) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.4081+/- 0.04244 (max: 0.72) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0975+/- 0.01856 (max: 0.21) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.3412+/- 0.0446 (max: 0.68) | +| eval/a1:test_return:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 2.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 16.4 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 2.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 13.2 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 6.94 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 10.16 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 6.94 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 11.39 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.06 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.05 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0 | +----------------------------------------------------------------------------------------------- +Evaluating PAIRED_CNN-LSTM_SEED3 against population in Overcooked-ForcedCoord6_9 for xpid SEED_3_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [2.5999999046325684, 0.0, 1.5999999046325684, 0.19999998807907104, 1.7999999523162842, 0.19999998807907104, 1.7999999523162842, 0.0, 1.5999999046325684, 0.3999999761581421, 0.5999999642372131, 0.0, 2.200000047683716, 0.0, 2.0, 0.0, 0.3999999761581421, 0.0, 3.0, 0.0, 1.7999999523162842, 0.19999998807907104, 0.3999999761581421, 0.0, 3.0, 0.3999999761581421, 4.599999904632568, 0.0, 4.199999809265137, 0.19999998807907104, 2.200000047683716, 0.19999998807907104, 0.5999999642372131, 0.19999998807907104, 0.7999999523162842, 0.19999998807907104, 1.0, 0.0, 1.1999999284744263, 0.19999998807907104, 0.7999999523162842, 0.19999998807907104, 2.200000047683716, 0.0, 3.3999998569488525, 0.0, 5.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [2.5999999046325684, 0.0, 1.7999999523162842, 0.0, 2.200000047683716, 0.0, 3.0, 0.0, 3.0, 0.3999999761581421, 2.200000047683716, 0.19999998807907104, 1.0, 0.0, 2.200000047683716, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [1.5999999046325684, 0.19999998807907104, 1.5999999046325684, 0.3999999761581421, 2.0, 0.0, 1.7999999523162842, 0.19999998807907104, 4.599999904632568, 0.0, 0.5999999642372131, 0.19999998807907104, 1.1999999284744263, 0.19999998807907104, 3.3999998569488525, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [1.7999999523162842, 0.19999998807907104, 0.5999999642372131, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 4.199999809265137, 0.19999998807907104, 0.7999999523162842, 0.19999998807907104, 0.7999999523162842, 0.19999998807907104, 5.0, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 1.071+/- 0.1928 (max: 5.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.925+/- 0.3781 (max: 5.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 1.162+/- 0.3034 (max: 3.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 1.125+/- 0.3351 (max: 4.6) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 3.34+/- 0.4048 (max: 9.539) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 3.014+/- 0.7032 (max: 9.539) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 3.416+/- 0.7681 (max: 7.141) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 3.589+/- 0.6667 (max: 8.417) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0004167+/- 0.0004167 (max: 0.02) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.00125+/- 0.00125 (max: 0.02) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating PAIRED_CNN-LSTM_SEED3 against population in Overcooked-CounterCircuit6_9 for xpid SEED_3_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [6.199999809265137, 4.400000095367432, 8.800000190734863, 4.599999904632568, 5.0, 5.0, 4.599999904632568, 3.1999998092651367, 2.5999999046325684, 3.1999998092651367, 1.5999999046325684, 1.0, 8.0, 4.0, 13.799999237060547, 10.800000190734863, 16.0, 8.199999809265137, 5.799999713897705, 4.199999809265137, 17.600000381469727, 12.0, 8.0, 5.599999904632568, 4.799999713897705, 7.399999618530273, 17.600000381469727, 12.59999942779541, 10.0, 4.400000095367432, 10.0, 6.799999713897705, 10.399999618530273, 7.0, 6.599999904632568, 5.199999809265137, 5.199999809265137, 4.599999904632568, 6.0, 6.199999809265137, 3.799999952316284, 3.0, 6.799999713897705, 5.400000095367432, 8.399999618530273, 7.199999809265137, 6.0, 2.799999952316284] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [6.199999809265137, 4.400000095367432, 4.599999904632568, 3.1999998092651367, 8.0, 4.0, 5.799999713897705, 4.199999809265137, 4.799999713897705, 7.399999618530273, 10.0, 6.799999713897705, 5.199999809265137, 4.599999904632568, 6.799999713897705, 5.400000095367432] +k eval/a1:test_return:Overcooked-CounterCircuit6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [8.800000190734863, 4.599999904632568, 2.5999999046325684, 3.1999998092651367, 13.799999237060547, 10.800000190734863, 17.600000381469727, 12.0, 17.600000381469727, 12.59999942779541, 10.399999618530273, 7.0, 6.0, 6.199999809265137, 8.399999618530273, 7.199999809265137] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [5.0, 5.0, 1.5999999046325684, 1.0, 16.0, 8.199999809265137, 8.0, 5.599999904632568, 10.0, 4.400000095367432, 6.599999904632568, 5.199999809265137, 3.799999952316284, 3.0, 6.0, 2.799999952316284] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 6.925+/- 0.5549 (max: 17.6) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 5.762+/- 0.9085 (max: 16.0) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 5.712+/- 0.4378 (max: 10.0) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 9.3+/- 1.146 (max: 17.6) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 9.858+/- 0.3225 (max: 15.28) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 8.5+/- 0.4769 (max: 12.33) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 9.462+/- 0.2841 (max: 11.83) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 11.61+/- 0.5752 (max: 15.28) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.02917+/- 0.006549 (max: 0.19) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0075+/- 0.006862 (max: 0.11) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.015+/- 0.003979 (max: 0.05) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.065+/- 0.01449 (max: 0.19) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 1.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 1.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 3.2 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 2.6 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 4.359 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 4.359 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 7.859 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 7.297 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating PAIRED_CNN-LSTM_SEED3 against population in Overcooked-AsymmAdvantages6_9 for xpid SEED_3_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [0.3999999761581421, 4.400000095367432, 1.399999976158142, 21.0, 1.0, 5.799999713897705, 0.19999998807907104, 8.399999618530273, 0.0, 40.599998474121094, 1.1999999284744263, 14.59999942779541, 0.19999998807907104, 42.39999771118164, 0.3999999761581421, 61.599998474121094, 1.0, 23.19999885559082, 0.5999999642372131, 25.599998474121094, 0.3999999761581421, 18.399999618530273, 0.3999999761581421, 16.600000381469727, 0.3999999761581421, 5.799999713897705, 0.19999998807907104, 29.599998474121094, 0.3999999761581421, 14.799999237060547, 0.3999999761581421, 6.799999713897705, 0.3999999761581421, 30.799999237060547, 0.7999999523162842, 12.199999809265137, 0.19999998807907104, 6.399999618530273, 0.5999999642372131, 28.599998474121094, 1.5999999046325684, 16.399999618530273, 1.0, 7.0, 0.3999999761581421, 27.0, 1.399999976158142, 4.400000095367432] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.3999999761581421, 4.400000095367432, 0.19999998807907104, 8.399999618530273, 0.19999998807907104, 42.39999771118164, 0.5999999642372131, 25.599998474121094, 0.3999999761581421, 5.799999713897705, 0.3999999761581421, 6.799999713897705, 0.19999998807907104, 6.399999618530273, 1.0, 7.0] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [1.399999976158142, 21.0, 0.0, 40.599998474121094, 0.3999999761581421, 61.599998474121094, 0.3999999761581421, 18.399999618530273, 0.19999998807907104, 29.599998474121094, 0.3999999761581421, 30.799999237060547, 0.5999999642372131, 28.599998474121094, 0.3999999761581421, 27.0] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [1.0, 5.799999713897705, 1.1999999284744263, 14.59999942779541, 1.0, 23.19999885559082, 0.3999999761581421, 16.600000381469727, 0.3999999761581421, 14.799999237060547, 0.7999999523162842, 12.199999809265137, 1.5999999046325684, 16.399999618530273, 1.399999976158142, 4.400000095367432] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 10.15+/- 2.007 (max: 61.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 7.237+/- 1.923 (max: 23.2) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 6.887+/- 2.855 (max: 42.4) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 16.34+/- 4.711 (max: 61.6) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 10.47+/- 1.246 (max: 33.55) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 9.743+/- 1.548 (max: 19.94) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 8.38+/- 1.786 (max: 27.02) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 13.29+/- 2.856 (max: 33.55) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.129+/- 0.02943 (max: 0.81) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.07937+/- 0.02664 (max: 0.33) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0775+/- 0.04376 (max: 0.65) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.23+/- 0.06705 (max: 0.81) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.4 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 0.2 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 2.8 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 1.99 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating PAIRED_CNN-LSTM_SEED3 against population in Overcooked-CrampedRoom6_9 for xpid SEED_3_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [8.800000190734863, 9.800000190734863, 64.4000015258789, 60.79999923706055, 89.4000015258789, 100.19999694824219, 16.19999885559082, 15.799999237060547, 69.4000015258789, 63.79999923706055, 96.0, 95.0, 16.399999618530273, 17.799999237060547, 77.0, 79.19999694824219, 90.5999984741211, 98.0, 17.600000381469727, 16.0, 81.5999984741211, 75.79999542236328, 66.79999542236328, 70.0, 11.800000190734863, 10.0, 67.4000015258789, 65.4000015258789, 84.4000015258789, 87.4000015258789, 17.19999885559082, 17.399999618530273, 81.0, 85.4000015258789, 92.0, 96.19999694824219, 18.19999885559082, 15.0, 70.5999984741211, 73.19999694824219, 93.4000015258789, 92.5999984741211, 18.19999885559082, 17.799999237060547, 74.19999694824219, 77.5999984741211, 96.5999984741211, 99.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [8.800000190734863, 9.800000190734863, 16.19999885559082, 15.799999237060547, 16.399999618530273, 17.799999237060547, 17.600000381469727, 16.0, 11.800000190734863, 10.0, 17.19999885559082, 17.399999618530273, 18.19999885559082, 15.0, 18.19999885559082, 17.799999237060547] +k eval/a1:test_return:Overcooked-CrampedRoom6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [64.4000015258789, 60.79999923706055, 69.4000015258789, 63.79999923706055, 77.0, 79.19999694824219, 81.5999984741211, 75.79999542236328, 67.4000015258789, 65.4000015258789, 81.0, 85.4000015258789, 70.5999984741211, 73.19999694824219, 74.19999694824219, 77.5999984741211] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [89.4000015258789, 100.19999694824219, 96.0, 95.0, 90.5999984741211, 98.0, 66.79999542236328, 70.0, 84.4000015258789, 87.4000015258789, 92.0, 96.19999694824219, 93.4000015258789, 92.5999984741211, 96.5999984741211, 99.0] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 59.55+/- 4.796 (max: 100.2) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 90.47+/- 2.405 (max: 100.2) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 15.25+/- 0.8121 (max: 18.2) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 72.92+/- 1.816 (max: 85.4) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 22.03+/- 0.9573 (max: 32.77) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 25.6+/- 0.6762 (max: 32.77) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 13.54+/- 0.3808 (max: 16.47) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 26.96+/- 0.8605 (max: 32.58) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.674+/- 0.05552 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.9669+/- 0.01106 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.1412+/- 0.01643 (max: 0.24) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.9137+/- 0.01036 (max: 0.98) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 8.8 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 66.8 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 8.8 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 60.8 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 10.7 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 20.73 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 10.7 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 22.03 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.83 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.82 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------- + + + + + + + +Evaluating ACCEL_CNN-LSTM_SEED1 against population in Overcooked-CoordRing6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [2.0, 2.0, 7.199999809265137, 5.599999904632568, 4.400000095367432, 5.0, 7.399999618530273, 7.599999904632568, 11.0, 12.399999618530273, 6.0, 7.399999618530273, 6.599999904632568, 8.0, 11.59999942779541, 9.399999618530273, 6.599999904632568, 8.59999942779541, 6.199999809265137, 6.799999713897705, 7.0, 10.0, 6.199999809265137, 8.800000190734863, 7.799999713897705, 8.800000190734863, 3.5999999046325684, 3.799999952316284, 6.199999809265137, 6.399999618530273, 3.3999998569488525, 5.400000095367432, 10.0, 11.59999942779541, 16.799999237060547, 15.799999237060547, 0.5999999642372131, 0.3999999761581421, 4.400000095367432, 7.0, 16.600000381469727, 21.600000381469727, 9.199999809265137, 11.399999618530273, 4.799999713897705, 6.199999809265137, 12.0, 17.399999618530273] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [2.0, 2.0, 7.399999618530273, 7.599999904632568, 6.599999904632568, 8.0, 6.199999809265137, 6.799999713897705, 7.799999713897705, 8.800000190734863, 3.3999998569488525, 5.400000095367432, 0.5999999642372131, 0.3999999761581421, 9.199999809265137, 11.399999618530273] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [7.199999809265137, 5.599999904632568, 11.0, 12.399999618530273, 11.59999942779541, 9.399999618530273, 7.0, 10.0, 3.5999999046325684, 3.799999952316284, 10.0, 11.59999942779541, 4.400000095367432, 7.0, 4.799999713897705, 6.199999809265137] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [4.400000095367432, 5.0, 6.0, 7.399999618530273, 6.599999904632568, 8.59999942779541, 6.199999809265137, 8.800000190734863, 6.199999809265137, 6.399999618530273, 16.799999237060547, 15.799999237060547, 16.600000381469727, 21.600000381469727, 12.0, 17.399999618530273] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 8.021+/- 0.6348 (max: 21.6) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 10.36+/- 1.37 (max: 21.6) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 5.85+/- 0.8148 (max: 11.4) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 7.85+/- 0.752 (max: 12.4) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 9.919+/- 0.3122 (max: 13.04) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 10.37+/- 0.3352 (max: 12.74) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 8.99+/- 0.7529 (max: 13.04) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 10.4+/- 0.3882 (max: 13.02) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.03146+/- 0.006267 (max: 0.22) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.04562+/- 0.01633 (max: 0.22) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.02062+/- 0.006421 (max: 0.09) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.02812+/- 0.006273 (max: 0.09) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 0.4 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 4.4 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 0.4 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 3.6 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 2.8 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 8.66 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 2.8 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 7.684 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------- +Evaluating ACCEL_CNN-LSTM_SEED1 against population in Overcooked-ForcedCoord6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [1.0, 0.0, 1.7999999523162842, 0.3999999761581421, 1.399999976158142, 0.3999999761581421, 2.0, 0.3999999761581421, 2.5999999046325684, 0.7999999523162842, 2.5999999046325684, 0.5999999642372131, 1.399999976158142, 0.5999999642372131, 2.0, 0.3999999761581421, 1.0, 0.5999999642372131, 2.3999998569488525, 0.7999999523162842, 2.3999998569488525, 0.7999999523162842, 1.0, 0.7999999523162842, 1.5999999046325684, 0.5999999642372131, 3.0, 0.7999999523162842, 3.1999998092651367, 0.5999999642372131, 2.3999998569488525, 0.3999999761581421, 1.7999999523162842, 0.3999999761581421, 1.0, 0.7999999523162842, 0.7999999523162842, 0.0, 1.0, 0.3999999761581421, 0.19999998807907104, 0.5999999642372131, 2.5999999046325684, 0.5999999642372131, 2.3999998569488525, 0.19999998807907104, 3.3999998569488525, 0.3999999761581421] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [1.0, 0.0, 2.0, 0.3999999761581421, 1.399999976158142, 0.5999999642372131, 2.3999998569488525, 0.7999999523162842, 1.5999999046325684, 0.5999999642372131, 2.3999998569488525, 0.3999999761581421, 0.7999999523162842, 0.0, 2.5999999046325684, 0.5999999642372131] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [1.7999999523162842, 0.3999999761581421, 2.5999999046325684, 0.7999999523162842, 2.0, 0.3999999761581421, 2.3999998569488525, 0.7999999523162842, 3.0, 0.7999999523162842, 1.7999999523162842, 0.3999999761581421, 1.0, 0.3999999761581421, 2.3999998569488525, 0.19999998807907104] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [1.399999976158142, 0.3999999761581421, 2.5999999046325684, 0.5999999642372131, 1.0, 0.5999999642372131, 1.0, 0.7999999523162842, 3.1999998092651367, 0.5999999642372131, 1.0, 0.7999999523162842, 0.19999998807907104, 0.5999999642372131, 3.3999998569488525, 0.3999999761581421] +------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 1.196+/- 0.1328 (max: 3.4) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 1.162+/- 0.2498 (max: 3.4) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 1.1+/- 0.2153 (max: 2.6) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 1.325+/- 0.2351 (max: 3.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 4.307+/- 0.2572 (max: 7.513) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 4.302+/- 0.4049 (max: 7.513) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 4.018+/- 0.5133 (max: 6.726) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 4.601+/- 0.4277 (max: 7.141) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.2 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.2 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 1.99 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 1.99 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------ +Evaluating ACCEL_CNN-LSTM_SEED1 against population in Overcooked-CounterCircuit6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [14.59999942779541, 9.0, 31.399999618530273, 16.600000381469727, 19.799999237060547, 17.799999237060547, 8.199999809265137, 5.799999713897705, 13.399999618530273, 9.399999618530273, 1.1999999284744263, 0.0, 0.0, 0.19999998807907104, 1.399999976158142, 0.3999999761581421, 1.5999999046325684, 1.399999976158142, 8.399999618530273, 5.599999904632568, 22.399999618530273, 13.799999237060547, 7.199999809265137, 2.3999998569488525, 12.199999809265137, 7.0, 20.0, 12.59999942779541, 3.3999998569488525, 2.200000047683716, 12.199999809265137, 9.399999618530273, 23.0, 10.199999809265137, 7.799999713897705, 2.799999952316284, 11.0, 5.0, 23.19999885559082, 12.199999809265137, 9.59999942779541, 6.0, 16.799999237060547, 11.59999942779541, 33.39999771118164, 19.399999618530273, 11.199999809265137, 5.799999713897705] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [14.59999942779541, 9.0, 8.199999809265137, 5.799999713897705, 0.0, 0.19999998807907104, 8.399999618530273, 5.599999904632568, 12.199999809265137, 7.0, 12.199999809265137, 9.399999618530273, 11.0, 5.0, 16.799999237060547, 11.59999942779541] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [31.399999618530273, 16.600000381469727, 13.399999618530273, 9.399999618530273, 1.399999976158142, 0.3999999761581421, 22.399999618530273, 13.799999237060547, 20.0, 12.59999942779541, 23.0, 10.199999809265137, 23.19999885559082, 12.199999809265137, 33.39999771118164, 19.399999618530273] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [19.799999237060547, 17.799999237060547, 1.1999999284744263, 0.0, 1.5999999046325684, 1.399999976158142, 7.199999809265137, 2.3999998569488525, 3.3999998569488525, 2.200000047683716, 7.799999713897705, 2.799999952316284, 9.59999942779541, 6.0, 11.199999809265137, 5.799999713897705] +----------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 10.42+/- 1.158 (max: 33.4) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 6.262+/- 1.468 (max: 19.8) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 8.562+/- 1.158 (max: 16.8) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 16.42+/- 2.308 (max: 33.4) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 10.45+/- 0.6433 (max: 20.41) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 7.941+/- 0.8221 (max: 13.41) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 9.878+/- 0.94 (max: 13.81) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 13.53+/- 1.122 (max: 20.41) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.09021+/- 0.01991 (max: 0.57) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.02437+/- 0.01565 (max: 0.22) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.04+/- 0.01114 (max: 0.14) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.2062+/- 0.04473 (max: 0.57) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 0.4 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 2.8 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +----------------------------------------------------------------------------------------------------- +Evaluating ACCEL_CNN-LSTM_SEED1 against population in Overcooked-AsymmAdvantages6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [5.0, 2.3999998569488525, 11.800000190734863, 9.59999942779541, 4.400000095367432, 0.19999998807907104, 3.799999952316284, 4.599999904632568, 7.799999713897705, 41.599998474121094, 4.199999809265137, 0.19999998807907104, 4.0, 32.0, 9.199999809265137, 57.79999923706055, 3.3999998569488525, 8.399999618530273, 5.599999904632568, 13.59999942779541, 8.0, 5.400000095367432, 4.400000095367432, 0.5999999642372131, 2.200000047683716, 1.399999976158142, 8.0, 13.59999942779541, 9.199999809265137, 1.399999976158142, 3.0, 1.5999999046325684, 7.199999809265137, 24.399999618530273, 3.5999999046325684, 0.19999998807907104, 2.799999952316284, 3.1999998092651367, 7.399999618530273, 15.399999618530273, 4.400000095367432, 2.200000047683716, 3.1999998092651367, 3.3999998569488525, 11.0, 18.19999885559082, 5.199999809265137, 0.5999999642372131] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [5.0, 2.3999998569488525, 3.799999952316284, 4.599999904632568, 4.0, 32.0, 5.599999904632568, 13.59999942779541, 2.200000047683716, 1.399999976158142, 3.0, 1.5999999046325684, 2.799999952316284, 3.1999998092651367, 3.1999998092651367, 3.3999998569488525] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [11.800000190734863, 9.59999942779541, 7.799999713897705, 41.599998474121094, 9.199999809265137, 57.79999923706055, 8.0, 5.400000095367432, 8.0, 13.59999942779541, 7.199999809265137, 24.399999618530273, 7.399999618530273, 15.399999618530273, 11.0, 18.19999885559082] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [4.400000095367432, 0.19999998807907104, 4.199999809265137, 0.19999998807907104, 3.3999998569488525, 8.399999618530273, 4.400000095367432, 0.5999999642372131, 9.199999809265137, 1.399999976158142, 3.5999999046325684, 0.19999998807907104, 4.400000095367432, 2.200000047683716, 5.199999809265137, 0.5999999642372131] +------------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 8.35+/- 1.562 (max: 57.8) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 3.287+/- 0.6988 (max: 9.2) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 5.737+/- 1.887 (max: 32.0) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 16.02+/- 3.578 (max: 57.8) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 11.03+/- 0.9006 (max: 30.19) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 6.996+/- 0.9011 (max: 13.39) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 9.519+/- 1.362 (max: 27.42) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 16.57+/- 1.294 (max: 30.19) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.08312+/- 0.02371 (max: 0.79) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.01312+/- 0.004539 (max: 0.06) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.04437+/- 0.02831 (max: 0.45) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.1919+/- 0.05697 (max: 0.79) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.2 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.2 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 1.4 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 5.4 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 1.99 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 1.99 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 5.103 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 10.95 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.03 | +------------------------------------------------------------------------------------------------------- +Evaluating ACCEL_CNN-LSTM_SEED1 against population in Overcooked-CrampedRoom6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [8.199999809265137, 6.399999618530273, 50.0, 56.19999694824219, 66.79999542236328, 68.4000015258789, 9.800000190734863, 8.800000190734863, 53.79999923706055, 54.39999771118164, 70.0, 66.5999984741211, 12.399999618530273, 13.0, 62.19999694824219, 63.0, 56.599998474121094, 59.39999771118164, 11.800000190734863, 14.799999237060547, 54.39999771118164, 64.5999984741211, 39.39999771118164, 49.0, 5.599999904632568, 8.199999809265137, 51.19999694824219, 47.0, 61.599998474121094, 62.39999771118164, 11.800000190734863, 11.199999809265137, 63.599998474121094, 71.4000015258789, 75.4000015258789, 77.0, 10.199999809265137, 11.0, 57.39999771118164, 65.5999984741211, 67.19999694824219, 69.0, 15.799999237060547, 14.59999942779541, 62.19999694824219, 74.79999542236328, 74.5999984741211, 72.79999542236328] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [8.199999809265137, 6.399999618530273, 9.800000190734863, 8.800000190734863, 12.399999618530273, 13.0, 11.800000190734863, 14.799999237060547, 5.599999904632568, 8.199999809265137, 11.800000190734863, 11.199999809265137, 10.199999809265137, 11.0, 15.799999237060547, 14.59999942779541] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [50.0, 56.19999694824219, 53.79999923706055, 54.39999771118164, 62.19999694824219, 63.0, 54.39999771118164, 64.5999984741211, 51.19999694824219, 47.0, 63.599998474121094, 71.4000015258789, 57.39999771118164, 65.5999984741211, 62.19999694824219, 74.79999542236328] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [66.79999542236328, 68.4000015258789, 70.0, 66.5999984741211, 56.599998474121094, 59.39999771118164, 39.39999771118164, 49.0, 61.599998474121094, 62.39999771118164, 75.4000015258789, 77.0, 67.19999694824219, 69.0, 74.5999984741211, 72.79999542236328] +-------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 45.03+/- 3.695 (max: 77.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 64.76+/- 2.499 (max: 77.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 10.85+/- 0.7368 (max: 15.8) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 59.49+/- 1.934 (max: 74.8) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 26.04+/- 1.36 (max: 36.25) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 32.06+/- 0.492 (max: 34.81) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 13.22+/- 0.4322 (max: 15.19) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 32.83+/- 0.7195 (max: 36.25) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.5617+/- 0.04909 (max: 0.93) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.8125+/- 0.02357 (max: 0.93) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.09687+/- 0.01341 (max: 0.19) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.7756+/- 0.01641 (max: 0.92) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 5.6 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 39.4 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 5.6 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 47.0 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 10.54 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 27.89 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 10.54 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 26.18 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.56 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.66 | +-------------------------------------------------------------------------------------------------- + + + + + + + +Evaluating ACCEL_CNN-LSTM_SEED2 against population in Overcooked-CoordRing6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [2.0, 3.799999952316284, 6.399999618530273, 7.199999809265137, 5.199999809265137, 5.400000095367432, 8.0, 8.399999618530273, 12.59999942779541, 16.19999885559082, 7.199999809265137, 10.199999809265137, 9.199999809265137, 9.59999942779541, 10.199999809265137, 13.199999809265137, 11.59999942779541, 11.0, 7.599999904632568, 7.399999618530273, 8.0, 8.199999809265137, 8.199999809265137, 8.199999809265137, 9.0, 10.199999809265137, 3.0, 3.0, 7.599999904632568, 8.800000190734863, 6.0, 7.799999713897705, 12.199999809265137, 15.199999809265137, 19.19999885559082, 17.399999618530273, 0.7999999523162842, 1.1999999284744263, 3.799999952316284, 6.0, 10.59999942779541, 16.0, 7.0, 10.59999942779541, 4.199999809265137, 7.0, 7.399999618530273, 14.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [2.0, 3.799999952316284, 8.0, 8.399999618530273, 9.199999809265137, 9.59999942779541, 7.599999904632568, 7.399999618530273, 9.0, 10.199999809265137, 6.0, 7.799999713897705, 0.7999999523162842, 1.1999999284744263, 7.0, 10.59999942779541] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [6.399999618530273, 7.199999809265137, 12.59999942779541, 16.19999885559082, 10.199999809265137, 13.199999809265137, 8.0, 8.199999809265137, 3.0, 3.0, 12.199999809265137, 15.199999809265137, 3.799999952316284, 6.0, 4.199999809265137, 7.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [5.199999809265137, 5.400000095367432, 7.199999809265137, 10.199999809265137, 11.59999942779541, 11.0, 8.199999809265137, 8.199999809265137, 7.599999904632568, 8.800000190734863, 19.19999885559082, 17.399999618530273, 10.59999942779541, 16.0, 7.399999618530273, 14.0] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 8.604+/- 0.5957 (max: 19.2) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 10.5+/- 1.048 (max: 19.2) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 6.787+/- 0.7919 (max: 10.6) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 8.525+/- 1.072 (max: 16.2) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 10.35+/- 0.2982 (max: 15.0) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 10.78+/- 0.3111 (max: 13.27) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 9.788+/- 0.6564 (max: 12.16) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 10.49+/- 0.5226 (max: 15.0) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.03708+/- 0.006489 (max: 0.19) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.04187+/- 0.01327 (max: 0.17) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0275+/- 0.005123 (max: 0.06) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.04187+/- 0.01358 (max: 0.19) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 0.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 5.2 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 0.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 3.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 3.919 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 8.773 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 3.919 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 7.681 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------- +Evaluating ACCEL_CNN-LSTM_SEED2 against population in Overcooked-ForcedCoord6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [2.5999999046325684, 0.0, 3.5999999046325684, 0.19999998807907104, 1.5999999046325684, 0.7999999523162842, 2.200000047683716, 0.19999998807907104, 5.799999713897705, 0.3999999761581421, 2.799999952316284, 1.0, 3.799999952316284, 0.7999999523162842, 3.1999998092651367, 1.5999999046325684, 2.200000047683716, 2.3999998569488525, 3.799999952316284, 1.0, 3.1999998092651367, 0.3999999761581421, 2.0, 0.19999998807907104, 3.3999998569488525, 0.3999999761581421, 4.400000095367432, 1.399999976158142, 5.599999904632568, 0.3999999761581421, 2.200000047683716, 1.0, 2.5999999046325684, 0.19999998807907104, 1.5999999046325684, 0.7999999523162842, 2.0, 0.0, 2.0, 0.3999999761581421, 2.200000047683716, 1.0, 4.400000095367432, 0.5999999642372131, 4.400000095367432, 1.0, 6.199999809265137, 0.19999998807907104] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [2.5999999046325684, 0.0, 2.200000047683716, 0.19999998807907104, 3.799999952316284, 0.7999999523162842, 3.799999952316284, 1.0, 3.3999998569488525, 0.3999999761581421, 2.200000047683716, 1.0, 2.0, 0.0, 4.400000095367432, 0.5999999642372131] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [3.5999999046325684, 0.19999998807907104, 5.799999713897705, 0.3999999761581421, 3.1999998092651367, 1.5999999046325684, 3.1999998092651367, 0.3999999761581421, 4.400000095367432, 1.399999976158142, 2.5999999046325684, 0.19999998807907104, 2.0, 0.3999999761581421, 4.400000095367432, 1.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [1.5999999046325684, 0.7999999523162842, 2.799999952316284, 1.0, 2.200000047683716, 2.3999998569488525, 2.0, 0.19999998807907104, 5.599999904632568, 0.3999999761581421, 1.5999999046325684, 0.7999999523162842, 2.200000047683716, 1.0, 6.199999809265137, 0.19999998807907104] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 1.962+/- 0.2372 (max: 6.2) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 1.937+/- 0.4362 (max: 6.2) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 1.775+/- 0.371 (max: 4.4) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 2.175+/- 0.4423 (max: 5.8) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 5.226+/- 0.3505 (max: 9.506) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 5.273+/- 0.5376 (max: 9.25) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 4.879+/- 0.6822 (max: 8.34) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 5.526+/- 0.6217 (max: 9.506) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0004167+/- 0.0002915 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.000625+/- 0.000625 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.000625+/- 0.000625 (max: 0.01) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.2 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.2 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 1.99 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 1.99 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating ACCEL_CNN-LSTM_SEED2 against population in Overcooked-CounterCircuit6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [10.59999942779541, 10.199999809265137, 17.399999618530273, 13.199999809265137, 13.799999237060547, 12.199999809265137, 5.400000095367432, 5.0, 11.399999618530273, 9.800000190734863, 0.3999999761581421, 0.0, 0.19999998807907104, 0.0, 4.199999809265137, 0.7999999523162842, 2.200000047683716, 2.200000047683716, 6.0, 4.799999713897705, 19.799999237060547, 11.399999618530273, 9.399999618530273, 5.199999809265137, 12.0, 7.199999809265137, 12.399999618530273, 9.59999942779541, 3.3999998569488525, 1.7999999523162842, 8.399999618530273, 8.399999618530273, 16.600000381469727, 9.199999809265137, 6.399999618530273, 5.400000095367432, 6.0, 4.799999713897705, 11.0, 9.0, 7.799999713897705, 7.199999809265137, 14.59999942779541, 9.59999942779541, 25.399999618530273, 13.799999237060547, 9.59999942779541, 5.599999904632568] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [10.59999942779541, 10.199999809265137, 5.400000095367432, 5.0, 0.19999998807907104, 0.0, 6.0, 4.799999713897705, 12.0, 7.199999809265137, 8.399999618530273, 8.399999618530273, 6.0, 4.799999713897705, 14.59999942779541, 9.59999942779541] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [17.399999618530273, 13.199999809265137, 11.399999618530273, 9.800000190734863, 4.199999809265137, 0.7999999523162842, 19.799999237060547, 11.399999618530273, 12.399999618530273, 9.59999942779541, 16.600000381469727, 9.199999809265137, 11.0, 9.0, 25.399999618530273, 13.799999237060547] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [13.799999237060547, 12.199999809265137, 0.3999999761581421, 0.0, 2.200000047683716, 2.200000047683716, 9.399999618530273, 5.199999809265137, 3.3999998569488525, 1.7999999523162842, 6.399999618530273, 5.400000095367432, 7.799999713897705, 7.199999809265137, 9.59999942779541, 5.599999904632568] +----------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 8.35+/- 0.7755 (max: 25.4) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 5.787+/- 1.02 (max: 13.8) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 7.075+/- 0.9791 (max: 14.6) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 12.19+/- 1.458 (max: 25.4) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 10.42+/- 0.629 (max: 23.13) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 8.145+/- 0.8287 (max: 13.17) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 9.386+/- 0.9118 (max: 13.22) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 13.73+/- 1.027 (max: 23.13) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.05792+/- 0.01221 (max: 0.41) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0125+/- 0.007042 (max: 0.11) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.02937+/- 0.00915 (max: 0.12) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.1319+/- 0.02666 (max: 0.41) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 0.8 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 3.919 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +----------------------------------------------------------------------------------------------------- +Evaluating ACCEL_CNN-LSTM_SEED2 against population in Overcooked-AsymmAdvantages6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [3.799999952316284, 2.200000047683716, 3.3999998569488525, 7.199999809265137, 3.799999952316284, 0.3999999761581421, 1.7999999523162842, 4.400000095367432, 6.199999809265137, 34.79999923706055, 3.1999998092651367, 0.7999999523162842, 5.599999904632568, 31.599998474121094, 4.599999904632568, 51.599998474121094, 5.799999713897705, 8.199999809265137, 3.5999999046325684, 14.0, 4.199999809265137, 5.199999809265137, 5.0, 1.5999999046325684, 3.0, 1.1999999284744263, 5.599999904632568, 16.399999618530273, 6.399999618530273, 1.7999999523162842, 3.5999999046325684, 2.3999998569488525, 4.199999809265137, 23.600000381469727, 5.799999713897705, 0.5999999642372131, 2.5999999046325684, 1.1999999284744263, 4.799999713897705, 14.199999809265137, 6.399999618530273, 1.399999976158142, 2.799999952316284, 3.5999999046325684, 4.799999713897705, 18.799999237060547, 6.199999809265137, 0.19999998807907104] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [3.799999952316284, 2.200000047683716, 1.7999999523162842, 4.400000095367432, 5.599999904632568, 31.599998474121094, 3.5999999046325684, 14.0, 3.0, 1.1999999284744263, 3.5999999046325684, 2.3999998569488525, 2.5999999046325684, 1.1999999284744263, 2.799999952316284, 3.5999999046325684] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [3.3999998569488525, 7.199999809265137, 6.199999809265137, 34.79999923706055, 4.599999904632568, 51.599998474121094, 4.199999809265137, 5.199999809265137, 5.599999904632568, 16.399999618530273, 4.199999809265137, 23.600000381469727, 4.799999713897705, 14.199999809265137, 4.799999713897705, 18.799999237060547] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [3.799999952316284, 0.3999999761581421, 3.1999998092651367, 0.7999999523162842, 5.799999713897705, 8.199999809265137, 5.0, 1.5999999046325684, 6.399999618530273, 1.7999999523162842, 5.799999713897705, 0.5999999642372131, 6.399999618530273, 1.399999976158142, 6.199999809265137, 0.19999998807907104] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 7.387+/- 1.42 (max: 51.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 3.6+/- 0.6651 (max: 8.2) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 5.462+/- 1.895 (max: 31.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 13.1+/- 3.398 (max: 51.6) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 10.14+/- 0.8291 (max: 29.15) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 7.411+/- 0.8157 (max: 10.91) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 9.115+/- 1.341 (max: 27.01) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 13.88+/- 1.58 (max: 29.15) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.07312+/- 0.02311 (max: 0.76) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.01312+/- 0.00395 (max: 0.04) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.04625+/- 0.03011 (max: 0.48) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.16+/- 0.05752 (max: 0.76) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.2 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.2 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 1.2 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 3.4 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 1.99 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 1.99 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 4.75 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 7.513 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating ACCEL_CNN-LSTM_SEED2 against population in Overcooked-CrampedRoom6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [8.0, 6.399999618530273, 63.39999771118164, 61.39999771118164, 70.4000015258789, 74.5999984741211, 10.800000190734863, 9.0, 60.19999694824219, 62.19999694824219, 76.79999542236328, 74.79999542236328, 13.199999809265137, 10.199999809265137, 66.5999984741211, 67.79999542236328, 61.79999923706055, 66.19999694824219, 11.800000190734863, 12.199999809265137, 73.4000015258789, 70.5999984741211, 55.0, 55.79999923706055, 5.799999713897705, 6.399999618530273, 53.599998474121094, 56.39999771118164, 64.4000015258789, 67.79999542236328, 12.799999237060547, 10.59999942779541, 70.19999694824219, 73.4000015258789, 81.4000015258789, 82.5999984741211, 11.399999618530273, 13.0, 62.19999694824219, 70.4000015258789, 72.0, 71.4000015258789, 14.799999237060547, 12.799999237060547, 66.5999984741211, 68.19999694824219, 74.5999984741211, 75.19999694824219] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [8.0, 6.399999618530273, 10.800000190734863, 9.0, 13.199999809265137, 10.199999809265137, 11.800000190734863, 12.199999809265137, 5.799999713897705, 6.399999618530273, 12.799999237060547, 10.59999942779541, 11.399999618530273, 13.0, 14.799999237060547, 12.799999237060547] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [63.39999771118164, 61.39999771118164, 60.19999694824219, 62.19999694824219, 66.5999984741211, 67.79999542236328, 73.4000015258789, 70.5999984741211, 53.599998474121094, 56.39999771118164, 70.19999694824219, 73.4000015258789, 62.19999694824219, 70.4000015258789, 66.5999984741211, 68.19999694824219] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [70.4000015258789, 74.5999984741211, 76.79999542236328, 74.79999542236328, 61.79999923706055, 66.19999694824219, 55.0, 55.79999923706055, 64.4000015258789, 67.79999542236328, 81.4000015258789, 82.5999984741211, 72.0, 71.4000015258789, 74.5999984741211, 75.19999694824219] +-------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 48.76+/- 4.038 (max: 82.6) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 70.3+/- 2.017 (max: 82.6) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 10.57+/- 0.6841 (max: 14.8) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 65.41+/- 1.454 (max: 73.4) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 25.92+/- 1.398 (max: 37.72) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 31.87+/- 0.6411 (max: 35.73) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 12.76+/- 0.4283 (max: 15.97) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 33.13+/- 0.649 (max: 37.72) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.5917+/- 0.05304 (max: 0.95) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.8687+/- 0.01793 (max: 0.95) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.08375+/- 0.01068 (max: 0.15) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.8225+/- 0.01185 (max: 0.9) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 5.8 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 55.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 5.8 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 53.6 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 9.506 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 26.96 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 9.506 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 29.13 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.01 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.7 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.01 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.71 | +-------------------------------------------------------------------------------------------------- + + + + + + + + +Evaluating ACCEL_CNN-LSTM_SEED3 against population in Overcooked-CoordRing6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [6.0, 7.599999904632568, 17.399999618530273, 15.399999618530273, 24.399999618530273, 22.799999237060547, 15.0, 16.799999237060547, 32.0, 33.39999771118164, 20.600000381469727, 19.600000381469727, 12.799999237060547, 15.199999809265137, 24.799999237060547, 24.399999618530273, 23.19999885559082, 22.19999885559082, 17.0, 13.399999618530273, 16.0, 17.0, 23.0, 22.399999618530273, 14.0, 11.59999942779541, 22.399999618530273, 20.600000381469727, 24.19999885559082, 22.799999237060547, 15.399999618530273, 17.799999237060547, 28.599998474121094, 28.19999885559082, 37.39999771118164, 35.79999923706055, 7.799999713897705, 4.599999904632568, 21.799999237060547, 19.19999885559082, 29.399999618530273, 28.399999618530273, 12.399999618530273, 11.199999809265137, 15.0, 14.199999809265137, 15.799999237060547, 16.399999618530273] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [6.0, 7.599999904632568, 15.0, 16.799999237060547, 12.799999237060547, 15.199999809265137, 17.0, 13.399999618530273, 14.0, 11.59999942779541, 15.399999618530273, 17.799999237060547, 7.799999713897705, 4.599999904632568, 12.399999618530273, 11.199999809265137] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [17.399999618530273, 15.399999618530273, 32.0, 33.39999771118164, 24.799999237060547, 24.399999618530273, 16.0, 17.0, 22.399999618530273, 20.600000381469727, 28.599998474121094, 28.19999885559082, 21.799999237060547, 19.19999885559082, 15.0, 14.199999809265137] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [24.399999618530273, 22.799999237060547, 20.600000381469727, 19.600000381469727, 23.19999885559082, 22.19999885559082, 23.0, 22.399999618530273, 24.19999885559082, 22.799999237060547, 37.39999771118164, 35.79999923706055, 29.399999618530273, 28.399999618530273, 15.799999237060547, 16.399999618530273] +----------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 19.53+/- 1.075 (max: 37.4) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 24.27+/- 1.49 (max: 37.4) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 12.41+/- 1.011 (max: 17.8) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 21.9+/- 1.542 (max: 33.4) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 12.34+/- 0.2959 (max: 18.35) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 10.98+/- 0.4412 (max: 14.23) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 12.52+/- 0.4541 (max: 15.25) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 13.51+/- 0.4522 (max: 18.35) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.2285+/- 0.02794 (max: 0.8) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.2969+/- 0.05599 (max: 0.8) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.1+/- 0.01732 (max: 0.22) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.2887+/- 0.04742 (max: 0.66) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 4.6 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 15.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 4.6 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 14.2 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 8.146 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 8.146 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 8.417 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 11.26 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.09 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.07 | +----------------------------------------------------------------------------------------------- +Evaluating ACCEL_CNN-LSTM_SEED3 against population in Overcooked-ForcedCoord6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [4.199999809265137, 0.0, 4.400000095367432, 0.3999999761581421, 4.199999809265137, 0.3999999761581421, 2.3999998569488525, 0.0, 3.0, 0.0, 3.0, 0.0, 4.599999904632568, 0.19999998807907104, 3.5999999046325684, 0.7999999523162842, 2.3999998569488525, 1.399999976158142, 4.0, 0.3999999761581421, 4.199999809265137, 0.0, 2.3999998569488525, 0.0, 3.5999999046325684, 0.19999998807907104, 5.199999809265137, 0.19999998807907104, 4.400000095367432, 0.3999999761581421, 2.799999952316284, 0.19999998807907104, 2.200000047683716, 0.0, 1.7999999523162842, 0.3999999761581421, 1.5999999046325684, 0.19999998807907104, 2.0, 0.19999998807907104, 1.7999999523162842, 0.19999998807907104, 2.200000047683716, 0.0, 5.599999904632568, 0.19999998807907104, 5.199999809265137, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [4.199999809265137, 0.0, 2.3999998569488525, 0.0, 4.599999904632568, 0.19999998807907104, 4.0, 0.3999999761581421, 3.5999999046325684, 0.19999998807907104, 2.799999952316284, 0.19999998807907104, 1.5999999046325684, 0.19999998807907104, 2.200000047683716, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [4.400000095367432, 0.3999999761581421, 3.0, 0.0, 3.5999999046325684, 0.7999999523162842, 4.199999809265137, 0.0, 5.199999809265137, 0.19999998807907104, 2.200000047683716, 0.0, 2.0, 0.19999998807907104, 5.599999904632568, 0.19999998807907104] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [4.199999809265137, 0.3999999761581421, 3.0, 0.0, 2.3999998569488525, 1.399999976158142, 2.3999998569488525, 0.0, 4.400000095367432, 0.3999999761581421, 1.7999999523162842, 0.3999999761581421, 1.7999999523162842, 0.19999998807907104, 5.199999809265137, 0.0] +---------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 1.804+/- 0.2608 (max: 5.6) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 1.75+/- 0.4307 (max: 5.2) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 1.662+/- 0.4323 (max: 4.6) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 2.0+/- 0.5128 (max: 5.6) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 4.572+/- 0.4702 (max: 9.319) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 4.618+/- 0.7886 (max: 8.773) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 4.346+/- 0.8285 (max: 9.319) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 4.753+/- 0.8742 (max: 9.217) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.002083+/- 0.000663 (max: 0.02) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.001875+/- 0.001008 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.001875+/- 0.00136 (max: 0.02) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0025+/- 0.001118 (max: 0.01) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +---------------------------------------------------------------------------------------------------- +Evaluating ACCEL_CNN-LSTM_SEED3 against population in Overcooked-CounterCircuit6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [4.0, 2.200000047683716, 4.599999904632568, 2.3999998569488525, 0.7999999523162842, 1.0, 1.5999999046325684, 1.1999999284744263, 2.0, 2.0, 0.3999999761581421, 0.0, 4.599999904632568, 2.200000047683716, 13.0, 5.599999904632568, 4.799999713897705, 3.0, 3.5999999046325684, 1.0, 12.199999809265137, 8.800000190734863, 3.0, 0.5999999642372131, 3.3999998569488525, 2.5999999046325684, 8.399999618530273, 5.0, 3.1999998092651367, 1.0, 4.400000095367432, 1.399999976158142, 7.399999618530273, 3.5999999046325684, 3.1999998092651367, 1.5999999046325684, 1.7999999523162842, 1.0, 4.599999904632568, 2.3999998569488525, 2.200000047683716, 0.3999999761581421, 3.799999952316284, 1.1999999284744263, 2.3999998569488525, 1.5999999046325684, 3.799999952316284, 0.19999998807907104] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [4.0, 2.200000047683716, 1.5999999046325684, 1.1999999284744263, 4.599999904632568, 2.200000047683716, 3.5999999046325684, 1.0, 3.3999998569488525, 2.5999999046325684, 4.400000095367432, 1.399999976158142, 1.7999999523162842, 1.0, 3.799999952316284, 1.1999999284744263] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [4.599999904632568, 2.3999998569488525, 2.0, 2.0, 13.0, 5.599999904632568, 12.199999809265137, 8.800000190734863, 8.399999618530273, 5.0, 7.399999618530273, 3.5999999046325684, 4.599999904632568, 2.3999998569488525, 2.3999998569488525, 1.5999999046325684] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [0.7999999523162842, 1.0, 0.3999999761581421, 0.0, 4.799999713897705, 3.0, 3.0, 0.5999999642372131, 3.1999998092651367, 1.0, 3.1999998092651367, 1.5999999046325684, 2.200000047683716, 0.3999999761581421, 3.799999952316284, 0.19999998807907104] +------------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 3.233+/- 0.4047 (max: 13.0) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 1.825+/- 0.3732 (max: 4.8) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 2.5+/- 0.3199 (max: 4.6) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 5.375+/- 0.9079 (max: 13.0) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 6.684+/- 0.3522 (max: 11.96) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 5.113+/- 0.6292 (max: 8.542) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 6.42+/- 0.3651 (max: 8.485) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 8.52+/- 0.4931 (max: 11.96) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.005+/- 0.001518 (max: 0.06) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.00125+/- 0.0008539 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.00125+/- 0.0008539 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0125+/- 0.003819 (max: 0.06) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 1.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 1.6 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 4.359 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 5.426 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------- +Evaluating ACCEL_CNN-LSTM_SEED3 against population in Overcooked-AsymmAdvantages6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [0.19999998807907104, 7.199999809265137, 0.19999998807907104, 16.799999237060547, 0.3999999761581421, 10.399999618530273, 0.3999999761581421, 10.399999618530273, 0.3999999761581421, 42.39999771118164, 0.5999999642372131, 11.0, 0.19999998807907104, 41.39999771118164, 0.19999998807907104, 59.39999771118164, 0.19999998807907104, 19.600000381469727, 0.19999998807907104, 20.600000381469727, 0.19999998807907104, 15.799999237060547, 0.3999999761581421, 10.199999809265137, 0.19999998807907104, 6.199999809265137, 0.19999998807907104, 25.19999885559082, 0.3999999761581421, 12.799999237060547, 0.19999998807907104, 6.799999713897705, 0.19999998807907104, 28.799999237060547, 0.3999999761581421, 10.0, 0.19999998807907104, 6.199999809265137, 0.19999998807907104, 28.19999885559082, 0.3999999761581421, 13.399999618530273, 0.3999999761581421, 9.800000190734863, 0.19999998807907104, 28.19999885559082, 0.19999998807907104, 10.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.19999998807907104, 7.199999809265137, 0.3999999761581421, 10.399999618530273, 0.19999998807907104, 41.39999771118164, 0.19999998807907104, 20.600000381469727, 0.19999998807907104, 6.199999809265137, 0.19999998807907104, 6.799999713897705, 0.19999998807907104, 6.199999809265137, 0.3999999761581421, 9.800000190734863] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.19999998807907104, 16.799999237060547, 0.3999999761581421, 42.39999771118164, 0.19999998807907104, 59.39999771118164, 0.19999998807907104, 15.799999237060547, 0.19999998807907104, 25.19999885559082, 0.19999998807907104, 28.799999237060547, 0.19999998807907104, 28.19999885559082, 0.19999998807907104, 28.19999885559082] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [0.3999999761581421, 10.399999618530273, 0.5999999642372131, 11.0, 0.19999998807907104, 19.600000381469727, 0.3999999761581421, 10.199999809265137, 0.3999999761581421, 12.799999237060547, 0.3999999761581421, 10.0, 0.3999999761581421, 13.399999618530273, 0.19999998807907104, 10.0] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 9.533+/- 1.922 (max: 59.4) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 6.275+/- 1.623 (max: 19.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 6.912+/- 2.703 (max: 41.4) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 15.41+/- 4.615 (max: 59.4) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 9.404+/- 1.199 (max: 31.81) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 7.925+/- 1.398 (max: 16.73) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 8.098+/- 1.813 (max: 26.57) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 12.19+/- 2.74 (max: 31.81) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.1154+/- 0.02953 (max: 0.84) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.04937+/- 0.01822 (max: 0.26) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.07187+/- 0.04141 (max: 0.64) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.225+/- 0.07024 (max: 0.84) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.2 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.2 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 0.2 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.2 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 1.99 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 1.99 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 1.99 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 1.99 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating ACCEL_CNN-LSTM_SEED3 against population in Overcooked-CrampedRoom6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [10.59999942779541, 8.199999809265137, 66.79999542236328, 63.79999923706055, 105.39999389648438, 105.19999694824219, 17.600000381469727, 16.399999618530273, 74.4000015258789, 69.4000015258789, 104.79999542236328, 102.0, 19.19999885559082, 17.799999237060547, 94.4000015258789, 93.19999694824219, 104.5999984741211, 103.5999984741211, 20.799999237060547, 19.600000381469727, 80.4000015258789, 80.79999542236328, 91.5999984741211, 88.5999984741211, 9.399999618530273, 10.800000190734863, 68.5999984741211, 71.0, 95.0, 96.5999984741211, 18.399999618530273, 17.799999237060547, 86.4000015258789, 84.0, 100.0, 102.79999542236328, 20.0, 18.19999885559082, 73.19999694824219, 73.0, 98.5999984741211, 103.5999984741211, 17.399999618530273, 16.600000381469727, 84.0, 82.5999984741211, 104.0, 104.39999389648438] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [10.59999942779541, 8.199999809265137, 17.600000381469727, 16.399999618530273, 19.19999885559082, 17.799999237060547, 20.799999237060547, 19.600000381469727, 9.399999618530273, 10.800000190734863, 18.399999618530273, 17.799999237060547, 20.0, 18.19999885559082, 17.399999618530273, 16.600000381469727] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [66.79999542236328, 63.79999923706055, 74.4000015258789, 69.4000015258789, 94.4000015258789, 93.19999694824219, 80.4000015258789, 80.79999542236328, 68.5999984741211, 71.0, 86.4000015258789, 84.0, 73.19999694824219, 73.0, 84.0, 82.5999984741211] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [105.39999389648438, 105.19999694824219, 104.79999542236328, 102.0, 104.5999984741211, 103.5999984741211, 91.5999984741211, 88.5999984741211, 95.0, 96.5999984741211, 100.0, 102.79999542236328, 98.5999984741211, 103.5999984741211, 104.0, 104.39999389648438] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 64.91+/- 5.288 (max: 105.4) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 100.7+/- 1.299 (max: 105.4) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 16.17+/- 1.009 (max: 20.8) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 77.87+/- 2.306 (max: 94.4) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 20.01+/- 0.7235 (max: 28.98) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 21.78+/- 0.568 (max: 25.34) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 13.89+/- 0.4176 (max: 15.98) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 24.36+/- 0.6765 (max: 28.98) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.7027+/- 0.05534 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.9912+/- 0.002016 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.1712+/- 0.02091 (max: 0.3) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.9456+/- 0.00532 (max: 0.98) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 8.2 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 88.6 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 8.2 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 63.8 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 10.62 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 16.4 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 10.62 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 18.7 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.98 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.91 | +------------------------------------------------------------------------------------------------- diff --git a/src/run_results_txt/eval_xpid_all_cnn_s5_out.txt b/src/run_results_txt/eval_xpid_all_cnn_s5_out.txt new file mode 100644 index 0000000..16f5ab5 --- /dev/null +++ b/src/run_results_txt/eval_xpid_all_cnn_s5_out.txt @@ -0,0 +1,2290 @@ +Evaluating DR_CNN-S5_SEED1 against population in Overcooked-CoordRing6_9 for xpid dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 + +k eval/a0:test_return:Overcooked-CoordRing6_9, v [3.0, 0.0, 0.3999999761581421, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.5999999046325684, 0.0, 0.0, 0.0, 0.0, 0.0, 10.800000190734863, 0.0, 9.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 2.200000047683716, 0.0, 12.799999237060547, 0.0, 0.0, 0.0, 0.3999999761581421, 0.0, 3.799999952316284, 0.0, 0.0, 0.0, 1.5999999046325684, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [3.0, 0.0, 0.0, 0.0, 1.5999999046325684, 0.0, 10.800000190734863, 0.0, 0.0, 0.0, 2.200000047683716, 0.0, 0.3999999761581421, 0.0, 1.5999999046325684, 0.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [0.3999999761581421, 0.0, 0.0, 0.0, 0.0, 0.0, 9.0, 0.0, 0.0, 0.0, 12.799999237060547, 0.0, 3.799999952316284, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +-------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 0.95+/- 0.3949 (max: 12.8) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 1.225+/- 0.6824 (max: 10.8) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 1.625+/- 0.9513 (max: 12.8) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 1.459+/- 0.4567 (max: 11.5) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 2.372+/- 0.8713 (max: 10.36) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 2.006+/- 0.992 (max: 11.5) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.001458+/- 0.001073 (max: 0.05) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.00125+/- 0.0008539 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.003125+/- 0.003125 (max: 0.05) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0 | +-------------------------------------------------------------------------------------------------- +Evaluating DR_CNN-S5_SEED1 against population in Overcooked-ForcedCoord6_9 for xpid dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 + +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +----------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +----------------------------------------------------------------------------------------- +Evaluating DR_CNN-S5_SEED1 against population in Overcooked-CounterCircuit6_9 for xpid dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 + +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.19999998807907104, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.19999998807907104, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.004167+/- 0.004167 (max: 0.2) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 0.0125+/- 0.0125 (max: 0.2) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 0.04146+/- 0.04146 (max: 1.99) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 0.1244+/- 0.1244 (max: 1.99) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating DR_CNN-S5_SEED1 against population in Overcooked-AsymmAdvantages6_9 for xpid dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 + +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [0.0, 1.0, 0.0, 6.399999618530273, 0.0, 0.0, 0.0, 4.599999904632568, 0.0, 44.20000076293945, 0.0, 0.19999998807907104, 0.0, 32.0, 0.0, 60.39999771118164, 0.0, 9.199999809265137, 0.0, 16.399999618530273, 0.0, 7.0, 0.0, 0.19999998807907104, 0.0, 0.5999999642372131, 0.0, 14.59999942779541, 0.0, 0.7999999523162842, 0.0, 1.7999999523162842, 0.0, 26.599998474121094, 0.0, 0.0, 0.0, 0.7999999523162842, 0.0, 15.59999942779541, 0.0, 1.1999999284744263, 0.0, 2.3999998569488525, 0.0, 18.799999237060547, 0.0, 0.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.0, 1.0, 0.0, 4.599999904632568, 0.0, 32.0, 0.0, 16.399999618530273, 0.0, 0.5999999642372131, 0.0, 1.7999999523162842, 0.0, 0.7999999523162842, 0.0, 2.3999998569488525] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.0, 6.399999618530273, 0.0, 44.20000076293945, 0.0, 60.39999771118164, 0.0, 7.0, 0.0, 14.59999942779541, 0.0, 26.599998474121094, 0.0, 15.59999942779541, 0.0, 18.799999237060547] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [0.0, 0.0, 0.0, 0.19999998807907104, 0.0, 9.199999809265137, 0.0, 0.19999998807907104, 0.0, 0.7999999523162842, 0.0, 0.0, 0.0, 1.1999999284744263, 0.0, 0.0] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 5.517+/- 1.782 (max: 60.4) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.725+/- 0.5715 (max: 9.2) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 3.725+/- 2.144 (max: 32.0) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 12.1+/- 4.503 (max: 60.4) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 5.472+/- 1.273 (max: 35.1) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 1.414+/- 0.6869 (max: 9.968) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 4.819+/- 1.914 (max: 27.71) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 10.18+/- 2.912 (max: 35.1) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.06771+/- 0.02541 (max: 0.76) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.04187+/- 0.03017 (max: 0.44) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.1612+/- 0.06501 (max: 0.76) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating DR_CNN-S5_SEED1 against population in Overcooked-CrampedRoom6_9 for xpid dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 + +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [7.799999713897705, 0.0, 95.5999984741211, 0.0, 65.79999542236328, 0.0, 15.0, 0.0, 69.4000015258789, 0.0, 19.0, 0.0, 14.399999618530273, 0.0, 98.79999542236328, 0.0, 6.599999904632568, 0.0, 17.0, 0.0, 89.0, 0.0, 21.19999885559082, 0.0, 8.800000190734863, 0.0, 97.0, 0.0, 23.399999618530273, 0.0, 16.799999237060547, 0.0, 59.79999923706055, 0.0, 12.799999237060547, 0.0, 14.199999809265137, 0.0, 105.19999694824219, 0.0, 13.0, 0.0, 21.0, 0.0, 90.79999542236328, 0.0, 10.0, 0.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [7.799999713897705, 0.0, 15.0, 0.0, 14.399999618530273, 0.0, 17.0, 0.0, 8.800000190734863, 0.0, 16.799999237060547, 0.0, 14.199999809265137, 0.0, 21.0, 0.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [95.5999984741211, 0.0, 69.4000015258789, 0.0, 98.79999542236328, 0.0, 89.0, 0.0, 97.0, 0.0, 59.79999923706055, 0.0, 105.19999694824219, 0.0, 90.79999542236328, 0.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [65.79999542236328, 0.0, 19.0, 0.0, 6.599999904632568, 0.0, 21.19999885559082, 0.0, 23.399999618530273, 0.0, 12.799999237060547, 0.0, 13.0, 0.0, 10.0, 0.0] +-------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 20.67+/- 4.772 (max: 105.2) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 10.74+/- 4.242 (max: 65.8) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 7.187+/- 1.998 (max: 21.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 44.1+/- 11.69 (max: 105.2) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 11.87+/- 2.129 (max: 47.28) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 9.75+/- 2.695 (max: 31.6) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 6.978+/- 1.844 (max: 17.52) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 18.89+/- 5.173 (max: 47.28) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.2175+/- 0.04867 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.1394+/- 0.05959 (max: 0.94) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.07687+/- 0.02623 (max: 0.33) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.4362+/- 0.1145 (max: 1.0) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.0 | +-------------------------------------------------------------------------------------------------- +Evaluating DR_CNN-S5_SEED2 against population in Overcooked-CoordRing6_9 for xpid SEED_2_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 + +k eval/a0:test_return:Overcooked-CoordRing6_9, v [5.599999904632568, 6.0, 20.399999618530273, 19.799999237060547, 16.799999237060547, 17.19999885559082, 14.799999237060547, 15.799999237060547, 28.799999237060547, 33.79999923706055, 17.0, 18.399999618530273, 12.799999237060547, 13.0, 26.399999618530273, 23.600000381469727, 17.0, 18.399999618530273, 14.799999237060547, 13.59999942779541, 19.799999237060547, 19.19999885559082, 13.59999942779541, 16.19999885559082, 15.199999809265137, 14.0, 21.0, 20.19999885559082, 17.600000381469727, 20.799999237060547, 16.0, 14.399999618530273, 28.0, 27.399999618530273, 31.399999618530273, 32.39999771118164, 3.799999952316284, 2.799999952316284, 20.399999618530273, 20.399999618530273, 22.799999237060547, 26.399999618530273, 11.399999618530273, 12.799999237060547, 16.399999618530273, 15.59999942779541, 15.59999942779541, 17.600000381469727] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [5.599999904632568, 6.0, 14.799999237060547, 15.799999237060547, 12.799999237060547, 13.0, 14.799999237060547, 13.59999942779541, 15.199999809265137, 14.0, 16.0, 14.399999618530273, 3.799999952316284, 2.799999952316284, 11.399999618530273, 12.799999237060547] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [20.399999618530273, 19.799999237060547, 28.799999237060547, 33.79999923706055, 26.399999618530273, 23.600000381469727, 19.799999237060547, 19.19999885559082, 21.0, 20.19999885559082, 28.0, 27.399999618530273, 20.399999618530273, 20.399999618530273, 16.399999618530273, 15.59999942779541] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [16.799999237060547, 17.19999885559082, 17.0, 18.399999618530273, 17.0, 18.399999618530273, 13.59999942779541, 16.19999885559082, 17.600000381469727, 20.799999237060547, 31.399999618530273, 32.39999771118164, 22.799999237060547, 26.399999618530273, 15.59999942779541, 17.600000381469727] +----------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 18.07+/- 0.9793 (max: 33.8) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 19.95+/- 1.385 (max: 32.4) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 11.67+/- 1.115 (max: 16.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 22.57+/- 1.24 (max: 33.8) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 12.57+/- 0.3182 (max: 17.95) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 12.31+/- 0.6169 (max: 17.95) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 12.03+/- 0.5932 (max: 15.05) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 13.38+/- 0.3891 (max: 17.66) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.184+/- 0.02202 (max: 0.6) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.1869+/- 0.04031 (max: 0.56) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.08437+/- 0.01557 (max: 0.2) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.2806+/- 0.03748 (max: 0.6) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 2.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 13.6 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 2.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 15.6 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 6.94 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 9.666 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 6.94 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 11.16 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.06 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.07 | +----------------------------------------------------------------------------------------------- +Evaluating DR_CNN-S5_SEED2 against population in Overcooked-ForcedCoord6_9 for xpid SEED_2_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 + +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +----------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +----------------------------------------------------------------------------------------- +Evaluating DR_CNN-S5_SEED2 against population in Overcooked-CounterCircuit6_9 for xpid SEED_2_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 + +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [6.0, 2.799999952316284, 10.399999618530273, 6.199999809265137, 7.599999904632568, 1.399999976158142, 3.799999952316284, 2.3999998569488525, 5.0, 3.5999999046325684, 4.199999809265137, 0.0, 2.3999998569488525, 1.1999999284744263, 11.0, 4.799999713897705, 6.199999809265137, 1.7999999523162842, 4.199999809265137, 3.0, 13.199999809265137, 10.800000190734863, 10.0, 2.3999998569488525, 4.599999904632568, 1.7999999523162842, 10.800000190734863, 6.799999713897705, 8.800000190734863, 1.7999999523162842, 8.399999618530273, 1.5999999046325684, 10.0, 4.199999809265137, 12.0, 6.399999618530273, 3.799999952316284, 2.200000047683716, 10.800000190734863, 8.0, 7.399999618530273, 1.1999999284744263, 3.1999998092651367, 2.200000047683716, 5.799999713897705, 6.799999713897705, 7.599999904632568, 0.19999998807907104] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [6.0, 2.799999952316284, 3.799999952316284, 2.3999998569488525, 2.3999998569488525, 1.1999999284744263, 4.199999809265137, 3.0, 4.599999904632568, 1.7999999523162842, 8.399999618530273, 1.5999999046325684, 3.799999952316284, 2.200000047683716, 3.1999998092651367, 2.200000047683716] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [10.399999618530273, 6.199999809265137, 5.0, 3.5999999046325684, 11.0, 4.799999713897705, 13.199999809265137, 10.800000190734863, 10.800000190734863, 6.799999713897705, 10.0, 4.199999809265137, 10.800000190734863, 8.0, 5.799999713897705, 6.799999713897705] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [7.599999904632568, 1.399999976158142, 4.199999809265137, 0.0, 6.199999809265137, 1.7999999523162842, 10.0, 2.3999998569488525, 8.800000190734863, 1.7999999523162842, 12.0, 6.399999618530273, 7.399999618530273, 1.1999999284744263, 7.599999904632568, 0.19999998807907104] +------------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 5.433+/- 0.506 (max: 13.2) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 4.937+/- 0.9451 (max: 12.0) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 3.35+/- 0.4573 (max: 8.4) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 8.012+/- 0.7453 (max: 13.2) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 8.302+/- 0.3819 (max: 13.03) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 7.307+/- 0.7994 (max: 10.58) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 7.157+/- 0.3573 (max: 10.27) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 10.44+/- 0.3814 (max: 13.03) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.01042+/- 0.002962 (max: 0.1) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0025+/- 0.001443 (max: 0.02) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.000625+/- 0.000625 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.02812+/- 0.006965 (max: 0.1) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 1.2 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 3.6 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 4.75 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 8.146 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------- +Evaluating DR_CNN-S5_SEED2 against population in Overcooked-AsymmAdvantages6_9 for xpid SEED_2_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 + +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [0.0, 0.7999999523162842, 0.0, 8.199999809265137, 0.0, 0.0, 0.0, 3.799999952316284, 0.0, 42.599998474121094, 0.0, 0.19999998807907104, 0.0, 30.19999885559082, 0.0, 50.39999771118164, 0.0, 8.59999942779541, 0.0, 13.799999237060547, 0.0, 5.599999904632568, 0.0, 0.5999999642372131, 0.0, 1.1999999284744263, 0.0, 16.799999237060547, 0.0, 0.3999999761581421, 0.0, 0.7999999523162842, 0.0, 25.399999618530273, 0.0, 0.19999998807907104, 0.0, 1.5999999046325684, 0.0, 13.199999809265137, 0.0, 2.5999999046325684, 0.0, 2.3999998569488525, 0.0, 19.799999237060547, 0.0, 0.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.0, 0.7999999523162842, 0.0, 3.799999952316284, 0.0, 30.19999885559082, 0.0, 13.799999237060547, 0.0, 1.1999999284744263, 0.0, 0.7999999523162842, 0.0, 1.5999999046325684, 0.0, 2.3999998569488525] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.0, 8.199999809265137, 0.0, 42.599998474121094, 0.0, 50.39999771118164, 0.0, 5.599999904632568, 0.0, 16.799999237060547, 0.0, 25.399999618530273, 0.0, 13.199999809265137, 0.0, 19.799999237060547] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [0.0, 0.0, 0.0, 0.19999998807907104, 0.0, 8.59999942779541, 0.0, 0.5999999642372131, 0.0, 0.3999999761581421, 0.0, 0.19999998807907104, 0.0, 2.5999999046325684, 0.0, 0.0] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 5.192+/- 1.612 (max: 50.4) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.7875+/- 0.5454 (max: 8.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 3.412+/- 1.981 (max: 30.2) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 11.37+/- 4.019 (max: 50.4) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 5.41+/- 1.211 (max: 31.68) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 1.676+/- 0.7238 (max: 9.902) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 5.024+/- 1.983 (max: 27.2) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 9.528+/- 2.696 (max: 31.68) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.06604+/- 0.02441 (max: 0.76) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.03875+/- 0.0274 (max: 0.4) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.1594+/- 0.0627 (max: 0.76) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating DR_CNN-S5_SEED2 against population in Overcooked-CrampedRoom6_9 for xpid SEED_2_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 + +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [9.199999809265137, 8.59999942779541, 67.79999542236328, 66.79999542236328, 103.19999694824219, 105.0, 15.399999618530273, 14.799999237060547, 76.5999984741211, 75.19999694824219, 102.5999984741211, 107.0, 20.0, 18.19999885559082, 95.4000015258789, 95.4000015258789, 100.5999984741211, 101.4000015258789, 17.799999237060547, 17.0, 89.4000015258789, 87.19999694824219, 90.19999694824219, 90.19999694824219, 9.59999942779541, 7.0, 76.4000015258789, 75.19999694824219, 100.0, 100.79999542236328, 18.19999885559082, 17.399999618530273, 85.79999542236328, 82.5999984741211, 102.0, 104.79999542236328, 17.600000381469727, 16.600000381469727, 84.19999694824219, 83.0, 100.4000015258789, 103.0, 21.19999885559082, 22.399999618530273, 86.0, 86.79999542236328, 100.79999542236328, 97.5999984741211] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [9.199999809265137, 8.59999942779541, 15.399999618530273, 14.799999237060547, 20.0, 18.19999885559082, 17.799999237060547, 17.0, 9.59999942779541, 7.0, 18.19999885559082, 17.399999618530273, 17.600000381469727, 16.600000381469727, 21.19999885559082, 22.399999618530273] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [67.79999542236328, 66.79999542236328, 76.5999984741211, 75.19999694824219, 95.4000015258789, 95.4000015258789, 89.4000015258789, 87.19999694824219, 76.4000015258789, 75.19999694824219, 85.79999542236328, 82.5999984741211, 84.19999694824219, 83.0, 86.0, 86.79999542236328] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [103.19999694824219, 105.0, 102.5999984741211, 107.0, 100.5999984741211, 101.4000015258789, 90.19999694824219, 90.19999694824219, 100.0, 100.79999542236328, 102.0, 104.79999542236328, 100.4000015258789, 103.0, 100.79999542236328, 97.5999984741211] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 66.13+/- 5.39 (max: 107.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 100.6+/- 1.16 (max: 107.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 15.69+/- 1.165 (max: 22.4) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 82.11+/- 2.127 (max: 95.4) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 18.98+/- 0.7133 (max: 32.68) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 21.69+/- 1.192 (max: 32.68) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 13.78+/- 0.3937 (max: 15.49) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 21.47+/- 0.7079 (max: 26.1) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.7112+/- 0.05656 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.99+/- 0.005083 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.1675+/- 0.02118 (max: 0.32) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.9762+/- 0.004366 (max: 1.0) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 7.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 90.2 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 7.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 66.8 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 10.34 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 17.46 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 10.34 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 16.84 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.94 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.95 | +------------------------------------------------------------------------------------------------- +Evaluating DR_CNN-S5_SEED3 against population in Overcooked-CoordRing6_9 for xpid SEED_3_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [3.0, 0.0, 0.3999999761581421, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.5999999046325684, 0.0, 0.0, 0.0, 0.0, 0.0, 10.800000190734863, 0.0, 9.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 2.200000047683716, 0.0, 12.799999237060547, 0.0, 0.0, 0.0, 0.3999999761581421, 0.0, 3.799999952316284, 0.0, 0.0, 0.0, 1.5999999046325684, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [3.0, 0.0, 0.0, 0.0, 1.5999999046325684, 0.0, 10.800000190734863, 0.0, 0.0, 0.0, 2.200000047683716, 0.0, 0.3999999761581421, 0.0, 1.5999999046325684, 0.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [0.3999999761581421, 0.0, 0.0, 0.0, 0.0, 0.0, 9.0, 0.0, 0.0, 0.0, 12.799999237060547, 0.0, 3.799999952316284, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +-------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 0.95+/- 0.3949 (max: 12.8) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 1.225+/- 0.6824 (max: 10.8) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 1.625+/- 0.9513 (max: 12.8) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 1.459+/- 0.4567 (max: 11.5) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 2.372+/- 0.8713 (max: 10.36) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 2.006+/- 0.992 (max: 11.5) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.001458+/- 0.001073 (max: 0.05) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.00125+/- 0.0008539 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.003125+/- 0.003125 (max: 0.05) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0 | +-------------------------------------------------------------------------------------------------- +Evaluating DR_CNN-S5_SEED3 against population in Overcooked-ForcedCoord6_9 for xpid SEED_3_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +----------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +----------------------------------------------------------------------------------------- +Evaluating DR_CNN-S5_SEED3 against population in Overcooked-CounterCircuit6_9 for xpid SEED_3_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.19999998807907104, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.19999998807907104, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.004167+/- 0.004167 (max: 0.2) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 0.0125+/- 0.0125 (max: 0.2) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 0.04146+/- 0.04146 (max: 1.99) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 0.1244+/- 0.1244 (max: 1.99) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating DR_CNN-S5_SEED3 against population in Overcooked-AsymmAdvantages6_9 for xpid SEED_3_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [0.0, 1.0, 0.0, 6.399999618530273, 0.0, 0.0, 0.0, 4.599999904632568, 0.0, 44.20000076293945, 0.0, 0.19999998807907104, 0.0, 32.0, 0.0, 60.39999771118164, 0.0, 9.199999809265137, 0.0, 16.399999618530273, 0.0, 7.0, 0.0, 0.19999998807907104, 0.0, 0.5999999642372131, 0.0, 14.59999942779541, 0.0, 0.7999999523162842, 0.0, 1.7999999523162842, 0.0, 26.599998474121094, 0.0, 0.0, 0.0, 0.7999999523162842, 0.0, 15.59999942779541, 0.0, 1.1999999284744263, 0.0, 2.3999998569488525, 0.0, 18.799999237060547, 0.0, 0.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.0, 1.0, 0.0, 4.599999904632568, 0.0, 32.0, 0.0, 16.399999618530273, 0.0, 0.5999999642372131, 0.0, 1.7999999523162842, 0.0, 0.7999999523162842, 0.0, 2.3999998569488525] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.0, 6.399999618530273, 0.0, 44.20000076293945, 0.0, 60.39999771118164, 0.0, 7.0, 0.0, 14.59999942779541, 0.0, 26.599998474121094, 0.0, 15.59999942779541, 0.0, 18.799999237060547] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [0.0, 0.0, 0.0, 0.19999998807907104, 0.0, 9.199999809265137, 0.0, 0.19999998807907104, 0.0, 0.7999999523162842, 0.0, 0.0, 0.0, 1.1999999284744263, 0.0, 0.0] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 5.517+/- 1.782 (max: 60.4) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.725+/- 0.5715 (max: 9.2) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 3.725+/- 2.144 (max: 32.0) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 12.1+/- 4.503 (max: 60.4) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 5.472+/- 1.273 (max: 35.1) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 1.414+/- 0.6869 (max: 9.968) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 4.819+/- 1.914 (max: 27.71) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 10.18+/- 2.912 (max: 35.1) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.06771+/- 0.02541 (max: 0.76) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.04187+/- 0.03017 (max: 0.44) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.1612+/- 0.06501 (max: 0.76) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating DR_CNN-S5_SEED3 against population in Overcooked-CrampedRoom6_9 for xpid SEED_3_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [7.799999713897705, 0.0, 95.5999984741211, 0.0, 65.79999542236328, 0.0, 15.0, 0.0, 69.4000015258789, 0.0, 19.0, 0.0, 14.399999618530273, 0.0, 98.79999542236328, 0.0, 6.599999904632568, 0.0, 17.0, 0.0, 89.0, 0.0, 21.19999885559082, 0.0, 8.800000190734863, 0.0, 97.0, 0.0, 23.399999618530273, 0.0, 16.799999237060547, 0.0, 59.79999923706055, 0.0, 12.799999237060547, 0.0, 14.199999809265137, 0.0, 105.19999694824219, 0.0, 13.0, 0.0, 21.0, 0.0, 90.79999542236328, 0.0, 10.0, 0.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [7.799999713897705, 0.0, 15.0, 0.0, 14.399999618530273, 0.0, 17.0, 0.0, 8.800000190734863, 0.0, 16.799999237060547, 0.0, 14.199999809265137, 0.0, 21.0, 0.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [95.5999984741211, 0.0, 69.4000015258789, 0.0, 98.79999542236328, 0.0, 89.0, 0.0, 97.0, 0.0, 59.79999923706055, 0.0, 105.19999694824219, 0.0, 90.79999542236328, 0.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [65.79999542236328, 0.0, 19.0, 0.0, 6.599999904632568, 0.0, 21.19999885559082, 0.0, 23.399999618530273, 0.0, 12.799999237060547, 0.0, 13.0, 0.0, 10.0, 0.0] +-------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 20.67+/- 4.772 (max: 105.2) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 10.74+/- 4.242 (max: 65.8) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 7.187+/- 1.998 (max: 21.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 44.1+/- 11.69 (max: 105.2) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 11.87+/- 2.129 (max: 47.28) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 9.75+/- 2.695 (max: 31.6) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 6.978+/- 1.844 (max: 17.52) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 18.89+/- 5.173 (max: 47.28) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.2175+/- 0.04867 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.1394+/- 0.05959 (max: 0.94) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.07687+/- 0.02623 (max: 0.33) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.4362+/- 0.1145 (max: 1.0) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.0 | +-------------------------------------------------------------------------------------------------- +Evaluating PLR_CNN-S5_SEED1 against population in Overcooked-CoordRing6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [6.599999904632568, 5.0, 16.19999885559082, 19.0, 23.0, 22.399999618530273, 16.0, 14.0, 31.599998474121094, 34.0, 21.0, 21.600000381469727, 12.199999809265137, 13.799999237060547, 23.399999618530273, 23.0, 23.19999885559082, 23.0, 14.799999237060547, 10.399999618530273, 17.19999885559082, 16.600000381469727, 20.19999885559082, 21.799999237060547, 17.0, 16.399999618530273, 22.399999618530273, 21.0, 23.19999885559082, 24.600000381469727, 15.59999942779541, 17.0, 26.399999618530273, 30.0, 36.0, 36.0, 6.599999904632568, 6.599999904632568, 21.799999237060547, 20.799999237060547, 31.599998474121094, 31.799999237060547, 15.0, 15.399999618530273, 21.19999885559082, 19.19999885559082, 22.399999618530273, 23.600000381469727] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [6.599999904632568, 5.0, 16.0, 14.0, 12.199999809265137, 13.799999237060547, 14.799999237060547, 10.399999618530273, 17.0, 16.399999618530273, 15.59999942779541, 17.0, 6.599999904632568, 6.599999904632568, 15.0, 15.399999618530273] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [16.19999885559082, 19.0, 31.599998474121094, 34.0, 23.399999618530273, 23.0, 17.19999885559082, 16.600000381469727, 22.399999618530273, 21.0, 26.399999618530273, 30.0, 21.799999237060547, 20.799999237060547, 21.19999885559082, 19.19999885559082] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [23.0, 22.399999618530273, 21.0, 21.600000381469727, 23.19999885559082, 23.0, 20.19999885559082, 21.799999237060547, 23.19999885559082, 24.600000381469727, 36.0, 36.0, 31.599998474121094, 31.799999237060547, 22.399999618530273, 23.600000381469727] +------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CoordRing6_9 | 20.24+/- 1.062 (max: 36.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 25.34+/- 1.324 (max: 36.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 12.65+/- 1.053 (max: 17.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 22.74+/- 1.322 (max: 34.0) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 12.17+/- 0.242 (max: 16.22) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 11.37+/- 0.4545 (max: 14.45) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 12.17+/- 0.4191 (max: 14.86) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 12.95+/- 0.2916 (max: 16.22) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.234+/- 0.02608 (max: 0.72) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.3194+/- 0.04962 (max: 0.72) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.09187+/- 0.01412 (max: 0.18) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.2906+/- 0.04108 (max: 0.69) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 5.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 20.2 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 5.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 16.2 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 8.818 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 8.818 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 9.11 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 11.6 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.15 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.12 | +------------------------------------------------------------------------------------------------ +Evaluating PLR_CNN-S5_SEED1 against population in Overcooked-ForcedCoord6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [4.400000095367432, 0.0, 6.0, 0.19999998807907104, 3.799999952316284, 0.7999999523162842, 3.0, 0.0, 3.799999952316284, 0.3999999761581421, 3.5999999046325684, 0.0, 4.199999809265137, 0.19999998807907104, 5.599999904632568, 0.5999999642372131, 2.3999998569488525, 0.7999999523162842, 4.400000095367432, 0.19999998807907104, 4.199999809265137, 0.0, 2.5999999046325684, 0.0, 3.1999998092651367, 0.0, 5.199999809265137, 0.0, 5.199999809265137, 0.0, 3.1999998092651367, 0.19999998807907104, 3.3999998569488525, 0.0, 1.5999999046325684, 0.0, 2.5999999046325684, 0.0, 2.200000047683716, 0.0, 1.1999999284744263, 0.0, 5.400000095367432, 0.0, 6.199999809265137, 0.0, 6.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [4.400000095367432, 0.0, 3.0, 0.0, 4.199999809265137, 0.19999998807907104, 4.400000095367432, 0.19999998807907104, 3.1999998092651367, 0.0, 3.1999998092651367, 0.19999998807907104, 2.5999999046325684, 0.0, 5.400000095367432, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [6.0, 0.19999998807907104, 3.799999952316284, 0.3999999761581421, 5.599999904632568, 0.5999999642372131, 4.199999809265137, 0.0, 5.199999809265137, 0.0, 3.3999998569488525, 0.0, 2.200000047683716, 0.0, 6.199999809265137, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [3.799999952316284, 0.7999999523162842, 3.5999999046325684, 0.0, 2.3999998569488525, 0.7999999523162842, 2.5999999046325684, 0.0, 5.199999809265137, 0.0, 1.5999999046325684, 0.0, 1.1999999284744263, 0.0, 6.0, 0.0] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 2.017+/- 0.31 (max: 6.2) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 1.75+/- 0.4968 (max: 6.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 1.937+/- 0.5075 (max: 5.4) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 2.362+/- 0.6211 (max: 6.2) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 4.349+/- 0.5361 (max: 10.08) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 4.044+/- 0.8899 (max: 9.165) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 4.286+/- 0.9403 (max: 8.879) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 4.716+/- 1.005 (max: 10.08) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0008333+/- 0.0005012 (max: 0.02) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.000625+/- 0.000625 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.001875+/- 0.00136 (max: 0.02) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating PLR_CNN-S5_SEED1 against population in Overcooked-CounterCircuit6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [5.0, 2.799999952316284, 7.399999618530273, 7.599999904632568, 1.5999999046325684, 2.5999999046325684, 3.3999998569488525, 3.0, 5.199999809265137, 4.0, 0.5999999642372131, 0.0, 4.799999713897705, 2.3999998569488525, 13.799999237060547, 5.400000095367432, 9.0, 4.199999809265137, 5.599999904632568, 2.200000047683716, 15.799999237060547, 12.199999809265137, 4.799999713897705, 1.1999999284744263, 4.400000095367432, 3.5999999046325684, 12.199999809265137, 11.399999618530273, 4.599999904632568, 0.5999999642372131, 7.0, 2.5999999046325684, 9.199999809265137, 5.400000095367432, 5.400000095367432, 2.200000047683716, 4.0, 2.3999998569488525, 7.199999809265137, 6.199999809265137, 2.799999952316284, 0.3999999761581421, 5.199999809265137, 1.7999999523162842, 7.199999809265137, 8.800000190734863, 4.599999904632568, 0.3999999761581421] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [5.0, 2.799999952316284, 3.3999998569488525, 3.0, 4.799999713897705, 2.3999998569488525, 5.599999904632568, 2.200000047683716, 4.400000095367432, 3.5999999046325684, 7.0, 2.5999999046325684, 4.0, 2.3999998569488525, 5.199999809265137, 1.7999999523162842] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [7.399999618530273, 7.599999904632568, 5.199999809265137, 4.0, 13.799999237060547, 5.400000095367432, 15.799999237060547, 12.199999809265137, 12.199999809265137, 11.399999618530273, 9.199999809265137, 5.400000095367432, 7.199999809265137, 6.199999809265137, 7.199999809265137, 8.800000190734863] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [1.5999999046325684, 2.5999999046325684, 0.5999999642372131, 0.0, 9.0, 4.199999809265137, 4.799999713897705, 1.1999999284744263, 4.599999904632568, 0.5999999642372131, 5.400000095367432, 2.200000047683716, 2.799999952316284, 0.3999999761581421, 4.599999904632568, 0.3999999761581421] +------------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 5.087+/- 0.5241 (max: 15.8) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 2.812+/- 0.616 (max: 9.0) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 3.762+/- 0.3652 (max: 7.0) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 8.687+/- 0.8618 (max: 15.8) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 8.045+/- 0.3831 (max: 12.11) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 5.964+/- 0.7107 (max: 9.95) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 7.664+/- 0.3013 (max: 9.95) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 10.51+/- 0.2844 (max: 12.11) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.01083+/- 0.002627 (max: 0.06) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.000625+/- 0.000625 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.00125+/- 0.0008539 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.03062+/- 0.004956 (max: 0.06) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 1.8 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 4.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 5.724 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 8.485 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------- +Evaluating PLR_CNN-S5_SEED1 against population in Overcooked-AsymmAdvantages6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [0.3999999761581421, 8.199999809265137, 0.0, 21.19999885559082, 0.19999998807907104, 10.0, 0.0, 10.199999809265137, 0.0, 45.79999923706055, 0.19999998807907104, 6.799999713897705, 0.0, 44.79999923706055, 0.5999999642372131, 63.39999771118164, 0.3999999761581421, 19.399999618530273, 0.0, 25.599998474121094, 0.0, 16.600000381469727, 0.0, 12.199999809265137, 0.0, 8.59999942779541, 0.0, 30.399999618530273, 0.0, 13.199999809265137, 0.0, 9.59999942779541, 0.0, 35.79999923706055, 0.3999999761581421, 7.599999904632568, 0.0, 7.799999713897705, 0.0, 30.599998474121094, 0.19999998807907104, 14.399999618530273, 0.19999998807907104, 9.59999942779541, 0.0, 27.0, 0.19999998807907104, 6.799999713897705] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.3999999761581421, 8.199999809265137, 0.0, 10.199999809265137, 0.0, 44.79999923706055, 0.0, 25.599998474121094, 0.0, 8.59999942779541, 0.0, 9.59999942779541, 0.0, 7.799999713897705, 0.19999998807907104, 9.59999942779541] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.0, 21.19999885559082, 0.0, 45.79999923706055, 0.5999999642372131, 63.39999771118164, 0.0, 16.600000381469727, 0.0, 30.399999618530273, 0.0, 35.79999923706055, 0.0, 30.599998474121094, 0.0, 27.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [0.19999998807907104, 10.0, 0.19999998807907104, 6.799999713897705, 0.3999999761581421, 19.399999618530273, 0.0, 12.199999809265137, 0.0, 13.199999809265137, 0.3999999761581421, 7.599999904632568, 0.19999998807907104, 14.399999618530273, 0.19999998807907104, 6.799999713897705] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 10.17+/- 2.111 (max: 63.4) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 5.75+/- 1.618 (max: 19.4) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 7.812+/- 3.011 (max: 44.8) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 16.96+/- 5.046 (max: 63.4) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 8.741+/- 1.279 (max: 28.15) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 7.385+/- 1.53 (max: 15.8) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 7.799+/- 2.103 (max: 27.87) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 11.04+/- 2.843 (max: 28.15) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.13+/- 0.03234 (max: 0.89) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.04687+/- 0.01736 (max: 0.23) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.08937+/- 0.0458 (max: 0.69) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.2537+/- 0.07644 (max: 0.89) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating PLR_CNN-S5_SEED1 against population in Overcooked-CrampedRoom6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [8.199999809265137, 6.799999713897705, 70.0, 70.0, 102.79999542236328, 103.5999984741211, 13.399999618530273, 13.399999618530273, 79.0, 70.0, 105.19999694824219, 108.79999542236328, 16.799999237060547, 15.199999809265137, 97.4000015258789, 95.79999542236328, 112.39999389648438, 110.0, 16.0, 15.0, 89.5999984741211, 80.19999694824219, 82.19999694824219, 84.79999542236328, 6.399999618530273, 7.0, 66.0, 68.19999694824219, 99.4000015258789, 107.39999389648438, 15.799999237060547, 16.799999237060547, 94.0, 87.5999984741211, 109.39999389648438, 104.79999542236328, 15.59999942779541, 15.59999942779541, 84.79999542236328, 76.4000015258789, 105.5999984741211, 104.0, 14.799999237060547, 15.199999809265137, 92.79999542236328, 86.0, 111.5999984741211, 108.79999542236328] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [8.199999809265137, 6.799999713897705, 13.399999618530273, 13.399999618530273, 16.799999237060547, 15.199999809265137, 16.0, 15.0, 6.399999618530273, 7.0, 15.799999237060547, 16.799999237060547, 15.59999942779541, 15.59999942779541, 14.799999237060547, 15.199999809265137] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [70.0, 70.0, 79.0, 70.0, 97.4000015258789, 95.79999542236328, 89.5999984741211, 80.19999694824219, 66.0, 68.19999694824219, 94.0, 87.5999984741211, 84.79999542236328, 76.4000015258789, 92.79999542236328, 86.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [102.79999542236328, 103.5999984741211, 105.19999694824219, 108.79999542236328, 112.39999389648438, 110.0, 82.19999694824219, 84.79999542236328, 99.4000015258789, 107.39999389648438, 109.39999389648438, 104.79999542236328, 105.5999984741211, 104.0, 111.5999984741211, 108.79999542236328] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 66.26+/- 5.743 (max: 112.4) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 103.8+/- 2.162 (max: 112.4) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 13.25+/- 0.9498 (max: 16.8) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 81.74+/- 2.679 (max: 97.4) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 20.45+/- 0.9273 (max: 32.84) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 21.97+/- 0.8477 (max: 29.82) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 13.05+/- 0.4577 (max: 15.31) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 26.32+/- 1.025 (max: 32.84) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.6823+/- 0.05864 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.9875+/- 0.005809 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.1181+/- 0.01651 (max: 0.18) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.9412+/- 0.01103 (max: 1.0) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 6.4 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 82.2 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 6.4 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 66.0 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 9.887 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 18.59 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 9.887 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 20.3 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.01 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.93 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.01 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.85 | +------------------------------------------------------------------------------------------------- +Evaluating PLR_CNN-S5_SEED2 against population in Overcooked-CoordRing6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [6.799999713897705, 5.400000095367432, 17.0, 18.19999885559082, 22.0, 21.19999885559082, 14.59999942779541, 13.0, 32.0, 30.599998474121094, 21.399999618530273, 21.600000381469727, 11.199999809265137, 12.59999942779541, 24.399999618530273, 24.0, 22.799999237060547, 22.600000381469727, 16.600000381469727, 11.800000190734863, 14.799999237060547, 15.199999809265137, 18.600000381469727, 19.399999618530273, 14.399999618530273, 13.59999942779541, 19.19999885559082, 17.0, 22.0, 23.399999618530273, 13.59999942779541, 14.199999809265137, 22.600000381469727, 24.799999237060547, 34.39999771118164, 34.79999923706055, 3.799999952316284, 5.0, 15.799999237060547, 17.600000381469727, 24.799999237060547, 25.0, 13.0, 12.0, 10.199999809265137, 12.199999809265137, 9.800000190734863, 10.399999618530273] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [6.799999713897705, 5.400000095367432, 14.59999942779541, 13.0, 11.199999809265137, 12.59999942779541, 16.600000381469727, 11.800000190734863, 14.399999618530273, 13.59999942779541, 13.59999942779541, 14.199999809265137, 3.799999952316284, 5.0, 13.0, 12.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [17.0, 18.19999885559082, 32.0, 30.599998474121094, 24.399999618530273, 24.0, 14.799999237060547, 15.199999809265137, 19.19999885559082, 17.0, 22.600000381469727, 24.799999237060547, 15.799999237060547, 17.600000381469727, 10.199999809265137, 12.199999809265137] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [22.0, 21.19999885559082, 21.399999618530273, 21.600000381469727, 22.799999237060547, 22.600000381469727, 18.600000381469727, 19.399999618530273, 22.0, 23.399999618530273, 34.39999771118164, 34.79999923706055, 24.799999237060547, 25.0, 9.800000190734863, 10.399999618530273] +------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CoordRing6_9 | 17.74+/- 1.045 (max: 34.8) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 22.14+/- 1.633 (max: 34.8) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 11.35+/- 0.972 (max: 16.6) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 19.72+/- 1.545 (max: 32.0) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 11.87+/- 0.265 (max: 14.73) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 10.88+/- 0.4691 (max: 14.16) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 11.98+/- 0.4862 (max: 14.17) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 12.76+/- 0.2903 (max: 14.73) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.1806+/- 0.02446 (max: 0.73) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.24+/- 0.05151 (max: 0.73) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.07625+/- 0.01217 (max: 0.13) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.2256+/- 0.04152 (max: 0.58) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 3.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 9.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 3.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 10.2 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 7.618 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 7.618 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 7.846 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 11.28 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.04 | +------------------------------------------------------------------------------------------------ +Evaluating PLR_CNN-S5_SEED2 against population in Overcooked-ForcedCoord6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [3.1999998092651367, 0.19999998807907104, 2.799999952316284, 0.7999999523162842, 2.799999952316284, 2.0, 2.799999952316284, 0.7999999523162842, 1.7999999523162842, 0.7999999523162842, 1.1999999284744263, 0.7999999523162842, 1.0, 0.5999999642372131, 2.3999998569488525, 0.7999999523162842, 2.3999998569488525, 1.399999976158142, 1.5999999046325684, 1.1999999284744263, 2.200000047683716, 1.399999976158142, 0.3999999761581421, 1.399999976158142, 2.3999998569488525, 1.0, 4.199999809265137, 1.7999999523162842, 1.5999999046325684, 0.5999999642372131, 1.0, 0.19999998807907104, 0.5999999642372131, 0.19999998807907104, 1.0, 0.5999999642372131, 1.1999999284744263, 0.3999999761581421, 1.0, 0.19999998807907104, 0.3999999761581421, 0.7999999523162842, 2.5999999046325684, 0.19999998807907104, 2.200000047683716, 0.3999999761581421, 1.7999999523162842, 0.19999998807907104] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [3.1999998092651367, 0.19999998807907104, 2.799999952316284, 0.7999999523162842, 1.0, 0.5999999642372131, 1.5999999046325684, 1.1999999284744263, 2.3999998569488525, 1.0, 1.0, 0.19999998807907104, 1.1999999284744263, 0.3999999761581421, 2.5999999046325684, 0.19999998807907104] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [2.799999952316284, 0.7999999523162842, 1.7999999523162842, 0.7999999523162842, 2.3999998569488525, 0.7999999523162842, 2.200000047683716, 1.399999976158142, 4.199999809265137, 1.7999999523162842, 0.5999999642372131, 0.19999998807907104, 1.0, 0.19999998807907104, 2.200000047683716, 0.3999999761581421] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [2.799999952316284, 2.0, 1.1999999284744263, 0.7999999523162842, 2.3999998569488525, 1.399999976158142, 0.3999999761581421, 1.399999976158142, 1.5999999046325684, 0.5999999642372131, 1.0, 0.5999999642372131, 0.3999999761581421, 0.7999999523162842, 1.7999999523162842, 0.19999998807907104] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 1.321+/- 0.1358 (max: 4.2) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 1.212+/- 0.1893 (max: 2.8) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 1.275+/- 0.2442 (max: 3.2) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 1.475+/- 0.2744 (max: 4.2) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 4.608+/- 0.2413 (max: 8.623) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 4.51+/- 0.3564 (max: 6.94) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 4.475+/- 0.4438 (max: 7.332) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 4.84+/- 0.467 (max: 8.623) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0002083+/- 0.0002083 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.000625+/- 0.000625 (max: 0.01) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.2 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.2 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.2 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.2 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 1.99 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 1.99 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 1.99 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 1.99 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating PLR_CNN-S5_SEED2 against population in Overcooked-CounterCircuit6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [2.200000047683716, 1.1999999284744263, 2.5999999046325684, 2.5999999046325684, 0.5999999642372131, 1.399999976158142, 1.0, 1.399999976158142, 1.5999999046325684, 2.799999952316284, 1.0, 0.0, 4.400000095367432, 1.399999976158142, 10.59999942779541, 4.799999713897705, 6.199999809265137, 3.0, 3.1999998092651367, 1.1999999284744263, 8.800000190734863, 7.799999713897705, 4.400000095367432, 1.7999999523162842, 2.5999999046325684, 2.799999952316284, 9.800000190734863, 5.0, 5.799999713897705, 2.0, 3.5999999046325684, 1.5999999046325684, 5.199999809265137, 3.799999952316284, 3.1999998092651367, 1.1999999284744263, 0.19999998807907104, 1.399999976158142, 1.0, 0.7999999523162842, 1.399999976158142, 1.1999999284744263, 3.799999952316284, 3.1999998092651367, 2.0, 1.7999999523162842, 1.7999999523162842, 0.3999999761581421] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [2.200000047683716, 1.1999999284744263, 1.0, 1.399999976158142, 4.400000095367432, 1.399999976158142, 3.1999998092651367, 1.1999999284744263, 2.5999999046325684, 2.799999952316284, 3.5999999046325684, 1.5999999046325684, 0.19999998807907104, 1.399999976158142, 3.799999952316284, 3.1999998092651367] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [2.5999999046325684, 2.5999999046325684, 1.5999999046325684, 2.799999952316284, 10.59999942779541, 4.799999713897705, 8.800000190734863, 7.799999713897705, 9.800000190734863, 5.0, 5.199999809265137, 3.799999952316284, 1.0, 0.7999999523162842, 2.0, 1.7999999523162842] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [0.5999999642372131, 1.399999976158142, 1.0, 0.0, 6.199999809265137, 3.0, 4.400000095367432, 1.7999999523162842, 5.799999713897705, 2.0, 3.1999998092651367, 1.1999999284744263, 1.399999976158142, 1.1999999284744263, 1.7999999523162842, 0.3999999761581421] +------------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 2.95+/- 0.3522 (max: 10.6) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 2.212+/- 0.4617 (max: 6.2) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 2.2+/- 0.3 (max: 4.4) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 4.437+/- 0.8009 (max: 10.6) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 6.484+/- 0.35 (max: 12.8) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 5.596+/- 0.5967 (max: 9.25) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 5.967+/- 0.4242 (max: 8.34) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 7.889+/- 0.6428 (max: 12.8) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.004167+/- 0.001903 (max: 0.08) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.000625+/- 0.000625 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.000625+/- 0.000625 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.01125+/- 0.005313 (max: 0.08) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 0.2 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 0.8 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 1.99 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 3.919 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------- +Evaluating PLR_CNN-S5_SEED2 against population in Overcooked-AsymmAdvantages6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [1.1999999284744263, 7.0, 5.799999713897705, 16.399999618530273, 1.7999999523162842, 7.799999713897705, 2.0, 10.59999942779541, 4.0, 47.79999923706055, 1.5999999046325684, 10.199999809265137, 3.3999998569488525, 40.599998474121094, 5.400000095367432, 66.79999542236328, 2.5999999046325684, 17.19999885559082, 4.0, 27.399999618530273, 4.199999809265137, 16.19999885559082, 1.5999999046325684, 13.59999942779541, 2.200000047683716, 6.599999904632568, 4.799999713897705, 28.19999885559082, 4.799999713897705, 15.199999809265137, 2.200000047683716, 8.199999809265137, 5.199999809265137, 33.20000076293945, 2.5999999046325684, 11.0, 1.0, 7.199999809265137, 5.400000095367432, 29.0, 1.0, 16.0, 1.399999976158142, 11.0, 5.599999904632568, 31.599998474121094, 2.0, 7.599999904632568] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [1.1999999284744263, 7.0, 2.0, 10.59999942779541, 3.3999998569488525, 40.599998474121094, 4.0, 27.399999618530273, 2.200000047683716, 6.599999904632568, 2.200000047683716, 8.199999809265137, 1.0, 7.199999809265137, 1.399999976158142, 11.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [5.799999713897705, 16.399999618530273, 4.0, 47.79999923706055, 5.400000095367432, 66.79999542236328, 4.199999809265137, 16.19999885559082, 4.799999713897705, 28.19999885559082, 5.199999809265137, 33.20000076293945, 5.400000095367432, 29.0, 5.599999904632568, 31.599998474121094] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [1.7999999523162842, 7.799999713897705, 1.5999999046325684, 10.199999809265137, 2.5999999046325684, 17.19999885559082, 1.5999999046325684, 13.59999942779541, 4.799999713897705, 15.199999809265137, 2.5999999046325684, 11.0, 1.0, 16.0, 2.0, 7.599999904632568] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 11.71+/- 1.985 (max: 66.8) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 7.287+/- 1.46 (max: 17.2) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 8.5+/- 2.688 (max: 40.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 19.35+/- 4.669 (max: 66.8) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 12.37+/- 0.9426 (max: 29.58) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 10.65+/- 1.203 (max: 17.67) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 10.46+/- 1.462 (max: 26.15) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 15.99+/- 1.858 (max: 29.58) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.1371+/- 0.03183 (max: 0.91) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.065+/- 0.01895 (max: 0.2) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.08812+/- 0.04367 (max: 0.61) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.2581+/- 0.07598 (max: 0.91) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 1.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 1.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 1.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 4.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 4.359 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 4.359 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 4.359 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 8.998 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.01 | +------------------------------------------------------------------------------------------------------ +Evaluating PLR_CNN-S5_SEED2 against population in Overcooked-CrampedRoom6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [7.599999904632568, 9.800000190734863, 65.4000015258789, 61.39999771118164, 92.79999542236328, 89.5999984741211, 14.799999237060547, 14.799999237060547, 67.5999984741211, 64.5999984741211, 85.4000015258789, 85.79999542236328, 16.799999237060547, 16.19999885559082, 79.0, 90.0, 83.0, 82.5999984741211, 14.799999237060547, 17.399999618530273, 79.4000015258789, 79.0, 73.19999694824219, 66.4000015258789, 8.399999618530273, 9.0, 73.19999694824219, 64.79999542236328, 80.79999542236328, 78.0, 16.799999237060547, 14.59999942779541, 77.0, 74.5999984741211, 89.0, 82.0, 14.199999809265137, 13.59999942779541, 73.79999542236328, 70.79999542236328, 82.79999542236328, 80.4000015258789, 17.600000381469727, 16.19999885559082, 83.5999984741211, 79.19999694824219, 82.19999694824219, 86.79999542236328] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [7.599999904632568, 9.800000190734863, 14.799999237060547, 14.799999237060547, 16.799999237060547, 16.19999885559082, 14.799999237060547, 17.399999618530273, 8.399999618530273, 9.0, 16.799999237060547, 14.59999942779541, 14.199999809265137, 13.59999942779541, 17.600000381469727, 16.19999885559082] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [65.4000015258789, 61.39999771118164, 67.5999984741211, 64.5999984741211, 79.0, 90.0, 79.4000015258789, 79.0, 73.19999694824219, 64.79999542236328, 77.0, 74.5999984741211, 73.79999542236328, 70.79999542236328, 83.5999984741211, 79.19999694824219] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [92.79999542236328, 89.5999984741211, 85.4000015258789, 85.79999542236328, 83.0, 82.5999984741211, 73.19999694824219, 66.4000015258789, 80.79999542236328, 78.0, 89.0, 82.0, 82.79999542236328, 80.4000015258789, 82.19999694824219, 86.79999542236328] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 56.81+/- 4.538 (max: 92.8) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 82.55+/- 1.595 (max: 92.8) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 13.91+/- 0.8347 (max: 17.6) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 73.96+/- 1.959 (max: 90.0) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 21.84+/- 0.9106 (max: 32.97) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 25.4+/- 0.6193 (max: 29.1) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 13.69+/- 0.433 (max: 16.42) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 26.42+/- 0.745 (max: 32.97) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.6762+/- 0.05639 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.965+/- 0.008165 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.1337+/- 0.01741 (max: 0.23) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.93+/- 0.006831 (max: 0.99) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 7.6 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 66.4 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 7.6 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 61.4 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 10.65 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 19.9 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 10.65 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 20.95 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.87 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.88 | +------------------------------------------------------------------------------------------------- +Evaluating PLR_CNN-S5_SEED3 against population in Overcooked-CoordRing6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [4.400000095367432, 3.5999999046325684, 13.59999942779541, 12.59999942779541, 16.19999885559082, 11.800000190734863, 16.19999885559082, 14.0, 25.19999885559082, 24.799999237060547, 12.59999942779541, 10.199999809265137, 12.0, 11.800000190734863, 15.199999809265137, 14.0, 14.59999942779541, 11.0, 9.399999618530273, 7.599999904632568, 17.0, 16.19999885559082, 11.800000190734863, 10.0, 16.799999237060547, 13.59999942779541, 17.0, 17.600000381469727, 15.59999942779541, 13.199999809265137, 9.800000190734863, 10.199999809265137, 19.19999885559082, 18.600000381469727, 24.0, 25.19999885559082, 2.5999999046325684, 2.5999999046325684, 16.600000381469727, 14.799999237060547, 20.0, 21.799999237060547, 6.0, 8.0, 12.199999809265137, 7.199999809265137, 15.399999618530273, 16.399999618530273] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [4.400000095367432, 3.5999999046325684, 16.19999885559082, 14.0, 12.0, 11.800000190734863, 9.399999618530273, 7.599999904632568, 16.799999237060547, 13.59999942779541, 9.800000190734863, 10.199999809265137, 2.5999999046325684, 2.5999999046325684, 6.0, 8.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [13.59999942779541, 12.59999942779541, 25.19999885559082, 24.799999237060547, 15.199999809265137, 14.0, 17.0, 16.19999885559082, 17.0, 17.600000381469727, 19.19999885559082, 18.600000381469727, 16.600000381469727, 14.799999237060547, 12.199999809265137, 7.199999809265137] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [16.19999885559082, 11.800000190734863, 12.59999942779541, 10.199999809265137, 14.59999942779541, 11.0, 11.800000190734863, 10.0, 15.59999942779541, 13.199999809265137, 24.0, 25.19999885559082, 20.0, 21.799999237060547, 15.399999618530273, 16.399999618530273] +------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CoordRing6_9 | 13.75+/- 0.8015 (max: 25.2) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 15.61+/- 1.204 (max: 25.2) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 9.287+/- 1.156 (max: 16.8) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 16.36+/- 1.113 (max: 25.2) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 11.34+/- 0.2731 (max: 15.56) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 11.22+/- 0.3897 (max: 15.13) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 10.89+/- 0.5991 (max: 14.34) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 11.9+/- 0.3934 (max: 15.56) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.09083+/- 0.01369 (max: 0.35) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.09562+/- 0.028 (max: 0.35) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.05437+/- 0.01369 (max: 0.17) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.1225+/- 0.02514 (max: 0.32) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 2.6 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 10.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 2.6 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 7.2 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 6.726 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 9.113 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 6.726 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 9.6 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------ +Evaluating PLR_CNN-S5_SEED3 against population in Overcooked-ForcedCoord6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.0, 0.0, 0.0, 0.0, 0.3999999761581421, 0.0, 0.19999998807907104, 0.0, 0.3999999761581421, 0.0, 0.19999998807907104, 0.0, 0.3999999761581421, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.0, 0.0, 0.0, 0.0, 0.19999998807907104, 0.0, 0.0, 0.0, 0.5999999642372131, 0.0, 0.19999998807907104, 0.0, 1.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [0.19999998807907104, 0.0, 0.0, 0.0, 0.19999998807907104, 0.0, 0.0, 0.0, 0.3999999761581421, 0.0, 0.19999998807907104, 0.0, 0.0, 0.0, 0.5999999642372131, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [0.19999998807907104, 0.0, 0.0, 0.0, 0.19999998807907104, 0.0, 0.3999999761581421, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.19999998807907104, 0.0, 0.3999999761581421, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.1+/- 0.02793 (max: 1.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.1+/- 0.06583 (max: 1.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.1+/- 0.04472 (max: 0.6) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.1+/- 0.03162 (max: 0.4) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.7515+/- 0.1719 (max: 4.359) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.5718+/- 0.3264 (max: 4.359) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.7614+/- 0.3034 (max: 3.412) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.9212+/- 0.274 (max: 2.8) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------- +Evaluating PLR_CNN-S5_SEED3 against population in Overcooked-CounterCircuit6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [2.200000047683716, 0.5999999642372131, 3.5999999046325684, 3.0, 3.0, 2.5999999046325684, 0.7999999523162842, 2.0, 2.5999999046325684, 2.0, 0.3999999761581421, 0.0, 3.1999998092651367, 1.7999999523162842, 10.399999618530273, 4.400000095367432, 8.0, 2.200000047683716, 3.3999998569488525, 2.3999998569488525, 11.800000190734863, 8.0, 3.1999998092651367, 0.19999998807907104, 4.400000095367432, 1.1999999284744263, 13.799999237060547, 9.199999809265137, 7.399999618530273, 1.399999976158142, 2.5999999046325684, 1.1999999284744263, 6.0, 4.799999713897705, 4.599999904632568, 0.7999999523162842, 0.7999999523162842, 0.5999999642372131, 4.0, 4.599999904632568, 1.0, 0.0, 1.5999999046325684, 0.19999998807907104, 2.5999999046325684, 2.0, 2.5999999046325684, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [2.200000047683716, 0.5999999642372131, 0.7999999523162842, 2.0, 3.1999998092651367, 1.7999999523162842, 3.3999998569488525, 2.3999998569488525, 4.400000095367432, 1.1999999284744263, 2.5999999046325684, 1.1999999284744263, 0.7999999523162842, 0.5999999642372131, 1.5999999046325684, 0.19999998807907104] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [3.5999999046325684, 3.0, 2.5999999046325684, 2.0, 10.399999618530273, 4.400000095367432, 11.800000190734863, 8.0, 13.799999237060547, 9.199999809265137, 6.0, 4.799999713897705, 4.0, 4.599999904632568, 2.5999999046325684, 2.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [3.0, 2.5999999046325684, 0.3999999761581421, 0.0, 8.0, 2.200000047683716, 3.1999998092651367, 0.19999998807907104, 7.399999618530273, 1.399999976158142, 4.599999904632568, 0.7999999523162842, 1.0, 0.0, 2.5999999046325684, 0.0] +------------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 3.317+/- 0.4574 (max: 13.8) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 2.337+/- 0.6255 (max: 8.0) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 1.812+/- 0.2935 (max: 4.4) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 5.8+/- 0.9293 (max: 13.8) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 6.457+/- 0.437 (max: 12.68) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 5.064+/- 0.8473 (max: 10.2) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 5.37+/- 0.4351 (max: 8.285) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 8.936+/- 0.5195 (max: 12.68) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.006458+/- 0.00257 (max: 0.08) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.00125+/- 0.0008539 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.01812+/- 0.006905 (max: 0.08) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 0.2 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 2.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 1.99 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 6.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------- +Evaluating PLR_CNN-S5_SEED3 against population in Overcooked-AsymmAdvantages6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [0.0, 6.599999904632568, 0.0, 15.59999942779541, 0.0, 5.400000095367432, 0.0, 7.599999904632568, 0.0, 44.599998474121094, 0.0, 2.5999999046325684, 0.0, 40.79999923706055, 0.0, 62.19999694824219, 0.0, 17.799999237060547, 0.0, 20.600000381469727, 0.0, 11.59999942779541, 0.0, 6.599999904632568, 0.0, 5.400000095367432, 0.0, 22.600000381469727, 0.0, 11.0, 0.0, 5.599999904632568, 0.0, 31.599998474121094, 0.0, 3.1999998092651367, 0.0, 6.599999904632568, 0.0, 18.600000381469727, 0.0, 10.199999809265137, 0.19999998807907104, 6.799999713897705, 0.0, 27.19999885559082, 0.0, 2.200000047683716] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.0, 6.599999904632568, 0.0, 7.599999904632568, 0.0, 40.79999923706055, 0.0, 20.600000381469727, 0.0, 5.400000095367432, 0.0, 5.599999904632568, 0.0, 6.599999904632568, 0.19999998807907104, 6.799999713897705] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.0, 15.59999942779541, 0.0, 44.599998474121094, 0.0, 62.19999694824219, 0.0, 11.59999942779541, 0.0, 22.600000381469727, 0.0, 31.599998474121094, 0.0, 18.600000381469727, 0.0, 27.19999885559082] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [0.0, 5.400000095367432, 0.0, 2.5999999046325684, 0.0, 17.799999237060547, 0.0, 6.599999904632568, 0.0, 11.0, 0.0, 3.1999998092651367, 0.0, 10.199999809265137, 0.0, 2.200000047683716] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 8.192+/- 1.951 (max: 62.2) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 3.687+/- 1.321 (max: 17.8) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 6.262+/- 2.672 (max: 40.8) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 14.62+/- 4.747 (max: 62.2) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 7.613+/- 1.283 (max: 30.45) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 5.605+/- 1.624 (max: 18.52) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 6.955+/- 2.099 (max: 29.11) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 10.28+/- 2.768 (max: 30.45) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.1017+/- 0.02945 (max: 0.84) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.03062+/- 0.01487 (max: 0.21) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.065+/- 0.04174 (max: 0.65) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.2094+/- 0.07065 (max: 0.84) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating PLR_CNN-S5_SEED3 against population in Overcooked-CrampedRoom6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [7.799999713897705, 7.199999809265137, 61.79999923706055, 61.0, 69.0, 75.0, 16.19999885559082, 16.799999237060547, 70.5999984741211, 71.19999694824219, 79.5999984741211, 71.0, 17.600000381469727, 16.799999237060547, 83.4000015258789, 92.0, 71.4000015258789, 67.5999984741211, 16.799999237060547, 19.399999618530273, 77.5999984741211, 77.4000015258789, 60.599998474121094, 52.19999694824219, 7.0, 7.199999809265137, 66.79999542236328, 70.0, 69.0, 60.79999923706055, 16.600000381469727, 17.19999885559082, 75.19999694824219, 78.79999542236328, 67.79999542236328, 64.19999694824219, 16.19999885559082, 16.399999618530273, 74.0, 70.79999542236328, 72.79999542236328, 65.5999984741211, 16.0, 17.600000381469727, 79.0, 80.79999542236328, 73.5999984741211, 65.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [7.799999713897705, 7.199999809265137, 16.19999885559082, 16.799999237060547, 17.600000381469727, 16.799999237060547, 16.799999237060547, 19.399999618530273, 7.0, 7.199999809265137, 16.600000381469727, 17.19999885559082, 16.19999885559082, 16.399999618530273, 16.0, 17.600000381469727] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [61.79999923706055, 61.0, 70.5999984741211, 71.19999694824219, 83.4000015258789, 92.0, 77.5999984741211, 77.4000015258789, 66.79999542236328, 70.0, 75.19999694824219, 78.79999542236328, 74.0, 70.79999542236328, 79.0, 80.79999542236328] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [69.0, 75.0, 79.5999984741211, 71.0, 71.4000015258789, 67.5999984741211, 60.599998474121094, 52.19999694824219, 69.0, 60.79999923706055, 67.79999542236328, 64.19999694824219, 72.79999542236328, 65.5999984741211, 73.5999984741211, 65.0] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 52.26+/- 4.014 (max: 92.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 67.82+/- 1.64 (max: 79.6) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 14.55+/- 1.099 (max: 19.4) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 74.4+/- 1.985 (max: 92.0) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 20.95+/- 0.9244 (max: 30.37) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 27.35+/- 0.4303 (max: 30.37) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 12.93+/- 0.4385 (max: 15.68) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 22.56+/- 0.6814 (max: 27.47) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.6635+/- 0.05573 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.9056+/- 0.01599 (max: 0.97) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.13+/- 0.01839 (max: 0.23) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.955+/- 0.007246 (max: 1.0) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 7.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 52.2 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 7.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 61.0 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 10.34 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 24.72 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 10.34 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 18.78 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.74 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.9 | +------------------------------------------------------------------------------------------------- +Evaluating PAIRED_CNN-S5_SEED1 against population in Overcooked-CoordRing6_9 for xpid paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [2.3999998569488525, 0.0, 0.5999999642372131, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.7999999523162842, 0.0, 0.0, 0.0, 0.0, 0.0, 7.799999713897705, 0.0, 4.400000095367432, 0.0, 0.0, 0.0, 0.19999998807907104, 0.0, 0.0, 0.0, 0.0, 0.0, 1.5999999046325684, 0.0, 10.59999942779541, 0.0, 0.0, 0.0, 1.1999999284744263, 0.0, 4.0, 0.0, 0.0, 0.0, 0.7999999523162842, 0.0, 0.19999998807907104, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [2.3999998569488525, 0.0, 0.0, 0.0, 0.7999999523162842, 0.0, 7.799999713897705, 0.0, 0.19999998807907104, 0.0, 1.5999999046325684, 0.0, 1.1999999284744263, 0.0, 0.7999999523162842, 0.0] +k eval/a1:test_return:Overcooked-CoordRing6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CoordRing6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [0.5999999642372131, 0.0, 0.0, 0.0, 0.0, 0.0, 4.400000095367432, 0.0, 0.0, 0.0, 10.59999942779541, 0.0, 4.0, 0.0, 0.19999998807907104, 0.0] +k eval/a1:test_return:Overcooked-CoordRing6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CoordRing6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +--------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 0.7208+/- 0.2941 (max: 10.6) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 0.925+/- 0.4926 (max: 7.8) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 1.237+/- 0.7177 (max: 10.6) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 1.474+/- 0.4333 (max: 11.47) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 2.291+/- 0.7855 (max: 10.16) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 2.132+/- 0.9643 (max: 11.47) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.001458+/- 0.0009411 (max: 0.04) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.000625+/- 0.000625 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.00375+/- 0.00272 (max: 0.04) | +| eval/a1:test_return:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0 | +--------------------------------------------------------------------------------------------------- +Evaluating PAIRED_CNN-S5_SEED1 against population in Overcooked-ForcedCoord6_9 for xpid paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +----------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +----------------------------------------------------------------------------------------- +Evaluating PAIRED_CNN-S5_SEED1 against population in Overcooked-CounterCircuit6_9 for xpid paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.19999998807907104, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CounterCircuit6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.19999998807907104, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.004167+/- 0.004167 (max: 0.2) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 0.0125+/- 0.0125 (max: 0.2) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 0.04146+/- 0.04146 (max: 1.99) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 0.1244+/- 0.1244 (max: 1.99) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating PAIRED_CNN-S5_SEED1 against population in Overcooked-AsymmAdvantages6_9 for xpid paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [0.0, 1.0, 0.0, 8.399999618530273, 0.0, 0.0, 0.0, 4.799999713897705, 0.0, 43.0, 0.0, 0.7999999523162842, 0.0, 29.0, 0.0, 58.0, 0.0, 7.799999713897705, 0.0, 15.399999618530273, 0.0, 8.59999942779541, 0.0, 0.3999999761581421, 0.0, 0.7999999523162842, 0.0, 17.399999618530273, 0.0, 0.5999999642372131, 0.0, 1.5999999046325684, 0.0, 26.599998474121094, 0.0, 0.0, 0.0, 2.200000047683716, 0.0, 13.799999237060547, 0.0, 2.0, 0.0, 2.5999999046325684, 0.0, 16.799999237060547, 0.0, 0.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.0, 1.0, 0.0, 4.799999713897705, 0.0, 29.0, 0.0, 15.399999618530273, 0.0, 0.7999999523162842, 0.0, 1.5999999046325684, 0.0, 2.200000047683716, 0.0, 2.5999999046325684] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.0, 8.399999618530273, 0.0, 43.0, 0.0, 58.0, 0.0, 8.59999942779541, 0.0, 17.399999618530273, 0.0, 26.599998474121094, 0.0, 13.799999237060547, 0.0, 16.799999237060547] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [0.0, 0.0, 0.0, 0.7999999523162842, 0.0, 7.799999713897705, 0.0, 0.3999999761581421, 0.0, 0.5999999642372131, 0.0, 0.0, 0.0, 2.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +----------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 5.45+/- 1.71 (max: 58.0) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.725+/- 0.4899 (max: 7.8) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 3.587+/- 1.948 (max: 29.0) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 12.04+/- 4.341 (max: 58.0) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 5.772+/- 1.305 (max: 34.35) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 1.618+/- 0.7191 (max: 9.755) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 5.192+/- 2.033 (max: 29.98) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 10.5+/- 2.948 (max: 34.35) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.06833+/- 0.0244 (max: 0.77) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.04+/- 0.0277 (max: 0.39) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.165+/- 0.06196 (max: 0.77) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +----------------------------------------------------------------------------------------------------- +Evaluating PAIRED_CNN-S5_SEED1 against population in Overcooked-CrampedRoom6_9 for xpid paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [9.0, 0.0, 97.79999542236328, 0.0, 65.0, 0.0, 13.59999942779541, 0.0, 75.19999694824219, 0.0, 17.19999885559082, 0.0, 14.59999942779541, 0.0, 94.19999694824219, 0.0, 6.399999618530273, 0.0, 14.59999942779541, 0.0, 79.4000015258789, 0.0, 14.399999618530273, 0.0, 6.399999618530273, 0.0, 94.0, 0.0, 21.19999885559082, 0.0, 13.59999942779541, 0.0, 58.599998474121094, 0.0, 11.399999618530273, 0.0, 14.0, 0.0, 107.79999542236328, 0.0, 13.0, 0.0, 24.600000381469727, 0.0, 102.39999389648438, 0.0, 9.0, 0.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [9.0, 0.0, 13.59999942779541, 0.0, 14.59999942779541, 0.0, 14.59999942779541, 0.0, 6.399999618530273, 0.0, 13.59999942779541, 0.0, 14.0, 0.0, 24.600000381469727, 0.0] +k eval/a1:test_return:Overcooked-CrampedRoom6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [97.79999542236328, 0.0, 75.19999694824219, 0.0, 94.19999694824219, 0.0, 79.4000015258789, 0.0, 94.0, 0.0, 58.599998474121094, 0.0, 107.79999542236328, 0.0, 102.39999389648438, 0.0] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [65.0, 0.0, 17.19999885559082, 0.0, 6.399999618530273, 0.0, 14.399999618530273, 0.0, 21.19999885559082, 0.0, 11.399999618530273, 0.0, 13.0, 0.0, 9.0, 0.0] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 20.36+/- 4.81 (max: 107.8) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 9.85+/- 4.106 (max: 65.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 6.9+/- 1.997 (max: 24.6) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 44.34+/- 11.78 (max: 107.8) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 11.5+/- 2.081 (max: 49.84) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 9.037+/- 2.414 (max: 23.22) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 6.978+/- 1.859 (max: 17.17) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 18.49+/- 5.137 (max: 49.84) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.21+/- 0.04861 (max: 0.98) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.1262+/- 0.05903 (max: 0.95) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.07+/- 0.02566 (max: 0.35) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.4337+/- 0.1138 (max: 0.98) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------- +Evaluating PAIRED_CNN-S5_SEED2 against population in Overcooked-CoordRing6_9 for xpid SEED_2_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [0.3999999761581421, 3.1999998092651367, 0.0, 0.19999998807907104, 0.0, 0.0, 1.5999999046325684, 8.800000190734863, 1.399999976158142, 2.799999952316284, 0.0, 0.0, 1.5999999046325684, 5.199999809265137, 0.0, 0.0, 0.0, 0.0, 0.5999999642372131, 1.0, 0.3999999761581421, 1.0, 0.0, 0.0, 1.0, 3.3999998569488525, 0.0, 0.0, 0.0, 0.0, 0.7999999523162842, 2.3999998569488525, 3.799999952316284, 4.799999713897705, 0.19999998807907104, 0.0, 0.3999999761581421, 0.0, 1.399999976158142, 3.0, 0.0, 0.0, 0.19999998807907104, 0.19999998807907104, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [0.3999999761581421, 3.1999998092651367, 1.5999999046325684, 8.800000190734863, 1.5999999046325684, 5.199999809265137, 0.5999999642372131, 1.0, 1.0, 3.3999998569488525, 0.7999999523162842, 2.3999998569488525, 0.3999999761581421, 0.0, 0.19999998807907104, 0.19999998807907104] +k eval/a1:test_return:Overcooked-CoordRing6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CoordRing6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [0.0, 0.19999998807907104, 1.399999976158142, 2.799999952316284, 0.0, 0.0, 0.3999999761581421, 1.0, 0.0, 0.0, 3.799999952316284, 4.799999713897705, 1.399999976158142, 3.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CoordRing6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.19999998807907104, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CoordRing6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 1.037+/- 0.2565 (max: 8.8) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 0.0125+/- 0.0125 (max: 0.2) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 1.925+/- 0.5819 (max: 8.8) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 1.175+/- 0.3945 (max: 4.8) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 2.755+/- 0.4724 (max: 11.77) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 0.1244+/- 0.1244 (max: 1.99) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 4.941+/- 0.7438 (max: 11.77) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 3.201+/- 0.858 (max: 9.432) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.001875+/- 0.00114 (max: 0.05) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.00375+/- 0.003146 (max: 0.05) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.001875+/- 0.00136 (max: 0.02) | +| eval/a1:test_return:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------- +Evaluating PAIRED_CNN-S5_SEED2 against population in Overcooked-ForcedCoord6_9 for xpid SEED_2_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +----------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +----------------------------------------------------------------------------------------- +Evaluating PAIRED_CNN-S5_SEED2 against population in Overcooked-CounterCircuit6_9 for xpid SEED_2_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [0.0, 0.0, 1.7999999523162842, 0.0, 0.3999999761581421, 0.0, 0.0, 0.19999998807907104, 1.7999999523162842, 0.0, 0.0, 0.0, 0.19999998807907104, 0.0, 0.0, 0.0, 0.3999999761581421, 0.19999998807907104, 1.5999999046325684, 0.19999998807907104, 14.399999618530273, 7.199999809265137, 0.5999999642372131, 0.0, 1.5999999046325684, 0.5999999642372131, 4.599999904632568, 1.7999999523162842, 2.799999952316284, 0.7999999523162842, 0.19999998807907104, 0.0, 3.3999998569488525, 0.5999999642372131, 0.5999999642372131, 0.3999999761581421, 0.19999998807907104, 0.0, 2.200000047683716, 0.0, 0.0, 0.0, 1.0, 0.3999999761581421, 6.599999904632568, 0.7999999523162842, 0.3999999761581421, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [0.0, 0.0, 0.0, 0.19999998807907104, 0.19999998807907104, 0.0, 1.5999999046325684, 0.19999998807907104, 1.5999999046325684, 0.5999999642372131, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 1.0, 0.3999999761581421] +k eval/a1:test_return:Overcooked-CounterCircuit6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [1.7999999523162842, 0.0, 1.7999999523162842, 0.0, 0.0, 0.0, 14.399999618530273, 7.199999809265137, 4.599999904632568, 1.7999999523162842, 3.3999998569488525, 0.5999999642372131, 2.200000047683716, 0.0, 6.599999904632568, 0.7999999523162842] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [0.3999999761581421, 0.0, 0.0, 0.0, 0.3999999761581421, 0.19999998807907104, 0.5999999642372131, 0.0, 2.799999952316284, 0.7999999523162842, 0.5999999642372131, 0.3999999761581421, 0.0, 0.0, 0.3999999761581421, 0.0] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +--------------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 1.208+/- 0.3632 (max: 14.4) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.4125+/- 0.1727 (max: 2.8) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 0.3875+/- 0.136 (max: 1.6) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 2.825+/- 0.9647 (max: 14.4) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 2.87+/- 0.4278 (max: 9.82) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 1.93+/- 0.5105 (max: 6.94) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 1.961+/- 0.4815 (max: 5.426) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 4.72+/- 0.9409 (max: 9.82) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0004167+/- 0.0002915 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.00125+/- 0.0008539 (max: 0.01) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +--------------------------------------------------------------------------------------------------------- +Evaluating PAIRED_CNN-S5_SEED2 against population in Overcooked-AsymmAdvantages6_9 for xpid SEED_2_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [0.0, 8.399999618530273, 0.0, 21.799999237060547, 0.0, 1.7999999523162842, 0.0, 8.800000190734863, 0.0, 28.0, 0.0, 1.399999976158142, 0.0, 36.39999771118164, 0.0, 68.19999694824219, 0.0, 13.799999237060547, 0.0, 22.600000381469727, 0.0, 7.799999713897705, 0.0, 7.0, 0.0, 10.59999942779541, 0.0, 24.0, 0.0, 20.0, 0.0, 9.800000190734863, 0.0, 26.599998474121094, 0.0, 1.0, 0.0, 9.800000190734863, 0.0, 12.59999942779541, 0.0, 11.0, 0.0, 15.59999942779541, 0.0, 27.399999618530273, 0.0, 1.5999999046325684] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.0, 8.399999618530273, 0.0, 8.800000190734863, 0.0, 36.39999771118164, 0.0, 22.600000381469727, 0.0, 10.59999942779541, 0.0, 9.800000190734863, 0.0, 9.800000190734863, 0.0, 15.59999942779541] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.0, 21.799999237060547, 0.0, 28.0, 0.0, 68.19999694824219, 0.0, 7.799999713897705, 0.0, 24.0, 0.0, 26.599998474121094, 0.0, 12.59999942779541, 0.0, 27.399999618530273] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [0.0, 1.7999999523162842, 0.0, 1.399999976158142, 0.0, 13.799999237060547, 0.0, 7.0, 0.0, 20.0, 0.0, 1.0, 0.0, 11.0, 0.0, 1.5999999046325684] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +-------------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 8.25+/- 1.906 (max: 68.2) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 3.6+/- 1.53 (max: 20.0) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 7.625+/- 2.583 (max: 36.4) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 13.52+/- 4.672 (max: 68.2) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 6.361+/- 1.15 (max: 34.65) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 3.48+/- 0.977 (max: 9.95) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 6.666+/- 1.93 (max: 24.88) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 8.937+/- 2.599 (max: 34.65) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.07937+/- 0.02494 (max: 0.83) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.003125+/- 0.003125 (max: 0.05) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.06187+/- 0.03528 (max: 0.51) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.1731+/- 0.05997 (max: 0.83) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +-------------------------------------------------------------------------------------------------------- +Evaluating PAIRED_CNN-S5_SEED2 against population in Overcooked-CrampedRoom6_9 for xpid SEED_2_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [0.7999999523162842, 0.7999999523162842, 4.799999713897705, 3.0, 2.0, 0.7999999523162842, 3.3999998569488525, 3.0, 4.799999713897705, 2.5999999046325684, 1.5999999046325684, 0.0, 3.1999998092651367, 3.1999998092651367, 3.1999998092651367, 3.1999998092651367, 1.399999976158142, 0.3999999761581421, 2.200000047683716, 2.0, 3.1999998092651367, 2.200000047683716, 1.0, 0.19999998807907104, 1.399999976158142, 1.399999976158142, 7.199999809265137, 4.400000095367432, 0.7999999523162842, 0.3999999761581421, 2.200000047683716, 3.1999998092651367, 1.5999999046325684, 1.399999976158142, 1.399999976158142, 0.3999999761581421, 3.1999998092651367, 3.1999998092651367, 4.799999713897705, 4.599999904632568, 1.1999999284744263, 0.3999999761581421, 4.0, 2.799999952316284, 6.0, 3.799999952316284, 1.1999999284744263, 0.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [0.7999999523162842, 0.7999999523162842, 3.3999998569488525, 3.0, 3.1999998092651367, 3.1999998092651367, 2.200000047683716, 2.0, 1.399999976158142, 1.399999976158142, 2.200000047683716, 3.1999998092651367, 3.1999998092651367, 3.1999998092651367, 4.0, 2.799999952316284] +k eval/a1:test_return:Overcooked-CrampedRoom6_9, v [0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:low, v [0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [4.799999713897705, 3.0, 4.799999713897705, 2.5999999046325684, 3.1999998092651367, 3.1999998092651367, 3.1999998092651367, 2.200000047683716, 7.199999809265137, 4.400000095367432, 1.5999999046325684, 1.399999976158142, 4.799999713897705, 4.599999904632568, 6.0, 3.799999952316284] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:mid, v [0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [2.0, 0.7999999523162842, 1.5999999046325684, 0.0, 1.399999976158142, 0.3999999761581421, 1.0, 0.19999998807907104, 0.7999999523162842, 0.3999999761581421, 1.399999976158142, 0.3999999761581421, 1.1999999284744263, 0.3999999761581421, 1.1999999284744263, 0.0] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:high, v [0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104] +---------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 2.375+/- 0.238 (max: 7.2) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 0.825+/- 0.1504 (max: 2.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 2.5+/- 0.2456 (max: 4.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 3.8+/- 0.3912 (max: 7.2) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 6.011+/- 0.3493 (max: 10.4) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 3.532+/- 0.4474 (max: 6.0) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 6.68+/- 0.3846 (max: 9.798) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 7.82+/- 0.3758 (max: 10.4) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.003333+/- 0.001127 (max: 0.04) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.005+/- 0.002582 (max: 0.04) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.005+/- 0.002041 (max: 0.02) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9 | 0.1+/- 0.01459 (max: 0.2) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:high | 0.1+/- 0.02582 (max: 0.2) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:low | 0.1+/- 0.02582 (max: 0.2) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:mid | 0.1+/- 0.02582 (max: 0.2) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9 | 0.995+/- 0.1451 (max: 1.99) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:high | 0.995+/- 0.2569 (max: 1.99) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:low | 0.995+/- 0.2569 (max: 1.99) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:mid | 0.995+/- 0.2569 (max: 1.99) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 0.8 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 1.4 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 3.919 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 5.103 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.0 | +---------------------------------------------------------------------------------------------------- +Evaluating PAIRED_CNN-S5_SEED3 against population in Overcooked-CoordRing6_9 for xpid SEED_3_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [4.799999713897705, 5.199999809265137, 17.799999237060547, 17.19999885559082, 23.399999618530273, 26.0, 13.59999942779541, 15.799999237060547, 27.19999885559082, 25.399999618530273, 22.19999885559082, 23.19999885559082, 13.0, 11.59999942779541, 25.599998474121094, 23.0, 22.399999618530273, 23.0, 13.59999942779541, 12.399999618530273, 16.19999885559082, 15.0, 17.19999885559082, 18.399999618530273, 14.59999942779541, 17.19999885559082, 20.399999618530273, 21.0, 21.19999885559082, 23.0, 13.59999942779541, 14.799999237060547, 24.399999618530273, 22.799999237060547, 33.79999923706055, 31.399999618530273, 3.0, 2.799999952316284, 17.399999618530273, 15.0, 27.799999237060547, 29.599998474121094, 10.59999942779541, 12.799999237060547, 12.799999237060547, 12.799999237060547, 19.19999885559082, 19.799999237060547] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [4.799999713897705, 5.199999809265137, 13.59999942779541, 15.799999237060547, 13.0, 11.59999942779541, 13.59999942779541, 12.399999618530273, 14.59999942779541, 17.19999885559082, 13.59999942779541, 14.799999237060547, 3.0, 2.799999952316284, 10.59999942779541, 12.799999237060547] +k eval/a1:test_return:Overcooked-CoordRing6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CoordRing6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [17.799999237060547, 17.19999885559082, 27.19999885559082, 25.399999618530273, 25.599998474121094, 23.0, 16.19999885559082, 15.0, 20.399999618530273, 21.0, 24.399999618530273, 22.799999237060547, 17.399999618530273, 15.0, 12.799999237060547, 12.799999237060547] +k eval/a1:test_return:Overcooked-CoordRing6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [23.399999618530273, 26.0, 22.19999885559082, 23.19999885559082, 22.399999618530273, 23.0, 17.19999885559082, 18.399999618530273, 21.19999885559082, 23.0, 33.79999923706055, 31.399999618530273, 27.799999237060547, 29.599998474121094, 19.19999885559082, 19.799999237060547] +k eval/a1:test_return:Overcooked-CoordRing6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +----------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 18.23+/- 1.013 (max: 33.8) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 23.85+/- 1.182 (max: 33.8) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 11.21+/- 1.157 (max: 17.2) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 19.62+/- 1.182 (max: 27.2) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 12.0+/- 0.2966 (max: 16.63) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 11.37+/- 0.4493 (max: 14.28) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 11.66+/- 0.6011 (max: 14.66) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 12.96+/- 0.4105 (max: 16.63) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.1858+/- 0.02108 (max: 0.66) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.265+/- 0.042 (max: 0.66) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0775+/- 0.01315 (max: 0.16) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.215+/- 0.03134 (max: 0.42) | +| eval/a1:test_return:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 2.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 17.2 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 2.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 12.8 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 6.94 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 8.818 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 6.94 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 10.4 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.11 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.02 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0 | +----------------------------------------------------------------------------------------------- +Evaluating PAIRED_CNN-S5_SEED3 against population in Overcooked-ForcedCoord6_9 for xpid SEED_3_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [1.5999999046325684, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 0.5999999642372131, 0.0, 1.7999999523162842, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.19999998807907104, 1.5999999046325684, 0.0, 0.19999998807907104, 0.19999998807907104, 1.1999999284744263, 0.0, 2.3999998569488525, 0.0, 1.0, 0.0, 1.7999999523162842, 0.0, 2.5999999046325684, 0.0, 1.0, 0.0, 1.1999999284744263, 0.0, 0.3999999761581421, 0.0, 0.19999998807907104, 0.19999998807907104, 0.3999999761581421, 0.0, 1.0, 0.19999998807907104, 1.0, 0.0, 2.0, 0.0, 3.1999998092651367, 0.0, 2.3999998569488525, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [1.5999999046325684, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.19999998807907104, 1.1999999284744263, 0.0, 1.7999999523162842, 0.0, 1.1999999284744263, 0.0, 0.3999999761581421, 0.0, 2.0, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [1.1999999284744263, 0.0, 1.7999999523162842, 0.0, 1.5999999046325684, 0.0, 2.3999998569488525, 0.0, 2.5999999046325684, 0.0, 0.3999999761581421, 0.0, 1.0, 0.19999998807907104, 3.1999998092651367, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [1.1999999284744263, 0.0, 0.5999999642372131, 0.0, 0.19999998807907104, 0.19999998807907104, 1.0, 0.0, 1.0, 0.0, 0.19999998807907104, 0.19999998807907104, 1.0, 0.0, 2.3999998569488525, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.6667+/- 0.1231 (max: 3.2) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.5+/- 0.1673 (max: 2.4) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.6+/- 0.1807 (max: 2.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.9+/- 0.2757 (max: 3.2) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 2.491+/- 0.3565 (max: 7.332) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 2.231+/- 0.5387 (max: 6.499) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 2.391+/- 0.6017 (max: 6.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 2.85+/- 0.7269 (max: 7.332) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------ +Evaluating PAIRED_CNN-S5_SEED3 against population in Overcooked-CounterCircuit6_9 for xpid SEED_3_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [9.59999942779541, 6.599999904632568, 15.0, 8.399999618530273, 8.59999942779541, 8.59999942779541, 5.0, 5.400000095367432, 8.0, 9.0, 4.0, 1.0, 5.0, 3.3999998569488525, 11.800000190734863, 11.800000190734863, 10.399999618530273, 6.0, 8.800000190734863, 6.0, 19.19999885559082, 15.59999942779541, 10.800000190734863, 7.799999713897705, 7.799999713897705, 7.599999904632568, 16.799999237060547, 13.399999618530273, 8.800000190734863, 2.3999998569488525, 11.199999809265137, 8.59999942779541, 16.399999618530273, 9.199999809265137, 8.399999618530273, 6.199999809265137, 9.199999809265137, 5.599999904632568, 9.800000190734863, 9.800000190734863, 8.0, 5.599999904632568, 12.0, 7.199999809265137, 13.399999618530273, 13.399999618530273, 9.59999942779541, 4.799999713897705] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [9.59999942779541, 6.599999904632568, 5.0, 5.400000095367432, 5.0, 3.3999998569488525, 8.800000190734863, 6.0, 7.799999713897705, 7.599999904632568, 11.199999809265137, 8.59999942779541, 9.199999809265137, 5.599999904632568, 12.0, 7.199999809265137] +k eval/a1:test_return:Overcooked-CounterCircuit6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [15.0, 8.399999618530273, 8.0, 9.0, 11.800000190734863, 11.800000190734863, 19.19999885559082, 15.59999942779541, 16.799999237060547, 13.399999618530273, 16.399999618530273, 9.199999809265137, 9.800000190734863, 9.800000190734863, 13.399999618530273, 13.399999618530273] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [8.59999942779541, 8.59999942779541, 4.0, 1.0, 10.399999618530273, 6.0, 10.800000190734863, 7.799999713897705, 8.800000190734863, 2.3999998569488525, 8.399999618530273, 6.199999809265137, 8.0, 5.599999904632568, 9.59999942779541, 4.799999713897705] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 8.979+/- 0.5517 (max: 19.2) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 6.937+/- 0.7049 (max: 10.8) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 7.437+/- 0.5939 (max: 12.0) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 12.56+/- 0.848 (max: 19.2) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 11.04+/- 0.3655 (max: 17.45) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 9.48+/- 0.4317 (max: 11.48) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 10.18+/- 0.3669 (max: 12.43) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 13.46+/- 0.577 (max: 17.45) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.04729+/- 0.008826 (max: 0.22) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.01187+/- 0.003561 (max: 0.04) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.02187+/- 0.006783 (max: 0.07) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.1081+/- 0.01733 (max: 0.22) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 1.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 1.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 3.4 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 8.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 5.196 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 5.196 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 7.513 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 9.837 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating PAIRED_CNN-S5_SEED3 against population in Overcooked-AsymmAdvantages6_9 for xpid SEED_3_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [0.3999999761581421, 2.3999998569488525, 0.3999999761581421, 7.0, 0.5999999642372131, 0.5999999642372131, 0.3999999761581421, 5.599999904632568, 0.3999999761581421, 41.599998474121094, 0.3999999761581421, 0.5999999642372131, 0.19999998807907104, 33.599998474121094, 0.19999998807907104, 55.79999923706055, 0.5999999642372131, 11.59999942779541, 0.19999998807907104, 17.19999885559082, 0.3999999761581421, 7.599999904632568, 1.0, 1.0, 0.5999999642372131, 2.200000047683716, 0.19999998807907104, 17.19999885559082, 0.5999999642372131, 2.200000047683716, 0.5999999642372131, 2.799999952316284, 0.19999998807907104, 27.799999237060547, 0.19999998807907104, 0.19999998807907104, 0.3999999761581421, 4.599999904632568, 0.7999999523162842, 15.199999809265137, 0.0, 2.200000047683716, 0.7999999523162842, 6.199999809265137, 0.3999999761581421, 17.799999237060547, 1.1999999284744263, 1.1999999284744263] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.3999999761581421, 2.3999998569488525, 0.3999999761581421, 5.599999904632568, 0.19999998807907104, 33.599998474121094, 0.19999998807907104, 17.19999885559082, 0.5999999642372131, 2.200000047683716, 0.5999999642372131, 2.799999952316284, 0.3999999761581421, 4.599999904632568, 0.7999999523162842, 6.199999809265137] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.3999999761581421, 7.0, 0.3999999761581421, 41.599998474121094, 0.19999998807907104, 55.79999923706055, 0.3999999761581421, 7.599999904632568, 0.19999998807907104, 17.19999885559082, 0.19999998807907104, 27.799999237060547, 0.7999999523162842, 15.199999809265137, 0.3999999761581421, 17.799999237060547] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [0.5999999642372131, 0.5999999642372131, 0.3999999761581421, 0.5999999642372131, 0.5999999642372131, 11.59999942779541, 1.0, 1.0, 0.5999999642372131, 2.200000047683716, 0.19999998807907104, 0.19999998807907104, 0.0, 2.200000047683716, 1.1999999284744263, 1.1999999284744263] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 6.154+/- 1.689 (max: 55.8) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 1.512+/- 0.6909 (max: 11.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 4.887+/- 2.193 (max: 33.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 12.06+/- 4.204 (max: 55.8) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 7.68+/- 1.121 (max: 33.14) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 4.076+/- 0.588 (max: 10.65) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 7.48+/- 1.785 (max: 28.83) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 11.49+/- 2.54 (max: 33.14) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.07417+/- 0.02551 (max: 0.73) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.00125+/- 0.00125 (max: 0.02) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.05437+/- 0.03504 (max: 0.5) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.1669+/- 0.06274 (max: 0.73) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 0.2 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.2 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 1.99 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 1.99 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating PAIRED_CNN-S5_SEED3 against population in Overcooked-CrampedRoom6_9 for xpid SEED_3_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [11.800000190734863, 8.59999942779541, 57.599998474121094, 56.39999771118164, 68.79999542236328, 63.0, 18.0, 15.0, 58.0, 53.79999923706055, 55.79999923706055, 51.79999923706055, 15.799999237060547, 16.0, 56.39999771118164, 60.0, 58.79999923706055, 55.19999694824219, 17.399999618530273, 19.600000381469727, 62.599998474121094, 59.39999771118164, 47.0, 47.20000076293945, 11.59999942779541, 11.59999942779541, 53.0, 46.39999771118164, 42.0, 46.79999923706055, 20.19999885559082, 16.0, 56.39999771118164, 58.0, 58.79999923706055, 51.79999923706055, 14.799999237060547, 16.399999618530273, 60.39999771118164, 61.79999923706055, 59.79999923706055, 55.599998474121094, 19.799999237060547, 18.600000381469727, 64.0, 58.0, 47.20000076293945, 44.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [11.800000190734863, 8.59999942779541, 18.0, 15.0, 15.799999237060547, 16.0, 17.399999618530273, 19.600000381469727, 11.59999942779541, 11.59999942779541, 20.19999885559082, 16.0, 14.799999237060547, 16.399999618530273, 19.799999237060547, 18.600000381469727] +k eval/a1:test_return:Overcooked-CrampedRoom6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [57.599998474121094, 56.39999771118164, 58.0, 53.79999923706055, 56.39999771118164, 60.0, 62.599998474121094, 59.39999771118164, 53.0, 46.39999771118164, 56.39999771118164, 58.0, 60.39999771118164, 61.79999923706055, 64.0, 58.0] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [68.79999542236328, 63.0, 55.79999923706055, 51.79999923706055, 58.79999923706055, 55.19999694824219, 47.0, 47.20000076293945, 42.0, 46.79999923706055, 58.79999923706055, 51.79999923706055, 59.79999923706055, 55.599998474121094, 47.20000076293945, 44.0] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 42.23+/- 2.848 (max: 68.8) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 53.35+/- 1.857 (max: 68.8) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 15.7+/- 0.8418 (max: 20.2) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 57.64+/- 1.053 (max: 64.0) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 24.3+/- 1.199 (max: 35.03) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 29.12+/- 0.5422 (max: 32.41) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 13.31+/- 0.2789 (max: 15.1) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 30.45+/- 1.004 (max: 35.03) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.5702+/- 0.04507 (max: 0.94) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.7775+/- 0.01811 (max: 0.92) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.1431+/- 0.01653 (max: 0.25) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.79+/- 0.01579 (max: 0.94) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 8.6 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 42.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 8.6 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 46.4 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 11.05 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 25.69 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 11.05 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 20.25 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.03 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.65 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.03 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.66 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------- +Evaluating ACCEL_CNN-S5_SEED1 against population in Overcooked-CoordRing6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [3.0, 3.0, 12.799999237060547, 9.0, 11.199999809265137, 12.0, 10.399999618530273, 8.0, 17.600000381469727, 21.19999885559082, 12.399999618530273, 13.799999237060547, 9.0, 11.0, 15.399999618530273, 15.799999237060547, 15.199999809265137, 17.0, 13.199999809265137, 10.399999618530273, 13.59999942779541, 12.59999942779541, 11.800000190734863, 11.399999618530273, 12.0, 8.800000190734863, 8.59999942779541, 8.0, 15.399999618530273, 17.0, 10.399999618530273, 9.800000190734863, 16.600000381469727, 19.600000381469727, 24.799999237060547, 26.399999618530273, 2.0, 2.200000047683716, 13.0, 15.59999942779541, 24.600000381469727, 31.399999618530273, 11.800000190734863, 14.399999618530273, 10.0, 10.59999942779541, 20.19999885559082, 23.600000381469727] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [3.0, 3.0, 10.399999618530273, 8.0, 9.0, 11.0, 13.199999809265137, 10.399999618530273, 12.0, 8.800000190734863, 10.399999618530273, 9.800000190734863, 2.0, 2.200000047683716, 11.800000190734863, 14.399999618530273] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [12.799999237060547, 9.0, 17.600000381469727, 21.19999885559082, 15.399999618530273, 15.799999237060547, 13.59999942779541, 12.59999942779541, 8.59999942779541, 8.0, 16.600000381469727, 19.600000381469727, 13.0, 15.59999942779541, 10.0, 10.59999942779541] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [11.199999809265137, 12.0, 12.399999618530273, 13.799999237060547, 15.199999809265137, 17.0, 11.800000190734863, 11.399999618530273, 15.399999618530273, 17.0, 24.799999237060547, 26.399999618530273, 24.600000381469727, 31.399999618530273, 20.19999885559082, 23.600000381469727] +------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CoordRing6_9 | 13.49+/- 0.8864 (max: 31.4) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 18.01+/- 1.589 (max: 31.4) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 8.712+/- 1.002 (max: 14.4) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 13.75+/- 0.9831 (max: 21.2) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 11.45+/- 0.3211 (max: 16.73) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 11.08+/- 0.5 (max: 14.77) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 10.94+/- 0.6586 (max: 14.55) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 12.34+/- 0.4533 (max: 16.73) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.09854+/- 0.01743 (max: 0.56) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.1444+/- 0.04506 (max: 0.56) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.05375+/- 0.01052 (max: 0.14) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0975+/- 0.02101 (max: 0.3) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 2.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 11.2 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 2.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 8.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 6.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 8.417 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 6.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 9.798 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------ +Evaluating ACCEL_CNN-S5_SEED1 against population in Overcooked-ForcedCoord6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [0.0, 0.19999998807907104, 0.0, 0.0, 0.19999998807907104, 0.19999998807907104, 0.19999998807907104, 0.19999998807907104, 0.19999998807907104, 0.0, 0.0, 0.3999999761581421, 0.19999998807907104, 0.5999999642372131, 0.3999999761581421, 0.5999999642372131, 0.19999998807907104, 1.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.19999998807907104, 0.0, 0.19999998807907104, 0.19999998807907104, 0.19999998807907104, 0.19999998807907104, 0.0, 0.0, 0.3999999761581421, 0.19999998807907104, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.19999998807907104, 0.0, 0.0, 0.0, 0.19999998807907104, 0.0, 0.0, 0.19999998807907104, 0.5999999642372131, 0.0, 0.19999998807907104] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [0.0, 0.19999998807907104, 0.19999998807907104, 0.19999998807907104, 0.19999998807907104, 0.5999999642372131, 0.3999999761581421, 0.0, 0.19999998807907104, 0.19999998807907104, 0.19999998807907104, 0.0, 0.0, 0.19999998807907104, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [0.0, 0.0, 0.19999998807907104, 0.0, 0.3999999761581421, 0.5999999642372131, 0.3999999761581421, 0.19999998807907104, 0.19999998807907104, 0.0, 0.0, 0.0, 0.0, 0.0, 0.19999998807907104, 0.5999999642372131] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [0.19999998807907104, 0.19999998807907104, 0.0, 0.3999999761581421, 0.19999998807907104, 1.0, 0.0, 0.19999998807907104, 0.0, 0.3999999761581421, 0.0, 0.0, 0.0, 0.19999998807907104, 0.0, 0.19999998807907104] +------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.175+/- 0.03073 (max: 1.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.1875+/- 0.06447 (max: 1.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.1625+/- 0.04171 (max: 0.6) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.175+/- 0.05439 (max: 0.6) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 1.342+/- 0.1859 (max: 4.359) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 1.369+/- 0.3444 (max: 4.359) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 1.383+/- 0.2926 (max: 3.412) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 1.274+/- 0.3466 (max: 3.412) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------ +Evaluating ACCEL_CNN-S5_SEED1 against population in Overcooked-CounterCircuit6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [14.59999942779541, 11.199999809265137, 33.0, 18.399999618530273, 17.19999885559082, 12.799999237060547, 6.0, 6.399999618530273, 13.199999809265137, 9.0, 2.799999952316284, 0.3999999761581421, 1.5999999046325684, 0.5999999642372131, 5.0, 3.799999952316284, 4.400000095367432, 3.0, 9.59999942779541, 6.799999713897705, 26.399999618530273, 21.19999885559082, 11.199999809265137, 6.399999618530273, 12.59999942779541, 10.800000190734863, 23.600000381469727, 17.399999618530273, 5.799999713897705, 3.0, 14.199999809265137, 12.59999942779541, 23.19999885559082, 15.59999942779541, 11.800000190734863, 7.199999809265137, 10.0, 7.799999713897705, 20.600000381469727, 13.799999237060547, 11.800000190734863, 6.799999713897705, 16.19999885559082, 12.199999809265137, 38.39999771118164, 28.799999237060547, 11.0, 6.799999713897705] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [14.59999942779541, 11.199999809265137, 6.0, 6.399999618530273, 1.5999999046325684, 0.5999999642372131, 9.59999942779541, 6.799999713897705, 12.59999942779541, 10.800000190734863, 14.199999809265137, 12.59999942779541, 10.0, 7.799999713897705, 16.19999885559082, 12.199999809265137] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [33.0, 18.399999618530273, 13.199999809265137, 9.0, 5.0, 3.799999952316284, 26.399999618530273, 21.19999885559082, 23.600000381469727, 17.399999618530273, 23.19999885559082, 15.59999942779541, 20.600000381469727, 13.799999237060547, 38.39999771118164, 28.799999237060547] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [17.19999885559082, 12.799999237060547, 2.799999952316284, 0.3999999761581421, 4.400000095367432, 3.0, 11.199999809265137, 6.399999618530273, 5.799999713897705, 3.0, 11.800000190734863, 7.199999809265137, 11.800000190734863, 6.799999713897705, 11.0, 6.799999713897705] +----------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 12.23+/- 1.203 (max: 38.4) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 7.65+/- 1.141 (max: 17.2) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 9.575+/- 1.114 (max: 16.2) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 19.46+/- 2.388 (max: 38.4) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 11.78+/- 0.5775 (max: 21.97) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 9.269+/- 0.5789 (max: 12.17) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 10.74+/- 0.704 (max: 13.52) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 15.35+/- 0.9815 (max: 21.97) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.1144+/- 0.02353 (max: 0.62) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.02+/- 0.007528 (max: 0.11) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.05062+/- 0.01142 (max: 0.14) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.2725+/- 0.04985 (max: 0.62) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.4 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.4 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 0.6 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 3.8 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 2.8 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 2.8 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 3.412 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 7.846 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +----------------------------------------------------------------------------------------------------- +Evaluating ACCEL_CNN-S5_SEED1 against population in Overcooked-AsymmAdvantages6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [1.7999999523162842, 2.5999999046325684, 0.5999999642372131, 11.0, 1.399999976158142, 3.5999999046325684, 1.7999999523162842, 6.599999904632568, 1.0, 45.599998474121094, 1.399999976158142, 3.5999999046325684, 1.399999976158142, 41.0, 1.5999999046325684, 57.79999923706055, 1.7999999523162842, 13.59999942779541, 1.0, 17.799999237060547, 1.7999999523162842, 10.0, 2.3999998569488525, 6.399999618530273, 1.1999999284744263, 3.1999998092651367, 1.5999999046325684, 19.0, 1.5999999046325684, 3.799999952316284, 0.19999998807907104, 2.200000047683716, 1.7999999523162842, 33.0, 2.0, 4.400000095367432, 1.0, 2.5999999046325684, 1.7999999523162842, 18.0, 1.1999999284744263, 7.599999904632568, 0.5999999642372131, 5.400000095367432, 0.5999999642372131, 20.600000381469727, 1.399999976158142, 2.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [1.7999999523162842, 2.5999999046325684, 1.7999999523162842, 6.599999904632568, 1.399999976158142, 41.0, 1.0, 17.799999237060547, 1.1999999284744263, 3.1999998092651367, 0.19999998807907104, 2.200000047683716, 1.0, 2.5999999046325684, 0.5999999642372131, 5.400000095367432] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.5999999642372131, 11.0, 1.0, 45.599998474121094, 1.5999999046325684, 57.79999923706055, 1.7999999523162842, 10.0, 1.5999999046325684, 19.0, 1.7999999523162842, 33.0, 1.7999999523162842, 18.0, 0.5999999642372131, 20.600000381469727] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [1.399999976158142, 3.5999999046325684, 1.399999976158142, 3.5999999046325684, 1.7999999523162842, 13.59999942779541, 2.3999998569488525, 6.399999618530273, 1.5999999046325684, 3.799999952316284, 2.0, 4.400000095367432, 1.1999999284744263, 7.599999904632568, 1.399999976158142, 2.0] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 7.8+/- 1.813 (max: 57.8) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 3.637+/- 0.8129 (max: 13.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 5.65+/- 2.583 (max: 41.0) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 14.11+/- 4.411 (max: 57.8) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 9.557+/- 0.9995 (max: 30.45) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 7.627+/- 0.6797 (max: 14.11) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 8.045+/- 1.619 (max: 28.34) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 13.0+/- 2.262 (max: 30.45) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.08854+/- 0.02773 (max: 0.78) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.01812+/- 0.00765 (max: 0.12) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.05688+/- 0.03978 (max: 0.61) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.1906+/- 0.06703 (max: 0.78) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.2 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 1.2 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 0.2 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.6 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 1.99 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 4.75 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 1.99 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 3.412 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating ACCEL_CNN-S5_SEED1 against population in Overcooked-CrampedRoom6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [8.399999618530273, 7.0, 65.19999694824219, 63.19999694824219, 76.4000015258789, 74.79999542236328, 13.59999942779541, 12.0, 63.39999771118164, 58.39999771118164, 82.0, 72.79999542236328, 17.0, 15.799999237060547, 77.5999984741211, 73.4000015258789, 71.5999984741211, 64.19999694824219, 15.799999237060547, 15.0, 70.0, 68.5999984741211, 53.39999771118164, 52.599998474121094, 8.199999809265137, 8.59999942779541, 66.4000015258789, 62.599998474121094, 67.0, 73.19999694824219, 16.799999237060547, 16.0, 65.5999984741211, 69.4000015258789, 83.19999694824219, 88.5999984741211, 16.19999885559082, 13.199999809265137, 70.19999694824219, 69.79999542236328, 73.19999694824219, 75.5999984741211, 17.19999885559082, 14.799999237060547, 75.4000015258789, 71.0, 73.79999542236328, 75.79999542236328] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [8.399999618530273, 7.0, 13.59999942779541, 12.0, 17.0, 15.799999237060547, 15.799999237060547, 15.0, 8.199999809265137, 8.59999942779541, 16.799999237060547, 16.0, 16.19999885559082, 13.199999809265137, 17.19999885559082, 14.799999237060547] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [65.19999694824219, 63.19999694824219, 63.39999771118164, 58.39999771118164, 77.5999984741211, 73.4000015258789, 70.0, 68.5999984741211, 66.4000015258789, 62.599998474121094, 65.5999984741211, 69.4000015258789, 70.19999694824219, 69.79999542236328, 75.4000015258789, 71.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [76.4000015258789, 74.79999542236328, 82.0, 72.79999542236328, 71.5999984741211, 64.19999694824219, 53.39999771118164, 52.599998474121094, 67.0, 73.19999694824219, 83.19999694824219, 88.5999984741211, 73.19999694824219, 75.5999984741211, 73.79999542236328, 75.79999542236328] +-------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 51.33+/- 4.021 (max: 88.6) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 72.39+/- 2.388 (max: 88.6) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 13.47+/- 0.8841 (max: 17.2) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 68.14+/- 1.26 (max: 77.6) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 25.21+/- 1.261 (max: 35.47) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 31.18+/- 0.5428 (max: 34.14) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 13.42+/- 0.4522 (max: 15.59) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 31.04+/- 0.7315 (max: 35.47) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.621+/- 0.05182 (max: 0.96) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.8762+/- 0.01604 (max: 0.96) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.125+/- 0.01725 (max: 0.22) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.8619+/- 0.008427 (max: 0.93) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 7.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 52.6 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 7.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 58.4 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 10.34 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 27.66 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 10.34 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 24.96 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.73 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.77 | +-------------------------------------------------------------------------------------------------- +Evaluating ACCEL_CNN-S5_SEED2 against population in Overcooked-CoordRing6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [1.7999999523162842, 3.0, 5.599999904632568, 6.599999904632568, 1.7999999523162842, 6.599999904632568, 4.799999713897705, 8.59999942779541, 9.399999618530273, 10.59999942779541, 3.1999998092651367, 7.199999809265137, 5.199999809265137, 6.199999809265137, 7.399999618530273, 7.599999904632568, 4.599999904632568, 5.199999809265137, 5.199999809265137, 3.5999999046325684, 4.599999904632568, 7.399999618530273, 3.1999998092651367, 4.599999904632568, 7.399999618530273, 8.199999809265137, 3.5999999046325684, 6.0, 1.7999999523162842, 3.5999999046325684, 5.599999904632568, 6.0, 10.199999809265137, 13.199999809265137, 14.59999942779541, 16.600000381469727, 1.1999999284744263, 1.0, 8.800000190734863, 8.800000190734863, 14.59999942779541, 15.0, 5.599999904632568, 8.199999809265137, 6.0, 7.199999809265137, 11.399999618530273, 13.399999618530273] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [1.7999999523162842, 3.0, 4.799999713897705, 8.59999942779541, 5.199999809265137, 6.199999809265137, 5.199999809265137, 3.5999999046325684, 7.399999618530273, 8.199999809265137, 5.599999904632568, 6.0, 1.1999999284744263, 1.0, 5.599999904632568, 8.199999809265137] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [5.599999904632568, 6.599999904632568, 9.399999618530273, 10.59999942779541, 7.399999618530273, 7.599999904632568, 4.599999904632568, 7.399999618530273, 3.5999999046325684, 6.0, 10.199999809265137, 13.199999809265137, 8.800000190734863, 8.800000190734863, 6.0, 7.199999809265137] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [1.7999999523162842, 6.599999904632568, 3.1999998092651367, 7.199999809265137, 4.599999904632568, 5.199999809265137, 3.1999998092651367, 4.599999904632568, 1.7999999523162842, 3.5999999046325684, 14.59999942779541, 16.600000381469727, 14.59999942779541, 15.0, 11.399999618530273, 13.399999618530273] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 6.917+/- 0.5506 (max: 16.6) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 7.962+/- 1.333 (max: 16.6) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 5.1+/- 0.6088 (max: 8.6) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 7.687+/- 0.6072 (max: 13.2) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 9.568+/- 0.2986 (max: 13.03) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 9.48+/- 0.5955 (max: 12.92) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 8.839+/- 0.5658 (max: 11.75) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 10.39+/- 0.2874 (max: 13.03) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.02437+/- 0.004334 (max: 0.11) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0325+/- 0.01074 (max: 0.11) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.01562+/- 0.00418 (max: 0.05) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.025+/- 0.005845 (max: 0.1) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 1.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 1.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 1.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 3.6 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 4.359 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 5.724 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 4.359 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 8.188 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------- +Evaluating ACCEL_CNN-S5_SEED2 against population in Overcooked-ForcedCoord6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [1.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.3999999761581421, 0.7999999523162842, 0.3999999761581421, 0.3999999761581421, 0.19999998807907104, 0.5999999642372131, 0.0, 0.7999999523162842, 0.19999998807907104, 0.0, 0.0, 0.7999999523162842, 0.3999999761581421, 0.5999999642372131, 0.0, 0.7999999523162842, 0.3999999761581421, 1.399999976158142, 0.0, 0.7999999523162842, 0.0, 0.3999999761581421, 0.0, 0.5999999642372131, 0.0, 0.3999999761581421, 0.19999998807907104, 0.0, 0.5999999642372131, 0.0, 0.0, 0.19999998807907104, 0.19999998807907104, 0.19999998807907104, 1.0, 1.0, 0.0, 2.0, 0.19999998807907104, 2.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [1.0, 0.0, 1.0, 0.3999999761581421, 0.5999999642372131, 0.0, 0.7999999523162842, 0.3999999761581421, 1.399999976158142, 0.0, 0.5999999642372131, 0.0, 0.0, 0.0, 1.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [1.0, 0.0, 0.7999999523162842, 0.3999999761581421, 0.7999999523162842, 0.19999998807907104, 0.5999999642372131, 0.0, 0.7999999523162842, 0.0, 0.3999999761581421, 0.19999998807907104, 0.19999998807907104, 0.19999998807907104, 2.0, 0.19999998807907104] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [0.0, 0.0, 0.3999999761581421, 0.19999998807907104, 0.0, 0.0, 0.7999999523162842, 0.3999999761581421, 0.3999999761581421, 0.0, 0.0, 0.5999999642372131, 0.19999998807907104, 1.0, 2.0, 0.0] +------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.4375+/- 0.07236 (max: 2.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.375+/- 0.134 (max: 2.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.45+/- 0.119 (max: 1.4) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.4875+/- 0.1291 (max: 2.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 2.201+/- 0.2716 (max: 6.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 1.879+/- 0.487 (max: 6.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 2.158+/- 0.5118 (max: 5.103) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 2.567+/- 0.422 (max: 6.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------ +Evaluating ACCEL_CNN-S5_SEED2 against population in Overcooked-CounterCircuit6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [14.59999942779541, 10.59999942779541, 35.0, 22.600000381469727, 28.599998474121094, 27.0, 13.59999942779541, 10.800000190734863, 20.399999618530273, 13.799999237060547, 6.599999904632568, 0.7999999523162842, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 1.5999999046325684, 2.5999999046325684, 10.0, 10.800000190734863, 27.799999237060547, 22.19999885559082, 15.199999809265137, 5.599999904632568, 15.0, 12.0, 32.79999923706055, 25.599998474121094, 8.0, 3.3999998569488525, 15.0, 15.0, 34.0, 23.799999237060547, 15.199999809265137, 5.599999904632568, 16.19999885559082, 8.800000190734863, 43.0, 19.0, 9.399999618530273, 3.1999998092651367, 16.399999618530273, 9.800000190734863, 44.599998474121094, 23.19999885559082, 9.800000190734863, 5.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [14.59999942779541, 10.59999942779541, 13.59999942779541, 10.800000190734863, 0.3999999761581421, 0.0, 10.0, 10.800000190734863, 15.0, 12.0, 15.0, 15.0, 16.19999885559082, 8.800000190734863, 16.399999618530273, 9.800000190734863] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [35.0, 22.600000381469727, 20.399999618530273, 13.799999237060547, 0.3999999761581421, 0.0, 27.799999237060547, 22.19999885559082, 32.79999923706055, 25.599998474121094, 34.0, 23.799999237060547, 43.0, 19.0, 44.599998474121094, 23.19999885559082] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [28.599998474121094, 27.0, 6.599999904632568, 0.7999999523162842, 1.5999999046325684, 2.5999999046325684, 15.199999809265137, 5.599999904632568, 8.0, 3.3999998569488525, 15.199999809265137, 5.599999904632568, 9.399999618530273, 3.1999998092651367, 9.800000190734863, 5.0] +---------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 14.89+/- 1.623 (max: 44.6) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 9.225+/- 2.1 (max: 28.6) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 11.19+/- 1.234 (max: 16.4) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 24.26+/- 3.149 (max: 44.6) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 11.82+/- 0.7045 (max: 19.93) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 9.154+/- 0.6016 (max: 12.45) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 10.99+/- 0.979 (max: 15.19) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 15.32+/- 1.418 (max: 19.93) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.1773+/- 0.03269 (max: 0.81) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.06625+/- 0.0372 (max: 0.46) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.07187+/- 0.0117 (max: 0.17) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.3937+/- 0.06194 (max: 0.81) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.8 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 3.919 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +---------------------------------------------------------------------------------------------------- +Evaluating ACCEL_CNN-S5_SEED2 against population in Overcooked-AsymmAdvantages6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [0.3999999761581421, 1.399999976158142, 0.0, 5.400000095367432, 0.0, 0.0, 0.0, 3.1999998092651367, 0.0, 36.20000076293945, 0.19999998807907104, 0.0, 0.19999998807907104, 31.19999885559082, 0.3999999761581421, 48.20000076293945, 0.3999999761581421, 7.599999904632568, 0.3999999761581421, 11.399999618530273, 0.3999999761581421, 5.400000095367432, 0.19999998807907104, 0.3999999761581421, 0.19999998807907104, 1.0, 0.0, 13.399999618530273, 0.19999998807907104, 0.7999999523162842, 0.0, 1.1999999284744263, 0.3999999761581421, 24.19999885559082, 0.19999998807907104, 0.3999999761581421, 0.19999998807907104, 1.399999976158142, 0.0, 13.399999618530273, 0.19999998807907104, 1.7999999523162842, 0.5999999642372131, 3.799999952316284, 0.0, 18.399999618530273, 0.3999999761581421, 0.19999998807907104] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.3999999761581421, 1.399999976158142, 0.0, 3.1999998092651367, 0.19999998807907104, 31.19999885559082, 0.3999999761581421, 11.399999618530273, 0.19999998807907104, 1.0, 0.0, 1.1999999284744263, 0.19999998807907104, 1.399999976158142, 0.5999999642372131, 3.799999952316284] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.0, 5.400000095367432, 0.0, 36.20000076293945, 0.3999999761581421, 48.20000076293945, 0.3999999761581421, 5.400000095367432, 0.0, 13.399999618530273, 0.3999999761581421, 24.19999885559082, 0.0, 13.399999618530273, 0.0, 18.399999618530273] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [0.0, 0.0, 0.19999998807907104, 0.0, 0.3999999761581421, 7.599999904632568, 0.19999998807907104, 0.3999999761581421, 0.19999998807907104, 0.7999999523162842, 0.19999998807907104, 0.3999999761581421, 0.19999998807907104, 1.7999999523162842, 0.3999999761581421, 0.19999998807907104] +-------------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 4.904+/- 1.492 (max: 48.2) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.8125+/- 0.4653 (max: 7.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 3.537+/- 1.974 (max: 31.2) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 10.36+/- 3.684 (max: 48.2) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 6.068+/- 1.091 (max: 31.38) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 2.697+/- 0.6093 (max: 9.708) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 5.737+/- 1.62 (max: 25.51) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 9.771+/- 2.553 (max: 31.38) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.05958+/- 0.02192 (max: 0.67) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.000625+/- 0.000625 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.03937+/- 0.02983 (max: 0.47) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.1387+/- 0.0543 (max: 0.67) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +-------------------------------------------------------------------------------------------------------- +Evaluating ACCEL_CNN-S5_SEED2 against population in Overcooked-CrampedRoom6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [9.800000190734863, 6.799999713897705, 58.599998474121094, 54.39999771118164, 67.4000015258789, 67.5999984741211, 9.399999618530273, 10.800000190734863, 44.599998474121094, 35.39999771118164, 51.19999694824219, 44.0, 12.799999237060547, 11.0, 54.0, 48.0, 57.79999923706055, 60.0, 13.799999237060547, 13.199999809265137, 55.0, 51.39999771118164, 39.599998474121094, 40.0, 9.800000190734863, 8.800000190734863, 37.79999923706055, 35.79999923706055, 35.39999771118164, 35.20000076293945, 16.600000381469727, 15.199999809265137, 51.39999771118164, 43.0, 50.0, 48.39999771118164, 10.399999618530273, 9.0, 50.0, 46.20000076293945, 43.79999923706055, 42.0, 16.19999885559082, 14.799999237060547, 53.19999694824219, 50.20000076293945, 46.39999771118164, 54.79999923706055] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [9.800000190734863, 6.799999713897705, 9.399999618530273, 10.800000190734863, 12.799999237060547, 11.0, 13.799999237060547, 13.199999809265137, 9.800000190734863, 8.800000190734863, 16.600000381469727, 15.199999809265137, 10.399999618530273, 9.0, 16.19999885559082, 14.799999237060547] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [58.599998474121094, 54.39999771118164, 44.599998474121094, 35.39999771118164, 54.0, 48.0, 55.0, 51.39999771118164, 37.79999923706055, 35.79999923706055, 51.39999771118164, 43.0, 50.0, 46.20000076293945, 53.19999694824219, 50.20000076293945] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [67.4000015258789, 67.5999984741211, 51.19999694824219, 44.0, 57.79999923706055, 60.0, 39.599998474121094, 40.0, 35.39999771118164, 35.20000076293945, 50.0, 48.39999771118164, 43.79999923706055, 42.0, 46.39999771118164, 54.79999923706055] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 36.27+/- 2.734 (max: 67.6) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 48.97+/- 2.562 (max: 67.6) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 11.77+/- 0.7335 (max: 16.6) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 48.06+/- 1.766 (max: 58.6) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 24.84+/- 1.26 (max: 37.12) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 30.03+/- 0.6819 (max: 33.59) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 13.28+/- 0.4056 (max: 15.63) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 31.22+/- 0.9342 (max: 37.12) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.4892+/- 0.04298 (max: 0.92) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.6844+/- 0.03431 (max: 0.92) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.09937+/- 0.0128 (max: 0.19) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.6837+/- 0.02876 (max: 0.91) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 6.8 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 35.2 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 6.8 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 35.4 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 10.28 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 25.1 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 10.28 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 23.2 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.52 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.49 | +------------------------------------------------------------------------------------------------- +Evaluating ACCEL_CNN-S5_SEED3 against population in Overcooked-CoordRing6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [2.799999952316284, 4.0, 14.0, 14.0, 11.800000190734863, 12.0, 12.399999618530273, 13.399999618530273, 22.399999618530273, 24.19999885559082, 9.59999942779541, 13.199999809265137, 9.59999942779541, 12.59999942779541, 12.799999237060547, 16.0, 10.59999942779541, 12.799999237060547, 12.199999809265137, 9.800000190734863, 16.19999885559082, 15.199999809265137, 11.0, 13.59999942779541, 11.399999618530273, 11.59999942779541, 10.0, 11.399999618530273, 12.0, 13.399999618530273, 8.0, 8.800000190734863, 18.19999885559082, 19.600000381469727, 23.19999885559082, 25.0, 3.5999999046325684, 2.5999999046325684, 11.59999942779541, 10.399999618530273, 19.399999618530273, 22.19999885559082, 7.199999809265137, 4.799999713897705, 9.199999809265137, 8.399999618530273, 7.799999713897705, 9.399999618530273] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [2.799999952316284, 4.0, 12.399999618530273, 13.399999618530273, 9.59999942779541, 12.59999942779541, 12.199999809265137, 9.800000190734863, 11.399999618530273, 11.59999942779541, 8.0, 8.800000190734863, 3.5999999046325684, 2.5999999046325684, 7.199999809265137, 4.799999713897705] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [14.0, 14.0, 22.399999618530273, 24.19999885559082, 12.799999237060547, 16.0, 16.19999885559082, 15.199999809265137, 10.0, 11.399999618530273, 18.19999885559082, 19.600000381469727, 11.59999942779541, 10.399999618530273, 9.199999809265137, 8.399999618530273] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [11.800000190734863, 12.0, 9.59999942779541, 13.199999809265137, 10.59999942779541, 12.799999237060547, 11.0, 13.59999942779541, 12.0, 13.399999618530273, 23.19999885559082, 25.0, 19.399999618530273, 22.19999885559082, 7.799999713897705, 9.399999618530273] +------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CoordRing6_9 | 12.4+/- 0.771 (max: 25.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 14.19+/- 1.317 (max: 25.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 8.425+/- 0.9527 (max: 13.4) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 14.6+/- 1.164 (max: 24.2) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 11.35+/- 0.2857 (max: 15.59) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 10.94+/- 0.4266 (max: 15.59) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 10.85+/- 0.5935 (max: 14.16) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 12.26+/- 0.3876 (max: 15.26) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.07625+/- 0.01365 (max: 0.38) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.075+/- 0.02958 (max: 0.38) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.045+/- 0.009531 (max: 0.12) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.1087+/- 0.02551 (max: 0.33) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 2.6 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 7.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 2.6 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 8.4 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 6.726 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 9.145 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 6.726 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 10.27 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.01 | +------------------------------------------------------------------------------------------------ +Evaluating ACCEL_CNN-S5_SEED3 against population in Overcooked-ForcedCoord6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [2.0, 0.0, 4.400000095367432, 0.0, 1.0, 0.0, 4.799999713897705, 0.0, 6.199999809265137, 0.0, 4.400000095367432, 0.0, 5.400000095367432, 0.0, 7.399999618530273, 0.0, 6.199999809265137, 0.0, 4.199999809265137, 0.0, 3.799999952316284, 0.0, 1.5999999046325684, 0.0, 1.1999999284744263, 0.0, 2.0, 0.0, 2.3999998569488525, 0.0, 4.0, 0.0, 5.799999713897705, 0.0, 2.0, 0.0, 3.5999999046325684, 0.0, 2.799999952316284, 0.0, 4.599999904632568, 0.0, 2.200000047683716, 0.0, 1.7999999523162842, 0.0, 4.199999809265137, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [2.0, 0.0, 4.799999713897705, 0.0, 5.400000095367432, 0.0, 4.199999809265137, 0.0, 1.1999999284744263, 0.0, 4.0, 0.0, 3.5999999046325684, 0.0, 2.200000047683716, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [4.400000095367432, 0.0, 6.199999809265137, 0.0, 7.399999618530273, 0.0, 3.799999952316284, 0.0, 2.0, 0.0, 5.799999713897705, 0.0, 2.799999952316284, 0.0, 1.7999999523162842, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [1.0, 0.0, 4.400000095367432, 0.0, 6.199999809265137, 0.0, 1.5999999046325684, 0.0, 2.3999998569488525, 0.0, 2.0, 0.0, 4.599999904632568, 0.0, 4.199999809265137, 0.0] +----------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 1.833+/- 0.3216 (max: 7.4) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 1.65+/- 0.5258 (max: 6.2) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 1.712+/- 0.5089 (max: 5.4) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 2.137+/- 0.6539 (max: 7.4) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 3.782+/- 0.5769 (max: 10.45) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 3.587+/- 0.9751 (max: 9.673) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 3.75+/- 1.012 (max: 9.739) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 4.008+/- 1.071 (max: 10.45) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.001875+/- 0.0007682 (max: 0.02) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.00125+/- 0.0008539 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0025+/- 0.001708 (max: 0.02) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.001875+/- 0.00136 (max: 0.02) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +----------------------------------------------------------------------------------------------------- +Evaluating ACCEL_CNN-S5_SEED3 against population in Overcooked-CounterCircuit6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [0.3999999761581421, 0.3999999761581421, 1.5999999046325684, 0.7999999523162842, 0.19999998807907104, 0.19999998807907104, 0.19999998807907104, 0.19999998807907104, 0.5999999642372131, 0.19999998807907104, 0.0, 0.0, 0.5999999642372131, 0.19999998807907104, 2.200000047683716, 1.399999976158142, 0.7999999523162842, 0.3999999761581421, 1.0, 0.0, 5.599999904632568, 2.3999998569488525, 0.7999999523162842, 0.0, 0.3999999761581421, 0.3999999761581421, 4.199999809265137, 2.0, 0.19999998807907104, 0.0, 1.399999976158142, 0.0, 1.399999976158142, 0.3999999761581421, 1.0, 0.0, 0.19999998807907104, 0.0, 0.7999999523162842, 0.3999999761581421, 0.0, 0.0, 0.5999999642372131, 0.0, 1.399999976158142, 0.7999999523162842, 0.19999998807907104, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [0.3999999761581421, 0.3999999761581421, 0.19999998807907104, 0.19999998807907104, 0.5999999642372131, 0.19999998807907104, 1.0, 0.0, 0.3999999761581421, 0.3999999761581421, 1.399999976158142, 0.0, 0.19999998807907104, 0.0, 0.5999999642372131, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [1.5999999046325684, 0.7999999523162842, 0.5999999642372131, 0.19999998807907104, 2.200000047683716, 1.399999976158142, 5.599999904632568, 2.3999998569488525, 4.199999809265137, 2.0, 1.399999976158142, 0.3999999761581421, 0.7999999523162842, 0.3999999761581421, 1.399999976158142, 0.7999999523162842] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [0.19999998807907104, 0.19999998807907104, 0.0, 0.0, 0.7999999523162842, 0.3999999761581421, 0.7999999523162842, 0.0, 0.19999998807907104, 0.0, 1.0, 0.0, 0.0, 0.0, 0.19999998807907104, 0.0] +--------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.75+/- 0.1558 (max: 5.6) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.2375+/- 0.0841 (max: 1.0) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 0.375+/- 0.09639 (max: 1.4) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 1.637+/- 0.3639 (max: 5.6) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 2.87+/- 0.328 (max: 8.98) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 1.435+/- 0.4106 (max: 4.359) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 2.215+/- 0.3926 (max: 5.103) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 4.961+/- 0.481 (max: 8.98) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 0.2 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 1.99 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +--------------------------------------------------------------------------------------------------- +Evaluating ACCEL_CNN-S5_SEED3 against population in Overcooked-AsymmAdvantages6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [0.0, 6.0, 0.0, 15.799999237060547, 0.3999999761581421, 11.0, 0.0, 8.59999942779541, 0.0, 44.39999771118164, 0.19999998807907104, 2.5999999046325684, 0.0, 40.599998474121094, 0.0, 63.599998474121094, 0.19999998807907104, 10.199999809265137, 0.0, 18.0, 0.0, 15.0, 0.0, 4.599999904632568, 0.0, 6.799999713897705, 0.0, 21.799999237060547, 0.0, 7.799999713897705, 0.0, 6.199999809265137, 0.0, 30.799999237060547, 0.0, 1.7999999523162842, 0.19999998807907104, 7.799999713897705, 0.0, 17.19999885559082, 0.19999998807907104, 7.199999809265137, 0.0, 7.199999809265137, 0.0, 25.19999885559082, 0.0, 6.399999618530273] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.0, 6.0, 0.0, 8.59999942779541, 0.0, 40.599998474121094, 0.0, 18.0, 0.0, 6.799999713897705, 0.0, 6.199999809265137, 0.19999998807907104, 7.799999713897705, 0.0, 7.199999809265137] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.0, 15.799999237060547, 0.0, 44.39999771118164, 0.0, 63.599998474121094, 0.0, 15.0, 0.0, 21.799999237060547, 0.0, 30.799999237060547, 0.0, 17.19999885559082, 0.0, 25.19999885559082] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [0.3999999761581421, 11.0, 0.19999998807907104, 2.5999999046325684, 0.19999998807907104, 10.199999809265137, 0.0, 4.599999904632568, 0.0, 7.799999713897705, 0.0, 1.7999999523162842, 0.19999998807907104, 7.199999809265137, 0.0, 6.399999618530273] +------------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 8.079+/- 1.93 (max: 63.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 3.287+/- 0.9941 (max: 11.0) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 6.337+/- 2.612 (max: 40.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 14.61+/- 4.756 (max: 63.6) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 7.927+/- 1.275 (max: 31.8) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 5.898+/- 1.337 (max: 14.25) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 7.021+/- 2.041 (max: 27.78) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 10.86+/- 2.903 (max: 31.8) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.09708+/- 0.02841 (max: 0.83) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.02062+/- 0.007983 (max: 0.09) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.06812+/- 0.04038 (max: 0.63) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.2025+/- 0.06859 (max: 0.83) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------- +Evaluating ACCEL_CNN-S5_SEED3 against population in Overcooked-CrampedRoom6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [7.799999713897705, 7.599999904632568, 58.79999923706055, 60.19999694824219, 86.5999984741211, 91.0, 14.59999942779541, 15.799999237060547, 68.79999542236328, 67.4000015258789, 84.5999984741211, 88.4000015258789, 16.799999237060547, 17.0, 89.0, 89.0, 86.5999984741211, 90.19999694824219, 13.199999809265137, 13.399999618530273, 69.19999694824219, 73.5999984741211, 69.0, 68.79999542236328, 7.799999713897705, 8.800000190734863, 65.0, 63.19999694824219, 76.0, 79.0, 14.199999809265137, 14.59999942779541, 73.5999984741211, 75.5999984741211, 83.0, 85.79999542236328, 17.600000381469727, 15.199999809265137, 60.39999771118164, 66.4000015258789, 86.19999694824219, 85.4000015258789, 16.0, 17.399999618530273, 72.5999984741211, 71.0, 80.5999984741211, 83.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [7.799999713897705, 7.599999904632568, 14.59999942779541, 15.799999237060547, 16.799999237060547, 17.0, 13.199999809265137, 13.399999618530273, 7.799999713897705, 8.800000190734863, 14.199999809265137, 14.59999942779541, 17.600000381469727, 15.199999809265137, 16.0, 17.399999618530273] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [58.79999923706055, 60.19999694824219, 68.79999542236328, 67.4000015258789, 89.0, 89.0, 69.19999694824219, 73.5999984741211, 65.0, 63.19999694824219, 73.5999984741211, 75.5999984741211, 60.39999771118164, 66.4000015258789, 72.5999984741211, 71.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [86.5999984741211, 91.0, 84.5999984741211, 88.4000015258789, 86.5999984741211, 90.19999694824219, 69.0, 68.79999542236328, 76.0, 79.0, 83.0, 85.79999542236328, 86.19999694824219, 85.4000015258789, 80.5999984741211, 83.0] +-------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 55.54+/- 4.491 (max: 91.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 82.76+/- 1.664 (max: 91.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 13.61+/- 0.8997 (max: 17.6) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 70.24+/- 2.229 (max: 89.0) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 19.7+/- 0.7193 (max: 27.94) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 22.86+/- 0.5345 (max: 25.83) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 13.36+/- 0.3594 (max: 14.93) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 22.87+/- 0.654 (max: 27.94) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.6775+/- 0.05719 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.9762+/- 0.005468 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.1269+/- 0.0158 (max: 0.2) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.9294+/- 0.008731 (max: 0.98) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 7.6 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 68.8 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 7.6 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 58.8 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 10.87 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 19.26 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 10.87 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 18.29 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.03 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.93 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.03 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.84 | +-------------------------------------------------------------------------------------------------- diff --git a/src/run_results_txt/eval_xpid_all_softmoe_out.txt b/src/run_results_txt/eval_xpid_all_softmoe_out.txt new file mode 100644 index 0000000..8b93e44 --- /dev/null +++ b/src/run_results_txt/eval_xpid_all_softmoe_out.txt @@ -0,0 +1,2280 @@ +Evaluating DR_SoftMoE_SEED1 against population in Overcooked-CoordRing6_9 for xpid dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [4.199999809265137, 3.5999999046325684, 3.799999952316284, 3.0, 1.399999976158142, 3.0, 3.3999998569488525, 4.799999713897705, 5.599999904632568, 9.399999618530273, 2.5999999046325684, 3.0, 6.799999713897705, 7.799999713897705, 4.0, 5.0, 2.5999999046325684, 2.799999952316284, 13.799999237060547, 7.199999809265137, 11.0, 10.199999809265137, 3.0, 0.7999999523162842, 3.1999998092651367, 2.799999952316284, 1.0, 3.3999998569488525, 2.3999998569488525, 2.200000047683716, 14.399999618530273, 14.399999618530273, 21.19999885559082, 21.0, 5.599999904632568, 4.799999713897705, 3.1999998092651367, 2.3999998569488525, 11.399999618530273, 16.600000381469727, 13.799999237060547, 21.0, 9.0, 13.59999942779541, 8.199999809265137, 9.199999809265137, 4.599999904632568, 8.59999942779541] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [4.199999809265137, 3.5999999046325684, 3.3999998569488525, 4.799999713897705, 6.799999713897705, 7.799999713897705, 13.799999237060547, 7.199999809265137, 3.1999998092651367, 2.799999952316284, 14.399999618530273, 14.399999618530273, 3.1999998092651367, 2.3999998569488525, 9.0, 13.59999942779541] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [3.799999952316284, 3.0, 5.599999904632568, 9.399999618530273, 4.0, 5.0, 11.0, 10.199999809265137, 1.0, 3.3999998569488525, 21.19999885559082, 21.0, 11.399999618530273, 16.600000381469727, 8.199999809265137, 9.199999809265137] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [1.399999976158142, 3.0, 2.5999999046325684, 3.0, 2.5999999046325684, 2.799999952316284, 3.0, 0.7999999523162842, 2.3999998569488525, 2.200000047683716, 5.599999904632568, 4.799999713897705, 13.799999237060547, 21.0, 4.599999904632568, 8.59999942779541] +------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CoordRing6_9 | 7.1+/- 0.7939 (max: 21.2) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 5.137+/- 1.323 (max: 21.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 7.162+/- 1.134 (max: 14.4) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 9.0+/- 1.546 (max: 21.2) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 10.38+/- 0.4978 (max: 18.63) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 9.118+/- 0.947 (max: 18.63) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 10.27+/- 0.6658 (max: 14.39) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 11.74+/- 0.8718 (max: 17.52) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.06292+/- 0.01227 (max: 0.34) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.04687+/- 0.02196 (max: 0.34) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.04687+/- 0.01251 (max: 0.14) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.095+/- 0.02596 (max: 0.33) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 0.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 0.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 2.4 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 1.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 3.919 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 3.919 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 6.94 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 5.196 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.01 | +------------------------------------------------------------------------------------------------ +Evaluating DR_SoftMoE_SEED1 against population in Overcooked-ForcedCoord6_9 for xpid dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [0.3999999761581421, 0.3999999761581421, 0.3999999761581421, 1.7999999523162842, 0.5999999642372131, 2.0, 0.3999999761581421, 0.7999999523162842, 0.3999999761581421, 0.5999999642372131, 0.0, 1.0, 0.19999998807907104, 3.3999998569488525, 0.0, 5.400000095367432, 0.19999998807907104, 7.199999809265137, 0.3999999761581421, 0.3999999761581421, 0.3999999761581421, 1.0, 0.19999998807907104, 1.7999999523162842, 0.0, 2.0, 0.3999999761581421, 1.7999999523162842, 1.399999976158142, 1.1999999284744263, 0.19999998807907104, 0.3999999761581421, 0.19999998807907104, 0.19999998807907104, 0.19999998807907104, 0.5999999642372131, 0.19999998807907104, 0.3999999761581421, 0.3999999761581421, 0.19999998807907104, 0.7999999523162842, 0.19999998807907104, 0.3999999761581421, 1.1999999284744263, 0.3999999761581421, 0.3999999761581421, 1.1999999284744263, 1.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [0.3999999761581421, 0.3999999761581421, 0.3999999761581421, 0.7999999523162842, 0.19999998807907104, 3.3999998569488525, 0.3999999761581421, 0.3999999761581421, 0.0, 2.0, 0.19999998807907104, 0.3999999761581421, 0.19999998807907104, 0.3999999761581421, 0.3999999761581421, 1.1999999284744263] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [0.3999999761581421, 1.7999999523162842, 0.3999999761581421, 0.5999999642372131, 0.0, 5.400000095367432, 0.3999999761581421, 1.0, 0.3999999761581421, 1.7999999523162842, 0.19999998807907104, 0.19999998807907104, 0.3999999761581421, 0.19999998807907104, 0.3999999761581421, 0.3999999761581421] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [0.5999999642372131, 2.0, 0.0, 1.0, 0.19999998807907104, 7.199999809265137, 0.19999998807907104, 1.7999999523162842, 1.399999976158142, 1.1999999284744263, 0.19999998807907104, 0.5999999642372131, 0.7999999523162842, 0.19999998807907104, 1.1999999284744263, 1.0] +------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.9333+/- 0.1917 (max: 7.2) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 1.225+/- 0.4254 (max: 7.2) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.7+/- 0.216 (max: 3.4) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.875+/- 0.3291 (max: 5.4) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 3.491+/- 0.2874 (max: 9.6) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 3.959+/- 0.5544 (max: 9.6) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 3.159+/- 0.4342 (max: 7.513) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 3.354+/- 0.5072 (max: 8.879) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------ +Evaluating DR_SoftMoE_SEED1 against population in Overcooked-CounterCircuit6_9 for xpid dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [6.399999618530273, 7.0, 10.59999942779541, 9.59999942779541, 6.399999618530273, 2.0, 5.0, 4.599999904632568, 6.199999809265137, 5.599999904632568, 1.0, 0.0, 0.0, 0.0, 1.0, 0.5999999642372131, 1.1999999284744263, 1.1999999284744263, 5.199999809265137, 3.0, 13.59999942779541, 10.59999942779541, 4.0, 2.3999998569488525, 7.399999618530273, 4.199999809265137, 9.800000190734863, 9.59999942779541, 1.5999999046325684, 1.399999976158142, 8.399999618530273, 3.3999998569488525, 10.199999809265137, 6.199999809265137, 5.599999904632568, 4.400000095367432, 6.399999618530273, 4.400000095367432, 7.399999618530273, 7.399999618530273, 7.599999904632568, 2.3999998569488525, 7.399999618530273, 5.799999713897705, 10.199999809265137, 10.0, 6.799999713897705, 1.1999999284744263] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [6.399999618530273, 7.0, 5.0, 4.599999904632568, 0.0, 0.0, 5.199999809265137, 3.0, 7.399999618530273, 4.199999809265137, 8.399999618530273, 3.3999998569488525, 6.399999618530273, 4.400000095367432, 7.399999618530273, 5.799999713897705] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [10.59999942779541, 9.59999942779541, 6.199999809265137, 5.599999904632568, 1.0, 0.5999999642372131, 13.59999942779541, 10.59999942779541, 9.800000190734863, 9.59999942779541, 10.199999809265137, 6.199999809265137, 7.399999618530273, 7.399999618530273, 10.199999809265137, 10.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [6.399999618530273, 2.0, 1.0, 0.0, 1.1999999284744263, 1.1999999284744263, 4.0, 2.3999998569488525, 1.5999999046325684, 1.399999976158142, 5.599999904632568, 4.400000095367432, 7.599999904632568, 2.3999998569488525, 6.799999713897705, 1.1999999284744263] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 5.342+/- 0.4983 (max: 13.6) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 3.075+/- 0.5977 (max: 7.6) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 4.912+/- 0.6099 (max: 8.4) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 8.037+/- 0.8747 (max: 13.6) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 8.861+/- 0.5576 (max: 16.37) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 6.63+/- 0.7181 (max: 11.59) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 8.289+/- 0.8692 (max: 12.71) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 11.67+/- 0.8794 (max: 16.37) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.02833+/- 0.005557 (max: 0.13) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.00625+/- 0.003521 (max: 0.05) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.01187+/- 0.003191 (max: 0.04) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.06687+/- 0.01087 (max: 0.13) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 0.6 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 3.412 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating DR_SoftMoE_SEED1 against population in Overcooked-AsymmAdvantages6_9 for xpid dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [10.59999942779541, 15.399999618530273, 9.0, 24.600000381469727, 16.19999885559082, 28.599998474121094, 10.59999942779541, 20.19999885559082, 14.199999809265137, 54.599998474121094, 11.59999942779541, 15.399999618530273, 16.19999885559082, 39.79999923706055, 14.199999809265137, 68.5999984741211, 15.199999809265137, 22.799999237060547, 12.0, 31.0, 13.59999942779541, 25.599998474121094, 13.799999237060547, 11.0, 11.800000190734863, 11.399999618530273, 10.399999618530273, 30.799999237060547, 13.799999237060547, 18.600000381469727, 12.199999809265137, 17.0, 12.0, 43.79999923706055, 13.199999809265137, 13.0, 10.800000190734863, 15.59999942779541, 15.799999237060547, 33.79999923706055, 12.59999942779541, 16.19999885559082, 9.399999618530273, 20.600000381469727, 8.59999942779541, 38.39999771118164, 18.19999885559082, 18.799999237060547] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [10.59999942779541, 15.399999618530273, 10.59999942779541, 20.19999885559082, 16.19999885559082, 39.79999923706055, 12.0, 31.0, 11.800000190734863, 11.399999618530273, 12.199999809265137, 17.0, 10.800000190734863, 15.59999942779541, 9.399999618530273, 20.600000381469727] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [9.0, 24.600000381469727, 14.199999809265137, 54.599998474121094, 14.199999809265137, 68.5999984741211, 13.59999942779541, 25.599998474121094, 10.399999618530273, 30.799999237060547, 12.0, 43.79999923706055, 15.799999237060547, 33.79999923706055, 8.59999942779541, 38.39999771118164] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [16.19999885559082, 28.599998474121094, 11.59999942779541, 15.399999618530273, 15.199999809265137, 22.799999237060547, 13.799999237060547, 11.0, 13.799999237060547, 18.600000381469727, 13.199999809265137, 13.0, 12.59999942779541, 16.19999885559082, 18.19999885559082, 18.799999237060547] +----------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 19.62+/- 1.775 (max: 68.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 16.19+/- 1.132 (max: 28.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 16.54+/- 2.067 (max: 39.8) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 26.12+/- 4.452 (max: 68.6) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 19.33+/- 0.8651 (max: 35.24) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 19.19+/- 1.115 (max: 27.83) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 17.0+/- 1.186 (max: 30.0) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 21.81+/- 1.901 (max: 35.24) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.2521+/- 0.02769 (max: 0.83) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.1987+/- 0.01832 (max: 0.38) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.2056+/- 0.03832 (max: 0.61) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.3519+/- 0.06639 (max: 0.83) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 8.6 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 11.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 9.4 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 8.6 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 12.08 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 13.92 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 12.68 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 12.08 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.04 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.11 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.08 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.04 | +----------------------------------------------------------------------------------------------------- +Evaluating DR_SoftMoE_SEED1 against population in Overcooked-CrampedRoom6_9 for xpid dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [10.800000190734863, 8.399999618530273, 67.4000015258789, 51.19999694824219, 96.5999984741211, 88.5999984741211, 16.19999885559082, 15.59999942779541, 76.79999542236328, 61.19999694824219, 91.5999984741211, 85.4000015258789, 18.19999885559082, 15.59999942779541, 89.4000015258789, 83.5999984741211, 99.5999984741211, 91.79999542236328, 19.0, 16.600000381469727, 86.0, 73.19999694824219, 86.4000015258789, 79.19999694824219, 9.399999618530273, 10.0, 74.19999694824219, 62.39999771118164, 95.4000015258789, 88.0, 17.799999237060547, 14.59999942779541, 82.0, 67.5999984741211, 98.5999984741211, 90.19999694824219, 17.0, 15.399999618530273, 80.79999542236328, 67.5999984741211, 94.5999984741211, 91.0, 21.600000381469727, 16.799999237060547, 85.79999542236328, 72.4000015258789, 92.4000015258789, 90.79999542236328] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [10.800000190734863, 8.399999618530273, 16.19999885559082, 15.59999942779541, 18.19999885559082, 15.59999942779541, 19.0, 16.600000381469727, 9.399999618530273, 10.0, 17.799999237060547, 14.59999942779541, 17.0, 15.399999618530273, 21.600000381469727, 16.799999237060547] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [67.4000015258789, 51.19999694824219, 76.79999542236328, 61.19999694824219, 89.4000015258789, 83.5999984741211, 86.0, 73.19999694824219, 74.19999694824219, 62.39999771118164, 82.0, 67.5999984741211, 80.79999542236328, 67.5999984741211, 85.79999542236328, 72.4000015258789] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [96.5999984741211, 88.5999984741211, 91.5999984741211, 85.4000015258789, 99.5999984741211, 91.79999542236328, 86.4000015258789, 79.19999694824219, 95.4000015258789, 88.0, 98.5999984741211, 90.19999694824219, 94.5999984741211, 91.0, 92.4000015258789, 90.79999542236328] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 60.1+/- 4.853 (max: 99.6) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 91.26+/- 1.298 (max: 99.6) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 15.19+/- 0.9285 (max: 21.6) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 73.85+/- 2.644 (max: 89.4) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 22.59+/- 0.9647 (max: 31.86) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 27.57+/- 0.8156 (max: 31.86) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 14.03+/- 0.3917 (max: 16.27) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 26.18+/- 0.7295 (max: 31.21) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.6775+/- 0.05446 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.96+/- 0.00677 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.1562+/- 0.01839 (max: 0.3) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.9162+/- 0.01615 (max: 0.98) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 8.4 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 79.2 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 8.4 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 51.2 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 11.02 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 20.8 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 11.02 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 20.91 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.03 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.91 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.03 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.76 | +------------------------------------------------------------------------------------------------- +Evaluating DR_SoftMoE_SEED2 against population in Overcooked-CoordRing6_9 for xpid SEED_2_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [4.799999713897705, 6.0, 17.19999885559082, 16.399999618530273, 20.799999237060547, 20.799999237060547, 17.600000381469727, 15.399999618530273, 33.599998474121094, 34.599998474121094, 17.0, 20.600000381469727, 16.19999885559082, 12.799999237060547, 22.0, 25.799999237060547, 18.19999885559082, 20.600000381469727, 14.399999618530273, 10.800000190734863, 19.0, 20.600000381469727, 17.799999237060547, 19.19999885559082, 18.600000381469727, 15.59999942779541, 23.399999618530273, 25.19999885559082, 19.399999618530273, 19.799999237060547, 13.59999942779541, 12.199999809265137, 29.399999618530273, 24.799999237060547, 36.599998474121094, 38.20000076293945, 5.0, 6.0, 18.600000381469727, 20.0, 27.399999618530273, 29.399999618530273, 8.399999618530273, 10.0, 15.199999809265137, 15.399999618530273, 17.799999237060547, 19.600000381469727] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [4.799999713897705, 6.0, 17.600000381469727, 15.399999618530273, 16.19999885559082, 12.799999237060547, 14.399999618530273, 10.800000190734863, 18.600000381469727, 15.59999942779541, 13.59999942779541, 12.199999809265137, 5.0, 6.0, 8.399999618530273, 10.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [17.19999885559082, 16.399999618530273, 33.599998474121094, 34.599998474121094, 22.0, 25.799999237060547, 19.0, 20.600000381469727, 23.399999618530273, 25.19999885559082, 29.399999618530273, 24.799999237060547, 18.600000381469727, 20.0, 15.199999809265137, 15.399999618530273] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [20.799999237060547, 20.799999237060547, 17.0, 20.600000381469727, 18.19999885559082, 20.600000381469727, 17.799999237060547, 19.19999885559082, 19.399999618530273, 19.799999237060547, 36.599998474121094, 38.20000076293945, 27.399999618530273, 29.399999618530273, 17.799999237060547, 19.600000381469727] +----------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 19.0+/- 1.113 (max: 38.2) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 22.7+/- 1.656 (max: 38.2) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 11.71+/- 1.147 (max: 18.6) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 22.57+/- 1.511 (max: 34.6) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 12.18+/- 0.2692 (max: 16.76) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 11.08+/- 0.3451 (max: 14.45) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 12.18+/- 0.5189 (max: 15.82) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 13.29+/- 0.3601 (max: 16.76) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.2125+/- 0.02894 (max: 0.81) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.2487+/- 0.06103 (max: 0.81) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.08875+/- 0.017 (max: 0.24) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.3+/- 0.04734 (max: 0.69) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 4.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 17.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 4.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 15.2 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 8.542 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 8.716 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 8.542 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 11.7 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.06 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.08 | +----------------------------------------------------------------------------------------------- +Evaluating DR_SoftMoE_SEED2 against population in Overcooked-ForcedCoord6_9 for xpid SEED_2_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [2.799999952316284, 0.0, 3.3999998569488525, 0.0, 2.3999998569488525, 0.3999999761581421, 2.3999998569488525, 0.0, 2.799999952316284, 0.0, 2.200000047683716, 0.0, 2.5999999046325684, 0.0, 3.3999998569488525, 0.0, 2.0, 0.19999998807907104, 3.1999998092651367, 0.0, 2.799999952316284, 0.0, 1.7999999523162842, 0.0, 2.3999998569488525, 0.19999998807907104, 3.5999999046325684, 0.3999999761581421, 2.3999998569488525, 0.0, 2.3999998569488525, 0.0, 3.1999998092651367, 0.0, 2.3999998569488525, 0.0, 1.399999976158142, 0.0, 2.200000047683716, 0.0, 2.200000047683716, 0.0, 2.0, 0.0, 3.1999998092651367, 0.19999998807907104, 3.5999999046325684, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [2.799999952316284, 0.0, 2.3999998569488525, 0.0, 2.5999999046325684, 0.0, 3.1999998092651367, 0.0, 2.3999998569488525, 0.19999998807907104, 2.3999998569488525, 0.0, 1.399999976158142, 0.0, 2.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [3.3999998569488525, 0.0, 2.799999952316284, 0.0, 3.3999998569488525, 0.0, 2.799999952316284, 0.0, 3.5999999046325684, 0.3999999761581421, 3.1999998092651367, 0.0, 2.200000047683716, 0.0, 3.1999998092651367, 0.19999998807907104] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [2.3999998569488525, 0.3999999761581421, 2.200000047683716, 0.0, 2.0, 0.19999998807907104, 1.7999999523162842, 0.0, 2.3999998569488525, 0.0, 2.3999998569488525, 0.0, 2.200000047683716, 0.0, 3.5999999046325684, 0.0] +------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 1.337+/- 0.1963 (max: 3.6) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 1.225+/- 0.3119 (max: 3.6) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 1.212+/- 0.3201 (max: 3.2) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 1.575+/- 0.3958 (max: 3.6) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 3.585+/- 0.468 (max: 7.684) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 3.513+/- 0.7831 (max: 7.684) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 3.349+/- 0.8175 (max: 7.332) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 3.894+/- 0.8759 (max: 7.684) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------ +Evaluating DR_SoftMoE_SEED2 against population in Overcooked-CounterCircuit6_9 for xpid SEED_2_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [1.7999999523162842, 1.5999999046325684, 3.3999998569488525, 1.5999999046325684, 1.399999976158142, 0.5999999642372131, 1.0, 1.1999999284744263, 2.3999998569488525, 1.399999976158142, 0.19999998807907104, 0.0, 3.3999998569488525, 1.0, 10.399999618530273, 5.0, 6.0, 1.7999999523162842, 1.399999976158142, 0.7999999523162842, 10.399999618530273, 8.59999942779541, 2.3999998569488525, 0.0, 2.3999998569488525, 1.7999999523162842, 9.0, 4.599999904632568, 4.199999809265137, 0.5999999642372131, 3.0, 0.0, 5.400000095367432, 1.399999976158142, 1.7999999523162842, 0.3999999761581421, 1.399999976158142, 1.0, 4.0, 1.7999999523162842, 0.19999998807907104, 0.19999998807907104, 2.5999999046325684, 0.5999999642372131, 4.199999809265137, 2.200000047683716, 0.7999999523162842, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [1.7999999523162842, 1.5999999046325684, 1.0, 1.1999999284744263, 3.3999998569488525, 1.0, 1.399999976158142, 0.7999999523162842, 2.3999998569488525, 1.7999999523162842, 3.0, 0.0, 1.399999976158142, 1.0, 2.5999999046325684, 0.5999999642372131] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [3.3999998569488525, 1.5999999046325684, 2.3999998569488525, 1.399999976158142, 10.399999618530273, 5.0, 10.399999618530273, 8.59999942779541, 9.0, 4.599999904632568, 5.400000095367432, 1.399999976158142, 4.0, 1.7999999523162842, 4.199999809265137, 2.200000047683716] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [1.399999976158142, 0.5999999642372131, 0.19999998807907104, 0.0, 6.0, 1.7999999523162842, 2.3999998569488525, 0.0, 4.199999809265137, 0.5999999642372131, 1.7999999523162842, 0.3999999761581421, 0.19999998807907104, 0.19999998807907104, 0.7999999523162842, 0.0] +------------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 2.529+/- 0.3802 (max: 10.4) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 1.287+/- 0.4231 (max: 6.0) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 1.562+/- 0.2275 (max: 3.4) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 4.737+/- 0.7985 (max: 10.4) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 5.618+/- 0.421 (max: 11.79) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 3.794+/- 0.7308 (max: 10.0) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 5.007+/- 0.4442 (max: 7.513) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 8.053+/- 0.541 (max: 11.79) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.003125+/- 0.001269 (max: 0.05) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.00125+/- 0.00125 (max: 0.02) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.008125+/- 0.003319 (max: 0.05) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 1.4 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 5.103 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------- +Evaluating DR_SoftMoE_SEED2 against population in Overcooked-AsymmAdvantages6_9 for xpid SEED_2_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [0.0, 19.19999885559082, 0.0, 37.599998474121094, 0.0, 31.399999618530273, 0.0, 18.0, 0.0, 56.19999694824219, 0.0, 33.599998474121094, 0.0, 56.19999694824219, 0.0, 79.79999542236328, 0.0, 41.79999923706055, 0.0, 37.39999771118164, 0.0, 38.0, 0.0, 37.39999771118164, 0.0, 18.19999885559082, 0.0, 43.39999771118164, 0.0, 37.20000076293945, 0.0, 14.799999237060547, 0.0, 48.20000076293945, 0.0, 32.599998474121094, 0.0, 17.19999885559082, 0.0, 54.79999923706055, 0.0, 37.599998474121094, 0.0, 21.19999885559082, 0.0, 50.599998474121094, 0.0, 31.799999237060547] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.0, 19.19999885559082, 0.0, 18.0, 0.0, 56.19999694824219, 0.0, 37.39999771118164, 0.0, 18.19999885559082, 0.0, 14.799999237060547, 0.0, 17.19999885559082, 0.0, 21.19999885559082] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.0, 37.599998474121094, 0.0, 56.19999694824219, 0.0, 79.79999542236328, 0.0, 38.0, 0.0, 43.39999771118164, 0.0, 48.20000076293945, 0.0, 54.79999923706055, 0.0, 50.599998474121094] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [0.0, 31.399999618530273, 0.0, 33.599998474121094, 0.0, 41.79999923706055, 0.0, 37.39999771118164, 0.0, 37.20000076293945, 0.0, 32.599998474121094, 0.0, 37.599998474121094, 0.0, 31.799999237060547] +----------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 18.63+/- 3.135 (max: 79.8) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 17.71+/- 4.616 (max: 41.8) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 12.64+/- 4.077 (max: 56.2) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 25.54+/- 6.988 (max: 79.8) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 11.13+/- 1.665 (max: 29.61) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 11.17+/- 2.889 (max: 23.61) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 9.704+/- 2.614 (max: 28.38) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 12.52+/- 3.26 (max: 29.61) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.2821+/- 0.04722 (max: 0.96) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.29+/- 0.076 (max: 0.69) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.1737+/- 0.05973 (max: 0.81) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.3825+/- 0.1008 (max: 0.96) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +----------------------------------------------------------------------------------------------------- +Evaluating DR_SoftMoE_SEED2 against population in Overcooked-CrampedRoom6_9 for xpid SEED_2_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [8.399999618530273, 7.0, 69.0, 65.0, 102.19999694824219, 99.79999542236328, 14.399999618530273, 14.0, 72.4000015258789, 68.79999542236328, 107.0, 109.0, 18.19999885559082, 18.600000381469727, 90.5999984741211, 91.0, 107.19999694824219, 108.5999984741211, 16.0, 17.600000381469727, 82.79999542236328, 85.79999542236328, 86.79999542236328, 86.19999694824219, 8.399999618530273, 7.799999713897705, 69.4000015258789, 69.0, 95.4000015258789, 94.19999694824219, 20.0, 20.19999885559082, 91.0, 80.5999984741211, 103.79999542236328, 102.5999984741211, 16.0, 16.799999237060547, 75.4000015258789, 75.0, 102.0, 103.19999694824219, 19.600000381469727, 15.59999942779541, 83.19999694824219, 82.0, 103.5999984741211, 103.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [8.399999618530273, 7.0, 14.399999618530273, 14.0, 18.19999885559082, 18.600000381469727, 16.0, 17.600000381469727, 8.399999618530273, 7.799999713897705, 20.0, 20.19999885559082, 16.0, 16.799999237060547, 19.600000381469727, 15.59999942779541] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [69.0, 65.0, 72.4000015258789, 68.79999542236328, 90.5999984741211, 91.0, 82.79999542236328, 85.79999542236328, 69.4000015258789, 69.0, 91.0, 80.5999984741211, 75.4000015258789, 75.0, 83.19999694824219, 82.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [102.19999694824219, 99.79999542236328, 107.0, 109.0, 107.19999694824219, 108.5999984741211, 86.79999542236328, 86.19999694824219, 95.4000015258789, 94.19999694824219, 103.79999542236328, 102.5999984741211, 102.0, 103.19999694824219, 103.5999984741211, 103.0] +-------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 64.67+/- 5.399 (max: 109.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 100.9+/- 1.737 (max: 109.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 14.91+/- 1.142 (max: 20.2) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 78.19+/- 2.205 (max: 91.0) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 19.09+/- 0.7209 (max: 28.33) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 20.97+/- 0.6191 (max: 25.88) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 13.14+/- 0.341 (max: 15.56) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 23.15+/- 0.8303 (max: 28.33) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.6927+/- 0.05804 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.9937+/- 0.002213 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.1337+/- 0.01765 (max: 0.24) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.9506+/- 0.006675 (max: 0.99) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 7.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 86.2 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 7.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 65.0 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 10.34 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 16.42 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 10.34 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 17.52 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.97 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.9 | +-------------------------------------------------------------------------------------------------- +Evaluating DR_SoftMoE_SEED3 against population in Overcooked-CoordRing6_9 for xpid SEED_3_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [14.199999809265137, 11.399999618530273, 34.599998474121094, 30.399999618530273, 25.399999618530273, 27.0, 25.799999237060547, 23.799999237060547, 39.20000076293945, 37.79999923706055, 28.19999885559082, 29.799999237060547, 27.799999237060547, 24.19999885559082, 40.79999923706055, 36.79999923706055, 21.0, 26.0, 28.399999618530273, 29.19999885559082, 30.599998474121094, 34.79999923706055, 25.0, 24.799999237060547, 24.799999237060547, 24.0, 33.79999923706055, 29.399999618530273, 27.399999618530273, 27.799999237060547, 23.799999237060547, 25.0, 34.39999771118164, 38.39999771118164, 38.79999923706055, 38.39999771118164, 8.800000190734863, 9.800000190734863, 27.599998474121094, 28.19999885559082, 25.399999618530273, 32.79999923706055, 22.19999885559082, 23.19999885559082, 27.399999618530273, 24.399999618530273, 24.600000381469727, 24.799999237060547] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [14.199999809265137, 11.399999618530273, 25.799999237060547, 23.799999237060547, 27.799999237060547, 24.19999885559082, 28.399999618530273, 29.19999885559082, 24.799999237060547, 24.0, 23.799999237060547, 25.0, 8.800000190734863, 9.800000190734863, 22.19999885559082, 23.19999885559082] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [34.599998474121094, 30.399999618530273, 39.20000076293945, 37.79999923706055, 40.79999923706055, 36.79999923706055, 30.599998474121094, 34.79999923706055, 33.79999923706055, 29.399999618530273, 34.39999771118164, 38.39999771118164, 27.599998474121094, 28.19999885559082, 27.399999618530273, 24.399999618530273] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [25.399999618530273, 27.0, 28.19999885559082, 29.799999237060547, 21.0, 26.0, 25.0, 24.799999237060547, 27.399999618530273, 27.799999237060547, 38.79999923706055, 38.39999771118164, 25.399999618530273, 32.79999923706055, 24.600000381469727, 24.799999237060547] +----------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 27.55+/- 1.037 (max: 40.8) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 27.95+/- 1.225 (max: 38.8) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 21.65+/- 1.669 (max: 29.2) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 33.04+/- 1.22 (max: 40.8) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 17.07+/- 0.334 (max: 22.03) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 18.27+/- 0.5251 (max: 21.69) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 15.29+/- 0.4934 (max: 17.64) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 17.65+/- 0.4483 (max: 22.03) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.4375+/- 0.02383 (max: 0.73) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.45+/- 0.02714 (max: 0.67) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.2987+/- 0.03773 (max: 0.5) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.5637+/- 0.02863 (max: 0.73) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 8.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 21.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 8.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 24.4 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 11.07 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 14.94 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 11.07 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 14.6 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.03 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.24 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.03 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.34 | +----------------------------------------------------------------------------------------------- +Evaluating DR_SoftMoE_SEED3 against population in Overcooked-ForcedCoord6_9 for xpid SEED_3_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [0.0, 1.0, 0.19999998807907104, 1.399999976158142, 0.0, 1.399999976158142, 0.19999998807907104, 1.1999999284744263, 0.0, 0.5999999642372131, 0.5999999642372131, 2.200000047683716, 1.0, 1.5999999046325684, 1.5999999046325684, 2.0, 0.19999998807907104, 0.7999999523162842, 0.0, 1.0, 0.3999999761581421, 1.1999999284744263, 0.0, 1.0, 0.3999999761581421, 2.200000047683716, 0.3999999761581421, 0.7999999523162842, 0.19999998807907104, 2.799999952316284, 0.0, 3.0, 0.5999999642372131, 3.0, 0.0, 3.5999999046325684, 0.5999999642372131, 1.1999999284744263, 0.19999998807907104, 2.200000047683716, 0.0, 0.7999999523162842, 0.19999998807907104, 0.3999999761581421, 0.19999998807907104, 0.7999999523162842, 0.5999999642372131, 4.799999713897705] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [0.0, 1.0, 0.19999998807907104, 1.1999999284744263, 1.0, 1.5999999046325684, 0.0, 1.0, 0.3999999761581421, 2.200000047683716, 0.0, 3.0, 0.5999999642372131, 1.1999999284744263, 0.19999998807907104, 0.3999999761581421] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [0.19999998807907104, 1.399999976158142, 0.0, 0.5999999642372131, 1.5999999046325684, 2.0, 0.3999999761581421, 1.1999999284744263, 0.3999999761581421, 0.7999999523162842, 0.5999999642372131, 3.0, 0.19999998807907104, 2.200000047683716, 0.19999998807907104, 0.7999999523162842] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [0.0, 1.399999976158142, 0.5999999642372131, 2.200000047683716, 0.19999998807907104, 0.7999999523162842, 0.0, 1.0, 0.19999998807907104, 2.799999952316284, 0.0, 3.5999999046325684, 0.0, 0.7999999523162842, 0.5999999642372131, 4.799999713897705] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 1.012+/- 0.1544 (max: 4.8) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 1.187+/- 0.3603 (max: 4.8) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.875+/- 0.212 (max: 3.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.975+/- 0.2144 (max: 3.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 3.61+/- 0.3338 (max: 8.998) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 3.624+/- 0.7192 (max: 8.998) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 3.4+/- 0.5479 (max: 7.141) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 3.807+/- 0.4738 (max: 7.141) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0002083+/- 0.0002083 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.000625+/- 0.000625 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating DR_SoftMoE_SEED3 against population in Overcooked-CounterCircuit6_9 for xpid SEED_3_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [14.799999237060547, 14.59999942779541, 25.599998474121094, 18.19999885559082, 11.199999809265137, 10.0, 10.0, 8.399999618530273, 15.399999618530273, 11.399999618530273, 3.5999999046325684, 2.0, 13.0, 12.799999237060547, 16.0, 14.199999809265137, 14.799999237060547, 19.799999237060547, 10.0, 7.599999904632568, 20.799999237060547, 17.19999885559082, 8.199999809265137, 7.799999713897705, 11.399999618530273, 7.599999904632568, 24.19999885559082, 17.600000381469727, 4.199999809265137, 2.799999952316284, 13.399999618530273, 13.59999942779541, 19.399999618530273, 14.199999809265137, 5.799999713897705, 4.799999713897705, 8.800000190734863, 6.199999809265137, 22.799999237060547, 12.59999942779541, 9.399999618530273, 7.799999713897705, 15.59999942779541, 10.399999618530273, 33.39999771118164, 19.399999618530273, 10.399999618530273, 8.199999809265137] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [14.799999237060547, 14.59999942779541, 10.0, 8.399999618530273, 13.0, 12.799999237060547, 10.0, 7.599999904632568, 11.399999618530273, 7.599999904632568, 13.399999618530273, 13.59999942779541, 8.800000190734863, 6.199999809265137, 15.59999942779541, 10.399999618530273] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [25.599998474121094, 18.19999885559082, 15.399999618530273, 11.399999618530273, 16.0, 14.199999809265137, 20.799999237060547, 17.19999885559082, 24.19999885559082, 17.600000381469727, 19.399999618530273, 14.199999809265137, 22.799999237060547, 12.59999942779541, 33.39999771118164, 19.399999618530273] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [11.199999809265137, 10.0, 3.5999999046325684, 2.0, 14.799999237060547, 19.799999237060547, 8.199999809265137, 7.799999713897705, 4.199999809265137, 2.799999952316284, 5.799999713897705, 4.799999713897705, 9.399999618530273, 7.799999713897705, 10.399999618530273, 8.199999809265137] +----------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 12.74+/- 0.9171 (max: 33.4) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 8.175+/- 1.153 (max: 19.8) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 11.14+/- 0.7311 (max: 15.6) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 18.9+/- 1.397 (max: 33.4) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 12.3+/- 0.4529 (max: 21.92) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 10.07+/- 0.531 (max: 13.71) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 11.43+/- 0.3169 (max: 13.52) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 15.4+/- 0.7245 (max: 21.92) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.1054+/- 0.01784 (max: 0.53) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.035+/- 0.01354 (max: 0.21) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.05062+/- 0.01105 (max: 0.13) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.2306+/- 0.03332 (max: 0.53) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 2.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 2.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 6.2 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 11.4 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 6.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 6.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 9.708 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 10.58 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.06 | +----------------------------------------------------------------------------------------------------- +Evaluating DR_SoftMoE_SEED3 against population in Overcooked-AsymmAdvantages6_9 for xpid SEED_3_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [7.599999904632568, 13.399999618530273, 0.5999999642372131, 23.399999618530273, 19.600000381469727, 18.0, 10.399999618530273, 14.799999237060547, 7.799999713897705, 47.599998474121094, 10.0, 12.399999618530273, 15.59999942779541, 45.0, 11.0, 58.19999694824219, 16.600000381469727, 26.399999618530273, 4.0, 25.799999237060547, 3.799999952316284, 17.799999237060547, 18.600000381469727, 13.199999809265137, 8.199999809265137, 11.399999618530273, 8.399999618530273, 28.19999885559082, 4.400000095367432, 18.799999237060547, 9.59999942779541, 14.199999809265137, 4.599999904632568, 36.20000076293945, 16.600000381469727, 9.800000190734863, 7.399999618530273, 11.59999942779541, 4.599999904632568, 25.399999618530273, 20.0, 17.799999237060547, 8.59999942779541, 10.800000190734863, 1.1999999284744263, 35.39999771118164, 15.399999618530273, 12.799999237060547] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [7.599999904632568, 13.399999618530273, 10.399999618530273, 14.799999237060547, 15.59999942779541, 45.0, 4.0, 25.799999237060547, 8.199999809265137, 11.399999618530273, 9.59999942779541, 14.199999809265137, 7.399999618530273, 11.59999942779541, 8.59999942779541, 10.800000190734863] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.5999999642372131, 23.399999618530273, 7.799999713897705, 47.599998474121094, 11.0, 58.19999694824219, 3.799999952316284, 17.799999237060547, 8.399999618530273, 28.19999885559082, 4.599999904632568, 36.20000076293945, 4.599999904632568, 25.399999618530273, 1.1999999284744263, 35.39999771118164] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [19.600000381469727, 18.0, 10.0, 12.399999618530273, 16.600000381469727, 26.399999618530273, 18.600000381469727, 13.199999809265137, 4.400000095367432, 18.799999237060547, 16.600000381469727, 9.800000190734863, 20.0, 17.799999237060547, 15.399999618530273, 12.799999237060547] +----------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 16.31+/- 1.731 (max: 58.2) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 15.65+/- 1.288 (max: 26.4) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 13.65+/- 2.419 (max: 45.0) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 19.64+/- 4.408 (max: 58.2) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 14.57+/- 0.7185 (max: 25.98) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 15.13+/- 0.6393 (max: 19.77) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 13.78+/- 1.065 (max: 25.98) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 14.8+/- 1.801 (max: 25.78) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.1917+/- 0.0299 (max: 0.83) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.1862+/- 0.02541 (max: 0.42) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.1306+/- 0.04228 (max: 0.67) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.2581+/- 0.07367 (max: 0.83) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.6 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 4.4 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 4.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.6 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 3.412 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 9.2 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 8.485 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 3.412 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.01 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +----------------------------------------------------------------------------------------------------- +Evaluating DR_SoftMoE_SEED3 against population in Overcooked-CrampedRoom6_9 for xpid SEED_3_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [24.600000381469727, 20.0, 81.79999542236328, 71.79999542236328, 97.19999694824219, 91.19999694824219, 28.799999237060547, 29.399999618530273, 83.0, 74.5999984741211, 91.0, 87.4000015258789, 31.599998474121094, 27.599998474121094, 103.5999984741211, 95.19999694824219, 96.0, 95.79999542236328, 29.399999618530273, 33.0, 93.79999542236328, 88.19999694824219, 87.19999694824219, 83.4000015258789, 21.600000381469727, 18.799999237060547, 82.19999694824219, 80.0, 104.0, 105.39999389648438, 33.599998474121094, 28.799999237060547, 95.5999984741211, 97.19999694824219, 94.19999694824219, 84.5999984741211, 31.599998474121094, 34.0, 90.79999542236328, 88.5999984741211, 103.79999542236328, 101.19999694824219, 33.39999771118164, 34.79999923706055, 89.79999542236328, 93.0, 92.5999984741211, 93.5999984741211] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [24.600000381469727, 20.0, 28.799999237060547, 29.399999618530273, 31.599998474121094, 27.599998474121094, 29.399999618530273, 33.0, 21.600000381469727, 18.799999237060547, 33.599998474121094, 28.799999237060547, 31.599998474121094, 34.0, 33.39999771118164, 34.79999923706055] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [81.79999542236328, 71.79999542236328, 83.0, 74.5999984741211, 103.5999984741211, 95.19999694824219, 93.79999542236328, 88.19999694824219, 82.19999694824219, 80.0, 95.5999984741211, 97.19999694824219, 90.79999542236328, 88.5999984741211, 89.79999542236328, 93.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [97.19999694824219, 91.19999694824219, 91.0, 87.4000015258789, 96.0, 95.79999542236328, 87.19999694824219, 83.4000015258789, 104.0, 105.39999389648438, 94.19999694824219, 84.5999984741211, 103.79999542236328, 101.19999694824219, 92.5999984741211, 93.5999984741211] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 70.39+/- 4.417 (max: 105.4) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 94.29+/- 1.712 (max: 105.4) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 28.81+/- 1.274 (max: 34.8) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 88.07+/- 2.151 (max: 103.6) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 27.71+/- 0.9204 (max: 43.02) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 33.13+/- 0.8861 (max: 43.02) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 19.64+/- 0.4019 (max: 22.48) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 30.35+/- 0.5063 (max: 34.45) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.7933+/- 0.03521 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.9619+/- 0.00754 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.4637+/- 0.02647 (max: 0.6) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.9544+/- 0.00555 (max: 0.99) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 18.8 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 83.4 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 18.8 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 71.8 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 17.52 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 27.86 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 17.52 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 27.44 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.25 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.89 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.25 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.9 | +------------------------------------------------------------------------------------------------- +Evaluating PLR_SoftMoE_SEED1 against population in Overcooked-CoordRing6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [7.199999809265137, 7.199999809265137, 19.0, 18.600000381469727, 27.399999618530273, 22.399999618530273, 15.59999942779541, 17.0, 32.0, 32.0, 25.0, 24.0, 14.59999942779541, 14.0, 25.799999237060547, 27.19999885559082, 26.599998474121094, 27.19999885559082, 15.0, 14.399999618530273, 17.799999237060547, 17.399999618530273, 22.19999885559082, 24.0, 14.799999237060547, 14.199999809265137, 21.19999885559082, 22.0, 27.799999237060547, 28.399999618530273, 17.600000381469727, 19.600000381469727, 26.799999237060547, 31.599998474121094, 40.599998474121094, 34.20000076293945, 4.0, 5.799999713897705, 20.399999618530273, 19.399999618530273, 24.600000381469727, 28.19999885559082, 14.799999237060547, 13.799999237060547, 15.799999237060547, 15.199999809265137, 12.199999809265137, 16.600000381469727] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [7.199999809265137, 7.199999809265137, 15.59999942779541, 17.0, 14.59999942779541, 14.0, 15.0, 14.399999618530273, 14.799999237060547, 14.199999809265137, 17.600000381469727, 19.600000381469727, 4.0, 5.799999713897705, 14.799999237060547, 13.799999237060547] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [19.0, 18.600000381469727, 32.0, 32.0, 25.799999237060547, 27.19999885559082, 17.799999237060547, 17.399999618530273, 21.19999885559082, 22.0, 26.799999237060547, 31.599998474121094, 20.399999618530273, 19.399999618530273, 15.799999237060547, 15.199999809265137] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [27.399999618530273, 22.399999618530273, 25.0, 24.0, 26.599998474121094, 27.19999885559082, 22.19999885559082, 24.0, 27.799999237060547, 28.399999618530273, 40.599998474121094, 34.20000076293945, 24.600000381469727, 28.19999885559082, 12.199999809265137, 16.600000381469727] +----------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 20.48+/- 1.116 (max: 40.6) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 25.71+/- 1.599 (max: 40.6) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 13.1+/- 1.128 (max: 19.6) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 22.64+/- 1.449 (max: 32.0) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 13.09+/- 0.2731 (max: 17.04) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 12.52+/- 0.4405 (max: 17.04) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 12.86+/- 0.5939 (max: 16.34) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 13.89+/- 0.2947 (max: 16.55) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.265+/- 0.02739 (max: 0.85) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.3687+/- 0.04959 (max: 0.85) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.1169+/- 0.01934 (max: 0.26) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.3094+/- 0.04365 (max: 0.62) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 4.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 12.2 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 4.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 15.2 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 8.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 10.2 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 8.0 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 12.27 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.1 | +----------------------------------------------------------------------------------------------- +Evaluating PLR_SoftMoE_SEED1 against population in Overcooked-ForcedCoord6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [4.400000095367432, 0.19999998807907104, 5.400000095367432, 0.19999998807907104, 5.599999904632568, 0.5999999642372131, 1.7999999523162842, 0.0, 2.5999999046325684, 0.19999998807907104, 3.0, 0.3999999761581421, 3.0, 0.19999998807907104, 4.599999904632568, 0.19999998807907104, 4.400000095367432, 0.3999999761581421, 3.1999998092651367, 0.5999999642372131, 3.5999999046325684, 0.3999999761581421, 4.400000095367432, 0.19999998807907104, 4.0, 0.3999999761581421, 4.0, 0.19999998807907104, 3.3999998569488525, 0.5999999642372131, 3.3999998569488525, 0.0, 2.5999999046325684, 0.3999999761581421, 2.5999999046325684, 0.0, 1.1999999284744263, 0.0, 2.5999999046325684, 0.3999999761581421, 2.200000047683716, 0.0, 3.1999998092651367, 0.0, 6.0, 0.0, 5.0, 0.3999999761581421] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [4.400000095367432, 0.19999998807907104, 1.7999999523162842, 0.0, 3.0, 0.19999998807907104, 3.1999998092651367, 0.5999999642372131, 4.0, 0.3999999761581421, 3.3999998569488525, 0.0, 1.1999999284744263, 0.0, 3.1999998092651367, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [5.400000095367432, 0.19999998807907104, 2.5999999046325684, 0.19999998807907104, 4.599999904632568, 0.19999998807907104, 3.5999999046325684, 0.3999999761581421, 4.0, 0.19999998807907104, 2.5999999046325684, 0.3999999761581421, 2.5999999046325684, 0.3999999761581421, 6.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [5.599999904632568, 0.5999999642372131, 3.0, 0.3999999761581421, 4.400000095367432, 0.3999999761581421, 4.400000095367432, 0.19999998807907104, 3.3999998569488525, 0.5999999642372131, 2.5999999046325684, 0.0, 2.200000047683716, 0.0, 5.0, 0.3999999761581421] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 1.921+/- 0.2739 (max: 6.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 2.075+/- 0.4983 (max: 5.6) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 1.6+/- 0.4119 (max: 4.4) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 2.087+/- 0.5263 (max: 6.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 4.706+/- 0.4548 (max: 9.539) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 5.028+/- 0.8035 (max: 9.539) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 4.171+/- 0.8164 (max: 8.754) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 4.918+/- 0.7769 (max: 9.165) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0008333+/- 0.0005012 (max: 0.02) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.001875+/- 0.00136 (max: 0.02) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.000625+/- 0.000625 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating PLR_SoftMoE_SEED1 against population in Overcooked-CounterCircuit6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [5.599999904632568, 3.1999998092651367, 8.0, 5.199999809265137, 5.400000095367432, 1.7999999523162842, 2.200000047683716, 2.5999999046325684, 5.0, 3.0, 1.399999976158142, 0.0, 3.1999998092651367, 1.7999999523162842, 11.399999618530273, 7.199999809265137, 5.0, 3.3999998569488525, 4.599999904632568, 1.7999999523162842, 13.799999237060547, 9.399999618530273, 6.0, 1.5999999046325684, 3.0, 3.0, 11.399999618530273, 7.199999809265137, 4.199999809265137, 2.200000047683716, 4.400000095367432, 2.200000047683716, 9.399999618530273, 4.599999904632568, 3.1999998092651367, 2.200000047683716, 2.3999998569488525, 2.5999999046325684, 5.799999713897705, 4.199999809265137, 3.0, 0.5999999642372131, 5.400000095367432, 3.3999998569488525, 7.399999618530273, 3.1999998092651367, 4.199999809265137, 1.399999976158142] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [5.599999904632568, 3.1999998092651367, 2.200000047683716, 2.5999999046325684, 3.1999998092651367, 1.7999999523162842, 4.599999904632568, 1.7999999523162842, 3.0, 3.0, 4.400000095367432, 2.200000047683716, 2.3999998569488525, 2.5999999046325684, 5.400000095367432, 3.3999998569488525] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [8.0, 5.199999809265137, 5.0, 3.0, 11.399999618530273, 7.199999809265137, 13.799999237060547, 9.399999618530273, 11.399999618530273, 7.199999809265137, 9.399999618530273, 4.599999904632568, 5.799999713897705, 4.199999809265137, 7.399999618530273, 3.1999998092651367] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [5.400000095367432, 1.7999999523162842, 1.399999976158142, 0.0, 5.0, 3.3999998569488525, 6.0, 1.5999999046325684, 4.199999809265137, 2.200000047683716, 3.1999998092651367, 2.200000047683716, 3.0, 0.5999999642372131, 4.199999809265137, 1.399999976158142] +------------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 4.442+/- 0.4257 (max: 13.8) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 2.85+/- 0.438 (max: 6.0) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 3.212+/- 0.2986 (max: 5.6) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 7.262+/- 0.7888 (max: 13.8) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 8.003+/- 0.3485 (max: 12.85) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 6.449+/- 0.6006 (max: 9.319) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 7.313+/- 0.2724 (max: 9.415) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 10.25+/- 0.4224 (max: 12.85) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.01146+/- 0.003234 (max: 0.09) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.00125+/- 0.0008539 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0025+/- 0.001118 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.03062+/- 0.007717 (max: 0.09) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 1.8 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 3.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 5.724 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 7.681 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------- +Evaluating PLR_SoftMoE_SEED1 against population in Overcooked-AsymmAdvantages6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [0.5999999642372131, 6.0, 0.0, 16.19999885559082, 0.5999999642372131, 6.599999904632568, 0.19999998807907104, 9.800000190734863, 0.0, 45.599998474121094, 0.3999999761581421, 6.599999904632568, 0.0, 41.599998474121094, 0.0, 61.79999923706055, 0.19999998807907104, 15.0, 0.0, 19.799999237060547, 0.0, 15.199999809265137, 0.19999998807907104, 9.0, 0.3999999761581421, 5.400000095367432, 0.0, 23.19999885559082, 0.0, 7.599999904632568, 0.5999999642372131, 7.599999904632568, 0.0, 28.399999618530273, 0.3999999761581421, 6.599999904632568, 0.19999998807907104, 5.400000095367432, 0.0, 22.0, 0.7999999523162842, 12.0, 0.19999998807907104, 7.399999618530273, 0.0, 24.399999618530273, 0.7999999523162842, 6.199999809265137] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.5999999642372131, 6.0, 0.19999998807907104, 9.800000190734863, 0.0, 41.599998474121094, 0.0, 19.799999237060547, 0.3999999761581421, 5.400000095367432, 0.5999999642372131, 7.599999904632568, 0.19999998807907104, 5.400000095367432, 0.19999998807907104, 7.399999618530273] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.0, 16.19999885559082, 0.0, 45.599998474121094, 0.0, 61.79999923706055, 0.0, 15.199999809265137, 0.0, 23.19999885559082, 0.0, 28.399999618530273, 0.0, 22.0, 0.0, 24.399999618530273] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [0.5999999642372131, 6.599999904632568, 0.3999999761581421, 6.599999904632568, 0.19999998807907104, 15.0, 0.19999998807907104, 9.0, 0.0, 7.599999904632568, 0.3999999761581421, 6.599999904632568, 0.7999999523162842, 12.0, 0.7999999523162842, 6.199999809265137] +------------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 8.646+/- 1.922 (max: 61.8) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 4.562+/- 1.2 (max: 15.0) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 6.575+/- 2.689 (max: 41.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 14.8+/- 4.704 (max: 61.8) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 8.702+/- 1.228 (max: 30.74) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 7.281+/- 1.246 (max: 13.56) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 7.979+/- 1.918 (max: 28.24) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 10.85+/- 2.908 (max: 30.74) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.1025+/- 0.02856 (max: 0.85) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.02937+/- 0.009809 (max: 0.13) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.06812+/- 0.03898 (max: 0.59) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.21+/- 0.06963 (max: 0.85) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------- +Evaluating PLR_SoftMoE_SEED1 against population in Overcooked-CrampedRoom6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [17.0, 17.600000381469727, 64.19999694824219, 64.0, 75.0, 75.0, 21.399999618530273, 25.0, 61.599998474121094, 53.79999923706055, 81.79999542236328, 83.4000015258789, 19.600000381469727, 21.799999237060547, 73.4000015258789, 70.0, 76.19999694824219, 70.5999984741211, 23.0, 23.19999885559082, 62.0, 64.5999984741211, 56.19999694824219, 57.599998474121094, 12.199999809265137, 16.0, 60.599998474121094, 56.79999923706055, 85.4000015258789, 79.79999542236328, 24.600000381469727, 24.399999618530273, 63.79999923706055, 60.0, 80.4000015258789, 82.4000015258789, 24.600000381469727, 26.399999618530273, 74.19999694824219, 68.4000015258789, 84.4000015258789, 84.79999542236328, 26.0, 25.599998474121094, 62.39999771118164, 63.599998474121094, 64.79999542236328, 73.4000015258789] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [17.0, 17.600000381469727, 21.399999618530273, 25.0, 19.600000381469727, 21.799999237060547, 23.0, 23.19999885559082, 12.199999809265137, 16.0, 24.600000381469727, 24.399999618530273, 24.600000381469727, 26.399999618530273, 26.0, 25.599998474121094] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [64.19999694824219, 64.0, 61.599998474121094, 53.79999923706055, 73.4000015258789, 70.0, 62.0, 64.5999984741211, 60.599998474121094, 56.79999923706055, 63.79999923706055, 60.0, 74.19999694824219, 68.4000015258789, 62.39999771118164, 63.599998474121094] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [75.0, 75.0, 81.79999542236328, 83.4000015258789, 76.19999694824219, 70.5999984741211, 56.19999694824219, 57.599998474121094, 85.4000015258789, 79.79999542236328, 80.4000015258789, 82.4000015258789, 84.4000015258789, 84.79999542236328, 64.79999542236328, 73.4000015258789] +-------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 53.81+/- 3.506 (max: 85.4) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 75.7+/- 2.32 (max: 85.4) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 21.77+/- 1.042 (max: 26.4) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 63.96+/- 1.36 (max: 74.2) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 29.84+/- 1.2 (max: 43.54) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 36.57+/- 0.8246 (max: 43.54) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 19.0+/- 0.4532 (max: 21.61) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 33.96+/- 0.8097 (max: 39.83) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.6675+/- 0.03748 (max: 0.95) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.8631+/- 0.01688 (max: 0.95) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.3169+/- 0.02188 (max: 0.41) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.8225+/- 0.009725 (max: 0.87) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 12.2 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 56.2 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 12.2 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 53.8 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 14.94 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 29.26 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 14.94 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 28.53 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.14 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.72 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.14 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.74 | +-------------------------------------------------------------------------------------------------- +Evaluating PLR_SoftMoE_SEED2 against population in Overcooked-CoordRing6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [4.799999713897705, 4.400000095367432, 24.0, 21.0, 27.19999885559082, 29.599998474121094, 16.799999237060547, 14.0, 34.599998474121094, 35.79999923706055, 28.399999618530273, 31.0, 14.399999618530273, 15.0, 28.0, 29.19999885559082, 30.19999885559082, 34.599998474121094, 18.799999237060547, 12.0, 20.799999237060547, 21.600000381469727, 25.599998474121094, 25.599998474121094, 18.19999885559082, 14.799999237060547, 26.19999885559082, 24.799999237060547, 29.599998474121094, 30.399999618530273, 19.0, 17.399999618530273, 28.19999885559082, 26.0, 38.599998474121094, 39.0, 6.0, 5.0, 19.799999237060547, 21.399999618530273, 32.79999923706055, 35.79999923706055, 17.19999885559082, 15.399999618530273, 19.799999237060547, 21.399999618530273, 20.799999237060547, 25.399999618530273] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [4.799999713897705, 4.400000095367432, 16.799999237060547, 14.0, 14.399999618530273, 15.0, 18.799999237060547, 12.0, 18.19999885559082, 14.799999237060547, 19.0, 17.399999618530273, 6.0, 5.0, 17.19999885559082, 15.399999618530273] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [24.0, 21.0, 34.599998474121094, 35.79999923706055, 28.0, 29.19999885559082, 20.799999237060547, 21.600000381469727, 26.19999885559082, 24.799999237060547, 28.19999885559082, 26.0, 19.799999237060547, 21.399999618530273, 19.799999237060547, 21.399999618530273] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [27.19999885559082, 29.599998474121094, 28.399999618530273, 31.0, 30.19999885559082, 34.599998474121094, 25.599998474121094, 25.599998474121094, 29.599998474121094, 30.399999618530273, 38.599998474121094, 39.0, 32.79999923706055, 35.79999923706055, 20.799999237060547, 25.399999618530273] +----------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 22.92+/- 1.261 (max: 39.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 30.29+/- 1.246 (max: 39.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 13.32+/- 1.319 (max: 19.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 25.16+/- 1.248 (max: 35.8) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 13.54+/- 0.3026 (max: 18.22) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 13.71+/- 0.5262 (max: 17.97) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 12.84+/- 0.5976 (max: 16.2) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 14.06+/- 0.418 (max: 18.22) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.336+/- 0.03009 (max: 0.75) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.5194+/- 0.03608 (max: 0.75) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.1231+/- 0.01947 (max: 0.26) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.3656+/- 0.03884 (max: 0.71) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 4.4 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 20.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 4.4 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 19.8 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 8.542 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 11.09 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 8.542 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 11.66 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.26 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.19 | +----------------------------------------------------------------------------------------------- +Evaluating PLR_SoftMoE_SEED2 against population in Overcooked-ForcedCoord6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [2.799999952316284, 0.0, 1.5999999046325684, 0.19999998807907104, 0.19999998807907104, 0.5999999642372131, 1.399999976158142, 0.19999998807907104, 0.7999999523162842, 0.3999999761581421, 1.1999999284744263, 0.0, 2.3999998569488525, 0.3999999761581421, 1.7999999523162842, 0.3999999761581421, 2.200000047683716, 0.7999999523162842, 1.7999999523162842, 0.0, 1.5999999046325684, 0.0, 1.0, 0.19999998807907104, 0.5999999642372131, 0.19999998807907104, 1.0, 0.19999998807907104, 1.399999976158142, 0.3999999761581421, 0.7999999523162842, 0.0, 0.5999999642372131, 0.5999999642372131, 0.3999999761581421, 0.19999998807907104, 0.5999999642372131, 0.0, 0.3999999761581421, 0.0, 0.5999999642372131, 0.0, 2.5999999046325684, 0.0, 2.5999999046325684, 0.0, 4.799999713897705, 0.19999998807907104] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [2.799999952316284, 0.0, 1.399999976158142, 0.19999998807907104, 2.3999998569488525, 0.3999999761581421, 1.7999999523162842, 0.0, 0.5999999642372131, 0.19999998807907104, 0.7999999523162842, 0.0, 0.5999999642372131, 0.0, 2.5999999046325684, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [1.5999999046325684, 0.19999998807907104, 0.7999999523162842, 0.3999999761581421, 1.7999999523162842, 0.3999999761581421, 1.5999999046325684, 0.0, 1.0, 0.19999998807907104, 0.5999999642372131, 0.5999999642372131, 0.3999999761581421, 0.0, 2.5999999046325684, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [0.19999998807907104, 0.5999999642372131, 1.1999999284744263, 0.0, 2.200000047683716, 0.7999999523162842, 1.0, 0.19999998807907104, 1.399999976158142, 0.3999999761581421, 0.3999999761581421, 0.19999998807907104, 0.5999999642372131, 0.0, 4.799999713897705, 0.19999998807907104] +------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.8375+/- 0.1423 (max: 4.8) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.8875+/- 0.2994 (max: 4.8) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.8625+/- 0.2521 (max: 2.8) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.7625+/- 0.1908 (max: 2.6) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 3.179+/- 0.3258 (max: 8.542) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 3.332+/- 0.5486 (max: 8.542) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 3.032+/- 0.6511 (max: 6.94) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 3.174+/- 0.5195 (max: 6.726) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------ +Evaluating PLR_SoftMoE_SEED2 against population in Overcooked-CounterCircuit6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [8.0, 5.599999904632568, 10.59999942779541, 12.199999809265137, 7.199999809265137, 3.1999998092651367, 6.199999809265137, 3.0, 5.599999904632568, 4.799999713897705, 1.7999999523162842, 0.0, 5.0, 3.0, 15.399999618530273, 9.800000190734863, 7.599999904632568, 4.599999904632568, 5.199999809265137, 3.1999998092651367, 18.600000381469727, 16.0, 9.399999618530273, 3.5999999046325684, 5.0, 4.199999809265137, 12.799999237060547, 10.800000190734863, 6.0, 1.5999999046325684, 9.399999618530273, 3.799999952316284, 11.800000190734863, 9.399999618530273, 6.799999713897705, 2.200000047683716, 6.599999904632568, 3.3999998569488525, 8.399999618530273, 8.0, 7.599999904632568, 1.7999999523162842, 8.800000190734863, 5.400000095367432, 11.399999618530273, 10.0, 8.800000190734863, 2.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [8.0, 5.599999904632568, 6.199999809265137, 3.0, 5.0, 3.0, 5.199999809265137, 3.1999998092651367, 5.0, 4.199999809265137, 9.399999618530273, 3.799999952316284, 6.599999904632568, 3.3999998569488525, 8.800000190734863, 5.400000095367432] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [10.59999942779541, 12.199999809265137, 5.599999904632568, 4.799999713897705, 15.399999618530273, 9.800000190734863, 18.600000381469727, 16.0, 12.799999237060547, 10.800000190734863, 11.800000190734863, 9.399999618530273, 8.399999618530273, 8.0, 11.399999618530273, 10.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [7.199999809265137, 3.1999998092651367, 1.7999999523162842, 0.0, 7.599999904632568, 4.599999904632568, 9.399999618530273, 3.5999999046325684, 6.0, 1.5999999046325684, 6.799999713897705, 2.200000047683716, 7.599999904632568, 1.7999999523162842, 8.800000190734863, 2.0] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 6.992+/- 0.5864 (max: 18.6) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 4.637+/- 0.7459 (max: 9.4) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 5.362+/- 0.5047 (max: 9.4) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 10.97+/- 0.904 (max: 18.6) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 9.638+/- 0.41 (max: 16.06) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 7.581+/- 0.6792 (max: 10.75) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 9.033+/- 0.3242 (max: 11.43) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 12.3+/- 0.4854 (max: 16.06) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.03042+/- 0.006671 (max: 0.17) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.001875+/- 0.00136 (max: 0.02) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.01062+/- 0.003091 (max: 0.04) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.07875+/- 0.01307 (max: 0.17) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 3.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 4.8 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 7.141 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 8.998 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating PLR_SoftMoE_SEED2 against population in Overcooked-AsymmAdvantages6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [0.3999999761581421, 1.5999999046325684, 0.3999999761581421, 9.800000190734863, 1.1999999284744263, 6.599999904632568, 0.19999998807907104, 6.599999904632568, 0.0, 43.20000076293945, 0.0, 7.599999904632568, 0.3999999761581421, 33.79999923706055, 0.19999998807907104, 55.599998474121094, 0.7999999523162842, 10.800000190734863, 0.19999998807907104, 17.0, 0.3999999761581421, 9.399999618530273, 1.0, 3.1999998092651367, 0.7999999523162842, 2.0, 0.19999998807907104, 19.399999618530273, 0.3999999761581421, 4.599999904632568, 0.5999999642372131, 3.3999998569488525, 0.3999999761581421, 27.799999237060547, 0.0, 5.199999809265137, 0.0, 4.400000095367432, 0.19999998807907104, 18.19999885559082, 0.5999999642372131, 6.599999904632568, 0.19999998807907104, 2.799999952316284, 0.0, 21.799999237060547, 0.7999999523162842, 2.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.3999999761581421, 1.5999999046325684, 0.19999998807907104, 6.599999904632568, 0.3999999761581421, 33.79999923706055, 0.19999998807907104, 17.0, 0.7999999523162842, 2.0, 0.5999999642372131, 3.3999998569488525, 0.0, 4.400000095367432, 0.19999998807907104, 2.799999952316284] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.3999999761581421, 9.800000190734863, 0.0, 43.20000076293945, 0.19999998807907104, 55.599998474121094, 0.3999999761581421, 9.399999618530273, 0.19999998807907104, 19.399999618530273, 0.3999999761581421, 27.799999237060547, 0.19999998807907104, 18.19999885559082, 0.0, 21.799999237060547] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [1.1999999284744263, 6.599999904632568, 0.0, 7.599999904632568, 0.7999999523162842, 10.800000190734863, 1.0, 3.1999998092651367, 0.3999999761581421, 4.599999904632568, 0.0, 5.199999809265137, 0.5999999642372131, 6.599999904632568, 0.7999999523162842, 2.0] +------------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 6.933+/- 1.716 (max: 55.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 3.212+/- 0.8243 (max: 10.8) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 4.65+/- 2.213 (max: 33.8) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 12.94+/- 4.292 (max: 55.6) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 8.045+/- 1.111 (max: 31.19) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 6.305+/- 0.9994 (max: 11.93) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 6.692+/- 1.656 (max: 25.56) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 11.14+/- 2.627 (max: 31.19) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.07646+/- 0.0253 (max: 0.77) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.01375+/- 0.005072 (max: 0.06) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.04437+/- 0.03089 (max: 0.46) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.1712+/- 0.06408 (max: 0.77) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------- +Evaluating PLR_SoftMoE_SEED2 against population in Overcooked-CrampedRoom6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [8.399999618530273, 7.399999618530273, 68.4000015258789, 69.5999984741211, 101.79999542236328, 103.5999984741211, 14.399999618530273, 12.199999809265137, 81.19999694824219, 70.0, 109.79999542236328, 109.19999694824219, 16.19999885559082, 15.199999809265137, 99.19999694824219, 94.79999542236328, 108.79999542236328, 104.0, 19.19999885559082, 17.0, 90.79999542236328, 88.19999694824219, 93.4000015258789, 95.5999984741211, 9.0, 8.199999809265137, 72.5999984741211, 68.5999984741211, 96.19999694824219, 103.79999542236328, 16.399999618530273, 19.0, 94.79999542236328, 86.79999542236328, 109.0, 109.39999389648438, 14.199999809265137, 17.0, 83.0, 80.4000015258789, 108.5999984741211, 107.5999984741211, 21.600000381469727, 17.399999618530273, 91.19999694824219, 89.19999694824219, 111.19999694824219, 109.19999694824219] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [8.399999618530273, 7.399999618530273, 14.399999618530273, 12.199999809265137, 16.19999885559082, 15.199999809265137, 19.19999885559082, 17.0, 9.0, 8.199999809265137, 16.399999618530273, 19.0, 14.199999809265137, 17.0, 21.600000381469727, 17.399999618530273] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [68.4000015258789, 69.5999984741211, 81.19999694824219, 70.0, 99.19999694824219, 94.79999542236328, 90.79999542236328, 88.19999694824219, 72.5999984741211, 68.5999984741211, 94.79999542236328, 86.79999542236328, 83.0, 80.4000015258789, 91.19999694824219, 89.19999694824219] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [101.79999542236328, 103.5999984741211, 109.79999542236328, 109.19999694824219, 108.79999542236328, 104.0, 93.4000015258789, 95.5999984741211, 96.19999694824219, 103.79999542236328, 109.0, 109.39999389648438, 108.5999984741211, 107.5999984741211, 111.19999694824219, 109.19999694824219] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 67.56+/- 5.716 (max: 111.2) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 105.1+/- 1.416 (max: 111.2) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 14.55+/- 1.087 (max: 21.6) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 83.05+/- 2.611 (max: 99.2) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 22.38+/- 0.9508 (max: 34.81) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 26.23+/- 0.5773 (max: 31.25) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 13.77+/- 0.4501 (max: 16.9) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 27.13+/- 0.7226 (max: 34.81) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.6887+/- 0.05664 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.9825+/- 0.003476 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.145+/- 0.01992 (max: 0.27) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.9387+/- 0.01076 (max: 0.99) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 7.4 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 93.4 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 7.4 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 68.4 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 10.62 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 23.4 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 10.62 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 23.82 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.95 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.82 | +------------------------------------------------------------------------------------------------- +Evaluating PLR_SoftMoE_SEED3 against population in Overcooked-CoordRing6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [5.599999904632568, 5.599999904632568, 25.19999885559082, 21.399999618530273, 29.19999885559082, 28.399999618530273, 15.799999237060547, 19.19999885559082, 35.79999923706055, 40.0, 29.19999885559082, 30.399999618530273, 14.59999942779541, 17.19999885559082, 30.799999237060547, 31.19999885559082, 28.19999885559082, 26.799999237060547, 14.59999942779541, 11.800000190734863, 20.19999885559082, 21.799999237060547, 24.19999885559082, 23.600000381469727, 18.600000381469727, 18.799999237060547, 26.799999237060547, 29.399999618530273, 31.599998474121094, 32.0, 15.399999618530273, 14.0, 23.0, 27.599998474121094, 41.599998474121094, 44.20000076293945, 3.1999998092651367, 5.400000095367432, 18.19999885559082, 20.19999885559082, 22.0, 22.19999885559082, 16.0, 16.19999885559082, 17.799999237060547, 17.799999237060547, 20.0, 17.799999237060547] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [5.599999904632568, 5.599999904632568, 15.799999237060547, 19.19999885559082, 14.59999942779541, 17.19999885559082, 14.59999942779541, 11.800000190734863, 18.600000381469727, 18.799999237060547, 15.399999618530273, 14.0, 3.1999998092651367, 5.400000095367432, 16.0, 16.19999885559082] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [25.19999885559082, 21.399999618530273, 35.79999923706055, 40.0, 30.799999237060547, 31.19999885559082, 20.19999885559082, 21.799999237060547, 26.799999237060547, 29.399999618530273, 23.0, 27.599998474121094, 18.19999885559082, 20.19999885559082, 17.799999237060547, 17.799999237060547] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [29.19999885559082, 28.399999618530273, 29.19999885559082, 30.399999618530273, 28.19999885559082, 26.799999237060547, 24.19999885559082, 23.600000381469727, 31.599998474121094, 32.0, 41.599998474121094, 44.20000076293945, 22.0, 22.19999885559082, 20.0, 17.799999237060547] +----------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 22.3+/- 1.31 (max: 44.2) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 28.21+/- 1.776 (max: 44.2) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 13.25+/- 1.329 (max: 19.2) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 25.45+/- 1.662 (max: 40.0) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 14.12+/- 0.3525 (max: 18.19) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 14.19+/- 0.4903 (max: 17.59) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 12.99+/- 0.6906 (max: 16.93) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 15.16+/- 0.541 (max: 18.19) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.3258+/- 0.03085 (max: 0.87) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.4612+/- 0.04632 (max: 0.87) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.1312+/- 0.02313 (max: 0.29) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.385+/- 0.04779 (max: 0.74) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 3.2 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 17.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 3.2 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 17.8 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 7.859 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 11.13 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 7.859 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 11.49 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.2 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.12 | +----------------------------------------------------------------------------------------------- +Evaluating PLR_SoftMoE_SEED3 against population in Overcooked-ForcedCoord6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [4.799999713897705, 0.19999998807907104, 8.199999809265137, 0.19999998807907104, 5.199999809265137, 0.19999998807907104, 5.599999904632568, 0.0, 4.799999713897705, 0.3999999761581421, 3.1999998092651367, 0.19999998807907104, 4.400000095367432, 0.19999998807907104, 4.0, 0.0, 3.5999999046325684, 0.0, 5.799999713897705, 0.19999998807907104, 3.799999952316284, 0.19999998807907104, 3.799999952316284, 0.7999999523162842, 4.400000095367432, 0.0, 6.0, 0.19999998807907104, 6.799999713897705, 0.3999999761581421, 3.5999999046325684, 0.3999999761581421, 3.3999998569488525, 0.19999998807907104, 2.0, 0.0, 3.0, 0.19999998807907104, 3.3999998569488525, 0.3999999761581421, 1.5999999046325684, 0.3999999761581421, 7.399999618530273, 0.19999998807907104, 8.0, 0.5999999642372131, 8.59999942779541, 0.7999999523162842] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [4.799999713897705, 0.19999998807907104, 5.599999904632568, 0.0, 4.400000095367432, 0.19999998807907104, 5.799999713897705, 0.19999998807907104, 4.400000095367432, 0.0, 3.5999999046325684, 0.3999999761581421, 3.0, 0.19999998807907104, 7.399999618530273, 0.19999998807907104] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [8.199999809265137, 0.19999998807907104, 4.799999713897705, 0.3999999761581421, 4.0, 0.0, 3.799999952316284, 0.19999998807907104, 6.0, 0.19999998807907104, 3.3999998569488525, 0.19999998807907104, 3.3999998569488525, 0.3999999761581421, 8.0, 0.5999999642372131] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [5.199999809265137, 0.19999998807907104, 3.1999998092651367, 0.19999998807907104, 3.5999999046325684, 0.0, 3.799999952316284, 0.7999999523162842, 6.799999713897705, 0.3999999761581421, 2.0, 0.0, 1.5999999046325684, 0.3999999761581421, 8.59999942779541, 0.7999999523162842] +--------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 2.537+/- 0.3839 (max: 8.6) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 2.35+/- 0.6607 (max: 8.6) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 2.525+/- 0.6514 (max: 7.4) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 2.737+/- 0.721 (max: 8.2) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 5.221+/- 0.5079 (max: 10.62) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 5.097+/- 0.8544 (max: 10.3) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 5.078+/- 0.93 (max: 9.831) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 5.487+/- 0.9074 (max: 10.62) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0025+/- 0.0008681 (max: 0.02) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0025+/- 0.001443 (max: 0.02) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.00125+/- 0.00125 (max: 0.02) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.00375+/- 0.001797 (max: 0.02) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +--------------------------------------------------------------------------------------------------- +Evaluating PLR_SoftMoE_SEED3 against population in Overcooked-CounterCircuit6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [7.0, 6.599999904632568, 8.399999618530273, 7.199999809265137, 6.599999904632568, 2.799999952316284, 4.0, 3.799999952316284, 7.199999809265137, 4.599999904632568, 1.0, 0.19999998807907104, 2.5999999046325684, 3.1999998092651367, 10.800000190734863, 6.799999713897705, 11.59999942779541, 5.0, 6.199999809265137, 2.799999952316284, 14.59999942779541, 13.59999942779541, 7.0, 1.5999999046325684, 5.400000095367432, 3.5999999046325684, 8.199999809265137, 5.799999713897705, 4.400000095367432, 1.399999976158142, 3.3999998569488525, 1.5999999046325684, 9.399999618530273, 5.199999809265137, 5.0, 2.3999998569488525, 3.3999998569488525, 1.7999999523162842, 8.0, 6.399999618530273, 3.1999998092651367, 1.0, 4.199999809265137, 2.5999999046325684, 10.59999942779541, 10.0, 5.599999904632568, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [7.0, 6.599999904632568, 4.0, 3.799999952316284, 2.5999999046325684, 3.1999998092651367, 6.199999809265137, 2.799999952316284, 5.400000095367432, 3.5999999046325684, 3.3999998569488525, 1.5999999046325684, 3.3999998569488525, 1.7999999523162842, 4.199999809265137, 2.5999999046325684] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [8.399999618530273, 7.199999809265137, 7.199999809265137, 4.599999904632568, 10.800000190734863, 6.799999713897705, 14.59999942779541, 13.59999942779541, 8.199999809265137, 5.799999713897705, 9.399999618530273, 5.199999809265137, 8.0, 6.399999618530273, 10.59999942779541, 10.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [6.599999904632568, 2.799999952316284, 1.0, 0.19999998807907104, 11.59999942779541, 5.0, 7.0, 1.5999999046325684, 4.400000095367432, 1.399999976158142, 5.0, 2.3999998569488525, 3.1999998092651367, 1.0, 5.599999904632568, 0.0] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 5.371+/- 0.4917 (max: 14.6) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 3.675+/- 0.7696 (max: 11.6) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 3.887+/- 0.4078 (max: 7.0) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 8.55+/- 0.7082 (max: 14.6) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 8.672+/- 0.4168 (max: 15.86) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 6.968+/- 0.7983 (max: 11.38) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 7.776+/- 0.3347 (max: 9.95) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 11.27+/- 0.4302 (max: 15.86) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.01937+/- 0.004683 (max: 0.17) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.00875+/- 0.0034 (max: 0.04) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0025+/- 0.001118 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.04687+/- 0.01079 (max: 0.17) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 1.6 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 4.6 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 5.426 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 8.773 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating PLR_SoftMoE_SEED3 against population in Overcooked-AsymmAdvantages6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [1.0, 1.7999999523162842, 0.3999999761581421, 11.399999618530273, 1.399999976158142, 3.5999999046325684, 1.0, 5.799999713897705, 1.1999999284744263, 45.39999771118164, 1.5999999046325684, 11.800000190734863, 1.0, 31.599998474121094, 1.1999999284744263, 54.39999771118164, 1.1999999284744263, 9.399999618530273, 1.0, 18.399999618530273, 1.1999999284744263, 8.199999809265137, 0.7999999523162842, 2.3999998569488525, 0.7999999523162842, 2.799999952316284, 0.19999998807907104, 17.0, 0.5999999642372131, 4.599999904632568, 0.7999999523162842, 4.0, 0.5999999642372131, 28.799999237060547, 1.0, 8.399999618530273, 0.19999998807907104, 4.400000095367432, 1.0, 21.19999885559082, 1.0, 6.599999904632568, 0.7999999523162842, 4.199999809265137, 0.19999998807907104, 22.0, 1.5999999046325684, 3.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [1.0, 1.7999999523162842, 1.0, 5.799999713897705, 1.0, 31.599998474121094, 1.0, 18.399999618530273, 0.7999999523162842, 2.799999952316284, 0.7999999523162842, 4.0, 0.19999998807907104, 4.400000095367432, 0.7999999523162842, 4.199999809265137] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.3999999761581421, 11.399999618530273, 1.1999999284744263, 45.39999771118164, 1.1999999284744263, 54.39999771118164, 1.1999999284744263, 8.199999809265137, 0.19999998807907104, 17.0, 0.5999999642372131, 28.799999237060547, 1.0, 21.19999885559082, 0.19999998807907104, 22.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [1.399999976158142, 3.5999999046325684, 1.5999999046325684, 11.800000190734863, 1.1999999284744263, 9.399999618530273, 0.7999999523162842, 2.3999998569488525, 0.5999999642372131, 4.599999904632568, 1.0, 8.399999618530273, 1.0, 6.599999904632568, 1.5999999046325684, 3.0] +------------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 7.354+/- 1.7 (max: 54.4) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 3.687+/- 0.8769 (max: 11.8) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 4.975+/- 2.086 (max: 31.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 13.4+/- 4.283 (max: 54.4) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 9.355+/- 1.051 (max: 32.26) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 7.904+/- 0.9881 (max: 16.52) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 7.804+/- 1.55 (max: 25.8) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 12.36+/- 2.471 (max: 32.26) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.08646+/- 0.02581 (max: 0.75) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.02562+/- 0.009397 (max: 0.12) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.05125+/- 0.03365 (max: 0.5) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.1825+/- 0.06401 (max: 0.75) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.2 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.6 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 0.2 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.2 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 1.99 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 3.412 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 1.99 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 1.99 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------- +Evaluating PLR_SoftMoE_SEED3 against population in Overcooked-CrampedRoom6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [13.59999942779541, 14.199999809265137, 66.0, 66.0, 87.79999542236328, 84.79999542236328, 19.399999618530273, 19.399999618530273, 66.0, 65.0, 85.19999694824219, 87.19999694824219, 22.600000381469727, 21.0, 81.19999694824219, 87.79999542236328, 93.0, 94.4000015258789, 19.799999237060547, 20.600000381469727, 78.0, 79.5999984741211, 69.5999984741211, 69.5999984741211, 15.0, 13.799999237060547, 67.4000015258789, 63.19999694824219, 77.5999984741211, 79.4000015258789, 19.19999885559082, 22.600000381469727, 83.19999694824219, 74.79999542236328, 96.19999694824219, 93.79999542236328, 20.600000381469727, 16.600000381469727, 75.5999984741211, 73.19999694824219, 90.0, 94.19999694824219, 22.399999618530273, 21.399999618530273, 82.79999542236328, 79.4000015258789, 83.5999984741211, 87.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [13.59999942779541, 14.199999809265137, 19.399999618530273, 19.399999618530273, 22.600000381469727, 21.0, 19.799999237060547, 20.600000381469727, 15.0, 13.799999237060547, 19.19999885559082, 22.600000381469727, 20.600000381469727, 16.600000381469727, 22.399999618530273, 21.399999618530273] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [66.0, 66.0, 66.0, 65.0, 81.19999694824219, 87.79999542236328, 78.0, 79.5999984741211, 67.4000015258789, 63.19999694824219, 83.19999694824219, 74.79999542236328, 75.5999984741211, 73.19999694824219, 82.79999542236328, 79.4000015258789] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [87.79999542236328, 84.79999542236328, 85.19999694824219, 87.19999694824219, 93.0, 94.4000015258789, 69.5999984741211, 69.5999984741211, 77.5999984741211, 79.4000015258789, 96.19999694824219, 93.79999542236328, 90.0, 94.19999694824219, 83.5999984741211, 87.0] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 59.68+/- 4.371 (max: 96.2) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 85.84+/- 2.074 (max: 96.2) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 18.89+/- 0.8015 (max: 22.6) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 74.32+/- 1.954 (max: 87.8) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 26.11+/- 1.206 (max: 39.14) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 33.15+/- 0.6656 (max: 39.14) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 14.98+/- 0.3526 (max: 17.29) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 30.19+/- 0.6522 (max: 34.48) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.6881+/- 0.04774 (max: 0.99) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.9256+/- 0.01411 (max: 0.99) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.2337+/- 0.02162 (max: 0.36) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.905+/- 0.009874 (max: 0.96) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 13.6 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 69.6 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 13.6 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 63.2 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 12.55 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 29.17 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 12.55 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 24.74 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.09 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.79 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.09 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.84 | +------------------------------------------------------------------------------------------------- +Evaluating PAIRED_SoftMoE_SEED1 against population in Overcooked-CoordRing6_9 for xpid paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [14.799999237060547, 16.19999885559082, 44.79999923706055, 40.79999923706055, 20.399999618530273, 22.600000381469727, 25.799999237060547, 26.799999237060547, 45.20000076293945, 44.0, 22.0, 26.599998474121094, 23.799999237060547, 24.799999237060547, 41.20000076293945, 43.20000076293945, 21.799999237060547, 23.19999885559082, 37.599998474121094, 36.39999771118164, 40.0, 44.20000076293945, 25.799999237060547, 26.399999618530273, 22.0, 26.0, 28.19999885559082, 29.599998474121094, 19.799999237060547, 22.799999237060547, 26.19999885559082, 23.799999237060547, 48.79999923706055, 45.599998474121094, 51.19999694824219, 60.0, 15.799999237060547, 16.600000381469727, 37.599998474121094, 37.20000076293945, 39.599998474121094, 39.79999923706055, 16.399999618530273, 19.799999237060547, 30.19999885559082, 32.79999923706055, 27.799999237060547, 27.599998474121094] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [14.799999237060547, 16.19999885559082, 25.799999237060547, 26.799999237060547, 23.799999237060547, 24.799999237060547, 37.599998474121094, 36.39999771118164, 22.0, 26.0, 26.19999885559082, 23.799999237060547, 15.799999237060547, 16.600000381469727, 16.399999618530273, 19.799999237060547] +k eval/a1:test_return:Overcooked-CoordRing6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CoordRing6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [44.79999923706055, 40.79999923706055, 45.20000076293945, 44.0, 41.20000076293945, 43.20000076293945, 40.0, 44.20000076293945, 28.19999885559082, 29.599998474121094, 48.79999923706055, 45.599998474121094, 37.599998474121094, 37.20000076293945, 30.19999885559082, 32.79999923706055] +k eval/a1:test_return:Overcooked-CoordRing6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [20.399999618530273, 22.600000381469727, 22.0, 26.599998474121094, 21.799999237060547, 23.19999885559082, 25.799999237060547, 26.399999618530273, 19.799999237060547, 22.799999237060547, 51.19999694824219, 60.0, 39.599998474121094, 39.79999923706055, 27.799999237060547, 27.599998474121094] +k eval/a1:test_return:Overcooked-CoordRing6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +----------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 30.91+/- 1.566 (max: 60.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 29.84+/- 2.937 (max: 60.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 23.3+/- 1.706 (max: 37.6) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 39.59+/- 1.596 (max: 48.8) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 16.96+/- 0.3158 (max: 23.38) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 17.3+/- 0.7194 (max: 23.38) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 16.28+/- 0.4104 (max: 18.63) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 17.32+/- 0.452 (max: 20.65) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.504+/- 0.03459 (max: 0.92) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.4494+/- 0.05703 (max: 0.92) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.3425+/- 0.04084 (max: 0.7) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.72+/- 0.03575 (max: 0.86) | +| eval/a1:test_return:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 14.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 19.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 14.8 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 28.2 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 12.84 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 14.06 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 12.84 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 13.86 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.11 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.2 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.11 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.44 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0 | +----------------------------------------------------------------------------------------------- +Evaluating PAIRED_SoftMoE_SEED1 against population in Overcooked-ForcedCoord6_9 for xpid paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [6.399999618530273, 0.0, 7.599999904632568, 0.0, 1.7999999523162842, 0.0, 5.199999809265137, 0.0, 5.599999904632568, 0.0, 2.0, 0.0, 5.400000095367432, 0.0, 3.799999952316284, 0.0, 4.0, 0.0, 3.0, 0.0, 3.3999998569488525, 0.0, 1.399999976158142, 0.0, 4.199999809265137, 0.0, 5.199999809265137, 0.0, 5.599999904632568, 0.0, 2.799999952316284, 0.0, 2.200000047683716, 0.0, 0.19999998807907104, 0.0, 3.0, 0.0, 1.7999999523162842, 0.0, 2.5999999046325684, 0.0, 6.599999904632568, 0.0, 6.399999618530273, 0.0, 7.799999713897705, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [6.399999618530273, 0.0, 5.199999809265137, 0.0, 5.400000095367432, 0.0, 3.0, 0.0, 4.199999809265137, 0.0, 2.799999952316284, 0.0, 3.0, 0.0, 6.599999904632568, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [7.599999904632568, 0.0, 5.599999904632568, 0.0, 3.799999952316284, 0.0, 3.3999998569488525, 0.0, 5.199999809265137, 0.0, 2.200000047683716, 0.0, 1.7999999523162842, 0.0, 6.399999618530273, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [1.7999999523162842, 0.0, 2.0, 0.0, 4.0, 0.0, 1.399999976158142, 0.0, 5.599999904632568, 0.0, 0.19999998807907104, 0.0, 2.5999999046325684, 0.0, 7.799999713897705, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +----------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 2.042+/- 0.3644 (max: 7.8) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 1.587+/- 0.5912 (max: 7.8) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 2.287+/- 0.647 (max: 6.6) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 2.25+/- 0.6776 (max: 7.6) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 3.866+/- 0.5951 (max: 10.16) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 3.398+/- 0.9812 (max: 10.16) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 4.139+/- 1.083 (max: 9.404) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 4.061+/- 1.082 (max: 10.11) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.001458+/- 0.0005148 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0025+/- 0.001118 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.000625+/- 0.000625 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.00125+/- 0.0008539 (max: 0.01) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +----------------------------------------------------------------------------------------------------- +Evaluating PAIRED_SoftMoE_SEED1 against population in Overcooked-CounterCircuit6_9 for xpid paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [14.799999237060547, 16.399999618530273, 18.399999618530273, 16.799999237060547, 7.0, 10.59999942779541, 10.800000190734863, 12.399999618530273, 13.399999618530273, 15.59999942779541, 3.799999952316284, 4.400000095367432, 5.599999904632568, 3.799999952316284, 7.0, 8.0, 11.199999809265137, 7.199999809265137, 11.800000190734863, 15.59999942779541, 19.799999237060547, 21.600000381469727, 11.800000190734863, 12.799999237060547, 10.800000190734863, 12.799999237060547, 20.399999618530273, 18.799999237060547, 8.199999809265137, 8.800000190734863, 13.0, 13.799999237060547, 13.799999237060547, 13.799999237060547, 8.0, 11.0, 10.59999942779541, 15.199999809265137, 15.0, 13.399999618530273, 11.0, 9.800000190734863, 12.0, 16.19999885559082, 21.600000381469727, 15.59999942779541, 11.800000190734863, 9.59999942779541] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [14.799999237060547, 16.399999618530273, 10.800000190734863, 12.399999618530273, 5.599999904632568, 3.799999952316284, 11.800000190734863, 15.59999942779541, 10.800000190734863, 12.799999237060547, 13.0, 13.799999237060547, 10.59999942779541, 15.199999809265137, 12.0, 16.19999885559082] +k eval/a1:test_return:Overcooked-CounterCircuit6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [18.399999618530273, 16.799999237060547, 13.399999618530273, 15.59999942779541, 7.0, 8.0, 19.799999237060547, 21.600000381469727, 20.399999618530273, 18.799999237060547, 13.799999237060547, 13.799999237060547, 15.0, 13.399999618530273, 21.600000381469727, 15.59999942779541] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [7.0, 10.59999942779541, 3.799999952316284, 4.400000095367432, 11.199999809265137, 7.199999809265137, 11.800000190734863, 12.799999237060547, 8.199999809265137, 8.800000190734863, 8.0, 11.0, 11.0, 9.800000190734863, 11.800000190734863, 9.59999942779541] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +----------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 12.41+/- 0.6386 (max: 21.6) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 9.187+/- 0.6554 (max: 12.8) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 12.22+/- 0.8788 (max: 16.4) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 15.81+/- 1.08 (max: 21.6) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 12.17+/- 0.3143 (max: 18.91) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 10.86+/- 0.354 (max: 13.59) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 11.73+/- 0.4026 (max: 13.97) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 13.91+/- 0.5584 (max: 18.91) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.08708+/- 0.01136 (max: 0.33) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0325+/- 0.007444 (max: 0.11) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.06812+/- 0.01212 (max: 0.17) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.1606+/- 0.02067 (max: 0.33) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 3.8 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 3.8 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 3.8 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 7.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 7.846 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 7.846 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 7.846 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 9.539 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +----------------------------------------------------------------------------------------------------- +Evaluating PAIRED_SoftMoE_SEED1 against population in Overcooked-AsymmAdvantages6_9 for xpid paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [1.7999999523162842, 37.79999923706055, 0.5999999642372131, 49.0, 1.5999999046325684, 80.0, 1.5999999046325684, 38.0, 0.5999999642372131, 74.79999542236328, 1.1999999284744263, 61.0, 1.7999999523162842, 70.19999694824219, 1.7999999523162842, 84.79999542236328, 1.399999976158142, 38.599998474121094, 1.0, 58.39999771118164, 3.5999999046325684, 63.79999923706055, 3.3999998569488525, 29.399999618530273, 0.5999999642372131, 30.599998474121094, 0.3999999761581421, 57.39999771118164, 0.7999999523162842, 51.39999771118164, 1.0, 36.20000076293945, 0.7999999523162842, 71.19999694824219, 1.1999999284744263, 66.79999542236328, 0.5999999642372131, 32.0, 0.5999999642372131, 73.0, 1.0, 29.0, 1.0, 36.20000076293945, 0.19999998807907104, 68.5999984741211, 3.5999999046325684, 55.39999771118164] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [1.7999999523162842, 37.79999923706055, 1.5999999046325684, 38.0, 1.7999999523162842, 70.19999694824219, 1.0, 58.39999771118164, 0.5999999642372131, 30.599998474121094, 1.0, 36.20000076293945, 0.5999999642372131, 32.0, 1.0, 36.20000076293945] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9, v [0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.5999999642372131, 49.0, 0.5999999642372131, 74.79999542236328, 1.7999999523162842, 84.79999542236328, 3.5999999046325684, 63.79999923706055, 0.3999999761581421, 57.39999771118164, 0.7999999523162842, 71.19999694824219, 0.5999999642372131, 73.0, 0.19999998807907104, 68.5999984741211] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [1.5999999046325684, 80.0, 1.1999999284744263, 61.0, 1.399999976158142, 38.599998474121094, 3.3999998569488525, 29.399999618530273, 0.7999999523162842, 51.39999771118164, 1.1999999284744263, 66.79999542236328, 1.0, 29.0, 3.5999999046325684, 55.39999771118164] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high, v [0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131] +----------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 27.62+/- 4.231 (max: 84.8) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 26.61+/- 7.127 (max: 80.0) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 21.8+/- 5.846 (max: 70.2) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 34.45+/- 8.824 (max: 84.8) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 17.5+/- 2.017 (max: 42.51) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 20.13+/- 4.016 (max: 42.51) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 14.07+/- 2.526 (max: 30.53) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 18.3+/- 3.803 (max: 41.32) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.3604+/- 0.05515 (max: 0.95) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.3331+/- 0.09095 (max: 0.88) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.3212+/- 0.0873 (max: 0.91) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.4269+/- 0.1107 (max: 0.95) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9 | 0.3+/- 0.04376 (max: 0.6) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high | 0.3+/- 0.07746 (max: 0.6) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low | 0.3+/- 0.07746 (max: 0.6) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.3+/- 0.07746 (max: 0.6) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9 | 1.706+/- 0.2488 (max: 3.412) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:high | 1.706+/- 0.4405 (max: 3.412) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:low | 1.706+/- 0.4405 (max: 3.412) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 1.706+/- 0.4405 (max: 3.412) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.2 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.8 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 0.6 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.2 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 1.99 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 3.919 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 3.412 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 1.99 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +----------------------------------------------------------------------------------------------------- +Evaluating PAIRED_SoftMoE_SEED1 against population in Overcooked-CrampedRoom6_9 for xpid paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [31.19999885559082, 28.799999237060547, 90.19999694824219, 94.79999542236328, 119.79999542236328, 125.39999389648438, 37.79999923706055, 38.39999771118164, 87.5999984741211, 94.0, 110.19999694824219, 111.5999984741211, 42.39999771118164, 37.39999771118164, 103.79999542236328, 106.39999389648438, 109.79999542236328, 112.19999694824219, 41.39999771118164, 41.20000076293945, 91.4000015258789, 96.0, 105.19999694824219, 103.79999542236328, 25.599998474121094, 26.0, 89.5999984741211, 88.4000015258789, 109.79999542236328, 113.19999694824219, 41.0, 39.599998474121094, 95.79999542236328, 103.39999389648438, 115.79999542236328, 106.19999694824219, 40.79999923706055, 40.39999771118164, 99.4000015258789, 97.5999984741211, 110.5999984741211, 117.19999694824219, 43.39999771118164, 43.599998474121094, 98.79999542236328, 95.0, 108.0, 111.79999542236328] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [31.19999885559082, 28.799999237060547, 37.79999923706055, 38.39999771118164, 42.39999771118164, 37.39999771118164, 41.39999771118164, 41.20000076293945, 25.599998474121094, 26.0, 41.0, 39.599998474121094, 40.79999923706055, 40.39999771118164, 43.39999771118164, 43.599998474121094] +k eval/a1:test_return:Overcooked-CrampedRoom6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [90.19999694824219, 94.79999542236328, 87.5999984741211, 94.0, 103.79999542236328, 106.39999389648438, 91.4000015258789, 96.0, 89.5999984741211, 88.4000015258789, 95.79999542236328, 103.39999389648438, 99.4000015258789, 97.5999984741211, 98.79999542236328, 95.0] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [119.79999542236328, 125.39999389648438, 110.19999694824219, 111.5999984741211, 109.79999542236328, 112.19999694824219, 105.19999694824219, 103.79999542236328, 109.79999542236328, 113.19999694824219, 115.79999542236328, 106.19999694824219, 110.5999984741211, 117.19999694824219, 108.0, 111.79999542236328] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 81.7+/- 4.736 (max: 125.4) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 111.9+/- 1.387 (max: 125.4) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 37.44+/- 1.515 (max: 43.6) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 95.76+/- 1.407 (max: 106.4) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 27.63+/- 0.8489 (max: 40.17) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 33.46+/- 1.016 (max: 40.17) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 21.09+/- 0.447 (max: 23.66) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 28.35+/- 0.6247 (max: 34.93) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.8594+/- 0.02583 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.98+/- 0.004564 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.6244+/- 0.02644 (max: 0.74) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.9737+/- 0.005313 (max: 1.0) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 25.6 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 103.8 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 25.6 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 87.6 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 18.44 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 27.13 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 18.44 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 25.24 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.41 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.94 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.41 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.92 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------- +Evaluating PAIRED_SoftMoE_SEED2 against population in Overcooked-CoordRing6_9 for xpid SEED_2_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [9.399999618530273, 11.800000190734863, 28.19999885559082, 27.0, 23.0, 28.19999885559082, 19.600000381469727, 23.0, 39.0, 44.599998474121094, 22.799999237060547, 34.599998474121094, 20.19999885559082, 18.0, 32.0, 37.79999923706055, 26.0, 33.599998474121094, 21.19999885559082, 22.799999237060547, 23.0, 26.799999237060547, 13.59999942779541, 28.799999237060547, 21.0, 22.799999237060547, 34.79999923706055, 37.0, 27.19999885559082, 35.39999771118164, 16.600000381469727, 20.600000381469727, 26.19999885559082, 31.19999885559082, 30.19999885559082, 30.19999885559082, 6.599999904632568, 9.0, 22.399999618530273, 25.399999618530273, 21.19999885559082, 24.19999885559082, 21.799999237060547, 20.399999618530273, 21.600000381469727, 22.600000381469727, 17.19999885559082, 16.19999885559082] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [9.399999618530273, 11.800000190734863, 19.600000381469727, 23.0, 20.19999885559082, 18.0, 21.19999885559082, 22.799999237060547, 21.0, 22.799999237060547, 16.600000381469727, 20.600000381469727, 6.599999904632568, 9.0, 21.799999237060547, 20.399999618530273] +k eval/a1:test_return:Overcooked-CoordRing6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CoordRing6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [28.19999885559082, 27.0, 39.0, 44.599998474121094, 32.0, 37.79999923706055, 23.0, 26.799999237060547, 34.79999923706055, 37.0, 26.19999885559082, 31.19999885559082, 22.399999618530273, 25.399999618530273, 21.600000381469727, 22.600000381469727] +k eval/a1:test_return:Overcooked-CoordRing6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [23.0, 28.19999885559082, 22.799999237060547, 34.599998474121094, 26.0, 33.599998474121094, 13.59999942779541, 28.799999237060547, 27.19999885559082, 35.39999771118164, 30.19999885559082, 30.19999885559082, 21.19999885559082, 24.19999885559082, 17.19999885559082, 16.19999885559082] +k eval/a1:test_return:Overcooked-CoordRing6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +----------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 24.52+/- 1.16 (max: 44.6) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 25.77+/- 1.635 (max: 35.4) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 17.8+/- 1.368 (max: 23.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 29.97+/- 1.735 (max: 44.6) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 17.12+/- 0.4028 (max: 24.08) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 17.2+/- 0.6717 (max: 24.08) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 15.23+/- 0.5825 (max: 17.67) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 18.94+/- 0.5247 (max: 21.88) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.3633+/- 0.02498 (max: 0.75) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.3931+/- 0.03747 (max: 0.65) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.2212+/- 0.02772 (max: 0.35) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.4756+/- 0.03807 (max: 0.75) | +| eval/a1:test_return:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 6.6 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 13.6 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 6.6 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 21.6 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 10.22 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 13.86 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 10.22 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 15.91 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.16 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.27 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0 | +----------------------------------------------------------------------------------------------- +Evaluating PAIRED_SoftMoE_SEED2 against population in Overcooked-ForcedCoord6_9 for xpid SEED_2_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [4.799999713897705, 0.7999999523162842, 5.799999713897705, 0.5999999642372131, 2.3999998569488525, 0.19999998807907104, 2.0, 0.0, 1.5999999046325684, 0.19999998807907104, 1.5999999046325684, 0.0, 3.5999999046325684, 1.5999999046325684, 3.3999998569488525, 0.3999999761581421, 3.1999998092651367, 1.1999999284744263, 3.3999998569488525, 0.5999999642372131, 2.200000047683716, 0.0, 2.0, 0.0, 4.599999904632568, 1.0, 5.599999904632568, 1.5999999046325684, 4.799999713897705, 0.3999999761581421, 2.3999998569488525, 0.3999999761581421, 2.3999998569488525, 0.3999999761581421, 0.7999999523162842, 0.19999998807907104, 1.7999999523162842, 0.19999998807907104, 2.0, 0.3999999761581421, 1.0, 0.3999999761581421, 3.0, 0.19999998807907104, 3.799999952316284, 0.19999998807907104, 7.0, 0.3999999761581421] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [4.799999713897705, 0.7999999523162842, 2.0, 0.0, 3.5999999046325684, 1.5999999046325684, 3.3999998569488525, 0.5999999642372131, 4.599999904632568, 1.0, 2.3999998569488525, 0.3999999761581421, 1.7999999523162842, 0.19999998807907104, 3.0, 0.19999998807907104] +k eval/a1:test_return:Overcooked-ForcedCoord6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [5.799999713897705, 0.5999999642372131, 1.5999999046325684, 0.19999998807907104, 3.3999998569488525, 0.3999999761581421, 2.200000047683716, 0.0, 5.599999904632568, 1.5999999046325684, 2.3999998569488525, 0.3999999761581421, 2.0, 0.3999999761581421, 3.799999952316284, 0.19999998807907104] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [2.3999998569488525, 0.19999998807907104, 1.5999999046325684, 0.0, 3.1999998092651367, 1.1999999284744263, 2.0, 0.0, 4.799999713897705, 0.3999999761581421, 0.7999999523162842, 0.19999998807907104, 1.0, 0.3999999761581421, 7.0, 0.3999999761581421] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +---------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 1.804+/- 0.2571 (max: 7.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 1.6+/- 0.4885 (max: 7.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 1.9+/- 0.3967 (max: 4.8) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 1.912+/- 0.4704 (max: 5.8) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 4.829+/- 0.3967 (max: 10.72) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 4.404+/- 0.7514 (max: 10.72) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 5.159+/- 0.6541 (max: 8.879) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 4.926+/- 0.6835 (max: 9.075) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.00125+/- 0.0007062 (max: 0.03) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0025+/- 0.001936 (max: 0.03) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.00125+/- 0.0008539 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +---------------------------------------------------------------------------------------------------- +Evaluating PAIRED_SoftMoE_SEED2 against population in Overcooked-CounterCircuit6_9 for xpid SEED_2_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [15.59999942779541, 9.399999618530273, 22.19999885559082, 14.59999942779541, 11.59999942779541, 11.59999942779541, 7.799999713897705, 9.800000190734863, 13.799999237060547, 12.0, 4.799999713897705, 1.7999999523162842, 8.0, 5.0, 13.59999942779541, 10.0, 10.800000190734863, 11.0, 11.399999618530273, 8.59999942779541, 24.399999618530273, 21.799999237060547, 12.199999809265137, 6.799999713897705, 9.399999618530273, 7.799999713897705, 16.0, 13.799999237060547, 8.199999809265137, 5.0, 14.799999237060547, 8.199999809265137, 20.0, 11.199999809265137, 8.800000190734863, 7.399999618530273, 10.399999618530273, 9.0, 20.399999618530273, 15.199999809265137, 12.399999618530273, 8.59999942779541, 13.199999809265137, 8.800000190734863, 20.0, 17.799999237060547, 11.0, 5.799999713897705] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [15.59999942779541, 9.399999618530273, 7.799999713897705, 9.800000190734863, 8.0, 5.0, 11.399999618530273, 8.59999942779541, 9.399999618530273, 7.799999713897705, 14.799999237060547, 8.199999809265137, 10.399999618530273, 9.0, 13.199999809265137, 8.800000190734863] +k eval/a1:test_return:Overcooked-CounterCircuit6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [22.19999885559082, 14.59999942779541, 13.799999237060547, 12.0, 13.59999942779541, 10.0, 24.399999618530273, 21.799999237060547, 16.0, 13.799999237060547, 20.0, 11.199999809265137, 20.399999618530273, 15.199999809265137, 20.0, 17.799999237060547] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [11.59999942779541, 11.59999942779541, 4.799999713897705, 1.7999999523162842, 10.800000190734863, 11.0, 12.199999809265137, 6.799999713897705, 8.199999809265137, 5.0, 8.800000190734863, 7.399999618530273, 12.399999618530273, 8.59999942779541, 11.0, 5.799999713897705] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 11.7+/- 0.7137 (max: 24.4) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 8.612+/- 0.7856 (max: 12.4) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 9.825+/- 0.6872 (max: 15.6) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 16.67+/- 1.086 (max: 24.4) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 12.1+/- 0.3574 (max: 18.74) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 10.57+/- 0.4904 (max: 14.54) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 11.2+/- 0.2517 (max: 12.83) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 14.52+/- 0.5448 (max: 18.74) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.08729+/- 0.01393 (max: 0.34) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.03062+/- 0.007498 (max: 0.1) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.04062+/- 0.007442 (max: 0.12) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.1906+/- 0.02534 (max: 0.34) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 1.8 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 1.8 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 5.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 10.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 5.724 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 5.724 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 8.66 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 11.27 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.04 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating PAIRED_SoftMoE_SEED2 against population in Overcooked-AsymmAdvantages6_9 for xpid SEED_2_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [18.19999885559082, 8.199999809265137, 6.0, 17.799999237060547, 36.79999923706055, 3.799999952316284, 19.600000381469727, 12.59999942779541, 24.799999237060547, 46.39999771118164, 26.19999885559082, 2.5999999046325684, 26.799999237060547, 37.20000076293945, 30.19999885559082, 64.79999542236328, 37.20000076293945, 15.799999237060547, 19.19999885559082, 22.0, 23.600000381469727, 11.199999809265137, 39.20000076293945, 6.599999904632568, 22.0, 8.59999942779541, 19.799999237060547, 20.600000381469727, 15.59999942779541, 8.399999618530273, 21.0, 6.799999713897705, 18.0, 31.799999237060547, 37.0, 3.3999998569488525, 20.600000381469727, 8.199999809265137, 25.19999885559082, 16.399999618530273, 46.39999771118164, 7.399999618530273, 17.0, 10.399999618530273, 10.399999618530273, 25.0, 35.39999771118164, 6.599999904632568] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [18.19999885559082, 8.199999809265137, 19.600000381469727, 12.59999942779541, 26.799999237060547, 37.20000076293945, 19.19999885559082, 22.0, 22.0, 8.59999942779541, 21.0, 6.799999713897705, 20.600000381469727, 8.199999809265137, 17.0, 10.399999618530273] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9, v [1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low, v [1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [6.0, 17.799999237060547, 24.799999237060547, 46.39999771118164, 30.19999885559082, 64.79999542236328, 23.600000381469727, 11.199999809265137, 19.799999237060547, 20.600000381469727, 18.0, 31.799999237060547, 25.19999885559082, 16.399999618530273, 10.399999618530273, 25.0] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid, v [1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [36.79999923706055, 3.799999952316284, 26.19999885559082, 2.5999999046325684, 37.20000076293945, 15.799999237060547, 39.20000076293945, 6.599999904632568, 15.59999942779541, 8.399999618530273, 37.0, 3.3999998569488525, 46.39999771118164, 7.399999618530273, 35.39999771118164, 6.599999904632568] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high, v [1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.19999998807907104] +----------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 20.81+/- 1.909 (max: 64.8) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 20.52+/- 3.944 (max: 46.4) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 17.4+/- 2.027 (max: 37.2) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 24.5+/- 3.593 (max: 64.8) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 16.82+/- 0.7477 (max: 31.89) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 15.61+/- 1.344 (max: 23.13) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 15.42+/- 1.057 (max: 28.99) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 19.44+/- 1.284 (max: 31.89) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.2815+/- 0.033 (max: 0.85) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.3006+/- 0.07319 (max: 0.75) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.2075+/- 0.03668 (max: 0.54) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.3362+/- 0.05425 (max: 0.85) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9 | 0.6+/- 0.05835 (max: 1.0) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high | 0.6+/- 0.1033 (max: 1.0) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low | 0.6+/- 0.1033 (max: 1.0) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.6+/- 0.1033 (max: 1.0) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9 | 3.174+/- 0.1728 (max: 4.359) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:high | 3.174+/- 0.3058 (max: 4.359) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:low | 3.174+/- 0.3058 (max: 4.359) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 3.174+/- 0.3058 (max: 4.359) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 2.6 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 2.6 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 6.8 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 6.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 7.297 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 7.297 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 9.887 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 10.77 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.01 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.01 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.01 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.04 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9 | 0.2 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high | 0.2 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low | 0.2 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.2 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9 | 1.99 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:high | 1.99 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:low | 1.99 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 1.99 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +----------------------------------------------------------------------------------------------------- +Evaluating PAIRED_SoftMoE_SEED2 against population in Overcooked-CrampedRoom6_9 for xpid SEED_2_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [21.600000381469727, 15.799999237060547, 85.4000015258789, 87.0, 112.19999694824219, 116.39999389648438, 29.599998474121094, 26.399999618530273, 80.19999694824219, 80.5999984741211, 106.79999542236328, 105.19999694824219, 29.0, 27.399999618530273, 104.39999389648438, 98.5999984741211, 111.19999694824219, 112.5999984741211, 25.19999885559082, 29.599998474121094, 94.5999984741211, 93.5999984741211, 97.19999694824219, 101.0, 18.19999885559082, 22.0, 84.19999694824219, 89.0, 112.0, 109.5999984741211, 31.0, 30.599998474121094, 96.0, 96.19999694824219, 107.19999694824219, 112.39999389648438, 30.0, 29.599998474121094, 92.5999984741211, 87.79999542236328, 109.0, 109.39999389648438, 33.39999771118164, 34.20000076293945, 99.79999542236328, 96.79999542236328, 106.19999694824219, 104.79999542236328] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [21.600000381469727, 15.799999237060547, 29.599998474121094, 26.399999618530273, 29.0, 27.399999618530273, 25.19999885559082, 29.599998474121094, 18.19999885559082, 22.0, 31.0, 30.599998474121094, 30.0, 29.599998474121094, 33.39999771118164, 34.20000076293945] +k eval/a1:test_return:Overcooked-CrampedRoom6_9, v [0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:low, v [0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [85.4000015258789, 87.0, 80.19999694824219, 80.5999984741211, 104.39999389648438, 98.5999984741211, 94.5999984741211, 93.5999984741211, 84.19999694824219, 89.0, 96.0, 96.19999694824219, 92.5999984741211, 87.79999542236328, 99.79999542236328, 96.79999542236328] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:mid, v [0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [112.19999694824219, 116.39999389648438, 106.79999542236328, 105.19999694824219, 111.19999694824219, 112.5999984741211, 97.19999694824219, 101.0, 112.0, 109.5999984741211, 107.19999694824219, 112.39999389648438, 109.0, 109.39999389648438, 106.19999694824219, 104.79999542236328] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:high, v [0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.3999999761581421] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 75.7+/- 5.175 (max: 116.4) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 108.3+/- 1.204 (max: 116.4) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 27.1+/- 1.316 (max: 34.2) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 91.67+/- 1.765 (max: 104.4) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 25.4+/- 0.8028 (max: 39.24) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 28.66+/- 0.9033 (max: 36.0) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 19.05+/- 0.4251 (max: 22.21) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 28.49+/- 1.016 (max: 39.24) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.7927+/- 0.03853 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.9812+/- 0.005692 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.4331+/- 0.02993 (max: 0.57) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.9637+/- 0.009349 (max: 1.0) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9 | 0.2+/- 0.02917 (max: 0.4) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:high | 0.2+/- 0.05164 (max: 0.4) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:low | 0.2+/- 0.05164 (max: 0.4) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:mid | 0.2+/- 0.05164 (max: 0.4) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9 | 1.4+/- 0.2042 (max: 2.8) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:high | 1.4+/- 0.3615 (max: 2.8) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:low | 1.4+/- 0.3615 (max: 2.8) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:mid | 1.4+/- 0.3615 (max: 2.8) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 15.8 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 97.2 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 15.8 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 80.2 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 15.52 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 22.51 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 15.52 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 22.95 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.2 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.92 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.2 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.85 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------- +Evaluating PAIRED_SoftMoE_SEED3 against population in Overcooked-CoordRing6_9 for xpid SEED_3_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [12.399999618530273, 11.0, 41.20000076293945, 36.0, 35.79999923706055, 40.39999771118164, 25.799999237060547, 26.399999618530273, 48.599998474121094, 53.39999771118164, 43.20000076293945, 48.0, 25.799999237060547, 24.600000381469727, 48.20000076293945, 49.79999923706055, 50.39999771118164, 54.79999923706055, 38.0, 35.20000076293945, 38.599998474121094, 37.0, 34.79999923706055, 38.0, 24.19999885559082, 28.799999237060547, 37.79999923706055, 40.0, 46.599998474121094, 49.599998474121094, 23.0, 24.799999237060547, 44.0, 43.79999923706055, 33.79999923706055, 43.39999771118164, 9.0, 7.599999904632568, 35.20000076293945, 27.399999618530273, 31.0, 34.599998474121094, 20.0, 24.600000381469727, 24.19999885559082, 22.799999237060547, 23.799999237060547, 22.600000381469727] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [12.399999618530273, 11.0, 25.799999237060547, 26.399999618530273, 25.799999237060547, 24.600000381469727, 38.0, 35.20000076293945, 24.19999885559082, 28.799999237060547, 23.0, 24.799999237060547, 9.0, 7.599999904632568, 20.0, 24.600000381469727] +k eval/a1:test_return:Overcooked-CoordRing6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CoordRing6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [41.20000076293945, 36.0, 48.599998474121094, 53.39999771118164, 48.20000076293945, 49.79999923706055, 38.599998474121094, 37.0, 37.79999923706055, 40.0, 44.0, 43.79999923706055, 35.20000076293945, 27.399999618530273, 24.19999885559082, 22.799999237060547] +k eval/a1:test_return:Overcooked-CoordRing6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [35.79999923706055, 40.39999771118164, 43.20000076293945, 48.0, 50.39999771118164, 54.79999923706055, 34.79999923706055, 38.0, 46.599998474121094, 49.599998474121094, 33.79999923706055, 43.39999771118164, 31.0, 34.599998474121094, 23.799999237060547, 22.600000381469727] +k eval/a1:test_return:Overcooked-CoordRing6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +----------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 33.75+/- 1.716 (max: 54.8) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 39.42+/- 2.333 (max: 54.8) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 22.57+/- 2.18 (max: 38.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 39.25+/- 2.235 (max: 53.4) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 16.77+/- 0.4238 (max: 25.72) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 19.48+/- 0.748 (max: 25.72) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 14.93+/- 0.4341 (max: 17.04) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 15.89+/- 0.4128 (max: 18.76) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.5819+/- 0.03686 (max: 0.95) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.68+/- 0.04593 (max: 0.93) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.3444+/- 0.05052 (max: 0.73) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.7212+/- 0.0491 (max: 0.95) | +| eval/a1:test_return:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 7.6 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 22.6 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 7.6 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 22.8 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 11.79 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 14.55 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 11.79 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 13.11 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.05 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.31 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.05 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.32 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CoordRing6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.0 | +----------------------------------------------------------------------------------------------- +Evaluating PAIRED_SoftMoE_SEED3 against population in Overcooked-ForcedCoord6_9 for xpid SEED_3_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [0.19999998807907104, 1.0, 0.19999998807907104, 2.5999999046325684, 0.0, 1.0, 0.0, 3.0, 0.19999998807907104, 2.799999952316284, 0.0, 0.7999999523162842, 0.3999999761581421, 2.3999998569488525, 0.19999998807907104, 3.0, 0.0, 3.0, 0.0, 2.0, 0.3999999761581421, 1.1999999284744263, 0.19999998807907104, 1.0, 0.19999998807907104, 1.1999999284744263, 0.5999999642372131, 0.7999999523162842, 0.7999999523162842, 0.19999998807907104, 0.0, 3.0, 0.0, 2.0, 0.0, 2.5999999046325684, 0.0, 2.0, 0.19999998807907104, 1.0, 0.0, 0.5999999642372131, 0.0, 1.5999999046325684, 0.0, 2.5999999046325684, 0.0, 0.5999999642372131] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [0.19999998807907104, 1.0, 0.0, 3.0, 0.3999999761581421, 2.3999998569488525, 0.0, 2.0, 0.19999998807907104, 1.1999999284744263, 0.0, 3.0, 0.0, 2.0, 0.0, 1.5999999046325684] +k eval/a1:test_return:Overcooked-ForcedCoord6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [0.19999998807907104, 2.5999999046325684, 0.19999998807907104, 2.799999952316284, 0.19999998807907104, 3.0, 0.3999999761581421, 1.1999999284744263, 0.5999999642372131, 0.7999999523162842, 0.0, 2.0, 0.19999998807907104, 1.0, 0.0, 2.5999999046325684] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [0.0, 1.0, 0.0, 0.7999999523162842, 0.0, 3.0, 0.19999998807907104, 1.0, 0.7999999523162842, 0.19999998807907104, 0.0, 2.5999999046325684, 0.0, 0.5999999642372131, 0.0, 0.5999999642372131] +k eval/a1:test_return:Overcooked-ForcedCoord6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.95+/- 0.1513 (max: 3.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.675+/- 0.2287 (max: 3.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 1.062+/- 0.2809 (max: 3.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 1.112+/- 0.2763 (max: 3.0) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 3.264+/- 0.3728 (max: 7.141) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 2.577+/- 0.6121 (max: 7.141) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 3.421+/- 0.7182 (max: 7.141) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 3.796+/- 0.6033 (max: 7.141) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0002083+/- 0.0002083 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.000625+/- 0.000625 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating PAIRED_SoftMoE_SEED3 against population in Overcooked-CounterCircuit6_9 for xpid SEED_3_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [3.5999999046325684, 2.5999999046325684, 8.59999942779541, 3.5999999046325684, 16.19999885559082, 13.399999618530273, 1.7999999523162842, 0.5999999642372131, 6.199999809265137, 2.5999999046325684, 9.59999942779541, 4.599999904632568, 12.59999942779541, 13.399999618530273, 14.399999618530273, 9.59999942779541, 33.79999923706055, 42.79999923706055, 6.599999904632568, 3.799999952316284, 13.0, 8.199999809265137, 15.399999618530273, 8.800000190734863, 5.199999809265137, 3.799999952316284, 18.0, 12.399999618530273, 15.399999618530273, 9.0, 8.399999618530273, 5.599999904632568, 11.800000190734863, 6.0, 13.0, 7.0, 3.0, 0.0, 4.799999713897705, 1.0, 14.799999237060547, 9.800000190734863, 6.0, 3.0, 5.799999713897705, 3.1999998092651367, 11.199999809265137, 8.59999942779541] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [3.5999999046325684, 2.5999999046325684, 1.7999999523162842, 0.5999999642372131, 12.59999942779541, 13.399999618530273, 6.599999904632568, 3.799999952316284, 5.199999809265137, 3.799999952316284, 8.399999618530273, 5.599999904632568, 3.0, 0.0, 6.0, 3.0] +k eval/a1:test_return:Overcooked-CounterCircuit6_9, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:low, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [8.59999942779541, 3.5999999046325684, 6.199999809265137, 2.5999999046325684, 14.399999618530273, 9.59999942779541, 13.0, 8.199999809265137, 18.0, 12.399999618530273, 11.800000190734863, 6.0, 4.799999713897705, 1.0, 5.799999713897705, 3.1999998092651367] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:mid, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [16.19999885559082, 13.399999618530273, 9.59999942779541, 4.599999904632568, 33.79999923706055, 42.79999923706055, 15.399999618530273, 8.800000190734863, 15.399999618530273, 9.0, 13.0, 7.0, 14.799999237060547, 9.800000190734863, 11.199999809265137, 8.59999942779541] +k eval/a1:test_return:Overcooked-CounterCircuit6_9:high, v [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +----------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 9.221+/- 1.118 (max: 42.8) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 14.59+/- 2.492 (max: 42.8) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 5.0+/- 0.951 (max: 13.4) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 8.075+/- 1.2 (max: 18.0) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 10.17+/- 0.4812 (max: 19.74) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 11.86+/- 0.7161 (max: 19.74) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 8.197+/- 0.8849 (max: 14.71) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 10.46+/- 0.6527 (max: 14.55) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.06312+/- 0.02012 (max: 0.77) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.12+/- 0.05657 (max: 0.77) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.02062+/- 0.01047 (max: 0.14) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.04875+/- 0.01214 (max: 0.16) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_return_std:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 4.6 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 1.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 8.879 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 4.359 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CounterCircuit6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +----------------------------------------------------------------------------------------------------- +Evaluating PAIRED_SoftMoE_SEED3 against population in Overcooked-AsymmAdvantages6_9 for xpid SEED_3_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [14.799999237060547, 23.19999885559082, 16.19999885559082, 32.0, 20.0, 34.39999771118164, 15.199999809265137, 20.0, 19.600000381469727, 58.599998474121094, 20.19999885559082, 31.19999885559082, 19.600000381469727, 51.0, 16.799999237060547, 71.5999984741211, 25.599998474121094, 16.399999618530273, 16.399999618530273, 37.20000076293945, 16.600000381469727, 34.0, 24.399999618530273, 17.0, 11.800000190734863, 18.799999237060547, 17.600000381469727, 35.20000076293945, 16.600000381469727, 23.799999237060547, 13.0, 21.600000381469727, 17.799999237060547, 47.20000076293945, 23.600000381469727, 25.399999618530273, 14.0, 16.399999618530273, 16.799999237060547, 40.79999923706055, 25.799999237060547, 18.0, 12.799999237060547, 23.399999618530273, 19.799999237060547, 42.79999923706055, 23.0, 27.799999237060547] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [14.799999237060547, 23.19999885559082, 15.199999809265137, 20.0, 19.600000381469727, 51.0, 16.399999618530273, 37.20000076293945, 11.800000190734863, 18.799999237060547, 13.0, 21.600000381469727, 14.0, 16.399999618530273, 12.799999237060547, 23.399999618530273] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9, v [0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [16.19999885559082, 32.0, 19.600000381469727, 58.599998474121094, 16.799999237060547, 71.5999984741211, 16.600000381469727, 34.0, 17.600000381469727, 35.20000076293945, 17.799999237060547, 47.20000076293945, 16.799999237060547, 40.79999923706055, 19.799999237060547, 42.79999923706055] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [20.0, 34.39999771118164, 20.19999885559082, 31.19999885559082, 25.599998474121094, 16.399999618530273, 24.399999618530273, 17.0, 16.600000381469727, 23.799999237060547, 23.600000381469727, 25.399999618530273, 25.799999237060547, 18.0, 23.0, 27.799999237060547] +k eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high, v [0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263, 0.0, 1.1999999284744263] +----------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 25.12+/- 1.806 (max: 71.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 23.32+/- 1.298 (max: 34.4) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 20.57+/- 2.548 (max: 51.0) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 31.46+/- 4.268 (max: 71.6) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 18.22+/- 0.8 (max: 34.55) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 17.81+/- 0.822 (max: 24.99) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 16.37+/- 1.116 (max: 27.48) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 20.47+/- 1.876 (max: 34.55) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.3327+/- 0.03005 (max: 0.87) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.3319+/- 0.02487 (max: 0.54) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.2475+/- 0.0495 (max: 0.76) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.4187+/- 0.06664 (max: 0.87) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9 | 0.6+/- 0.08752 (max: 1.2) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high | 0.6+/- 0.1549 (max: 1.2) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low | 0.6+/- 0.1549 (max: 1.2) | +| eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.6+/- 0.1549 (max: 1.2) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9 | 2.375+/- 0.3464 (max: 4.75) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:high | 2.375+/- 0.6132 (max: 4.75) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:low | 2.375+/- 0.6132 (max: 4.75) | +| eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 2.375+/- 0.6132 (max: 4.75) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 11.8 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 16.4 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 11.8 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 16.2 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 11.66 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 13.56 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 11.84 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 11.66 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.06 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.21 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.06 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.16 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +----------------------------------------------------------------------------------------------------- +Evaluating PAIRED_SoftMoE_SEED3 against population in Overcooked-CrampedRoom6_9 for xpid SEED_3_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [45.39999771118164, 47.0, 98.4000015258789, 95.19999694824219, 107.19999694824219, 114.79999542236328, 60.39999771118164, 59.19999694824219, 101.5999984741211, 106.5999984741211, 113.79999542236328, 115.19999694824219, 58.0, 54.0, 116.0, 117.5999984741211, 94.0, 104.79999542236328, 56.19999694824219, 58.39999771118164, 104.5999984741211, 96.5999984741211, 80.5999984741211, 93.0, 45.599998474121094, 48.39999771118164, 101.79999542236328, 100.4000015258789, 95.79999542236328, 117.79999542236328, 53.19999694824219, 50.599998474121094, 114.19999694824219, 107.39999389648438, 104.39999389648438, 109.39999389648438, 62.599998474121094, 61.79999923706055, 103.79999542236328, 104.0, 117.19999694824219, 112.39999389648438, 67.0, 60.0, 118.39999389648438, 109.5999984741211, 81.19999694824219, 96.4000015258789] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [45.39999771118164, 47.0, 60.39999771118164, 59.19999694824219, 58.0, 54.0, 56.19999694824219, 58.39999771118164, 45.599998474121094, 48.39999771118164, 53.19999694824219, 50.599998474121094, 62.599998474121094, 61.79999923706055, 67.0, 60.0] +k eval/a1:test_return:Overcooked-CrampedRoom6_9, v [0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:low, v [0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [98.4000015258789, 95.19999694824219, 101.5999984741211, 106.5999984741211, 116.0, 117.5999984741211, 104.5999984741211, 96.5999984741211, 101.79999542236328, 100.4000015258789, 114.19999694824219, 107.39999389648438, 103.79999542236328, 104.0, 118.39999389648438, 109.5999984741211] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:mid, v [0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [107.19999694824219, 114.79999542236328, 113.79999542236328, 115.19999694824219, 94.0, 104.79999542236328, 80.5999984741211, 93.0, 95.79999542236328, 117.79999542236328, 104.39999389648438, 109.39999389648438, 117.19999694824219, 112.39999389648438, 81.19999694824219, 96.4000015258789] +k eval/a1:test_return:Overcooked-CrampedRoom6_9:high, v [0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863, 0.0, 8.800000190734863] +-------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 88.37+/- 3.628 (max: 118.4) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 103.6+/- 3.045 (max: 117.8) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 55.49+/- 1.645 (max: 67.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 106.0+/- 1.84 (max: 118.4) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 30.95+/- 1.134 (max: 51.5) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 40.5+/- 1.371 (max: 51.5) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 24.7+/- 0.3532 (max: 27.13) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 27.65+/- 0.8123 (max: 32.75) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.9142+/- 0.01214 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.9225+/- 0.0158 (max: 0.99) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.8287+/- 0.01557 (max: 0.92) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.9912+/- 0.002394 (max: 1.0) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9 | 4.4+/- 0.6418 (max: 8.8) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:high | 4.4+/- 1.136 (max: 8.8) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:low | 4.4+/- 1.136 (max: 8.8) | +| eval/a1:test_return:Overcooked-CrampedRoom6_9:mid | 4.4+/- 1.136 (max: 8.8) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9 | 5.161+/- 0.7529 (max: 10.32) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:high | 5.161+/- 1.333 (max: 10.32) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:low | 5.161+/- 1.333 (max: 10.32) | +| eval/a1:test_return_std:Overcooked-CrampedRoom6_9:mid | 5.161+/- 1.333 (max: 10.32) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.005+/- 0.0007293 (max: 0.01) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.005+/- 0.001291 (max: 0.01) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.005+/- 0.001291 (max: 0.01) | +| eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.005+/- 0.001291 (max: 0.01) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 45.4 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 80.6 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 45.4 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 95.2 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 21.19 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 31.24 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 21.24 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 21.19 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.71 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.78 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.71 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.97 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_return:Overcooked-CrampedRoom6_9:mid | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_return_std:Overcooked-CrampedRoom6_9:mid | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.0 | +| min:eval/a1:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.0 | +-------------------------------------------------------------------------------------------------- +Evaluating ACCEL_SoftMoE_SEED1 against population in Overcooked-CoordRing6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [4.799999713897705, 2.3999998569488525, 19.0, 18.600000381469727, 24.600000381469727, 25.799999237060547, 15.0, 14.0, 29.399999618530273, 33.0, 26.399999618530273, 29.0, 12.799999237060547, 12.199999809265137, 24.799999237060547, 28.0, 28.19999885559082, 30.0, 14.799999237060547, 11.199999809265137, 18.799999237060547, 14.59999942779541, 23.0, 22.399999618530273, 16.19999885559082, 15.59999942779541, 20.399999618530273, 22.799999237060547, 28.0, 31.0, 14.59999942779541, 12.0, 25.0, 20.0, 38.0, 39.39999771118164, 3.1999998092651367, 4.400000095367432, 19.0, 19.19999885559082, 36.0, 34.39999771118164, 15.399999618530273, 13.799999237060547, 16.399999618530273, 13.799999237060547, 25.599998474121094, 25.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [4.799999713897705, 2.3999998569488525, 15.0, 14.0, 12.799999237060547, 12.199999809265137, 14.799999237060547, 11.199999809265137, 16.19999885559082, 15.59999942779541, 14.59999942779541, 12.0, 3.1999998092651367, 4.400000095367432, 15.399999618530273, 13.799999237060547] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [19.0, 18.600000381469727, 29.399999618530273, 33.0, 24.799999237060547, 28.0, 18.799999237060547, 14.59999942779541, 20.399999618530273, 22.799999237060547, 25.0, 20.0, 19.0, 19.19999885559082, 16.399999618530273, 13.799999237060547] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [24.600000381469727, 25.799999237060547, 26.399999618530273, 29.0, 28.19999885559082, 30.0, 23.0, 22.399999618530273, 28.0, 31.0, 38.0, 39.39999771118164, 36.0, 34.39999771118164, 25.599998474121094, 25.0] +----------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 20.67+/- 1.287 (max: 39.4) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 29.17+/- 1.318 (max: 39.4) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 11.4+/- 1.204 (max: 16.2) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 21.42+/- 1.341 (max: 33.0) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 13.38+/- 0.3369 (max: 16.97) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 13.57+/- 0.5546 (max: 16.97) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 12.36+/- 0.7336 (max: 15.15) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 14.21+/- 0.3214 (max: 16.84) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.2894+/- 0.02954 (max: 0.83) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.4837+/- 0.04042 (max: 0.83) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.1019+/- 0.01708 (max: 0.19) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.2825+/- 0.03753 (max: 0.65) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 2.4 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 22.4 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 2.4 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 13.8 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 6.499 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 10.2 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 6.499 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 11.79 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.31 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.08 | +----------------------------------------------------------------------------------------------- +Evaluating ACCEL_SoftMoE_SEED1 against population in Overcooked-ForcedCoord6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [0.7999999523162842, 0.19999998807907104, 0.5999999642372131, 0.19999998807907104, 0.0, 0.19999998807907104, 1.5999999046325684, 0.3999999761581421, 1.399999976158142, 1.0, 0.3999999761581421, 0.3999999761581421, 0.5999999642372131, 0.0, 1.0, 0.0, 0.3999999761581421, 0.0, 1.5999999046325684, 0.0, 1.399999976158142, 0.5999999642372131, 0.0, 0.3999999761581421, 0.3999999761581421, 0.0, 1.399999976158142, 0.19999998807907104, 1.0, 0.19999998807907104, 1.0, 0.0, 1.0, 0.3999999761581421, 0.3999999761581421, 0.0, 0.3999999761581421, 0.0, 0.19999998807907104, 0.19999998807907104, 0.7999999523162842, 0.19999998807907104, 1.1999999284744263, 0.19999998807907104, 0.7999999523162842, 0.5999999642372131, 1.5999999046325684, 0.5999999642372131] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [0.7999999523162842, 0.19999998807907104, 1.5999999046325684, 0.3999999761581421, 0.5999999642372131, 0.0, 1.5999999046325684, 0.0, 0.3999999761581421, 0.0, 1.0, 0.0, 0.3999999761581421, 0.0, 1.1999999284744263, 0.19999998807907104] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [0.5999999642372131, 0.19999998807907104, 1.399999976158142, 1.0, 1.0, 0.0, 1.399999976158142, 0.5999999642372131, 1.399999976158142, 0.19999998807907104, 1.0, 0.3999999761581421, 0.19999998807907104, 0.19999998807907104, 0.7999999523162842, 0.5999999642372131] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [0.0, 0.19999998807907104, 0.3999999761581421, 0.3999999761581421, 0.3999999761581421, 0.0, 0.0, 0.3999999761581421, 1.0, 0.19999998807907104, 0.3999999761581421, 0.0, 0.7999999523162842, 0.19999998807907104, 1.5999999046325684, 0.5999999642372131] +------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.5417+/- 0.0712 (max: 1.6) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.4125+/- 0.1072 (max: 1.6) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.525+/- 0.1401 (max: 1.6) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.6875+/- 0.1183 (max: 1.4) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 2.71+/- 0.251 (max: 5.426) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 2.318+/- 0.4111 (max: 5.426) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 2.479+/- 0.5021 (max: 5.426) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 3.331+/- 0.3624 (max: 5.103) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0+/- 0.0 (max: 0.0) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------ +Evaluating ACCEL_SoftMoE_SEED1 against population in Overcooked-CounterCircuit6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [9.800000190734863, 7.599999904632568, 13.199999809265137, 12.399999618530273, 12.59999942779541, 12.799999237060547, 6.399999618530273, 8.0, 11.199999809265137, 9.59999942779541, 3.0, 1.399999976158142, 5.599999904632568, 3.3999998569488525, 14.0, 12.399999618530273, 16.600000381469727, 9.0, 11.399999618530273, 8.199999809265137, 20.0, 17.0, 11.399999618530273, 6.0, 10.399999618530273, 8.800000190734863, 21.19999885559082, 18.0, 11.59999942779541, 6.199999809265137, 11.800000190734863, 12.199999809265137, 18.799999237060547, 12.399999618530273, 11.199999809265137, 5.599999904632568, 9.199999809265137, 5.199999809265137, 13.799999237060547, 11.199999809265137, 6.799999713897705, 6.399999618530273, 11.800000190734863, 8.59999942779541, 14.799999237060547, 15.199999809265137, 9.0, 4.199999809265137] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [9.800000190734863, 7.599999904632568, 6.399999618530273, 8.0, 5.599999904632568, 3.3999998569488525, 11.399999618530273, 8.199999809265137, 10.399999618530273, 8.800000190734863, 11.800000190734863, 12.199999809265137, 9.199999809265137, 5.199999809265137, 11.800000190734863, 8.59999942779541] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [13.199999809265137, 12.399999618530273, 11.199999809265137, 9.59999942779541, 14.0, 12.399999618530273, 20.0, 17.0, 21.19999885559082, 18.0, 18.799999237060547, 12.399999618530273, 13.799999237060547, 11.199999809265137, 14.799999237060547, 15.199999809265137] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [12.59999942779541, 12.799999237060547, 3.0, 1.399999976158142, 16.600000381469727, 9.0, 11.399999618530273, 6.0, 11.59999942779541, 6.199999809265137, 11.199999809265137, 5.599999904632568, 6.799999713897705, 6.399999618530273, 9.0, 4.199999809265137] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 10.57+/- 0.6443 (max: 21.2) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 8.362+/- 1.026 (max: 16.6) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 8.65+/- 0.6464 (max: 12.2) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 14.7+/- 0.8503 (max: 21.2) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 11.37+/- 0.3286 (max: 15.9) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 9.852+/- 0.4803 (max: 12.54) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 10.64+/- 0.3431 (max: 12.99) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 13.62+/- 0.3743 (max: 15.9) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.06312+/- 0.009956 (max: 0.25) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.02375+/- 0.00875 (max: 0.11) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.02875+/- 0.006575 (max: 0.09) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.1369+/- 0.01635 (max: 0.25) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 1.4 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 1.4 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 3.4 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 9.6 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 5.103 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 5.103 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 7.513 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 10.39 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.03 | +------------------------------------------------------------------------------------------------------ +Evaluating ACCEL_SoftMoE_SEED1 against population in Overcooked-AsymmAdvantages6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [0.5999999642372131, 4.199999809265137, 0.7999999523162842, 15.199999809265137, 0.7999999523162842, 3.5999999046325684, 1.0, 5.799999713897705, 1.399999976158142, 36.39999771118164, 0.3999999761581421, 6.399999618530273, 0.7999999523162842, 37.599998474121094, 0.5999999642372131, 60.0, 1.0, 11.59999942779541, 0.3999999761581421, 17.399999618530273, 1.5999999046325684, 12.799999237060547, 2.0, 8.0, 0.5999999642372131, 4.599999904632568, 1.0, 22.19999885559082, 0.19999998807907104, 8.0, 0.3999999761581421, 4.199999809265137, 0.3999999761581421, 29.19999885559082, 0.7999999523162842, 4.599999904632568, 0.5999999642372131, 5.799999713897705, 1.7999999523162842, 21.19999885559082, 0.5999999642372131, 9.800000190734863, 0.7999999523162842, 6.399999618530273, 0.7999999523162842, 23.0, 0.7999999523162842, 4.400000095367432] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [0.5999999642372131, 4.199999809265137, 1.0, 5.799999713897705, 0.7999999523162842, 37.599998474121094, 0.3999999761581421, 17.399999618530273, 0.5999999642372131, 4.599999904632568, 0.3999999761581421, 4.199999809265137, 0.5999999642372131, 5.799999713897705, 0.7999999523162842, 6.399999618530273] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [0.7999999523162842, 15.199999809265137, 1.399999976158142, 36.39999771118164, 0.5999999642372131, 60.0, 1.5999999046325684, 12.799999237060547, 1.0, 22.19999885559082, 0.3999999761581421, 29.19999885559082, 1.7999999523162842, 21.19999885559082, 0.7999999523162842, 23.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [0.7999999523162842, 3.5999999046325684, 0.3999999761581421, 6.399999618530273, 1.0, 11.59999942779541, 2.0, 8.0, 0.19999998807907104, 8.0, 0.7999999523162842, 4.599999904632568, 0.5999999642372131, 9.800000190734863, 0.7999999523162842, 4.400000095367432] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 7.971+/- 1.755 (max: 60.0) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 3.937+/- 0.9409 (max: 11.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 5.7+/- 2.385 (max: 37.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 14.27+/- 4.279 (max: 60.0) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 9.27+/- 0.9981 (max: 29.66) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 7.492+/- 1.044 (max: 15.79) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 7.73+/- 1.436 (max: 24.05) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 12.59+/- 2.261 (max: 29.66) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.09062+/- 0.02645 (max: 0.85) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.02375+/- 0.00841 (max: 0.12) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.05625+/- 0.03769 (max: 0.58) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.1919+/- 0.06343 (max: 0.85) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.2 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 0.2 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 0.4 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 0.4 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 1.99 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 1.99 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 2.8 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 2.8 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating ACCEL_SoftMoE_SEED1 against population in Overcooked-CrampedRoom6_9 for xpid plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [11.0, 9.800000190734863, 68.4000015258789, 68.0, 102.39999389648438, 99.19999694824219, 16.399999618530273, 15.59999942779541, 74.5999984741211, 74.4000015258789, 115.5999984741211, 114.19999694824219, 19.19999885559082, 17.399999618530273, 95.79999542236328, 100.4000015258789, 114.79999542236328, 113.0, 16.799999237060547, 18.600000381469727, 95.4000015258789, 95.5999984741211, 92.0, 93.5999984741211, 9.199999809265137, 9.199999809265137, 75.4000015258789, 75.79999542236328, 108.5999984741211, 110.79999542236328, 18.399999618530273, 19.19999885559082, 95.5999984741211, 96.4000015258789, 117.0, 121.0, 15.59999942779541, 15.59999942779541, 89.0, 85.0, 113.0, 111.39999389648438, 16.600000381469727, 20.799999237060547, 97.4000015258789, 94.79999542236328, 110.19999694824219, 107.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [11.0, 9.800000190734863, 16.399999618530273, 15.59999942779541, 19.19999885559082, 17.399999618530273, 16.799999237060547, 18.600000381469727, 9.199999809265137, 9.199999809265137, 18.399999618530273, 19.19999885559082, 15.59999942779541, 15.59999942779541, 16.600000381469727, 20.799999237060547] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [68.4000015258789, 68.0, 74.5999984741211, 74.4000015258789, 95.79999542236328, 100.4000015258789, 95.4000015258789, 95.5999984741211, 75.4000015258789, 75.79999542236328, 95.5999984741211, 96.4000015258789, 89.0, 85.0, 97.4000015258789, 94.79999542236328] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [102.39999389648438, 99.19999694824219, 115.5999984741211, 114.19999694824219, 114.79999542236328, 113.0, 92.0, 93.5999984741211, 108.5999984741211, 110.79999542236328, 117.0, 121.0, 113.0, 111.39999389648438, 110.19999694824219, 107.0] +------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 70.32+/- 5.926 (max: 121.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 109.0+/- 2.066 (max: 121.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 15.59+/- 0.9417 (max: 20.8) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 86.37+/- 2.897 (max: 100.4) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 21.84+/- 0.9275 (max: 33.95) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 25.13+/- 1.029 (max: 31.98) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 13.99+/- 0.4335 (max: 16.95) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 26.4+/- 0.7863 (max: 33.95) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.7017+/- 0.05652 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.9887+/- 0.00427 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.1587+/- 0.02091 (max: 0.3) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.9575+/- 0.008342 (max: 1.0) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 9.2 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 92.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 9.2 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 68.0 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 10.74 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 16.09 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 10.74 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 21.61 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.93 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.89 | +------------------------------------------------------------------------------------------------- +Evaluating ACCEL_SoftMoE_SEED2 against population in Overcooked-CoordRing6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [3.3999998569488525, 4.400000095367432, 15.59999942779541, 17.399999618530273, 19.399999618530273, 20.399999618530273, 14.199999809265137, 13.799999237060547, 25.599998474121094, 25.599998474121094, 17.799999237060547, 20.0, 12.799999237060547, 13.799999237060547, 22.399999618530273, 20.600000381469727, 18.600000381469727, 20.399999618530273, 11.399999618530273, 12.199999809265137, 12.799999237060547, 16.399999618530273, 15.199999809265137, 17.600000381469727, 13.399999618530273, 13.199999809265137, 18.399999618530273, 19.0, 20.799999237060547, 21.19999885559082, 11.0, 13.0, 19.799999237060547, 21.399999618530273, 32.0, 29.399999618530273, 3.1999998092651367, 3.0, 13.799999237060547, 17.19999885559082, 26.0, 29.799999237060547, 12.799999237060547, 15.59999942779541, 10.199999809265137, 15.199999809265137, 21.0, 24.19999885559082] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [3.3999998569488525, 4.400000095367432, 14.199999809265137, 13.799999237060547, 12.799999237060547, 13.799999237060547, 11.399999618530273, 12.199999809265137, 13.399999618530273, 13.199999809265137, 11.0, 13.0, 3.1999998092651367, 3.0, 12.799999237060547, 15.59999942779541] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [15.59999942779541, 17.399999618530273, 25.599998474121094, 25.599998474121094, 22.399999618530273, 20.600000381469727, 12.799999237060547, 16.399999618530273, 18.399999618530273, 19.0, 19.799999237060547, 21.399999618530273, 13.799999237060547, 17.19999885559082, 10.199999809265137, 15.199999809265137] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [19.399999618530273, 20.399999618530273, 17.799999237060547, 20.0, 18.600000381469727, 20.399999618530273, 15.199999809265137, 17.600000381469727, 20.799999237060547, 21.19999885559082, 32.0, 29.399999618530273, 26.0, 29.799999237060547, 21.0, 24.19999885559082] +---------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CoordRing6_9 | 17.01+/- 0.9417 (max: 32.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 22.11+/- 1.208 (max: 32.0) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 10.7+/- 1.108 (max: 15.6) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 18.21+/- 1.079 (max: 25.6) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 12.48+/- 0.2989 (max: 16.31) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 12.41+/- 0.4374 (max: 15.35) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 11.54+/- 0.6308 (max: 14.67) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 13.5+/- 0.349 (max: 16.31) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.174+/- 0.02075 (max: 0.65) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.2569+/- 0.0447 (max: 0.65) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.07+/- 0.01278 (max: 0.14) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.195+/- 0.02603 (max: 0.39) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 3.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 15.2 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 3.0 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 10.2 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 7.141 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 10.16 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 7.141 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 11.49 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.07 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.04 | +---------------------------------------------------------------------------------------------- +Evaluating ACCEL_SoftMoE_SEED2 against population in Overcooked-ForcedCoord6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [3.1999998092651367, 0.19999998807907104, 4.0, 0.3999999761581421, 3.799999952316284, 0.7999999523162842, 5.0, 0.0, 3.3999998569488525, 0.7999999523162842, 3.1999998092651367, 0.5999999642372131, 2.799999952316284, 0.19999998807907104, 1.399999976158142, 0.7999999523162842, 0.7999999523162842, 0.0, 4.0, 0.19999998807907104, 4.599999904632568, 0.19999998807907104, 2.799999952316284, 1.399999976158142, 2.200000047683716, 0.19999998807907104, 2.799999952316284, 0.7999999523162842, 4.0, 1.0, 3.799999952316284, 0.19999998807907104, 2.5999999046325684, 0.0, 1.7999999523162842, 0.5999999642372131, 3.0, 0.0, 3.3999998569488525, 0.19999998807907104, 2.0, 0.3999999761581421, 5.400000095367432, 0.0, 4.0, 0.3999999761581421, 6.0, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [3.1999998092651367, 0.19999998807907104, 5.0, 0.0, 2.799999952316284, 0.19999998807907104, 4.0, 0.19999998807907104, 2.200000047683716, 0.19999998807907104, 3.799999952316284, 0.19999998807907104, 3.0, 0.0, 5.400000095367432, 0.0] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [4.0, 0.3999999761581421, 3.3999998569488525, 0.7999999523162842, 1.399999976158142, 0.7999999523162842, 4.599999904632568, 0.19999998807907104, 2.799999952316284, 0.7999999523162842, 2.5999999046325684, 0.0, 3.3999998569488525, 0.19999998807907104, 4.0, 0.3999999761581421] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [3.799999952316284, 0.7999999523162842, 3.1999998092651367, 0.5999999642372131, 0.7999999523162842, 0.0, 2.799999952316284, 1.399999976158142, 4.0, 1.0, 1.7999999523162842, 0.5999999642372131, 2.0, 0.3999999761581421, 6.0, 0.0] +----------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 1.862+/- 0.2511 (max: 6.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 1.825+/- 0.4262 (max: 6.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 1.9+/- 0.4956 (max: 5.4) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 1.862+/- 0.4065 (max: 4.6) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 4.794+/- 0.4211 (max: 9.165) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 4.871+/- 0.6694 (max: 9.165) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 4.51+/- 0.8705 (max: 8.879) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 5.002+/- 0.6724 (max: 8.485) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.000625+/- 0.0003531 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.00125+/- 0.0008539 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.000625+/- 0.000625 (max: 0.01) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +----------------------------------------------------------------------------------------------------- +Evaluating ACCEL_SoftMoE_SEED2 against population in Overcooked-CounterCircuit6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [8.0, 6.599999904632568, 14.199999809265137, 10.59999942779541, 15.199999809265137, 13.59999942779541, 8.0, 6.0, 12.399999618530273, 10.59999942779541, 2.3999998569488525, 0.7999999523162842, 2.200000047683716, 2.200000047683716, 9.399999618530273, 6.0, 10.59999942779541, 6.599999904632568, 7.399999618530273, 8.800000190734863, 17.19999885559082, 15.59999942779541, 7.799999713897705, 6.199999809265137, 8.199999809265137, 8.800000190734863, 18.799999237060547, 13.59999942779541, 7.199999809265137, 4.799999713897705, 9.199999809265137, 10.399999618530273, 14.799999237060547, 11.59999942779541, 9.199999809265137, 7.199999809265137, 6.799999713897705, 6.399999618530273, 16.19999885559082, 10.800000190734863, 7.799999713897705, 4.199999809265137, 9.0, 8.399999618530273, 18.600000381469727, 12.59999942779541, 7.0, 4.599999904632568] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [8.0, 6.599999904632568, 8.0, 6.0, 2.200000047683716, 2.200000047683716, 7.399999618530273, 8.800000190734863, 8.199999809265137, 8.800000190734863, 9.199999809265137, 10.399999618530273, 6.799999713897705, 6.399999618530273, 9.0, 8.399999618530273] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [14.199999809265137, 10.59999942779541, 12.399999618530273, 10.59999942779541, 9.399999618530273, 6.0, 17.19999885559082, 15.59999942779541, 18.799999237060547, 13.59999942779541, 14.799999237060547, 11.59999942779541, 16.19999885559082, 10.800000190734863, 18.600000381469727, 12.59999942779541] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [15.199999809265137, 13.59999942779541, 2.3999998569488525, 0.7999999523162842, 10.59999942779541, 6.599999904632568, 7.799999713897705, 6.199999809265137, 7.199999809265137, 4.799999713897705, 9.199999809265137, 7.199999809265137, 7.799999713897705, 4.199999809265137, 7.0, 4.599999904632568] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 9.262+/- 0.6191 (max: 18.8) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 7.2+/- 0.9313 (max: 15.2) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 7.275+/- 0.5733 (max: 10.4) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 13.31+/- 0.8754 (max: 18.8) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 11.28+/- 0.4246 (max: 18.6) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 9.457+/- 0.53 (max: 13.6) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 10.17+/- 0.4309 (max: 12.8) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 14.2+/- 0.5951 (max: 18.6) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.06229+/- 0.01171 (max: 0.32) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.01812+/- 0.008814 (max: 0.14) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.02312+/- 0.005456 (max: 0.08) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.1456+/- 0.02204 (max: 0.32) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 0.8 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 0.8 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 2.2 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 6.0 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 3.919 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 3.919 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 6.258 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 9.165 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating ACCEL_SoftMoE_SEED2 against population in Overcooked-AsymmAdvantages6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [2.5999999046325684, 7.199999809265137, 2.3999998569488525, 19.399999618530273, 3.1999998092651367, 8.199999809265137, 2.799999952316284, 9.199999809265137, 2.799999952316284, 43.20000076293945, 2.799999952316284, 7.399999618530273, 3.1999998092651367, 38.0, 3.1999998092651367, 57.599998474121094, 3.1999998092651367, 11.199999809265137, 3.1999998092651367, 19.799999237060547, 3.3999998569488525, 13.399999618530273, 4.0, 6.199999809265137, 2.5999999046325684, 4.799999713897705, 2.799999952316284, 23.0, 2.200000047683716, 7.0, 2.5999999046325684, 7.799999713897705, 4.0, 29.599998474121094, 3.5999999046325684, 6.0, 2.799999952316284, 8.399999618530273, 2.5999999046325684, 21.600000381469727, 3.799999952316284, 8.0, 2.0, 7.599999904632568, 2.5999999046325684, 26.19999885559082, 4.199999809265137, 4.0] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [2.5999999046325684, 7.199999809265137, 2.799999952316284, 9.199999809265137, 3.1999998092651367, 38.0, 3.1999998092651367, 19.799999237060547, 2.5999999046325684, 4.799999713897705, 2.5999999046325684, 7.799999713897705, 2.799999952316284, 8.399999618530273, 2.0, 7.599999904632568] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [2.3999998569488525, 19.399999618530273, 2.799999952316284, 43.20000076293945, 3.1999998092651367, 57.599998474121094, 3.3999998569488525, 13.399999618530273, 2.799999952316284, 23.0, 4.0, 29.599998474121094, 2.5999999046325684, 21.600000381469727, 2.5999999046325684, 26.19999885559082] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [3.1999998092651367, 8.199999809265137, 2.799999952316284, 7.399999618530273, 3.1999998092651367, 11.199999809265137, 4.0, 6.199999809265137, 2.200000047683716, 7.0, 3.5999999046325684, 6.0, 3.799999952316284, 8.0, 4.199999809265137, 4.0] +------------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 9.737+/- 1.712 (max: 57.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 5.312+/- 0.6242 (max: 11.2) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 7.787+/- 2.305 (max: 38.0) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 16.11+/- 4.192 (max: 57.6) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 12.02+/- 0.9337 (max: 31.69) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 9.808+/- 0.547 (max: 13.66) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 11.32+/- 1.618 (max: 31.69) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 14.92+/- 2.087 (max: 31.66) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.1087+/- 0.02705 (max: 0.81) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.03312+/- 0.006875 (max: 0.09) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.08+/- 0.03578 (max: 0.55) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.2131+/- 0.06624 (max: 0.81) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 2.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 2.2 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 2.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 2.4 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 6.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 6.258 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 6.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 6.499 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------- +Evaluating ACCEL_SoftMoE_SEED2 against population in Overcooked-CrampedRoom6_9 for xpid SEED_2_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [10.59999942779541, 10.0, 66.4000015258789, 65.79999542236328, 86.4000015258789, 90.19999694824219, 13.799999237060547, 15.199999809265137, 71.0, 70.19999694824219, 96.5999984741211, 97.19999694824219, 16.600000381469727, 15.399999618530273, 90.4000015258789, 83.79999542236328, 95.19999694824219, 93.19999694824219, 16.399999618530273, 14.0, 84.0, 79.79999542236328, 64.5999984741211, 72.5999984741211, 7.199999809265137, 7.399999618530273, 70.79999542236328, 67.5999984741211, 91.0, 91.0, 15.0, 16.19999885559082, 80.79999542236328, 84.79999542236328, 104.5999984741211, 100.19999694824219, 15.0, 14.399999618530273, 81.0, 78.79999542236328, 96.79999542236328, 93.0, 18.600000381469727, 16.0, 90.19999694824219, 79.4000015258789, 96.5999984741211, 97.5999984741211] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [10.59999942779541, 10.0, 13.799999237060547, 15.199999809265137, 16.600000381469727, 15.399999618530273, 16.399999618530273, 14.0, 7.199999809265137, 7.399999618530273, 15.0, 16.19999885559082, 15.0, 14.399999618530273, 18.600000381469727, 16.0] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [66.4000015258789, 65.79999542236328, 71.0, 70.19999694824219, 90.4000015258789, 83.79999542236328, 84.0, 79.79999542236328, 70.79999542236328, 67.5999984741211, 80.79999542236328, 84.79999542236328, 81.0, 78.79999542236328, 90.19999694824219, 79.4000015258789] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [86.4000015258789, 90.19999694824219, 96.5999984741211, 97.19999694824219, 95.19999694824219, 93.19999694824219, 64.5999984741211, 72.5999984741211, 91.0, 91.0, 104.5999984741211, 100.19999694824219, 96.79999542236328, 93.0, 96.5999984741211, 97.5999984741211] +-------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 61.11+/- 5.062 (max: 104.6) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 91.67+/- 2.52 (max: 104.6) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 13.86+/- 0.8296 (max: 18.6) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 77.8+/- 2.035 (max: 90.4) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 24.45+/- 1.174 (max: 36.21) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 29.64+/- 0.6716 (max: 33.11) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 13.65+/- 0.4103 (max: 16.01) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 30.06+/- 0.8101 (max: 36.21) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.6667+/- 0.05556 (max: 0.99) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.9556+/- 0.008849 (max: 0.99) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.1325+/- 0.0159 (max: 0.23) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.9119+/- 0.009274 (max: 0.95) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 7.2 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 64.6 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 7.2 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 65.8 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 10.45 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 24.41 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 10.45 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 24.8 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.86 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.02 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.83 | +-------------------------------------------------------------------------------------------------- +Evaluating ACCEL_SoftMoE_SEED3 against population in Overcooked-CoordRing6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CoordRing6_9, v [3.5999999046325684, 4.400000095367432, 20.0, 16.600000381469727, 24.19999885559082, 24.0, 13.799999237060547, 16.399999618530273, 30.19999885559082, 35.0, 25.19999885559082, 25.399999618530273, 14.59999942779541, 11.59999942779541, 23.19999885559082, 25.399999618530273, 26.19999885559082, 28.399999618530273, 18.19999885559082, 12.799999237060547, 19.0, 20.600000381469727, 21.600000381469727, 21.0, 16.399999618530273, 14.799999237060547, 22.399999618530273, 24.600000381469727, 23.19999885559082, 28.799999237060547, 12.799999237060547, 12.399999618530273, 25.0, 23.799999237060547, 30.0, 36.20000076293945, 3.1999998092651367, 2.3999998569488525, 18.399999618530273, 20.19999885559082, 26.599998474121094, 30.599998474121094, 12.799999237060547, 14.0, 14.59999942779541, 17.19999885559082, 18.600000381469727, 25.19999885559082] +k eval/a0:test_return:Overcooked-CoordRing6_9:low, v [3.5999999046325684, 4.400000095367432, 13.799999237060547, 16.399999618530273, 14.59999942779541, 11.59999942779541, 18.19999885559082, 12.799999237060547, 16.399999618530273, 14.799999237060547, 12.799999237060547, 12.399999618530273, 3.1999998092651367, 2.3999998569488525, 12.799999237060547, 14.0] +k eval/a0:test_return:Overcooked-CoordRing6_9:mid, v [20.0, 16.600000381469727, 30.19999885559082, 35.0, 23.19999885559082, 25.399999618530273, 19.0, 20.600000381469727, 22.399999618530273, 24.600000381469727, 25.0, 23.799999237060547, 18.399999618530273, 20.19999885559082, 14.59999942779541, 17.19999885559082] +k eval/a0:test_return:Overcooked-CoordRing6_9:high, v [24.19999885559082, 24.0, 25.19999885559082, 25.399999618530273, 26.19999885559082, 28.399999618530273, 21.600000381469727, 21.0, 23.19999885559082, 28.799999237060547, 30.0, 36.20000076293945, 26.599998474121094, 30.599998474121094, 18.600000381469727, 25.19999885559082] +------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CoordRing6_9 | 19.91+/- 1.128 (max: 36.2) | +| eval/a0:test_return:Overcooked-CoordRing6_9:high | 25.95+/- 1.063 (max: 36.2) | +| eval/a0:test_return:Overcooked-CoordRing6_9:low | 11.51+/- 1.283 (max: 18.2) | +| eval/a0:test_return:Overcooked-CoordRing6_9:mid | 22.26+/- 1.3 (max: 35.0) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9 | 13.51+/- 0.3541 (max: 19.29) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 13.57+/- 0.58 (max: 19.29) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 12.26+/- 0.7193 (max: 15.17) | +| eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 14.71+/- 0.3518 (max: 17.37) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.2606+/- 0.02445 (max: 0.68) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.38+/- 0.03378 (max: 0.67) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.09625+/- 0.01622 (max: 0.23) | +| eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.3056+/- 0.03643 (max: 0.68) | +| min:eval/a0:test_return:Overcooked-CoordRing6_9 | 2.4 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:high | 18.6 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:low | 2.4 | +| min:eval/a0:test_return:Overcooked-CoordRing6_9:mid | 14.6 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9 | 6.499 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:high | 10.65 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:low | 6.499 | +| min:eval/a0:test_return_std:Overcooked-CoordRing6_9:mid | 12.6 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:high | 0.17 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CoordRing6_9:mid | 0.1 | +------------------------------------------------------------------------------------------------ +Evaluating ACCEL_SoftMoE_SEED3 against population in Overcooked-ForcedCoord6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-ForcedCoord6_9, v [6.399999618530273, 0.3999999761581421, 6.599999904632568, 1.0, 3.1999998092651367, 0.3999999761581421, 7.799999713897705, 0.0, 6.399999618530273, 0.5999999642372131, 4.400000095367432, 0.5999999642372131, 5.199999809265137, 2.200000047683716, 4.599999904632568, 1.399999976158142, 1.5999999046325684, 1.0, 7.199999809265137, 0.3999999761581421, 5.599999904632568, 0.0, 5.799999713897705, 0.3999999761581421, 4.0, 0.3999999761581421, 5.599999904632568, 0.19999998807907104, 8.0, 0.3999999761581421, 5.799999713897705, 0.3999999761581421, 5.0, 0.0, 3.5999999046325684, 0.3999999761581421, 2.799999952316284, 0.19999998807907104, 4.0, 1.5999999046325684, 3.5999999046325684, 0.7999999523162842, 8.0, 0.19999998807907104, 7.599999904632568, 0.5999999642372131, 9.800000190734863, 0.3999999761581421] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:low, v [6.399999618530273, 0.3999999761581421, 7.799999713897705, 0.0, 5.199999809265137, 2.200000047683716, 7.199999809265137, 0.3999999761581421, 4.0, 0.3999999761581421, 5.799999713897705, 0.3999999761581421, 2.799999952316284, 0.19999998807907104, 8.0, 0.19999998807907104] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:mid, v [6.599999904632568, 1.0, 6.399999618530273, 0.5999999642372131, 4.599999904632568, 1.399999976158142, 5.599999904632568, 0.0, 5.599999904632568, 0.19999998807907104, 5.0, 0.0, 4.0, 1.5999999046325684, 7.599999904632568, 0.5999999642372131] +k eval/a0:test_return:Overcooked-ForcedCoord6_9:high, v [3.1999998092651367, 0.3999999761581421, 4.400000095367432, 0.5999999642372131, 1.5999999046325684, 1.0, 5.799999713897705, 0.3999999761581421, 8.0, 0.3999999761581421, 3.5999999046325684, 0.3999999761581421, 3.5999999046325684, 0.7999999523162842, 9.800000190734863, 0.3999999761581421] +---------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-ForcedCoord6_9 | 3.054+/- 0.4147 (max: 9.8) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 2.775+/- 0.7398 (max: 9.8) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 3.212+/- 0.7708 (max: 8.0) | +| eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 3.175+/- 0.6836 (max: 7.6) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 5.863+/- 0.4681 (max: 10.2) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 5.686+/- 0.6958 (max: 9.998) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 5.847+/- 0.8881 (max: 10.2) | +| eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 6.056+/- 0.8845 (max: 10.11) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.00125+/- 0.0004824 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0+/- 0.0 (max: 0.0) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.00125+/- 0.0008539 (max: 0.01) | +| eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0025+/- 0.001118 (max: 0.01) | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:high | 0.4 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:high | 2.8 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_return_std:Overcooked-ForcedCoord6_9:mid | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-ForcedCoord6_9:mid | 0.0 | +---------------------------------------------------------------------------------------------------- +Evaluating ACCEL_SoftMoE_SEED3 against population in Overcooked-CounterCircuit6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CounterCircuit6_9, v [10.59999942779541, 10.800000190734863, 21.799999237060547, 13.59999942779541, 15.59999942779541, 13.199999809265137, 6.599999904632568, 6.599999904632568, 11.0, 11.199999809265137, 5.799999713897705, 1.399999976158142, 1.5999999046325684, 1.5999999046325684, 9.199999809265137, 9.199999809265137, 10.399999618530273, 9.0, 8.199999809265137, 7.399999618530273, 22.399999618530273, 17.799999237060547, 13.199999809265137, 7.399999618530273, 10.800000190734863, 8.399999618530273, 19.19999885559082, 15.799999237060547, 8.800000190734863, 4.199999809265137, 12.0, 9.800000190734863, 19.0, 13.59999942779541, 10.399999618530273, 8.0, 7.599999904632568, 7.199999809265137, 15.0, 13.799999237060547, 12.799999237060547, 5.0, 10.199999809265137, 9.0, 20.399999618530273, 17.600000381469727, 12.799999237060547, 4.199999809265137] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:low, v [10.59999942779541, 10.800000190734863, 6.599999904632568, 6.599999904632568, 1.5999999046325684, 1.5999999046325684, 8.199999809265137, 7.399999618530273, 10.800000190734863, 8.399999618530273, 12.0, 9.800000190734863, 7.599999904632568, 7.199999809265137, 10.199999809265137, 9.0] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:mid, v [21.799999237060547, 13.59999942779541, 11.0, 11.199999809265137, 9.199999809265137, 9.199999809265137, 22.399999618530273, 17.799999237060547, 19.19999885559082, 15.799999237060547, 19.0, 13.59999942779541, 15.0, 13.799999237060547, 20.399999618530273, 17.600000381469727] +k eval/a0:test_return:Overcooked-CounterCircuit6_9:high, v [15.59999942779541, 13.199999809265137, 5.799999713897705, 1.399999976158142, 10.399999618530273, 9.0, 13.199999809265137, 7.399999618530273, 8.800000190734863, 4.199999809265137, 10.399999618530273, 8.0, 12.799999237060547, 5.0, 12.799999237060547, 4.199999809265137] +------------------------------------------------------------------------------------------------------ +| eval/a0:test_return:Overcooked-CounterCircuit6_9 | 10.86+/- 0.7338 (max: 22.4) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 8.887+/- 1.011 (max: 15.6) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 8.025+/- 0.7492 (max: 12.0) | +| eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 15.66+/- 1.071 (max: 22.4) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 11.78+/- 0.4498 (max: 19.79) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 10.08+/- 0.437 (max: 12.07) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 10.35+/- 0.5525 (max: 12.65) | +| eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 14.91+/- 0.6494 (max: 19.79) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.07833+/- 0.01431 (max: 0.38) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.02625+/- 0.006575 (max: 0.08) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.02625+/- 0.006382 (max: 0.07) | +| eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.1825+/- 0.02747 (max: 0.38) | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9 | 1.4 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:high | 1.4 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:low | 1.6 | +| min:eval/a0:test_return:Overcooked-CounterCircuit6_9:mid | 9.2 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9 | 5.103 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:high | 5.103 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:low | 5.426 | +| min:eval/a0:test_return_std:Overcooked-CounterCircuit6_9:mid | 9.968 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-CounterCircuit6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------ +Evaluating ACCEL_SoftMoE_SEED3 against population in Overcooked-AsymmAdvantages6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9, v [1.0, 6.0, 1.7999999523162842, 14.399999618530273, 2.200000047683716, 3.0, 1.5999999046325684, 9.199999809265137, 1.399999976158142, 42.39999771118164, 2.3999998569488525, 3.1999998092651367, 0.7999999523162842, 37.0, 1.0, 56.0, 1.0, 11.59999942779541, 1.5999999046325684, 19.799999237060547, 3.0, 9.59999942779541, 2.200000047683716, 4.799999713897705, 0.5999999642372131, 3.3999998569488525, 1.0, 18.19999885559082, 1.1999999284744263, 5.799999713897705, 0.7999999523162842, 6.599999904632568, 1.1999999284744263, 25.0, 2.0, 5.400000095367432, 0.7999999523162842, 5.400000095367432, 1.7999999523162842, 24.399999618530273, 1.7999999523162842, 8.800000190734863, 0.19999998807907104, 9.199999809265137, 1.1999999284744263, 22.600000381469727, 2.3999998569488525, 2.200000047683716] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low, v [1.0, 6.0, 1.5999999046325684, 9.199999809265137, 0.7999999523162842, 37.0, 1.5999999046325684, 19.799999237060547, 0.5999999642372131, 3.3999998569488525, 0.7999999523162842, 6.599999904632568, 0.7999999523162842, 5.400000095367432, 0.19999998807907104, 9.199999809265137] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid, v [1.7999999523162842, 14.399999618530273, 1.399999976158142, 42.39999771118164, 1.0, 56.0, 3.0, 9.59999942779541, 1.0, 18.19999885559082, 1.1999999284744263, 25.0, 1.7999999523162842, 24.399999618530273, 1.1999999284744263, 22.600000381469727] +k eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high, v [2.200000047683716, 3.0, 2.3999998569488525, 3.1999998092651367, 1.0, 11.59999942779541, 2.200000047683716, 4.799999713897705, 1.1999999284744263, 5.799999713897705, 2.0, 5.400000095367432, 1.7999999523162842, 8.800000190734863, 2.3999998569488525, 2.200000047683716] +------------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 8.104+/- 1.707 (max: 56.0) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 3.75+/- 0.7297 (max: 11.6) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 6.5+/- 2.401 (max: 37.0) | +| eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 14.06+/- 4.16 (max: 56.0) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 9.987+/- 0.9854 (max: 30.59) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 7.933+/- 0.6595 (max: 12.43) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 8.856+/- 1.681 (max: 26.89) | +| eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 13.17+/- 2.196 (max: 30.59) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.08833+/- 0.0261 (max: 0.75) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.01812+/- 0.005643 (max: 0.07) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.06938+/- 0.04139 (max: 0.62) | +| eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.1775+/- 0.06143 (max: 0.75) | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9 | 0.2 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:high | 1.0 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:low | 0.2 | +| min:eval/a0:test_return:Overcooked-AsymmAdvantages6_9:mid | 1.0 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9 | 1.99 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:high | 4.359 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:low | 1.99 | +| min:eval/a0:test_return_std:Overcooked-AsymmAdvantages6_9:mid | 4.359 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9 | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:high | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:low | 0.0 | +| min:eval/a0:test_solved_rate:Overcooked-AsymmAdvantages6_9:mid | 0.0 | +------------------------------------------------------------------------------------------------------- +Evaluating ACCEL_SoftMoE_SEED3 against population in Overcooked-CrampedRoom6_9 for xpid SEED_3_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 +k eval/a0:test_return:Overcooked-CrampedRoom6_9, v [12.799999237060547, 12.0, 70.4000015258789, 67.0, 97.0, 94.5999984741211, 17.799999237060547, 14.799999237060547, 75.19999694824219, 69.0, 104.0, 102.79999542236328, 20.0, 21.399999618530273, 89.79999542236328, 93.0, 104.0, 110.0, 21.0, 23.799999237060547, 84.0, 81.5999984741211, 76.4000015258789, 78.5999984741211, 9.0, 12.399999618530273, 77.19999694824219, 67.19999694824219, 98.4000015258789, 96.5999984741211, 19.0, 20.799999237060547, 84.5999984741211, 86.0, 105.5999984741211, 97.79999542236328, 19.799999237060547, 18.399999618530273, 80.4000015258789, 75.79999542236328, 95.4000015258789, 104.19999694824219, 22.799999237060547, 21.19999885559082, 86.79999542236328, 80.79999542236328, 99.0, 98.4000015258789] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:low, v [12.799999237060547, 12.0, 17.799999237060547, 14.799999237060547, 20.0, 21.399999618530273, 21.0, 23.799999237060547, 9.0, 12.399999618530273, 19.0, 20.799999237060547, 19.799999237060547, 18.399999618530273, 22.799999237060547, 21.19999885559082] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:mid, v [70.4000015258789, 67.0, 75.19999694824219, 69.0, 89.79999542236328, 93.0, 84.0, 81.5999984741211, 77.19999694824219, 67.19999694824219, 84.5999984741211, 86.0, 80.4000015258789, 75.79999542236328, 86.79999542236328, 80.79999542236328] +k eval/a0:test_return:Overcooked-CrampedRoom6_9:high, v [97.0, 94.5999984741211, 104.0, 102.79999542236328, 104.0, 110.0, 76.4000015258789, 78.5999984741211, 98.4000015258789, 96.5999984741211, 105.5999984741211, 97.79999542236328, 95.4000015258789, 104.19999694824219, 99.0, 98.4000015258789] +-------------------------------------------------------------------------------------------------- +| eval/a0:test_return:Overcooked-CrampedRoom6_9 | 64.97+/- 5.082 (max: 110.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 97.67+/- 2.235 (max: 110.0) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 17.94+/- 1.098 (max: 23.8) | +| eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 79.3+/- 2.015 (max: 93.0) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 25.19+/- 1.1 (max: 35.14) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 30.55+/- 0.7829 (max: 35.14) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 15.09+/- 0.382 (max: 17.16) | +| eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 29.93+/- 0.6353 (max: 33.49) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.7021+/- 0.05005 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.96+/- 0.007692 (max: 1.0) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.225+/- 0.02422 (max: 0.37) | +| eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.9212+/- 0.009259 (max: 0.99) | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9 | 9.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:high | 76.4 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:low | 9.0 | +| min:eval/a0:test_return:Overcooked-CrampedRoom6_9:mid | 67.0 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9 | 11.45 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:high | 23.75 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:low | 11.45 | +| min:eval/a0:test_return_std:Overcooked-CrampedRoom6_9:mid | 25.51 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9 | 0.04 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:high | 0.88 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:low | 0.04 | +| min:eval/a0:test_solved_rate:Overcooked-CrampedRoom6_9:mid | 0.85 | +-------------------------------------------------------------------------------------------------- diff --git a/src/train_baseline_dr_lstm.sh b/src/train_baseline_dr_lstm.sh new file mode 100755 index 0000000..1ea62df --- /dev/null +++ b/src/train_baseline_dr_lstm.sh @@ -0,0 +1,67 @@ +DEFAULTVALUE=4 +DEFAULTSEED=2 +device="${1:-$DEFAULTVALUE}" +seed="${2:-$DEFAULTSEED}" +CUDA_VISIBLE_DEVICES=${device} XLA_PYTHON_CLIENT_MEM_FRACTION=.40 LD_LIBRARY_PATH="" nice -n 5 python3 -m minimax.train \ +--wandb_mode=online \ +--wandb_project=overcooked-minimax-jax \ +--wandb_entity=${WANDB_ENTITY} \ +--seed=${seed} \ +--agent_rl_algo=ppo \ +--n_total_updates=30000 \ +--train_runner=dr \ +--n_devices=1 \ +--student_model_name=default_student_actor_cnn \ +--student_critic_model_name=default_student_critic_cnn \ +--env_name=Overcooked \ +--is_multi_agent=True \ +--verbose=False \ +--log_dir=~/logs/minimax \ +--log_interval=10 \ +--from_last_checkpoint=False \ +--checkpoint_interval=1000 \ +--archive_interval=0 \ +--archive_init_checkpoint=False \ +--test_interval=100 \ +--n_students=1 \ +--n_parallel=32 \ +--n_eval=1 \ +--n_rollout_steps=400 \ +--lr=0.0003 \ +--lr_anneal_steps=0 \ +--max_grad_norm=0.5 \ +--adam_eps=1e-05 \ +--track_env_metrics=True \ +--discount=0.999 \ +--n_unroll_rollout=10 \ +--render=False \ +--student_gae_lambda=0.98 \ +--student_entropy_coef=0.01 \ +--student_value_loss_coef=0.5 \ +--student_n_unroll_update=5 \ +--student_ppo_n_epochs=8 \ +--student_ppo_n_minibatches=4 \ +--student_ppo_clip_eps=0.2 \ +--student_ppo_clip_value_loss=True \ +--student_recurrent_arch=lstm \ +--student_recurrent_hidden_dim=64 \ +--student_hidden_dim=64 \ +--student_n_hidden_layers=3 \ +--student_n_conv_layers=3 \ +--student_n_conv_filters=32 \ +--student_n_scalar_embeddings=4 \ +--student_scalar_embed_dim=5 \ +--student_agent_kind=mappo \ +--overcooked_height=6 \ +--overcooked_width=9 \ +--overcooked_n_walls=15 \ +--overcooked_replace_wall_pos=True \ +--overcooked_sample_n_walls=True \ +--overcooked_normalize_obs=True \ +--overcooked_max_steps=400 \ +--overcooked_random_reset=False \ +--n_shaped_reward_updates=30000 \ +--test_n_episodes=10 \ +--test_env_names=Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9 \ +--overcooked_test_normalize_obs=True \ +--xpid=SEED_${seed}_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 \ No newline at end of file diff --git a/src/train_baseline_dr_s5.sh b/src/train_baseline_dr_s5.sh new file mode 100755 index 0000000..db78691 --- /dev/null +++ b/src/train_baseline_dr_s5.sh @@ -0,0 +1,71 @@ +DEFAULTVALUE=4 +DEFAULTSEED=2 +device="${1:-$DEFAULTVALUE}" +seed="${2:-$DEFAULTSEED}" +CUDA_VISIBLE_DEVICES=${device} XLA_PYTHON_CLIENT_MEM_FRACTION=.40 LD_LIBRARY_PATH="" nice -n 5 python3 -m minimax.train \ +--wandb_mode=online \ +--wandb_project=overcooked-minimax-jax \ +--wandb_entity=${WANDB_ENTITY} \ +--seed=${seed} \ +--agent_rl_algo=ppo \ +--n_total_updates=30000 \ +--train_runner=dr \ +--n_devices=1 \ +--student_model_name=default_student_actor_cnn \ +--student_critic_model_name=default_student_critic_cnn \ +--env_name=Overcooked \ +--is_multi_agent=True \ +--verbose=False \ +--log_dir=~/logs/minimax \ +--log_interval=10 \ +--from_last_checkpoint=False \ +--checkpoint_interval=1000 \ +--archive_interval=0 \ +--archive_init_checkpoint=False \ +--test_interval=100 \ +--n_students=1 \ +--n_parallel=32 \ +--n_eval=1 \ +--n_rollout_steps=400 \ +--lr=0.0003 \ +--lr_anneal_steps=0 \ +--max_grad_norm=0.5 \ +--adam_eps=1e-05 \ +--track_env_metrics=True \ +--discount=0.999 \ +--n_unroll_rollout=10 \ +--render=False \ +--student_gae_lambda=0.98 \ +--student_entropy_coef=0.01 \ +--student_value_loss_coef=0.5 \ +--student_n_unroll_update=5 \ +--student_ppo_n_epochs=8 \ +--student_ppo_n_minibatches=4 \ +--student_ppo_clip_eps=0.2 \ +--student_ppo_clip_value_loss=True \ +--student_recurrent_arch=s5 \ +--student_recurrent_hidden_dim=64 \ +--student_hidden_dim=64 \ +--student_n_hidden_layers=3 \ +--student_n_conv_layers=3 \ +--student_n_conv_filters=32 \ +--student_n_scalar_embeddings=4 \ +--student_scalar_embed_dim=5 \ +--student_s5_n_blocks=2 \ +--student_s5_n_layers=2 \ +--student_s5_layernorm_pos=pre \ +--student_s5_activation=half_glu1 \ +--student_agent_kind=mappo \ +--overcooked_height=6 \ +--overcooked_width=9 \ +--overcooked_n_walls=15 \ +--overcooked_replace_wall_pos=True \ +--overcooked_sample_n_walls=True \ +--overcooked_normalize_obs=True \ +--overcooked_max_steps=400 \ +--overcooked_random_reset=False \ +--n_shaped_reward_updates=30000 \ +--test_n_episodes=10 \ +--test_env_names=Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9 \ +--overcooked_test_normalize_obs=True \ +--xpid=SEED_${seed}_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 \ No newline at end of file diff --git a/src/train_baseline_dr_softmoe_lstm.sh b/src/train_baseline_dr_softmoe_lstm.sh new file mode 100755 index 0000000..21ca311 --- /dev/null +++ b/src/train_baseline_dr_softmoe_lstm.sh @@ -0,0 +1,70 @@ +DEFAULTVALUE=4 +DEFAULTSEED=2 +device="${1:-$DEFAULTVALUE}" +seed="${2:-$DEFAULTSEED}" +CUDA_VISIBLE_DEVICES=${device} XLA_PYTHON_CLIENT_MEM_FRACTION=.40 LD_LIBRARY_PATH="" nice -n 5 python3 -m minimax.train \ +--wandb_mode=online \ +--wandb_project=overcooked-minimax-jax \ +--wandb_entity=${WANDB_ENTITY} \ +--seed=${seed} \ +--agent_rl_algo=ppo \ +--n_total_updates=30000 \ +--train_runner=dr \ +--n_devices=1 \ +--student_model_name=default_student_actor_cnn \ +--student_critic_model_name=default_student_critic_cnn \ +--env_name=Overcooked \ +--is_multi_agent=True \ +--verbose=False \ +--log_dir=~/logs/minimax \ +--log_interval=10 \ +--from_last_checkpoint=False \ +--checkpoint_interval=1000 \ +--archive_interval=0 \ +--archive_init_checkpoint=False \ +--test_interval=100 \ +--n_students=1 \ +--n_parallel=32 \ +--n_eval=1 \ +--n_rollout_steps=400 \ +--lr=0.0003 \ +--lr_anneal_steps=0 \ +--max_grad_norm=0.5 \ +--adam_eps=1e-05 \ +--track_env_metrics=True \ +--discount=0.999 \ +--n_unroll_rollout=10 \ +--render=False \ +--student_gae_lambda=0.98 \ +--student_entropy_coef=0.01 \ +--student_value_loss_coef=0.5 \ +--student_n_unroll_update=5 \ +--student_ppo_n_epochs=8 \ +--student_ppo_n_minibatches=4 \ +--student_ppo_clip_eps=0.2 \ +--student_ppo_clip_value_loss=True \ +--student_recurrent_arch=lstm \ +--student_recurrent_hidden_dim=64 \ +--student_hidden_dim=64 \ +--student_n_hidden_layers=2 \ +--student_is_soft_moe=True \ +--student_soft_moe_num_experts=4 \ +--student_soft_moe_num_slots=32 \ +--student_n_conv_layers=3 \ +--student_n_conv_filters=32 \ +--student_n_scalar_embeddings=4 \ +--student_scalar_embed_dim=5 \ +--student_agent_kind=mappo \ +--overcooked_height=6 \ +--overcooked_width=9 \ +--overcooked_n_walls=15 \ +--overcooked_replace_wall_pos=True \ +--overcooked_sample_n_walls=True \ +--overcooked_normalize_obs=True \ +--overcooked_max_steps=400 \ +--overcooked_random_reset=False \ +--n_shaped_reward_updates=30000 \ +--test_n_episodes=10 \ +--test_env_names=Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9 \ +--overcooked_test_normalize_obs=True \ +--xpid=SEED_${seed}_dr-overcooked6x9w15_fs_IMAGE-r1s_32p_1e_400t_ae1e-05-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 \ No newline at end of file diff --git a/src/train_baseline_p_accel_lstm.sh b/src/train_baseline_p_accel_lstm.sh new file mode 100755 index 0000000..1e1d893 --- /dev/null +++ b/src/train_baseline_p_accel_lstm.sh @@ -0,0 +1,81 @@ +DEFAULTVALUE=4 +DEFAULTSEED=2 +device="${1:-$DEFAULTVALUE}" +seed="${2:-$DEFAULTSEED}" +CUDA_VISIBLE_DEVICES=${device} XLA_PYTHON_CLIENT_MEM_FRACTION=.40 LD_LIBRARY_PATH="" nice -n 5 python3 -m minimax.train \ +--wandb_mode=online \ +--wandb_project=overcooked-minimax-jax \ +--wandb_entity=${WANDB_ENTITY} \ +--seed=${seed} \ +--agent_rl_algo=ppo \ +--n_total_updates=30000 \ +--train_runner=plr \ +--n_devices=1 \ +--student_model_name=default_student_actor_cnn \ +--student_critic_model_name=default_student_critic_cnn \ +--env_name=Overcooked \ +--is_multi_agent=True \ +--verbose=False \ +--log_dir=~/logs/minimax \ +--log_interval=10 \ +--from_last_checkpoint=False \ +--checkpoint_interval=1000 \ +--archive_interval=0 \ +--archive_init_checkpoint=False \ +--test_interval=100 \ +--n_students=1 \ +--n_parallel=32 \ +--n_eval=1 \ +--n_rollout_steps=400 \ +--lr=0.0003 \ +--lr_anneal_steps=0 \ +--max_grad_norm=0.5 \ +--adam_eps=1e-05 \ +--track_env_metrics=True \ +--discount=0.999 \ +--n_unroll_rollout=10 \ +--render=False \ +--ued_score=max_mc \ +--plr_replay_prob=0.8 \ +--plr_buffer_size=4000 \ +--plr_staleness_coef=0.3 \ +--plr_temp=0.1 \ +--plr_use_score_ranks=True \ +--plr_min_fill_ratio=0.5 \ +--plr_use_robust_plr=True \ +--plr_use_parallel_eval=True \ +--plr_force_unique=True \ +--plr_mutation_fn=default \ +--plr_n_mutations=20 \ +--plr_mutation_criterion=batch \ +--plr_mutation_subsample_size=4 \ +--student_gae_lambda=0.98 \ +--student_entropy_coef=0.01 \ +--student_value_loss_coef=0.5 \ +--student_n_unroll_update=5 \ +--student_ppo_n_epochs=8 \ +--student_ppo_n_minibatches=4 \ +--student_ppo_clip_eps=0.2 \ +--student_ppo_clip_value_loss=True \ +--student_recurrent_arch=lstm \ +--student_recurrent_hidden_dim=64 \ +--student_hidden_dim=64 \ +--student_n_hidden_layers=3 \ +--student_n_conv_layers=3 \ +--student_n_conv_filters=32 \ +--student_n_scalar_embeddings=4 \ +--student_scalar_embed_dim=5 \ +--student_agent_kind=mappo \ +--overcooked_height=6 \ +--overcooked_width=9 \ +--overcooked_n_walls=15 \ +--overcooked_replace_wall_pos=True \ +--overcooked_sample_n_walls=True \ +--overcooked_normalize_obs=True \ +--overcooked_max_steps=400 \ +--overcooked_random_reset=False \ +--n_shaped_reward_updates=30000 \ +--test_n_episodes=10 \ +--test_env_names=Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9 \ +--overcooked_test_normalize_obs=True \ +--xpid=SEED_${seed}_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 \ No newline at end of file diff --git a/src/train_baseline_p_accel_s5.sh b/src/train_baseline_p_accel_s5.sh new file mode 100755 index 0000000..60416e5 --- /dev/null +++ b/src/train_baseline_p_accel_s5.sh @@ -0,0 +1,85 @@ +DEFAULTVALUE=4 +DEFAULTSEED=2 +device="${1:-$DEFAULTVALUE}" +seed="${2:-$DEFAULTSEED}" +CUDA_VISIBLE_DEVICES=${device} XLA_PYTHON_CLIENT_MEM_FRACTION=.40 LD_LIBRARY_PATH="" nice -n 5 python3 -m minimax.train \ +--wandb_mode=online \ +--wandb_project=overcooked-minimax-jax \ +--wandb_entity=${WANDB_ENTITY} \ +--seed=${seed} \ +--agent_rl_algo=ppo \ +--n_total_updates=30000 \ +--train_runner=plr \ +--n_devices=1 \ +--student_model_name=default_student_actor_cnn \ +--student_critic_model_name=default_student_critic_cnn \ +--env_name=Overcooked \ +--is_multi_agent=True \ +--verbose=False \ +--log_dir=~/logs/minimax \ +--log_interval=10 \ +--from_last_checkpoint=False \ +--checkpoint_interval=1000 \ +--archive_interval=0 \ +--archive_init_checkpoint=False \ +--test_interval=100 \ +--n_students=1 \ +--n_parallel=32 \ +--n_eval=1 \ +--n_rollout_steps=400 \ +--lr=0.0003 \ +--lr_anneal_steps=0 \ +--max_grad_norm=0.5 \ +--adam_eps=1e-05 \ +--track_env_metrics=True \ +--discount=0.999 \ +--n_unroll_rollout=10 \ +--render=False \ +--ued_score=max_mc \ +--plr_replay_prob=0.8 \ +--plr_buffer_size=4000 \ +--plr_staleness_coef=0.3 \ +--plr_temp=0.1 \ +--plr_use_score_ranks=True \ +--plr_min_fill_ratio=0.5 \ +--plr_use_robust_plr=True \ +--plr_use_parallel_eval=True \ +--plr_force_unique=True \ +--plr_mutation_fn=default \ +--plr_n_mutations=20 \ +--plr_mutation_criterion=batch \ +--plr_mutation_subsample_size=4 \ +--student_gae_lambda=0.98 \ +--student_entropy_coef=0.01 \ +--student_value_loss_coef=0.5 \ +--student_n_unroll_update=5 \ +--student_ppo_n_epochs=8 \ +--student_ppo_n_minibatches=4 \ +--student_ppo_clip_eps=0.2 \ +--student_ppo_clip_value_loss=True \ +--student_recurrent_arch=s5 \ +--student_recurrent_hidden_dim=64 \ +--student_hidden_dim=64 \ +--student_n_hidden_layers=3 \ +--student_n_conv_layers=3 \ +--student_n_conv_filters=32 \ +--student_n_scalar_embeddings=4 \ +--student_scalar_embed_dim=5 \ +--student_s5_n_blocks=2 \ +--student_s5_n_layers=2 \ +--student_s5_layernorm_pos=pre \ +--student_s5_activation=half_glu1 \ +--student_agent_kind=mappo \ +--overcooked_height=6 \ +--overcooked_width=9 \ +--overcooked_n_walls=15 \ +--overcooked_replace_wall_pos=True \ +--overcooked_sample_n_walls=True \ +--overcooked_normalize_obs=True \ +--overcooked_max_steps=400 \ +--overcooked_random_reset=False \ +--n_shaped_reward_updates=30000 \ +--test_n_episodes=10 \ +--test_env_names=Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9 \ +--overcooked_test_normalize_obs=True \ +--xpid=SEED_${seed}_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 \ No newline at end of file diff --git a/src/train_baseline_p_accel_softmoe_lstm.sh b/src/train_baseline_p_accel_softmoe_lstm.sh new file mode 100755 index 0000000..b085621 --- /dev/null +++ b/src/train_baseline_p_accel_softmoe_lstm.sh @@ -0,0 +1,84 @@ +DEFAULTVALUE=4 +DEFAULTSEED=2 +device="${1:-$DEFAULTVALUE}" +seed="${2:-$DEFAULTSEED}" +CUDA_VISIBLE_DEVICES=${device} XLA_PYTHON_CLIENT_MEM_FRACTION=.40 LD_LIBRARY_PATH="" nice -n 5 python3 -m minimax.train \ +--wandb_mode=online \ +--wandb_project=overcooked-minimax-jax \ +--wandb_entity=${WANDB_ENTITY} \ +--seed=${seed} \ +--agent_rl_algo=ppo \ +--n_total_updates=30000 \ +--train_runner=plr \ +--n_devices=1 \ +--student_model_name=default_student_actor_cnn \ +--student_critic_model_name=default_student_critic_cnn \ +--env_name=Overcooked \ +--is_multi_agent=True \ +--verbose=False \ +--log_dir=~/logs/minimax \ +--log_interval=10 \ +--from_last_checkpoint=False \ +--checkpoint_interval=1000 \ +--archive_interval=0 \ +--archive_init_checkpoint=False \ +--test_interval=100 \ +--n_students=1 \ +--n_parallel=32 \ +--n_eval=1 \ +--n_rollout_steps=400 \ +--lr=0.0003 \ +--lr_anneal_steps=0 \ +--max_grad_norm=0.5 \ +--adam_eps=1e-05 \ +--track_env_metrics=True \ +--discount=0.999 \ +--n_unroll_rollout=10 \ +--render=False \ +--ued_score=max_mc \ +--plr_replay_prob=0.8 \ +--plr_buffer_size=4000 \ +--plr_staleness_coef=0.3 \ +--plr_temp=0.1 \ +--plr_use_score_ranks=True \ +--plr_min_fill_ratio=0.5 \ +--plr_use_robust_plr=True \ +--plr_use_parallel_eval=True \ +--plr_force_unique=True \ +--plr_mutation_fn=default \ +--plr_n_mutations=20 \ +--plr_mutation_criterion=batch \ +--plr_mutation_subsample_size=4 \ +--student_gae_lambda=0.98 \ +--student_entropy_coef=0.01 \ +--student_value_loss_coef=0.5 \ +--student_n_unroll_update=5 \ +--student_ppo_n_epochs=8 \ +--student_ppo_n_minibatches=4 \ +--student_ppo_clip_eps=0.2 \ +--student_ppo_clip_value_loss=True \ +--student_recurrent_arch=lstm \ +--student_recurrent_hidden_dim=64 \ +--student_hidden_dim=64 \ +--student_n_hidden_layers=2 \ +--student_is_soft_moe=True \ +--student_soft_moe_num_experts=4 \ +--student_soft_moe_num_slots=32 \ +--student_n_conv_layers=3 \ +--student_n_conv_filters=32 \ +--student_n_scalar_embeddings=4 \ +--student_scalar_embed_dim=5 \ +--student_agent_kind=mappo \ +--overcooked_height=6 \ +--overcooked_width=9 \ +--overcooked_n_walls=15 \ +--overcooked_replace_wall_pos=True \ +--overcooked_sample_n_walls=True \ +--overcooked_normalize_obs=True \ +--overcooked_max_steps=400 \ +--overcooked_random_reset=False \ +--n_shaped_reward_updates=30000 \ +--test_n_episodes=10 \ +--test_env_names=Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9 \ +--overcooked_test_normalize_obs=True \ +--xpid=SEED_${seed}_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.8b4000t0.1s0.3m0.5r_mdef20bat_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 \ No newline at end of file diff --git a/src/train_baseline_p_plr_lstm.sh b/src/train_baseline_p_plr_lstm.sh new file mode 100755 index 0000000..bb0ccd5 --- /dev/null +++ b/src/train_baseline_p_plr_lstm.sh @@ -0,0 +1,77 @@ +DEFAULTVALUE=4 +DEFAULTSEED=2 +device="${1:-$DEFAULTVALUE}" +seed="${2:-$DEFAULTSEED}" +CUDA_VISIBLE_DEVICES=${device} XLA_PYTHON_CLIENT_MEM_FRACTION=.40 LD_LIBRARY_PATH="" nice -n 5 python3 -m minimax.train \ +--wandb_mode=online \ +--wandb_project=overcooked-minimax-jax \ +--wandb_entity=${WANDB_ENTITY} \ +--seed=${seed} \ +--agent_rl_algo=ppo \ +--n_total_updates=30000 \ +--train_runner=plr \ +--n_devices=1 \ +--student_model_name=default_student_actor_cnn \ +--student_critic_model_name=default_student_critic_cnn \ +--env_name=Overcooked \ +--is_multi_agent=True \ +--verbose=False \ +--log_dir=~/logs/minimax \ +--log_interval=10 \ +--from_last_checkpoint=False \ +--checkpoint_interval=1000 \ +--archive_interval=0 \ +--archive_init_checkpoint=False \ +--test_interval=100 \ +--n_students=1 \ +--n_parallel=32 \ +--n_eval=1 \ +--n_rollout_steps=400 \ +--lr=0.0003 \ +--lr_anneal_steps=0 \ +--max_grad_norm=0.5 \ +--adam_eps=1e-05 \ +--track_env_metrics=True \ +--discount=0.999 \ +--n_unroll_rollout=10 \ +--render=False \ +--ued_score=max_mc \ +--plr_replay_prob=0.5 \ +--plr_buffer_size=4000 \ +--plr_staleness_coef=0.3 \ +--plr_temp=0.1 \ +--plr_use_score_ranks=True \ +--plr_min_fill_ratio=0.5 \ +--plr_use_robust_plr=True \ +--plr_use_parallel_eval=True \ +--plr_force_unique=True \ +--student_gae_lambda=0.98 \ +--student_entropy_coef=0.01 \ +--student_value_loss_coef=0.5 \ +--student_n_unroll_update=5 \ +--student_ppo_n_epochs=8 \ +--student_ppo_n_minibatches=4 \ +--student_ppo_clip_eps=0.2 \ +--student_ppo_clip_value_loss=True \ +--student_recurrent_arch=lstm \ +--student_recurrent_hidden_dim=64 \ +--student_hidden_dim=64 \ +--student_n_hidden_layers=3 \ +--student_n_conv_layers=3 \ +--student_n_conv_filters=32 \ +--student_n_scalar_embeddings=4 \ +--student_scalar_embed_dim=5 \ +--student_agent_kind=mappo \ +--overcooked_height=6 \ +--overcooked_width=9 \ +--overcooked_n_walls=15 \ +--overcooked_replace_wall_pos=True \ +--overcooked_sample_n_walls=True \ +--overcooked_normalize_obs=True \ +--overcooked_max_steps=400 \ +--overcooked_random_reset=False \ +--n_shaped_reward_updates=30000 \ +--test_n_episodes=10 \ +--test_env_names=Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9 \ +--overcooked_test_normalize_obs=True \ +--xpid=SEED_${seed}_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_0 \ No newline at end of file diff --git a/src/train_baseline_p_plr_s5.sh b/src/train_baseline_p_plr_s5.sh new file mode 100755 index 0000000..c8e31ec --- /dev/null +++ b/src/train_baseline_p_plr_s5.sh @@ -0,0 +1,81 @@ +DEFAULTVALUE=4 +DEFAULTSEED=2 +device="${1:-$DEFAULTVALUE}" +seed="${2:-$DEFAULTSEED}" +CUDA_VISIBLE_DEVICES=${device} XLA_PYTHON_CLIENT_MEM_FRACTION=.40 LD_LIBRARY_PATH="" nice -n 5 python3 -m minimax.train \ +--wandb_mode=online \ +--wandb_project=overcooked-minimax-jax \ +--wandb_entity=${WANDB_ENTITY} \ +--seed=${seed} \ +--agent_rl_algo=ppo \ +--n_total_updates=30000 \ +--train_runner=plr \ +--n_devices=1 \ +--student_model_name=default_student_actor_cnn \ +--student_critic_model_name=default_student_critic_cnn \ +--env_name=Overcooked \ +--is_multi_agent=True \ +--verbose=False \ +--log_dir=~/logs/minimax \ +--log_interval=10 \ +--from_last_checkpoint=False \ +--checkpoint_interval=1000 \ +--archive_interval=0 \ +--archive_init_checkpoint=False \ +--test_interval=100 \ +--n_students=1 \ +--n_parallel=32 \ +--n_eval=1 \ +--n_rollout_steps=400 \ +--lr=0.0003 \ +--lr_anneal_steps=0 \ +--max_grad_norm=0.5 \ +--adam_eps=1e-05 \ +--track_env_metrics=True \ +--discount=0.999 \ +--n_unroll_rollout=10 \ +--render=False \ +--ued_score=max_mc \ +--plr_replay_prob=0.5 \ +--plr_buffer_size=4000 \ +--plr_staleness_coef=0.3 \ +--plr_temp=0.1 \ +--plr_use_score_ranks=True \ +--plr_min_fill_ratio=0.5 \ +--plr_use_robust_plr=True \ +--plr_use_parallel_eval=True \ +--plr_force_unique=True \ +--student_gae_lambda=0.98 \ +--student_entropy_coef=0.01 \ +--student_value_loss_coef=0.5 \ +--student_n_unroll_update=5 \ +--student_ppo_n_epochs=8 \ +--student_ppo_n_minibatches=4 \ +--student_ppo_clip_eps=0.2 \ +--student_ppo_clip_value_loss=True \ +--student_recurrent_arch=s5 \ +--student_recurrent_hidden_dim=64 \ +--student_hidden_dim=64 \ +--student_n_hidden_layers=3 \ +--student_n_conv_layers=3 \ +--student_n_conv_filters=32 \ +--student_n_scalar_embeddings=4 \ +--student_scalar_embed_dim=5 \ +--student_s5_n_blocks=2 \ +--student_s5_n_layers=2 \ +--student_s5_layernorm_pos=pre \ +--student_s5_activation=half_glu1 \ +--student_agent_kind=mappo \ +--overcooked_height=6 \ +--overcooked_width=9 \ +--overcooked_n_walls=15 \ +--overcooked_replace_wall_pos=True \ +--overcooked_sample_n_walls=True \ +--overcooked_normalize_obs=True \ +--overcooked_max_steps=400 \ +--overcooked_random_reset=False \ +--n_shaped_reward_updates=30000 \ +--test_n_episodes=10 \ +--test_env_names=Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9 \ +--overcooked_test_normalize_obs=True \ +--xpid=SEED_${seed}_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_0 \ No newline at end of file diff --git a/src/train_baseline_p_plr_softmoe_lstm.sh b/src/train_baseline_p_plr_softmoe_lstm.sh new file mode 100755 index 0000000..03f17d4 --- /dev/null +++ b/src/train_baseline_p_plr_softmoe_lstm.sh @@ -0,0 +1,80 @@ +DEFAULTVALUE=4 +DEFAULTSEED=2 +device="${1:-$DEFAULTVALUE}" +seed="${2:-$DEFAULTSEED}" +CUDA_VISIBLE_DEVICES=${device} XLA_PYTHON_CLIENT_MEM_FRACTION=.40 LD_LIBRARY_PATH="" nice -n 5 python3 -m minimax.train \ +--wandb_mode=online \ +--wandb_project=overcooked-minimax-jax \ +--wandb_entity=${WANDB_ENTITY} \ +--seed=${seed} \ +--agent_rl_algo=ppo \ +--n_total_updates=30000 \ +--train_runner=plr \ +--n_devices=1 \ +--student_model_name=default_student_actor_cnn \ +--student_critic_model_name=default_student_critic_cnn \ +--env_name=Overcooked \ +--is_multi_agent=True \ +--verbose=False \ +--log_dir=~/logs/minimax \ +--log_interval=10 \ +--from_last_checkpoint=False \ +--checkpoint_interval=1000 \ +--archive_interval=0 \ +--archive_init_checkpoint=False \ +--test_interval=100 \ +--n_students=1 \ +--n_parallel=32 \ +--n_eval=1 \ +--n_rollout_steps=400 \ +--lr=0.0003 \ +--lr_anneal_steps=0 \ +--max_grad_norm=0.5 \ +--adam_eps=1e-05 \ +--track_env_metrics=True \ +--discount=0.999 \ +--n_unroll_rollout=10 \ +--render=False \ +--ued_score=max_mc \ +--plr_replay_prob=0.5 \ +--plr_buffer_size=4000 \ +--plr_staleness_coef=0.3 \ +--plr_temp=0.1 \ +--plr_use_score_ranks=True \ +--plr_min_fill_ratio=0.5 \ +--plr_use_robust_plr=True \ +--plr_use_parallel_eval=True \ +--plr_force_unique=True \ +--student_gae_lambda=0.98 \ +--student_entropy_coef=0.01 \ +--student_value_loss_coef=0.5 \ +--student_n_unroll_update=5 \ +--student_ppo_n_epochs=8 \ +--student_ppo_n_minibatches=4 \ +--student_ppo_clip_eps=0.2 \ +--student_ppo_clip_value_loss=True \ +--student_recurrent_arch=lstm \ +--student_recurrent_hidden_dim=64 \ +--student_hidden_dim=64 \ +--student_n_hidden_layers=2 \ +--student_is_soft_moe=True \ +--student_soft_moe_num_experts=4 \ +--student_soft_moe_num_slots=32 \ +--student_n_conv_layers=3 \ +--student_n_conv_filters=32 \ +--student_n_scalar_embeddings=4 \ +--student_scalar_embed_dim=5 \ +--student_agent_kind=mappo \ +--overcooked_height=6 \ +--overcooked_width=9 \ +--overcooked_n_walls=15 \ +--overcooked_replace_wall_pos=True \ +--overcooked_sample_n_walls=True \ +--overcooked_normalize_obs=True \ +--overcooked_max_steps=400 \ +--overcooked_random_reset=False \ +--n_shaped_reward_updates=30000 \ +--test_n_episodes=10 \ +--test_env_names=Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9 \ +--overcooked_test_normalize_obs=True \ +--xpid=SEED_${seed}_plr-overcooked6x9w15_fs_IMAGE-rpf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_400t_ae1e-05_smm-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___0 \ No newline at end of file diff --git a/src/train_baseline_pop_paired_lstm.sh b/src/train_baseline_pop_paired_lstm.sh new file mode 100755 index 0000000..e0b1953 --- /dev/null +++ b/src/train_baseline_pop_paired_lstm.sh @@ -0,0 +1,89 @@ +DEFAULTVALUE=4 +DEFAULTSEED=2 +device="${1:-$DEFAULTVALUE}" +seed="${2:-$DEFAULTSEED}" +CUDA_VISIBLE_DEVICES=${device} XLA_PYTHON_CLIENT_MEM_FRACTION=.40 LD_LIBRARY_PATH="" nice -n 5 python3 -m minimax.train \ +--wandb_mode=online \ +--wandb_project=overcooked-minimax-jax \ +--wandb_entity=${WANDB_ENTITY} \ +--seed=${seed} \ +--agent_rl_algo=ppo \ +--n_total_updates=30000 \ +--train_runner=paired \ +--n_devices=1 \ +--student_model_name=default_student_actor_cnn \ +--student_critic_model_name=default_student_critic_cnn \ +--env_name=Overcooked \ +--verbose=False \ +--is_multi_agent=True \ +--log_dir=~/logs/minimax \ +--log_interval=10 \ +--from_last_checkpoint=False \ +--checkpoint_interval=1000 \ +--archive_interval=0 \ +--archive_init_checkpoint=False \ +--test_interval=100 \ +--n_students=2 \ +--n_parallel=32 \ +--n_eval=1 \ +--n_rollout_steps=400 \ +--lr=0.0003 \ +--lr_anneal_steps=0 \ +--max_grad_norm=0.5 \ +--adam_eps=1e-05 \ +--track_env_metrics=True \ +--discount=0.999 \ +--n_unroll_rollout=10 \ +--render=False \ +--ued_score=relative_regret \ +--student_gae_lambda=0.98 \ +--teacher_discount=0.999 \ +--teacher_lr_anneal_steps=0 \ +--teacher_gae_lambda=0.98 \ +--student_entropy_coef=0.01 \ +--student_value_loss_coef=0.5 \ +--student_n_unroll_update=5 \ +--student_ppo_n_epochs=8 \ +--student_ppo_n_minibatches=4 \ +--student_ppo_clip_eps=0.2 \ +--student_ppo_clip_value_loss=True \ +--teacher_entropy_coef=0.01 \ +--teacher_value_loss_coef=0.5 \ +--teacher_n_unroll_update=5 \ +--teacher_ppo_n_epochs=8 \ +--teacher_ppo_n_minibatches=4 \ +--teacher_ppo_clip_eps=0.2 \ +--teacher_ppo_clip_value_loss=True \ +--student_recurrent_arch=lstm \ +--student_recurrent_hidden_dim=64 \ +--student_hidden_dim=64 \ +--student_n_hidden_layers=3 \ +--student_n_conv_layers=3 \ +--student_n_conv_filters=32 \ +--student_n_scalar_embeddings=4 \ +--student_scalar_embed_dim=5 \ +--student_agent_kind=mappo \ +--teacher_model_name=default_teacher_cnn \ +--teacher_recurrent_arch=lstm \ +--teacher_recurrent_hidden_dim=64 \ +--teacher_hidden_dim=64 \ +--teacher_n_hidden_layers=1 \ +--teacher_n_conv_filters=128 \ +--teacher_scalar_embed_dim=10 \ +--overcooked_height=6 \ +--overcooked_width=9 \ +--overcooked_n_walls=5 \ +--overcooked_normalize_obs=True \ +--overcooked_max_steps=400 \ +--overcooked_random_reset=False \ +--overcooked_ued_replace_wall_pos=True \ +--overcooked_ued_fixed_n_wall_steps=False \ +--overcooked_ued_first_wall_pos_sets_budget=True \ +--overcooked_ued_noise_dim=50 \ +--overcooked_ued_n_walls=15 \ +--overcooked_ued_normalize_obs=True \ +--n_shaped_reward_updates=30000 \ +--test_n_episodes=10 \ +--test_env_names=Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9 \ +--overcooked_test_normalize_obs=True \ +--xpid=SEED_${seed}_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lstm_h64_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 \ No newline at end of file diff --git a/src/train_baseline_pop_paired_s5.sh b/src/train_baseline_pop_paired_s5.sh new file mode 100755 index 0000000..06fdd82 --- /dev/null +++ b/src/train_baseline_pop_paired_s5.sh @@ -0,0 +1,93 @@ +DEFAULTVALUE=4 +DEFAULTSEED=2 +device="${1:-$DEFAULTVALUE}" +seed="${2:-$DEFAULTSEED}" +CUDA_VISIBLE_DEVICES=${device} XLA_PYTHON_CLIENT_MEM_FRACTION=.40 LD_LIBRARY_PATH="" nice -n 5 python3 -m minimax.train \ +--wandb_mode=online \ +--wandb_project=overcooked-minimax-jax \ +--wandb_entity=${WANDB_ENTITY} \ +--seed=${seed} \ +--agent_rl_algo=ppo \ +--n_total_updates=30000 \ +--train_runner=paired \ +--n_devices=1 \ +--student_model_name=default_student_actor_cnn \ +--student_critic_model_name=default_student_critic_cnn \ +--env_name=Overcooked \ +--verbose=False \ +--is_multi_agent=True \ +--log_dir=~/logs/minimax \ +--log_interval=10 \ +--from_last_checkpoint=False \ +--checkpoint_interval=1000 \ +--archive_interval=0 \ +--archive_init_checkpoint=False \ +--test_interval=100 \ +--n_students=2 \ +--n_parallel=32 \ +--n_eval=1 \ +--n_rollout_steps=400 \ +--lr=0.0003 \ +--lr_anneal_steps=0 \ +--max_grad_norm=0.5 \ +--adam_eps=1e-05 \ +--track_env_metrics=True \ +--discount=0.999 \ +--n_unroll_rollout=10 \ +--render=False \ +--ued_score=relative_regret \ +--student_gae_lambda=0.98 \ +--teacher_discount=0.999 \ +--teacher_lr_anneal_steps=0 \ +--teacher_gae_lambda=0.98 \ +--student_entropy_coef=0.01 \ +--student_value_loss_coef=0.5 \ +--student_n_unroll_update=5 \ +--student_ppo_n_epochs=8 \ +--student_ppo_n_minibatches=4 \ +--student_ppo_clip_eps=0.2 \ +--student_ppo_clip_value_loss=True \ +--teacher_entropy_coef=0.01 \ +--teacher_value_loss_coef=0.5 \ +--teacher_n_unroll_update=5 \ +--teacher_ppo_n_epochs=8 \ +--teacher_ppo_n_minibatches=4 \ +--teacher_ppo_clip_eps=0.2 \ +--teacher_ppo_clip_value_loss=True \ +--student_recurrent_arch=s5 \ +--student_recurrent_hidden_dim=64 \ +--student_hidden_dim=64 \ +--student_n_hidden_layers=3 \ +--student_n_conv_layers=3 \ +--student_n_conv_filters=32 \ +--student_n_scalar_embeddings=4 \ +--student_scalar_embed_dim=5 \ +--student_s5_n_blocks=2 \ +--student_s5_n_layers=2 \ +--student_s5_layernorm_pos=pre \ +--student_s5_activation=half_glu1 \ +--student_agent_kind=mappo \ +--teacher_model_name=default_teacher_cnn \ +--teacher_recurrent_arch=lstm \ +--teacher_recurrent_hidden_dim=64 \ +--teacher_hidden_dim=64 \ +--teacher_n_hidden_layers=1 \ +--teacher_n_conv_filters=128 \ +--teacher_scalar_embed_dim=10 \ +--overcooked_height=6 \ +--overcooked_width=9 \ +--overcooked_n_walls=5 \ +--overcooked_normalize_obs=True \ +--overcooked_max_steps=400 \ +--overcooked_random_reset=False \ +--overcooked_ued_replace_wall_pos=True \ +--overcooked_ued_fixed_n_wall_steps=False \ +--overcooked_ued_first_wall_pos_sets_budget=True \ +--overcooked_ued_noise_dim=50 \ +--overcooked_ued_n_walls=15 \ +--overcooked_ued_normalize_obs=True \ +--n_shaped_reward_updates=30000 \ +--test_n_episodes=10 \ +--test_env_names=Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9 \ +--overcooked_test_normalize_obs=True \ +--xpid=SEED_${seed}_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc3se5ba_re_lpr_ahg1_s5_h64nb2nl2_tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 \ No newline at end of file diff --git a/src/train_baseline_pop_paired_softmoe_lstm.sh b/src/train_baseline_pop_paired_softmoe_lstm.sh new file mode 100755 index 0000000..ffdfe1a --- /dev/null +++ b/src/train_baseline_pop_paired_softmoe_lstm.sh @@ -0,0 +1,92 @@ +DEFAULTVALUE=4 +DEFAULTSEED=2 +device="${1:-$DEFAULTVALUE}" +seed="${2:-$DEFAULTSEED}" +CUDA_VISIBLE_DEVICES=${device} XLA_PYTHON_CLIENT_MEM_FRACTION=.40 LD_LIBRARY_PATH="" nice -n 5 python3 -m minimax.train \ +--wandb_mode=online \ +--wandb_project=overcooked-minimax-jax \ +--wandb_entity=${WANDB_ENTITY} \ +--seed=${seed} \ +--agent_rl_algo=ppo \ +--n_total_updates=30000 \ +--train_runner=paired \ +--n_devices=1 \ +--student_model_name=default_student_actor_cnn \ +--student_critic_model_name=default_student_critic_cnn \ +--env_name=Overcooked \ +--verbose=False \ +--is_multi_agent=True \ +--log_dir=~/logs/minimax \ +--log_interval=10 \ +--from_last_checkpoint=False \ +--checkpoint_interval=1000 \ +--archive_interval=0 \ +--archive_init_checkpoint=False \ +--test_interval=100 \ +--n_students=2 \ +--n_parallel=32 \ +--n_eval=1 \ +--n_rollout_steps=400 \ +--lr=0.0003 \ +--lr_anneal_steps=0 \ +--max_grad_norm=0.5 \ +--adam_eps=1e-05 \ +--track_env_metrics=True \ +--discount=0.999 \ +--n_unroll_rollout=10 \ +--render=False \ +--ued_score=relative_regret \ +--student_gae_lambda=0.98 \ +--teacher_discount=0.999 \ +--teacher_lr_anneal_steps=0 \ +--teacher_gae_lambda=0.98 \ +--student_entropy_coef=0.01 \ +--student_value_loss_coef=0.5 \ +--student_n_unroll_update=5 \ +--student_ppo_n_epochs=8 \ +--student_ppo_n_minibatches=4 \ +--student_ppo_clip_eps=0.2 \ +--student_ppo_clip_value_loss=True \ +--teacher_entropy_coef=0.01 \ +--teacher_value_loss_coef=0.5 \ +--teacher_n_unroll_update=5 \ +--teacher_ppo_n_epochs=8 \ +--teacher_ppo_n_minibatches=4 \ +--teacher_ppo_clip_eps=0.2 \ +--teacher_ppo_clip_value_loss=True \ +--student_recurrent_arch=lstm \ +--student_recurrent_hidden_dim=64 \ +--student_hidden_dim=64 \ +--student_n_hidden_layers=2 \ +--student_is_soft_moe=True \ +--student_soft_moe_num_experts=4 \ +--student_soft_moe_num_slots=32 \ +--student_n_conv_layers=3 \ +--student_n_conv_filters=32 \ +--student_n_scalar_embeddings=4 \ +--student_scalar_embed_dim=5 \ +--student_agent_kind=mappo \ +--teacher_model_name=default_teacher_cnn \ +--teacher_recurrent_arch=lstm \ +--teacher_recurrent_hidden_dim=64 \ +--teacher_hidden_dim=64 \ +--teacher_n_hidden_layers=1 \ +--teacher_n_conv_filters=128 \ +--teacher_scalar_embed_dim=10 \ +--overcooked_height=6 \ +--overcooked_width=9 \ +--overcooked_n_walls=5 \ +--overcooked_normalize_obs=True \ +--overcooked_max_steps=400 \ +--overcooked_random_reset=False \ +--overcooked_ued_replace_wall_pos=True \ +--overcooked_ued_fixed_n_wall_steps=False \ +--overcooked_ued_first_wall_pos_sets_budget=True \ +--overcooked_ued_noise_dim=50 \ +--overcooked_ued_n_walls=15 \ +--overcooked_ued_normalize_obs=True \ +--n_shaped_reward_updates=30000 \ +--test_n_episodes=10 \ +--test_env_names=Overcooked-CoordRing6_9,Overcooked-ForcedCoord6_9,Overcooked-CounterCircuit6_9,Overcooked-AsymmAdvantages6_9,Overcooked-CrampedRoom6_9 \ +--overcooked_test_normalize_obs=True \ +--xpid=SEED_${seed}_paired-overcooked6x9w5_ld50_rb-r2s_32p_1e_400t_ae1e-05_sr-ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98_pc0.2_h64cf32fc2se5ba_re_lstm_h64__SoftMoE_4E_32S___tch_ppo_lr0.0003g0.999cv0.5ce0.01e8mb4l0.98pc0.2_h64cf128fc1se10ba_re_lstm_h64_0 \ No newline at end of file diff --git a/src/train_baselines_lstm6x9.sh b/src/train_baselines_lstm6x9.sh new file mode 100755 index 0000000..ee83274 --- /dev/null +++ b/src/train_baselines_lstm6x9.sh @@ -0,0 +1,10 @@ +DEFAULTVALUE=4 +DEFAULTSEED=1 +device="${1:-$DEFAULTVALUE}" +seed="${2:-$DEFAULTSEED}" +echo "Using device ${device} and seed ${seed}" + +./train_baseline_p_plr_lstm.sh $device $seed +./train_baseline_p_accel_lstm.sh $device $seed +./train_baseline_pop_paired_lstm.sh $device $seed +./train_baseline_dr_lstm.sh $device $seed diff --git a/src/train_baselines_s56x9.sh b/src/train_baselines_s56x9.sh new file mode 100755 index 0000000..67e746c --- /dev/null +++ b/src/train_baselines_s56x9.sh @@ -0,0 +1,10 @@ +DEFAULTVALUE=4 +DEFAULTSEED=1 +device="${1:-$DEFAULTVALUE}" +seed="${2:-$DEFAULTSEED}" +echo "Using device ${device} and seed ${seed}" + +./train_baseline_p_plr_s5.sh $device $seed +./train_baseline_p_accel_s5.sh $device $seed +./train_baseline_pop_paired_s5.sh $device $seed +./train_baseline_dr_s5.sh $device $seed diff --git a/src/train_baselines_softmoe_lstm6x9.sh b/src/train_baselines_softmoe_lstm6x9.sh new file mode 100755 index 0000000..6c04370 --- /dev/null +++ b/src/train_baselines_softmoe_lstm6x9.sh @@ -0,0 +1,10 @@ +DEFAULTVALUE=4 +DEFAULTSEED=1 +device="${1:-$DEFAULTVALUE}" +seed="${2:-$DEFAULTSEED}" +echo "Using device ${device} and seed ${seed}" + +./train_baseline_p_plr_softmoe_lstm.sh $device $seed +./train_baseline_p_accel_softmoe_lstm.sh $device $seed +./train_baseline_pop_paired_softmoe_lstm.sh $device $seed +./train_baseline_dr_softmoe_lstm.sh $device $seed diff --git a/src/train_maze.sh b/src/train_maze.sh new file mode 100755 index 0000000..7dcddfc --- /dev/null +++ b/src/train_maze.sh @@ -0,0 +1,67 @@ +CUDA_VISIBLE_DEVICES=$1 LD_LIBRARY_PATH="" nice -n 5 python3 -m minimax.train \ +--seed=1 \ +--agent_rl_algo=ppo \ +--n_total_updates=30000 \ +--train_runner=plr \ +--n_devices=1 \ +--student_model_name=default_student_cnn \ +--env_name=Maze \ +--verbose=False \ +--log_dir=~/logs/minimax \ +--log_interval=10 \ +--from_last_checkpoint=False \ +--checkpoint_interval=1000 \ +--archive_interval=0 \ +--archive_init_checkpoint=False \ +--test_interval=100 \ +--n_students=1 \ +--n_parallel=32 \ +--n_eval=1 \ +--n_rollout_steps=256 \ +--lr=5e-05 \ +--lr_anneal_steps=0 \ +--max_grad_norm=0.5 \ +--adam_eps=1e-05 \ +--track_env_metrics=True \ +--discount=0.995 \ +--n_unroll_rollout=10 \ +--render=False \ +--ued_score=max_mc \ +--plr_replay_prob=0.5 \ +--plr_buffer_size=4000 \ +--plr_staleness_coef=0.3 \ +--plr_temp=0.1 \ +--plr_use_score_ranks=True \ +--plr_min_fill_ratio=0.5 \ +--plr_use_robust_plr=True \ +--plr_use_parallel_eval=False \ +--plr_force_unique=True \ +--student_gae_lambda=0.98 \ +--student_entropy_coef=0.0 \ +--student_value_loss_coef=0.5 \ +--student_n_unroll_update=5 \ +--student_ppo_n_epochs=5 \ +--student_ppo_n_minibatches=1 \ +--student_ppo_clip_eps=0.2 \ +--student_ppo_clip_value_loss=True \ +--student_recurrent_arch=lstm \ +--student_recurrent_hidden_dim=256 \ +--student_hidden_dim=32 \ +--student_n_hidden_layers=1 \ +--student_n_conv_filters=16 \ +--student_n_scalar_embeddings=4 \ +--student_scalar_embed_dim=5 \ +--maze_height=13 \ +--maze_width=13 \ +--maze_n_walls=60 \ +--maze_replace_wall_pos=True \ +--maze_sample_n_walls=False \ +--maze_see_agent=False \ +--maze_normalize_obs=True \ +--maze_obs_agent_pos=False \ +--maze_max_episode_steps=250 \ +--test_n_episodes=10 \ +--test_env_names=Maze-SixteenRooms,Maze-Labyrinth,Maze-StandardMaze \ +--maze_test_see_agent=False \ +--maze_test_normalize_obs=True \ +--xpid=plr-maze13x13w60na_f-rf_p0.5b4000t0.1s0.3m0.5r_r1s_32p_1e_256t_ae1e-05_smm-ppo_lr5e-05g0.995cv0.5ce0.0e5mb1l0.98_pc0.2_h32cf16fc1se5ba_re_lstm_h256_0 \ No newline at end of file diff --git a/src/train_maze_s5.sh b/src/train_maze_s5.sh new file mode 100755 index 0000000..0d96b18 --- /dev/null +++ b/src/train_maze_s5.sh @@ -0,0 +1,76 @@ +DEFAULTVALUE=4 +device="${1:-$DEFAULTVALUE}" +CUDA_VISIBLE_DEVICES=${device} XLA_PYTHON_CLIENT_MEM_FRACTION=.40 LD_LIBRARY_PATH="" nice -n 5 python3 -m minimax.train \ +--wandb_mode=online \ +--wandb_project=overcooked-minimax-jax \ +--wandb_entity=${WANDB_ENTITY} \ +--seed=1 \ +--agent_rl_algo=ppo \ +--n_total_updates=30000 \ +--train_runner=plr \ +--n_devices=1 \ +--student_model_name=default_student_cnn \ +--env_name=Maze \ +--verbose=False \ +--log_dir=~/logs/minimax \ +--log_interval=10 \ +--from_last_checkpoint=True \ +--checkpoint_interval=1000 \ +--archive_interval=0 \ +--archive_init_checkpoint=False \ +--test_interval=100 \ +--n_students=1 \ +--n_parallel=32 \ +--n_eval=1 \ +--n_rollout_steps=256 \ +--lr=3e-05 \ +--lr_anneal_steps=0 \ +--max_grad_norm=0.5 \ +--adam_eps=1e-05 \ +--track_env_metrics=True \ +--discount=0.999 \ +--n_unroll_rollout=10 \ +--render=False \ +--ued_score=max_mc \ +--plr_replay_prob=0.5 \ +--plr_buffer_size=4000 \ +--plr_staleness_coef=0.3 \ +--plr_temp=0.3 \ +--plr_use_score_ranks=True \ +--plr_min_fill_ratio=0.5 \ +--plr_use_robust_plr=True \ +--plr_use_parallel_eval=False \ +--plr_force_unique=True \ +--student_gae_lambda=0.98 \ +--student_entropy_coef=0.001 \ +--student_value_loss_coef=0.5 \ +--student_n_unroll_update=5 \ +--student_ppo_n_epochs=5 \ +--student_ppo_n_minibatches=1 \ +--student_ppo_clip_eps=0.2 \ +--student_ppo_clip_value_loss=True \ +--student_recurrent_arch=s5 \ +--student_recurrent_hidden_dim=256 \ +--student_hidden_dim=32 \ +--student_n_hidden_layers=1 \ +--student_n_conv_filters=16 \ +--student_n_scalar_embeddings=4 \ +--student_scalar_embed_dim=5 \ +--student_s5_n_blocks=2 \ +--student_s5_n_layers=2 \ +--student_s5_layernorm_pos=pre \ +--student_s5_activation=half_glu1 \ +--maze_height=13 \ +--maze_width=13 \ +--maze_n_walls=60 \ +--maze_replace_wall_pos=True \ +--maze_sample_n_walls=False \ +--maze_see_agent=False \ +--maze_normalize_obs=True \ +--maze_obs_agent_pos=False \ +--maze_max_episode_steps=250 \ +--test_n_episodes=10 \ +--test_env_names=Maze-SixteenRooms,Maze-Labyrinth,Maze-StandardMaze \ +--maze_test_see_agent=False \ +--maze_test_normalize_obs=True \ +--xpid=plr-maze13x13w60na_f-rf_p0.5b4000t0.3s0.3m0.5r_r1s_32p_1e_256t_ae1e-05_smm-ppo_lr3e-05g0.999cv0.5ce0.001e5mb1l0.98_pc0.2_h32cf16fc1se5ba_re_lpr_ahg1_s5_h256nb2nl2_0

    5EWEIjpc!FdiZ1?E4_Bwj_?eNXhBzi4N9kp5M0XPG!p${9d4 z^I8V$_7$~EZ5>*2D>b9`$ya5IM4{K86xN3qHNhh^$T8|nza%k9SWGfWm_(qLk zTl}}#&+y#ZmTC00#4Kjz+e7^YV@9aE(jUI{{$;^=P{>n$w^1g?FaWWw$Oa6mn^)-0 zOG=KazxFNvN27-PlL>~XedQ1((kekHq*Fr{dgY;8s0ZF|k+M=U|?;I@>p8?6Y1BK$43C4Dz#{4uB!PxTd?gtc0Vj1G!~W z3X@&7<(N8DVhPI)CXWUqMLpj+c&=_veK@c`#?*sAe71k*`GAMpKruL?xD3pg&NY8{ zYdvWd>9n(Qrj(ThssMlC$q624mCo7htgzf%ja9&dmN@sULYjK5@L>O?(s~*fRb3)Y zkTU0~xm?e)nNfc3)hTrD<4r9*c}xO`oy96CXcKqsob1+zBAi|I#nhm`II*r!NdQj5 zp~P=yn%>8q<+Es`ska?qvAR(Kr$aE19HfB{nqJzdvOXF**Sn%xy31qw);w;v~;)y~)gk(J}$)TBe;kj9GWh z84Oe~Rkq-TjV3hYx6!!^k={6syu5TExtlgihVc%u#aZ3JR2YS93^)^e*B(x@1%eSy zxCW`H^}_S)7PPijIyF9#9#vXn_cnZ_<;Eo55ekxMGZ& zE}bg0c7QJ99Rr4OK3cwA=gM^3#GJ|6WcyU|!%s#ah>sQfZu~VJyV|a}mtk5GtT|WA z@l}Vk9h;@+LX+Pt;x3EdJZZ0>2~tq<$9!59o~aEzCPrhMQR4n*uk};CP~YQt&v<_J zz9fEAYHZGBZq!W#27Wre0yZwRQt`@DIzv|T*n!|)ZmgoD6J#A)97*=W(RNk1fZYqRDzorqAg5O`(vEWAN4i# zn2b&Ry5G|ouF}X5oC-Fe{4?>9HvfroB_?;@=0#*k4~U6X>k}rGM|Mb9)#pG7%>Kmg zv=h{{iN;@G1Jy9u+!Doj-60}rF99y=hoK-JtqVJlIE=sE~V@Tw)3O1K*;ux~D##9VGxn z=O4)j`X0%}uh#;snv+yo8wS8d39J32u|OhZ2C>*#wW+G$0J|h}O4+=VHG&BS-|!ao zT=iESlmUB_86U@(3P1L#ax3{UxgA!O53kZ?@bGmeL_~9I+71#Df$|~ zh)=|*cij}59L-a-FVExXRN*5qUkV+Gx7DF@+S}<=J*Jyya#l55*=4q%B((u-1&bzX zK5aP`u8g#8l^tj#sN`)VJHu}|8i!>L_YKZWViaC*C-o?RR!3u&e8=o^+{Kabu+8(~ zZ^ITGR_L2D6;ceb3A1{H>f@VVC3teoyR`M{H|xl5C9`W)5T5H)-V0utT#4zy(GJ5b z`d~*_k$uHz@u2-%7ut;9sxI_Knd0VlC;5)xT&HjZ&9hp6sm z+8AC^TDg2(SPoO-Ai5(ZW&ohDCtk*H3^v+@2!3Fv3-{J1q8)_SJjp=dK4vwDXd zEo_*$Vc5=CP}XGGz}F*GsnX;b&x78Evgv@#&OL`A!;WcLOl8x&1^7l4?j6c@nrJDF zp8VwNX)2JgVraM|^Pw_-a<(w(ZF`!Fc~~O(nVzU}I`iDi)*_Dm_fjsuAAZZfJ74H= zWWp3Q=|{R$i`SKGYFXlThnqO3uGws_%=pNB5VTOzwf+d~P%UVY4)ZB8b*goqH``q? z*?Q!A^hMS{PZV>HkT+~SUDHFRt?b|YPR?Nd9P{RZgbwD zKC)f42!^mjS8Cs0&g0?y-M#F^VXSJwO_yi9_OAQzLHSp%eg)wE!gl{X$-qxjyf*8? z7KuY=km&{v{ZXUmqsEq2;gS`}Pli#5p*wV(_Oz=pemx%y#@yX6f$n3Dki`emZh*Qp z<0gtXL};pc{q{))iB%@YRD9G{^Bpu_G7#Lx*|rU#~j(OZ^t93Hz!Q2 zrf|7O=#Me{BKt{tek|x}TF)!;fqd~duRU-So`V!#^Ztde8xHp)#jl&<&Uav!<%@fE zN0qh4yo>2b@(P8H*P_1HyozOkyYoX;0!5e|>D>ybA5WSaMw8x~Tz6-JK*BhN7hJLo zinER`mh;x&PE*q3-2a3Q)}ed%>JV{?vf2>0ac_PXnfB>~?HN-F(eALpT>XKh*PDJV z4!44B z_6*3;eIG044aXH;~4x1g#&u3`b1W2|9T6b0x{eM=#e9w$eO9t=b@9FOHH zy(ov>6jgFIjoyt9KDxk?WWr}~N=2e5)rM;pGQRRl=;mrIpuO zTcgAgmmFTGOO!7+V8vou@cq;l09a}Rh@aq#U~NoG=lUiRD`D%zsj>*Bl?Prfq*|`U zC$oX1EJ-fm%?|MS6^iW^%BH*K;$nysELb5`_G5@SQ_6!3)oA{`t+^sZE|)d*X@I!6 z?D}`N-Uh*<+k>Jxr7q59>G7pkv1rqp#khrkUy8@2sh%^rhOnY#4}WQ}k?6Bl0@I6F5SZd32o3@0P|IZH*rfl=QXd0p3e- zD_%Sp_?N!FNC&;ig*7oooHJ|f6AVT{hv9aXJUv0P1In{P=5Qhl`3+F4e6whQ8-RirrWnNTbRg*WZ`G#+S#62eUU! zf=%9`x+AJ>&|?~b93>OMoIa)IeJnXGjN_#cV^j)s1lzF+6@6i2$I zPH++t<`XB zTf<4WBu`uT_AW<@g*%A5B4tDMdy^J^z5JB)$cc*?*01|bZqlqy#A)R3eO;T~X=O1| z+j|emp%>>zM@82-$WfnuXVpwF`(2juDPw!}1Oh)kIZKfpS|1xdu**+;=qd4dL^J+v z%ZZ6y3~f7BEfk~7HHMU2R~e3_bT$T^ITQXfz^ghPMfCK(p$v0HZ-M!+cd>o8hSHUl zCaW2M&@78;Kva_X*l=&+m5)3Xk-r>GfDDH6YQSsu_iuW0i6OET8I1@m)p6j;8i3zX z=oJL^nPwR@>Y+p}UTOOL7CO^%y5c(9a#2QRxsB@$OSw!7p(ufsbr?&62NUWJy!2XV zBC>fB&!sS!g;G379BhHSzyiS$U{&dSlJhfh6upu98l~p$Te`bBC8GPDq(AOE@Qiy9 z2{MweS;Sqlep(eoZskEl{%SDMM4|$*7j4IaQ`72(nG*W)K)4-D4JloRDyobR)ddyT zKVfJ!7SrnaMkSSmMIz*U)q6V@KsoO?)1XZfSKPquNY6^g>M_EftG93>#Mr@9$8isN4zVGg?G34 z4qyx~Za)uwB2culwEEmf4BGk?OSQH>__u*9~B=6?~P6l3i&3$ zN5^N(q@MxKbYBb_gzi{hB-EK^=_`~D`C!r=<#(+s z5=we2)%4GR$7vDUQUJYP(fz=FR^Dvc*hOyVZF-ki(}St7FZwkE%)iByhP^(;DVX*c zZgQ9ojHGy6zs!R%yi0dD+3xlCJHC}Pb{Tq&{RD=0Bh}zcF;miRpOXyVjTjBT>*>H1 zvi1U(Q9A6S(l@-mFudAlEA-Xx+y#hy-esHjb|08Y$*cUDltyyb^3LhsfL_{j`|EpU z3axJRlt!x%cQoFyy6!E_O|t8 z9gCtf1*9T+Q>i5cs5O)dtpZA?w7n{Z6;d4;%WS^9A!nt+4V0%pgNbT~{issXHa>z~8 zK84M;->G9(_xh1y_8)TsT5}YzxWh0WLN&zPZNhKfTC?-I&H+2?xIoE{jk6~IbntG8 z*J)Y+$0>~lcI~&uKS)7)*q@!MW_mbAhIPpIKJkE4UQ?yd4zspOwxXCn{cNHt-aZ^| zbACV80*kvR;Uvy&6bwC)$#P18-=?^^-)>m#n$>tZxP9%T65M{q+t-3$XcB4c{`l7+6}u&{gjo6Q@ul&SIcMPii}uZO2{9 zYA9HbIvZe$f+Il`YHhv!E-qn9vL0;^R^19_M|unBn%hQqdHU_Vlf;67kT^4}rzb!7 z-Jd~&&y=M~IYDDjs|ezaRr79pt(V!aIoF^)?RQj5wl9MWB7&7y4nenc0peb;iCgDO znQaGD@GO6}^)O7nvsNaS{Gos7pvjv}>eQf_amHmY6iz9xZ$0?Soa}F(`JXGYem@GT zJ?z)F>evVVDtq14Cs2phf&RBi7VPleYT}JTVq1ZU9}Fe(66L9sk`&IK5EuurJ_E|s zg5k0Ht#eo8N6;PdGE@{X50W2n8TD`L@>q|!?)TlaK1hh)pyH@@!`+J%bW)|I>QI7g z1<9$l2|*^6ToYPPy^~v2LJ=82Kl}>QuOZ({A^UyQW9viIPG3nKQ_xAxt@SUsctuT5 z-l%I)p;hkthuiq+vk{CK+s%N>=AgWcuyPWj3|e!p_InM9neEBWI~_n`I{?g^1fiVG zl)T)W5+?_`jMb;4Qc1tQc*DMHi)ksDDGEs)U%Tk%^Q|U4XCfS^E#0SkH*^E*xrdHt zQT(jA{Lile@ef30naGU=?s7p~`+wm=#T};=Ugi`IHdu?fN0V<*lciHT>JiJ3`j-9Z z_IR_%G?sAIBA6@SyELW-O6xxVAz7?5L;$j*OWKo_+S}6@x;ZRl%tp62FepmuB9-Av zmn=xe=t5J-WdX=akg2iT8N!(`N6N-4l2m$?h?q!L-lHCcUM%W{8M8Kou;qZ-zNapD zSgka=NxUtIcWavCz|Iz@pK*=`ZP5DTKqZA6sUhrYv5&i2xB=ZE}@8n$& z+ly((hx9t!>N2=2={OK!7oGQH&F68+uDy?OS0>9bX5|JC62RJvId~hEmt@XIpGMB7 z|6G@!eXV)~)SQ&Nk~nQIpgs}t;SJj{<=Asn7;~h6OL*Y$Eom2^VG)v4;vYzaV&SqSbf=e!UAa3I)z#i=tv*)u$g&(7Ct*;1FN6Nq*lI7;PA ze4QKe==gxwb(-}TzEf#)iSjN_t&*T&J~w56e*<&~PGffbUg`bT@)k&exd_ZV(ed4L z?IN9+CrJ3(IhN&dbEj&4rr+*gXeY(c9;A`SWJQ1FiDI1S!Us%aH`n!HwN9}387s8< zJ(A+y<<}1Xl~>GhxMC+%>U1e#t$2_ZNDtVP_s#9yWIMaZ^VB~5l#=V##Y|$AA)WFx%e~1j zoG4%<xw;0)< z?UFyf_N$e9@DL-rM?7(_y^#u-Nh=yRgl*V-58zR%eHWV$_mDPkRtLXeYTK|Ja{Ktv7N3U=vO^;6W&mXH>hwp&~X6?A$hu&ybTq*0m!o}#rNpgp-Y?2E@Ge20Sf z5fcpi#p%oZBrlf;V2{C*}Ho~KxD=EiB$ymvEHXwUZ<#Sd4f^KDVTKmg}dxNK?ZCyl~idND0K9i zw}<%Aki&#k0&fnpF^c@GEBSNbfBzHq7BPL#Lm)6d25RNYhKnKNHSQ#G$0ot^QwOYY zbD>wdu21TDh}MiB=;2bAD|VS#ngGef5u&l{1o?Nqy?uBMr6d>C`3iG@MpnQRb>)9) zT#r?Yp^_BSaxX{NeN+cZq#V^MFk<11(*eFv7O$~Cn?^rkPz6jCuPnAC zuKN7nTK{Hb{~wR}??2A(I2uM~CLlKBHci}spP^gvut|@AJ!c=LPH(cdcJN=H%ur#D z6C>H=CIIO;fT^!BQ2tf{&_vp>6FDAWd}jhe;B;Vg7Vsl15}09F*CJWRfI;;jUR$tf zG9d^PxxdHq@RnnerECbr4yL0Of^hw8LDPnK#@l?M3w!|6o0)E-v>ecI<}bl8cE)D| zT*jw;g?7`Sap;E0q%qKhSEi0?e9hAH{Z$JJQ$v+Z?U&>%%5BiL86)0QYEQlg3_S-n zPvY6%nWvBpX#^k!7rM75Tdl|jH}UPQ^0%yCfNMR~6C0>*2=$Q#pw3dcuwvanqE&N& zqVij+j%?2nn~oJBVAXq5a#ZU0BY0~HAQnh;`xNj^=guT}1Ee3kSpc|(EuQD`S53&P z&II6|Zx6l=c$akDlmu>35<;;x3C)ay5QjBxki_(WNGHdjwGxt458C?LU8ajWI@dcQ zh%jlP_mi`Afy?EyVMp>rF|@DxpyIu6l5>yWcVF{Gb8|$-@a76>I1rmb!$7yKe@v!Z=1pyRd=RW(bpbx{SnrVV280me^#&S;9c z?ws0)C!>^k5xv~om2dK+gu8&(-2`okw;u-u^Mk<-!lS3PAtqY#$_w6+3bJGn0TIpf z{_jY_KTovkL=;RPt*|Eja!W+7{yDlK?5oQcXAx zb({Q1Lh^?in+;$D7EK3VmWntc*p3!qFL1efy=dwrTY~R-Th4!?yEaPwe{l6(g_7h}! zl%Xk9HKhbr`~?;&PMQ=hS5Z6E>3E4xj00y>t5pQ^$!OC?$5(DpTh$}?>a<6#J+Y&5u zuE%l40XOcUrs`sMC@{u-ojvR&zQrVaE3i-#64nWb!m4 zxsZjTLxM^+slvWwf%_|*cmfpD{8|-<0kS&Qj!=%(sathPUoRk$T%sPL|Y3837;owK+R-( zBTSu!i_X!kDJGZcMVqJZ1BiIBjrRs)jYtwTrJG=$jd$nlD{N4dDN9!5HtE%e^V>QJ z?%ZYWt;QO!#7%Pjgo*ACYMkv3>%5@8m|e4mgDn^I7TR16&7u^~2%g&q(7PvPmM7{z zR+qs^paT@(8Y7E24NTaxAH<`GaB?zcGqZg~c(rqlM8(T)A~~OkNxXHb&K<4Hiq=%z zVU7pBFXKhtwrirnk02j&HZu%yfq|k-z*I5uI^c(N9rN`4PFp^r3A%|0IlD!JcAJZ3 zn~QD7;V$XFx?*YZ1a5I5iqa9dkbzPc7klf=F={$XUUJ2>H)lNk6F87C?sUx?wuvZ8 zeDvdYjEG)O2weyyNT3Qc5)sJ{!hORiubL~4gokzM^(HrHT1gL(&- zEUivVbeUWl{V@(cI$mLpZAWkU?{LuPBI;xorgcmgw_@^xhc&p<3FGQ5pO|WrPbb_e zbmnIbF}`KUn1LEfW1c1hZ-Y=1_mNd6DI_Qa&p;IeX?duyuN<0 zw_r({w&i*p97`8G93(nn3BWw=48PTNKiM7WYOH9M=@zq)WpoZ0F00 z!Flz>X*PcLq$n~N{zSS9$kVT(8k)-h{A2>35pM4#@gDq%Gt865tyvV0DT`TJEX2BB z$Z4!|&-FL0BKawZBe3&}(phxP!N}n`uWt~lDDzQnvJZ4!-7imGR`rJSA$w(N??}wC zhGCSb<>*BvBmejW2%T!V3wq^~*yY|T)I!v>IjqVG(7V;kRW-;HGbQ7 zlvunn zV~m!%U|rUmQA3ib*(~U^8N@D`$gEb__B*Ro?ndrmoe`_cbPJK|ZkqDm?8dlkyBkrj zCl8af$X7}?-mHSapFred2?Za*FigP8GB}s4g?~FUUwV}P()$zM+$RzTddB*wlq0V` z^2p-@<3p+hC9vUmlNz1W2<{ccl9!WR6V-}HithM_yhjiO+CKEMGQ|LoSPS0L>fasx zEfQs+hP&T9*>aIwbI|rYy#J2*Ufyo}kUoLXLDI5NYx&k3gdu6CSX>MKFNeB10jg>x z8$m`-a5;QL-(!V1b&0qV>nQ~&Y_`G>$CsN8p5FtW;&7xs3B)89k$EK&ytK=?D4!^o z0m?EJC?b1BDj_iNT8iaObLM8l3s+bB@$VmLI4+11H`JXT*g(utOcmvj(F1uUCV-4p z^{da>YoqKhZ)YecBHfVIZQ(S_f7@?k@1lTcY=)@^5ydd#B#TjKu9E|n-ZZpZ*HxFb6jtF-iOKGBBMHL-sTk)<#{K zv3VgTjdc#xI`*;O1g4U+#4%Se$lG6cKB|)wpG)U&ay`o{!2Dmm{lYZ_WMs;DCTfab zz;EUuxVYU@dUn++mx3U^F22Zi&L>Y#!)Ya8WxA`C)3`X(EwOqCqA9Ww%@**pZ#J}N zFF_teAW{UGY<($PZU=sob@kPZV$h@a+7*p?i(tx`c7bG8!&I#xdiymT{HEZ-ZEqp^ zh3c(rOSJdkuHBkPZo}&O71JV!X^#r=@m60c@d_In ztN6HyDCFQ}P>7d)?>9^DIR=h4y}58$Y;F}tDH*@`jSyu7jL((Edxoz1%GpDY4vg zz{B|E{C+1&hV{@Fp*(Y-QoJ7->-+I}hgj|Ber^>6Ry`qFTMz+fe#8F@ z-@vsv$S^W72lqbKdCAt-f%GyTL7+Bx_6@EPTAMuJC-tGr5SkBNzifvT-Jq;eiDLo$ zO(XibAs#HJc1u8$Z#10_zZd|gOFmv1b*dAphHFMvQ%(a37Qy>5;<*a18Fs^66{ zR~J=T_ouDVM@udDRHUP(krEY=hIFpNCBwsZMS%q1v1}+xiarELtTbGkJCb%jajMRp zT3(Xzhl3B`8%CmoeNKZnZqSmW!C(BW>{f3NqpcZ)V>A`~VXB?6db;>sv2w&Ale(9(6^i;Vy;O5Hq?k8E2yM#J)J|+1E z{n)nRehu_1Rr{N$_sFa!)j*$f&svza^(E9nR@@AY{@vRq(RLE9REoOiMY%JhRsQfB z7o8zef&7Q?G-uGgdu4wTeXOiy6v1%J=h9#ORye{<(dSf_Ni8Kc_%s=6q2 z;U6XLAOAX|4!6#5)H&g+s~74M3vJyauB$eH`st;zYFBHA%BRbanhbak$h}xxRmGko zIuQ$T0-r_>Ja{Yl-7QUvK}_BN0&hW)nCmKcXXOHEyaP?wO*xLR7V2IM#3?ocr^W{C zy&NNSX*hID;aam(ZeYD=4@@5F4>RJA<^p4`HZ;j+(4f;NrPfXU+H_;ZiNN85y_H50) z^rg>>7t-}f_BedtJ3K+ck8|8eh=7yFL+hC!GWyQ4>y#rxRm~-Ug;4$}xj-|5MShPM z8UI-zqF91u5jQ5h_xSHXQkW87!1)w1CA&THS=TlbHU{VxqO|&rc}V;36@KkKIh&3A zh=`j8$oRZ|8XRo@8lQ`*QH*ZuPpMAq`YDk2oFX+C_YbR(Nh#0Wi$1}+Z#DZpM*sOq^ zkQD>6&eTouesNsp7nv+S(*u8&Wr?TTp(h^1dX#}ZFf%`u67lb||6II?%8bx~zx5OH z(KxIyn(RA!ry$y?M>Ew7Mphw@R05k-D35{7Ud|cPn*<hFz=TtzSTt=v+D*J((Fh<2L(!A-g|2Y!BDwDRB-_92)5 z3|8U}7en%`^JbKP{tbVg7)*+U01i_gJ@AL|6L{1R^|goO;Y$&O-x7)-5Yz|@xUZO6 zI)x1Xy!L@bhorND*@7uBfShq+h)jH9Q|7?`X&E8MDe65n{p`w*uKG` zP2RJvEu$AkGLivrveEZT7k@d(L3Ndk{Bhiz&wn56pJygiEs+uFC6{S4~=@!(g%%u-R$<89adJVOF2}0U9(dAoq0^Zlsqu z&GlUZ2~mDl9#B6Ov-numuJ7;2cI~DGF2tV?{XhOf-lB;p4wxIUYUCTd@uzvJ3G!Ef z$}&9R8vMHd<+;i_BZhMKRr4Azkf%sXkQUfALPXr>Ktm`D3OX`K28h9_aq3~x;Qz;u z1pCmF@pFkV0nP_%?0_uakP(fJtq)>N!pkqoS(pvee6Z^l0W^x)eFK2f)R3~TDKv5^ z6B&r1msEi$B!9{>x399(cO9zg7JVUVhIA(O$U2#y5i|Cu4)ugtN_F$v3Y`51#;-k4`KKk zd6bXCVh5EH5?S`r9}`-MM7W@=@QT5=X15Z!X7Rr*i|EsXh@7_vdKp&_91Ki6W+7G^ z?#~~co>ke9F^CtjP|ULqJM{U8njeB8EhU&^cC zJ^nXa!VVSUngs&3HRvfTV-P?C%5eS#A4@SN(tYol=x!LH$+|nnMCc!MDOWN?^jE4QB=jx3xe)qKpCTK-^s6T#FZQbvc+- z;c(3XJoDt{eRaqnvV!*VVbmBR_yqEL_2|$VXo&idAYS+uz|y5gTt>P<<_Gyf$1cU* zhSPryNp9r_cmK&kHzY|OaziZPYAT?&F0Vt%$xaDwHN?x)d=i;=fZaL>y06u<OPx8&90MjTJdZzWdK@9+uhJ%2yR)WH-4ngoLc^ywEvvyP__uP}y zlJ_=!+M3F@uc)S?YGYZWpzubvXvbCKH4v{l zP^Jz=lN*hwJ0xtnKj|FL9@Rr-WCrSO0O~UTjBnVsp(|Hcz{aj{(p2{a&#QVGijreV zN2>rUj1Qw^aF^bcc;bG{9L zFo4H8Y`&~ zl2DLJlrG~Vi0y$n36;Nf`6lFWXaOsP&1+{1*eq%^7oY>h$G3zYQlpiC>??p(zI9iq z_NdKk_lv835jLpxj17=hBJaR7VOc3Go?zQi}Q9b5F6e>YuNW3#*`o79@2G3sh@sltP1PUS0kzUZGLD=WWbq{m| z3|q3^!%K$m%1M$VdxK`pM_Adlif!v(=-bwzI9@4=V^|NW`UEA6!=tgSZy7-_tWYkE z6LxP7Cv@LIGzuG{+R!u|l1bye1ZiUsK~UOTEa9%IsHTcu@A?XTL=#8;H#qg|#oPQ# zfv?W!I!XXojlMx>s~Aj1=OQg~?IM&%00LNMQ<;VV#3ie=ZI;i7J3rx|`S*1I#XC_u z=I`CnoSpz|vfB)Qxyqt!-?JLOM|E3YsV)uBbK*Jiy1bdZ!uNix@Kk!O<@Ca~=DvJM zYI$R*OlRQLk^k34l6LUHFz(u-+tu+aSILl@C3xS8e?fHwpY_Kp>j!fE7MZSa2e&eL z-uSCzyti-V#I?n(D&#%)6~yvxdv1Cc^rIZ8b(XeH^1DlR7T{G(PzJPaAXZ&|?I^vgqj31*;#T|VpP%># zfGf&`K|vI^KragYQV`~Cr=2CmBeLcIIo=Y_@~D4j;aCNtca&9)X-8sh`?8*pDuQa! z3+v&C7@nyBoNoqKyGzz&TO4EQ0B9bLIlCFNtE9^0T`)A(dcw!O4O<*c<(*p_2p;ts$1{fqq{DbQDyMa6+0n19mb z1cT(SvfK`QI_W^G2Nb9BDe{;<*5Jo>`>|l;1e~9fX#!|7)L{;NFdGksOD)95x^|ok==VNEwZ$wC%fQ?woIgcIC*?#2O<=v6aSRK^oqsAe` zKW=3hqSrc_!x80p5pCKXUIF^GnG=mKSi%tv8)VQh++yD}gD*O@AknX*szjHt1nRXi zu-Pd0R*b2KAJqqktZWnZ6FK#2*!0yGMZVrx*dtUnT>Z;%(Z*OtP``P7*0TPp>q&yd zKNlkCEur{~@mI2bU2Dqrd@{Uvv;N8f9Y6$RMkv{!Q1zn1{W{r1GYRR4%%Kbe4-1At zh-Qhe9(L%;(dL`Gc-u!>=I`FSFLXXI2`ElW>f}+Bzc)H5t|4R zw3BScb^A51){HjZ(^R{wpYx6q(|CDQ+>YGS@zFNys>JUXz=z>|zU5MD;Ir@ZneGd*S}t?|Xf9-V3{K3gXGIHUF4D#st61;5TWA7raj4 z5zh}Ab&r{;cH76#N{1ZG9Ds~Bz6TY4`f-A8g-t|PJ@Jm4h{h4yp3ty;NGaC!)1v_K z%uN?1V1Vpe8t}uB?lYbJs(rV&>P0L`;(!FL`08#YT%}jfqpTZmmcqTk-C}WU13~2A zM)@*-E^`uwNrP1CI)}dtDrrguUY$5*0nt4kVCnW6wWjB_y~@0B{)NMj1AUr~1t#1E z!y*aZeNtxybWW|FJsftFDe|DnBjG;=*-Lq~P)y%v10P(kr^vbUV_AGmQ7f_67TH(i zjq@Hg56L+7U{}_z(S4N=&pv;x|aPx^>5tJVfAhQuS(;4-^K z&t((<=3lLlHp=G#(A+$v3OtLvF5bY3$*R7hmjvnIU>?#9R`Hq`puCZ>DaV_;KPJ?| zCR&_P@-{W6mF|{Mt8m?x?V8-R|9WobaJ$X#JwdR~KDZW0PWPYlrX08P2MMf1t?%e$Q&Ml z+4&aPT`uqLEFB1dxnxrfOaBU#XbXOMUh6|HFLRn^P*6bfpdMwW>!XONWt3^4-Th1` zija_eMYrjhFQ*l{vc&fgC&z&Ftw!1}4UK`JRN1y+pb$0h6&2QSc5UiDN$CjTy`guH z5?ukjBwzbQXGJvl=-Fg>ohxcOt0{TJl7Jxbkn`)QUzrOqMO&a=+PSbyxg~Dy+T{9s z_QJ(Nfge7#f|ZmQL<{`gUmbnYc05$Qa&L$264a48efK*lrVxKIZFNu(!|Ll5x&ta- z9SXib+&bI9b({Cg=7h98T`<@9FvGGJDL1CWL_l*@3c@BMr(#jD1cW|_cnU$sTi^5H z@k4;eN&(mF0~}~*!7&;E?IT@sg6|n+%>!jiPT9c7lo^a;RblSU-g9xO-~zHs;v=LY z#A?2OBIZlRs$aqqBmzI9?5Nz8IxAt*cBa7!hqPXmQ8y%q0vgBt6WB*{_FvcH(%be0 zoSTVA8u@iB{(Ag>?C#(1`_GSwQWJss_En(c36wQ}t`xb|7W~evSHkIH&nE+5UKn(h zdLHvUHmdDKg=Tu7F1C*FlTxpkN_-+CmN@pMSHusiCO`JKtl|i zT@#tW+{IZaxD7nB&xcU0K;Ll?k`5bLIj)p|Q(~%j3lPep{O)sV7ADI^aJ>alN^q{V zD=*Y{F8z|@GgK%W;oYs}3ACANz7<Pt?=lTrW&(aLG|0k{ZhEA_azGQ11 zmzS-7cOd@rqR~l^q$qx;F4 zJ9-O%CwdJ=Ygad$6u$^YUdvZQQ2Um`Jc_T`9mXL@nUp&A9?KR*^k!>hcgMt^!9JkO ztT_%Cn=}U{WHY`(J3I(cve{r%%&ba`k*h01Se~b3pqilE&c_9N8IKa6e;A$Kk*h@|St@|FX%9!o3ME zLvgiOXmrATDuKgx36`Y@iEne7h6==?|D3$w*Ef`y^rxW>3=U7Tl13X~#FC=1i&XVu z8Rq)^*S)sBO5*NOF@T&}Z}wwF(QGd$?&IzVL=-5ASF>uU5`@k)K+mqrWFi6Xk0QrM zzI=EYQ$WbA3f~9~WUoiepHqF=;($lUM~U&%3sj|rc#(FBKtfH*hP3Km#zSjvfFg0mt+V@5yxd}=8G!tJn<`+(Cpo( zp;hHbR}=5`XQdm~qt#Jng^bVK3wvil<8JmtHM~D7JD*5Gbx)qJE%qCnYi=)nCRg|^ z%JJ}c24fl!v9IUUmiyUGJEJ#mEiUs3!xc;4#^3P&GqZmqDoPOPZiH5hH-hyVI?cKs zb$nB)M+Rj!6Rdi}C$VuUvi~fwsZ!(BggUk1Eya31^~di%Tzw~QGqv~U#r4lG;eXn4 zqFtbWl8}|!`=7q(?+!%KyNEtQfAan>P~`8v={#*Y1K3~rUiBwmL z+x+`Y{Ih=hw@)zf5hGm<*$F{Y81s~zy74&q(H&^F5Ksrm4S~S6=9V3F4GrVaH3!{v z%5kSIMHtFz+t@|MVypeKqM^P7wT(i#DjR}Y7hVSuo#yLixqtgX|5_US{)L#>2@&WX zP-9F39Q3%tM?=8G&AkV7w@#H2eKSBjoxufY04FapuOFQ{4Usel@H4CZ9>7%22R>?B zBS7|WSeF9RxuUk0n@8p>+{sM);jz8j6MXLXFLM53a&83pLj3-~-~TH>OH}hZ=rzri z!&Zw-Bap|!EUa3l%C13#!<`8trW@M8nDjrW8v53J^aQE2xc(bQoQL-uPlwv3__L2+^fr#I_0Z151d4PaT^>%p>@8 z!3nrr_#8iYus`Bl(to4=iIG?mfWLhcckwAhUE|c+yPxI72AP8#-Y2zbOSp~RfJ zQyXzNH@&7l9GJ_b$L!1T+Ku_Zi~h7bXVB?<+W#5%@ubG^n*=$ABd!f$JKu7~Id|IT zukmhs2p=iN4lwSlJaL}I@(n(p7qEQOu0H?NVQ@bxV5elB6SLIy#InQJ=69x3&8C>$A(z1CQ=&&Y@{DDV>*ly-Y^Z*wUs^q1J{KBdyb1=x8+Al zde2pxtGi&hn43&a&@VSeIe*ZMnTM$#nMVNbWuc!l_RD^OVi_~cVuHuqc$t1aW^>3) zvNwBTMm}oAM`-uUQf(=_7GIgPDr?1tiv;$l#tHreZ<)~3EFon*R`O)J$L)63W*3T0 z_6wObc4ja&@eR|P;i{wWB#n3tSyy8)RLoj$P{nJNC1u-HW8wMpGIEUz8KVl=4b#p! zR~m+_=2*R;?B^T7n~Y9_yw>i~Ld%V~wc+zevv+ai6+7m;NfJ*F4X&}PuiDlnoRtpC zAKvTHk6{dS2^?Hp9BZc3aM$h?kR>SnucyXeOCArdJ_w1IhB$aiyoHr|ymB_T2oV>> z^gSe|(W<$(edPmOBciE9faYdYai0hzM6!c6Jyz^&PRw-WS;cWipnKQLTjeafyIQM9 zWjy9KBwrHV&@TX#bKqm!WdHX|N9#|#%cRQ6v99M|8()tbjPbNCTNa2lSr@bom))5Y z%F@`lP}VZb`P_c9=1onp1~V>A&Fy+~n(T`2lQ(#s z);3E&t)tg$m6p;P2UIQQ{aS~HzD9d=T;*kEj$iA{5~#Qh=F3hKy^Ilp>vO}OXEyg* zgqp3^f4~jdYp#|rQY+VHpTOcrObpYo5i5QRaW-$NHfFSQPl)_)hldL>6lV>Hp)>`U zUmpMuX!Y0kKZ3PBadhX^6@bwL^a=c41&%m~w{hJxSP9^>dRxe- zGb{dr%ix5)X@-#p|LoS~ob42vK7pi{g@$Vk1~j@o!kWj~U5%u?SoIQJ+bioEPlOfE zKeT>@AK~3LuGw9~9M@a0RiAvpDllM3E_j=cY-ihLxU31IM_oMc|5Bt-WVdf7jQKlP z&N#+~da&R;gJF&3M0P(~UV<2FbSL@o_4_yLdtC%iyVdYzk4zCNtCpFsW)7e8F+r;x z7yM*4xYO%QV??J>_aC78h%*3Z)pZejKkl*(0NWMcPs*D;DuOHy~$Cq!G8m#KzC>F*Z6Ut)^dP1r_ z6SI@X8WZzs9G=JvEK~AD-!11Sy0*%QQBQNvNy@R@D701+Z>d>nZu;O?QS<#9nX^+F ztL`~<8v%r#}kmUs2+BlQsB3Mu~16%D=m0IlaTOu zwea-z0ZvzBx)@Ou)sv%rfKIP6q>X0pF&GSrHlqN6aZsXr>f+Nal5MB^vCO7M*UPLd zU}&h&?Uq4I>DNMUcVa37I+cCM|&iAsF5%2r~Zr}OMvm=A#P3i-Ui|PZNi(&npHsL5g%sTn* zf{k9o6e<`WZh;^4+IyW%MYHXofr8 zFfGn0GbU{1N`WdaAoc^DA;JG$FDnvV$VxlWVRTsb7ERT5atepjKi=He#E3sO&;>2v zjx9ypgXW^d{YVb!UnK}b+@{-`V2-;lH3B&{@2~^GB>RRL=K@M?JAYu+ZaFW?>XY_j zYDSHPT24tSyZpgSkM}j=c?a0HT+TsZKSqr~&vOV4N{6vb6P|5=`bKC0^On7rVgY*p zqEk1X^lco+i8MbYl!^1Y^XeJHj+VhIcdzq}jGDuxTh(t#Z5H(x;?{gsx0|@JhMUht z+f7X4Elr2MNrn;Yewhe8t}?h*fz2a++Wi`R(*V;SlOLS9puSMk z_ZUXEXTMx(Hv=vQ32-WoeirO#Iz~h(OH`|3tw2gXGLC2M?6QAp{0yhU$sS5X!)!_8 z%s&DJ8!5dAzGhEHL0+IkRNP>K?wi}6PM6ed=gm7SMm_goi)$ntNVgNu8qCk_w$M0l zRvFxFq0&ot!YO*Y@N2kK%k}2bjQH+r;4Bb3=!!}=X#c-6Q>D@i$s66o>R;0o^eCTpfVSxL5L)B;Q z-+_Z)t`XRzZ$K@7{_N<01)cdl(EP&Gq2K|o4~z@q+W>GpD03O2U=F^~f`lfGC!Kom zPUB#g$@q`&P^+v4KEaqYo;R&I!n{(;SGJ;d)>7n>YI}L$XNfw-~3S2*!VC|^4VG` ztbX@!yFcPIV0>(XxbOk6(FhoqaJcMB4c((lDb*e+;1oD~rVXhNno-~0Ro$Sw&eV!} zp1aY;*>m=01k3K$=NcQM2hN+zfi?LamFKa+Qgk~y{p^OC&jrRbvQ*1bE$nCB?Dl(n z9(KV|WvLX0r4>u9=NilIHd`s@DGFiM>o6~Y20acZ#aWc!3CC5kxskofoS#UJ^LHTwk?Yg$ z@o_tQI^cTQANH?LUFYuG8>wtzbR1Z(HiFxm7!hXgZ$22tLUn(vahuRiRVaV5N}qcN zn+mmJ=!c`JeVdCi&bXM|xhedFap%{1VgprMbh-U#ErCbjF3v}lzj`H1djZ=Ya#KaD zM1GYq+R|bEbArd_0bhk}s-9fE$x^r3k^297`G^LQyck0 z9$tjE^vkd8wp)*$&A&(%l?mnI^%Ls5v(!WFpZTFG2rA<@ zZllp3u3Je(e|>${TAxS@%e}Y0DNCF*FlP~8!p+P&Ban1{AIi+Szv?Y|AzJXBbo}CA zsbAY3x^xQTMB^B`o`R;g*3W#6G5U5z%$L*s0(RlfaURQ{=|8aObVn7U$YZd>fnb*L1L>xW%iN6!3)5&wOKj2?M&-II&tWonaq6fq1JME7?vxGG&u?m{&(=)F7F z5@|{||0SPv^HKfk;3*aN#=P-LWucHhd|vEk3Z`0Qu{ z!^r#LSl)LtXHXTHmN1Q=FV*W;fQrGA6U+1Qoty`bV}oyRy00P4y^;&ha(2~o3=(|T z=Cbb3_Xms5`QB_O60&&ix!!UoJVDh;W_M<<#lW`*@$ zyYy_BWrdrI-=xxt8R#@-kGw6CGdN19yGr_!pVa&S$~+dApxbxvS@e>$mh<6s+;rpf zRLSx}DC5h@jWUjWP%9Hmc{u%b)20S<&)gLcGa9Y#8P&@?6nvVJ8c6AB6jmEl8PUx( z{|DaqyCgO85oIe3cp=11EJum?-QVv}rE8bvA}RnNd$n2eDXO)Dv()KyM!G}Jh(s2w z3Zl5sk6}30nuA~=QJjjW+$?V>>}^I~SVibU>CQ4mO_KO@`iDA;SDnXqz;vsDMNPO- z=#D2_Ud@%piF?AmbPR*FsBS-t-OUu%7k)t_U(~v-j$M_Krqx;f#XJ#hJ=_Q2(ez*i zBKRHYYPl*&y&>gT#cv%7M2D#AjOc(HyU~!i7qMyV1oj#^kW=YAmEdnYm;fWc1~5Zj z6daE{c=mAK!?~FvmwT%Zt6&m&=Jj;TVNV@U@E`&w;Bpv3Oi(ez{?uVxuaMWJh0k8@ zyMOhs&0wG99lq@VNImVTrP8+ccb5MUE(e|T0cqbLDUS%avXEFYW<8_a>IYi7N)#BL zRw6p&NQgHw*JVK#rGw~!eDE==EX%+>gjeMty=04b?U_q<+wtwW&*Ua&1*$=l(U+q> zVvJ~c`qj0|t}<&Lr~Y~ViMCQ*05Jj;2Sw$4c2TQ+gg(F@rht(YBW@h98rg6oq&6qQQTHtA1TOSC zMPG?UII$>$pliM?+Y5;ki}qhGrH#iU zR9S#S?-NH`0mQ7un?UySf`Xcpli1X~D8PMi{2CY8?{w6FidrZtl0A?kXrB{3tFSfQQK zm!$>^y$iK@i*_-qbhi4f*GJQTR||0n5n}GaQ90YdIffTQy(!%C=B{?=nJxz~owfqd z6-~NC&phJL7 z#g4Xvv&A4x-4^t4AcYJZpIDA3`2ciPvL|s-y03kahX%iej}AvruuME{|GDtX4&VRf z&W>P%dqie`s@p#M7ottdT79pbi04XZhVv`p+KQYkdIxi3FfR%ph86iW@Z`&>I~Th@ zn;B0QpV0XA=Ygn4^0-q-Q#ozHHHN(Jzz~`TjN4hLYGWd5KEIyK0S7H#ag=rC2gqqr z#!)nbRM{v&l=lePWx$k=JW zu9kuALL?NLDTa_vc$gL5<&|?o4v^GMxJ>6B3%SUTn3sV0Uh(|Amfz<$U8%i4S1U*v z6>``S+;7eJ1q=OCA3c)(mu1vsgJzcce$hYC0s`2UAo`_E7-%`Ti`|Ikh<>LEIRSyE zaiCB~VnC~(Lm$F?U?f-3bPow2698dtHUga@p)cU6fxP1-R0Hqey|hB`=vPkYAp!hU z5(GpTgB6y#7IaTAao({$itbfWI~EGBR3CtH)2W=8Ij$`2^_tNou;c=l%r$L`j>LyC zK>hk&W&(DikbIV^N z(a}>cS{(oin+Gl5AvVOE`{Qf{RAUPz$yEH|#55d^^B*BH)2Fh)y~XHD3B%NNmZKdy zP18amv_jt$$)`IAlRd_=U-{(zc#%n|GsZ$JZrzY z@Xwsj%Jh1@c?<5T?VMQFK~$f)*tLO>NGYcn zUpB2!zKn5tH|0eu##FL0l?V<$`==Kh}A4MD-hJf`6UOVKDrjJwst!MvFZSCsUItCcIMDWlW8RRFFhQ{59;rAjF^ zxlJkrLF_c?Sxf`%N{GXZ&K&A1zSm@>od6e^GA63$Ro%~CB^NiIizn6l`f6_;+KtqAMZlW3EdXuH3b!DNcM=Eq8; zpVYd0m}YVDF)sKouo5+|_~J{p#mR*zb2B~RT;y01;0cE_vLapy<>}8# zB;CD8hw9tB#g=R1<)L{AKqkJzjcqlxV@Nk36(t(mrgcDd3)6jGY`K{!h>DPijC5Oo z%dy5ia1pzBP`5!4YJi{^!v39MVFB`3&-2OPtmrFGpEqU@*c^+a$=ifkWm>fTV=SCp z5m?Ntk~XI!m!;{hF*Oi0?KjX8qV;Z+I;uTQc%*%#)> z>*KQ%Li~3mu6Qw?lFhO~vMiY?lnC-0htn)w8dEcx9q>Vlno{ET!;s_I^N?=4aJj4h7ywAL7hMq^4WtT7Wqbf>P@pLkP_8nPyr1Hxk%ViR^Js*%k`D zA$aQCsYrD-d35(25N(=t1l+zE(NLtZWxDG=WfT!kuj#0Ee!W^>Bjc&QN6(+^_(wZR zs72BDyJxR8q#9I$vgj4zoALBo5!7`u=<9eKfD{;2=szK+-3bHT5Oo1)4fiI z6JN47U(p1mXGajJsiYcEdF_`gV`>h}F5i97Z`GbSn~Q0M;Mpg+gh{J(3i*YP^)QX6 zvY%EpQfJ2BLEEZH+(rolBXOD6Yv(SwNwy7@qU-9i;G)wYw)bQ;aB>emEJ;tmvndp}K* z&|iD(WRufYY?-fkmTK07FCP^Jt(DtcK?7l30>1?6uGHVhPjAd&?Ml^y6s`P2fBkXgi2s?6i!sCEL!N>z#WY# z#DI*31c*@1d|5Jn5Jyl}PbSdZGV9)1(w=yOnD1jHq1PgeOKC35SS@;~1r_*;QEh1z zn<|EDu0uHFs`{ZwHSn4rq?DSYLicw$b*9SF)97R}iOfSFW0f;hd9YSug8+jH8k?MyizKbRI3wi-^4Or+Q5V{36prIh)a~;nmUY zS?Zj5+CGy{^+tU5|0T|W;@$G-6)=A}ZG5Zk_ zZ3;H%*UHq@8kD4lIf>c1h}OdjCB!1CGAY+jP)~(o#NQWJm7MIeuz@m-qqb{MZBn^T zlcn*<3_02AOf(e51RiL`RT|REIij*uqy_URfBdX#ZyZ;ExyE-}G#t-hemy_^Q66zk z2z;0|%dQ-z%(s3flqe@TB zPGo89N*}9*C7gReFRbD70B&3RsATSFJg{Dm<@R&4<80@N9EsGoJe4S3swZAOm-j3r zmgh`);hfnkTPo6sWq^0f%~r+xc|vdESn1R>HBmTaM>%Y5T?p3-7TRWTQX|9VFO5Cs z+Ih%&Prz#_oNIsP6eK=|+YWlrQiCRHkYtzx