[Feature] Make PPO compatible with composite actions and log-probs #2665
Dr.CI classification results
{"FAILED":[{"workflowId":12712552356,"workflowUniqueId":55958309,"id":35438401849,"runnerName":"i-03c3a70e5ec320267","authorEmail":"[email protected]","name":"Lint / python-source-and-configs / linux-job","jobName":"python-source-and-configs / linux-job","conclusion":"failure","completed_at":"2025-01-10T15:42:52.000000000Z","html_url":"https://github.com/pytorch/rl/actions/runs/12712552356/job/35438401849","head_branch":"gh/vmoens/58/head","pr_number":2665,"head_sha":"14e639d0393df511e07efbdc1ab8ffa474eb2e04","head_sha_timestamp":"2025-01-10T15:38:08.000000000Z","failure_captures":["RuntimeError: Command docker exec -t c2b0e48713e98b877d9a703af3eff1dceaf0752d2eede17f2cd6135ce8b6a95a /exec failed with exit code 1"],"failure_lines":["RuntimeError: Command docker exec -t c2b0e48713e98b877d9a703af3eff1dceaf0752d2eede17f2cd6135ce8b6a95a /exec failed with exit code 1"],"failure_context":[],"time":"2025-01-10T15:38:41.000000000Z"},{"workflowId":12712552245,"workflowUniqueId":61790681,"id":35438388541,"runnerName":"GitHub Actions 492","authorEmail":"[email protected]","name":"Continuous Benchmark (PR) / CPU Pytest benchmark","jobName":"CPU Pytest benchmark","conclusion":"failure","completed_at":"2025-01-10T16:00:16.000000000Z","html_url":"https://github.com/pytorch/rl/actions/runs/12712552245/job/35438388541","head_branch":"gh/vmoens/58/head","pr_number":2665,"head_sha":"14e639d0393df511e07efbdc1ab8ffa474eb2e04","head_sha_timestamp":"2025-01-10T15:38:08.000000000Z","failure_captures":["Process completed with exit code 1."],"failure_lines":["##[error]Process completed with exit code 1."],"failure_context":[],"time":"2025-01-10T15:38:25.000000000Z"},{"workflowId":12712552245,"workflowUniqueId":61790681,"id":35438389468,"runnerName":"i-02a885b2ef3b01135","authorEmail":"[email protected]","name":"Continuous Benchmark (PR) / GPU Pytest benchmark","jobName":"GPU Pytest benchmark","conclusion":"failure","completed_at":"2025-01-10T16:05:07.000000000Z","html_url":"https://github.com/pytorch/rl/actions/runs/12712552245/job/35438389468","head_branch":"gh/vmoens/58/head","pr_number":2665,"head_sha":"14e639d0393df511e07efbdc1ab8ffa474eb2e04","head_sha_timestamp":"2025-01-10T15:38:08.000000000Z","failure_captures":["Process completed with exit code 1."],"failure_lines":["##[error]Process completed with exit code 1."],"failure_context":[],"time":"2025-01-10T15:38:26.000000000Z"},{"workflowId":12712552335,"workflowUniqueId":60501564,"id":35438398583,"runnerName":"i-00ad7b8eaa4ee8113","authorEmail":"[email protected]","name":"Habitat Tests on Linux / tests (3.9, 12.1) / linux-job","jobName":"tests (3.9, 12.1) / linux-job","conclusion":"failure","completed_at":"2025-01-10T15:50:04.000000000Z","html_url":"https://github.com/pytorch/rl/actions/runs/12712552335/job/35438398583","head_branch":"gh/vmoens/58/head","pr_number":2665,"head_sha":"14e639d0393df511e07efbdc1ab8ffa474eb2e04","head_sha_timestamp":"2025-01-10T15:38:08.000000000Z","failure_captures":["AttributeError: ARRAY_API not found"],"failure_lines":["AttributeError: ARRAY_API not found"],"failure_context":[],"time":"2025-01-10T15:38:37.000000000Z"},{"workflowId":12712552341,"workflowUniqueId":79519303,"id":35438400452,"runnerName":"i-063b9e63593d78635","authorEmail":"[email protected]","name":"Unit-tests on Windows / unittests-cpu / windows-job","jobName":"unittests-cpu / windows-job","conclusion":"failure","completed_at":"2025-01-10T15:45:21.000000000Z","html_url":"https://github.com/pytorch/rl/actions/runs/12712552341/job/35438400452","head_branch":"gh/vmoens/58/head","pr_number":2665,"head_sha":"14e639d0393df511e07efbdc1ab8ffa474eb2e04","head_sha_timestamp":"2025-01-10T15:38:08.000000000Z","failure_captures":["Process completed with exit code 1."],"failure_lines":["##[error]Process completed with exit code 1."],"failure_context":[],"time":"2025-01-10T15:38:40.000000000Z"},{"workflowId":12712552352,"workflowUniqueId":82165822,"id":35438403792,"runnerName":"i-08d3013a6d303dcb7","authorEmail":"[email protected]","name":"Unit-tests on Linux / tests-cpu (3.10) / linux-job","jobName":"tests-cpu (3.10) / linux-job","conclusion":"failure","completed_at":"2025-01-10T16:16:55.000000000Z","html_url":"https://github.com/pytorch/rl/actions/runs/12712552352/job/35438403792","head_branch":"gh/vmoens/58/head","pr_number":2665,"head_sha":"14e639d0393df511e07efbdc1ab8ffa474eb2e04","head_sha_timestamp":"2025-01-10T15:38:08.000000000Z","failure_captures":["test/test_cost.py::TestPPO::test_ppo_value_clipping[True-clip_value4-KLPENPPOLoss-device0]"],"failure_lines":["FAILED test/test_cost.py::TestPPO::test_ppo_value_clipping[True-clip_value4-KLPENPPOLoss-device0] - TypeError: Got key of type <class 'NoneType'> when a string was expected."],"failure_context":[],"time":"2025-01-10T15:38:44.000000000Z"},{"workflowId":12712552352,"workflowUniqueId":82165822,"id":35438418213,"runnerName":"i-0da7c534af5d4a4b7","authorEmail":"[email protected]","name":"Unit-tests on Linux / tests-stable-gpu (3.10, 11.8) / linux-job","jobName":"tests-stable-gpu (3.10, 11.8) / linux-job","conclusion":"failure","completed_at":"2025-01-10T16:43:42.000000000Z","html_url":"https://github.com/pytorch/rl/actions/runs/12712552352/job/35438418213","head_branch":"gh/vmoens/58/head","pr_number":2665,"head_sha":"14e639d0393df511e07efbdc1ab8ffa474eb2e04","head_sha_timestamp":"2025-01-10T15:38:08.000000000Z","failure_captures":["test/test_cost.py::TestPPO::test_ppo_value_clipping[True-clip_value4-KLPENPPOLoss-device0]"],"failure_lines":["FAILED test/test_cost.py::TestPPO::test_ppo_value_clipping[True-clip_value4-KLPENPPOLoss-device0] - TypeError: Got key of type <class 'NoneType'> when a string was expected."],"failure_context":[],"time":"2025-01-10T15:38:59.000000000Z"},{"workflowId":12712552352,"workflowUniqueId":82165822,"id":35438406509,"runnerName":"i-029fcd47ae52c729f","authorEmail":"[email protected]","name":"Unit-tests on Linux / tests-cpu (3.11) / linux-job","jobName":"tests-cpu (3.11) / linux-job","conclusion":"failure","completed_at":"2025-01-10T16:16:16.000000000Z","html_url":"https://github.com/pytorch/rl/actions/runs/12712552352/job/35438406509","head_branch":"gh/vmoens/58/head","pr_number":2665,"head_sha":"14e639d0393df511e07efbdc1ab8ffa474eb2e04","head_sha_timestamp":"2025-01-10T15:38:08.000000000Z","failure_captures":["test/test_cost.py::TestPPO::test_ppo_value_clipping[True-clip_value4-KLPENPPOLoss-device0]"],"failure_lines":["FAILED test/test_cost.py::TestPPO::test_ppo_value_clipping[True-clip_value4-KLPENPPOLoss-device0] - TypeError: Got key of type <class 'NoneType'> when a string was expected."],"failure_context":[],"time":"2025-01-10T15:38:47.000000000Z"},{"workflowId":12712552352,"workflowUniqueId":82165822,"id":35438408047,"runnerName":"i-09d25aeb81be048b5","authorEmail":"[email protected]","name":"Unit-tests on Linux / tests-cpu (3.12) / linux-job","jobName":"tests-cpu (3.12) / linux-job","conclusion":"failure","completed_at":"2025-01-10T16:18:07.000000000Z","html_url":"https://github.com/pytorch/rl/actions/runs/12712552352/job/35438408047","head_branch":"gh/vmoens/58/head","pr_number":2665,"head_sha":"14e639d0393df511e07efbdc1ab8ffa474eb2e04","head_sha_timestamp":"2025-01-10T15:38:08.000000000Z","failure_captures":["test/test_cost.py::TestPPO::test_ppo_value_clipping[True-clip_value4-KLPENPPOLoss-device0]"],"failure_lines":["FAILED test/test_cost.py::TestPPO::test_ppo_value_clipping[True-clip_value4-KLPENPPOLoss-device0] - TypeError: Got key of type <class 'NoneType'> when a string was expected."],"failure_context":[],"time":"2025-01-10T15:38:49.000000000Z"},{"workflowId":12712552352,"workflowUniqueId":82165822,"id":35438409557,"runnerName":"i-0474400eb3b3ac9f5","authorEmail":"[email protected]","name":"Unit-tests on Linux / tests-cpu (3.9) / linux-job","jobName":"tests-cpu (3.9) / linux-job","conclusion":"failure","completed_at":"2025-01-10T16:19:03.000000000Z","html_url":"https://github.com/pytorch/rl/actions/runs/12712552352/job/35438409557","head_branch":"gh/vmoens/58/head","pr_number":2665,"head_sha":"14e639d0393df511e07efbdc1ab8ffa474eb2e04","head_sha_timestamp":"2025-01-10T15:38:08.000000000Z","failure_captures":["test/test_cost.py::TestPPO::test_ppo_value_clipping[True-clip_value4-KLPENPPOLoss-device0]"],"failure_lines":["FAILED test/test_cost.py::TestPPO::test_ppo_value_clipping[True-clip_value4-KLPENPPOLoss-device0] - TypeError: Got key of type <class 'NoneType'> when a string was expected."],"failure_context":[],"time":"2025-01-10T15:38:50.000000000Z"},{"workflowId":12712552352,"workflowUniqueId":82165822,"id":35438411139,"runnerName":"i-05cd616031eefd273","authorEmail":"[email protected]","name":"Unit-tests on Linux / tests-cpu-oldget (3.12) / linux-job","jobName":"tests-cpu-oldget (3.12) / linux-job","conclusion":"failure","completed_at":"2025-01-10T16:19:53.000000000Z","html_url":"https://github.com/pytorch/rl/actions/runs/12712552352/job/35438411139","head_branch":"gh/vmoens/58/head","pr_number":2665,"head_sha":"14e639d0393df511e07efbdc1ab8ffa474eb2e04","head_sha_timestamp":"2025-01-10T15:38:08.000000000Z","failure_captures":["test/test_cost.py::TestPPO::test_ppo_value_clipping[True-clip_value4-KLPENPPOLoss-device0]"],"failure_lines":["FAILED test/test_cost.py::TestPPO::test_ppo_value_clipping[True-clip_value4-KLPENPPOLoss-device0] - TypeError: Got key of type <class 'NoneType'> when a string was expected."],"failure_context":[],"time":"2025-01-10T15:38:52.000000000Z"},{"workflowId":12712552352,"workflowUniqueId":82165822,"id":35438413764,"runnerName":"i-0b0033623710b288a","authorEmail":"[email protected]","name":"Unit-tests on Linux / tests-gpu (3.11, 12.1) / linux-job","jobName":"tests-gpu (3.11, 12.1) / linux-job","conclusion":"failure","completed_at":"2025-01-10T16:39:37.000000000Z","html_url":"https://github.com/pytorch/rl/actions/runs/12712552352/job/35438413764","head_branch":"gh/vmoens/58/head","pr_number":2665,"head_sha":"14e639d0393df511e07efbdc1ab8ffa474eb2e04","head_sha_timestamp":"2025-01-10T15:38:08.000000000Z","failure_captures":["test/test_loggers.py::TestMLFlowLogger::test_log_video[steps1]"],"failure_lines":["FAILED test/test_loggers.py::TestMLFlowLogger::test_log_video[steps1] - TypeError: an integer is required"],"failure_context":[],"time":"2025-01-10T15:38:55.000000000Z"},{"workflowId":12712552352,"workflowUniqueId":82165822,"id":35438415560,"runnerName":"i-078f9813dce7dcbb8","authorEmail":"[email protected]","name":"Unit-tests on Linux / tests-optdeps (3.11, 12.1) / linux-job","jobName":"tests-optdeps (3.11, 12.1) / linux-job","conclusion":"failure","completed_at":"2025-01-10T16:19:38.000000000Z","html_url":"https://github.com/pytorch/rl/actions/runs/12712552352/job/35438415560","head_branch":"gh/vmoens/58/head","pr_number":2665,"head_sha":"14e639d0393df511e07efbdc1ab8ffa474eb2e04","head_sha_timestamp":"2025-01-10T15:38:08.000000000Z","failure_captures":["test/test_cost.py::TestPPO::test_ppo_value_clipping[True-clip_value4-KLPENPPOLoss-device0]"],"failure_lines":["FAILED test/test_cost.py::TestPPO::test_ppo_value_clipping[True-clip_value4-KLPENPPOLoss-device0] - TypeError: Got key of type <class 'NoneType'> when a string was expected."],"failure_context":[],"time":"2025-01-10T15:38:57.000000000Z"},{"workflowId":12712552352,"workflowUniqueId":82165822,"id":35438417268,"runnerName":"i-0f33687c9bdb52414","authorEmail":"[email protected]","name":"Unit-tests on Linux / tests-olddeps (3.8, 11.6) / linux-job","jobName":"tests-olddeps (3.8, 11.6) / linux-job","conclusion":"failure","completed_at":"2025-01-10T16:21:45.000000000Z","html_url":"https://github.com/pytorch/rl/actions/runs/12712552352/job/35438417268","head_branch":"gh/vmoens/58/head","pr_number":2665,"head_sha":"14e639d0393df511e07efbdc1ab8ffa474eb2e04","head_sha_timestamp":"2025-01-10T15:38:08.000000000Z","failure_captures":["test/test_storage_map.py::TestMCTSForest::test_forest_parent[simple-False]"],"failure_lines":["FAILED test/test_storage_map.py::TestMCTSForest::test_forest_parent[simple-False] - gym.error.DeprecatedEnv: Env Pendulum-v1 not found (valid versions include ['Pendulum-v0'])"],"failure_context":[],"time":"2025-01-10T15:38:58.000000000Z"},{"workflowId":12712552381,"workflowUniqueId":106193614,"id":35439160463,"runnerName":"GitHub Actions 63","authorEmail":"[email protected]","name":"Build Windows Wheels / pytorch/rl / upload / wheel-py3_9-cuda12_6","jobName":"pytorch/rl / upload / wheel-py3_9-cuda12_6","conclusion":"failure","completed_at":"2025-01-10T15:53:31.000000000Z","html_url":"https://github.com/pytorch/rl/actions/runs/12712552381/job/35439160463","head_branch":"gh/vmoens/58/head","pr_number":2665,"head_sha":"14e639d0393df511e07efbdc1ab8ffa474eb2e04","head_sha_timestamp":"2025-01-10T15:38:08.000000000Z","failure_captures":["Unable to download artifact(s): Artifact not found for name: pytorch_rl__3.9_cu126"],"failure_lines":["##[error]Unable to download artifact(s): Artifact not found for name: pytorch_rl__3.9_cu126"],"failure_context":[],"time":"2025-01-10T15:53:16.000000000Z"},{"workflowId":12712552381,"workflowUniqueId":106193614,"id":35439159277,"runnerName":"GitHub Actions 154","authorEmail":"[email protected]","name":"Build Windows Wheels / pytorch/rl / upload / wheel-py3_9-cpu","jobName":"pytorch/rl / upload / wheel-py3_9-cpu","conclusion":"failure","completed_at":"2025-01-10T15:53:29.000000000Z","html_url":"https://github.com/pytorch/rl/actions/runs/12712552381/job/35439159277","head_branch":"gh/vmoens/58/head","pr_number":2665,"head_sha":"14e639d0393df511e07efbdc1ab8ffa474eb2e04","head_sha_timestamp":"2025-01-10T15:38:08.000000000Z","failure_captures":["Unable to download artifact(s): Artifact not found for name: pytorch_rl__3.9_cpu_"],"failure_lines":["##[error]Unable to download artifact(s): Artifact not found for name: pytorch_rl__3.9_cpu_"],"failure_context":[],"time":"2025-01-10T15:53:15.000000000Z"},{"workflowId":12712552381,"workflowUniqueId":106193614,"id":35439159652,"runnerName":"GitHub Actions 408","authorEmail":"[email protected]","name":"Build Windows Wheels / pytorch/rl / upload / wheel-py3_9-cuda11_8","jobName":"pytorch/rl / upload / wheel-py3_9-cuda11_8","conclusion":"failure","completed_at":"2025-01-10T15:53:36.000000000Z","html_url":"https://github.com/pytorch/rl/actions/runs/12712552381/job/35439159652","head_branch":"gh/vmoens/58/head","pr_number":2665,"head_sha":"14e639d0393df511e07efbdc1ab8ffa474eb2e04","head_sha_timestamp":"2025-01-10T15:38:08.000000000Z","failure_captures":["Unable to download artifact(s): Artifact not found for name: pytorch_rl__3.9_cu118_"],"failure_lines":["##[error]Unable to download artifact(s): Artifact not found for name: pytorch_rl__3.9_cu118_"],"failure_context":[],"time":"2025-01-10T15:53:16.000000000Z"}],"FLAKY":[],"BROKEN_TRUNK":[{"workflowId":12712552381,"workflowUniqueId":106193614,"id":35439160095,"runnerName":"GitHub Actions 131","authorEmail":"[email protected]","name":"Build Windows Wheels / pytorch/rl / upload / wheel-py3_9-cuda12_4","jobName":"pytorch/rl / upload / wheel-py3_9-cuda12_4","conclusion":"failure","completed_at":"2025-01-10T15:53:35.000000000Z","html_url":"https://github.com/pytorch/rl/actions/runs/12712552381/job/35439160095","head_branch":"gh/vmoens/58/head","pr_number":2665,"head_sha":"14e639d0393df511e07efbdc1ab8ffa474eb2e04","head_sha_timestamp":"2025-01-10T15:38:08.000000000Z","failure_captures":[],"failure_lines":[],"failure_context":[],"time":"2025-01-10T15:53:16.000000000Z"}],"UNSTABLE":[]}