kastan

Kastan's group workspace

Group: Jul-27__20:49

1-1

of 1

Timestamps visible

2022-07-28 02:21:29

frame #5: c10d::PrefixStore::get(std::string const&) + 0x31 (0x7f1f029308d1 in /u/kastanday/.conda/envs/nice_base/envs/col_ai_old_v5/lib/python3.9/site-packages/torch/lib/libtorch_cpu.so)

2022-07-28 02:21:29

frame #6: c10d::PrefixStore::get(std::string const&) + 0x31 (0x7f1f029308d1 in /u/kastanday/.conda/envs/nice_base/envs/col_ai_old_v5/lib/python3.9/site-packages/torch/lib/libtorch_cpu.so)

2022-07-28 02:21:29

frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::string const&, int) + 0xab (0x7f1f1845733b in /u/kastanday/.conda/envs/nice_base/envs/col_ai_old_v5/lib/python3.9/site-packages/torch/lib/libtorch_cuda_cpp.so)

2022-07-28 02:21:29

frame #8: c10d::ProcessGroupNCCL::getNCCLComm(std::string const&, std::vector<c10::Device, std::allocator<c10::Device> > const&, c10d::OpType, int, bool) + 0x1fe (0x7f1f1845b0de in /u/kastanday/.conda/envs/nice_base/envs/col_ai_old_v5/lib/python3.9/site-packages/torch/lib/libtorch_cuda_cpp.so)

2022-07-28 02:21:29

frame #9: <unknown function> + 0x1ff1d6 (0x7f1f184621d6 in /u/kastanday/.conda/envs/nice_base/envs/col_ai_old_v5/lib/python3.9/site-packages/torch/lib/libtorch_cuda_cpp.so)

2022-07-28 02:21:29

frame #10: c10d::ProcessGroupNCCL::allreduce_impl(std::vector<at::Tensor, std::allocator<at::Tensor> >&, c10d::AllreduceOptions const&) + 0x10 (0x7f1f184635d0 in /u/kastanday/.conda/envs/nice_base/envs/col_ai_old_v5/lib/python3.9/site-packages/torch/lib/libtorch_cuda_cpp.so)

2022-07-28 02:21:29

frame #11: c10d::ProcessGroupNCCL::allreduce(std::vector<at::Tensor, std::allocator<at::Tensor> >&, c10d::AllreduceOptions const&) + 0x2ac (0x7f1f1846531c in /u/kastanday/.conda/envs/nice_base/envs/col_ai_old_v5/lib/python3.9/site-packages/torch/lib/libtorch_cuda_cpp.so)

2022-07-28 02:21:29

frame #12: <unknown function> + 0x9f9f93 (0x7f1f25f22f93 in /u/kastanday/.conda/envs/nice_base/envs/col_ai_old_v5/lib/python3.9/site-packages/torch/lib/libtorch_python.so)

2022-07-28 02:21:29

frame #13: <unknown function> + 0x36bc3d (0x7f1f25894c3d in /u/kastanday/.conda/envs/nice_base/envs/col_ai_old_v5/lib/python3.9/site-packages/torch/lib/libtorch_python.so)

2022-07-28 02:21:29

2022-07-28 02:21:29

frame #54: __libc_start_main + 0xf3 (0x7f1f540e1493 in /lib64/libc.so.6)