kastan

Kastan's group workspace

Group: Jul-27__21:26

1-1

of 1

Timestamps visible

2022-07-28 02:57:10

frame #5: c10d::PrefixStore::get(std::string const&) + 0x31 (0x7f01bef868d1 in /u/kastanday/.conda/envs/nice_base/envs/col_ai_old_v5/lib/python3.9/site-packages/torch/lib/libtorch_cpu.so)

2022-07-28 02:57:10

frame #6: c10d::PrefixStore::get(std::string const&) + 0x31 (0x7f01bef868d1 in /u/kastanday/.conda/envs/nice_base/envs/col_ai_old_v5/lib/python3.9/site-packages/torch/lib/libtorch_cpu.so)

2022-07-28 02:57:10

frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::string const&, int) + 0xab (0x7f01d4aad33b in /u/kastanday/.conda/envs/nice_base/envs/col_ai_old_v5/lib/python3.9/site-packages/torch/lib/libtorch_cuda_cpp.so)

2022-07-28 02:57:10

frame #8: c10d::ProcessGroupNCCL::getNCCLComm(std::string const&, std::vector<c10::Device, std::allocator<c10::Device> > const&, c10d::OpType, int, bool) + 0x1fe (0x7f01d4ab10de in /u/kastanday/.conda/envs/nice_base/envs/col_ai_old_v5/lib/python3.9/site-packages/torch/lib/libtorch_cuda_cpp.so)

2022-07-28 02:57:10

frame #9: <unknown function> + 0x1ff1d6 (0x7f01d4ab81d6 in /u/kastanday/.conda/envs/nice_base/envs/col_ai_old_v5/lib/python3.9/site-packages/torch/lib/libtorch_cuda_cpp.so)

2022-07-28 02:57:10

frame #10: c10d::ProcessGroupNCCL::allreduce_impl(std::vector<at::Tensor, std::allocator<at::Tensor> >&, c10d::AllreduceOptions const&) + 0x10 (0x7f01d4ab95d0 in /u/kastanday/.conda/envs/nice_base/envs/col_ai_old_v5/lib/python3.9/site-packages/torch/lib/libtorch_cuda_cpp.so)

2022-07-28 02:57:10

frame #11: c10d::ProcessGroupNCCL::allreduce(std::vector<at::Tensor, std::allocator<at::Tensor> >&, c10d::AllreduceOptions const&) + 0x2ac (0x7f01d4abb31c in /u/kastanday/.conda/envs/nice_base/envs/col_ai_old_v5/lib/python3.9/site-packages/torch/lib/libtorch_cuda_cpp.so)

2022-07-28 02:57:10

frame #12: <unknown function> + 0x9f9f93 (0x7f01e2578f93 in /u/kastanday/.conda/envs/nice_base/envs/col_ai_old_v5/lib/python3.9/site-packages/torch/lib/libtorch_python.so)

2022-07-28 02:57:10

frame #13: <unknown function> + 0x36bc3d (0x7f01e1eeac3d in /u/kastanday/.conda/envs/nice_base/envs/col_ai_old_v5/lib/python3.9/site-packages/torch/lib/libtorch_python.so)

2022-07-28 02:57:10

2022-07-28 02:57:10

frame #54: __libc_start_main + 0xf3 (0x7f0210737493 in /lib64/libc.so.6)