Skip to main content
kimsehun725
Projects
my_vae
Run groups
20221015_1624
Logs
Group
Workspace
Runs
Logs
Files
Artifacts

Kimsehun725's group workspace

Timestamps visible
2022-10-15 07:32:32
  rank_zero_warn("Detected KeyboardInterrupt, attempting graceful shutdown...")
2022-10-15 07:32:32
/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/loggers/wandb.py:352: UserWarning: There is a wandb run already in progress and newly created instances of `WandbLogger` will reuse this run. If this is not desired, call `wandb.finish()` before instantiating `WandbLogger`.
2022-10-15 07:32:32
  rank_zero_warn(
2022-10-15 07:32:32
GPU available: True (cuda), used: True
2022-10-15 07:32:32
TPU available: False, using: 0 TPU cores
2022-10-15 07:32:32
IPU available: False, using: 0 IPUs
2022-10-15 07:32:32
HPU available: False, using: 0 HPUs
2022-10-15 07:32:32
wandb: logging graph, to disable use `wandb.watch(log_graph=False)`
2022-10-15 07:32:32
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]
2022-10-15 07:32:32
  | Name                   | Type             | Params
2022-10-15 07:32:32
------------------------------------------------------------
2022-10-15 07:32:32
0 | conv_stack             | ConvStack        | 1.6 M
2022-10-15 07:32:32
1 | self_attention_block   | ConformerEncoder | 92.0 K
2022-10-15 07:32:32
2 | frame_tab_output_layer | Sequential       | 24.6 K
2022-10-15 07:32:32
3 | softmax_by_string      | Softmax          | 0
2022-10-15 07:32:32
------------------------------------------------------------
2022-10-15 07:32:32
1.7 M     Trainable params
2022-10-15 07:32:32
0         Non-trainable params
2022-10-15 07:32:32
1.7 M     Total params
2022-10-15 07:32:32
6.873     Total estimated model params size (MB)
2022-10-15 07:32:33
Training test number 02 ...
2022-10-15 07:32:34
Exception ignored in: <function _releaseLock at 0x7f1119fe6310>
2022-10-15 07:32:34
Traceback (most recent call last):
2022-10-15 07:32:34
  File "/usr/lib/python3.8/logging/__init__.py", line 227, in _releaseLock
2022-10-15 07:32:34
    def _releaseLock():
2022-10-15 07:32:34
KeyboardInterrupt:
2022-10-15 07:32:34
Exception ignored in: <function _MultiProcessingDataLoaderIter.__del__ at 0x7f110f737550>
2022-10-15 07:32:34
Traceback (most recent call last):
2022-10-15 07:32:34
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 1510, in __del__
2022-10-15 07:32:34
    self._shutdown_workers()
2022-10-15 07:32:34
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 1468, in _shutdown_workers
2022-10-15 07:32:34
    if self._persistent_workers or self._workers_status[worker_id]:
2022-10-15 07:32:34
AttributeError: '_MultiProcessingDataLoaderIter' object has no attribute '_workers_status'
2022-10-15 07:32:34
Exception ignored in: <function _releaseLock at 0x7f1119fe6310>
2022-10-15 07:32:34
Traceback (most recent call last):
2022-10-15 07:32:34
  File "/usr/lib/python3.8/logging/__init__.py", line 227, in _releaseLock
2022-10-15 07:32:34
    def _releaseLock():
2022-10-15 07:32:34
KeyboardInterrupt:
2022-10-15 07:32:34
Exception ignored in: <function _releaseLock at 0x7f1119fe6310>
2022-10-15 07:32:34
Traceback (most recent call last):
2022-10-15 07:32:34
  File "/usr/lib/python3.8/logging/__init__.py", line 227, in _releaseLock
2022-10-15 07:32:34
    def _releaseLock():
2022-10-15 07:32:34
KeyboardInterrupt:
2022-10-15 07:32:34
Exception ignored in: <function _releaseLock at 0x7f1119fe6310>
2022-10-15 07:32:34
Traceback (most recent call last):
2022-10-15 07:32:34
  File "/usr/lib/python3.8/logging/__init__.py", line 227, in _releaseLock
2022-10-15 07:32:34
    def _releaseLock():
2022-10-15 07:32:34
KeyboardInterrupt:
2022-10-15 07:32:34
Traceback (most recent call last):
2022-10-15 07:32:34
  File "src/train.py", line 65, in <module>
2022-10-15 07:32:34
    main(kwargs)
2022-10-15 07:32:34
  File "src/train.py", line 25, in main
2022-10-15 07:32:34
    step3(now, kwargs)
2022-10-15 07:32:34
  File "/data/group1/z44543r/vae_separation/src/step3.py", line 213, in step3
2022-10-15 07:32:34
    train(kwargs, use_pretrained_model, pretrained_time, pretrained_epoch, now, test_num, train_data_list,
2022-10-15 07:32:34
  File "/data/group1/z44543r/vae_separation/src/step3.py", line 110, in train
2022-10-15 07:32:34
    trainer = pl.Trainer(max_epochs=epochs,
2022-10-15 07:32:34
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/utilities/argparse.py", line 345, in insert_env_defaults
2022-10-15 07:32:34
    return fn(self, **kwargs)
2022-10-15 07:32:34
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 433, in __init__
2022-10-15 07:32:34
    self._accelerator_connector = AcceleratorConnector(
2022-10-15 07:32:34
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/accelerator_connector.py", line 214, in __init__
2022-10-15 07:32:34
    self._set_parallel_devices_and_init_accelerator()
2022-10-15 07:32:34
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/accelerator_connector.py", line 545, in _set_parallel_devices_and_init_accelerator
2022-10-15 07:32:34
    self._devices_flag = self.accelerator.parse_devices(self._devices_flag)
2022-10-15 07:32:34
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/accelerators/cuda.py", line 77, in parse_devices
2022-10-15 07:32:34
    return device_parser.parse_gpu_ids(devices, include_cuda=True)
2022-10-15 07:32:34
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/utilities/device_parser.py", line 125, in parse_gpu_ids
2022-10-15 07:32:34
    return _sanitize_gpu_ids(gpus, include_cuda=include_cuda, include_mps=include_mps)
2022-10-15 07:32:34
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/utilities/device_parser.py", line 206, in _sanitize_gpu_ids
2022-10-15 07:32:34
    all_available_gpus = _get_all_available_gpus(include_cuda=include_cuda, include_mps=include_mps)
2022-10-15 07:32:34
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/utilities/device_parser.py", line 236, in _get_all_available_gpus
2022-10-15 07:32:34
    cuda_gpus = _get_all_available_cuda_gpus() if include_cuda else []
2022-10-15 07:32:34
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/utilities/device_parser.py", line 257, in _get_all_available_cuda_gpus
2022-10-15 07:32:34
    return list(range(num_cuda_devices()))
2022-10-15 07:32:34
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/utilities/device_parser.py", line 348, in num_cuda_devices
2022-10-15 07:32:34
    return pool.apply(torch.cuda.device_count)
2022-10-15 07:32:34
  File "/usr/lib/python3.8/multiprocessing/pool.py", line 736, in __exit__
2022-10-15 07:32:34
    self.terminate()
2022-10-15 07:32:34
  File "/usr/lib/python3.8/multiprocessing/pool.py", line 654, in terminate
2022-10-15 07:32:34
    self._terminate()
2022-10-15 07:32:34
  File "/usr/lib/python3.8/multiprocessing/util.py", line 224, in __call__
2022-10-15 07:32:34
    res = self._callback(*self._args, **self._kwargs)
2022-10-15 07:32:34
  File "/usr/lib/python3.8/multiprocessing/pool.py", line 729, in _terminate_pool
2022-10-15 07:32:34
    p.join()
2022-10-15 07:32:34
  File "/usr/lib/python3.8/multiprocessing/process.py", line 149, in join
2022-10-15 07:32:34
    res = self._popen.wait(timeout)
2022-10-15 07:32:34
  File "/usr/lib/python3.8/multiprocessing/popen_fork.py", line 47, in wait
2022-10-15 07:32:34
    return self.poll(os.WNOHANG if timeout == 0.0 else 0)
2022-10-15 07:32:34
  File "/usr/lib/python3.8/multiprocessing/popen_fork.py", line 27, in poll
2022-10-15 07:32:34
    pid, sts = os.waitpid(self.pid, flag)
2022-10-15 07:32:34
KeyboardInterrupt