Skip to main content
kimsehun725
Projects
my_vae
Run groups
20221014_2240
Logs
Group
Workspace
Runs
Logs
Files
Artifacts

Kimsehun725's group workspace

Timestamps visible
2022-10-14 13:45:08
Epoch 14:  89%|██████████████████████████████████████████████████████▉       | 31/35 [00:06<00:00,  5.08it/s, loss=0.0371, v_num=23rn, val/loss=0.031, val/precision=0.703, val/recall=0.734, val/f1=0.718, train/loss=0.040]
2022-10-14 13:45:26
Epoch 15:  89%|█████████████████████████████████████████████████████▏      | 31/35 [00:05<00:00,  5.25it/s, loss=0.0384, v_num=23rn, val/loss=0.0313, val/precision=0.687, val/recall=0.760, val/f1=0.722, train/loss=0.0384]
2022-10-14 13:45:46
Epoch 16:  89%|█████████████████████████████████████████████████████▏      | 31/35 [00:06<00:00,  5.12it/s, loss=0.0368, v_num=23rn, val/loss=0.0312, val/precision=0.706, val/recall=0.756, val/f1=0.730, train/loss=0.0379]
2022-10-14 13:45:46
Validation: 0it [00:00, ?it/s]
2022-10-14 13:45:48
Exception ignored in: <function _after_fork at 0x2b365681c310>
2022-10-14 13:45:48
Traceback (most recent call last):
2022-10-14 13:45:48
  File "/usr/lib/python3.8/threading.py", line 1456, in _after_fork
2022-10-14 13:45:48
    thread._reset_internal_locks(False)
2022-10-14 13:45:48
  File "/usr/lib/python3.8/threading.py", line 811, in _reset_internal_locks
2022-10-14 13:45:48
    self._started._reset_internal_locks()
2022-10-14 13:45:48
  File "/usr/lib/python3.8/threading.py", line 511, in _reset_internal_locks
2022-10-14 13:45:48
    self._cond.__init__(Lock())
2022-10-14 13:45:48
KeyboardInterrupt:
2022-10-14 13:45:48
Exception ignored in: <function _releaseLock at 0x2b361f8f81f0>
2022-10-14 13:45:48
Traceback (most recent call last):
2022-10-14 13:45:48
  File "/usr/lib/python3.8/logging/__init__.py", line 227, in _releaseLock
2022-10-14 13:45:48
    def _releaseLock():
2022-10-14 13:45:48
KeyboardInterrupt:
2022-10-14 13:45:54
Traceback (most recent call last):
2022-10-14 13:45:54
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 1163, in _try_get_data
2022-10-14 13:45:54
    data = self._data_queue.get(timeout=timeout)
2022-10-14 13:45:54
  File "/usr/lib/python3.8/multiprocessing/queues.py", line 108, in get
2022-10-14 13:45:54
    raise Empty
2022-10-14 13:45:54
_queue.Empty
2022-10-14 13:45:54
The above exception was the direct cause of the following exception:
2022-10-14 13:45:54
Traceback (most recent call last):
2022-10-14 13:45:54
  File "src/train.py", line 65, in <module>
2022-10-14 13:45:54
    main(kwargs)
2022-10-14 13:45:54
  File "src/train.py", line 25, in main
2022-10-14 13:45:54
    step3(now, kwargs)
2022-10-14 13:45:54
  File "/data/group1/z44543r/vae_separation/src/step3.py", line 209, in step3
2022-10-14 13:45:54
    train(kwargs, use_pretrained_model, pretrained_time, pretrained_epoch, now, test_num, train_data_list,
2022-10-14 13:45:54
  File "/data/group1/z44543r/vae_separation/src/step3.py", line 151, in train
2022-10-14 13:45:54
    trainer.fit(model, train_dataloaders=train_loader,
2022-10-14 13:45:54
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 696, in fit
2022-10-14 13:45:54
    self._call_and_handle_interrupt(
2022-10-14 13:45:54
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 650, in _call_and_handle_interrupt
2022-10-14 13:45:54
    return trainer_fn(*args, **kwargs)
2022-10-14 13:45:54
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 735, in _fit_impl
2022-10-14 13:45:54
    results = self._run(model, ckpt_path=self.ckpt_path)
2022-10-14 13:45:54
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1166, in _run
2022-10-14 13:45:54
    results = self._run_stage()
2022-10-14 13:45:54
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1252, in _run_stage
2022-10-14 13:45:54
    return self._run_train()
2022-10-14 13:45:54
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1283, in _run_train
2022-10-14 13:45:54
    self.fit_loop.run()
2022-10-14 13:45:54
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/loops/loop.py", line 200, in run
2022-10-14 13:45:54
    self.advance(*args, **kwargs)
2022-10-14 13:45:54
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/loops/fit_loop.py", line 271, in advance
2022-10-14 13:45:54
    self._outputs = self.epoch_loop.run(self._data_fetcher)
2022-10-14 13:45:54
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/loops/loop.py", line 201, in run
2022-10-14 13:45:54
    self.on_advance_end()
2022-10-14 13:45:54
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/loops/epoch/training_epoch_loop.py", line 241, in on_advance_end
2022-10-14 13:45:54
    self._run_validation()
2022-10-14 13:45:54
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/loops/epoch/training_epoch_loop.py", line 299, in _run_validation
2022-10-14 13:45:54
    self.val_loop.run()
2022-10-14 13:45:54
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/loops/loop.py", line 200, in run
2022-10-14 13:45:54
    self.advance(*args, **kwargs)
2022-10-14 13:45:54
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/loops/dataloader/evaluation_loop.py", line 155, in advance
2022-10-14 13:45:54
    dl_outputs = self.epoch_loop.run(self._data_fetcher, dl_max_batches, kwargs)
2022-10-14 13:45:54
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/loops/loop.py", line 200, in run
2022-10-14 13:45:54
    self.advance(*args, **kwargs)
2022-10-14 13:45:54
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/loops/epoch/evaluation_epoch_loop.py", line 127, in advance
2022-10-14 13:45:54
    batch = next(data_fetcher)
2022-10-14 13:45:54
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/utilities/fetching.py", line 184, in __next__
2022-10-14 13:45:54
    return self.fetching_function()
2022-10-14 13:45:54
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/utilities/fetching.py", line 263, in fetching_function
2022-10-14 13:45:54
    self._fetch_next_batch(self.dataloader_iter)
2022-10-14 13:45:54
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/utilities/fetching.py", line 277, in _fetch_next_batch
2022-10-14 13:45:54
    batch = next(iterator)
2022-10-14 13:45:54
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 681, in __next__
2022-10-14 13:45:54
    data = self._next_data()
2022-10-14 13:45:54
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 1359, in _next_data
2022-10-14 13:45:54
    idx, data = self._get_data()
2022-10-14 13:45:54
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 1325, in _get_data
2022-10-14 13:45:54
    success, data = self._try_get_data()
2022-10-14 13:45:54
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 1176, in _try_get_data
2022-10-14 13:45:54
    raise RuntimeError('DataLoader worker (pid(s) {}) exited unexpectedly'.format(pids_str)) from e
2022-10-14 13:45:54
RuntimeError: DataLoader worker (pid(s) 140153, 140193, 140233, 140273, 140313, 140353, 140393, 140433, 140473, 140513, 140553, 140593, 140633, 140673, 140713, 140753, 140793, 140833, 140873) exited unexpectedly