Skip to main content
kimsehun725
Projects
my_vae
Run groups
20221015_0234
Logs
Group
Workspace
Runs
Logs
Files
Artifacts

Kimsehun725's group workspace

Timestamps visible
2022-10-14 17:35:45
  File "src/train.py", line 25, in main
2022-10-14 17:35:45
    step3(now, kwargs)
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/src/step3.py", line 212, in step3
2022-10-14 17:35:45
    train(kwargs, use_pretrained_model, pretrained_time, pretrained_epoch, now, test_num, train_data_list,
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/src/step3.py", line 152, in train
2022-10-14 17:35:45
    trainer.fit(model, train_dataloaders=train_loader,
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 696, in fit
2022-10-14 17:35:45
    self._call_and_handle_interrupt(
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 650, in _call_and_handle_interrupt
2022-10-14 17:35:45
    return trainer_fn(*args, **kwargs)
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 735, in _fit_impl
2022-10-14 17:35:45
    results = self._run(model, ckpt_path=self.ckpt_path)
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1166, in _run
2022-10-14 17:35:45
    results = self._run_stage()
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1252, in _run_stage
2022-10-14 17:35:45
    return self._run_train()
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1274, in _run_train
2022-10-14 17:35:45
    self._run_sanity_check()
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1343, in _run_sanity_check
2022-10-14 17:35:45
    val_loop.run()
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/loops/loop.py", line 200, in run
2022-10-14 17:35:45
    self.advance(*args, **kwargs)
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/loops/dataloader/evaluation_loop.py", line 155, in advance
2022-10-14 17:35:45
    dl_outputs = self.epoch_loop.run(self._data_fetcher, dl_max_batches, kwargs)
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/loops/loop.py", line 200, in run
2022-10-14 17:35:45
    self.advance(*args, **kwargs)
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/loops/epoch/evaluation_epoch_loop.py", line 143, in advance
2022-10-14 17:35:45
    output = self._evaluation_step(**kwargs)
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/loops/epoch/evaluation_epoch_loop.py", line 240, in _evaluation_step
2022-10-14 17:35:45
    output = self.trainer._call_strategy_hook(hook_name, *kwargs.values())
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1704, in _call_strategy_hook
2022-10-14 17:35:45
    output = fn(*args, **kwargs)
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/strategies/dp.py", line 139, in validation_step
2022-10-14 17:35:45
    return self.model(*args, **kwargs)
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
2022-10-14 17:35:45
    return forward_call(*input, **kwargs)
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/torch/nn/parallel/data_parallel.py", line 168, in forward
2022-10-14 17:35:45
    outputs = self.parallel_apply(replicas, inputs, kwargs)
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/torch/nn/parallel/data_parallel.py", line 178, in parallel_apply
2022-10-14 17:35:45
    return parallel_apply(replicas, inputs, kwargs, self.device_ids[:len(replicas)])
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/torch/nn/parallel/parallel_apply.py", line 86, in parallel_apply
2022-10-14 17:35:45
    output.reraise()
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/torch/_utils.py", line 461, in reraise
2022-10-14 17:35:45
    raise exception
2022-10-14 17:35:45
RuntimeError: Caught RuntimeError in replica 0 on device 0.
2022-10-14 17:35:45
Original Traceback (most recent call last):
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/torch/nn/parallel/parallel_apply.py", line 61, in _worker
2022-10-14 17:35:45
    output = module(*input, **kwargs)
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
2022-10-14 17:35:45
    return forward_call(*input, **kwargs)
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/overrides/data_parallel.py", line 65, in forward
2022-10-14 17:35:45
    output = super().forward(*inputs, **kwargs)
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/pytorch_lightning/overrides/base.py", line 90, in forward
2022-10-14 17:35:45
    return self.module.validation_step(*inputs, **kwargs)
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/src/models.py", line 653, in validation_step
2022-10-14 17:35:45
    y_hat = self.forward(x1, ilens)
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/src/models.py", line 633, in forward
2022-10-14 17:35:45
    memory, _, _ = self.self_attention_block(z1, ilens)
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1148, in _call_impl
2022-10-14 17:35:45
    result = forward_call(*input, **kwargs)
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/espnet2/asr/encoder/conformer_encoder.py", line 331, in forward
2022-10-14 17:35:45
    xs_pad, masks = self.encoders(xs_pad, masks)
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
2022-10-14 17:35:45
    return forward_call(*input, **kwargs)
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/espnet/nets/pytorch_backend/transformer/repeat.py", line 18, in forward
2022-10-14 17:35:45
    args = m(*args)
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
2022-10-14 17:35:45
    return forward_call(*input, **kwargs)
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/espnet/nets/pytorch_backend/conformer/encoder_layer.py", line 139, in forward
2022-10-14 17:35:45
    x_att = self.self_attn(x_q, x, x, pos_emb, mask)
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
2022-10-14 17:35:45
    return forward_call(*input, **kwargs)
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/espnet/nets/pytorch_backend/transformer/attention.py", line 305, in forward
2022-10-14 17:35:45
    return self.forward_attention(v, scores, mask)
2022-10-14 17:35:45
  File "/data/group1/z44543r/vae_separation/venv/lib/python3.8/site-packages/espnet/nets/pytorch_backend/transformer/attention.py", line 80, in forward_attention
2022-10-14 17:35:45
    scores = scores.masked_fill(mask, min_value)
2022-10-14 17:35:45
RuntimeError: The size of tensor a (413) must match the size of tensor b (492) at non-singleton dimension 3