Created
August 13, 2019 23:26
-
-
Save judywawira/f39f3102424996632d15e047e57d0aa3 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
fold: 0 | |
0.00% [0/7 00:00<00:00] | |
epoch train_loss valid_loss dice time | |
Interrupted | |
--------------------------------------------------------------------------- | |
RuntimeError Traceback (most recent call last) | |
<ipython-input-21-2b9600a80866> in <module> | |
15 lr = 1e-4 | |
16 #learn.load('fold_4') | |
---> 17 learn.fit_one_cycle(7, lr, callbacks = [AccumulateStep(learn,n_acc)]) | |
18 | |
19 #fit entire model with saving on the best epoch | |
~/.local/lib/python3.6/site-packages/fastai/train.py in fit_one_cycle(learn, cyc_len, max_lr, moms, div_factor, pct_start, final_div, wd, callbacks, tot_epochs, start_epoch) | |
20 callbacks.append(OneCycleScheduler(learn, max_lr, moms=moms, div_factor=div_factor, pct_start=pct_start, | |
21 final_div=final_div, tot_epochs=tot_epochs, start_epoch=start_epoch)) | |
---> 22 learn.fit(cyc_len, max_lr, wd=wd, callbacks=callbacks) | |
23 | |
24 def lr_find(learn:Learner, start_lr:Floats=1e-7, end_lr:Floats=10, num_it:int=100, stop_div:bool=True, wd:float=None): | |
~/.local/lib/python3.6/site-packages/fastai/basic_train.py in fit(self, epochs, lr, wd, callbacks) | |
200 callbacks = [cb(self) for cb in self.callback_fns + listify(defaults.extra_callback_fns)] + listify(callbacks) | |
201 self.cb_fns_registered = True | |
--> 202 fit(epochs, self, metrics=self.metrics, callbacks=self.callbacks+callbacks) | |
203 | |
204 def create_opt(self, lr:Floats, wd:Floats=0.)->None: | |
~/.local/lib/python3.6/site-packages/fastai/basic_train.py in fit(epochs, learn, callbacks, metrics) | |
99 for xb,yb in progress_bar(learn.data.train_dl, parent=pbar): | |
100 xb, yb = cb_handler.on_batch_begin(xb, yb) | |
--> 101 loss = loss_batch(learn.model, xb, yb, learn.loss_func, learn.opt, cb_handler) | |
102 if cb_handler.on_batch_end(loss): break | |
103 | |
~/.local/lib/python3.6/site-packages/fastai/basic_train.py in loss_batch(model, xb, yb, loss_func, opt, cb_handler) | |
24 if not is_listy(xb): xb = [xb] | |
25 if not is_listy(yb): yb = [yb] | |
---> 26 out = model(*xb) | |
27 out = cb_handler.on_loss_begin(out) | |
28 | |
~/.local/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs) | |
545 result = self._slow_forward(*input, **kwargs) | |
546 else: | |
--> 547 result = self.forward(*input, **kwargs) | |
548 for hook in self._forward_hooks.values(): | |
549 hook_result = hook(self, input, result) | |
~/.local/lib/python3.6/site-packages/torch/nn/parallel/data_parallel.py in forward(self, *inputs, **kwargs) | |
150 return self.module(*inputs[0], **kwargs[0]) | |
151 replicas = self.replicate(self.module, self.device_ids[:len(inputs)]) | |
--> 152 outputs = self.parallel_apply(replicas, inputs, kwargs) | |
153 return self.gather(outputs, self.output_device) | |
154 | |
~/.local/lib/python3.6/site-packages/torch/nn/parallel/data_parallel.py in parallel_apply(self, replicas, inputs, kwargs) | |
160 | |
161 def parallel_apply(self, replicas, inputs, kwargs): | |
--> 162 return parallel_apply(replicas, inputs, kwargs, self.device_ids[:len(replicas)]) | |
163 | |
164 def gather(self, outputs, output_device): | |
~/.local/lib/python3.6/site-packages/torch/nn/parallel/parallel_apply.py in parallel_apply(modules, inputs, kwargs_tup, devices) | |
83 output = results[i] | |
84 if isinstance(output, ExceptionWrapper): | |
---> 85 output.reraise() | |
86 outputs.append(output) | |
87 return outputs | |
~/.local/lib/python3.6/site-packages/torch/_utils.py in reraise(self) | |
367 # (https://bugs.python.org/issue2651), so we work around it. | |
368 msg = KeyErrorMessage(msg) | |
--> 369 raise self.exc_type(msg) | |
RuntimeError: Caught RuntimeError in replica 1 on device 1. | |
Original Traceback (most recent call last): | |
File "/home/jupyter-judy/.local/lib/python3.6/site-packages/torch/nn/parallel/parallel_apply.py", line 60, in _worker | |
output = module(*input, **kwargs) | |
File "/home/jupyter-judy/.local/lib/python3.6/site-packages/torch/nn/modules/module.py", line 547, in __call__ | |
result = self.forward(*input, **kwargs) | |
File "/home/jupyter-judy/.local/lib/python3.6/site-packages/fastai/layers.py", line 136, in forward | |
nres = l(res) | |
File "/home/jupyter-judy/.local/lib/python3.6/site-packages/torch/nn/modules/module.py", line 547, in __call__ | |
result = self.forward(*input, **kwargs) | |
File "/home/jupyter-judy/.local/lib/python3.6/site-packages/fastai/vision/models/unet.py", line 33, in forward | |
cat_x = self.relu(torch.cat([up_out, self.bn(s)], dim=1)) | |
File "/home/jupyter-judy/.local/lib/python3.6/site-packages/torch/nn/modules/module.py", line 547, in __call__ | |
result = self.forward(*input, **kwargs) | |
File "/home/jupyter-judy/.local/lib/python3.6/site-packages/torch/nn/modules/batchnorm.py", line 81, in forward | |
exponential_average_factor, self.eps) | |
File "/home/jupyter-judy/.local/lib/python3.6/site-packages/torch/nn/functional.py", line 1656, in batch_norm | |
training, momentum, eps, torch.backends.cudnn.enabled | |
RuntimeError: Expected tensor for argument #1 'input' to have the same device as tensor for argument #2 'weight'; but device 0 does not equal 1 (while checking arguments for cudnn_batch_norm) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment