Skip to content

Instantly share code, notes, and snippets.

@ian-whitestone

ian-whitestone/stack_trace.txt

Last active July 2, 2020 17:27
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save ian-whitestone/a214d15a34357f2a31548435c760157b to your computer and use it in GitHub Desktop.
Save ian-whitestone/a214d15a34357f2a31548435c760157b to your computer and use it in GitHub Desktop.
Download ZIP
Dask + pydantic pickling error
Raw
stack_trace.txt
---------------------------------------------------------------------------
PicklingError Traceback (most recent call last)
<ipython-input-67-59601d7f9a60> in <module>
15
16 bag = db.from_sequence(data)
---> 17 bag.map(validate_data).compute()
~/Library/Caches/pypoetry/virtualenvs/domi-IWOYYLRr-py3.7/lib/python3.7/site-packages/dask/base.py in compute(self, **kwargs)
164 dask.base.compute
165 """
--> 166 (result,) = compute(self, traverse=False, **kwargs)
167 return result
168
~/Library/Caches/pypoetry/virtualenvs/domi-IWOYYLRr-py3.7/lib/python3.7/site-packages/dask/base.py in compute(*args, **kwargs)
442 postcomputes.append(x.__dask_postcompute__())
443
--> 444 results = schedule(dsk, keys, **kwargs)
445 return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
446
~/Library/Caches/pypoetry/virtualenvs/domi-IWOYYLRr-py3.7/lib/python3.7/site-packages/dask/multiprocessing.py in get(dsk, keys, num_workers, func_loads, func_dumps, optimize_graph, pool, **kwargs)
216 pack_exception=pack_exception,
217 raise_exception=reraise,
--> 218 **kwargs
219 )
220 finally:
~/Library/Caches/pypoetry/virtualenvs/domi-IWOYYLRr-py3.7/lib/python3.7/site-packages/dask/local.py in get_async(apply_async, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, **kwargs)
469 # Seed initial tasks into the thread pool
470 while state["ready"] and len(state["running"]) < num_workers:
--> 471 fire_task()
472
473 # Main loop, wait on tasks to finish, insert new ones
~/Library/Caches/pypoetry/virtualenvs/domi-IWOYYLRr-py3.7/lib/python3.7/site-packages/dask/local.py in fire_task()
458 args=(
459 key,
--> 460 dumps((dsk[key], data)),
461 dumps,
462 loads,
~/Library/Caches/pypoetry/virtualenvs/domi-IWOYYLRr-py3.7/lib/python3.7/site-packages/cloudpickle/cloudpickle.py in dumps(obj, protocol)
1081 try:
1082 cp = CloudPickler(file, protocol=protocol)
-> 1083 cp.dump(obj)
1084 return file.getvalue()
1085 finally:
~/Library/Caches/pypoetry/virtualenvs/domi-IWOYYLRr-py3.7/lib/python3.7/site-packages/cloudpickle/cloudpickle.py in dump(self, obj)
475 self.inject_addons()
476 try:
--> 477 return Pickler.dump(self, obj)
478 except RuntimeError as e:
479 if 'recursion' in e.args[0]:
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in dump(self, obj)
435 if self.proto >= 4:
436 self.framer.start_framing()
--> 437 self.save(obj)
438 self.write(STOP)
439 self.framer.end_framing()
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
502 f = self.dispatch.get(t)
503 if f is not None:
--> 504 f(self, obj) # Call unbound method with explicit self
505 return
506
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in save_tuple(self, obj)
772 if n <= 3 and self.proto >= 2:
773 for element in obj:
--> 774 save(element)
775 # Subtle. Same as in the big comment below.
776 if id(obj) in memo:
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
502 f = self.dispatch.get(t)
503 if f is not None:
--> 504 f(self, obj) # Call unbound method with explicit self
505 return
506
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in save_tuple(self, obj)
772 if n <= 3 and self.proto >= 2:
773 for element in obj:
--> 774 save(element)
775 # Subtle. Same as in the big comment below.
776 if id(obj) in memo:
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
502 f = self.dispatch.get(t)
503 if f is not None:
--> 504 f(self, obj) # Call unbound method with explicit self
505 return
506
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in save_tuple(self, obj)
787 write(MARK)
788 for element in obj:
--> 789 save(element)
790
791 if id(obj) in memo:
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
502 f = self.dispatch.get(t)
503 if f is not None:
--> 504 f(self, obj) # Call unbound method with explicit self
505 return
506
~/Library/Caches/pypoetry/virtualenvs/domi-IWOYYLRr-py3.7/lib/python3.7/site-packages/cloudpickle/cloudpickle.py in save_function(self, obj, name)
542 return self.save_pypy_builtin_func(obj)
543 else:
--> 544 return self.save_function_tuple(obj)
545
546 dispatch[types.FunctionType] = save_function
~/Library/Caches/pypoetry/virtualenvs/domi-IWOYYLRr-py3.7/lib/python3.7/site-packages/cloudpickle/cloudpickle.py in save_function_tuple(self, func)
754 if hasattr(func, '__kwdefaults__'):
755 state['kwdefaults'] = func.__kwdefaults__
--> 756 save(state)
757 write(pickle.TUPLE)
758 write(pickle.REDUCE) # applies _fill_function on the tuple
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
502 f = self.dispatch.get(t)
503 if f is not None:
--> 504 f(self, obj) # Call unbound method with explicit self
505 return
506
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in save_dict(self, obj)
857
858 self.memoize(obj)
--> 859 self._batch_setitems(obj.items())
860
861 dispatch[dict] = save_dict
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in _batch_setitems(self, items)
883 for k, v in tmp:
884 save(k)
--> 885 save(v)
886 write(SETITEMS)
887 elif n:
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
502 f = self.dispatch.get(t)
503 if f is not None:
--> 504 f(self, obj) # Call unbound method with explicit self
505 return
506
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in save_dict(self, obj)
857
858 self.memoize(obj)
--> 859 self._batch_setitems(obj.items())
860
861 dispatch[dict] = save_dict
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in _batch_setitems(self, items)
888 k, v = tmp[0]
889 save(k)
--> 890 save(v)
891 write(SETITEM)
892 # else tmp is empty, and we're done
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
516 issc = False
517 if issc:
--> 518 self.save_global(obj)
519 return
520
~/Library/Caches/pypoetry/virtualenvs/domi-IWOYYLRr-py3.7/lib/python3.7/site-packages/cloudpickle/cloudpickle.py in save_global(self, obj, name, pack)
842 Pickler.save_global(self, obj, name=name)
843 elif not _is_importable_by_name(obj, name=name):
--> 844 self.save_dynamic_class(obj)
845 else:
846 Pickler.save_global(self, obj, name=name)
~/Library/Caches/pypoetry/virtualenvs/domi-IWOYYLRr-py3.7/lib/python3.7/site-packages/cloudpickle/cloudpickle.py in save_dynamic_class(self, obj)
684 # Now save the rest of obj's __dict__. Any references to obj
685 # encountered while saving will point to the skeleton class.
--> 686 save(clsdict)
687
688 # Write a tuple of (skeleton_class, clsdict).
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
502 f = self.dispatch.get(t)
503 if f is not None:
--> 504 f(self, obj) # Call unbound method with explicit self
505 return
506
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in save_dict(self, obj)
857
858 self.memoize(obj)
--> 859 self._batch_setitems(obj.items())
860
861 dispatch[dict] = save_dict
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in _batch_setitems(self, items)
883 for k, v in tmp:
884 save(k)
--> 885 save(v)
886 write(SETITEMS)
887 elif n:
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
502 f = self.dispatch.get(t)
503 if f is not None:
--> 504 f(self, obj) # Call unbound method with explicit self
505 return
506
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in save_dict(self, obj)
857
858 self.memoize(obj)
--> 859 self._batch_setitems(obj.items())
860
861 dispatch[dict] = save_dict
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in _batch_setitems(self, items)
883 for k, v in tmp:
884 save(k)
--> 885 save(v)
886 write(SETITEMS)
887 elif n:
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
547
548 # Save the reduce() output and finally memoize the object
--> 549 self.save_reduce(obj=obj, *rv)
550
551 def persistent_id(self, obj):
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in save_reduce(self, func, args, state, listitems, dictitems, obj)
660
661 if state is not None:
--> 662 save(state)
663 write(BUILD)
664
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
502 f = self.dispatch.get(t)
503 if f is not None:
--> 504 f(self, obj) # Call unbound method with explicit self
505 return
506
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in save_tuple(self, obj)
772 if n <= 3 and self.proto >= 2:
773 for element in obj:
--> 774 save(element)
775 # Subtle. Same as in the big comment below.
776 if id(obj) in memo:
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
502 f = self.dispatch.get(t)
503 if f is not None:
--> 504 f(self, obj) # Call unbound method with explicit self
505 return
506
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in save_dict(self, obj)
857
858 self.memoize(obj)
--> 859 self._batch_setitems(obj.items())
860
861 dispatch[dict] = save_dict
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in _batch_setitems(self, items)
883 for k, v in tmp:
884 save(k)
--> 885 save(v)
886 write(SETITEMS)
887 elif n:
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
502 f = self.dispatch.get(t)
503 if f is not None:
--> 504 f(self, obj) # Call unbound method with explicit self
505 return
506
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in save_list(self, obj)
817
818 self.memoize(obj)
--> 819 self._batch_appends(obj)
820
821 dispatch[list] = save_list
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in _batch_appends(self, items)
844 write(APPENDS)
845 elif n:
--> 846 save(tmp[0])
847 write(APPEND)
848 # else tmp is empty, and we're done
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
533 # Check for string returned by reduce(), meaning "save as global"
534 if isinstance(rv, str):
--> 535 self.save_global(obj, rv)
536 return
537
~/Library/Caches/pypoetry/virtualenvs/domi-IWOYYLRr-py3.7/lib/python3.7/site-packages/cloudpickle/cloudpickle.py in save_global(self, obj, name, pack)
840 self._save_parametrized_type_hint(obj)
841 elif name is not None:
--> 842 Pickler.save_global(self, obj, name=name)
843 elif not _is_importable_by_name(obj, name=name):
844 self.save_dynamic_class(obj)
~/.pyenv/versions/3.7.6/lib/python3.7/pickle.py in save_global(self, obj, name)
958 raise PicklingError(
959 "Can't pickle %r: it's not found as %s.%s" %
--> 960 (obj, module_name, name)) from None
961 else:
962 if obj2 is not obj:
PicklingError: Can't pickle <cyfunction int_validator at 0x116503460>: it's not found as pydantic.validators.lambda11
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment