Skip to content

Instantly share code, notes, and snippets.

@johnfelipe
Created June 22, 2022 16:17
Show Gist options
  • Save johnfelipe/f6dca7da93d1549cfa8391c17fa70a77 to your computer and use it in GitHub Desktop.
Save johnfelipe/f6dca7da93d1549cfa8391c17fa70a77 to your computer and use it in GitHub Desktop.
root@ubuntu20dockers:~/datahub# datahub ingest -c /root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml
/usr/local/lib/python3.8/dist-packages/cryptography/hazmat/backends/openssl/x509.py:14: CryptographyDeprecationWarning: This version of cryptography contains a temporary pyOpenSSL fallback path. Upgrade pyOpenSSL now.
warnings.warn(
[2022-06-22 11:05:33,519] INFO {datahub.cli.ingest_cli:99} - DataHub CLI version: 0.8.38.3
[2022-06-22 11:05:33,898] ERROR {datahub.entrypoints:167} - File "/usr/local/lib/python3.8/dist-packages/datahub/entrypoints.py", line 149, in main
146 def main(**kwargs):
147 # This wrapper prevents click from suppressing errors.
148 try:
--> 149 sys.exit(datahub(standalone_mode=False, **kwargs))
150 except click.exceptions.Abort:
..................................................
kwargs = {}
datahub = <Group datahub>
click.exceptions.Abort = <class 'click.exceptions.Abort'>
..................................................
File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 1130, in __call__
1128 def __call__(self, *args: t.Any, **kwargs: t.Any) -> t.Any:
1129 """Alias for :meth:`main`."""
--> 1130 return self.main(*args, **kwargs)
..................................................
self = <Group datahub>
args = ()
t.Any = typing.Any
kwargs = {'standalone_mode': False}
self.main = <method 'BaseCommand.main' of <Group datahub> core.py:987>
..................................................
File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 1055, in main
987 def main(
988 self,
989 args: t.Optional[t.Sequence[str]] = None,
990 prog_name: t.Optional[str] = None,
991 complete_var: t.Optional[str] = None,
992 standalone_mode: bool = True,
993 windows_expand_args: bool = True,
994 **extra: t.Any,
995 ) -> t.Any:
(...)
1051
1052 try:
1053 try:
1054 with self.make_context(prog_name, args, **extra) as ctx:
--> 1055 rv = self.invoke(ctx)
1056 if not standalone_mode:
..................................................
self = <Group datahub>
args = ['ingest', '-c', '/root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml', ]
t.Optional = typing.Optional
t.Sequence = typing.Sequence
prog_name = 'datahub'
complete_var = None
standalone_mode = False
windows_expand_args = True
extra = {}
t.Any = typing.Any
self.make_context = <method 'BaseCommand.make_context' of <Group datahub> core.py:884>
ctx = <click.core.Context object at 0x7fe037b50400>
self.invoke = <method 'MultiCommand.invoke' of <Group datahub> core.py:1623>
..................................................
File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 1657, in invoke
1623 def invoke(self, ctx: Context) -> t.Any:
(...)
1653 ctx.invoked_subcommand = cmd_name
1654 super().invoke(ctx)
1655 sub_ctx = cmd.make_context(cmd_name, args, parent=ctx)
1656 with sub_ctx:
--> 1657 return _process_result(sub_ctx.command.invoke(sub_ctx))
1658
..................................................
self = <Group datahub>
ctx = <click.core.Context object at 0x7fe037b50400>
Context = <class 'click.core.Context'>
t.Any = typing.Any
ctx.invoked_subcommand = 'ingest'
cmd_name = 'ingest'
sub_ctx = <click.core.Context object at 0x7fe037c6aa90>
cmd.make_context = <method 'BaseCommand.make_context' of <DefaultGroup ingest> core.py:884>
args = ['/root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml', ]
_process_result = <function 'MultiCommand.invoke.<locals>._process_result' core.py:1624>
sub_ctx.command.invoke = <method 'MultiCommand.invoke' of <DefaultGroup ingest> core.py:1623>
..................................................
File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 1657, in invoke
1623 def invoke(self, ctx: Context) -> t.Any:
(...)
1653 ctx.invoked_subcommand = cmd_name
1654 super().invoke(ctx)
1655 sub_ctx = cmd.make_context(cmd_name, args, parent=ctx)
1656 with sub_ctx:
--> 1657 return _process_result(sub_ctx.command.invoke(sub_ctx))
1658
..................................................
self = <DefaultGroup ingest>
ctx = <click.core.Context object at 0x7fe037c6aa90>
Context = <class 'click.core.Context'>
t.Any = typing.Any
ctx.invoked_subcommand = 'run'
cmd_name = 'run'
sub_ctx = <click.core.Context object at 0x7fe029679190>
cmd.make_context = <method 'BaseCommand.make_context' of <Command run> core.py:884>
args = []
_process_result = <function 'MultiCommand.invoke.<locals>._process_result' core.py:1624>
sub_ctx.command.invoke = <method 'Command.invoke' of <Command run> core.py:1393>
..................................................
File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 1404, in invoke
1393 def invoke(self, ctx: Context) -> t.Any:
(...)
1400 ).format(name=self.name)
1401 echo(style(message, fg="red"), err=True)
1402
1403 if self.callback is not None:
--> 1404 return ctx.invoke(self.callback, **ctx.params)
..................................................
self = <Command run>
ctx = <click.core.Context object at 0x7fe029679190>
Context = <class 'click.core.Context'>
t.Any = typing.Any
self.name = 'run'
self.callback = <function 'run' decorators.py:25>
ctx.invoke = <method 'Context.invoke' of <click.core.Context object at 0x7fe029679190> core.py:709>
ctx.params = {'config': '/root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml',
'dry_run': False,
'preview': False,
'preview_workunits': 10,
'strict_warnings': False,
'suppress_error_logs': False}
..................................................
File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 760, in invoke
709 def invoke(
710 __self, # noqa: B902
711 __callback: t.Union["Command", t.Callable[..., t.Any]],
712 *args: t.Any,
713 **kwargs: t.Any,
714 ) -> t.Any:
(...)
756 ctx = __self
757
758 with augment_usage_errors(__self):
759 with ctx:
--> 760 return __callback(*args, **kwargs)
..................................................
t.Union = typing.Union
t.Callable = typing.Callable
t.Any = typing.Any
args = ()
kwargs = {'config': '/root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml',
'dry_run': False,
'preview': False,
'preview_workunits': 10,
'strict_warnings': False,
'suppress_error_logs': False}
ctx = <click.core.Context object at 0x7fe029679190>
..................................................
File "/usr/local/lib/python3.8/dist-packages/click/decorators.py", line 26, in new_func
25 def new_func(*args, **kwargs): # type: ignore
--> 26 return f(get_current_context(), *args, **kwargs)
..................................................
args = ()
kwargs = {'config': '/root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml',
'dry_run': False,
'preview': False,
'preview_workunits': 10,
'strict_warnings': False,
'suppress_error_logs': False}
f = <function 'run' upgrade.py:330>
..................................................
File "/usr/local/lib/python3.8/dist-packages/datahub/upgrade/upgrade.py", line 333, in wrapper
330 @wraps(func)
331 def wrapper(*args: Any, **kwargs: Any) -> Any:
332
--> 333 res = func(*args, **kwargs)
334 try:
..................................................
func = <function 'run' telemetry.py:280>
args = (<click.core.Context object at 0x7fe029679190>, )
Any = typing.Any
kwargs = {'config': '/root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml',
'dry_run': False,
'preview': False,
'preview_workunits': 10,
'strict_warnings': False,
'suppress_error_logs': False}
..................................................
File "/usr/local/lib/python3.8/dist-packages/datahub/telemetry/telemetry.py", line 338, in wrapper
281 def wrapper(*args: Any, **kwargs: Any) -> Any:
(...)
334 "status": "error",
335 "error": get_full_class_name(e),
336 },
337 )
--> 338 raise e
..................................................
args = (<click.core.Context object at 0x7fe029679190>, )
Any = typing.Any
kwargs = {'config': '/root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml',
'dry_run': False,
'preview': False,
'preview_workunits': 10,
'strict_warnings': False,
'suppress_error_logs': False}
..................................................
File "/usr/local/lib/python3.8/dist-packages/datahub/telemetry/telemetry.py", line 290, in wrapper
281 def wrapper(*args: Any, **kwargs: Any) -> Any:
(...)
286 telemetry_instance.ping(
287 "function-call", {"function": function, "status": "start"}
288 )
289 try:
--> 290 res = func(*args, **kwargs)
291 telemetry_instance.ping(
..................................................
args = (<click.core.Context object at 0x7fe029679190>, )
Any = typing.Any
kwargs = {'config': '/root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml',
'dry_run': False,
'preview': False,
'preview_workunits': 10,
'strict_warnings': False,
'suppress_error_logs': False}
telemetry_instance.ping = <method 'Telemetry.ping' of <datahub.telemetry.telemetry.Telemetry object at 0x7fe029841880> telemetry.py:215>
function = 'datahub.cli.ingest_cli.run'
func = <function 'run' memory_leak_detector.py:92>
..................................................
File "/usr/local/lib/python3.8/dist-packages/datahub/utilities/memory_leak_detector.py", line 102, in wrapper
93 def wrapper(*args: Any, **kwargs: Any) -> Any:
(...)
98 )
99 _init_leak_detection()
100
101 try:
--> 102 res = func(*args, **kwargs)
103 return res
..................................................
args = (<click.core.Context object at 0x7fe029679190>, )
Any = typing.Any
kwargs = {'config': '/root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml',
'dry_run': False,
'preview': False,
'preview_workunits': 10,
'strict_warnings': False,
'suppress_error_logs': False}
func = <function 'run' ingest_cli.py:43>
..................................................
File "/usr/local/lib/python3.8/dist-packages/datahub/cli/ingest_cli.py", line 102, in run
88 def run(
89 ctx: click.Context,
90 config: str,
91 dry_run: bool,
92 preview: bool,
93 strict_warnings: bool,
94 preview_workunits: int,
95 suppress_error_logs: bool,
96 ) -> None:
(...)
98
99 logger.info("DataHub CLI version: %s", datahub_package.nice_version_name())
100
101 config_file = pathlib.Path(config)
--> 102 pipeline_config = load_config_file(config_file)
103
..................................................
run = <Command run>
ctx = <click.core.Context object at 0x7fe029679190>
click.Context = <class 'click.core.Context'>
config = '/root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml'
dry_run = False
preview = False
strict_warnings = False
preview_workunits = 10
suppress_error_logs = False
logger.info = <method 'Logger.info' of <Logger datahub.cli.ingest_cli (INFO)> __init__.py:1436>
datahub_package.nice_version_name = <function 'nice_version_name' __init__.py:10>
config_file = PosixPath('/root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml')
pathlib.Path = <class 'pathlib.Path'>
..................................................
File "/usr/local/lib/python3.8/dist-packages/datahub/configuration/config_loader.py", line 72, in load_config_file
51 def load_config_file(config_file: Union[pathlib.Path, str]) -> dict:
(...)
68
69 with config_file.open() as raw_config_fp:
70 raw_config_file = raw_config_fp.read()
71 config_fp = io.StringIO(raw_config_file)
--> 72 config = config_mech.load_config(config_fp)
73 resolve_env_variables(config)
..................................................
config_file = PosixPath('/root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml')
Union = typing.Union
pathlib.Path = <class 'pathlib.Path'>
config_file.open = <method 'Path.open' of /root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml pathlib.py
:1214>
raw_config_fp = <_io.TextIOWrapper name='/root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml' mode='r
' encoding='UTF-8'>
raw_config_file = '---\nsource:\n type: bigquery\n config:\n project_id: "bigquery-public-data"\n options:\n credentials_path
: "./bq_credentials.json"\n table_pattern:\n allow:\n # Allow anything that starts with "covid19"\n -
"bigquery-public-data\\\\.covid19.*"\n deny:\n # Except for tables that end with an underscore.\n - ".*_\
\$" # also escapes the variable expansion on $\n\nsink:\n type: "file"\n config:\n filename: "./examples/demo_data/
all_covid19_datasets.json"\n'
config_fp = <_io.StringIO object at 0x7fe029674310>
io.StringIO = <class '_io.StringIO'>
config_mech.load_config = <method 'YamlConfigurationMechanism.load_config' of <datahub.configuration.yaml.YamlConfigurationMechanism object at 0x7
fe029679280> yaml.py:11>
..................................................
File "/usr/local/lib/python3.8/dist-packages/datahub/configuration/yaml.py", line 12, in load_config
11 def load_config(self, config_fp: IO) -> dict:
--> 12 config = yaml.safe_load(config_fp)
13 return config
..................................................
self = <datahub.configuration.yaml.YamlConfigurationMechanism object at 0x7fe029679280>
config_fp = <_io.StringIO object at 0x7fe029674310>
IO = <class 'typing.IO'>
yaml.safe_load = <function 'safe_load' __init__.py:154>
..................................................
File "/usr/lib/python3/dist-packages/yaml/__init__.py", line 162, in safe_load
154 def safe_load(stream):
(...)
158
159 Resolve only basic YAML tags. This is known
160 to be safe for untrusted input.
161 """
--> 162 return load(stream, SafeLoader)
..................................................
stream = <_io.StringIO object at 0x7fe029674310>
SafeLoader = <class 'yaml.loader.SafeLoader'>
..................................................
File "/usr/lib/python3/dist-packages/yaml/__init__.py", line 114, in load
103 def load(stream, Loader=None):
(...)
110 Loader = FullLoader
111
112 loader = Loader(stream)
113 try:
--> 114 return loader.get_single_data()
115 finally:
..................................................
stream = <_io.StringIO object at 0x7fe029674310>
Loader = <class 'yaml.loader.SafeLoader'>
FullLoader = <class 'yaml.loader.FullLoader'>
loader = <yaml.loader.SafeLoader object at 0x7fe029679df0>
loader.get_single_data = <method 'BaseConstructor.get_single_data' of <yaml.loader.SafeLoader object at 0x7fe029679df0> constructor.py:47>
..................................................
File "/usr/lib/python3/dist-packages/yaml/constructor.py", line 49, in get_single_data
47 def get_single_data(self):
48 # Ensure that the stream contains a single document and construct it.
--> 49 node = self.get_single_node()
50 if node is not None:
..................................................
self = <yaml.loader.SafeLoader object at 0x7fe029679df0>
self.get_single_node = <method 'Composer.get_single_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:29>
..................................................
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 36, in get_single_node
29 def get_single_node(self):
(...)
32
33 # Compose a document if the stream is not empty.
34 document = None
35 if not self.check_event(StreamEndEvent):
--> 36 document = self.compose_document()
37
..................................................
self = <yaml.loader.SafeLoader object at 0x7fe029679df0>
document = None
self.check_event = <method 'Parser.check_event' of <yaml.loader.SafeLoader object at 0x7fe029679df0> parser.py:94>
StreamEndEvent = <class 'yaml.events.StreamEndEvent'>
self.compose_document = <method 'Composer.compose_document' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:50>
..................................................
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 55, in compose_document
50 def compose_document(self):
51 # Drop the DOCUMENT-START event.
52 self.get_event()
53
54 # Compose the root node.
--> 55 node = self.compose_node(None, None)
56
..................................................
self = <yaml.loader.SafeLoader object at 0x7fe029679df0>
self.get_event = <method 'Parser.get_event' of <yaml.loader.SafeLoader object at 0x7fe029679df0> parser.py:114>
self.compose_node = <method 'Composer.compose_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:63>
..................................................
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 84, in compose_node
63 def compose_node(self, parent, index):
(...)
80 node = self.compose_scalar_node(anchor)
81 elif self.check_event(SequenceStartEvent):
82 node = self.compose_sequence_node(anchor)
83 elif self.check_event(MappingStartEvent):
--> 84 node = self.compose_mapping_node(anchor)
85 self.ascend_resolver()
..................................................
self = <yaml.loader.SafeLoader object at 0x7fe029679df0>
parent = None
index = None
self.compose_scalar_node = <method 'Composer.compose_scalar_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:88>
anchor = None
self.check_event = <method 'Parser.check_event' of <yaml.loader.SafeLoader object at 0x7fe029679df0> parser.py:94>
SequenceStartEvent = <class 'yaml.events.SequenceStartEvent'>
self.compose_sequence_node = <method 'Composer.compose_sequence_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:99>
MappingStartEvent = <class 'yaml.events.MappingStartEvent'>
self.compose_mapping_node = <method 'Composer.compose_mapping_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:117>
self.ascend_resolver = <method 'BaseResolver.ascend_resolver' of <yaml.loader.SafeLoader object at 0x7fe029679df0> resolver.py:114>
..................................................
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 133, in compose_mapping_node
117 def compose_mapping_node(self, anchor):
(...)
129 item_key = self.compose_node(node, None)
130 #if item_key in node.value:
131 # raise ComposerError("while composing a mapping", start_event.start_mark,
132 # "found duplicate key", key_event.start_mark)
--> 133 item_value = self.compose_node(node, item_key)
134 #node.value[item_key] = item_value
..................................................
self = <yaml.loader.SafeLoader object at 0x7fe029679df0>
anchor = None
item_key = ScalarNode(tag='tag:yaml.org,2002:str', value='source')
self.compose_node = <method 'Composer.compose_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:63>
node = MappingNode(tag='tag:yaml.org,2002:map', value=[])
..................................................
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 84, in compose_node
63 def compose_node(self, parent, index):
(...)
80 node = self.compose_scalar_node(anchor)
81 elif self.check_event(SequenceStartEvent):
82 node = self.compose_sequence_node(anchor)
83 elif self.check_event(MappingStartEvent):
--> 84 node = self.compose_mapping_node(anchor)
85 self.ascend_resolver()
..................................................
self = <yaml.loader.SafeLoader object at 0x7fe029679df0>
parent = MappingNode(tag='tag:yaml.org,2002:map', value=[])
index = ScalarNode(tag='tag:yaml.org,2002:str', value='source')
self.compose_scalar_node = <method 'Composer.compose_scalar_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:88>
anchor = None
self.check_event = <method 'Parser.check_event' of <yaml.loader.SafeLoader object at 0x7fe029679df0> parser.py:94>
SequenceStartEvent = <class 'yaml.events.SequenceStartEvent'>
self.compose_sequence_node = <method 'Composer.compose_sequence_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:99>
MappingStartEvent = <class 'yaml.events.MappingStartEvent'>
self.compose_mapping_node = <method 'Composer.compose_mapping_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:117>
self.ascend_resolver = <method 'BaseResolver.ascend_resolver' of <yaml.loader.SafeLoader object at 0x7fe029679df0> resolver.py:114>
..................................................
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 133, in compose_mapping_node
117 def compose_mapping_node(self, anchor):
(...)
129 item_key = self.compose_node(node, None)
130 #if item_key in node.value:
131 # raise ComposerError("while composing a mapping", start_event.start_mark,
132 # "found duplicate key", key_event.start_mark)
--> 133 item_value = self.compose_node(node, item_key)
134 #node.value[item_key] = item_value
..................................................
self = <yaml.loader.SafeLoader object at 0x7fe029679df0>
anchor = None
item_key = ScalarNode(tag='tag:yaml.org,2002:str', value='config')
self.compose_node = <method 'Composer.compose_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:63>
node = MappingNode(tag='tag:yaml.org,2002:map', value=[(ScalarNode(tag='tag:yaml.org,2002:str', value='type'), ScalarNode(tag='
tag:yaml.org,2002:str', value='bigquery'))])
item_value = ScalarNode(tag='tag:yaml.org,2002:str', value='bigquery')
..................................................
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 84, in compose_node
63 def compose_node(self, parent, index):
(...)
80 node = self.compose_scalar_node(anchor)
81 elif self.check_event(SequenceStartEvent):
82 node = self.compose_sequence_node(anchor)
83 elif self.check_event(MappingStartEvent):
--> 84 node = self.compose_mapping_node(anchor)
85 self.ascend_resolver()
..................................................
self = <yaml.loader.SafeLoader object at 0x7fe029679df0>
parent = MappingNode(tag='tag:yaml.org,2002:map', value=[(ScalarNode(tag='tag:yaml.org,2002:str', value='type'), ScalarNode(tag='
tag:yaml.org,2002:str', value='bigquery'))])
index = ScalarNode(tag='tag:yaml.org,2002:str', value='config')
self.compose_scalar_node = <method 'Composer.compose_scalar_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:88>
anchor = None
self.check_event = <method 'Parser.check_event' of <yaml.loader.SafeLoader object at 0x7fe029679df0> parser.py:94>
SequenceStartEvent = <class 'yaml.events.SequenceStartEvent'>
self.compose_sequence_node = <method 'Composer.compose_sequence_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:99>
MappingStartEvent = <class 'yaml.events.MappingStartEvent'>
self.compose_mapping_node = <method 'Composer.compose_mapping_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:117>
self.ascend_resolver = <method 'BaseResolver.ascend_resolver' of <yaml.loader.SafeLoader object at 0x7fe029679df0> resolver.py:114>
..................................................
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 133, in compose_mapping_node
117 def compose_mapping_node(self, anchor):
(...)
129 item_key = self.compose_node(node, None)
130 #if item_key in node.value:
131 # raise ComposerError("while composing a mapping", start_event.start_mark,
132 # "found duplicate key", key_event.start_mark)
--> 133 item_value = self.compose_node(node, item_key)
134 #node.value[item_key] = item_value
..................................................
self = <yaml.loader.SafeLoader object at 0x7fe029679df0>
anchor = None
item_key = ScalarNode(tag='tag:yaml.org,2002:str', value='table_pattern')
self.compose_node = <method 'Composer.compose_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:63>
node = MappingNode(tag='tag:yaml.org,2002:map', value=[(ScalarNode(tag='tag:yaml.org,2002:str', value='project_id'), ScalarNode
(tag='tag:yaml.org,2002:str', value='bigquery-public-data')), (ScalarNode(tag='tag:yaml.org,2002:str', value='options'),
MappingNode(tag='tag:yaml.org,2002:map', value=[(ScalarNode(tag='tag:yaml.org,2002:str', value='credentials_path'), Sca
larNode(tag='tag:yaml.org,2002:str', value='./bq_credentials.json'))]))])
item_value = MappingNode(tag='tag:yaml.org,2002:map', value=[(ScalarNode(tag='tag:yaml.org,2002:str', value='credentials_path'), Scal
arNode(tag='tag:yaml.org,2002:str', value='./bq_credentials.json'))])
..................................................
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 84, in compose_node
63 def compose_node(self, parent, index):
(...)
80 node = self.compose_scalar_node(anchor)
81 elif self.check_event(SequenceStartEvent):
82 node = self.compose_sequence_node(anchor)
83 elif self.check_event(MappingStartEvent):
--> 84 node = self.compose_mapping_node(anchor)
85 self.ascend_resolver()
..................................................
self = <yaml.loader.SafeLoader object at 0x7fe029679df0>
parent = MappingNode(tag='tag:yaml.org,2002:map', value=[(ScalarNode(tag='tag:yaml.org,2002:str', value='project_id'), ScalarNode
(tag='tag:yaml.org,2002:str', value='bigquery-public-data')), (ScalarNode(tag='tag:yaml.org,2002:str', value='options'),
MappingNode(tag='tag:yaml.org,2002:map', value=[(ScalarNode(tag='tag:yaml.org,2002:str', value='credentials_path'), Sca
larNode(tag='tag:yaml.org,2002:str', value='./bq_credentials.json'))]))])
index = ScalarNode(tag='tag:yaml.org,2002:str', value='table_pattern')
self.compose_scalar_node = <method 'Composer.compose_scalar_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:88>
anchor = None
self.check_event = <method 'Parser.check_event' of <yaml.loader.SafeLoader object at 0x7fe029679df0> parser.py:94>
SequenceStartEvent = <class 'yaml.events.SequenceStartEvent'>
self.compose_sequence_node = <method 'Composer.compose_sequence_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:99>
MappingStartEvent = <class 'yaml.events.MappingStartEvent'>
self.compose_mapping_node = <method 'Composer.compose_mapping_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:117>
self.ascend_resolver = <method 'BaseResolver.ascend_resolver' of <yaml.loader.SafeLoader object at 0x7fe029679df0> resolver.py:114>
..................................................
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 133, in compose_mapping_node
117 def compose_mapping_node(self, anchor):
(...)
129 item_key = self.compose_node(node, None)
130 #if item_key in node.value:
131 # raise ComposerError("while composing a mapping", start_event.start_mark,
132 # "found duplicate key", key_event.start_mark)
--> 133 item_value = self.compose_node(node, item_key)
134 #node.value[item_key] = item_value
..................................................
self = <yaml.loader.SafeLoader object at 0x7fe029679df0>
anchor = None
item_key = ScalarNode(tag='tag:yaml.org,2002:str', value='deny')
self.compose_node = <method 'Composer.compose_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:63>
node = MappingNode(tag='tag:yaml.org,2002:map', value=[(ScalarNode(tag='tag:yaml.org,2002:str', value='allow'), SequenceNode(ta
g='tag:yaml.org,2002:seq', value=[ScalarNode(tag='tag:yaml.org,2002:str', value='bigquery-public-data\\.covid19.*')]))])
item_value = SequenceNode(tag='tag:yaml.org,2002:seq', value=[ScalarNode(tag='tag:yaml.org,2002:str', value='bigquery-public-data\\.c
ovid19.*')])
..................................................
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 82, in compose_node
63 def compose_node(self, parent, index):
(...)
78 self.descend_resolver(parent, index)
79 if self.check_event(ScalarEvent):
80 node = self.compose_scalar_node(anchor)
81 elif self.check_event(SequenceStartEvent):
--> 82 node = self.compose_sequence_node(anchor)
83 elif self.check_event(MappingStartEvent):
..................................................
self = <yaml.loader.SafeLoader object at 0x7fe029679df0>
parent = MappingNode(tag='tag:yaml.org,2002:map', value=[(ScalarNode(tag='tag:yaml.org,2002:str', value='allow'), SequenceNode(ta
g='tag:yaml.org,2002:seq', value=[ScalarNode(tag='tag:yaml.org,2002:str', value='bigquery-public-data\\.covid19.*')]))])
index = ScalarNode(tag='tag:yaml.org,2002:str', value='deny')
self.descend_resolver = <method 'BaseResolver.descend_resolver' of <yaml.loader.SafeLoader object at 0x7fe029679df0> resolver.py:91>
self.check_event = <method 'Parser.check_event' of <yaml.loader.SafeLoader object at 0x7fe029679df0> parser.py:94>
ScalarEvent = <class 'yaml.events.ScalarEvent'>
self.compose_scalar_node = <method 'Composer.compose_scalar_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:88>
anchor = None
SequenceStartEvent = <class 'yaml.events.SequenceStartEvent'>
self.compose_sequence_node = <method 'Composer.compose_sequence_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:99>
MappingStartEvent = <class 'yaml.events.MappingStartEvent'>
..................................................
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 110, in compose_sequence_node
99 def compose_sequence_node(self, anchor):
(...)
106 flow_style=start_event.flow_style)
107 if anchor is not None:
108 self.anchors[anchor] = node
109 index = 0
--> 110 while not self.check_event(SequenceEndEvent):
111 node.value.append(self.compose_node(node, index))
..................................................
self = <yaml.loader.SafeLoader object at 0x7fe029679df0>
anchor = None
start_event.flow_style = None
self.anchors = {}
node = SequenceNode(tag='tag:yaml.org,2002:seq', value=[])
index = 0
self.check_event = <method 'Parser.check_event' of <yaml.loader.SafeLoader object at 0x7fe029679df0> parser.py:94>
SequenceEndEvent = <class 'yaml.events.SequenceEndEvent'>
self.compose_node = <method 'Composer.compose_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:63>
..................................................
File "/usr/lib/python3/dist-packages/yaml/parser.py", line 98, in check_event
94 def check_event(self, *choices):
95 # Check the type of the next event.
96 if self.current_event is None:
97 if self.state:
--> 98 self.current_event = self.state()
99 if self.current_event is not None:
..................................................
self = <yaml.loader.SafeLoader object at 0x7fe029679df0>
choices = (<class 'yaml.events.SequenceEndEvent'>, )
self.current_event = None
self.state = None
..................................................
File "/usr/lib/python3/dist-packages/yaml/parser.py", line 405, in parse_indentless_sequence_entry
402 def parse_indentless_sequence_entry(self):
403 if self.check_token(BlockEntryToken):
404 token = self.get_token()
--> 405 if not self.check_token(BlockEntryToken,
406 KeyToken, ValueToken, BlockEndToken):
..................................................
self = <yaml.loader.SafeLoader object at 0x7fe029679df0>
self.check_token = <method 'Scanner.check_token' of <yaml.loader.SafeLoader object at 0x7fe029679df0> scanner.py:113>
BlockEntryToken = <class 'yaml.tokens.BlockEntryToken'>
token = BlockEntryToken()
self.get_token = <method 'Scanner.get_token' of <yaml.loader.SafeLoader object at 0x7fe029679df0> scanner.py:135>
KeyToken = <class 'yaml.tokens.KeyToken'>
ValueToken = <class 'yaml.tokens.ValueToken'>
BlockEndToken = <class 'yaml.tokens.BlockEndToken'>
..................................................
File "/usr/lib/python3/dist-packages/yaml/scanner.py", line 116, in check_token
113 def check_token(self, *choices):
114 # Check if the next token is one of the given types.
115 while self.need_more_tokens():
--> 116 self.fetch_more_tokens()
117 if self.tokens:
..................................................
self = <yaml.loader.SafeLoader object at 0x7fe029679df0>
choices = (<class 'yaml.tokens.BlockEntryToken'>, <class 'yaml.tokens.KeyToken'>, <class 'yaml.tokens.ValueToken'>, <class 'yaml.t
okens.BlockEndToken'>, )
self.need_more_tokens = <method 'Scanner.need_more_tokens' of <yaml.loader.SafeLoader object at 0x7fe029679df0> scanner.py:145>
self.fetch_more_tokens = <method 'Scanner.fetch_more_tokens' of <yaml.loader.SafeLoader object at 0x7fe029679df0> scanner.py:156>
self.tokens = []
..................................................
File "/usr/lib/python3/dist-packages/yaml/scanner.py", line 251, in fetch_more_tokens
156 def fetch_more_tokens(self):
(...)
247 return self.fetch_single()
248
249 # Is it a double quoted scalar?
250 if ch == '\"':
--> 251 return self.fetch_double()
252
..................................................
self = <yaml.loader.SafeLoader object at 0x7fe029679df0>
self.fetch_single = <method 'Scanner.fetch_single' of <yaml.loader.SafeLoader object at 0x7fe029679df0> scanner.py:651>
ch = '"'
self.fetch_double = <method 'Scanner.fetch_double' of <yaml.loader.SafeLoader object at 0x7fe029679df0> scanner.py:654>
..................................................
File "/usr/lib/python3/dist-packages/yaml/scanner.py", line 655, in fetch_double
654 def fetch_double(self):
--> 655 self.fetch_flow_scalar(style='"')
..................................................
self = <yaml.loader.SafeLoader object at 0x7fe029679df0>
self.fetch_flow_scalar = <method 'Scanner.fetch_flow_scalar' of <yaml.loader.SafeLoader object at 0x7fe029679df0> scanner.py:657>
..................................................
File "/usr/lib/python3/dist-packages/yaml/scanner.py", line 666, in fetch_flow_scalar
657 def fetch_flow_scalar(self, style):
(...)
662 # No simple keys after flow scalars.
663 self.allow_simple_key = False
664
665 # Scan and add SCALAR.
--> 666 self.tokens.append(self.scan_flow_scalar(style))
..................................................
self = <yaml.loader.SafeLoader object at 0x7fe029679df0>
style = '"'
self.allow_simple_key = False
self.scan_flow_scalar = <method 'Scanner.scan_flow_scalar' of <yaml.loader.SafeLoader object at 0x7fe029679df0> scanner.py:1134>
..................................................
File "/usr/lib/python3/dist-packages/yaml/scanner.py", line 1149, in scan_flow_scalar
1134 def scan_flow_scalar(self, style):
(...)
1145 chunks = []
1146 start_mark = self.get_mark()
1147 quote = self.peek()
1148 self.forward()
--> 1149 chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark))
1150 while self.peek() != quote:
..................................................
self = <yaml.loader.SafeLoader object at 0x7fe029679df0>
style = '"'
chunks = []
start_mark = <yaml.error.Mark object at 0x7fe0295a27c0>
self.get_mark = <method 'Reader.get_mark' of <yaml.loader.SafeLoader object at 0x7fe029679df0> reader.py:114>
quote = '"'
self.peek = <method 'Reader.peek' of <yaml.loader.SafeLoader object at 0x7fe029679df0> reader.py:87>
self.forward = <method 'Reader.forward' of <yaml.loader.SafeLoader object at 0x7fe029679df0> reader.py:99>
self.scan_flow_scalar_non_spaces = <method 'Scanner.scan_flow_scalar_non_spaces' of <yaml.loader.SafeLoader object at 0x7fe029679df0> scanner.py:1185>
double = True
..................................................
File "/usr/lib/python3/dist-packages/yaml/scanner.py", line 1223, in scan_flow_scalar_non_spaces
1185 def scan_flow_scalar_non_spaces(self, double, start_mark):
(...)
1219 elif ch in '\r\n\x85\u2028\u2029':
1220 self.scan_line_break()
1221 chunks.extend(self.scan_flow_scalar_breaks(double, start_mark))
1222 else:
--> 1223 raise ScannerError("while scanning a double-quoted scalar", start_mark,
1224 "found unknown escape character %r" % ch, self.get_mark())
..................................................
self = <yaml.loader.SafeLoader object at 0x7fe029679df0>
double = True
start_mark = <yaml.error.Mark object at 0x7fe0295a27c0>
ch = '$'
self.scan_line_break = <method 'Scanner.scan_line_break' of <yaml.loader.SafeLoader object at 0x7fe029679df0> scanner.py:1416>
self.scan_flow_scalar_breaks = <method 'Scanner.scan_flow_scalar_breaks' of <yaml.loader.SafeLoader object at 0x7fe029679df0> scanner.py:1252>
ScannerError = <class 'yaml.scanner.ScannerError'>
self.get_mark = <method 'Reader.get_mark' of <yaml.loader.SafeLoader object at 0x7fe029679df0> reader.py:114>
..................................................
---- (full traceback above) ----
File "/usr/local/lib/python3.8/dist-packages/datahub/entrypoints.py", line 149, in main
sys.exit(datahub(standalone_mode=False, **kwargs))
File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 1130, in __call__
return self.main(*args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 1055, in main
rv = self.invoke(ctx)
File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 1657, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 1657, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 1404, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 760, in invoke
return __callback(*args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/click/decorators.py", line 26, in new_func
return f(get_current_context(), *args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/datahub/upgrade/upgrade.py", line 333, in wrapper
res = func(*args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/datahub/telemetry/telemetry.py", line 338, in wrapper
raise e
File "/usr/local/lib/python3.8/dist-packages/datahub/telemetry/telemetry.py", line 290, in wrapper
res = func(*args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/datahub/utilities/memory_leak_detector.py", line 102, in wrapper
res = func(*args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/datahub/cli/ingest_cli.py", line 102, in run
pipeline_config = load_config_file(config_file)
File "/usr/local/lib/python3.8/dist-packages/datahub/configuration/config_loader.py", line 72, in load_config_file
config = config_mech.load_config(config_fp)
File "/usr/local/lib/python3.8/dist-packages/datahub/configuration/yaml.py", line 12, in load_config
config = yaml.safe_load(config_fp)
File "/usr/lib/python3/dist-packages/yaml/__init__.py", line 162, in safe_load
return load(stream, SafeLoader)
File "/usr/lib/python3/dist-packages/yaml/__init__.py", line 114, in load
return loader.get_single_data()
File "/usr/lib/python3/dist-packages/yaml/constructor.py", line 49, in get_single_data
node = self.get_single_node()
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 36, in get_single_node
document = self.compose_document()
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 55, in compose_document
node = self.compose_node(None, None)
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 84, in compose_node
node = self.compose_mapping_node(anchor)
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 133, in compose_mapping_node
item_value = self.compose_node(node, item_key)
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 84, in compose_node
node = self.compose_mapping_node(anchor)
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 133, in compose_mapping_node
item_value = self.compose_node(node, item_key)
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 84, in compose_node
node = self.compose_mapping_node(anchor)
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 133, in compose_mapping_node
item_value = self.compose_node(node, item_key)
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 84, in compose_node
node = self.compose_mapping_node(anchor)
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 133, in compose_mapping_node
item_value = self.compose_node(node, item_key)
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 82, in compose_node
node = self.compose_sequence_node(anchor)
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 110, in compose_sequence_node
while not self.check_event(SequenceEndEvent):
File "/usr/lib/python3/dist-packages/yaml/parser.py", line 98, in check_event
self.current_event = self.state()
File "/usr/lib/python3/dist-packages/yaml/parser.py", line 405, in parse_indentless_sequence_entry
if not self.check_token(BlockEntryToken,
File "/usr/lib/python3/dist-packages/yaml/scanner.py", line 116, in check_token
self.fetch_more_tokens()
File "/usr/lib/python3/dist-packages/yaml/scanner.py", line 251, in fetch_more_tokens
return self.fetch_double()
File "/usr/lib/python3/dist-packages/yaml/scanner.py", line 655, in fetch_double
self.fetch_flow_scalar(style='"')
File "/usr/lib/python3/dist-packages/yaml/scanner.py", line 666, in fetch_flow_scalar
self.tokens.append(self.scan_flow_scalar(style))
File "/usr/lib/python3/dist-packages/yaml/scanner.py", line 1149, in scan_flow_scalar
chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark))
File "/usr/lib/python3/dist-packages/yaml/scanner.py", line 1223, in scan_flow_scalar_non_spaces
raise ScannerError("while scanning a double-quoted scalar", start_mark,
ScannerError: while scanning a double-quoted scalar
in "<file>", line 14, column 9
found unknown escape character '$'
in "<file>", line 14, column 14
[2022-06-22 11:05:33,909] INFO {datahub.entrypoints:176} - DataHub CLI version: 0.8.38.3 at /usr/local/lib/python3.8/dist-packages/datahub/__init__.py
[2022-06-22 11:05:33,909] INFO {datahub.entrypoints:179} - Python version: 3.8.10 (default, Mar 15 2022, 12:22:08)
[GCC 9.4.0] at /usr/bin/python3 on Linux-5.4.0-117-generic-x86_64-with-glibc2.29
[2022-06-22 11:05:33,909] INFO {datahub.entrypoints:182} - GMS config {}
root@ubuntu20dockers:~/datahub#
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment