-
-
Save johnfelipe/f6dca7da93d1549cfa8391c17fa70a77 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
root@ubuntu20dockers:~/datahub# datahub ingest -c /root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml | |
/usr/local/lib/python3.8/dist-packages/cryptography/hazmat/backends/openssl/x509.py:14: CryptographyDeprecationWarning: This version of cryptography contains a temporary pyOpenSSL fallback path. Upgrade pyOpenSSL now. | |
warnings.warn( | |
[2022-06-22 11:05:33,519] INFO {datahub.cli.ingest_cli:99} - DataHub CLI version: 0.8.38.3 | |
[2022-06-22 11:05:33,898] ERROR {datahub.entrypoints:167} - File "/usr/local/lib/python3.8/dist-packages/datahub/entrypoints.py", line 149, in main | |
146 def main(**kwargs): | |
147 # This wrapper prevents click from suppressing errors. | |
148 try: | |
--> 149 sys.exit(datahub(standalone_mode=False, **kwargs)) | |
150 except click.exceptions.Abort: | |
.................................................. | |
kwargs = {} | |
datahub = <Group datahub> | |
click.exceptions.Abort = <class 'click.exceptions.Abort'> | |
.................................................. | |
File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 1130, in __call__ | |
1128 def __call__(self, *args: t.Any, **kwargs: t.Any) -> t.Any: | |
1129 """Alias for :meth:`main`.""" | |
--> 1130 return self.main(*args, **kwargs) | |
.................................................. | |
self = <Group datahub> | |
args = () | |
t.Any = typing.Any | |
kwargs = {'standalone_mode': False} | |
self.main = <method 'BaseCommand.main' of <Group datahub> core.py:987> | |
.................................................. | |
File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 1055, in main | |
987 def main( | |
988 self, | |
989 args: t.Optional[t.Sequence[str]] = None, | |
990 prog_name: t.Optional[str] = None, | |
991 complete_var: t.Optional[str] = None, | |
992 standalone_mode: bool = True, | |
993 windows_expand_args: bool = True, | |
994 **extra: t.Any, | |
995 ) -> t.Any: | |
(...) | |
1051 | |
1052 try: | |
1053 try: | |
1054 with self.make_context(prog_name, args, **extra) as ctx: | |
--> 1055 rv = self.invoke(ctx) | |
1056 if not standalone_mode: | |
.................................................. | |
self = <Group datahub> | |
args = ['ingest', '-c', '/root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml', ] | |
t.Optional = typing.Optional | |
t.Sequence = typing.Sequence | |
prog_name = 'datahub' | |
complete_var = None | |
standalone_mode = False | |
windows_expand_args = True | |
extra = {} | |
t.Any = typing.Any | |
self.make_context = <method 'BaseCommand.make_context' of <Group datahub> core.py:884> | |
ctx = <click.core.Context object at 0x7fe037b50400> | |
self.invoke = <method 'MultiCommand.invoke' of <Group datahub> core.py:1623> | |
.................................................. | |
File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 1657, in invoke | |
1623 def invoke(self, ctx: Context) -> t.Any: | |
(...) | |
1653 ctx.invoked_subcommand = cmd_name | |
1654 super().invoke(ctx) | |
1655 sub_ctx = cmd.make_context(cmd_name, args, parent=ctx) | |
1656 with sub_ctx: | |
--> 1657 return _process_result(sub_ctx.command.invoke(sub_ctx)) | |
1658 | |
.................................................. | |
self = <Group datahub> | |
ctx = <click.core.Context object at 0x7fe037b50400> | |
Context = <class 'click.core.Context'> | |
t.Any = typing.Any | |
ctx.invoked_subcommand = 'ingest' | |
cmd_name = 'ingest' | |
sub_ctx = <click.core.Context object at 0x7fe037c6aa90> | |
cmd.make_context = <method 'BaseCommand.make_context' of <DefaultGroup ingest> core.py:884> | |
args = ['/root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml', ] | |
_process_result = <function 'MultiCommand.invoke.<locals>._process_result' core.py:1624> | |
sub_ctx.command.invoke = <method 'MultiCommand.invoke' of <DefaultGroup ingest> core.py:1623> | |
.................................................. | |
File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 1657, in invoke | |
1623 def invoke(self, ctx: Context) -> t.Any: | |
(...) | |
1653 ctx.invoked_subcommand = cmd_name | |
1654 super().invoke(ctx) | |
1655 sub_ctx = cmd.make_context(cmd_name, args, parent=ctx) | |
1656 with sub_ctx: | |
--> 1657 return _process_result(sub_ctx.command.invoke(sub_ctx)) | |
1658 | |
.................................................. | |
self = <DefaultGroup ingest> | |
ctx = <click.core.Context object at 0x7fe037c6aa90> | |
Context = <class 'click.core.Context'> | |
t.Any = typing.Any | |
ctx.invoked_subcommand = 'run' | |
cmd_name = 'run' | |
sub_ctx = <click.core.Context object at 0x7fe029679190> | |
cmd.make_context = <method 'BaseCommand.make_context' of <Command run> core.py:884> | |
args = [] | |
_process_result = <function 'MultiCommand.invoke.<locals>._process_result' core.py:1624> | |
sub_ctx.command.invoke = <method 'Command.invoke' of <Command run> core.py:1393> | |
.................................................. | |
File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 1404, in invoke | |
1393 def invoke(self, ctx: Context) -> t.Any: | |
(...) | |
1400 ).format(name=self.name) | |
1401 echo(style(message, fg="red"), err=True) | |
1402 | |
1403 if self.callback is not None: | |
--> 1404 return ctx.invoke(self.callback, **ctx.params) | |
.................................................. | |
self = <Command run> | |
ctx = <click.core.Context object at 0x7fe029679190> | |
Context = <class 'click.core.Context'> | |
t.Any = typing.Any | |
self.name = 'run' | |
self.callback = <function 'run' decorators.py:25> | |
ctx.invoke = <method 'Context.invoke' of <click.core.Context object at 0x7fe029679190> core.py:709> | |
ctx.params = {'config': '/root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml', | |
'dry_run': False, | |
'preview': False, | |
'preview_workunits': 10, | |
'strict_warnings': False, | |
'suppress_error_logs': False} | |
.................................................. | |
File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 760, in invoke | |
709 def invoke( | |
710 __self, # noqa: B902 | |
711 __callback: t.Union["Command", t.Callable[..., t.Any]], | |
712 *args: t.Any, | |
713 **kwargs: t.Any, | |
714 ) -> t.Any: | |
(...) | |
756 ctx = __self | |
757 | |
758 with augment_usage_errors(__self): | |
759 with ctx: | |
--> 760 return __callback(*args, **kwargs) | |
.................................................. | |
t.Union = typing.Union | |
t.Callable = typing.Callable | |
t.Any = typing.Any | |
args = () | |
kwargs = {'config': '/root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml', | |
'dry_run': False, | |
'preview': False, | |
'preview_workunits': 10, | |
'strict_warnings': False, | |
'suppress_error_logs': False} | |
ctx = <click.core.Context object at 0x7fe029679190> | |
.................................................. | |
File "/usr/local/lib/python3.8/dist-packages/click/decorators.py", line 26, in new_func | |
25 def new_func(*args, **kwargs): # type: ignore | |
--> 26 return f(get_current_context(), *args, **kwargs) | |
.................................................. | |
args = () | |
kwargs = {'config': '/root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml', | |
'dry_run': False, | |
'preview': False, | |
'preview_workunits': 10, | |
'strict_warnings': False, | |
'suppress_error_logs': False} | |
f = <function 'run' upgrade.py:330> | |
.................................................. | |
File "/usr/local/lib/python3.8/dist-packages/datahub/upgrade/upgrade.py", line 333, in wrapper | |
330 @wraps(func) | |
331 def wrapper(*args: Any, **kwargs: Any) -> Any: | |
332 | |
--> 333 res = func(*args, **kwargs) | |
334 try: | |
.................................................. | |
func = <function 'run' telemetry.py:280> | |
args = (<click.core.Context object at 0x7fe029679190>, ) | |
Any = typing.Any | |
kwargs = {'config': '/root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml', | |
'dry_run': False, | |
'preview': False, | |
'preview_workunits': 10, | |
'strict_warnings': False, | |
'suppress_error_logs': False} | |
.................................................. | |
File "/usr/local/lib/python3.8/dist-packages/datahub/telemetry/telemetry.py", line 338, in wrapper | |
281 def wrapper(*args: Any, **kwargs: Any) -> Any: | |
(...) | |
334 "status": "error", | |
335 "error": get_full_class_name(e), | |
336 }, | |
337 ) | |
--> 338 raise e | |
.................................................. | |
args = (<click.core.Context object at 0x7fe029679190>, ) | |
Any = typing.Any | |
kwargs = {'config': '/root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml', | |
'dry_run': False, | |
'preview': False, | |
'preview_workunits': 10, | |
'strict_warnings': False, | |
'suppress_error_logs': False} | |
.................................................. | |
File "/usr/local/lib/python3.8/dist-packages/datahub/telemetry/telemetry.py", line 290, in wrapper | |
281 def wrapper(*args: Any, **kwargs: Any) -> Any: | |
(...) | |
286 telemetry_instance.ping( | |
287 "function-call", {"function": function, "status": "start"} | |
288 ) | |
289 try: | |
--> 290 res = func(*args, **kwargs) | |
291 telemetry_instance.ping( | |
.................................................. | |
args = (<click.core.Context object at 0x7fe029679190>, ) | |
Any = typing.Any | |
kwargs = {'config': '/root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml', | |
'dry_run': False, | |
'preview': False, | |
'preview_workunits': 10, | |
'strict_warnings': False, | |
'suppress_error_logs': False} | |
telemetry_instance.ping = <method 'Telemetry.ping' of <datahub.telemetry.telemetry.Telemetry object at 0x7fe029841880> telemetry.py:215> | |
function = 'datahub.cli.ingest_cli.run' | |
func = <function 'run' memory_leak_detector.py:92> | |
.................................................. | |
File "/usr/local/lib/python3.8/dist-packages/datahub/utilities/memory_leak_detector.py", line 102, in wrapper | |
93 def wrapper(*args: Any, **kwargs: Any) -> Any: | |
(...) | |
98 ) | |
99 _init_leak_detection() | |
100 | |
101 try: | |
--> 102 res = func(*args, **kwargs) | |
103 return res | |
.................................................. | |
args = (<click.core.Context object at 0x7fe029679190>, ) | |
Any = typing.Any | |
kwargs = {'config': '/root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml', | |
'dry_run': False, | |
'preview': False, | |
'preview_workunits': 10, | |
'strict_warnings': False, | |
'suppress_error_logs': False} | |
func = <function 'run' ingest_cli.py:43> | |
.................................................. | |
File "/usr/local/lib/python3.8/dist-packages/datahub/cli/ingest_cli.py", line 102, in run | |
88 def run( | |
89 ctx: click.Context, | |
90 config: str, | |
91 dry_run: bool, | |
92 preview: bool, | |
93 strict_warnings: bool, | |
94 preview_workunits: int, | |
95 suppress_error_logs: bool, | |
96 ) -> None: | |
(...) | |
98 | |
99 logger.info("DataHub CLI version: %s", datahub_package.nice_version_name()) | |
100 | |
101 config_file = pathlib.Path(config) | |
--> 102 pipeline_config = load_config_file(config_file) | |
103 | |
.................................................. | |
run = <Command run> | |
ctx = <click.core.Context object at 0x7fe029679190> | |
click.Context = <class 'click.core.Context'> | |
config = '/root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml' | |
dry_run = False | |
preview = False | |
strict_warnings = False | |
preview_workunits = 10 | |
suppress_error_logs = False | |
logger.info = <method 'Logger.info' of <Logger datahub.cli.ingest_cli (INFO)> __init__.py:1436> | |
datahub_package.nice_version_name = <function 'nice_version_name' __init__.py:10> | |
config_file = PosixPath('/root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml') | |
pathlib.Path = <class 'pathlib.Path'> | |
.................................................. | |
File "/usr/local/lib/python3.8/dist-packages/datahub/configuration/config_loader.py", line 72, in load_config_file | |
51 def load_config_file(config_file: Union[pathlib.Path, str]) -> dict: | |
(...) | |
68 | |
69 with config_file.open() as raw_config_fp: | |
70 raw_config_file = raw_config_fp.read() | |
71 config_fp = io.StringIO(raw_config_file) | |
--> 72 config = config_mech.load_config(config_fp) | |
73 resolve_env_variables(config) | |
.................................................. | |
config_file = PosixPath('/root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml') | |
Union = typing.Union | |
pathlib.Path = <class 'pathlib.Path'> | |
config_file.open = <method 'Path.open' of /root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml pathlib.py | |
:1214> | |
raw_config_fp = <_io.TextIOWrapper name='/root/datahub/metadata-ingestion/examples/demo_data/bigquery_covid19_to_file.dhub.yaml' mode='r | |
' encoding='UTF-8'> | |
raw_config_file = '---\nsource:\n type: bigquery\n config:\n project_id: "bigquery-public-data"\n options:\n credentials_path | |
: "./bq_credentials.json"\n table_pattern:\n allow:\n # Allow anything that starts with "covid19"\n - | |
"bigquery-public-data\\\\.covid19.*"\n deny:\n # Except for tables that end with an underscore.\n - ".*_\ | |
\$" # also escapes the variable expansion on $\n\nsink:\n type: "file"\n config:\n filename: "./examples/demo_data/ | |
all_covid19_datasets.json"\n' | |
config_fp = <_io.StringIO object at 0x7fe029674310> | |
io.StringIO = <class '_io.StringIO'> | |
config_mech.load_config = <method 'YamlConfigurationMechanism.load_config' of <datahub.configuration.yaml.YamlConfigurationMechanism object at 0x7 | |
fe029679280> yaml.py:11> | |
.................................................. | |
File "/usr/local/lib/python3.8/dist-packages/datahub/configuration/yaml.py", line 12, in load_config | |
11 def load_config(self, config_fp: IO) -> dict: | |
--> 12 config = yaml.safe_load(config_fp) | |
13 return config | |
.................................................. | |
self = <datahub.configuration.yaml.YamlConfigurationMechanism object at 0x7fe029679280> | |
config_fp = <_io.StringIO object at 0x7fe029674310> | |
IO = <class 'typing.IO'> | |
yaml.safe_load = <function 'safe_load' __init__.py:154> | |
.................................................. | |
File "/usr/lib/python3/dist-packages/yaml/__init__.py", line 162, in safe_load | |
154 def safe_load(stream): | |
(...) | |
158 | |
159 Resolve only basic YAML tags. This is known | |
160 to be safe for untrusted input. | |
161 """ | |
--> 162 return load(stream, SafeLoader) | |
.................................................. | |
stream = <_io.StringIO object at 0x7fe029674310> | |
SafeLoader = <class 'yaml.loader.SafeLoader'> | |
.................................................. | |
File "/usr/lib/python3/dist-packages/yaml/__init__.py", line 114, in load | |
103 def load(stream, Loader=None): | |
(...) | |
110 Loader = FullLoader | |
111 | |
112 loader = Loader(stream) | |
113 try: | |
--> 114 return loader.get_single_data() | |
115 finally: | |
.................................................. | |
stream = <_io.StringIO object at 0x7fe029674310> | |
Loader = <class 'yaml.loader.SafeLoader'> | |
FullLoader = <class 'yaml.loader.FullLoader'> | |
loader = <yaml.loader.SafeLoader object at 0x7fe029679df0> | |
loader.get_single_data = <method 'BaseConstructor.get_single_data' of <yaml.loader.SafeLoader object at 0x7fe029679df0> constructor.py:47> | |
.................................................. | |
File "/usr/lib/python3/dist-packages/yaml/constructor.py", line 49, in get_single_data | |
47 def get_single_data(self): | |
48 # Ensure that the stream contains a single document and construct it. | |
--> 49 node = self.get_single_node() | |
50 if node is not None: | |
.................................................. | |
self = <yaml.loader.SafeLoader object at 0x7fe029679df0> | |
self.get_single_node = <method 'Composer.get_single_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:29> | |
.................................................. | |
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 36, in get_single_node | |
29 def get_single_node(self): | |
(...) | |
32 | |
33 # Compose a document if the stream is not empty. | |
34 document = None | |
35 if not self.check_event(StreamEndEvent): | |
--> 36 document = self.compose_document() | |
37 | |
.................................................. | |
self = <yaml.loader.SafeLoader object at 0x7fe029679df0> | |
document = None | |
self.check_event = <method 'Parser.check_event' of <yaml.loader.SafeLoader object at 0x7fe029679df0> parser.py:94> | |
StreamEndEvent = <class 'yaml.events.StreamEndEvent'> | |
self.compose_document = <method 'Composer.compose_document' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:50> | |
.................................................. | |
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 55, in compose_document | |
50 def compose_document(self): | |
51 # Drop the DOCUMENT-START event. | |
52 self.get_event() | |
53 | |
54 # Compose the root node. | |
--> 55 node = self.compose_node(None, None) | |
56 | |
.................................................. | |
self = <yaml.loader.SafeLoader object at 0x7fe029679df0> | |
self.get_event = <method 'Parser.get_event' of <yaml.loader.SafeLoader object at 0x7fe029679df0> parser.py:114> | |
self.compose_node = <method 'Composer.compose_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:63> | |
.................................................. | |
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 84, in compose_node | |
63 def compose_node(self, parent, index): | |
(...) | |
80 node = self.compose_scalar_node(anchor) | |
81 elif self.check_event(SequenceStartEvent): | |
82 node = self.compose_sequence_node(anchor) | |
83 elif self.check_event(MappingStartEvent): | |
--> 84 node = self.compose_mapping_node(anchor) | |
85 self.ascend_resolver() | |
.................................................. | |
self = <yaml.loader.SafeLoader object at 0x7fe029679df0> | |
parent = None | |
index = None | |
self.compose_scalar_node = <method 'Composer.compose_scalar_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:88> | |
anchor = None | |
self.check_event = <method 'Parser.check_event' of <yaml.loader.SafeLoader object at 0x7fe029679df0> parser.py:94> | |
SequenceStartEvent = <class 'yaml.events.SequenceStartEvent'> | |
self.compose_sequence_node = <method 'Composer.compose_sequence_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:99> | |
MappingStartEvent = <class 'yaml.events.MappingStartEvent'> | |
self.compose_mapping_node = <method 'Composer.compose_mapping_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:117> | |
self.ascend_resolver = <method 'BaseResolver.ascend_resolver' of <yaml.loader.SafeLoader object at 0x7fe029679df0> resolver.py:114> | |
.................................................. | |
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 133, in compose_mapping_node | |
117 def compose_mapping_node(self, anchor): | |
(...) | |
129 item_key = self.compose_node(node, None) | |
130 #if item_key in node.value: | |
131 # raise ComposerError("while composing a mapping", start_event.start_mark, | |
132 # "found duplicate key", key_event.start_mark) | |
--> 133 item_value = self.compose_node(node, item_key) | |
134 #node.value[item_key] = item_value | |
.................................................. | |
self = <yaml.loader.SafeLoader object at 0x7fe029679df0> | |
anchor = None | |
item_key = ScalarNode(tag='tag:yaml.org,2002:str', value='source') | |
self.compose_node = <method 'Composer.compose_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:63> | |
node = MappingNode(tag='tag:yaml.org,2002:map', value=[]) | |
.................................................. | |
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 84, in compose_node | |
63 def compose_node(self, parent, index): | |
(...) | |
80 node = self.compose_scalar_node(anchor) | |
81 elif self.check_event(SequenceStartEvent): | |
82 node = self.compose_sequence_node(anchor) | |
83 elif self.check_event(MappingStartEvent): | |
--> 84 node = self.compose_mapping_node(anchor) | |
85 self.ascend_resolver() | |
.................................................. | |
self = <yaml.loader.SafeLoader object at 0x7fe029679df0> | |
parent = MappingNode(tag='tag:yaml.org,2002:map', value=[]) | |
index = ScalarNode(tag='tag:yaml.org,2002:str', value='source') | |
self.compose_scalar_node = <method 'Composer.compose_scalar_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:88> | |
anchor = None | |
self.check_event = <method 'Parser.check_event' of <yaml.loader.SafeLoader object at 0x7fe029679df0> parser.py:94> | |
SequenceStartEvent = <class 'yaml.events.SequenceStartEvent'> | |
self.compose_sequence_node = <method 'Composer.compose_sequence_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:99> | |
MappingStartEvent = <class 'yaml.events.MappingStartEvent'> | |
self.compose_mapping_node = <method 'Composer.compose_mapping_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:117> | |
self.ascend_resolver = <method 'BaseResolver.ascend_resolver' of <yaml.loader.SafeLoader object at 0x7fe029679df0> resolver.py:114> | |
.................................................. | |
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 133, in compose_mapping_node | |
117 def compose_mapping_node(self, anchor): | |
(...) | |
129 item_key = self.compose_node(node, None) | |
130 #if item_key in node.value: | |
131 # raise ComposerError("while composing a mapping", start_event.start_mark, | |
132 # "found duplicate key", key_event.start_mark) | |
--> 133 item_value = self.compose_node(node, item_key) | |
134 #node.value[item_key] = item_value | |
.................................................. | |
self = <yaml.loader.SafeLoader object at 0x7fe029679df0> | |
anchor = None | |
item_key = ScalarNode(tag='tag:yaml.org,2002:str', value='config') | |
self.compose_node = <method 'Composer.compose_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:63> | |
node = MappingNode(tag='tag:yaml.org,2002:map', value=[(ScalarNode(tag='tag:yaml.org,2002:str', value='type'), ScalarNode(tag=' | |
tag:yaml.org,2002:str', value='bigquery'))]) | |
item_value = ScalarNode(tag='tag:yaml.org,2002:str', value='bigquery') | |
.................................................. | |
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 84, in compose_node | |
63 def compose_node(self, parent, index): | |
(...) | |
80 node = self.compose_scalar_node(anchor) | |
81 elif self.check_event(SequenceStartEvent): | |
82 node = self.compose_sequence_node(anchor) | |
83 elif self.check_event(MappingStartEvent): | |
--> 84 node = self.compose_mapping_node(anchor) | |
85 self.ascend_resolver() | |
.................................................. | |
self = <yaml.loader.SafeLoader object at 0x7fe029679df0> | |
parent = MappingNode(tag='tag:yaml.org,2002:map', value=[(ScalarNode(tag='tag:yaml.org,2002:str', value='type'), ScalarNode(tag=' | |
tag:yaml.org,2002:str', value='bigquery'))]) | |
index = ScalarNode(tag='tag:yaml.org,2002:str', value='config') | |
self.compose_scalar_node = <method 'Composer.compose_scalar_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:88> | |
anchor = None | |
self.check_event = <method 'Parser.check_event' of <yaml.loader.SafeLoader object at 0x7fe029679df0> parser.py:94> | |
SequenceStartEvent = <class 'yaml.events.SequenceStartEvent'> | |
self.compose_sequence_node = <method 'Composer.compose_sequence_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:99> | |
MappingStartEvent = <class 'yaml.events.MappingStartEvent'> | |
self.compose_mapping_node = <method 'Composer.compose_mapping_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:117> | |
self.ascend_resolver = <method 'BaseResolver.ascend_resolver' of <yaml.loader.SafeLoader object at 0x7fe029679df0> resolver.py:114> | |
.................................................. | |
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 133, in compose_mapping_node | |
117 def compose_mapping_node(self, anchor): | |
(...) | |
129 item_key = self.compose_node(node, None) | |
130 #if item_key in node.value: | |
131 # raise ComposerError("while composing a mapping", start_event.start_mark, | |
132 # "found duplicate key", key_event.start_mark) | |
--> 133 item_value = self.compose_node(node, item_key) | |
134 #node.value[item_key] = item_value | |
.................................................. | |
self = <yaml.loader.SafeLoader object at 0x7fe029679df0> | |
anchor = None | |
item_key = ScalarNode(tag='tag:yaml.org,2002:str', value='table_pattern') | |
self.compose_node = <method 'Composer.compose_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:63> | |
node = MappingNode(tag='tag:yaml.org,2002:map', value=[(ScalarNode(tag='tag:yaml.org,2002:str', value='project_id'), ScalarNode | |
(tag='tag:yaml.org,2002:str', value='bigquery-public-data')), (ScalarNode(tag='tag:yaml.org,2002:str', value='options'), | |
MappingNode(tag='tag:yaml.org,2002:map', value=[(ScalarNode(tag='tag:yaml.org,2002:str', value='credentials_path'), Sca | |
larNode(tag='tag:yaml.org,2002:str', value='./bq_credentials.json'))]))]) | |
item_value = MappingNode(tag='tag:yaml.org,2002:map', value=[(ScalarNode(tag='tag:yaml.org,2002:str', value='credentials_path'), Scal | |
arNode(tag='tag:yaml.org,2002:str', value='./bq_credentials.json'))]) | |
.................................................. | |
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 84, in compose_node | |
63 def compose_node(self, parent, index): | |
(...) | |
80 node = self.compose_scalar_node(anchor) | |
81 elif self.check_event(SequenceStartEvent): | |
82 node = self.compose_sequence_node(anchor) | |
83 elif self.check_event(MappingStartEvent): | |
--> 84 node = self.compose_mapping_node(anchor) | |
85 self.ascend_resolver() | |
.................................................. | |
self = <yaml.loader.SafeLoader object at 0x7fe029679df0> | |
parent = MappingNode(tag='tag:yaml.org,2002:map', value=[(ScalarNode(tag='tag:yaml.org,2002:str', value='project_id'), ScalarNode | |
(tag='tag:yaml.org,2002:str', value='bigquery-public-data')), (ScalarNode(tag='tag:yaml.org,2002:str', value='options'), | |
MappingNode(tag='tag:yaml.org,2002:map', value=[(ScalarNode(tag='tag:yaml.org,2002:str', value='credentials_path'), Sca | |
larNode(tag='tag:yaml.org,2002:str', value='./bq_credentials.json'))]))]) | |
index = ScalarNode(tag='tag:yaml.org,2002:str', value='table_pattern') | |
self.compose_scalar_node = <method 'Composer.compose_scalar_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:88> | |
anchor = None | |
self.check_event = <method 'Parser.check_event' of <yaml.loader.SafeLoader object at 0x7fe029679df0> parser.py:94> | |
SequenceStartEvent = <class 'yaml.events.SequenceStartEvent'> | |
self.compose_sequence_node = <method 'Composer.compose_sequence_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:99> | |
MappingStartEvent = <class 'yaml.events.MappingStartEvent'> | |
self.compose_mapping_node = <method 'Composer.compose_mapping_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:117> | |
self.ascend_resolver = <method 'BaseResolver.ascend_resolver' of <yaml.loader.SafeLoader object at 0x7fe029679df0> resolver.py:114> | |
.................................................. | |
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 133, in compose_mapping_node | |
117 def compose_mapping_node(self, anchor): | |
(...) | |
129 item_key = self.compose_node(node, None) | |
130 #if item_key in node.value: | |
131 # raise ComposerError("while composing a mapping", start_event.start_mark, | |
132 # "found duplicate key", key_event.start_mark) | |
--> 133 item_value = self.compose_node(node, item_key) | |
134 #node.value[item_key] = item_value | |
.................................................. | |
self = <yaml.loader.SafeLoader object at 0x7fe029679df0> | |
anchor = None | |
item_key = ScalarNode(tag='tag:yaml.org,2002:str', value='deny') | |
self.compose_node = <method 'Composer.compose_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:63> | |
node = MappingNode(tag='tag:yaml.org,2002:map', value=[(ScalarNode(tag='tag:yaml.org,2002:str', value='allow'), SequenceNode(ta | |
g='tag:yaml.org,2002:seq', value=[ScalarNode(tag='tag:yaml.org,2002:str', value='bigquery-public-data\\.covid19.*')]))]) | |
item_value = SequenceNode(tag='tag:yaml.org,2002:seq', value=[ScalarNode(tag='tag:yaml.org,2002:str', value='bigquery-public-data\\.c | |
ovid19.*')]) | |
.................................................. | |
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 82, in compose_node | |
63 def compose_node(self, parent, index): | |
(...) | |
78 self.descend_resolver(parent, index) | |
79 if self.check_event(ScalarEvent): | |
80 node = self.compose_scalar_node(anchor) | |
81 elif self.check_event(SequenceStartEvent): | |
--> 82 node = self.compose_sequence_node(anchor) | |
83 elif self.check_event(MappingStartEvent): | |
.................................................. | |
self = <yaml.loader.SafeLoader object at 0x7fe029679df0> | |
parent = MappingNode(tag='tag:yaml.org,2002:map', value=[(ScalarNode(tag='tag:yaml.org,2002:str', value='allow'), SequenceNode(ta | |
g='tag:yaml.org,2002:seq', value=[ScalarNode(tag='tag:yaml.org,2002:str', value='bigquery-public-data\\.covid19.*')]))]) | |
index = ScalarNode(tag='tag:yaml.org,2002:str', value='deny') | |
self.descend_resolver = <method 'BaseResolver.descend_resolver' of <yaml.loader.SafeLoader object at 0x7fe029679df0> resolver.py:91> | |
self.check_event = <method 'Parser.check_event' of <yaml.loader.SafeLoader object at 0x7fe029679df0> parser.py:94> | |
ScalarEvent = <class 'yaml.events.ScalarEvent'> | |
self.compose_scalar_node = <method 'Composer.compose_scalar_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:88> | |
anchor = None | |
SequenceStartEvent = <class 'yaml.events.SequenceStartEvent'> | |
self.compose_sequence_node = <method 'Composer.compose_sequence_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:99> | |
MappingStartEvent = <class 'yaml.events.MappingStartEvent'> | |
.................................................. | |
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 110, in compose_sequence_node | |
99 def compose_sequence_node(self, anchor): | |
(...) | |
106 flow_style=start_event.flow_style) | |
107 if anchor is not None: | |
108 self.anchors[anchor] = node | |
109 index = 0 | |
--> 110 while not self.check_event(SequenceEndEvent): | |
111 node.value.append(self.compose_node(node, index)) | |
.................................................. | |
self = <yaml.loader.SafeLoader object at 0x7fe029679df0> | |
anchor = None | |
start_event.flow_style = None | |
self.anchors = {} | |
node = SequenceNode(tag='tag:yaml.org,2002:seq', value=[]) | |
index = 0 | |
self.check_event = <method 'Parser.check_event' of <yaml.loader.SafeLoader object at 0x7fe029679df0> parser.py:94> | |
SequenceEndEvent = <class 'yaml.events.SequenceEndEvent'> | |
self.compose_node = <method 'Composer.compose_node' of <yaml.loader.SafeLoader object at 0x7fe029679df0> composer.py:63> | |
.................................................. | |
File "/usr/lib/python3/dist-packages/yaml/parser.py", line 98, in check_event | |
94 def check_event(self, *choices): | |
95 # Check the type of the next event. | |
96 if self.current_event is None: | |
97 if self.state: | |
--> 98 self.current_event = self.state() | |
99 if self.current_event is not None: | |
.................................................. | |
self = <yaml.loader.SafeLoader object at 0x7fe029679df0> | |
choices = (<class 'yaml.events.SequenceEndEvent'>, ) | |
self.current_event = None | |
self.state = None | |
.................................................. | |
File "/usr/lib/python3/dist-packages/yaml/parser.py", line 405, in parse_indentless_sequence_entry | |
402 def parse_indentless_sequence_entry(self): | |
403 if self.check_token(BlockEntryToken): | |
404 token = self.get_token() | |
--> 405 if not self.check_token(BlockEntryToken, | |
406 KeyToken, ValueToken, BlockEndToken): | |
.................................................. | |
self = <yaml.loader.SafeLoader object at 0x7fe029679df0> | |
self.check_token = <method 'Scanner.check_token' of <yaml.loader.SafeLoader object at 0x7fe029679df0> scanner.py:113> | |
BlockEntryToken = <class 'yaml.tokens.BlockEntryToken'> | |
token = BlockEntryToken() | |
self.get_token = <method 'Scanner.get_token' of <yaml.loader.SafeLoader object at 0x7fe029679df0> scanner.py:135> | |
KeyToken = <class 'yaml.tokens.KeyToken'> | |
ValueToken = <class 'yaml.tokens.ValueToken'> | |
BlockEndToken = <class 'yaml.tokens.BlockEndToken'> | |
.................................................. | |
File "/usr/lib/python3/dist-packages/yaml/scanner.py", line 116, in check_token | |
113 def check_token(self, *choices): | |
114 # Check if the next token is one of the given types. | |
115 while self.need_more_tokens(): | |
--> 116 self.fetch_more_tokens() | |
117 if self.tokens: | |
.................................................. | |
self = <yaml.loader.SafeLoader object at 0x7fe029679df0> | |
choices = (<class 'yaml.tokens.BlockEntryToken'>, <class 'yaml.tokens.KeyToken'>, <class 'yaml.tokens.ValueToken'>, <class 'yaml.t | |
okens.BlockEndToken'>, ) | |
self.need_more_tokens = <method 'Scanner.need_more_tokens' of <yaml.loader.SafeLoader object at 0x7fe029679df0> scanner.py:145> | |
self.fetch_more_tokens = <method 'Scanner.fetch_more_tokens' of <yaml.loader.SafeLoader object at 0x7fe029679df0> scanner.py:156> | |
self.tokens = [] | |
.................................................. | |
File "/usr/lib/python3/dist-packages/yaml/scanner.py", line 251, in fetch_more_tokens | |
156 def fetch_more_tokens(self): | |
(...) | |
247 return self.fetch_single() | |
248 | |
249 # Is it a double quoted scalar? | |
250 if ch == '\"': | |
--> 251 return self.fetch_double() | |
252 | |
.................................................. | |
self = <yaml.loader.SafeLoader object at 0x7fe029679df0> | |
self.fetch_single = <method 'Scanner.fetch_single' of <yaml.loader.SafeLoader object at 0x7fe029679df0> scanner.py:651> | |
ch = '"' | |
self.fetch_double = <method 'Scanner.fetch_double' of <yaml.loader.SafeLoader object at 0x7fe029679df0> scanner.py:654> | |
.................................................. | |
File "/usr/lib/python3/dist-packages/yaml/scanner.py", line 655, in fetch_double | |
654 def fetch_double(self): | |
--> 655 self.fetch_flow_scalar(style='"') | |
.................................................. | |
self = <yaml.loader.SafeLoader object at 0x7fe029679df0> | |
self.fetch_flow_scalar = <method 'Scanner.fetch_flow_scalar' of <yaml.loader.SafeLoader object at 0x7fe029679df0> scanner.py:657> | |
.................................................. | |
File "/usr/lib/python3/dist-packages/yaml/scanner.py", line 666, in fetch_flow_scalar | |
657 def fetch_flow_scalar(self, style): | |
(...) | |
662 # No simple keys after flow scalars. | |
663 self.allow_simple_key = False | |
664 | |
665 # Scan and add SCALAR. | |
--> 666 self.tokens.append(self.scan_flow_scalar(style)) | |
.................................................. | |
self = <yaml.loader.SafeLoader object at 0x7fe029679df0> | |
style = '"' | |
self.allow_simple_key = False | |
self.scan_flow_scalar = <method 'Scanner.scan_flow_scalar' of <yaml.loader.SafeLoader object at 0x7fe029679df0> scanner.py:1134> | |
.................................................. | |
File "/usr/lib/python3/dist-packages/yaml/scanner.py", line 1149, in scan_flow_scalar | |
1134 def scan_flow_scalar(self, style): | |
(...) | |
1145 chunks = [] | |
1146 start_mark = self.get_mark() | |
1147 quote = self.peek() | |
1148 self.forward() | |
--> 1149 chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark)) | |
1150 while self.peek() != quote: | |
.................................................. | |
self = <yaml.loader.SafeLoader object at 0x7fe029679df0> | |
style = '"' | |
chunks = [] | |
start_mark = <yaml.error.Mark object at 0x7fe0295a27c0> | |
self.get_mark = <method 'Reader.get_mark' of <yaml.loader.SafeLoader object at 0x7fe029679df0> reader.py:114> | |
quote = '"' | |
self.peek = <method 'Reader.peek' of <yaml.loader.SafeLoader object at 0x7fe029679df0> reader.py:87> | |
self.forward = <method 'Reader.forward' of <yaml.loader.SafeLoader object at 0x7fe029679df0> reader.py:99> | |
self.scan_flow_scalar_non_spaces = <method 'Scanner.scan_flow_scalar_non_spaces' of <yaml.loader.SafeLoader object at 0x7fe029679df0> scanner.py:1185> | |
double = True | |
.................................................. | |
File "/usr/lib/python3/dist-packages/yaml/scanner.py", line 1223, in scan_flow_scalar_non_spaces | |
1185 def scan_flow_scalar_non_spaces(self, double, start_mark): | |
(...) | |
1219 elif ch in '\r\n\x85\u2028\u2029': | |
1220 self.scan_line_break() | |
1221 chunks.extend(self.scan_flow_scalar_breaks(double, start_mark)) | |
1222 else: | |
--> 1223 raise ScannerError("while scanning a double-quoted scalar", start_mark, | |
1224 "found unknown escape character %r" % ch, self.get_mark()) | |
.................................................. | |
self = <yaml.loader.SafeLoader object at 0x7fe029679df0> | |
double = True | |
start_mark = <yaml.error.Mark object at 0x7fe0295a27c0> | |
ch = '$' | |
self.scan_line_break = <method 'Scanner.scan_line_break' of <yaml.loader.SafeLoader object at 0x7fe029679df0> scanner.py:1416> | |
self.scan_flow_scalar_breaks = <method 'Scanner.scan_flow_scalar_breaks' of <yaml.loader.SafeLoader object at 0x7fe029679df0> scanner.py:1252> | |
ScannerError = <class 'yaml.scanner.ScannerError'> | |
self.get_mark = <method 'Reader.get_mark' of <yaml.loader.SafeLoader object at 0x7fe029679df0> reader.py:114> | |
.................................................. | |
---- (full traceback above) ---- | |
File "/usr/local/lib/python3.8/dist-packages/datahub/entrypoints.py", line 149, in main | |
sys.exit(datahub(standalone_mode=False, **kwargs)) | |
File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 1130, in __call__ | |
return self.main(*args, **kwargs) | |
File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 1055, in main | |
rv = self.invoke(ctx) | |
File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 1657, in invoke | |
return _process_result(sub_ctx.command.invoke(sub_ctx)) | |
File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 1657, in invoke | |
return _process_result(sub_ctx.command.invoke(sub_ctx)) | |
File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 1404, in invoke | |
return ctx.invoke(self.callback, **ctx.params) | |
File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 760, in invoke | |
return __callback(*args, **kwargs) | |
File "/usr/local/lib/python3.8/dist-packages/click/decorators.py", line 26, in new_func | |
return f(get_current_context(), *args, **kwargs) | |
File "/usr/local/lib/python3.8/dist-packages/datahub/upgrade/upgrade.py", line 333, in wrapper | |
res = func(*args, **kwargs) | |
File "/usr/local/lib/python3.8/dist-packages/datahub/telemetry/telemetry.py", line 338, in wrapper | |
raise e | |
File "/usr/local/lib/python3.8/dist-packages/datahub/telemetry/telemetry.py", line 290, in wrapper | |
res = func(*args, **kwargs) | |
File "/usr/local/lib/python3.8/dist-packages/datahub/utilities/memory_leak_detector.py", line 102, in wrapper | |
res = func(*args, **kwargs) | |
File "/usr/local/lib/python3.8/dist-packages/datahub/cli/ingest_cli.py", line 102, in run | |
pipeline_config = load_config_file(config_file) | |
File "/usr/local/lib/python3.8/dist-packages/datahub/configuration/config_loader.py", line 72, in load_config_file | |
config = config_mech.load_config(config_fp) | |
File "/usr/local/lib/python3.8/dist-packages/datahub/configuration/yaml.py", line 12, in load_config | |
config = yaml.safe_load(config_fp) | |
File "/usr/lib/python3/dist-packages/yaml/__init__.py", line 162, in safe_load | |
return load(stream, SafeLoader) | |
File "/usr/lib/python3/dist-packages/yaml/__init__.py", line 114, in load | |
return loader.get_single_data() | |
File "/usr/lib/python3/dist-packages/yaml/constructor.py", line 49, in get_single_data | |
node = self.get_single_node() | |
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 36, in get_single_node | |
document = self.compose_document() | |
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 55, in compose_document | |
node = self.compose_node(None, None) | |
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 84, in compose_node | |
node = self.compose_mapping_node(anchor) | |
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 133, in compose_mapping_node | |
item_value = self.compose_node(node, item_key) | |
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 84, in compose_node | |
node = self.compose_mapping_node(anchor) | |
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 133, in compose_mapping_node | |
item_value = self.compose_node(node, item_key) | |
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 84, in compose_node | |
node = self.compose_mapping_node(anchor) | |
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 133, in compose_mapping_node | |
item_value = self.compose_node(node, item_key) | |
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 84, in compose_node | |
node = self.compose_mapping_node(anchor) | |
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 133, in compose_mapping_node | |
item_value = self.compose_node(node, item_key) | |
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 82, in compose_node | |
node = self.compose_sequence_node(anchor) | |
File "/usr/lib/python3/dist-packages/yaml/composer.py", line 110, in compose_sequence_node | |
while not self.check_event(SequenceEndEvent): | |
File "/usr/lib/python3/dist-packages/yaml/parser.py", line 98, in check_event | |
self.current_event = self.state() | |
File "/usr/lib/python3/dist-packages/yaml/parser.py", line 405, in parse_indentless_sequence_entry | |
if not self.check_token(BlockEntryToken, | |
File "/usr/lib/python3/dist-packages/yaml/scanner.py", line 116, in check_token | |
self.fetch_more_tokens() | |
File "/usr/lib/python3/dist-packages/yaml/scanner.py", line 251, in fetch_more_tokens | |
return self.fetch_double() | |
File "/usr/lib/python3/dist-packages/yaml/scanner.py", line 655, in fetch_double | |
self.fetch_flow_scalar(style='"') | |
File "/usr/lib/python3/dist-packages/yaml/scanner.py", line 666, in fetch_flow_scalar | |
self.tokens.append(self.scan_flow_scalar(style)) | |
File "/usr/lib/python3/dist-packages/yaml/scanner.py", line 1149, in scan_flow_scalar | |
chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark)) | |
File "/usr/lib/python3/dist-packages/yaml/scanner.py", line 1223, in scan_flow_scalar_non_spaces | |
raise ScannerError("while scanning a double-quoted scalar", start_mark, | |
ScannerError: while scanning a double-quoted scalar | |
in "<file>", line 14, column 9 | |
found unknown escape character '$' | |
in "<file>", line 14, column 14 | |
[2022-06-22 11:05:33,909] INFO {datahub.entrypoints:176} - DataHub CLI version: 0.8.38.3 at /usr/local/lib/python3.8/dist-packages/datahub/__init__.py | |
[2022-06-22 11:05:33,909] INFO {datahub.entrypoints:179} - Python version: 3.8.10 (default, Mar 15 2022, 12:22:08) | |
[GCC 9.4.0] at /usr/bin/python3 on Linux-5.4.0-117-generic-x86_64-with-glibc2.29 | |
[2022-06-22 11:05:33,909] INFO {datahub.entrypoints:182} - GMS config {} | |
root@ubuntu20dockers:~/datahub# |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment