mauliksoneji

class GCSClient(object):
    def __init__(self, spark, projectId):
        self.spark = spark
        self.spark._jsc.hadoopConfiguration().set("fs.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem")
        self.spark._jsc.hadoopConfiguration().set("fs.AbstractFileSystem.gs.impl",
                                             "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS")
        self.spark._jsc.hadoopConfiguration().set("fs.gs.project.id", projectId)
        #spark._jsc.hadoopConfiguration().set("fs.gs.auth.service.account.email", "/hadoop/bq/key.json")
        self.spark._jsc.hadoopConfiguration().set("fs.gs.auth.service.account.enable", "true")

mauliksoneji

class BigQueryClient(object):

    def __init__(self, project_id):
        self.project_id = project_id

    def _get_conf(self, bucket, dataset_id, table_id):
        return {
            "fs.gs.project.id": self.project_id,
            "mapred.bq.project.id": self.project_id,  # default project
            "mapred.bq.gcs.bucket": bucket,  # gcs bucket holding the temperory path

mauliksoneji

{"lastUpload":"2019-05-06T04:58:27.314Z","extensionVersion":"v3.2.9"}

mauliksoneji

0x44Ab6709Fc0723C9ac86b192DC057B59B56DbAC9