WriteToGcsHadoop.java

package org.bk.gcs;

import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.Test;

import java.net.URI;
import java.util.List;

class WriteToGcsTest {

    @Test
    void testWriteToGcsUsingHadoopConnector() throws Exception {
        var fileName1 = "someroot/somepath1.txt";
        var fileName2 = "someroot/somepath2.txt";
        var fileName3 = "someroot/somepath3.txt";
        List<String> lines = List.of("one", "two", "three");
        Path path = new Path("gs://root-bucket/");
        Configuration configuration = getConfiguration();
        GoogleHadoopFileSystem fileSystem = createGhfs(path.toUri(), configuration);

        writeToFile(fileName1, lines, fileSystem);
        writeToFile(fileName2, lines, fileSystem);
        writeToFile(fileName3, lines, fileSystem);
    }

    private void writeToFile(String fileName, List<String> lines, GoogleHadoopFileSystem fileSystem) throws Exception {
        Path pathRem = new Path(fileName);
        try (FSDataOutputStream fsDataOutputStream = fileSystem.create(pathRem)) {
            for (String line : lines) {
                fsDataOutputStream.writeBytes(line + "\n");
            }
        }
    }

    private GoogleHadoopFileSystem createGhfs(URI path, Configuration config) throws Exception {
        GoogleHadoopFileSystem ghfs = new GoogleHadoopFileSystem();
        ghfs.initialize(path, config);
        return ghfs;
    }

    private Configuration getConfiguration() {
        Configuration configuration = new Configuration();
        configuration.set("fs.gs.auth.type", "SERVICE_ACCOUNT_JSON_KEYFILE");
        configuration.set("fs.gs.auth.service.account.json.keyfile", "/somecreds.json");
        configuration.set("fs.gs.create.items.conflict.check.enable", "false");
        return configuration;
    }
}