rpmoore

plugins {
    kotlin("jvm") version "1.3.10" apply false
}

allprojects {
    group = "rpmoore.example"
    version = "0.0.1-SNAPSHOT"
}

subprojects {

rpmoore

final Ds3Client client = Ds3Client.fromEnv();
final String bucketName = "bucket";

client.deleteBucketSpectraS3(new DeleteBucketSpectraS3Request(bucketName).withForce(true));

rpmoore

import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;

public class MappingObjectChannelBuilder implements Ds3ClientHelpers.ObjectChannelBuilder {
  
    private final ObjectChannelBuilder wrappedChannelBuilder;
    private Map<String, String> fileNameMapping;
  
    public class MappingObjectChannelBuilder(final ObjectChannelBuilder wrappedChannelBuilder, final Map<String, String> fileNameMapping) {
        this.wrappedChannelBuilder = wrappedChannelBuilder;
        this.fileNameMapping = fileNameMapping;

rpmoore

sudo docker run -it spectralogic/ds3_c_docker
cd opt
git clone https://github.com/SpectraLogic/ds3_c_sdk.git
cd ds3_c_sdk
autoreconf --install
./configure
make install
ldconfig
cd ..
git clone https://github.com/SpectraLogic/ds3_python_sdk.git

rpmoore

final Configuration conf = new Configuration(); // hadoop configuration object
final JobClient jobClient = new JobClient(conf); // hadoop JobClient
final Ds3HadoopHelper wrapper = Ds3HadoopHelper.wrap(ds3Client, hdfsFilesystem, conf);

final Job job = wrapper.startWriteJob(...);

final HadoopJobIterator iterator = job.iterator();

while(iterator.hasNext()) {
    final JobConf jobConf = iterator.nextJobConf();

rpmoore

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobConf;

import com.spectralogic.ds3.hadoop.AbstractJobConfFactory;

public class CustomJobConfFactory extends AbstractJobConfFactory {
  
    @Override
    public JobConf createNewJobConf(final Configuration baseConfig) {
        final JobConf jobConf = new JobConf(baseConfig);

rpmoore

package com.spectralogic.hadoop.sample;

import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import java.io.IOException;
import java.nio.ByteBuffer;