Simple example of writing into hdfs via Boot CLI and Spring Hadoop Store

App.groovy

@GrabResolver(name='spring-release', root='http://repo.spring.io/libs-release')
@Grab('org.springframework.data:spring-data-hadoop-boot:2.3.0.RELEASE')
@Grab('org.springframework.data:spring-data-hadoop-store:2.3.0.RELEASE')
@Grab('org.springframework:spring-tx:4.2.4.RELEASE')

import org.springframework.data.hadoop.fs.FsShell
import org.apache.hadoop.conf.Configuration
import org.springframework.data.hadoop.store.config.annotation.EnableDataStoreTextWriter
import org.springframework.data.hadoop.store.config.annotation.SpringDataStoreTextWriterConfigurerAdapter
import org.springframework.data.hadoop.store.DataStoreWriter
import org.springframework.data.hadoop.store.config.annotation.builders.DataStoreTextWriterConfigurer

public class App implements CommandLineRunner {

  @Autowired FsShell fsShell
  @Autowired DataStoreWriter<String> writer

  void run(String... args) {
    fsShell.rm("/tmp/foo/data")
    writer.write("foo1")
    writer.write("foo2")
    writer.close()
    fsShell.lsr("/tmp/foo").each() {
      println "> ${it.path}"
    }
    fsShell.text("/tmp/foo/data").each() {
      println it
    }
  }

  @org.springframework.context.annotation.Configuration
  @EnableDataStoreTextWriter
  static class Config extends SpringDataStoreTextWriterConfigurerAdapter {

    @Override
    public void configure(DataStoreTextWriterConfigurer writer) throws Exception {
      writer
        .basePath("/tmp/foo/data")
    }

  }

}

application.yml

spring:
  main:
    show_banner: false
  hadoop:
    fsUri: hdfs://localhost:8020

logback.xml

<?xml version="1.0" encoding="UTF-8"?>
<configuration>
  <include resource="org/springframework/boot/logging/logback/base.xml"/>
  <root level="WARN"/>
  <logger name="org.apache.hadoop" level="ERROR"/>
</configuration>

Install Boot CLI to ease demos http://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/#getting-started-installing-the-cli.

Add all above files to same directory and execute:

$ spring run App.groovy 
> hdfs://localhost:8020/tmp/foo
> hdfs://localhost:8020/tmp/foo/data
foo1
foo2