spring-batch to fetch PhraseApp translations

BatchConfiguration.java

package hello;

import hello.dto.PostDto;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.json.JacksonJsonObjectReader;
import org.springframework.batch.item.json.JsonItemReader;
import org.springframework.batch.item.json.builder.JsonItemReaderBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.UrlResource;

import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.StringJoiner;

@Configuration
@EnableBatchProcessing
public class BatchConfiguration {
    private static final Logger log = LoggerFactory.getLogger(BatchConfiguration.class);
    @Autowired
    public JobBuilderFactory jobBuilderFactory;

    @Autowired
    public StepBuilderFactory stepBuilderFactory;

    @Bean
    public JsonItemReader<PostDto> itemReader() throws Exception {
        URL url = new URL(buildUrl());
        HttpURLConnection con = (HttpURLConnection) url.openConnection();
        initConnection(con);

        UrlResource phraseAppResource = new UrlResource(url);
        int responseCode = con.getResponseCode();
        System.out.println("+++++++++ RESPONSE++++++++++++++ : " + responseCode);

        final JsonItemReader<PostDto> jsonReader = new JsonItemReaderBuilder<PostDto>()
                .name("jsonReader")
                .resource(phraseAppResource)
                .jsonObjectReader(new JacksonJsonObjectReader<>(PostDto.class))
                .strict(false)
                .build();

        return jsonReader;
    }

    private void initConnection(HttpURLConnection con) throws IOException {
        String apiToken = "token {phrase app token value}";
        con.setRequestMethod("GET");
        con.setRequestProperty("Content-Type", "application/json");
        con.setRequestProperty("Authorization", apiToken);
        con.connect();
    }

    private String buildUrl() {
        String apiUrl = "https://classic-json-api.herokuapp.com";
        String postsUrl = "posts";
        StringJoiner joiner = new StringJoiner("/");
        joiner.add(apiUrl).add(postsUrl);

        return joiner.toString();
    }

    @Bean
    public ItemWriter<PostDto> itemWriter() {
        return items -> {
            for (PostDto item : items) {
                System.out.println("item = " + item);
            }
        };
    }

    @Bean
    public Job job() throws Exception {
        return jobBuilderFactory.get("job")
                .start(step())
                .build();
    }

    @Bean
    public Step step() throws Exception {
        return stepBuilderFactory.get("step")
                .<PostDto, PostDto>chunk(5)
                .reader(itemReader())
                .writer(itemWriter())
                .build();
    }

    /*@Bean
    public JsonItemReader<TranslationDto> itemReader() throws Exception {
        URL url = new URL(buildUrl());
        HttpURLConnection con = (HttpURLConnection) url.openConnection();
        initConnection(con);

        UrlResource phraseAppResource = new UrlResource(url);
        int responseCode = con.getResponseCode();
        System.out.println("+++++++++ RESPONSE++++++++++++++ : " + responseCode);


        final JsonItemReader<TranslationDto> jsonReader = new JsonItemReaderBuilder<TranslationDto>()
                .name("jsonReader")
                .resource(phraseAppResource)
                .jsonObjectReader(new JacksonJsonObjectReader<>(TranslationDto.class))
                .strict(false)
                .build();

        return jsonReader;
    }

    private void initConnection(HttpURLConnection con) throws IOException {
        String apiToken = "token {phrase app token}";
        con.setRequestMethod("GET");
        con.setRequestProperty("Content-Type", "application/json");
        con.setRequestProperty("Authorization", apiToken);
        con.connect();
    }

    private String buildUrl() {
        String apiUrl = "https://api.phraseapp.com/api/v2/projects";
        String projectId = "{phrase app project id}";
        String translationsUrl = "translations";
        StringJoiner joiner = new StringJoiner("/");
        joiner.add(apiUrl).add(projectId).add(translationsUrl);

        return joiner.toString();
    }

    @Bean
    public ItemWriter<TranslationDto> itemWriter() {
        return items -> {
            for (TranslationDto item : items) {
                System.out.println("item = " + item);
            }
        };
    }

    @Bean
    public Job job() throws Exception {
        log.info("+++++++++++++ importTranslationsJob +++++++++++++++");
        return jobBuilderFactory.get("job")
                .start(step())
                .build();
    }

    @Bean
    public Step step() throws Exception {
        log.info("+++++++++++++ step ++++++++++++++++++");
        return stepBuilderFactory.get("step")
                .<TranslationDto, TranslationDto>chunk(25)
                .reader(itemReader())
                .writer(itemWriter())
                .build();
    }*/
}

KeyDto.java

package hello.dto;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;

@Getter
@Setter
@ToString
@JsonIgnoreProperties(ignoreUnknown = true)
public class KeyDto {
    private String name;
}

LocaleDto.java

package hello.dto;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;

@Getter
@Setter
@ToString
@JsonIgnoreProperties(ignoreUnknown = true)
public class LocaleDto {

    private String name;

    private String code;
}

PostDTo.java

package hello.dto;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;

@Getter
@Setter
@ToString
@JsonIgnoreProperties(ignoreUnknown = true)
public class PostDto {
    private String title;
    private String body;
}

TranslationDto.java

package hello.dto;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;

@Getter
@Setter
@ToString
@JsonIgnoreProperties(ignoreUnknown = true)
public class TranslationDto {

    private String content;
    private LocaleDto locale;
    private KeyDto key;
}

@benas, Can we use another third library to just hit the resource and pass it to JsonItemReader? Javalite Http works pretty well. Or use RestTemplate ? Any other ideas?

Yes, I'm preparing a working sample with your url and will share it with you asap.

Cool, thank you 👍

Here is an example with plain Java net APIs:

import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;

import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.json.JacksonJsonObjectReader;
import org.springframework.batch.item.json.JsonItemReader;
import org.springframework.batch.item.json.builder.JsonItemReaderBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.InputStreamResource;

@Configuration
@EnableBatchProcessing
public class MyJob {

	public static void main(String[] args) throws Exception {
		ApplicationContext context = new AnnotationConfigApplicationContext(MyJob.class);
		JobLauncher jobLauncher = context.getBean(JobLauncher.class);
		Job job = context.getBean(Job.class);
		jobLauncher.run(job, new JobParameters());
	}
	
	@Autowired
	private JobBuilderFactory jobs;

	@Autowired
	private StepBuilderFactory steps;

	@Bean(destroyMethod = "close")
	public InputStream urlResource() throws IOException {
		URL url = new URL("https://classic-json-api.herokuapp.com");
		URLConnection urlConnection = url.openConnection();
		// urlConnection.setRequestProperty("", ""); // set auth headers if necessary
		return urlConnection.getInputStream();
	}

	@Bean
	public JsonItemReader<Pojo> itemReader(InputStream urlResource) {
		return new JsonItemReaderBuilder<Pojo>()
				.name("restReader")
				.resource(new InputStreamResource(urlResource))
				.strict(true)
				.jsonObjectReader(new JacksonJsonObjectReader<>(Pojo.class))
				.build();
	}

	@Bean
	public ItemWriter<Pojo> itemWriter() {
		return items -> {
			for (Pojo item : items) {
				System.out.println("item = " + item.getTitle());
			}
		};
	}

	@Bean
	public Step step() {
		return steps.get("step")
				.<Pojo, Pojo>chunk(5)
				.reader(itemReader(null))
				.writer(itemWriter())
				.build();
	}

	@Bean
	public Job job() {
		return jobs.get("job")
				.start(step())
				.build();
	}

	@JsonIgnoreProperties(ignoreUnknown = true)
	public static class Pojo {

		private String title;

		public Pojo() {
		}

		public String getTitle() {
			return title;
		}

		public void setTitle(String title) {
			this.title = title;
		}
	}

}

prints:

[warn 2019/07/04 11:50:39.925 CEST <main> tid=0x1] No datasource was provided...using a Map based JobRepository

[warn 2019/07/04 11:50:39.930 CEST <main> tid=0x1] No transaction manager was provided, using a ResourcelessTransactionManager

[info 2019/07/04 11:50:40.077 CEST <main> tid=0x1] No TaskExecutor has been set, defaulting to synchronous executor.

[info 2019/07/04 11:50:40.114 CEST <main> tid=0x1] Job: [SimpleJob: [name=job]] launched with the following parameters: [{}]

[info 2019/07/04 11:50:40.159 CEST <main> tid=0x1] Executing step: [step]

item = title-0
item = title-1
item = title-2
item = title-3
item = title-4
item = title-5
item = title-6
item = title-7
item = title-8
item = title-9
[info 2019/07/04 11:50:40.243 CEST <main> tid=0x1] Step: [step] executed in 82ms

[info 2019/07/04 11:50:40.250 CEST <main> tid=0x1] Job: [SimpleJob: [name=job]] completed with the following parameters: [{}] and the following status: [COMPLETED] in 103ms

Let me know if it helps.

@benas, yep, it works now, 🎉 , thank you. Using

.reader(itemReader(null))

looks a little bit weird though.

Great! Glad to help.

For the .reader(itemReader(null)), you can do something like:

	@Bean
	public JsonItemReader<Pojo> itemReader() throws IOException {
		return new JsonItemReaderBuilder<Pojo>()
				.name("restReader")
				.resource(new InputStreamResource(urlResource()))
				.strict(true)
				.jsonObjectReader(new JacksonJsonObjectReader<>(Pojo.class))
				.build();
	}

then remove the null from the reader definition in the step. However, you might need to propagate the exception declaration in all methods with this approach.

May I ask for my bounty 🙃

Cool, it was just a prototype to see how things work together, I'll refactor all this later, thank you.
Sure, I'll update the SO question and apply you the bounty 😄

@benas I can attribute the bounty only in 8 hours. No worries, I shall not forget ...

No worries. The most important thing is to be able to help you!