Skip to content

Instantly share code, notes, and snippets.

@belgoros
Last active May 17, 2020 21:08
Show Gist options
  • Save belgoros/4ccdb2731223451c73c211de1746755f to your computer and use it in GitHub Desktop.
Save belgoros/4ccdb2731223451c73c211de1746755f to your computer and use it in GitHub Desktop.
spring-batch to fetch PhraseApp translations
package hello;
import hello.dto.PostDto;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.json.JacksonJsonObjectReader;
import org.springframework.batch.item.json.JsonItemReader;
import org.springframework.batch.item.json.builder.JsonItemReaderBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.UrlResource;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.StringJoiner;
@Configuration
@EnableBatchProcessing
public class BatchConfiguration {
private static final Logger log = LoggerFactory.getLogger(BatchConfiguration.class);
@Autowired
public JobBuilderFactory jobBuilderFactory;
@Autowired
public StepBuilderFactory stepBuilderFactory;
@Bean
public JsonItemReader<PostDto> itemReader() throws Exception {
URL url = new URL(buildUrl());
HttpURLConnection con = (HttpURLConnection) url.openConnection();
initConnection(con);
UrlResource phraseAppResource = new UrlResource(url);
int responseCode = con.getResponseCode();
System.out.println("+++++++++ RESPONSE++++++++++++++ : " + responseCode);
final JsonItemReader<PostDto> jsonReader = new JsonItemReaderBuilder<PostDto>()
.name("jsonReader")
.resource(phraseAppResource)
.jsonObjectReader(new JacksonJsonObjectReader<>(PostDto.class))
.strict(false)
.build();
return jsonReader;
}
private void initConnection(HttpURLConnection con) throws IOException {
String apiToken = "token {phrase app token value}";
con.setRequestMethod("GET");
con.setRequestProperty("Content-Type", "application/json");
con.setRequestProperty("Authorization", apiToken);
con.connect();
}
private String buildUrl() {
String apiUrl = "https://classic-json-api.herokuapp.com";
String postsUrl = "posts";
StringJoiner joiner = new StringJoiner("/");
joiner.add(apiUrl).add(postsUrl);
return joiner.toString();
}
@Bean
public ItemWriter<PostDto> itemWriter() {
return items -> {
for (PostDto item : items) {
System.out.println("item = " + item);
}
};
}
@Bean
public Job job() throws Exception {
return jobBuilderFactory.get("job")
.start(step())
.build();
}
@Bean
public Step step() throws Exception {
return stepBuilderFactory.get("step")
.<PostDto, PostDto>chunk(5)
.reader(itemReader())
.writer(itemWriter())
.build();
}
/*@Bean
public JsonItemReader<TranslationDto> itemReader() throws Exception {
URL url = new URL(buildUrl());
HttpURLConnection con = (HttpURLConnection) url.openConnection();
initConnection(con);
UrlResource phraseAppResource = new UrlResource(url);
int responseCode = con.getResponseCode();
System.out.println("+++++++++ RESPONSE++++++++++++++ : " + responseCode);
final JsonItemReader<TranslationDto> jsonReader = new JsonItemReaderBuilder<TranslationDto>()
.name("jsonReader")
.resource(phraseAppResource)
.jsonObjectReader(new JacksonJsonObjectReader<>(TranslationDto.class))
.strict(false)
.build();
return jsonReader;
}
private void initConnection(HttpURLConnection con) throws IOException {
String apiToken = "token {phrase app token}";
con.setRequestMethod("GET");
con.setRequestProperty("Content-Type", "application/json");
con.setRequestProperty("Authorization", apiToken);
con.connect();
}
private String buildUrl() {
String apiUrl = "https://api.phraseapp.com/api/v2/projects";
String projectId = "{phrase app project id}";
String translationsUrl = "translations";
StringJoiner joiner = new StringJoiner("/");
joiner.add(apiUrl).add(projectId).add(translationsUrl);
return joiner.toString();
}
@Bean
public ItemWriter<TranslationDto> itemWriter() {
return items -> {
for (TranslationDto item : items) {
System.out.println("item = " + item);
}
};
}
@Bean
public Job job() throws Exception {
log.info("+++++++++++++ importTranslationsJob +++++++++++++++");
return jobBuilderFactory.get("job")
.start(step())
.build();
}
@Bean
public Step step() throws Exception {
log.info("+++++++++++++ step ++++++++++++++++++");
return stepBuilderFactory.get("step")
.<TranslationDto, TranslationDto>chunk(25)
.reader(itemReader())
.writer(itemWriter())
.build();
}*/
}
package hello.dto;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
@Getter
@Setter
@ToString
@JsonIgnoreProperties(ignoreUnknown = true)
public class KeyDto {
private String name;
}
package hello.dto;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
@Getter
@Setter
@ToString
@JsonIgnoreProperties(ignoreUnknown = true)
public class LocaleDto {
private String name;
private String code;
}
package hello.dto;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
@Getter
@Setter
@ToString
@JsonIgnoreProperties(ignoreUnknown = true)
public class PostDto {
private String title;
private String body;
}
package hello.dto;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
@Getter
@Setter
@ToString
@JsonIgnoreProperties(ignoreUnknown = true)
public class TranslationDto {
private String content;
private LocaleDto locale;
private KeyDto key;
}
@belgoros
Copy link
Author

belgoros commented Jul 3, 2019

After a long debugging of 2 different APIs, - one at http schema, another one - at https one, I discovered that in case of https end-point API, the following condition was evaluated to false:

package org.springframework.batch.item.json

public class JsonItemReader<T> extends AbstractItemCountingItemStreamItemReader<T> implements
		ResourceAwareItemReaderItemStream<T> {
...
@Override
	protected void doOpen() throws Exception {
		if (!this.resource.exists()) {
			if (this.strict) {
				throw new IllegalStateException("Input resource must exist (reader is in 'strict' mode)");
			}
			LOGGER.warn("Input resource does not exist " + this.resource.getDescription());
			return;
		}
...

i.e. the resource didn't exist! In case of the same processing but for nother API end-point at http it was evaluated totrueand everything worked. Why so ?

@belgoros
Copy link
Author

belgoros commented Jul 4, 2019

It seems like in AbstractFileResolvingResource, in exists method, line 55, a new connection is opened:

URLConnection con = url.openConnection();

which has has no Authorization values set up, that's why I'm getting 401 response code in the below lines in the same method:

if (httpCon != null) {
  int code = httpCon.getResponseCode();
  if (code == HttpURLConnection.HTTP_OK) {
    return true;
  }
  else if (code == HttpURLConnection.HTTP_NOT_FOUND) {
    return false;
  }
}

if (con.getContentLengthLong() > 0) {
  return true;
}
if (httpCon != null) {
  // No HTTP OK status, and no content-length header: give up
  httpCon.disconnect();
  return false;
}

So even if I set the necessary header value in my BatchConfiguration class, it seems like all of theam are just ignored 😢 Here is wjat the documentation to URL#openConnection says:

A new instance of URLConnection is created every time when invoking the URLStreamHandler.openConnection(URL) method of the protocol handler for this URL.

It should be noted that a URLConnection instance does not establish the actual network connection on creation. This will happen only when calling URLConnection.connect().

@fmbenhassine
Copy link

fmbenhassine commented Jul 4, 2019

I was looking at the same place and came to the same conculsion. Here is how I located the issue:

public static void main(String[] args) throws Exception {

	System.out.println("http:");
	UrlResource urlResource = new UrlResource(new URL("http://api.phraseapp.com/api/v2/formats"));

	boolean exists = urlResource.exists();
	System.out.println("exists = " + exists);
	boolean isReadable = urlResource.isReadable();
	System.out.println("isReadable = " + isReadable);

	// with https
	System.out.println("https:");
	urlResource = new UrlResource(new URL("https://api.phraseapp.com/api/v2/formats"));

	exists = urlResource.exists();
	System.out.println("exists = " + exists);
	isReadable = urlResource.isReadable();
	System.out.println("isReadable = " + isReadable);
}

prints:

http:
exists = true
isReadable = false
https:
exists = false
isReadable = false

So it won't work in both cases as the json reader checks if the resource exists and isReadable when in strict mode (the json reader should be used in strict mode in this case).

On the other hand, UrlResource does open the URL behind the scene, so I don't know how to pass authentication headers to it.

@belgoros
Copy link
Author

belgoros commented Jul 4, 2019

@benas, Can we use another third library to just hit the resource and pass it to JsonItemReader? Javalite Http works pretty well. Or use RestTemplate ? Any other ideas?

@fmbenhassine
Copy link

Yes, I'm preparing a working sample with your url and will share it with you asap.

@belgoros
Copy link
Author

belgoros commented Jul 4, 2019

Cool, thank you 👍

@fmbenhassine
Copy link

fmbenhassine commented Jul 4, 2019

Here is an example with plain Java net APIs:

import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;

import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.json.JacksonJsonObjectReader;
import org.springframework.batch.item.json.JsonItemReader;
import org.springframework.batch.item.json.builder.JsonItemReaderBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.InputStreamResource;

@Configuration
@EnableBatchProcessing
public class MyJob {

	public static void main(String[] args) throws Exception {
		ApplicationContext context = new AnnotationConfigApplicationContext(MyJob.class);
		JobLauncher jobLauncher = context.getBean(JobLauncher.class);
		Job job = context.getBean(Job.class);
		jobLauncher.run(job, new JobParameters());
	}
	
	@Autowired
	private JobBuilderFactory jobs;

	@Autowired
	private StepBuilderFactory steps;

	@Bean(destroyMethod = "close")
	public InputStream urlResource() throws IOException {
		URL url = new URL("https://classic-json-api.herokuapp.com");
		URLConnection urlConnection = url.openConnection();
		// urlConnection.setRequestProperty("", ""); // set auth headers if necessary
		return urlConnection.getInputStream();
	}

	@Bean
	public JsonItemReader<Pojo> itemReader(InputStream urlResource) {
		return new JsonItemReaderBuilder<Pojo>()
				.name("restReader")
				.resource(new InputStreamResource(urlResource))
				.strict(true)
				.jsonObjectReader(new JacksonJsonObjectReader<>(Pojo.class))
				.build();
	}

	@Bean
	public ItemWriter<Pojo> itemWriter() {
		return items -> {
			for (Pojo item : items) {
				System.out.println("item = " + item.getTitle());
			}
		};
	}

	@Bean
	public Step step() {
		return steps.get("step")
				.<Pojo, Pojo>chunk(5)
				.reader(itemReader(null))
				.writer(itemWriter())
				.build();
	}

	@Bean
	public Job job() {
		return jobs.get("job")
				.start(step())
				.build();
	}

	@JsonIgnoreProperties(ignoreUnknown = true)
	public static class Pojo {

		private String title;

		public Pojo() {
		}

		public String getTitle() {
			return title;
		}

		public void setTitle(String title) {
			this.title = title;
		}
	}

}

prints:

[warn 2019/07/04 11:50:39.925 CEST <main> tid=0x1] No datasource was provided...using a Map based JobRepository

[warn 2019/07/04 11:50:39.930 CEST <main> tid=0x1] No transaction manager was provided, using a ResourcelessTransactionManager

[info 2019/07/04 11:50:40.077 CEST <main> tid=0x1] No TaskExecutor has been set, defaulting to synchronous executor.

[info 2019/07/04 11:50:40.114 CEST <main> tid=0x1] Job: [SimpleJob: [name=job]] launched with the following parameters: [{}]

[info 2019/07/04 11:50:40.159 CEST <main> tid=0x1] Executing step: [step]

item = title-0
item = title-1
item = title-2
item = title-3
item = title-4
item = title-5
item = title-6
item = title-7
item = title-8
item = title-9
[info 2019/07/04 11:50:40.243 CEST <main> tid=0x1] Step: [step] executed in 82ms

[info 2019/07/04 11:50:40.250 CEST <main> tid=0x1] Job: [SimpleJob: [name=job]] completed with the following parameters: [{}] and the following status: [COMPLETED] in 103ms

Let me know if it helps.

@belgoros
Copy link
Author

belgoros commented Jul 4, 2019

@benas, yep, it works now, 🎉 , thank you. Using

.reader(itemReader(null))

looks a little bit weird though.

@fmbenhassine
Copy link

fmbenhassine commented Jul 4, 2019

Great! Glad to help.

For the .reader(itemReader(null)), you can do something like:

	@Bean
	public JsonItemReader<Pojo> itemReader() throws IOException {
		return new JsonItemReaderBuilder<Pojo>()
				.name("restReader")
				.resource(new InputStreamResource(urlResource()))
				.strict(true)
				.jsonObjectReader(new JacksonJsonObjectReader<>(Pojo.class))
				.build();
	}

then remove the null from the reader definition in the step. However, you might need to propagate the exception declaration in all methods with this approach.

@fmbenhassine
Copy link

May I ask for my bounty 🙃

@belgoros
Copy link
Author

belgoros commented Jul 4, 2019

Cool, it was just a prototype to see how things work together, I'll refactor all this later, thank you.
Sure, I'll update the SO question and apply you the bounty 😄

@belgoros
Copy link
Author

belgoros commented Jul 4, 2019

@benas I can attribute the bounty only in 8 hours. No worries, I shall not forget ...

@fmbenhassine
Copy link

No worries. The most important thing is to be able to help you!

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment