HttpComponentsFeedFetcher

gistfile1.java

import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.zip.GZIPInputStream;

import org.apache.http.Header;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;

import com.google.common.base.Preconditions;
import com.sun.syndication.feed.synd.SyndFeed;
import com.sun.syndication.fetcher.FetcherEvent;
import com.sun.syndication.fetcher.FetcherException;
import com.sun.syndication.fetcher.impl.AbstractFeedFetcher;
import com.sun.syndication.fetcher.impl.FeedFetcherCache;
import com.sun.syndication.fetcher.impl.SyndFeedInfo;
import com.sun.syndication.io.FeedException;
import com.sun.syndication.io.SyndFeedInput;
import com.sun.syndication.io.XmlReader;

public class HttpComponentsFeedFetcher extends AbstractFeedFetcher {

 private CloseableHttpClient client;

 private FeedFetcherCache feedInfoCache;

 private volatile HttpClientMethodCallbackIntf httpClientMethodCallback;

 public HttpComponentsFeedFetcher(CloseableHttpClient client) {
  this.client = client;
 }

 public interface HttpClientMethodCallbackIntf {
  public void afterHttpClientMethodCreate(HttpGet request);
 }

 @Override
 public SyndFeed retrieveFeed(URL feedUrl) throws IllegalArgumentException,
   IOException, FeedException, FetcherException {
  Preconditions.checkNotNull(feedUrl, "null is not a valid URL");

  String urlStr = feedUrl.toString();

  HttpGet request = new HttpGet(urlStr);
  request.addHeader("Accept-Encoding", "gzip");
  request.addHeader("User-Agent", getUserAgent());

  if (httpClientMethodCallback != null) {
   httpClientMethodCallback.afterHttpClientMethodCreate(request);
  }

  FeedFetcherCache cache = feedInfoCache;
  if (cache != null) {
   // retrieve feed
   if (isUsingDeltaEncoding()) {
    request.addHeader("A-IM", "feed");
   }

   // get the feed info from the cache
   // Note that syndFeedInfo will be null if it is not in the
   // cache
   SyndFeedInfo syndFeedInfo = cache.getFeedInfo(feedUrl);
   if (syndFeedInfo != null) {
    request.addHeader("If-None-Match", syndFeedInfo.getETag());

    if (syndFeedInfo.getLastModified() instanceof String) {
     request.addHeader("If-Modified-Since",
       (String) syndFeedInfo.getLastModified());
    }
   }

   try (CloseableHttpResponse response = client.execute(request)) {
    int statusCode = response.getStatusLine().getStatusCode();
    fireEvent(FetcherEvent.EVENT_TYPE_FEED_POLLED, urlStr);
    handleErrorCodes(statusCode);

    SyndFeed feed = getFeed(syndFeedInfo, urlStr, response,
      statusCode);

    syndFeedInfo = buildSyndFeedInfo(feedUrl, urlStr, response,
      feed, statusCode);

    cache.setFeedInfo(new URL(urlStr), syndFeedInfo);

    // the feed may have been modified to pick up cached values
    // (eg - for delta encoding)
    feed = syndFeedInfo.getSyndFeed();

    return feed;
   } finally {
    request.releaseConnection();
   }

  } else {
   // cache is not in use
   try (CloseableHttpResponse response = client.execute(request)) {
    int statusCode = response.getStatusLine().getStatusCode();
    fireEvent(FetcherEvent.EVENT_TYPE_FEED_POLLED, urlStr);
    handleErrorCodes(statusCode);
    return getFeed(null, urlStr, response, statusCode);
   } finally {
    request.releaseConnection();
   }
  }
 }

 private SyndFeed getFeed(SyndFeedInfo syndFeedInfo, String urlStr,
   CloseableHttpResponse response, int statusCode) throws IOException,
   FetcherException, FeedException {

  if (statusCode == HttpURLConnection.HTTP_NOT_MODIFIED
    && syndFeedInfo != null) {
   fireEvent(FetcherEvent.EVENT_TYPE_FEED_UNCHANGED, urlStr);
   return syndFeedInfo.getSyndFeed();
  }

  SyndFeed feed = retrieveFeed(urlStr, response);
  fireEvent(FetcherEvent.EVENT_TYPE_FEED_RETRIEVED, urlStr, feed);
  return feed;
 }

 private SyndFeed retrieveFeed(String urlStr, CloseableHttpResponse response)
   throws IOException, FetcherException, FeedException {

  InputStream stream = null;
  if ((response.getFirstHeader("Content-Encoding") != null)
    && ("gzip".equalsIgnoreCase(response.getFirstHeader(
      "Content-Encoding").getValue()))) {
   stream = new GZIPInputStream(response.getEntity().getContent());
  } else {
   stream = response.getEntity().getContent();
  }
  try {
   XmlReader reader = null;
   if (response.getFirstHeader("Content-Type") != null) {
    reader = new XmlReader(stream, response.getFirstHeader(
      "Content-Type").getValue(), true);
   } else {
    reader = new XmlReader(stream, true);
   }
   SyndFeedInput syndFeedInput = new SyndFeedInput();
   syndFeedInput.setPreserveWireFeed(isPreserveWireFeed());

   return syndFeedInput.build(reader);
  } finally {
   if (stream != null) {
    stream.close();
   }
  }
 }

 private SyndFeedInfo buildSyndFeedInfo(URL feedUrl, String urlStr,
   CloseableHttpResponse response, SyndFeed feed, int statusCode)
   throws MalformedURLException {
  SyndFeedInfo syndFeedInfo;
  syndFeedInfo = new SyndFeedInfo();

  // this may be different to feedURL because of 3XX redirects
  syndFeedInfo.setUrl(new URL(urlStr));
  syndFeedInfo.setId(feedUrl.toString());

  Header imHeader = response.getFirstHeader("IM");
  if (imHeader != null && imHeader.getValue().indexOf("feed") >= 0
    && isUsingDeltaEncoding()) {
   FeedFetcherCache cache = feedInfoCache;
   if (cache != null && statusCode == 226) {
    // client is setup to use http delta encoding and the server
    // supports it and has returned a delta encoded response
    // This response only includes new items
    SyndFeedInfo cachedInfo = cache.getFeedInfo(feedUrl);
    if (cachedInfo != null) {
     SyndFeed cachedFeed = cachedInfo.getSyndFeed();

     // set the new feed to be the orginal feed plus the new
     // items
     feed = combineFeeds(cachedFeed, feed);
    }
   }
  }

  Header lastModifiedHeader = response.getFirstHeader("Last-Modified");
  if (lastModifiedHeader != null) {
   syndFeedInfo.setLastModified(lastModifiedHeader.getValue());
  }

  Header eTagHeader = response.getFirstHeader("ETag");
  if (eTagHeader != null) {
   syndFeedInfo.setETag(eTagHeader.getValue());
  }

  syndFeedInfo.setSyndFeed(feed);

  return syndFeedInfo;
 }

 public synchronized void setFeedInfoCache(FeedFetcherCache feedInfoCache) {
  this.feedInfoCache = feedInfoCache;
 }

 public synchronized void setHttpClientMethodCallback(
   HttpClientMethodCallbackIntf httpClientMethodCallback) {
  this.httpClientMethodCallback = httpClientMethodCallback;
 }
}