Make an HTTPS connection in Java using a certificate file

URLTools.java

import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.security.cert.Certificate;
import java.security.cert.CertificateException;
import java.security.cert.CertificateFactory;
import java.security.cert.X509Certificate;
import java.security.KeyManagementException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.zip.Checksum;
import java.util.zip.Adler32;

import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLPeerUnverifiedException;
import javax.net.ssl.SSLSession;
import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
import javax.net.ssl.X509TrustManager;

/**
 * Use this code to make a connection to an external HTTP server using the HTTP or HTTPS protocol,
 * for both GET and POST requests. This code works without having to rely on the global system properties
 * javax.net.ssl.keyStore and javax.net.ssl.trustStore in order to make your HTTPS connection work.
 *
 * This is useful if some other object(s) running in your JVM requires javax.net.ssl.keyStore or
 * javax.net.ssl.trustStore set to something other than what you want to use. Eg. the proxy rather
 * then the HTTP Server. Since those system properties can only be set once during the lifetime of
 * the JVM, it is not possible to change them at runtime and expect the change to take effect. And
 * even if you could change them, it would break the other objects that do need those properties to
 * be set to particular values.
 *
      // Perform a GET on a URL
      remoteConn = getConnection (remoteUrl);
      String rawData = fetchUrlString (remoteConn);

      // Perform a GET on an SSL URL with SSL cert validation
      SSLSocketFactory sslSocketFactory = getSSLSocketFactory (certFile);
      remoteConn = getConnection (remoteUrl, sslSocketFactory);
      String rawData = fetchUrlString (remoteConn);

      // Perform a POST on a URL
      remoteConn = getConnection (remoteUrl);
      String rawData = new String (fetchUrl (remoteUrl, postData));

      // Perform a POST on a URL with SSL cert validation
      SSLSocketFactory sslSocketFactory = getSSLSocketFactory (certFile);
      remoteConn = getConnection (remoteUrl, sslSocketFactory);
      String rawData = new String (fetchUrl (remoteUrl, postData));

 *
 */
public class URLTools
{
  /**
   * Test Only
   */
  public static void main (String [] args) throws CertificateException, KeyStoreException, NoSuchAlgorithmException, KeyManagementException, IOException
  {
    String remoteUrl = "";
    URLConnection remoteConn = null;
    if (args.length == 0)
    {
      System.out.println ("USAGE: java URLTools <url> [<cert file>]");
      System.exit (-1);
    }
    else if (args.length == 1)
    {
      remoteUrl = args [0];
      remoteConn = getConnection (remoteUrl);
    }
    else if (args.length == 2)
    {
      remoteUrl = args [0];
      String certFile = args [1];
      // Construct an SSLSocketFactory with certFile
      SSLSocketFactory sslSocketFactory = getSSLSocketFactory (certFile);
      // Construct a URLConnection with empty headers
      remoteConn = getConnection (remoteUrl, new HashMap <String, String> (), sslSocketFactory);
    }

    // Fetch the data
    String rawData = fetchUrlString (remoteConn);
    System.out.println (rawData);
  }

  /**
   * Read an X.509 certificate from a single long string and return it as a properly formatted certificate.
   *
   * @param inputCert the long string containing the certificate to use
   */
  public static String formatCert (String inputCert)
  {
    /** The prefix string for an X.509 certificate */
    String beginCert = "-----BEGIN CERTIFICATE-----";
    /** The suffix string for an X.509 certificate */
    String endCert = "-----END CERTIFICATE-----";

    StringBuilder resultBuilder = new StringBuilder ();

    // Add the certificate begin and end lines in case they are missing
    if (!inputCert.startsWith (beginCert))
      inputCert = beginCert + "\n" + inputCert;
    if (!inputCert.endsWith (endCert))
      inputCert = inputCert + "\n" + endCert;

    // Ensure that the certificate begin and end are on their own individual lines
    inputCert = inputCert.replaceAll (beginCert, "\n" + beginCert + "\n").replaceAll (endCert, "\n" + endCert + "\n").replaceAll ("\n\n", "\n");

    String certLines[] = inputCert.split ("[\r\n]+"); // Array of all non-empty lines

    // The certificate begin and end lines can't exceed 64 char length, but the other(s) can. So wrap them.
    for (String currLine : certLines)
    {
      if ((currLine.length () > 64))
      {
        currLine =  currLine.replaceAll ("(.{64})", "$1\n");
      }
      resultBuilder.append (currLine).append ("\n");
    }
    return resultBuilder.toString ().replaceAll ("\n\n", "\n").trim ();
  }

  /**
   * Get an SSLSocketFactory that uses the input certificate file.
   *
   * @param certFile the full path to the certificate file to use
   */
  public static SSLSocketFactory getSSLSocketFactory (String certFile) throws CertificateException, IOException, KeyStoreException, NoSuchAlgorithmException, KeyManagementException
  {
    return getSSLSocketFactory (true, certFile, "TLS", "PKIX");
  }

  /**
   * Get an SSLSocketFactory that uses the provided certificate file, protocol and algorithm.
   *
   * The standard protocols are listed here:
   * https://docs.oracle.com/javase/7/docs/technotes/guides/security/StandardNames.html#SSLContext
   *
   * @param certInFile if true then the next parameter the the name of the certificate file - otherwise it is the certificate itself
   * @param theCert the String containing the certificate itself or name of the certificate file depending on the first parameter
   * @param secureProtocol the protocol to use or null for default (TLS)
   * @param tmAlgorithm the TrustManagerFactory algorithm name or null for the default (PKIX)
   * @return
   * @throws CertificateException
   * @throws IOException
   * @throws KeyStoreException
   * @throws NoSuchAlgorithmException
   * @throws KeyManagementException
   */
  public static SSLSocketFactory getSSLSocketFactory (boolean certInFile, String theCert, String secureProtocol, String tmAlgorithm) throws CertificateException, IOException, KeyStoreException, NoSuchAlgorithmException, KeyManagementException
  {
    Certificate theCertificate;
    if (certInFile)
      theCertificate = getCertificateFromFile (theCert);
    else
      theCertificate = getCertificateFromString (theCert);

    KeyStore keyStore = getKeyStore (theCertificate);
    TrustManagerFactory tmFactory = getTrustManagerFactory (keyStore, tmAlgorithm);
    return getSSLSocketFactory (tmFactory, secureProtocol);
  }

  /**
   * Create an SSLContext that uses a given TrustManager Factory and protocol.
   * https://docs.oracle.com/javase/7/docs/technotes/guides/security/StandardNames.html#SSLContext
   *
   * @param tmFactory the TrustManagerFactory for which we need an SSLContext
   * @param secureProtocol the protocol to use or null for default
   * @return
   * @throws NoSuchAlgorithmException
   * @throws KeyManagementException
   */
  public static SSLSocketFactory getSSLSocketFactory (TrustManagerFactory tmFactory, String secureProtocol) throws NoSuchAlgorithmException, KeyManagementException
  {
    if (secureProtocol == null)
    {
      secureProtocol = "TLS";
    }
    SSLContext sslContext = SSLContext.getInstance (secureProtocol);
    // System.out.println ("Created SSLContext using protocol [" + secureProtocol + "]");
    sslContext.init (null, tmFactory.getTrustManagers (), null);
    return sslContext.getSocketFactory ();
  }

  /**
   * Create a TrustManager Factory that trusts the CAs in a given KeyStore
   *
   * @param keyStore
   * @param tmAlgorithm the TrustManagerFactory algorithm name or null for the default
   * @return
   * @throws NoSuchAlgorithmException
   * @throws KeyStoreException
   */
  public static TrustManagerFactory getTrustManagerFactory (KeyStore keyStore, String tmAlgorithm) throws NoSuchAlgorithmException, KeyStoreException
  {
    if (tmAlgorithm == null)
    {
      tmAlgorithm = TrustManagerFactory.getDefaultAlgorithm ();
    }
    // System.out.println ("Created TrustManagerFactory using Algorithm [" + tmAlgorithm + "]");

    TrustManagerFactory tmFactory = TrustManagerFactory.getInstance (tmAlgorithm);
    tmFactory.init (keyStore);
    return tmFactory;
  }

  /**
   * Create a KeyStore that trusts the CAs in a given Certificate
   *
   * @param theCert
   * @return
   * @throws KeyStoreException
   * @throws IOException
   * @throws NoSuchAlgorithmException
   * @throws CertificateException
   */
  public static KeyStore getKeyStore (Certificate theCert) throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException
  {
    String keyStoreType = KeyStore.getDefaultType ();
    KeyStore keyStore = KeyStore.getInstance (keyStoreType);
    keyStore.load (null, null);
    keyStore.setCertificateEntry ("theCert", theCert);
    return keyStore;
  }

  /**
   * Load an X.509 certificate from a PEM formatted string
   *
   * @param pemString a String in PEM format encoded in "UTF-8"
   * @return an X.509 Certificate
   * @throws CertificateException
   * @throws IOException
   */
  public static Certificate getCertificateFromString (String pemString) throws CertificateException, IOException
  {
    if (pemString == null || pemString.length () == 0)
    {
      throw new CertificateException ("Null or zero length Certificate string");
    }

    // System.out.println ("DEBUG: theCertString = " + pemString);
    CertificateFactory certFactory = CertificateFactory.getInstance ("X.509");

    //before decoding we need to get rid of the prefix and suffix
    //byte [] decodedBytes = Base64.decode (certStr.replaceAll (X509Factory.BEGIN_CERT, "").replaceAll (X509Factory.END_CERT, ""));
    //ByteArrayInputStream certStream = new ByteArrayInputStream (decodedBytes);

    ByteArrayInputStream certStream = new ByteArrayInputStream (pemString.getBytes ("US-ASCII")); // Not "UTF-8" since certs are base64 encoded

    Certificate theCert;
    try
    {
      theCert = certFactory.generateCertificate (certStream);
      // System.out.println ("DEBUG: theCert = " + ((X509Certificate) theCert).getSubjectDN ());
    }
    finally
    {
      if (certStream != certStream)
      {
        certStream.close ();
      }
    }
    return theCert;
  }

  /**
   * Load an X.509 certificate from a file
   *
   * @param certFile
   * @return an X.509 Certificate
   * @throws CertificateException
   * @throws IOException
   */
  public static Certificate getCertificateFromFile (String certFile) throws CertificateException, IOException
  {
    if (certFile == null || certFile.length () == 0)
    {
      throw new CertificateException ("ERROR: Null or zero length certificate file names are not allowed.");
    }

    CertificateFactory certFactory = CertificateFactory.getInstance ("X.509");
    InputStream certStream = new BufferedInputStream (new FileInputStream (certFile));
    Certificate theCert;
    try
    {
      theCert = certFactory.generateCertificate (certStream);
      // System.out.println ("theCert=" + ((X509Certificate) theCert).getSubjectDN ());
    }
    finally
    {
      if (certStream != certStream)
      {
        certStream.close ();
      }
    }
    return theCert;
  }

  /**
   * Create an HttpsURLConnection that does encryption but not authentication and then connect it to a specified URL
   *
   * @param theUrl the URL string to which we are connecting
   * @param secureSocketProtocol the protocol to use when creating the encryption (Hint: just use "TLS" or maybe "SSL")
   * @throws IOException if unable to create the secure connection
   */
  public static HttpsURLConnection getEncryptedNoauthConnection (URL theUrl, String secureSocketProtocol) throws IOException
  {
    SSLContext sslContext = null;
    // Assume the URL is https - if not, it is an error so we just let the cast throw an exception
    HttpsURLConnection connection = (HttpsURLConnection) theUrl.openConnection ();

    TrustManager [] trustManagers;
    try
    {
      sslContext = SSLContext.getInstance (secureSocketProtocol);
    }
    catch (NoSuchAlgorithmException ne)
    {
      throw new IOException ("ERROR: Unable to create SSL Context using protocol: " + secureSocketProtocol + ".\nMessage: " + ne.getMessage ());
    }
    // Here we do not care about authenticating the server, so just encrypt but don't worry about certificates
    trustManagers = new TrustManager[] { openTrustManager };
    connection.setHostnameVerifier (openHostnameVerifier);

    try
    {
      sslContext.init (null, trustManagers, null);
    }
    catch (KeyManagementException ke)
    {
      throw new IOException ("ERROR: Unable to initialize SSL Context using trust manager: " + trustManagers + ".\nMessage: " + ke.getMessage ());
    }
    connection.setSSLSocketFactory (sslContext.getSocketFactory ());

    return connection;
  }

  /**
   * This is a fully OPEN trust manager. It will trust EVERY client, server and certificate issuer.
   */
  private static TrustManager openTrustManager = new X509TrustManager ()
  {
    @Override
    public X509Certificate [] getAcceptedIssuers () { return new X509Certificate [0]; }

    @Override
    public void checkClientTrusted (X509Certificate[] certs, String authType) { return; }

    @Override
    public void checkServerTrusted (X509Certificate[] certs, String authType) { return; }
  };

  /**
   * This is a fully OPEN hostname verifier. It will verify SSL sessions to EVERY host.
   */
  private static HostnameVerifier openHostnameVerifier = new HostnameVerifier ()
  {
    @Override
    public boolean verify (String s, SSLSession sslSession) { return true; }
  };

  /**
   * Read the entire contents of a URLConnection and get results as a byte array.
   *
   * @param theConnection the URLConnection from which we are reading data
   */
  public static byte [] getData (URLConnection theConnection) throws IOException
  {
    return emptyStream (theConnection.getInputStream ());
  }

  /**
   * Get detailed certificate information about an HttpsURLConnection into a string
   * This is quite useful for debugging purposes but not much else.
   *
   * @param theConnection the HttpsURLConnection which we are debugging
   */
  public static String getCertInfo (HttpsURLConnection theConnection) throws IOException
  {
    StringBuilder resultBuffer = new StringBuilder ();
    if (theConnection != null)
    {
      try
      {
        resultBuffer.append ("Response Code : " + theConnection.getResponseCode ()).append ("\n");
        resultBuffer.append ("Cipher Suite : " + theConnection.getCipherSuite ()).append ("\n");
        resultBuffer.append ("\n");

        Certificate[] certificateList = theConnection.getServerCertificates ();
        for (Certificate currCert : certificateList)
        {
          resultBuffer.append ("Certificate Type : " + currCert.getType ()).append ("\n");
          resultBuffer.append ("Certificate Hash Code : " + currCert.hashCode ()).append ("\n");
          resultBuffer.append ("Certificate Public Key Algorithm : " + currCert.getPublicKey ().getAlgorithm ()).append ("\n");
          resultBuffer.append ("Certificate Public Key Format : " + currCert.getPublicKey ().getFormat ()).append ("\n");
          resultBuffer.append ("\n");
        }
      }
      catch (SSLPeerUnverifiedException e)
      {
        throw new IOException ("ERROR: SSL Peer Unverified. URL: " + theConnection + ".\nMessage: " + e.getMessage ());
      }
    }
    return resultBuffer.toString ();
  }

  /**
   * Create a new URLConnection to a remote server. Read all headers from a Map
   *
   * @param theUrl  the <code>String</code> containing the remote target to which we are creating a new connection
   * @param reqHeaders  the <code>Map <String, String></code> from which we to extract headers, etc for the new connection
   * @param sslSocketFactory  the <code>SSLSocketFactory</code> to use for creation of secure sockets or null if https is not needed
   * @throws IOException if an I/O error occurs
   */
  public static URLConnection getConnection (String theUrl, Map <String, String> reqHeaders, SSLSocketFactory sslSocketFactory) throws IOException
  {
    URLConnection theConnection;
    if (theUrl.toLowerCase ().startsWith ("https://"))
    {
      if (sslSocketFactory == null)
      {
        // We have an https:// URL but no certificate - so no authentication
      	// Ideally, we should just throw an exception here as this is insecure
        theConnection = getEncryptedNoauthConnection (new URL (theUrl), "TLS");
      }
      else
      {
        // We have an https:// URL with a certificate inside the SSL Socket Factory so we can authenticate the server
        HttpsURLConnection secConnection = (HttpsURLConnection) new URL (theUrl).openConnection ();
        secConnection.setSSLSocketFactory (sslSocketFactory);
        theConnection = secConnection;
      }
    }
    else
    {
      // We have a plain http:// URL
      theConnection = new URL (theUrl).openConnection ();
    }

    if (reqHeaders != null)
    {
      setHeaders (theConnection, reqHeaders);
    }

    return theConnection;
  }

  /**
   * Create a new URLConnection to a remote server. Copy all headers from a Map
   *
   * @param theUrl  the <code>String</code> containing the remote target to which we are creating a new connection
   * @param reqHeaders  the <code>Map <String, String></code> from which we to extract headers, etc for the new connection
   * @throws IOException if an I/O error occurs
   */
  public static URLConnection getConnection (String theUrl, Map <String, String> reqHeaders) throws IOException
  {
    return getConnection (theUrl, reqHeaders, null);
  }

  /**
   * Make a URLConnection to an HTTP or HTTPS URL (using TLS)
   *
   * @param theUrl the URL string to which we are connecting
   * @throws IOException if unable to fetch url
   */
  public static URLConnection getConnection (String theUrl) throws IOException
  {
    return getConnection (theUrl, null, null);
  }

  /**
   * Fetch the contents of an HTTPS URL into a byte array using a given SSL Socket Factory
   *
   * @param theUrl the URL that needs to be retreived
   * @param sslSocketFactory the socket factory to be used for creation of the SSL connection
   * @return a byte array containing the contents of the url
   * @throws IOException
   */
  public static URLConnection getConnection (String theUrl, SSLSocketFactory sslSocketFactory) throws IOException
  {
    return getConnection (theUrl, null, sslSocketFactory);
  }

  /**
   * Make a GET request to an HTTP or HTTPS URL and get results as a String.
   *
   * @param theUrl the URL string to which we are connecting
   * @throws IOException if unable to fetch url
   */
  public static String fetchUrlString (String theUrl) throws IOException
  {
    return new String (fetchUrl (getConnection (theUrl), null, "GET"));
  }

  /**
   * Make a GET request to an HTTP or HTTPS URLConnection and get results as a String.
   *
   * @param theConnection the URLConnection to which we are connecting
   * @throws IOException if unable to fetch url
   */
  public static String fetchUrlString (URLConnection theConnection) throws IOException
  {
    return new String (fetchUrl (theConnection, null, "GET"));
  }

  /*
   * Make a GET request to an HTTP or HTTPS URL and get results as a byte array.
   *
   * @param theUrl the URL string to which we are connecting
   * @throws IOException if unable to fetch url
   */
  public static byte [] fetchUrl (String theUrl) throws IOException
  {
    return fetchUrl (getConnection (theUrl), null, "GET");
  }

  /**
   * Make a GET request to an HTTP or HTTPS URL and get results as a byte array.
   *
   * @param theConnection the URLConnection to which we are connecting
   * @throws IOException if unable to fetch url
   */
  public static byte [] fetchUrl (URLConnection theConnection) throws IOException
  {
    return fetchUrl (theConnection, null, "GET");
  }

  /**
   * GET or POST a request to an HTTP or HTTPS URL and return results as a byte array.
   *
   * @param theConnection the URLConnection to which we are connecting
   * @param dataBuffer the String containing data to be POSTed (or null for GET) Eg: "name1=value1&name2=value2"
   * @throws IOException if unable to fetch url
   */
  public static byte [] fetchUrl (String theUrl, String dataBuffer) throws IOException
  {
    return fetchUrl (getConnection (theUrl), dataBuffer.getBytes (), dataBuffer == null ? "GET" : "POST");
  }

  /**
   * GET or POST a request to an HTTP or HTTPS URL and return results as a byte array.
   *
   * @param theConnection the URLConnection to which we are connecting
   * @param dataBuffer the byte array of data to be POSTed (or null for GET)
   * @throws IOException if unable to fetch url
   */
  public static byte [] fetchUrl (URLConnection theConnection, byte [] dataBuffer, String reqMethod) throws IOException
  {
    byte[] responseBuffer = null;

    try
    {
      theConnection.setUseCaches (false);
      theConnection.setDoInput (true);
      theConnection.setRequestProperty ("Cache-Control", "no-cache");

      if (reqMethod.toUpperCase ().equals ("GET"))
      {
        theConnection.setDoOutput (false);
        if (theConnection instanceof HttpURLConnection)    // Block probably not needed - check
        {
          HttpURLConnection theHttpConnection = (HttpURLConnection) theConnection;
          theHttpConnection.setRequestMethod ("GET");
        }
      }
      else                                                 // This is a POST
      {
        theConnection.setDoOutput (true);
        if (theConnection instanceof HttpURLConnection)    // Block probably not needed - check
        {
          HttpURLConnection theHttpConnection = (HttpURLConnection) theConnection;
          theHttpConnection.setRequestMethod ("POST");
        }
        fillStream (theConnection.getOutputStream (), dataBuffer);
      }

      /* FOR DEBUG ONLY - Dump HTTPS Certificate information
      if (theConnection instanceof HttpsURLConnection)
      {
        HttpsURLConnection secureConnection = (HttpsURLConnection) theConnection;
        System.out.println (getCertInfo (secureConnection));
      }
      */

      // Read the entire contents of the URLConnection and get results as a byte array.
      responseBuffer = getData (theConnection);
    }
    catch (MalformedURLException e)
    {
      throw new IOException ("ERROR: Invalid URL: " + theConnection + ".\nMessage: " + e.getMessage ());
    }
    return responseBuffer;
  }

  /**
   * Read all of the response headers from an HttpURLConnection into a Map of String pairs
   *
   * @param remoteConn the HttpURLConnection from which we are extracting response headers
   */
  public static Map <String, String> getResponseHeaders (URLConnection remoteConn)
  {
    Map <String, String> headerMap = new HashMap <String, String> ();
    Map <String, List<String>> headerFields = remoteConn.getHeaderFields ();
    Set <String> headerFieldsSet = headerFields.keySet ();
    Iterator <String> headerFieldsIter = headerFieldsSet.iterator ();

    while (headerFieldsIter.hasNext ())
    {
      String headerFieldKey = headerFieldsIter.next ();
      if (headerFieldKey != null)
      {
        List <String> headerFieldValue = headerFields.get (headerFieldKey);
        headerMap.put (headerFieldKey, csvList (headerFieldValue));
      }
    }
    return headerMap;
  }

  /**
   * Merge a List of Strings into a single csv string
   *
   * @param headerValues a List of Strings containing the values of a header
   */
  public static String csvList (List<String> theList)
  {
    StringBuilder mergedValue = new StringBuilder ();
    for (String currValue : theList)
    {
      if (mergedValue.length () > 0)
      {
        mergedValue.append (",");
      }
      mergedValue.append (currValue);
    }
    return mergedValue.toString ();
  }

  /**
   * Dump a java map of string keys and string values into a formatted string
   * of name value pairs delimited by semi-colons.
   *
   * @param theMap  the map which we are dumping
   */
  public static String dumpMap (Map <String, String> theMap)
  {
    String entryTemplate = "%s=\"%s\";";
    StringBuilder resultBuffer = new StringBuilder ();
    for (Map.Entry <String, String> currEntry : theMap.entrySet ())
    {
      resultBuffer.append (String.format (entryTemplate, currEntry.getKey (), currEntry.getValue ()));
    }
    if (resultBuffer.length () > 1)
    {
      resultBuffer.deleteCharAt (resultBuffer.length () - 1);
    }
    return resultBuffer.toString ();
  }

  /**
   * Write out the an entire contents of a byte array into an output stream
   *
   * @param outStream  the <code>OutputStream</code> to which we read the data
   * @param streamData  the data to be written
   * @throws IOException if an I/O error occurs
   */
  public static void fillStream (OutputStream outStream, byte [] streamData) throws IOException
  {
    try
    {
      outStream.write (streamData);
      outStream.flush ();
    }
    finally
    {
      if (outStream != null) outStream.close ();
    }
  }

  /**
   * Empty out the an entire contents of an input stream into a byte array.
   *
   * @param inStream  the <code>InputStream</code> to which we read the data
   * @throws IOException if an I/O error occurs
   */
  public static byte [] emptyStream (InputStream inStream) throws IOException
  {
    /** size of buffer. */
    int DEFAULT_BLOCK_SIZE = 4096;
    ByteArrayOutputStream outStream = null;

    if (inStream == null)
    {
      return new byte[0];
    }

    try
    {
      outStream = new ByteArrayOutputStream ();
      int numBytes;
      byte [] byteBuffer = new byte [DEFAULT_BLOCK_SIZE];
      while ((numBytes = inStream.read (byteBuffer)) > -1)
      {
        outStream.write (byteBuffer, 0, numBytes);
      }
    }
    finally
    {
      if (outStream != null)
      {
        outStream.close ();
      }
      if (inStream != null)
      {
        inStream.close ();
      }
    }
    return outStream.toByteArray ();
  }

  /**
   * Compute the Adler32 checksum (fast algorithm) of a byte array
   *
   * @param rawData  the data for which we are getting the checksum
   */
  public static long getAdler32 (byte rawData[])
  {
    if (rawData == null)
    {
      return 0L;
    }
    Checksum checksum = new Adler32 ();
    checksum.update (rawData, 0, rawData.length);
    return checksum.getValue ();
  }

  /**
   * Set the value of a request header in a URL connection
   *
   * @param remoteConn  the <code>URLConnection</code> whose header we are setting
   * @param headerName  the <code>String</code> containing name of the header we are setting
   * @param headerValue  the <code>String</code> containing value of the header we are setting
   */
  public static void setHeader (URLConnection remoteConn, String headerName, String headerValue)
  {
    remoteConn.setRequestProperty (headerName, headerValue);
  }

  /**
   * Set the values of a bunch of request header in a URL connection from a Map of String pairs
   *
   * @param remoteConn  the <code>URLConnection</code> whose header we are setting
   * @param httpHeaders  the <code>Map</code> containing name of the name/value of headers we are setting
   */
  public static void setHeaders (URLConnection remoteConn, Map <String, String> httpHeaders)
  {
    // Copy over all headers from the map to the remote connection
    for (String headerName : httpHeaders.keySet ())
    {
      remoteConn.setRequestProperty (headerName, httpHeaders.get (headerName));
    }
  }

  /**
   * Print the full set of request headers in a URL connection to a logger
   *
   * @param logger  the <code>Logger</code> to which we are printing the headers
   * @param remoteConn  the <code>URLConnection</code> from which we are printing the request headers
   * @param prefix  the <code>String</code> which we can use to identify what is being printed
   */
  public static void dumpRequestHeaders (URLConnection remoteConn, String prefix)
  {
    for (String currHeader : remoteConn.getRequestProperties().keySet ())
    {
      if (currHeader != null)
      {
        for (String headerValue : remoteConn.getRequestProperties ().get (currHeader))
        {
          System.out.println (prefix + " " + currHeader + ": " + headerValue);
        }
      }
    }
  }

  /**
   * Print the full set of response headers in a URL connection to the console
   *
   * @param remoteConn  the <code>URLConnection</code> from which we are printing the response headers
   * @param prefix  the <code>String</code> which we can use to identify what is being printed
   */
  public static void dumpResponseHeaders (URLConnection remoteConn, String prefixStr)
  {
    for (String currHeader : remoteConn.getHeaderFields ().keySet ())
    {
      if (currHeader != null)
      {
        for (String headerValue : remoteConn.getHeaderFields ().get (currHeader))
        {
          System.out.println (prefixStr + " " + currHeader + ": " + headerValue);
        }
      }
    }
  }

  /**
   * Create a checksum of a byte array and store the result in a Map
   */
  public static void addChecksumHeader (String headerName, Map <String, String> theMap, byte[] rawData)
  {
    if (rawData == null)
    {
      theMap.put (headerName, "");
    }
    else
    {
      theMap.put (headerName, Long.toString (getAdler32 (rawData)));
    }
  }

  /**
   * Read a map and fetch the value of the checksum header
   */
  public static String getChecksumHeader (String headerName, Map <String, String> theMap)
  {
    return theMap.get (headerName);
  }

  /**
   * Log all messages in a standard way.
   *
   * If called from method boo in class Foo like this:
   *   logPrefix ("ERROR", Thread.currentThread ())
   * it will return something like:
   *   ERROR: [Foo :: boo] -
   */
  public static String logPrefix (String msgType, Thread theThread)
  {
    StringBuffer resultStr = new StringBuffer (msgType);
    resultStr.append (": [");
    resultStr.append (theThread.getStackTrace ()[2].getClassName ().toString ());
    resultStr.append (" :: ");
    resultStr.append (theThread.getStackTrace ()[2].getMethodName ().toString ());
    resultStr.append ("] ");
    return resultStr.toString ();
  }

}