Last active
October 17, 2017 10:21
-
-
Save angrycub/1510c873942867b9a31b355316f13125 to your computer and use it in GitHub Desktop.
Sample code to read through nasty multipart/mixed response to get to individual json chunks,
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> | |
<modelVersion>4.0.0</modelVersion> | |
<groupId>Streaming2iReader</groupId> | |
<artifactId>Streaming2iReader</artifactId> | |
<version>0.0.1-SNAPSHOT</version> | |
<build> | |
<sourceDirectory>src</sourceDirectory> | |
<plugins> | |
<plugin> | |
<artifactId>maven-compiler-plugin</artifactId> | |
<version>3.3</version> | |
<configuration> | |
<source>1.7</source> | |
<target>1.7</target> | |
</configuration> | |
</plugin> | |
</plugins> | |
</build> | |
<dependencies> | |
<dependency> | |
<groupId>org.json</groupId> | |
<artifactId>json</artifactId> | |
<version>20160212</version> | |
</dependency> | |
<dependency> | |
<groupId>org.apache.wink</groupId> | |
<artifactId>wink-common</artifactId> | |
<version>1.4</version> | |
</dependency> | |
</dependencies> | |
</project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# build sample data | |
for USER in alan bob charlie; do echo "Creating Postings for $USER"; for I in {1..100}; do curl -XPOST localhost:8098/buckets/postings/keys/$USER_$I -H "x-riak-index-user_bin: $USER" -H 'Content-Type: application/json' -d "{\"userData\":\"$USER: I wrote message number $I.\"}"; done; done | |
# run sample query | |
curl -v localhost:8098/buckets/postings/index/user_bin/charlie?stream=true |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package com.basho; | |
import java.io.IOException; | |
import java.io.InputStream; | |
import java.io.UnsupportedEncodingException; | |
import java.net.URL; | |
import java.net.URLConnection; | |
import java.net.URLEncoder; | |
import org.apache.wink.common.internal.providers.multipart.MultiPartParser; | |
import org.apache.wink.common.model.multipart.InMultiPart; | |
import org.apache.wink.common.model.multipart.InPart; | |
import org.json.JSONArray; | |
import org.json.JSONObject; | |
public class Streaming2iReader { | |
String host, port, bucketType, bucket, index, queryValue = ""; | |
URL url = null; | |
URLConnection connection = null; | |
InputStream response = null; | |
public Streaming2iReader(String host, String port, String bucketType, String bucket, String index, String queryValue) throws IOException { | |
this.host = host; | |
this.port = port; | |
this.bucketType = bucketType; | |
this.bucket = bucket; | |
this.index = index; | |
this.queryValue = queryValue; | |
String url = "http://"+host+":"+port+"/types/"+bucketType+"/buckets/"+bucket+"/index/"+index+"/"+queryValue; | |
String charset = "UTF-8"; // Or in Java 7 and later, use the constant: java.nio.charset.StandardCharsets.UTF_8.name() | |
String stream = "true"; | |
String query=""; | |
try { | |
query = String.format("stream=%s", | |
URLEncoder.encode(stream, charset)); | |
} catch (UnsupportedEncodingException e) { | |
e.printStackTrace(); | |
} | |
if (! query.equals("")) { | |
this.url = new URL(url + "?" + query); | |
} else { | |
this.url = new URL(url); | |
} | |
this.connection = this.url.openConnection(); | |
connection.setRequestProperty("Accept-Charset", charset); | |
this.response = connection.getInputStream(); | |
} | |
String getContentType() { | |
return this.connection.getContentType(); | |
} | |
InputStream getResponse() { | |
return this.response; | |
} | |
boolean isMultiPart() { | |
return connection.getContentType().startsWith("multipart/mixed"); | |
} | |
String getBoundary() { | |
if (isMultiPart()) { | |
return getContentType().substring(25); | |
} else { | |
return null; | |
} | |
} | |
public static void main(String[] args) throws Exception { | |
String host = "172.28.128.7"; | |
String port = "8098"; | |
String bucketType = "default"; | |
String bucket = "postings"; | |
String index = "user_bin"; | |
String queryValue = "charlie"; | |
Streaming2iReader reader = new Streaming2iReader(host, port, bucketType, bucket, index, queryValue); | |
if (reader.isMultiPart()) { | |
InMultiPart imp = new InMultiPart(new MultiPartParser(reader.getResponse(), reader.getBoundary())); | |
reader.processMessage(imp); | |
} | |
} | |
public void processMessage(InMultiPart inMultiPart) throws Exception { | |
while(inMultiPart.hasNext()) { | |
InPart part = inMultiPart.next(); | |
if (part.getContentType().startsWith("application/json")) { | |
String jsonString = convertStreamToString(part.getInputStream()); | |
JSONObject jo = new JSONObject(jsonString); | |
JSONArray keys = (JSONArray) jo.get("keys"); | |
for (Object key : keys) { | |
String keyString = (String) key; | |
processKey(keyString); | |
} | |
} else { | |
throw new Exception("Unexpected content type: "+ part.getContentType()); | |
} | |
} | |
} | |
public static void processKey(String keyString) { | |
System.out.println(keyString); | |
} | |
public static void debug(String message) { | |
System.out.println("DEBUG: " + message); | |
} | |
static String convertStreamToString(java.io.InputStream is) { | |
java.util.Scanner s = new java.util.Scanner(is).useDelimiter("\\A"); | |
return s.hasNext() ? s.next() : ""; | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment