secret
Last active

Excerpt of the MetadataLookup EvalFunc.

  • Download Gist
gistfile1.java
Java
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53
package com.parsely.pig.cache;
 
import java.io.IOException;
import java.net.UnknownHostException;
import java.security.NoSuchAlgorithmException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Arrays;
 
import org.apache.commons.logging.Log;
 
import org.apache.pig.EvalFunc;
import org.apache.pig.PigWarning;
import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
 
import com.parsely.util.cache.Metadata;
import com.parsely.util.cache.ParselyMetadataService;
 
public class MetadataLookup extends EvalFunc<Tuple> {
private ParselyMetadataService meta;
private TupleFactory mTupleFactory = TupleFactory.getInstance();
private DateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mmZ");
private Log logger;
public MetadataLookup(int lruSize) throws UnknownHostException {
// This service opens up a connection to both mongo and redis
this.meta = new ParselyMetadataService(lruSize);
this.logger = getLogger();
}
public MetadataLookup() throws UnknownHostException {
this(500);
}
 
@Override
public Tuple exec(Tuple input) throws IOException {
// Returns a 5 element tuple by using calls to meta.fetch()
// which queries Redis/Mongo depending on the parameters
}
@Override
public void finish() {
// Closes the private mongo and redis connections within meta
meta.close();
}
}

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.