public
Last active

  • Download Gist
gistfile1.java
Java
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80
public static void oldMakeTable(HTable table, int numRows, int numCols) throws IOException {
table.setAutoFlush(false);
Random random = new Random();
byte[] family = Bytes.toBytes("family");
for (int i = 0; i < numRows; i++) {
byte[] rowBytes = Bytes.toBytes(i);
for (int j = 0 ; j < numCols; j++) {
table.put(new Put(rowBytes).add(family, Bytes.toBytes(j), Bytes.toBytes(random.nextDouble())));
}
System.out.println(String.format("Done ingesting %d rows", i));
}
table.flushCommits();
}
 
public static void oldScanTable(HTable table, int numRows, int numCols) throws IOException {
Scan scan = new Scan();
byte[] family = Bytes.toBytes("family");
scan.addFamily(family);
ResultScanner results = table.getScanner(scan);
int i = 0;
for (Result result : results) {
for (KeyValue kv : result.list()) {
// System.out.print(Bytes.toDouble(kv.getValue()) + "\t");
}
// System.out.println("");
System.out.println(String.format("Done scanning %d rows", i++));
}
results.close();
}
 
public static void newMakeTable(HTable table, int numRows, int numCols, int segmentSize) throws IOException {
table.setAutoFlush(false);
Random random = new Random();
byte[] family = Bytes.toBytes("family");
for (int i = 0; i < numRows; i++) {
byte[] rowBytes = Bytes.toBytes(i);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(baos);
dos.writeInt(segmentSize);
int k = 0;
int segmentNum = 0;
for (int j = 0 ; j < numCols; j++) {
dos.writeDouble(random.nextDouble());
k++;
if (k >= segmentSize || j == numCols - 1) {
k = 0;
table.put(new Put(rowBytes).add(family, Bytes.toBytes(segmentNum++), baos.toByteArray()));
baos = new ByteArrayOutputStream();
dos = new DataOutputStream(baos);
if (numCols - j - 1 > segmentSize)
dos.writeInt(segmentSize);
else if (numCols - j - 1 != 0)
dos.writeInt(numCols - j - 1);
}
}
System.out.println(String.format("Done ingesting %d rows", i));
}
table.flushCommits();
}
 
public static void newScanTable(HTable table, int numRows, int numCols, int segmentSize) throws IOException {
Scan scan = new Scan();
byte[] family = Bytes.toBytes("family");
scan.addFamily(family);
ResultScanner results = table.getScanner(scan);
int j = 0;
for (Result result : results) {
for (KeyValue kv : result.list()) {
int segmentNum = Bytes.toInt(kv.getQualifier());
DataInputStream dis = new DataInputStream(new ByteArrayInputStream(kv.getValue()));
int valuesInSegment = dis.readInt();
for (int i = 0; i < valuesInSegment; i++) {
// System.out.print(dis.readDouble() + "\t");
}
}
System.out.println(String.format("Done scanning %d rows", j++));
// System.out.println("");
}
results.close();
}

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.