This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import java.util.LinkedList; | |
import java.util.List; | |
import java.util.Queue; | |
/* | |
* A solution using Proxy Pattern to limited instances problem | |
* Requirement: | |
* class RareResource must have no more than 3 instances at any run time. | |
* Note: | |
* Black magic like reflection is out of consideration as you can never prevent it. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
keys = ['a', 'b'] | |
values = ['c', 'd'] | |
dictionary = dict(zip(keys, values)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import java.io.*; | |
public class CVJNI { | |
//Load jni library | |
static { | |
try { | |
System.loadLibrary("cvjni"); | |
} catch (Exception e) { | |
e.printStackTrace(); | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import java.util.*; | |
public class Iter { | |
public static void main(String[] args) { | |
Aggregate<Integer, Integer, Integer> ag = new DataList<Integer, Integer, Integer>(); | |
((DataList)ag).add(new ThreeD<Integer, Integer, Integer>(1, 2, 3)); | |
((DataList)ag).add(new ThreeD<Integer, Integer, Integer>(1, 2, 3)); | |
((DataList)ag).add(new ThreeD<Integer, Integer, Integer>(1, 2, 3)); | |
Iterator<Integer> ia = ag.createIteratorA(); | |
while (ia.next() != null) { |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
//Here is HelloJNI.java | |
public class HelloJNI { | |
static { | |
try { | |
Class c = HelloJNI.class; | |
URL location = | |
c.getProtectionDomain().getCodeSource().getLocation(); | |
ZipFile zf = new ZipFile(location.getPath()); | |
// libhellojni.so is put in the lib folder |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// code snippet in nutch | |
Injector injector = new Injector(getConf()); | |
Generator generator = new Generator(getConf()); | |
Fetcher fetcher = new Fetcher(getConf()); | |
ParseSegment parseSegment = new ParseSegment(getConf()); | |
CrawlDb crawlDbTool = new CrawlDb(getConf()); | |
LinkDb linkDbTool = new LinkDb(getConf()); | |
injector.inject(crawlDb, rootUrlDir); | |
int i; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
linkDbTool.invert(linkDb, segments, true, true, false); // invert links | |
if (solrUrl != null) { | |
// index, dedup & merge | |
FileStatus[] fstats = fs.listStatus(segments, HadoopFSUtil.getPassDirectoriesFilter(fs)); | |
SolrIndexer indexer = new SolrIndexer(getConf()); | |
indexer.indexSolr(solrUrl, crawlDb, linkDb, | |
Arrays.asList(HadoopFSUtil.getPaths(fstats))); | |
SolrDeleteDuplicates dedup = new SolrDeleteDuplicates(); | |
dedup.setConf(getConf()); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
if (LOG.isInfoEnabled()) { | |
LOG.info("Injector: Converting injected urls to crawl db entries."); | |
} | |
JobConf sortJob = new NutchJob(getConf()); | |
sortJob.setJobName("inject " + urlDir); | |
FileInputFormat.addInputPath(sortJob, urlDir); | |
sortJob.setMapperClass(InjectMapper.class); | |
FileOutputFormat.setOutputPath(sortJob, tempDir); | |
sortJob.setOutputFormat(SequenceFileOutputFormat.class); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
public static JobConf createJob(Configuration config, Path crawlDb) | |
throws IOException { | |
Path newCrawlDb = new Path(crawlDb, | |
Integer.toString(new Random().nextInt(Integer.MAX_VALUE))); | |
JobConf job = new NutchJob(config); | |
job.setJobName("crawldb " + crawlDb); | |
Path current = new Path(crawlDb, CURRENT_NAME); | |
if (FileSystem.get(job).exists(current)) { |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
public static void install(JobConf job, Path crawlDb) throws IOException { | |
Path newCrawlDb = FileOutputFormat.getOutputPath(job); | |
FileSystem fs = new JobClient(job).getFs(); | |
Path old = new Path(crawlDb, "old"); | |
Path current = new Path(crawlDb, CURRENT_NAME); | |
if (fs.exists(current)) { | |
if (fs.exists(old)) fs.delete(old, true); | |
fs.rename(current, old); | |
} | |
fs.mkdirs(crawlDb); |
OlderNewer