Pig script for counting words in small text comments stored in one column of a flat file.

pig script for counting words

unstructuredText = load '<file name>' using PigStorage('|') 
  as 
		(
		CUSTOMER_NUMBER:chararray,
		VISIT_TYPE:chararray,
		REVIEW_DATE:chararray,
		NOTE:chararray
		);

tokenized = foreach unstructuredText 
	generate 
		CUSTOMER_NUMBER,
		VISIT_TYPE, 
		REVIEW_DATE,
		FLATTEN(TOKENIZE(NOTE)) as word;

grouped = group tokenized by 
		(CUSTOMER_NUMBER,
		VISIT_TYPE, 
		REVIEW_DATE);

counts = foreach grouped
	generate
		group, COUNT(tokenized) as wc;

store counts into 'output' using PigStorage('\t');