create_ohsumed.sh

gistfile1.txt

#!/bin/sh
#wget http://disi.unitn.it/moschitti/corpora/ohsumed-all-docs.tar.gz
#tar xvf ohsumed-all-docs.tar.gz
cd ohsumed-all
filename=ohsumed_clusters.txt
rm -f $filename
for category in C*; do 
  echo %%%$category >> $filename
  for abstract in $category/*; do 
    tr '\n' ' ' < $abstract | ../tocorpus.pl >> $filename 
  done
done
mv $filename tmp.txt
awk '!seen[$0]++' tmp.txt > $filename # remove duplicate lines