gistfile1.txt

// Check if the body is empty
			if(doc.body() != null) {
				// Find all text on the page
				final String text = doc.body().text();
				
				// Split the text into tokens
				final ArrayList<String> tokens = new ArrayList<>();
				final StringTokenizer tokenizer = new StringTokenizer(text);
				
				// Add all valid tokens
				while(tokenizer.hasMoreTokens()) {
					final String token = tokenizer.nextToken();
					if(!Constants.stopWords.contains(token)) {
						// Add the stemmed token
						tokens.add(Porter.stem(token));
					}
				}
				
				// Add all the words to the index
				for (String term : tokens) {
					Index.addTerm(term, url);
				}
				
				// System.out.println("Found " + tokens.size() + " terms.");
			}