kris-sigur

#!/bin/bash

OUTBACK_INDEX_URL='http://localhost:8080/lbs?badLines=skip'

# FUNCTIONS
containsElement () {
  local e
  for e in "${@:2}"; do [[ "$e" == "$1" ]] && return 0; done
  return 1
}

kris-sigur

package is.landsbokasafn.crawler.reporting;

import java.util.Date;

import org.archive.crawler.event.StatSnapshotEvent;
import org.archive.crawler.framework.CrawlController;
import org.archive.crawler.reporting.CrawlStatSnapshot;
import org.archive.crawler.reporting.StatisticsTracker;
import org.archive.util.ArchiveUtils;
import org.archive.util.PaddingStringBuffer;

kris-sigur

package is.landsbokasafn.crawler.deciderules;

import static org.archive.modules.CoreAttributeConstants.A_DNS_SERVER_IP_LABEL;

import java.net.InetAddress;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.logging.Level;

kris-sigur

package is.landsbokasafn.crawler.deciderules;

import static org.archive.modules.CoreAttributeConstants.A_DNS_SERVER_IP_LABEL;

import java.net.InetAddress;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;