autoscroll-fetch-links-snippet.js

// IN DEV CONSOLE
// go to Sources, Snippets
// FOR Pocket

const uniqueURLs = new Set();
let isScrolling = false;
let lastLoggedCount = 0; // To avoid logging the same count multiple times

function startAutoScroll() {
  isScrolling = true;
  const scrollInterval = setInterval(() => {
    if (!isScrolling) {
      clearInterval(scrollInterval);
    } else {
      window.scrollBy(0, 500); // Scroll 500 pixels every second
      extractURLs();
    }
  }, 1000);
}

function stopAutoScroll() {
  isScrolling = false;
}

function extractURLs() {
  const urls = document.querySelectorAll('footer.footer cite.details a.publisher');
  for (let url of urls) {
    uniqueURLs.add(url.href);
  }

  const size = uniqueURLs.size;
  if (size !== lastLoggedCount && size % 100 <= 10) {
    console.log(`URL count is around ${Math.floor(size / 100) * 100}, current count: ${size}`);
    lastLoggedCount = size;
    console.log({uniqueURLs});
  }
}

// Usage: Start and stop scrolling as needed
startAutoScroll(); // Start scrolling
// stopAutoScroll(); // Call this to stop scrolling

// Access unique URLs
function getUniqueURLs() {
  return Array.from(uniqueURLs);
}

console.log(getUniqueURLs());