nextdns.io Block Youtube Ads

nextdns.js

// ID of the config, e.g. A1234BCD.
const configID = "A1234BCD";
// API key, found at the bottom of your account page in https://my.nextdns.io/account
const APIKey = "xxxxxxxxxxxxxxxxxxxxxxxxxx";
// Mark true or false. If true, failed links will be retried 3 times at progressively increasing intervals.
// If false, failed links will not be retried.
const retryFailedLinks = true;
// Time delay between requests in milliseconds.
// 800 seems to not give any errors but is rather slow while anything faster will give errors (in limited testing).
// If you want to go lower, it is recommended that "retryFailedLinks" is true
const timeDelay = 200;

const ignoreDomainsSet = new Set([
  "clients.l.google.com",
  "clients1.google.com",
  "clients2.google.com",
  "clients3.google.com",
  "clients4.google.com",
  "clients5.google.com",
  "clients6.google.com",
  "akamaiedge.net",
]);

const youtubeAdDomainsSet = new Set();
const existingLinkSet = new Set();

const fetchEwprattenDomains = async () => {
  const response = await fetch(
    "https://raw.githubusercontent.com/Ewpratten/youtube_ad_blocklist/master/blocklist.txt"
  );
  const text = await response.text();
  text.split("\n").forEach((line) => youtubeAdDomainsSet.add(line));
  return;
};

const fetchkboghdadyDomains = async () => {
  const response = await fetch(
    "https://raw.githubusercontent.com/kboghdady/youTube_ads_4_pi-hole/master/youtubelist.txt"
  );
  const text = await response.text();
  text.split("\n").forEach((line) => youtubeAdDomainsSet.add(line));
  return;
};

const fetchGoodbyeAdsDomains = async () => {
  const response = await fetch(
    "https://raw.githubusercontent.com/jerryn70/GoodbyeAds/master/Formats/GoodbyeAds-YouTube-AdBlock-Filter.txt"
  );
  const text = await response.text();
  text.split("\n").forEach((line) => {
    if (line.startsWith("||") && line.endsWith("^")) {
      const domain = line.substring(2, line.length - 1);
      youtubeAdDomainsSet.add(domain);
    }
  });
  return;
};

const fetchExistingLinks = async () => {
  const response = await fetch(
    `https://api.nextdns.io/profiles/${configID}/denylist`,
    {
      headers: {
        accept: "application/json, text/plain, */*",
        "content-type": "application/json",
        "X-Api-Key": JSON.stringify(APIKey),
      },
      method: "GET",
      mode: "cors",
      credentials: "include",
    }
  );
  const domains = await response.json();
  domains.data.map((domain) => {
    existingLinkSet.add(domain.id);
  });
  return;
};

function sleep(milliseconds) {
  return new Promise((resolve) => setTimeout(resolve, milliseconds));
}

const blockDomain = async (domain) =>
  fetch(`https://api.nextdns.io/profiles/${configID}/denylist`, {
    headers: {
      accept: "application/json, text/plain, */*",
      "content-type": "application/json",
      "X-Api-Key": JSON.stringify(APIKey),
    },
    body: JSON.stringify({ id: domain, active: true }),
    method: "POST",
    mode: "cors",
    credentials: "include",
  }).then((response) => {
    if (response.ok) {
      return response.json;
    }
    throw "Failed to block domain";
  });

const blockDomains = async (domains, delay = timeDelay) => {
  console.log(`Preparing to block ${domains.length} domains`);

  const failedLinksSet = new Set();

  for (let idx = 0; idx < domains.length; idx++) {
    const domain = domains[idx];

    if (ignoreDomainsSet.has(domain)) {
      console.log(`Ignoring ${domain} ${idx + 1}/${domains.length}`);
      continue;
    }

    if (existingLinkSet.has(domain)) {
      console.log(`Skipping ${domain} ${idx + 1}/${domains.length}`);
      continue;
    }

    try {
      console.log(`Blocking ${domain} ${idx + 1}/${domains.length}`);
      await blockDomain(domain);
    } catch (error) {
      console.error(error);
      failedLinksSet.add(domain);
    }
    await sleep(delay);
  }

  return Array.from(failedLinksSet);
};

const stringToHex = (str) =>
  str
    .split("")
    .map((c) => c.charCodeAt(0).toString(16).padStart(2, "0"))
    .join("");

const unblockDomain = async (domain) =>
  fetch(
    `https://api.nextdns.io/profiles/${configID}/denylist/hex:${stringToHex(
      domain
    )}`,
    {
      headers: {
        accept: "application/json, text/plain, */*",
        "content-type": "application/json",
        "X-Api-Key": JSON.stringify(APIKey),
      },
      body: null,
      method: "DELETE",
      mode: "cors",
      credentials: "include",
    }
  );

const unblockDomains = async (domains, delay = timeDelay) => {
  console.log(`Preparing to unblock ${domains.length} domains`);
  for (let idx = 0; idx < domains.length; idx++) {
    try {
      console.log(`Unblocking ${domains[idx]} ${idx}/${domains.length}`);
      await unblockDomain(domains[idx]);
    } catch (error) {
      console.error(error);
    }

    await sleep(delay);
  }
};

const retry = async (fn, initialInput, retries, delayMs) => {
  let attempt = 0;
  let delay = timeDelay;
  let nextInput = initialInput;

  while (true) {
    nextInput = await fn(nextInput, delay);
    if (nextInput.size == 0) {
      return;
    }

    console.log(`Retry ${attempt}/${retries} failed. Retrying again...`);

    if (attempt++ > retries) {
      console.error("Failed domains", nextInput);
      throw "Retries exceeded";
    }

    delay += delayMs * attempt;
  }
};

const run = async () => {
  console.log(`Downloading domains to block ...`);
  await fetchEwprattenDomains();
  await fetchkboghdadyDomains();
  await fetchGoodbyeAdsDomains();
  await fetchExistingLinks();

  const youtubeAdDomains = Array.from(youtubeAdDomainsSet);

  const retries = retryFailedLinks ? 3 : 0;
  await retry(blockDomains, youtubeAdDomains, retries, 100);

  // await unblockDomains(Array.from(existingLinkSet));

  console.log("Have fun!");
};

run();

Probably not, but it is so cheap, great to support.

…

On Feb 12, 2022, at 8:18 AM, ICeZer0 ***@***.***> wrote: ***@***.*** commented on this gist. I'm guessing this will not work on the free trial of nextDNS? I get 404's — Reply to this email directly, view it on GitHub, or unsubscribe. Triage notifications on the go with GitHub Mobile for iOS or Android. You are receiving this because you commented.

I just signed up, the 404's was my fault, I didn't change the configId properly.

Check this:
https://github.com/aguyonp/NextDNS-BlockYoutubeAds

can this work on pihole?

No, because it use the NextDNS API. But you can add manualy execption list of domain with pihole.

Check this: https://github.com/aguyonp/NextDNS-BlockYoutubeAds

Does it have to take forever? I thought it would just load it in a snap, but it's loading page by page, second by second.

Check this: https://github.com/aguyonp/NextDNS-BlockYoutubeAds

Does it have to take forever? I thought it would just load it in a snap, but it's loading page by page, second by second.

We are forced to issue requests second by second. Otherwise the NEXTDNS API will block our IP for a while. Let the operation run in a container on a VPS, a dedicated server or a Raspberry PI.

I just signed up, the 404's was my fault, I didn't change the configId properly.

@ICeZer0 how did you configure the configId variable?
I am selecting the config then copying the string from the Setup page so it looks like this: const configID = "111abc";

Still getting 404's though.

I just signed up, the 404's was my fault, I didn't change the configId properly.

@ICeZer0 how did you configure the configId variable? I am selecting the config then copying the string from the Setup page so it looks like this: const configID = "111abc";

Still getting 404's though.

NextDNS made changes to their API. Below is updated code.

Edit: I didn't know a thing about JS before I started this, so if anyone sees anything wrong or that could be improved, feel free to make the changes.

Edit2: If you encounter lag, your computer hangs up, or for some other reason you need to stop the process, that's okay. Running the code again will read your deny list first and skip anything already on it.

const configID = "ABC123";  //the ID of the config, e.g. A1234BCD. Keep the parentheses
const APIKey = "YOURAPIKEY"; //your API key, found at the bottom of your account page. Keep the parentheses. https://my.nextdns.io/account
const retryFailedLinks = "Y";  //Mark "Y" or "N". If "Y", failed links will be retried 4 times at progressively increasing invtervals. If "N", failed links will not be retried. Keep the parentheses
const timeDelay = 800; //time delay between requests in milliseconds. 800 seems to not give any errors but is rather slow while anything faster will give errors (in limited testing). If you want to go lower, it is recommended that "retryFailedLinks" is marked "Y"

const ignoreDomainsSet = new Set([
  "clients.l.google.com",
  "clients1.google.com",
  "clients2.google.com",
  "clients3.google.com",
  "clients4.google.com",
  "clients5.google.com",
  "clients6.google.com",
  "akamaiedge.net",
]);

const youtubeAdDomainsSet = new Set();
const existingLinkSet = new Set();
const failedLinksSet = new Set();
const failedLinksSet2 = new Set();


const fetchEwprattenDomains = async () => {
  const response = await fetch(
    "https://raw.githubusercontent.com/Ewpratten/youtube_ad_blocklist/master/blocklist.txt"
  );
  const text = await response.text();
  text.split("\n").forEach((line) => youtubeAdDomainsSet.add(line));
  return;
};

const fetchkboghdadyDomains = async () => {
  const response = await fetch(
    "https://raw.githubusercontent.com/kboghdady/youTube_ads_4_pi-hole/master/youtubelist.txt"
  );
  const text = await response.text();
  text.split("\n").forEach((line) => youtubeAdDomainsSet.add(line));
  return;
};

const fetchGoodbyeAdsDomains = async () => {
  const response = await fetch(
    "https://raw.githubusercontent.com/jerryn70/GoodbyeAds/master/Formats/GoodbyeAds-YouTube-AdBlock-Filter.txt"
  );
  const text = await response.text();
  text.split("\n").forEach((line) => {
    if (line.startsWith("||") && line.endsWith("^")) {
      const domain = line.substring(2, line.length - 1);
      youtubeAdDomainsSet.add(domain);
    }
  });
  return;
};

const fetchExistingLinks = async() => {
  const response = await fetch(
    `https://api.nextdns.io/profiles/${configID}/denylist`,
    {
      headers: {
        "accept": "application/json, text/plain, */*",
        "content-type": "application/json",
        "X-Api-Key": JSON.stringify(APIKey),
      },
      method: "GET",
      mode: "cors",
      credentials: "include",
    }
  );
  const text = await response.text();
  text.split("{").forEach((line) => {
    if (line.startsWith("\"id\"") && line.endsWith("\}\,")){
      const oldLink = line.substring(6, line.length - 17);
      existingLinkSet.add(oldLink);
    }
  })
  return;
};


function sleep (milliseconds) {
    return new Promise((resolve) => setTimeout(resolve, milliseconds))
}

const blockDomain = async (domain) =>
  fetch(
    `https://api.nextdns.io/profiles/${configID}/denylist`,
    {
      headers: {
        "accept": "application/json, text/plain, */*",
        "content-type": "application/json",
        "X-Api-Key": JSON.stringify(APIKey),
      },
      body: JSON.stringify({id: domain, active: true}),
      method: "POST",
      mode: "cors",
      credentials: "include",
    }
  ).then((response) => {
    if (response.ok) {
      return (response.json)
    }
    throw "Error, going to reattempt 3 times at progressively increasing intervals";
  });

const retryFailed = async() => {
  const failedLinksArray = Array.from(failedLinksSet);
  for (x = 0; x < failedLinksArray.length; x++) {
    try {
      console.log(
        `Retrying to Block ${failedLinksArray[x]} Attempt 1/3 | ${x+1}/${failedLinksArray.length}`
      );
      await blockDomain(failedLinksArray[x]);
      var retryCount = 1
    } catch (error) {
      console.error(error);
      for (var retryCount=2; retryCount < 4; retryCount++){
        try {
          console.log(
            `Retrying to Block ${failedLinksArray[x]} Attempt ${retryCount}/3 | ${x+1}/${failedLinksArray.length}`
          );
        await sleep(5000*(retryCount)); 
        await blockDomain(failedLinksArray[x]);
        }
        catch (error) {
          console.error(error);
          if(retryCount==3){
            failedLinksSet2.add(failedLinksArray[x])
          };
      };
    };
  };
};
};

const blockDomains = async () => {
  console.log(`Downloading domains to block ...`);
  await fetchEwprattenDomains(); //I recommend starting with only this list; you can comment out the the two lines below by preceding them with two backwards slashes, as can be seen preceding this comment.
  await fetchkboghdadyDomains();
  await fetchGoodbyeAdsDomains();
  await fetchExistingLinks();
  const youtubeAdDomains = Array.from(youtubeAdDomainsSet);
  console.log(`Preparing to block ${youtubeAdDomains.length} domains`);
    for (let idx = 0; idx < 1300; idx++) {
    if (ignoreDomainsSet.has(youtubeAdDomains[idx])) {
      console.log(
        `Skipping ${youtubeAdDomains[idx]} ${idx}/${youtubeAdDomains.length}`
      );
      continue;
    }

    if (existingLinkSet.has(youtubeAdDomains[idx])) {
      console.log(
        `Skipping ${youtubeAdDomains[idx]} ${idx+1}/${youtubeAdDomains.length}`
      );
      continue;
    }

    try {
      console.log(
        `Blocking ${youtubeAdDomains[idx]} ${idx+1}/${youtubeAdDomains.length}`
      );
      await blockDomain(youtubeAdDomains[idx]);
    } catch (error) {
      console.error(error);
      failedLinksSet.add(youtubeAdDomains[idx])
    }
    await sleep(timeDelay);
  }
  if (retryFailedLinks=="Y"){
    await retryFailed();
    console.log("Failed Links:", failedLinksSet2);
  };
  console.log("Have fun!");
};


blockDomains();

I just signed up, the 404's was my fault, I didn't change the configId properly.

@ICeZer0 how did you configure the configId variable? I am selecting the config then copying the string from the Setup page so it looks like this: const configID = "111abc";
Still getting 404's though.

NextDNS made changes to their API. Below is updated code.

Edit: I didn't know a thing about JS before I started this, so if anyone sees anything wrong or that could be improved, feel free to make the changes.

Edit2: If you encounter lag, your computer hangs up, or for some other reason you need to stop the process, that's okay. Running the code again will read your deny list first and skip anything already on it.

const configID = "ABC123";  //the ID of the config, e.g. A1234BCD. Keep the parentheses
const APIKey = "YOURAPIKEY"; //your API key, found at the bottom of your account page. Keep the parentheses. https://my.nextdns.io/account
const retryFailedLinks = "Y";  //Mark "Y" or "N". If "Y", failed links will be retried 4 times at progressively increasing invtervals. If "N", failed links will not be retried. Keep the parentheses
const timeDelay = 800; //time delay between requests in milliseconds. 800 seems to not give any errors but is rather slow while anything faster will give errors (in limited testing). If you want to go lower, it is recommended that "retryFailedLinks" is marked "Y"

const ignoreDomainsSet = new Set([
  "clients.l.google.com",
  "clients1.google.com",
  "clients2.google.com",
  "clients3.google.com",
  "clients4.google.com",
  "clients5.google.com",
  "clients6.google.com",
  "akamaiedge.net",
]);

const youtubeAdDomainsSet = new Set();
const existingLinkSet = new Set();
const failedLinksSet = new Set();
const failedLinksSet2 = new Set();


const fetchEwprattenDomains = async () => {
  const response = await fetch(
    "https://raw.githubusercontent.com/Ewpratten/youtube_ad_blocklist/master/blocklist.txt"
  );
  const text = await response.text();
  text.split("\n").forEach((line) => youtubeAdDomainsSet.add(line));
  return;
};

const fetchkboghdadyDomains = async () => {
  const response = await fetch(
    "https://raw.githubusercontent.com/kboghdady/youTube_ads_4_pi-hole/master/youtubelist.txt"
  );
  const text = await response.text();
  text.split("\n").forEach((line) => youtubeAdDomainsSet.add(line));
  return;
};

const fetchGoodbyeAdsDomains = async () => {
  const response = await fetch(
    "https://raw.githubusercontent.com/jerryn70/GoodbyeAds/master/Formats/GoodbyeAds-YouTube-AdBlock-Filter.txt"
  );
  const text = await response.text();
  text.split("\n").forEach((line) => {
    if (line.startsWith("||") && line.endsWith("^")) {
      const domain = line.substring(2, line.length - 1);
      youtubeAdDomainsSet.add(domain);
    }
  });
  return;
};

const fetchExistingLinks = async() => {
  const response = await fetch(
    `https://api.nextdns.io/profiles/${configID}/denylist`,
    {
      headers: {
        "accept": "application/json, text/plain, */*",
        "content-type": "application/json",
        "X-Api-Key": JSON.stringify(APIKey),
      },
      method: "GET",
      mode: "cors",
      credentials: "include",
    }
  );
  const text = await response.text();
  text.split("{").forEach((line) => {
    if (line.startsWith("\"id\"") && line.endsWith("\}\,")){
      const oldLink = line.substring(6, line.length - 17);
      existingLinkSet.add(oldLink);
    }
  })
  return;
};


function sleep (milliseconds) {
    return new Promise((resolve) => setTimeout(resolve, milliseconds))
}

const blockDomain = async (domain) =>
  fetch(
    `https://api.nextdns.io/profiles/${configID}/denylist`,
    {
      headers: {
        "accept": "application/json, text/plain, */*",
        "content-type": "application/json",
        "X-Api-Key": JSON.stringify(APIKey),
      },
      body: JSON.stringify({id: domain, active: true}),
      method: "POST",
      mode: "cors",
      credentials: "include",
    }
  ).then((response) => {
    if (response.ok) {
      return (response.json)
    }
    throw "Error, going to reattempt 3 times at progressively increasing intervals";
  });

const retryFailed = async() => {
  const failedLinksArray = Array.from(failedLinksSet);
  for (x = 0; x < failedLinksArray.length; x++) {
    try {
      console.log(
        `Retrying to Block ${failedLinksArray[x]} Attempt 1/3 | ${x+1}/${failedLinksArray.length}`
      );
      await blockDomain(failedLinksArray[x]);
      var retryCount = 1
    } catch (error) {
      console.error(error);
      for (var retryCount=2; retryCount < 4; retryCount++){
        try {
          console.log(
            `Retrying to Block ${failedLinksArray[x]} Attempt ${retryCount}/3 | ${x+1}/${failedLinksArray.length}`
          );
        await sleep(5000*(retryCount)); 
        await blockDomain(failedLinksArray[x]);
        }
        catch (error) {
          console.error(error);
          if(retryCount==3){
            failedLinksSet2.add(failedLinksArray[x])
          };
      };
    };
  };
};
};

const blockDomains = async () => {
  console.log(`Downloading domains to block ...`);
  await fetchEwprattenDomains(); //I recommend starting with only this list; you can comment out the the two lines below by preceding them with two backwards slashes, as can be seen preceding this comment.
  await fetchkboghdadyDomains();
  await fetchGoodbyeAdsDomains();
  await fetchExistingLinks();
  const youtubeAdDomains = Array.from(youtubeAdDomainsSet);
  console.log(`Preparing to block ${youtubeAdDomains.length} domains`);
    for (let idx = 0; idx < 1300; idx++) {
    if (ignoreDomainsSet.has(youtubeAdDomains[idx])) {
      console.log(
        `Skipping ${youtubeAdDomains[idx]} ${idx}/${youtubeAdDomains.length}`
      );
      continue;
    }

    if (existingLinkSet.has(youtubeAdDomains[idx])) {
      console.log(
        `Skipping ${youtubeAdDomains[idx]} ${idx+1}/${youtubeAdDomains.length}`
      );
      continue;
    }

    try {
      console.log(
        `Blocking ${youtubeAdDomains[idx]} ${idx+1}/${youtubeAdDomains.length}`
      );
      await blockDomain(youtubeAdDomains[idx]);
    } catch (error) {
      console.error(error);
      failedLinksSet.add(youtubeAdDomains[idx])
    }
    await sleep(timeDelay);
  }
  if (retryFailedLinks=="Y"){
    await retryFailed();
    console.log("Failed Links:", failedLinksSet2);
  };
  console.log("Have fun!");
};


blockDomains();

** Did not work :(**
At https://api.nextdns.io/profiles/******/denylist it adds a list to block, but I can access it all the time. Maybe it is about a page parameter?

Thanks ducktapeonmydesk. That updated code did the trick. For anyone else reading, I changed:
for (let idx = 0; idx < 1300; idx++) {

to:
for (let idx = 0; idx < youtubeAdDomains.length; idx++) {

so it adds all the domains.

TEST:
not work... I can access it all the time

up

Updated

Thanks ducktapeonmydesk. That updated code did the trick. For anyone else reading, I changed: for (let idx = 0; idx < 1300; idx++) {

to: for (let idx = 0; idx < youtubeAdDomains.length; idx++) {

so it adds all the domains.

Whoops, it should have been that! I was doing some testing with the retries and didn't want to wait for the entire thing. Glad it worked for you. But it looks to have been updated and written better by someone anyways :)

Is this actually working for anyone?
I have run the script and the addresses are added to the denylist in NextDNS - but they don't block anything. Youtube shows just as many ads as before.

same problem i only stuck denylist and browser crashing when im trying to enter on that page. also there is no bulk remove so its pain in the ass to remove few thousands links manualy pffff

same problem i only stuck denylist and browser crashing when im trying to enter on that page. also there is no bulk remove so its pain in the ass to remove few thousands links manualy pffff

To remove, comment out line 203, uncomment line 205.

ut line 203, uncomment line 205.

tnx man

same problem i only stuck denylist and browser crashing when im trying to enter on that page. also there is no bulk remove so its pain in the ass to remove few thousands links manualy pffff

To remove, comment out line 203, uncomment line 205.

Thank you! glad to know there is an easy way to undo this :)
by the way, why no one creates an official list and add it to NextDNS that is only and specifically for blocking YouTube ads? why do we have to do it like this?

This is a spiffy way to 'subscribe' to lists on demand. I had been hoping I could chain decloudus (DC) to nextdns (ND) for pattern matching on DC and subscriptions on ND. DC profile allows specifying an upstream resolver in ipv4 or ipv6. ND profile provides unique ipv6, but DC rejects as invalid.

Could the browser be crashing because the whole list is grabbed instead of chunks? or all lists into ram?

Are we using the posted gist or the ducktapeonmydesk + MarkDarwin mod?

This is a spiffy way to 'subscribe' to lists on demand. I had been hoping I could chain decloudus (DC) to nextdns (ND) for pattern matching on DC and subscriptions on ND. DC profile allows specifying an upstream resolver in ipv4 or ipv6. ND profile provides unique ipv6, but DC rejects as invalid.

Could the browser be crashing because the whole list is grabbed instead of chunks? or all lists into ram?

Are we using the posted gist or the ducktapeonmydesk + MarkDarwin mod?

The posted gist. Each list is being grabbed, combined into one "master" list, and then each URL is added to the deny list in NextDNS. Lists remain true, no manipulation.

The only "mod" I had was commenting out certain lists so they wouldn't be run.

For testing purposes, because I didn't want to wait for ~100k lines to be added to the deny list, I had it set to "finish" at 1300 lines. I forgot to change the code back to run the entire list before posting. MarkDarwin caught my mistake and fixed it.

As far as browser crashing: Each list is grabbed in it's entirety before being combined. The amount of data could be causing your browser to crash. If you want to test it with only one of the lists, you can comment out any combination of lines 195 through 197.

Sadly, doesn’t seem to work for the iOS, iPadOS and tvOS apps.

Sadly, doesn’t seem to work for the iOS, iPadOS and tvOS apps.

no need to use those awful OSes. use Windows or Android only.

Sadly, doesn’t seem to work for the iOS, iPadOS and tvOS apps.

no need to use those awful OSes. use Windows or Android only.

Thank you for your unwanted opinion :)