Skip to content

Instantly share code, notes, and snippets.

@estevecastells
Last active October 14, 2023 13:57
Show Gist options
  • Star 2 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save estevecastells/579a2ecc08fbaeb10c9b2ea9d2b2e5ef to your computer and use it in GitHub Desktop.
Save estevecastells/579a2ecc08fbaeb10c9b2ea9d2b2e5ef to your computer and use it in GitHub Desktop.
Minified Bookmarklet - Visualize URLs blocked by Robots.txt in a page
javascript:!function(){let e=Array.from(document.querySelectorAll("a")).map(e=>new URL(e.href,window.location.origin).href).filter(e=>{try{return new URL(e).origin===window.location.origin&&!e.includes("#")}catch(t){return console.error(%60Error parsing URL: ${e}%60),!1}});e=Array.from(new Set(e));let t=window.location.origin+"/robots.txt";fetch("https://tools.estevecastells.com/api/bulk-robots-txt/v1",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({robots_txt_url:t,links:e})}).then(e=>e.json()).then(e=>{let o="box-"+Math.random().toString(36).substr(2,9),r=document.createElement("div");r.attachShadow({mode:"open"});let n=document.createElement("div");if(n.style="position:fixed;bottom:5px;right:5px;background-color:white;width:300px;padding:10px;border:1px solid grey;z-index:9999;",n.id=o,document.body.appendChild(r),r.shadowRoot.appendChild(n),e.error)n.innerHTML=%60<span style="position:absolute;right:5px;top:0px;cursor:pointer;padding:0 4px;background-color:white;" id="close">X</span> <p>${e.error}</p>%60,r.shadowRoot.getElementById("close").addEventListener("click",()=>{r.remove()});else{let i=[];for(let l in e)if(!e[l]){let s=document.querySelectorAll(%60a[href="${l}"]%60);if(0===s.length){let a=new URL(l),p=a.pathname+a.search+a.hash;s=document.querySelectorAll(%60a[href="${p}"]%60)}s.length>0&&(s.forEach(e=>{e.style.setProperty("background-color","red","important"),e.style.setProperty("color","black","important"),e.style.setProperty("border","2px solid black","important"),e.style.setProperty("padding","3px","important")}),i.push(l))}n.innerHTML+=%60<span style="position:absolute;right:5px;top:0px;cursor:pointer;padding:0 4px;background-color:white;" id="close">X</span> <p>Blocked URLs count: ${i.length}</p> <a href="${t}" target="_blank" style="display: block; margin-bottom: 10px;">Open robots.txt</a> <button id="copy" style="margin-bottom: 10px;">Copy Blocked URLs</button> <p><small>Note: Blocked by robots.txt links will be highlighted as red, if there is any.</small></p>%60,r.shadowRoot.getElementById("copy").addEventListener("click",()=>{navigator.clipboard.writeText(i.join("\n")).then(()=>{alert("Blocked URLs copied to clipboard.")})}),r.shadowRoot.getElementById("close").addEventListener("click",()=>{r.remove()})}}).catch(e=>{console.error(%60Error fetching or processing data: ${e}%60)})}();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment