Last active
October 14, 2023 13:57
-
-
Save estevecastells/579a2ecc08fbaeb10c9b2ea9d2b2e5ef to your computer and use it in GitHub Desktop.
Minified Bookmarklet - Visualize URLs blocked by Robots.txt in a page
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
javascript:!function(){let e=Array.from(document.querySelectorAll("a")).map(e=>new URL(e.href,window.location.origin).href).filter(e=>{try{return new URL(e).origin===window.location.origin&&!e.includes("#")}catch(t){return console.error(%60Error parsing URL: ${e}%60),!1}});e=Array.from(new Set(e));let t=window.location.origin+"/robots.txt";fetch("https://tools.estevecastells.com/api/bulk-robots-txt/v1",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({robots_txt_url:t,links:e})}).then(e=>e.json()).then(e=>{let o="box-"+Math.random().toString(36).substr(2,9),r=document.createElement("div");r.attachShadow({mode:"open"});let n=document.createElement("div");if(n.style="position:fixed;bottom:5px;right:5px;background-color:white;width:300px;padding:10px;border:1px solid grey;z-index:9999;",n.id=o,document.body.appendChild(r),r.shadowRoot.appendChild(n),e.error)n.innerHTML=%60<span style="position:absolute;right:5px;top:0px;cursor:pointer;padding:0 4px;background-color:white;" id="close">X</span> <p>${e.error}</p>%60,r.shadowRoot.getElementById("close").addEventListener("click",()=>{r.remove()});else{let i=[];for(let l in e)if(!e[l]){let s=document.querySelectorAll(%60a[href="${l}"]%60);if(0===s.length){let a=new URL(l),p=a.pathname+a.search+a.hash;s=document.querySelectorAll(%60a[href="${p}"]%60)}s.length>0&&(s.forEach(e=>{e.style.setProperty("background-color","red","important"),e.style.setProperty("color","black","important"),e.style.setProperty("border","2px solid black","important"),e.style.setProperty("padding","3px","important")}),i.push(l))}n.innerHTML+=%60<span style="position:absolute;right:5px;top:0px;cursor:pointer;padding:0 4px;background-color:white;" id="close">X</span> <p>Blocked URLs count: ${i.length}</p> <a href="${t}" target="_blank" style="display: block; margin-bottom: 10px;">Open robots.txt</a> <button id="copy" style="margin-bottom: 10px;">Copy Blocked URLs</button> <p><small>Note: Blocked by robots.txt links will be highlighted as red, if there is any.</small></p>%60,r.shadowRoot.getElementById("copy").addEventListener("click",()=>{navigator.clipboard.writeText(i.join("\n")).then(()=>{alert("Blocked URLs copied to clipboard.")})}),r.shadowRoot.getElementById("close").addEventListener("click",()=>{r.remove()})}}).catch(e=>{console.error(%60Error fetching or processing data: ${e}%60)})}(); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment