Skip to content

Instantly share code, notes, and snippets.

@DrI-T
Last active May 28, 2021 15:11
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save DrI-T/27ce2878f50aaee5b9ab0ac1975adb4c to your computer and use it in GitHub Desktop.
Save DrI-T/27ce2878f50aaee5b9ab0ac1975adb4c to your computer and use it in GitHub Desktop.
Locally (distributed) Wikipedia

local wikipedia w/ IPFS

I have put the compete wikipedia in a gist; so you can consult it locally without censorship !

You need to have a local IPFS node running (see INSTALL)

  1. run an ipfs node
  2. browse the "distributed" wikipedia

local Wikipedia : gist:27ce2878f50aaee5b9ab0ac1975adb4c

Git repository: https://hologit-ml.ipns.dweb.link/afend/localWikipedia.git

Reference:

fetch('wikiChain.json').then(resp => resp.json())
.then(obj => { hashes = obj; })
.catch(console.warn)
.finally(_ => {
let els = document.getElementsByTagName('a');
for (let a of els) {
for (let qm of Object.keys(hashes).sort().reverse()) {
rex = RegExp(':'+qm+'\\b','g');
if (a.href.match(rex)) {
a.href = a.href.replace(rex,hashes[qm]);
console.debug(':'+qm+' -> ',a.href);
}
if (a.innerText.match(rex)) {
a.innerText = a.innerText.replace(rex,hashes[qm]);
console.debug(':'+qm+' -> ',a.innerText)
}
}
}
});
#
set -e
find . -name '*~1' -delete
symb='wikiChain'
# ipfs key gen -t rsa -s 3072 wikiChain
headkey=$(ipfs key list -l --ipns-base=b58mh | grep -w $symb | cut -d' ' -f1)
echo headkey: $headkey
head=$(ipfs resolve /ipns/$headkey)
qmhead=${head#*/ipfs/}
echo qmhead: $qmhead
gitrepo=$(ipfs resolve /ipns/hologit.ml/afend/localWikipedia.git)
qmgit=${gitrepo#*/ipfs/}
tic=$(date +%s%N)
tics=$(date +%s)
date=$(date +%D)
# Get immutable such that the wikipedia is "blockchained"
# xdg-opne https://www.google.com/search?hl=en&q=inurl:snapshot-hashes.yml
if [ ! -e snapshot-hashes.yml ]; then
curl -sL https://raw.githubusercontent.com/ipfs/distributed-wikipedia-mirror/main/snapshot-hashes.yml -o snapshot-hashes.yml
fi
pqm=$(tail -1 qm.log | cut -d' ' -f2)
qm=$(ipfs add snapshot-hashes.yml -Q)
if [ "x$qm" = "x$pqm" ]; then
echo $tics: $qm
echo no changes ...
exit 0;
fi
echo $tics: $qm >> qm.log;
qmlog=$(ipfs add qm.log -Q)
wiki_url=$(cat snapshot-hashes.yml | json_xs -f yaml -t string -e '$_ = $_->{en}{ipfs}')
echo wiki_url: $wiki_url
bafy=${wiki_url##*/}
qmprv=$(cat wikiChain.json | json_xs -t string -e '$_ = $_->{qmhead}')
idx=$(ipfs resolve /ipfs/$qmprv/index.html)
qmidx=${idx#*/ipfs/}
mv wikiHeadChain.html wikiPrevBlock.htm
sed -e "s/hashes.qmlog = '.*'/hashes.qmlog = '$qmlog'/" \
-e "s/hashes.qmhead = '.*'/hashes.qmhead = '$qmhead'/" \
-e "s/hashes.qmprv = '.*'/hashes.qmprv = '$qmprv'/" \
-e "s/hashes.qm = '.*'/hashes.qm = '$qm'/" wikiPrevBlock.htm > wikiHeadChain.html
qmprv=$(ipfs add -w wikiPrevBlock.htm wikiChain.json snapshot-hashes.yml qm.log style.css app.js -Q)
qmprv=$(ipfs object patch add-link $qmprv index.html $qmidx)
qmhc=$(ipfs add wikiHeadChain.html -Q)
qmhead=$(ipfs object patch add-link $qmprv index.html $qmhc)
echo qmhead: $qmhead
qmwiki=$(ipfs add -r . -Q)
cat > wikiChain.json <<EOT
{
"tic": $tic,
"date": "$date",
"headkey": "$headkey",
"wiki-url": "$wiki_url",
"repository": "https://hologit-ml.ipns.cf-ipfs.com/afend/localWikipedia.git",
"qmgit": "$qmgit",
"qmwiki": "$qmwiki",
"bafy": "$bafy",
"qm": "$qm",
"qmlog": "$qmlog",
"qmhead": "$qmhead",
"qmprv": "$qmprv"
}
EOT
gwport=$(ipfs config Addresses.Gateway | cut -d'/' -f 5)
qmwiki=$(ipfs add -r . -Q)
qmidx=$(sed -e "s/en\.wikipedia-on-ipfs\.org\.ipns/$bafy.ipfs/" \
-e "s/en-wikipedia--on--ipfs-org\.ipns/$bafy\.ipfs/" \
index.html | ipfs add -Q -)
qmidx=$(ipfs object patch add-link $qmwiki index.html $qmidx)
echo wikipedia_url: http://127.0.0.1:$gwport/ipfs/$qmidx
qmmut=$(ipfs object patch add-link $qmwiki index.html $qmhc)
qmmut=$(ipfs object patch add-link $qmmut wikipedia.html $qmidx)
echo wikichain_url: http://127.0.0.1:$gwport/ipfs/$qmmut
ipfs name publish --key=$headkey /ipfs/$qmmut
Display the source blob
Display the rendered blob
Raw
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
<!DOCTYPE html><meta charset=utf8>
<link rel=stylesheet href=style.css>
<style>
iframe {
margin:0;
width: 100%;
max-width: none;
height: calc(100vh - 2.4rem - 4px)
}
div { max-width: none; }
</style>
<header>
<img src=favicon.webp class=logo>
WikiWord: <input name=page value=""><button>view</button>
<div id=court>courtesy <a href="https://www.drit.ml/">Doctor I·T
<img src=drit-pin.svg alt=drit-pin class=logo></a>
</div>
</header>
<div class=modal>
<div class=modal-popup>
<h3>LOCALLY DISTRIBUTED WIKIPEDIA</h3>
<p>To access to the wikipedia locally you would need to run
<!--
echo "Your IPFS node is running locally on port 8080" | ipfs add --offline -Q
-- -->
an <b>IPFS node</b> on port <a href=http://127.0.0.1:8080/ipfs/QmeFswrU1ovgRxsiBnuAwMFbqjwDUk8rYpFHJstkEWF4P1>8080</a>.
Alternatively you can still browse this wikipedia via a public gateway, however it defeats the purpose
of having a "unstoppable" P2P version of the <a href="https://wikipedia.org">official wikipedia</a>.
<p>
To browse a specific page on wikipedia, please specify the page name in the "WikiWord" input field
and click the view button.
<p>To install <a href="">IPFS</a> follow the steps below:
<p><b>INSTALL</b>
<p>Run the following commands in a bash shell; this will install a docker instance of IPFS
and run the IPFS daemon within this docker.
<pre><code>
sudo apt-get curl # you need to have <em>curl</em> installed
curl -sL https://gist.github.com/DrI-T/882ae8a852ab2bc58a333e3c6f279d03/raw/one-liner-install.sh | sh -
</code></pre>
<button style="float:right" onclick="modal_close(0)">close</button>
<br>
</div>
</div>
<iframe src=""></iframe>
<script>
// console.info('hash:',location.hash, typeof location.hash)
var fragment = (typeof location.hash == 'undefined' || location.hash == '') ? 'IPFS': location.hash.slice(1);
var wkp_url='http://en.wikipedia-on-ipfs.org.ipns.localhost:8080/wiki/'
var iframe = document.getElementsByTagName('iframe')[0]
let input = document.getElementsByName('page')[0]
let button = document.getElementsByTagName('button')[0]
input.value = cap(fragment);
fetch(wkp_url+input.value,{method:'HEAD'})
.then(_ => { modal_close(0); })
.catch(async _ => {
await fetch('/ipns/en.wikipedia-on-ipfs.org/wiki/'+input.value,{method:'HEAD'})
.then(resp => {
if (resp.ok) {
wkp_url = '/ipns/en.wikipedia-on-ipfs.org/wiki/';
console.info('switching to wkp_url:',wkp_url);
modal_close(0);
} else {
wkp_url='https://en-wikipedia--on--ipfs-org.ipns.cf-ipfs.com/wiki/'
console.info('switching to wkp_url:',wkp_url);
}
})
.catch(_ => {
wkp_url='https://en-wikipedia--on--ipfs-org.ipns.dweb.link/wiki/'
console.info('switching to wkp_url:',wkp_url);
})
})
.finally(_ => { update_page({target:input})});
input.addEventListener('change',update_page,true);
button.addEventListener('click',e => { update_page({target:input}) },false);
function nop(ev) {
return void(0);
}
function modal_close(i) {
let popup = document.getElementsByClassName('modal')[i];
popup.style.display='none';
}
function update_page(ev) {
//let wikiWord = ev.target.value.split(' ').map(cap).join('');
let wikiWord = cap(ev.target.value.replace(/ /g,'_'))
console.log('wikiWord:',wikiWord);
iframe.src=wkp_url + wikiWord
return iframe.src
}
function cap(w) { return w.replace(/^\w/, x => x.toUpperCase()); }
</script>
<style>
.modal {
position: fixed;
top: 0;
left: 0;
z-index: 1025;
display: block;
width: 100%;
height: 100%;
overflow: hidden;
outline: 0;
}
.modal-popup {
max-width: 500px;
margin: 20vh auto;
background-color: white;
padding: 1rem;
box-shadow: 5px 5px 15px 2px rgba(4,3,5,0.9);
}
.modal-content {
position: relative;
width: 100%;
pointer-events: auto;
border: 1px solid #002b36;
border-radius: .3rem;
outline: 0;
}
</style>

INSTALL

Run the following commands in a bash shell; this will install a docker instance of IPFS and run the IPFS daemon within this docker.

sudo apt-get curl # you need to have <code>curl</code> installed

curl -sL https://gist.github.com/DrI-T/882ae8a852ab2bc58a333e3c6f279d03/raw/one-liner-install.sh | sh -

-- gist:882ae8a852ab2bc58a333e3c6f279d03

--- # QmLog for snapshot-hashes.yml (wikiChain)
1621930301: QmeXzmFVxvG5gHwfApadsCCfwJqkJXF9jupdJPzreiAssN
%YAML 1.2
---
# This is a stopgap measure for keeping track of hashes until we set up a better system.
en:
name: English
original: en.wikipedia.org
date: 2021-03-09
source: wikipedia_en_all_maxi_2021-02.zim
ipns:
ipfs: https://dweb.link/ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze
tr:
name: Turkish
original: tr.wikipedia.org
source: wikipedia_tr_all_maxi_2021-02.zim
date: 2021-02-19
ipns:
ipfs: https://dweb.link/ipfs/bafybeieuutdavvf55sh3jktq2dpi2hkle6dtmebe7uklod3ramihyf3xa4
my:
name: Myanmar
original: my.wikipedia.org
source: wikipedia_my_all_maxi_2021-02.zim
date: 2021-02-22
ipns:
ipfs: https://dweb.link/ipfs/bafybeib66xujztkiq7lqbupfz6arzhlncwagva35dx54nj7ipyoqpyozhy
ar:
name: Arabic
original: ar.wikipedia.org
source: wikipedia_ar_all_maxi_2021-03.zim
date: 2021-03-26
ipns:
ipfs: https://dweb.link/ipfs/bafybeih4a6ylafdki6ailjrdvmr7o4fbbeceeeuty4v3qyyouiz5koqlpi
zh:
name: Chinese
original: zh.wikipedia.org
source: wikipedia_zh_all_maxi_2021-02.zim
date: 2021-03-16
ipns:
ipfs: https://dweb.link/ipfs/bafybeiazgazbrj6qprr4y5hx277u4g2r5nzgo3jnxkhqx56doxdqrzms6y
ru:
name: Russian
original: ru.wikipedia.org
source: wikipedia_ru_all_maxi_2021-03.zim
date: 2021-03-25
ipns:
ipfs: https://dweb.link/ipfs/bafybeieto6mcuvqlechv4iadoqvnffondeiwxc2bcfcewhvpsd2odvbmvm
* { box-sizing: border-box; }
body {background-color: #415; margin:0; }
body > div:nth-child(1) {
margin-top: 10vh;
}
iframe {
display: block;
width: 100%;
max-width: 768px;
margin: auto;
margin-top: 4px;
box-sizing: border-box;
padding:0;
background-color: #edf;
border: 1px solid orange ;
height: calc(100vh - 2.4rem - 4px)
}
header {
display: flex;
justify-content: space-between;
align-items: center;
color: #213;
min-height: 2rem;
background-color: #1a9;
overflow: hidden;
padding-right: 0.8rem;
padding-left: 0.8rem;
}
pre,code {
overflow-x: scroll !important;
}
.logo {
display: inline-block;
vertical-align: middle;
height: 1.6rem;
}
#court {
display: inline;
margin-right: 10px;
text-align: right;
vertical-align: middle;
}
{
"tic": 1621930301711551161,
"date": "05/25/21",
"headkey": "Qmd4gaXBDzXPUxuwWDnyCZvhwi7TjbMqLh2heWcf62J9gJ",
"wiki-url": "https://dweb.link/ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze",
"repository": "https://hologit-ml.ipns.cf-ipfs.com/afend/localWikipedia.git",
"qmgit": "QmTXkLjkDNitkSttiMadSbxdgSZLbGLbNQX8hBGPAmnFyY",
"qmwiki": "QmNs8iMKao4QdT7TKaM2KvgPsaKfed8hKXzNLWABsZsaUZ",
"bafy": "bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze",
"qm": "QmeXzmFVxvG5gHwfApadsCCfwJqkJXF9jupdJPzreiAssN",
"qmlog": "QmPGiv6t2Pqy5pL66zQGuzK8UfMudvnhEAAGZZzd8AFQd1",
"qmhead": "QmYATxtANUKjVF7PyX2Hv3a76KZon2MRdE22LLwgbzN2aj",
"qmprv": "QmZPYADLvL8FfQCurMWHtYd8x1iWimsY6btiJUNdBCEQ34"
}
<!DOCTYPE html><meta charset="utf8">
<title>The wikiChain (the tamper Proof wikipedia)</title>
<link rel="stylesheet" href="style.css">
<style>
body > div {
max-width: 768px;
background-color: white;
margin: auto;
padding: 0.8rem;
box-shadow: 5px 5px 15px 2px rgba(4,3,5,0.9);
}
</style>
<script>
const mutable = 'Qm123';
var hashes = {};
hashes.headkey = mutable;
hashes.qmlog = 'QmPGiv6t2Pqy5pL66zQGuzK8UfMudvnhEAAGZZzd8AFQd1';
hashes.qmhead = 'QmcqzYQBQ1EuQycz4qCUHMjF85hYnQ5pj4UDp4DvSfuqgv'; // previous one need to be re-evaluated by app.js
hashes.qmprv = 'QmNq64roQD95tPiAsgDU9KCjh7c6kzbQNFXq6pkLwvCHdx';
hashes.qm = 'QmeXzmFVxvG5gHwfApadsCCfwJqkJXF9jupdJPzreiAssN'; // snapshot-hashes.yml
</script>
<div>
<h2>The wikiChain (the tamper Proof wikipedia)</h2>
<p>We put the <a href="https://dweb.link/ipfs/:qmwiki/">(uncensorable) wikipedia</a>
on the <a href="http://ipfs.blockring™.ml/ipfs/:qmhead/index.html">blockchain</a>
such that we'll learn from our mistakes
and won't repeat the past, as we will remember !
<br>--&nbsp;<br>
<p>~<a href=https://www.drit.ml/>Doctor I·T</a>
</div>
<br>
<div>
<h3>Cryptographic hashes :</h3>
<ul>
<li>qmwiki: <a href="https://ipfs.blockringtm.ml/ipfs/:qmwiki/#blockchain">:qmwiki</a>
<li>head: <a href="https://ipfs.blockringtm.ml/ipns/:headkey/">:headkey</a> (mutable)
<li>qm: <a href="https://gateway.ipfs.io/ipfs/:qm/">:qm</a> (snapshot-hashes.yml)
<li>log: <a href="qm.log">:qmlog</a> (qm.log)
<li>current: <a href="https://ipfs.blockringtm.ml/ipfs/:qmhead/">:qmhead</a>
<li>prev: <a href="http://127.0.0.1:8080/ipfs/:qmprv/">:qmprv</a>
</ul>
</div>
<br>
<iframe src=snapshot-hashes.yml></iframe>
<script src=app.js></script>
<!DOCTYPE html><meta charset="utf8">
<title>The wikiChain (the tamper Proof wikipedia)</title>
<link rel="stylesheet" href="style.css">
<style>
body > div {
max-width: 768px;
background-color: white;
margin: auto;
padding: 0.8rem;
box-shadow: 5px 5px 15px 2px rgba(4,3,5,0.9);
}
</style>
<script>
const mutable = 'Qm123';
var hashes = {};
hashes.headkey = mutable;
hashes.qmlog = 'QmNTtj9MdCT3d6sELuNqjnRyA4vAuqoUJvjFzJ5bVBX4Qx';
hashes.qmhead = 'QmNcPzwwzjvnizDT8APRe2bVMjksGG49iQYS9SWjmrvFcz'; // previous one need to be re-evaluated by app.js
hashes.qmprv = 'QmXEunhjyvuF3SWnLydqWUkXp8Ck68uSFr7nSuGUkLMnp2';
hashes.qm = 'QmeXzmFVxvG5gHwfApadsCCfwJqkJXF9jupdJPzreiAssN'; // snapshot-hashes.yml
</script>
<div>
<h2>The wikiChain (the tamper Proof wikipedia)</h2>
<p>We put the <a href="https://dweb.link/ipfs/:qmwiki/">(uncensorable) wikipedia</a>
on the <a href="http://ipfs.blockring™.ml/ipfs/:qmhead/index.html">blockchain</a>
such that we'll learn from our mistakes
and won't repeat the past, as we will remember !
<br>--&nbsp;<br>
<p>~<a href=https://www.drit.ml/>Doctor I·T</a>
</div>
<br>
<div>
<h3>Cryptographic hashes :</h3>
<ul>
<li>qmwiki: <a href="https://ipfs.blockringtm.ml/ipfs/:qmwiki/#blockchain">:qmwiki</a>
<li>head: <a href="https://ipfs.blockringtm.ml/ipns/:headkey/">:headkey</a> (mutable)
<li>qm: <a href="https://gateway.ipfs.io/ipfs/:qm/">:qm</a> (snapshot-hashes.yml)
<li>log: <a href="qm.log">:qmlog</a> (qm.log)
<li>current: <a href="https://ipfs.blockringtm.ml/ipfs/:qmhead/">:qmhead</a>
<li>prev: <a href="http://127.0.0.1:8080/ipfs/:qmprv/">:qmprv</a>
</ul>
</div>
<br>
<iframe src=snapshot-hashes.yml></iframe>
<script src=app.js></script>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment