Extract data from wiki-pedia

README.md

Various snippets to extract data from pages (wiki pedia)

• extractMilitarySpending.js
• extractPopulation.js

extractMilitarySpending.js

var csv = '';
Array.from(document.querySelectorAll('#mw-content-text > div > table:nth-child(9) > tbody > tr:nth-child(2) > td:nth-child(1) > table thead tr')).map((tr,i) => {
    var td = Array.from(tr.querySelectorAll('th')).map(td => td.textContent);
    csv+=td.join(';')+'\n'
});
Array.from(document.querySelectorAll('#mw-content-text > div > table:nth-child(9) > tbody > tr:nth-child(2) > td:nth-child(1) > table tbody tr')).map((tr,i) => {
  var td = Array.from(tr.querySelectorAll('td')).map(td => td.textContent);

  td[2] = parseFloat(td[2].toString().replace(/,/g, '.'))*1000000000;  
  csv+=td.join(';')+'\n'
});
copy(csv)

extractPopulation.js

// https://en.wikipedia.org/wiki/List_of_countries_and_dependencies_by_population
var csv = '';
Array.from(document.querySelectorAll('#mw-content-text > div > table:nth-child(9) > tbody > tr:nth-child(2) > td:nth-child(1) > table thead tr')).map((tr,i) => {
    var td = Array.from(tr.querySelectorAll('th')).map(td => td.textContent);
    csv+=td.join(';')+'\n'
});
var csv = '';
Array.from(document.querySelectorAll('#mw-content-text > div > table.wikitable.sortable.jquery-tablesorter tbody tr')).map((tr,i) => {
  var tds = Array.from(tr.querySelectorAll('td')).map(td => td.textContent);
  if (tds[0].indexOf('–')===-1) {
    tds[2] = parseFloat(tds[2].toString().replace(/,/g, ''));  
    tds = tds.filter((td, i) => [1,2].indexOf(i)!=-1)
    csv+=tds.join(';')+'\n'
  }
});
copy(csv.replace(/\[.*\]/ig, ''))


var rows = csv.split(`
`);
var population = {};
var headers = ['Country','Population'];
rows.map((cols, i) => {
   var col = cols.split(';');
   population[col[0].trim()] = col[1];
})
console.log(population)

militarySpendings.csv

Rank;Country;Spending $; % of GDP;World total;1686000000;2.2
01; United States;611200000000;3.3
02; People's Republic of China[a];215700000000;1.9
03; Russia;69200000000;5.3
04; Saudi Arabia[a][b];63700000000;10
05; India;55900000000;2.5
06; France;55700000000;2.3
07; United Kingdom;48300000000;1.9
08; Japan;46100000000;1.0
09; Germany;41100000000;1.2
10; South Korea;36800000000;2.7
11; Italy;27900000000;1.5
12; Australia;24300000000;2.0
13; Brazil;22800000000;1.3
14; United Arab Emirates[c];22800000000;5.7
15; Israel;17800000000;5.8
16; Canada;15500000000;1.0
17; Spain;14900000000;1.2
18; Turkey;14900000000;2.0
19; Iran;12300000000;3.0
20; Algeria;10600000000;6.7
21; Singapore;9900000000;3.4
22; Republic of China (Taiwan);9900000000;1.9
23; Pakistan;9900000000;3.4
24; Colombia;9900000000;3.4
25; Poland;9700000000;2.0
26; Netherlands;9200000000;1.2
27; Oman;9000000000;16.7
28; Indonesia;7700000000;0.9
29; Mexico;6800000000;0.6
30; Kuwait;6300000000;6.5

militarySpendings.json

{
  "United States": { "net": 611200000000, "pct": 3.3 },
  "People's Republic of China[a]": { "net": 215700000000, "pct": 1.9 },
  "Russia": { "net": 69200000000, "pct": 5.3 },
  "Saudi Arabia[a][b]": { "net": 63700000000, "pct": 10 },
  "India": { "net": 55900000000, "pct": 2.5 },
  "France": { "net": 55700000000, "pct": 2.3 },
  "United Kingdom": { "net": 48300000000, "pct": 1.9 },
  "Japan": { "net": 46100000000, "pct": 1 },
  "Germany": { "net": 41100000000, "pct": 1.2 },
  "South Korea": { "net": 36800000000, "pct": 2.7 },
  "Italy": { "net": 27900000000, "pct": 1.5 },
  "Australia": { "net": 24300000000, "pct": 2 },
  "Brazil": { "net": 22800000000, "pct": 1.3 },
  "United Arab Emirates[c]": { "net": 22800000000, "pct": 5.7 },
  "Israel": { "net": 17800000000, "pct": 5.8 },
  "Canada": { "net": 15500000000, "pct": 1 },
  "Spain": { "net": 14900000000, "pct": 1.2 },
  "Turkey": { "net": 14900000000, "pct": 2 },
  "Iran": { "net": 12300000000, "pct": 3 },
  "Algeria": { "net": 10600000000, "pct": 6.7 },
  "Singapore": { "net": 9900000000, "pct": 3.4 },
  "Republic of China (Taiwan)": { "net": 9900000000, "pct": 1.9 },
  "Pakistan": { "net": 9900000000, "pct": 3.4 },
  "Colombia": { "net": 9900000000, "pct": 3.4 },
  "Poland": { "net": 9700000000, "pct": 2 },
  "Netherlands": { "net": 9200000000, "pct": 1.2 },
  "Oman": { "net": 9000000000, "pct": 16.7 },
  "Indonesia": { "net": 7700000000, "pct": 0.9 },
  "Mexico": { "net": 6800000000, "pct": 0.6 },
  "Kuwait": { "net": 6300000000, "pct": 6.5 }
}

population.json

{ China: '1390280000',
  India: '1330310000',
  'United States': '326944000',
  Indonesia: '261890900',
  Pakistan: '211202000',
  Brazil: '208903000',
  Nigeria: '193392500',
  Bangladesh: '164310000',
  Russia: '146877088',
  Japan: '126520000',
  Mexico: '123982528',
  Philippines: '105708000',
  Egypt: '96772100',
  Ethiopia: '94352000',
  Vietnam: '93700000',
  Germany: '82576900',
  Iran: '81361300',
  'Democratic Republic of the Congo': '81339900',
  Turkey: '80810525',
  France: '67206000',
  Thailand: '66061000',
  'United Kingdom': '66028930',
  Italy: '60497174',
  'South Africa': '56717000',
  Myanmar: '53370609',
  Tanzania: '51557365',
  'South Korea': '51446201',
  Colombia: '49713500',
  Kenya: '49699862',
  Spain: '46549045',
  Argentina: '44044811',
  Ukraine: '42364933',
  Algeria: '41697498',
  Sudan: '40782742',
  Poland: '38422346',
  Iraq: '38274618',
  Uganda: '37673800',
  Canada: '37072200',
  Morocco: '34634700',
  'Saudi Arabia': '33413660',
  Malaysia: '32474500',
  Uzbekistan: '32345000',
  Peru: '31826018',
  Venezuela: '31431164',
  Afghanistan: '29724323',
  Ghana: '29614337',
  Angola: '29250009',
  Mozambique: '28861863',
  Nepal: '28825709',
  Yemen: '28250000',
  Madagascar: '25571000',
  'North Korea': '25491000',
  Australia: '24900200',
  'Ivory Coast': '24294750',
  Cameroon: '23794164',
  Taiwan: '23571408',
  'Sri Lanka': '21444000',
  Niger: '20651070',
  Romania: '19638000',
  'Burkina Faso': '19632147',
  Mali: '18542000',
  Syria: '18270000',
  Kazakhstan: '18177700',
  Chile: '17574003',
  Malawi: '17373185',
  Netherlands: '17211800',
  Ecuador: '16968700',
  Zambia: '16405229',
  Guatemala: '16176133',
  Cambodia: '15848495',
  Senegal: '15726037',
  Chad: '14900000',
  Somalia: '14742523',
  Zimbabwe: '14542235',
  Guinea: '11883516',
  'South Sudan': '11868209',
  Rwanda: '11809300',
  Tunisia: '11446300',
  Belgium: '11403187',
  Cuba: '11239224',
  Bolivia: '11145770',
  Benin: '11002578',
  Haiti: '10911819',
  Greece: '10768193',
  'Czech Republic': '10610055',
  Burundi: '10400938',
  Portugal: '10309573',
  'Dominican Republic': '10169172',
  Sweden: '10135303',
  Jordan: '10110870',
  Azerbaijan: '9898085',
  Hungary: '9771000',
  Belarus: '9491900',
  'United Arab Emirates': '9400000',
  Tajikistan: '8931000',
  Honduras: '8866351',
  Israel: '8841320',
  Austria: '8823054',
  Switzerland: '8482200',
  'Papua New Guinea': '8151300',
  Togo: '7178000',
  Bulgaria: '7101859',
  'Sierra Leone': '7092113',
  Paraguay: '7052983',
  Serbia: '7040272',
  'El Salvador': '6581940',
  Laos: '6492400',
  Libya: '6374616',
  Nicaragua: '6305956',
  Kyrgyzstan: '6140200',
  Lebanon: '6082000',
  Denmark: '5781190',
  Turkmenistan: '5758000',
  Singapore: '5612300',
  Finland: '5513849',
  Slovakia: '5443120',
  Norway: '5295619',
  'Republic of the Congo': '5260750',
  Eritrea: '5068831',
  'Costa Rica': '4947490',
  'New Zealand': '4872090',
  Palestine: '4816503',
  Ireland: '4792500',
  'Central African Republic': '4659080',
  Oman: '4639678',
  Liberia: '4289520',
  Croatia: '4154213',
  Kuwait: '4132415',
  Panama: '4098135',
  Mauritania: '3806719',
  Georgia: '3718200',
  Moldova: '3550900',
  'Bosnia and Herzegovina': '3511372',
  Uruguay: '3493205',
  Mongolia: '3205275',
  Armenia: '2972900',
  Albania: '2876591',
  Lithuania: '2803332',
  Jamaica: '2730894',
  Qatar: '2700390',
  Namibia: '2368747',
  Lesotho: '2233339',
  Botswana: '2230905',
  'The Gambia': '2101000',
  Macedonia: '2073702',
  Slovenia: '2065895',
  Gabon: '2025137',
  Latvia: '1926600',
  'Guinea-Bissau': '1553822',
  Bahrain: '1451200',
  'Trinidad and Tobago': '1356633',
  Estonia: '1352320',
  Mauritius: '1264887',
  'Equatorial Guinea': '1222442',
  'East Timor': '1167242',
  Swaziland: '1145970',
  Djibouti: '956985',
  Fiji: '884887',
  Cyprus: '854800',
  Comoros: '806153',
  Bhutan: '798700',
  Guyana: '777859',
  'Solomon Islands': '651700',
  Montenegro: '622387',
  Luxembourg: '590667',
  Suriname: '541638',
  'Cape Verde': '537661',
  Malta: '434403',
  Brunei: '422678',
  Belize: '380010',
  Bahamas: '382460',
  Iceland: '348580',
  Maldives: '344023',
  Vanuatu: '289700',
  Barbados: '285719',
  Samoa: '197448',
  'São Tomé and Príncipe': '187356',
  'Saint Lucia': '178844',
  Kiribati: '115300',
  'Saint Vincent and the Grenadines': '109557',
  'Federated States of Micronesia': '104600',
  Grenada: '103328',
  Tonga: '100600',
  Seychelles: '94205',
  'Antigua and Barbuda': '86295',
  Andorra: '78264',
  Dominica: '71293',
  'Marshall Islands': '55000',
  'Saint Kitts and Nevis': '46204',
  Liechtenstein: '38111',
  Monaco: '37550',
  'San Marino': '33332',
  Palau: '17800',
  Nauru: '10800',
  Tuvalu: '10100',
  'Vatican City': '800' }

toJSON.js

var csv = `Rank;Country;Spending $; % of GDP;World total;1686000000;2.2
01; United States;611200000000;3.3
02; People's Republic of China[a];215700000000;1.9
03; Russia;69200000000;5.3
04; Saudi Arabia[a][b];63700000000;10
05; India;55900000000;2.5
06; France;55700000000;2.3
07; United Kingdom;48300000000;1.9
08; Japan;46100000000;1.0
09; Germany;41100000000;1.2
10; South Korea;36800000000;2.7
11; Italy;27900000000;1.5
12; Australia;24300000000;2.0
13; Brazil;22800000000;1.3
14; United Arab Emirates[c];22800000000;5.7
15; Israel;17800000000;5.8
16; Canada;15500000000;1.0
17; Spain;14900000000;1.2
18; Turkey;14900000000;2.0
19; Iran;12300000000;3.0
20; Algeria;10600000000;6.7
21; Singapore;9900000000;3.4
22; Republic of China (Taiwan);9900000000;1.9
23; Pakistan;9900000000;3.4
24; Colombia;9900000000;3.4
25; Poland;9700000000;2.0
26; Netherlands;9200000000;1.2
27; Oman;9000000000;16.7
28; Indonesia;7700000000;0.9
29; Mexico;6800000000;0.6
30; Kuwait;6300000000;6.5`
var rows = csv.split(`
`);
var militarySpendings = {};
var headers = ['Rank','Country','Spending $','% of GDP'];
rows.map((cols, i) => {
  if (i>0) {
   var col = cols.split(';');
   militarySpendings[col[1].trim()] = {net:parseInt(col[2]),pct:parseFloat(col[3])};
  }
})