Skip to content

Instantly share code, notes, and snippets.

@restorer
Last active September 29, 2020 06:32
Show Gist options
  • Save restorer/e848e2b1f9573699b0c4a1d88eee52de to your computer and use it in GitHub Desktop.
Save restorer/e848e2b1f9573699b0c4a1d88eee52de to your computer and use it in GitHub Desktop.
Phone chooser
'use strict';
// На текущий момент страница со списком устройств на сайте /e/ поменяла дизайн,
// так что проверка поддерживает ли устройство /e/ не работает. Но раньше работало :)
const fs = require('fs').promises;
const path = require('path');
const axios = require('axios');
const cheerio = require('cheerio');
const Iconv = require('iconv').Iconv;
const cacheDir = path.join(__dirname, '.cache');
const cachedIconvs = Object.create(null);
function convertEncoding(buffer, encoding) {
if (!cachedIconvs[encoding]) {
cachedIconvs[encoding] = new Iconv(encoding, 'utf8');
}
return cachedIconvs[encoding].convert(buffer).toString();
}
async function retrieveRawPageContent(url) {
const cachedPath = path.join(cacheDir, encodeURIComponent(url));
const isCached = await fs.access(cachedPath).then(() => true).catch(() => false);
let responseBody;
if (isCached) {
responseBody = await fs.readFile(cachedPath, 'utf8');
} else {
console.log(`Retrieving "${url}"...`);
const response = await axios.get(url, {
responseType: 'arraybuffer'
});
let encoding = 'utf8';
if (response.headers['content-type']) {
const mt = response.headers['content-type'].match(/charset\s*=\s*(.+)/);
if (mt) {
encoding = mt[1].trim().toLowerCase();
}
}
if (encoding === 'utf8' || encoding === 'utf-8') {
responseBody = response.data.toString('utf-8');
} else {
responseBody = convertEncoding(response.data, encoding);
}
await fs.writeFile(cachedPath, Buffer.from(responseBody));
}
return responseBody;
}
async function retrieveDomPageContent(url) {
return cheerio.load(await retrieveRawPageContent(url));
}
async function retrieveJsonPageContent(url) {
return JSON.parse(await retrieveRawPageContent(url));
}
async function retrieveLineageOsSupportedDevices() {
const $ = await retrieveDomPageContent('https://download.lineageos.org/');
const result = [];
$('a.collapsible-header').each((_, vendorElem) => {
const $vendor = $(vendorElem);
const vendorName = $vendor.text();
$vendor.parent().find('a.device-link').each((_, deviceElem) => {
const $device = $(deviceElem);
const deviceName = $device.find(':not(.device-model)').text();
result.push({
phoneName: vendorName + ' ' + deviceName,
phoneModel: $device.find('.device-model').text().toLowerCase().trim(),
});
});
});
return result;
}
async function retrieveMicrogSupportedDevicesMap() {
const $ = await retrieveDomPageContent('https://download.lineage.microg.org/');
const result = Object.create(null);
$('a').each((_, elem) => {
const $elem = $(elem);
const phoneModel = $elem.text().toLowerCase().trim();
if ($elem.attr('href').toLowerCase().trim() === `/${phoneModel}/`) {
result[phoneModel] = true;
}
});
return result;
}
async function retrieveESupportedDevicesMap() {
const $ = await retrieveDomPageContent('https://doc.e.foundation/devices/');
const result = Object.create(null);
$('a').each((_, elem) => {
const $elem = $(elem);
const phoneModel = $elem.text().toLowerCase().trim();
if ($elem.attr('href').toLowerCase().trim() === phoneModel) {
result[phoneModel] = true;
}
});
return result;
}
function makeMatcherList(value) {
return value.toLowerCase()
.replace(/[^0-9a-z ]/g, ' ')
.replace(/[ ]{2,}/g, ' ')
.trim()
.split(/ /);
}
/*
function computeMatchRatio(srcMl, compareValue) {
if (!srcMl.length) {
return 0;
}
const compareMl = makeMatcherList(compareValue);
let matchCount = 0;
for (part of srcMl) {
if (compareMl.includes(part)) {
++matchCount;
}
}
return matchCount / srcMl.length;
}
*/
// https://4pda.ru/devdb/search?s=Asus%20ZenFone%206
// https://4pda.ru/forum/index.php?act=search&source=all&forums%5B%5D=570&x=0&y=0&subforums=1&query=Asus+ZenFone+6+ZS630KL
// phoneName.replace(/\s+\([^)]+\)\s*$/, ''); -- убрать то, что в скобочках
async function retrieve4PdaPhoneSpecs(phoneName) {
let $ = await retrieveDomPageContent(
'https://4pda.ru/forum/index.php?act=search&source=all&forums%5B%5D=570&x=0&y=0&subforums=1&query=' +
encodeURIComponent(makeMatcherList(phoneName).join(' ')))
let deviceUrl = null;
$('.cat_name > a').each((_, topicLinkElem) => {
const $topicLink = $(topicLinkElem);
const $deviceLink = $topicLink.parent().parent().find('.post_body > a[href*=devdb]:contains("Описание")');
if ($deviceLink.length) {
deviceUrl = $deviceLink.attr('href');
return false;
}
});
if (deviceUrl === null) {
$('.ddb-ft-title a[href*=devdb]').each((_, deviceLinkElem) => {
deviceUrl = $(deviceLinkElem).attr('href');
return false;
});
}
// const $ = await retrieveDomPageContent('https://4pda.ru/devdb/search?s=' + encodeURIComponent(phoneName));
// let deviceUrl = null;
//
// $('a.dev-compare-item-link').parent().parent().find('.name > a[href*=devdb]').each((_, deviceLinkElem) => {
// deviceUrl = $(deviceLinkElem).attr('href');
// return false;
// });
if (deviceUrl === null) {
return {
osText: '',
yearText: '',
osCompare: 0,
year: 0,
};
}
$ = await retrieveDomPageContent(deviceUrl.startsWith('//') ? `https:${deviceUrl}` : deviceUrl);
let osText = null;
let yearText = null;
$('.specifications-row > dt:contains("ОС")').parent().find('dd').each((_, ddElem) => {
osText = $(ddElem).text();
return false;
});
$('.specifications-row > dt:contains("Год выпуска")').parent().find('dd').each((_, ddElem) => {
yearText = $(ddElem).text();
return false;
});
if (osText === null || yearText === null) {
return {
osText: '',
yearText: '',
osCompare: 0,
year: 0,
};
}
const mtOs = osText.match(/^Android\s+(\d+)(?:\.(\d+))?/);
return {
isOn4pda: true,
osText,
yearText,
osCompare: mtOs ? (parseInt(mtOs[1], 10) * 10 + parseInt(mtOs[2] || '0', 10)) : 0,
year: parseInt(yearText, 10),
};
}
// https://www.onliner.by/sdapi/catalog.api/search/products?query=Asus+Zenfone+6+ZS630KL
async function retrieveOnlinerPhoneSpecs(phoneName) {
const phoneNameNormalized = makeMatcherList(phoneName).join(' ');
const json = await retrieveJsonPageContent(
'https://www.onliner.by/sdapi/catalog.api/search/products?query=' +
encodeURIComponent(phoneNameNormalized))
let deviceUrl = null;
for (let product of json.products) {
const fullNameNormalized = makeMatcherList(product.full_name).join(' ');
if (fullNameNormalized.startsWith(phoneNameNormalized) && product.html_url.indexOf('phoneaccum') === -1) {
deviceUrl = product.html_url;
break;
}
}
if (deviceUrl === null) {
return {
osText: '',
yearText: '',
osCompare: 0,
year: 0,
};
}
const $ = await retrieveDomPageContent(deviceUrl);
let osText = null;
let yearText = null;
$('.product-specs__table > tbody > tr > td:contains("Версия операционной системы")').parent().find('td > span.value__text').each((_, elem) => {
osText = $(elem).text();
return false;
});
$('.product-specs__table > tbody > tr > td:contains("Дата выхода на рынок")').parent().find('td > span.value__text').each((_, elem) => {
yearText = $(elem).text();
return false;
});
if (osText === null || yearText === null) {
return {
osText: '',
yearText: '',
osCompare: 0,
year: 0,
};
}
const mtOs = osText.match(/^Android\s+(\d+)(?:\.(\d+))?/);
return {
isOnOnliner: true,
osText,
yearText,
osCompare: mtOs ? (parseInt(mtOs[1], 10) * 10 + parseInt(mtOs[2] || '0', 10)) : 0,
year: parseInt(yearText, 10),
};
}
async function retrievePhoneSpecs(phoneName) {
const specsA = await retrieve4PdaPhoneSpecs(phoneName);
const specsB = await retrieveOnlinerPhoneSpecs(phoneName);
if (specsA.year === 0 && specsB.year === 0) {
return specsA;
}
if (specsA.year === 0) {
return specsB;
}
if (specsB.year === 0) {
return specsA;
}
return {
isOn4pda: true,
isOnOnliner: true,
osText: (specsA.osText.startsWith(specsB.osText) || specsB.osText.startsWith(specsA.osText))
? specsA.osText
: `${specsA.osText}, ${specsB.osText}`,
yearText: (specsA.yearText.startsWith(specsB.yearText) || specsB.yearText.startsWith(specsA.yearText))
? specsA.yearText
: `${specsA.yearText}, ${specsB.yearText}`,
osCompare: Math.min(specsA.osCompare, specsB.osCompare),
year: Math.min(specsA.year, specsB.year),
};
}
async function process() {
await fs.mkdir(cacheDir, { recursive: true });
const lineageOsSupportedDevices = await retrieveLineageOsSupportedDevices();
const microgSupportedDevicesMap = await retrieveMicrogSupportedDevicesMap();
const eSupportedDevicesMap = await retrieveESupportedDevicesMap();
const specList = [];
for (const device of lineageOsSupportedDevices) {
const specs = await retrievePhoneSpecs(device.phoneName);
const hasMicrog = !!microgSupportedDevicesMap[device.phoneModel];
const hasE = !!eSupportedDevicesMap[device.phoneModel];
specList.push(Object.assign({
hasMicrog,
hasE,
hasMicrogOrECompare: (hasMicrog || hasE) ? 1 : 0,
}, device, specs));
}
specList.sort((a, b) => {
const microgOrECmp = b.hasMicrogOrECompare - a.hasMicrogOrECompare;
if (microgOrECmp !== 0) {
return microgOrECmp;
}
const yearCmp = b.year - a.year;
if (yearCmp !== 0) {
return yearCmp;
}
return b.osCompare - a.osCompare;
});
for (const spec of specList) {
const parts = [];
if (spec.hasMicrog) {
parts.push('microG');
} else {
parts.push('------');
}
if (spec.hasE) {
parts.push('E');
} else {
parts.push('-');
}
if (spec.isOnOnliner) {
parts.push('Onliner');
} else {
parts.push('-------');
}
if (spec.isOn4pda) {
parts.push('4pda');
} else {
parts.push('----');
}
parts.push(spec.yearText === '' ? '-' : spec.yearText);
parts.push(spec.osText === '' ? '-' : spec.osText);
parts.push(spec.phoneModel);
parts.push(spec.phoneName);
console.log(parts.join(' | '));
}
}
process();
{
"name": "phone-chooser.local",
"version": "0.0.0",
"description": "Phone Chooser",
"main": "chooser.js",
"scripts": {
"choose": "node chooser.js"
},
"engines": {
"node": ">=7.6.0"
},
"dependencies": {
"axios": "^0.19.2",
"cheerio": "^1.0.0-rc.3",
"iconv": "^2.3.5"
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment