restorer/chooser.js

## chooser.js
'use strict';

// На текущий момент страница со списком устройств на сайте /e/ поменяла дизайн,
// так что проверка поддерживает ли устройство /e/ не работает. Но раньше работало :)

const fs = require('fs').promises;
const path = require('path');
const axios = require('axios');
const cheerio = require('cheerio');
const Iconv = require('iconv').Iconv;

const cacheDir = path.join(__dirname, '.cache');
const cachedIconvs = Object.create(null);

function convertEncoding(buffer, encoding) {
    if (!cachedIconvs[encoding]) {
        cachedIconvs[encoding] = new Iconv(encoding, 'utf8');
    }

    return cachedIconvs[encoding].convert(buffer).toString();
}

async function retrieveRawPageContent(url) {
    const cachedPath = path.join(cacheDir, encodeURIComponent(url));
    const isCached = await fs.access(cachedPath).then(() => true).catch(() => false);
    let responseBody;

    if (isCached) {
        responseBody = await fs.readFile(cachedPath, 'utf8');
    } else {
        console.log(`Retrieving "${url}"...`);

        const response = await axios.get(url, {
            responseType: 'arraybuffer'
        });

        let encoding = 'utf8';

        if (response.headers['content-type']) {
            const mt = response.headers['content-type'].match(/charset\s*=\s*(.+)/);

            if (mt) {
                encoding = mt[1].trim().toLowerCase();
            }
        }

        if (encoding === 'utf8' || encoding === 'utf-8') {
            responseBody = response.data.toString('utf-8');
        } else {
            responseBody = convertEncoding(response.data, encoding);
        }

        await fs.writeFile(cachedPath, Buffer.from(responseBody));
    }

    return responseBody;
}

async function retrieveDomPageContent(url) {
    return cheerio.load(await retrieveRawPageContent(url));
}

async function retrieveJsonPageContent(url) {
    return JSON.parse(await retrieveRawPageContent(url));
}

async function retrieveLineageOsSupportedDevices() {
    const $ = await retrieveDomPageContent('https://download.lineageos.org/');
    const result = [];

    $('a.collapsible-header').each((_, vendorElem) => {
        const $vendor = $(vendorElem);
        const vendorName = $vendor.text();

        $vendor.parent().find('a.device-link').each((_, deviceElem) => {
            const $device = $(deviceElem);
            const deviceName = $device.find(':not(.device-model)').text();

            result.push({
                phoneName: vendorName + ' ' + deviceName,
                phoneModel: $device.find('.device-model').text().toLowerCase().trim(),
            });
        });
    });

    return result;
}

async function retrieveMicrogSupportedDevicesMap() {
    const $ = await retrieveDomPageContent('https://download.lineage.microg.org/');
    const result = Object.create(null);

    $('a').each((_, elem) => {
        const $elem = $(elem);
        const phoneModel = $elem.text().toLowerCase().trim();

        if ($elem.attr('href').toLowerCase().trim() === `/${phoneModel}/`) {
            result[phoneModel] = true;
        }
    });

    return result;
}

async function retrieveESupportedDevicesMap() {
    const $ = await retrieveDomPageContent('https://doc.e.foundation/devices/');
    const result = Object.create(null);

    $('a').each((_, elem) => {
        const $elem = $(elem);
        const phoneModel = $elem.text().toLowerCase().trim();

        if ($elem.attr('href').toLowerCase().trim() === phoneModel) {
            result[phoneModel] = true;
        }
    });

    return result;
}

function makeMatcherList(value) {
    return value.toLowerCase()
        .replace(/[^0-9a-z ]/g, ' ')
        .replace(/[ ]{2,}/g, ' ')
        .trim()
        .split(/ /);
}

/*
function computeMatchRatio(srcMl, compareValue) {
    if (!srcMl.length) {
        return 0;
    }

    const compareMl = makeMatcherList(compareValue);
    let matchCount = 0;

    for (part of srcMl) {
        if (compareMl.includes(part)) {
            ++matchCount;
        }
    }

    return matchCount / srcMl.length;
}
*/

// https://4pda.ru/devdb/search?s=Asus%20ZenFone%206
// https://4pda.ru/forum/index.php?act=search&source=all&forums%5B%5D=570&x=0&y=0&subforums=1&query=Asus+ZenFone+6+ZS630KL
// phoneName.replace(/\s+\([^)]+\)\s*$/, ''); -- убрать то, что в скобочках

async function retrieve4PdaPhoneSpecs(phoneName) {
    let $ = await retrieveDomPageContent(
        'https://4pda.ru/forum/index.php?act=search&source=all&forums%5B%5D=570&x=0&y=0&subforums=1&query=' +
        encodeURIComponent(makeMatcherList(phoneName).join(' ')))

    let deviceUrl = null;

    $('.cat_name > a').each((_, topicLinkElem) => {
        const $topicLink = $(topicLinkElem);
        const $deviceLink = $topicLink.parent().parent().find('.post_body > a[href*=devdb]:contains("Описание")');

        if ($deviceLink.length) {
            deviceUrl = $deviceLink.attr('href');
            return false;
        }
    });

    if (deviceUrl === null) {
        $('.ddb-ft-title a[href*=devdb]').each((_, deviceLinkElem) => {
            deviceUrl = $(deviceLinkElem).attr('href');
            return false;
        });
    }

    // const $ = await retrieveDomPageContent('https://4pda.ru/devdb/search?s=' + encodeURIComponent(phoneName));
    // let deviceUrl = null;
    //
    // $('a.dev-compare-item-link').parent().parent().find('.name > a[href*=devdb]').each((_, deviceLinkElem) => {
    //     deviceUrl = $(deviceLinkElem).attr('href');
    //     return false;
    // });

    if (deviceUrl === null) {
        return {
            osText: '',
            yearText: '',
            osCompare: 0,
            year: 0,
        };
    }

    $ = await retrieveDomPageContent(deviceUrl.startsWith('//') ? `https:${deviceUrl}` : deviceUrl);
    let osText = null;
    let yearText = null;

    $('.specifications-row > dt:contains("ОС")').parent().find('dd').each((_, ddElem) => {
        osText = $(ddElem).text();
        return false;
    });

    $('.specifications-row > dt:contains("Год выпуска")').parent().find('dd').each((_, ddElem) => {
        yearText = $(ddElem).text();
        return false;
    });

    if (osText === null || yearText === null) {
        return {
            osText: '',
            yearText: '',
            osCompare: 0,
            year: 0,
        };
    }

    const mtOs = osText.match(/^Android\s+(\d+)(?:\.(\d+))?/);

    return {
        isOn4pda: true,
        osText,
        yearText,
        osCompare: mtOs ? (parseInt(mtOs[1], 10) * 10 + parseInt(mtOs[2] || '0', 10)) : 0,
        year: parseInt(yearText, 10),
    };
}

// https://www.onliner.by/sdapi/catalog.api/search/products?query=Asus+Zenfone+6+ZS630KL

async function retrieveOnlinerPhoneSpecs(phoneName) {
    const phoneNameNormalized = makeMatcherList(phoneName).join(' ');

    const json = await retrieveJsonPageContent(
        'https://www.onliner.by/sdapi/catalog.api/search/products?query=' +
        encodeURIComponent(phoneNameNormalized))

    let deviceUrl = null;

    for (let product of json.products) {
        const fullNameNormalized = makeMatcherList(product.full_name).join(' ');

        if (fullNameNormalized.startsWith(phoneNameNormalized) && product.html_url.indexOf('phoneaccum') === -1) {
            deviceUrl = product.html_url;
            break;
        }
    }

    if (deviceUrl === null) {
        return {
            osText: '',
            yearText: '',
            osCompare: 0,
            year: 0,
        };
    }

    const $ = await retrieveDomPageContent(deviceUrl);
    let osText = null;
    let yearText = null;

    $('.product-specs__table > tbody > tr > td:contains("Версия операционной системы")').parent().find('td > span.value__text').each((_, elem) => {
        osText = $(elem).text();
        return false;
    });

    $('.product-specs__table > tbody > tr > td:contains("Дата выхода на рынок")').parent().find('td > span.value__text').each((_, elem) => {
        yearText = $(elem).text();
        return false;
    });

    if (osText === null || yearText === null) {
        return {
            osText: '',
            yearText: '',
            osCompare: 0,
            year: 0,
        };
    }

    const mtOs = osText.match(/^Android\s+(\d+)(?:\.(\d+))?/);

    return {
        isOnOnliner: true,
        osText,
        yearText,
        osCompare: mtOs ? (parseInt(mtOs[1], 10) * 10 + parseInt(mtOs[2] || '0', 10)) : 0,
        year: parseInt(yearText, 10),
    };
}

async function retrievePhoneSpecs(phoneName) {
    const specsA = await retrieve4PdaPhoneSpecs(phoneName);
    const specsB = await retrieveOnlinerPhoneSpecs(phoneName);

    if (specsA.year === 0 && specsB.year === 0) {
        return specsA;
    }

    if (specsA.year === 0) {
        return specsB;
    }

    if (specsB.year === 0) {
        return specsA;
    }

    return {
        isOn4pda: true,
        isOnOnliner: true,
        osText: (specsA.osText.startsWith(specsB.osText) || specsB.osText.startsWith(specsA.osText))
            ? specsA.osText
            : `${specsA.osText}, ${specsB.osText}`,
        yearText: (specsA.yearText.startsWith(specsB.yearText) || specsB.yearText.startsWith(specsA.yearText))
            ? specsA.yearText
            : `${specsA.yearText}, ${specsB.yearText}`,
        osCompare: Math.min(specsA.osCompare, specsB.osCompare),
        year: Math.min(specsA.year, specsB.year),
    };
}

async function process() {
    await fs.mkdir(cacheDir, { recursive: true });
    const lineageOsSupportedDevices = await retrieveLineageOsSupportedDevices();
    const microgSupportedDevicesMap = await retrieveMicrogSupportedDevicesMap();
    const eSupportedDevicesMap = await retrieveESupportedDevicesMap();
    const specList = [];

    for (const device of lineageOsSupportedDevices) {
        const specs = await retrievePhoneSpecs(device.phoneName);
        const hasMicrog = !!microgSupportedDevicesMap[device.phoneModel];
        const hasE = !!eSupportedDevicesMap[device.phoneModel];

        specList.push(Object.assign({
            hasMicrog,
            hasE,
            hasMicrogOrECompare: (hasMicrog || hasE) ? 1 : 0,
        }, device, specs));
    }

    specList.sort((a, b) => {
        const microgOrECmp = b.hasMicrogOrECompare - a.hasMicrogOrECompare;

        if (microgOrECmp !== 0) {
            return microgOrECmp;
        }

        const yearCmp = b.year - a.year;

        if (yearCmp !== 0) {
            return yearCmp;
        }

        return b.osCompare - a.osCompare;
    });

    for (const spec of specList) {
        const parts = [];

        if (spec.hasMicrog) {
            parts.push('microG');
        } else {
            parts.push('------');
        }

        if (spec.hasE) {
            parts.push('E');
        } else {
            parts.push('-');
        }

        if (spec.isOnOnliner) {
            parts.push('Onliner');
        } else {
            parts.push('-------');
        }

        if (spec.isOn4pda) {
            parts.push('4pda');
        } else {
            parts.push('----');
        }

        parts.push(spec.yearText === '' ? '-' : spec.yearText);
        parts.push(spec.osText === '' ? '-' : spec.osText);
        parts.push(spec.phoneModel);
        parts.push(spec.phoneName);

        console.log(parts.join(' | '));
    }
}

process();

## package.json
{
    "name": "phone-chooser.local",
    "version": "0.0.0",
    "description": "Phone Chooser",
    "main": "chooser.js",
    "scripts": {
        "choose": "node chooser.js"
    },
    "engines": {
        "node": ">=7.6.0"
    },
    "dependencies": {
        "axios": "^0.19.2",
        "cheerio": "^1.0.0-rc.3",
        "iconv": "^2.3.5"
    }
}
	'use strict';

	// На текущий момент страница со списком устройств на сайте /e/ поменяла дизайн,
	// так что проверка поддерживает ли устройство /e/ не работает. Но раньше работало :)

	const fs = require('fs').promises;
	const path = require('path');
	const axios = require('axios');
	const cheerio = require('cheerio');
	const Iconv = require('iconv').Iconv;

	const cacheDir = path.join(__dirname, '.cache');
	const cachedIconvs = Object.create(null);

	function convertEncoding(buffer, encoding) {
	if (!cachedIconvs[encoding]) {
	cachedIconvs[encoding] = new Iconv(encoding, 'utf8');
	}

	return cachedIconvs[encoding].convert(buffer).toString();
	}

	async function retrieveRawPageContent(url) {
	const cachedPath = path.join(cacheDir, encodeURIComponent(url));
	const isCached = await fs.access(cachedPath).then(() => true).catch(() => false);
	let responseBody;

	if (isCached) {
	responseBody = await fs.readFile(cachedPath, 'utf8');
	} else {
	console.log(`Retrieving "${url}"...`);

	const response = await axios.get(url, {
	responseType: 'arraybuffer'
	});

	let encoding = 'utf8';

	if (response.headers['content-type']) {
	const mt = response.headers['content-type'].match(/charset\s=\s(.+)/);

	if (mt) {
	encoding = mt[1].trim().toLowerCase();
	}
	}

	if (encoding === 'utf8' \|\| encoding === 'utf-8') {
	responseBody = response.data.toString('utf-8');
	} else {
	responseBody = convertEncoding(response.data, encoding);
	}

	await fs.writeFile(cachedPath, Buffer.from(responseBody));
	}

	return responseBody;
	}

	async function retrieveDomPageContent(url) {
	return cheerio.load(await retrieveRawPageContent(url));
	}

	async function retrieveJsonPageContent(url) {
	return JSON.parse(await retrieveRawPageContent(url));
	}

	async function retrieveLineageOsSupportedDevices() {
	const $ = await retrieveDomPageContent('https://download.lineageos.org/');
	const result = [];

	$('a.collapsible-header').each((_, vendorElem) => {
	const $vendor = $(vendorElem);
	const vendorName = $vendor.text();

	$vendor.parent().find('a.device-link').each((_, deviceElem) => {
	const $device = $(deviceElem);
	const deviceName = $device.find(':not(.device-model)').text();

	result.push({
	phoneName: vendorName + ' ' + deviceName,
	phoneModel: $device.find('.device-model').text().toLowerCase().trim(),
	});
	});
	});

	return result;
	}

	async function retrieveMicrogSupportedDevicesMap() {
	const $ = await retrieveDomPageContent('https://download.lineage.microg.org/');
	const result = Object.create(null);

	$('a').each((_, elem) => {
	const $elem = $(elem);
	const phoneModel = $elem.text().toLowerCase().trim();

	if ($elem.attr('href').toLowerCase().trim() === `/${phoneModel}/`) {
	result[phoneModel] = true;
	}
	});

	return result;
	}

	async function retrieveESupportedDevicesMap() {
	const $ = await retrieveDomPageContent('https://doc.e.foundation/devices/');
	const result = Object.create(null);

	$('a').each((_, elem) => {
	const $elem = $(elem);
	const phoneModel = $elem.text().toLowerCase().trim();

	if ($elem.attr('href').toLowerCase().trim() === phoneModel) {
	result[phoneModel] = true;
	}
	});

	return result;
	}

	function makeMatcherList(value) {
	return value.toLowerCase()
	.replace(/[^0-9a-z ]/g, ' ')
	.replace(/[ ]{2,}/g, ' ')
	.trim()
	.split(/ /);
	}

	/*
	function computeMatchRatio(srcMl, compareValue) {
	if (!srcMl.length) {
	return 0;
	}

	const compareMl = makeMatcherList(compareValue);
	let matchCount = 0;

	for (part of srcMl) {
	if (compareMl.includes(part)) {
	++matchCount;
	}
	}

	return matchCount / srcMl.length;
	}
	*/

	// https://4pda.ru/devdb/search?s=Asus%20ZenFone%206
	// https://4pda.ru/forum/index.php?act=search&source=all&forums%5B%5D=570&x=0&y=0&subforums=1&query=Asus+ZenFone+6+ZS630KL
	// phoneName.replace(/\s+\([^)]+\)\s*$/, ''); -- убрать то, что в скобочках

	async function retrieve4PdaPhoneSpecs(phoneName) {
	let $ = await retrieveDomPageContent(
	'https://4pda.ru/forum/index.php?act=search&source=all&forums%5B%5D=570&x=0&y=0&subforums=1&query=' +
	encodeURIComponent(makeMatcherList(phoneName).join(' ')))

	let deviceUrl = null;

	$('.cat_name > a').each((_, topicLinkElem) => {
	const $topicLink = $(topicLinkElem);
	const $deviceLink = $topicLink.parent().parent().find('.post_body > a[href*=devdb]:contains("Описание")');

	if ($deviceLink.length) {
	deviceUrl = $deviceLink.attr('href');
	return false;
	}
	});

	if (deviceUrl === null) {
	$('.ddb-ft-title a[href*=devdb]').each((_, deviceLinkElem) => {
	deviceUrl = $(deviceLinkElem).attr('href');
	return false;
	});
	}

	// const $ = await retrieveDomPageContent('https://4pda.ru/devdb/search?s=' + encodeURIComponent(phoneName));
	// let deviceUrl = null;
	//
	// $('a.dev-compare-item-link').parent().parent().find('.name > a[href*=devdb]').each((_, deviceLinkElem) => {
	// deviceUrl = $(deviceLinkElem).attr('href');
	// return false;
	// });

	if (deviceUrl === null) {
	return {
	osText: '',
	yearText: '',
	osCompare: 0,
	year: 0,
	};
	}

	$ = await retrieveDomPageContent(deviceUrl.startsWith('//') ? `https:${deviceUrl}` : deviceUrl);
	let osText = null;
	let yearText = null;

	$('.specifications-row > dt:contains("ОС")').parent().find('dd').each((_, ddElem) => {
	osText = $(ddElem).text();
	return false;
	});

	$('.specifications-row > dt:contains("Год выпуска")').parent().find('dd').each((_, ddElem) => {
	yearText = $(ddElem).text();
	return false;
	});

	if (osText === null \|\| yearText === null) {
	return {
	osText: '',
	yearText: '',
	osCompare: 0,
	year: 0,
	};
	}

	const mtOs = osText.match(/^Android\s+(\d+)(?:\.(\d+))?/);

	return {
	isOn4pda: true,
	osText,
	yearText,
	osCompare: mtOs ? (parseInt(mtOs[1], 10) * 10 + parseInt(mtOs[2] \|\| '0', 10)) : 0,
	year: parseInt(yearText, 10),
	};
	}

	// https://www.onliner.by/sdapi/catalog.api/search/products?query=Asus+Zenfone+6+ZS630KL

	async function retrieveOnlinerPhoneSpecs(phoneName) {
	const phoneNameNormalized = makeMatcherList(phoneName).join(' ');

	const json = await retrieveJsonPageContent(
	'https://www.onliner.by/sdapi/catalog.api/search/products?query=' +
	encodeURIComponent(phoneNameNormalized))

	let deviceUrl = null;

	for (let product of json.products) {
	const fullNameNormalized = makeMatcherList(product.full_name).join(' ');

	if (fullNameNormalized.startsWith(phoneNameNormalized) && product.html_url.indexOf('phoneaccum') === -1) {
	deviceUrl = product.html_url;
	break;
	}
	}

	if (deviceUrl === null) {
	return {
	osText: '',
	yearText: '',
	osCompare: 0,
	year: 0,
	};
	}

	const $ = await retrieveDomPageContent(deviceUrl);
	let osText = null;
	let yearText = null;

	$('.product-specs__table > tbody > tr > td:contains("Версия операционной системы")').parent().find('td > span.value__text').each((_, elem) => {
	osText = $(elem).text();
	return false;
	});

	$('.product-specs__table > tbody > tr > td:contains("Дата выхода на рынок")').parent().find('td > span.value__text').each((_, elem) => {
	yearText = $(elem).text();
	return false;
	});

	if (osText === null \|\| yearText === null) {
	return {
	osText: '',
	yearText: '',
	osCompare: 0,
	year: 0,
	};
	}

	const mtOs = osText.match(/^Android\s+(\d+)(?:\.(\d+))?/);

	return {
	isOnOnliner: true,
	osText,
	yearText,
	osCompare: mtOs ? (parseInt(mtOs[1], 10) * 10 + parseInt(mtOs[2] \|\| '0', 10)) : 0,
	year: parseInt(yearText, 10),
	};
	}

	async function retrievePhoneSpecs(phoneName) {
	const specsA = await retrieve4PdaPhoneSpecs(phoneName);
	const specsB = await retrieveOnlinerPhoneSpecs(phoneName);

	if (specsA.year === 0 && specsB.year === 0) {
	return specsA;
	}

	if (specsA.year === 0) {
	return specsB;
	}

	if (specsB.year === 0) {
	return specsA;
	}

	return {
	isOn4pda: true,
	isOnOnliner: true,
	osText: (specsA.osText.startsWith(specsB.osText) \|\| specsB.osText.startsWith(specsA.osText))
	? specsA.osText
	: `${specsA.osText}, ${specsB.osText}`,
	yearText: (specsA.yearText.startsWith(specsB.yearText) \|\| specsB.yearText.startsWith(specsA.yearText))
	? specsA.yearText
	: `${specsA.yearText}, ${specsB.yearText}`,
	osCompare: Math.min(specsA.osCompare, specsB.osCompare),
	year: Math.min(specsA.year, specsB.year),
	};
	}

	async function process() {
	await fs.mkdir(cacheDir, { recursive: true });
	const lineageOsSupportedDevices = await retrieveLineageOsSupportedDevices();
	const microgSupportedDevicesMap = await retrieveMicrogSupportedDevicesMap();
	const eSupportedDevicesMap = await retrieveESupportedDevicesMap();
	const specList = [];

	for (const device of lineageOsSupportedDevices) {
	const specs = await retrievePhoneSpecs(device.phoneName);
	const hasMicrog = !!microgSupportedDevicesMap[device.phoneModel];
	const hasE = !!eSupportedDevicesMap[device.phoneModel];

	specList.push(Object.assign({
	hasMicrog,
	hasE,
	hasMicrogOrECompare: (hasMicrog \|\| hasE) ? 1 : 0,
	}, device, specs));
	}

	specList.sort((a, b) => {
	const microgOrECmp = b.hasMicrogOrECompare - a.hasMicrogOrECompare;

	if (microgOrECmp !== 0) {
	return microgOrECmp;
	}

	const yearCmp = b.year - a.year;

	if (yearCmp !== 0) {
	return yearCmp;
	}

	return b.osCompare - a.osCompare;
	});

	for (const spec of specList) {
	const parts = [];

	if (spec.hasMicrog) {
	parts.push('microG');
	} else {
	parts.push('------');
	}

	if (spec.hasE) {
	parts.push('E');
	} else {
	parts.push('-');
	}

	if (spec.isOnOnliner) {
	parts.push('Onliner');
	} else {
	parts.push('-------');
	}

	if (spec.isOn4pda) {
	parts.push('4pda');
	} else {
	parts.push('----');
	}

	parts.push(spec.yearText === '' ? '-' : spec.yearText);
	parts.push(spec.osText === '' ? '-' : spec.osText);
	parts.push(spec.phoneModel);
	parts.push(spec.phoneName);

	console.log(parts.join(' \| '));
	}
	}

	process();
	{
	"name": "phone-chooser.local",
	"version": "0.0.0",
	"description": "Phone Chooser",
	"main": "chooser.js",
	"scripts": {
	"choose": "node chooser.js"
	},
	"engines": {
	"node": ">=7.6.0"
	},
	"dependencies": {
	"axios": "^0.19.2",
	"cheerio": "^1.0.0-rc.3",
	"iconv": "^2.3.5"
	}
	}