Slider Captcha Solution Action

slider-captcha.ts

import { Element, Ctx } from '@ubio/engine';

export async function captchaSlider(el: Element, ctx: Ctx) {
    const page = el.page;

    // Obtain elements to interact with
    const sliderEl = (await el.queryOne('.yidun_jigsaw', false))!;
    const imageEl = (await el.queryOne('.yidun_bg-img', false))!;

    // Get image resources (base64) so that we can send them for offscreen canvas processing
    const sliderSrc: string = await sliderEl.evaluateJson(el => el.src);
    const imageSrc: string = await imageEl.evaluateJson(el => el.src);
    const sliderRes = await page.send('Page.getResourceContent', {
        frameId: page.target.targetId,
        url: sliderSrc
    });
    const imageRes = await page.send('Page.getResourceContent', {
        frameId: page.target.targetId,
        url: imageSrc
    });

    // Interesting part: send images back to page to process and evaluate the offset
    const answer = await page.evaluateJson(async (sliderDataB64: string, imageDataB64: string) => {
        // Note: slider is always png, image is always jpg (so this might not work on other websites)
        const sliderData = getImageData(await loadImageBase64(sliderDataB64, 'png'));
        const imageData = getImageData(await loadImageBase64(imageDataB64, 'jpg'));
        const { width, height } = imageData;
        const maxDistance = width - sliderData.width;

        const canvas = document.createElement('canvas');
        canvas.width = width;
        canvas.height = height;
        // Uncomment for debugging:
        // document.body.appendChild(canvas);
        const ctx = canvas.getContext('2d')!;
        ctx.fillStyle = 'rgb(255,0,0)';
        ctx.putImageData(imageData, 0, 0, 0, 0, width, height);

        const sobelData = calcSobel(imageData);
        const points = findFeaturePoints(sliderData);
        const best = findAnswer(sobelData, points, maxDistance);

        // ctx.putImageData(sliderData, best.pos, 0, 0, 0, width, height);
        // Uncomment to visualize feature points
        // for (const [x, y] of points) {
        //     ctx.fillRect(best.pos + x, y, 1, 1,);
        // }

        return { width, height, pos: best.pos };

        function loadImageBase64(base64: string, ext: string): Promise<HTMLImageElement> {
            return new Promise((resolve, reject) => {
                const img = document.createElement('img');
                img.src = `data:image/${ext};base64,${base64}`;
                img.onload = () => {
                    resolve(img);
                };
                img.onerror = err => reject(err);
            });
        }

        function getImageData(img: HTMLImageElement) {
            const canvas = document.createElement('canvas');
            const { width, height } = img;
            canvas.width = width;
            canvas.height = height;
            const ctx = canvas.getContext('2d')!;
            ctx.drawImage(img, 0, 0);
            return ctx.getImageData(0, 0, width, height);
        }

        function findFeaturePoints(sliderData: ImageData): Array<[number, number]> {
            const { data, width, height } = sliderData;
            const points: Array<[number, number]> = [];
            for (let y = 1; y < height - 1; y += 2) {
                for (let x = 1; x < width - 1; x += 2) {
                    let val = 0;
                    for (const ox of [-1, 0, 1]) {
                        for (const oy of [-1, 0, 1]) {
                            const i = getIndex(width, x + ox, y + oy);
                            const a = data[i + 3];
                            val += a >= 255 ? 1 : 0;
                        }
                    }
                    if (val > 1 && val < 8) {
                        points.push([x, y]);
                    }
                }
            }
            return points;
        }

        function calcSobel(imageData: ImageData): ImageData {
            const Kx = [
                [-1, 0, +1],
                [-2, 0, +2],
                [-1, 0, +1]
            ];
            const Ky = [
                [-1, -2, -1],
                [0, 0, 0],
                [+1, +2, +1]
            ];
            const { width, height, data } = imageData;
            const output = ctx.createImageData(width, height);
            for (let y = 1; y < height - 1; y += 1) {
                for (let x = 1; x < width - 1; x += 1) {
                    let vx = 0;
                    let vy = 0;
                    for (let oy of [-1, 0, 1]) {
                        for (let ox of [-1, 0, 1]) {
                            const i = getIndex(width, x + ox, y + oy);
                            const l = lum(data[i], data[i + 1], data[i + 2]);
                            const kx = Kx[oy + 1][ox + 1];
                            const ky = Ky[oy + 1][ox + 1];
                            vx += l * kx;
                            vy += l * ky;
                        }
                    }
                    const val = Math.hypot(vx, vy);
                    const i = getIndex(width, x, y);
                    output.data[i + 0] = val;
                    output.data[i + 1] = val;
                    output.data[i + 2] = val;
                    output.data[i + 3] = 255;
                }
            }
            return output;
        }

        function findAnswer(sobelData: ImageData, points: Array<[number, number]>, maxDistance: number) {
            const { width, data } = sobelData;
            const best = {
                pos: 0,
                val: 0,
            };
            for (let pos = 0; pos < maxDistance; pos++) {
                let val = 0;
                for (const [x, y] of points) {
                    const i = getIndex(width, pos + x, y);
                    val += data[i];
                }
                if (val > best.val) {
                    best.val = val;
                    best.pos = pos;
                }
            }
            return best;
        }

        function getIndex(width: number, x: number, y: number) {
            return (y * width + x) * 4;
        }

        function lum(r: number, g: number, b: number) {
            return 0.2126 * r + 0.7152 * g + 0.0722 * b;
        }

    }, sliderRes.content, imageRes.content);

    // Ok, that was a mouthful!
    // We now have 'pos' which is the distance in pixels we need to move the slider,
    // but it's in "original image space", so we have to transform it into "element space".
    // Plus, there appears to be some easing when
    const { width, pos } = answer;
    const imageBox = await imageEl.remote.getBoxModel();
    const offset = Math.round(pos * imageBox.width / width);

    // Now let's carefully move the slider element with CDP events
    const { x, y } = await sliderEl.remote.getStablePoint();
    await page.inputManager.mouseMove(x, y);
    await page.inputManager.mouseDown(x, y);
    for (let dx = 0; dx < offset; dx += 5) {
        const dy = Math.random() * 20 - 10;
        await page.inputManager.mouseMove(x + dx, y + dy);
        await new Promise(r => setTimeout(r, 5));
    }
    // Ok so here's the deal: there appears to be some easing on dragging,
    // so if we drag the exact `offset` pixels to the right, it will still
    // stay not aligned.
    // To fix this we need to figure the correction value between offset
    // and current left style.
    const left = await sliderEl.evaluateJson(el => Math.round(Number(el.style.left.replace('px', ''))));
    const diff = offset - left;
    const dy = Math.random() * 20 - 10;
    await page.inputManager.mouseMove(x + offset + diff, y + dy);
    await page.inputManager.mouseUp(x + offset + diff, y + dy);

}