redfin_scraper.js

// what is AVM?
const sleep = (n) => new Promise((resolve) => setTimeout(resolve, n));

async function getAboveTheFold(propertyId) {
    const r = await fetch(`https://www.redfin.com/stingray/api/home/details/aboveTheFold?propertyId=${propertyId}&accessLevel=1`);
    const t = await r.text();
    const data = JSON.parse(t.replace('{}&&', ''));
    const payload = data.payload;
    const addressSectionInfo = payload.addressSectionInfo;

    const interestingData = {};
    // Big number they display up front
    interestingData.predictedValue = addressSectionInfo.avmInfo.predictedValue;

    // Address / ID info
    interestingData.streetAddress = addressSectionInfo.streetAddress;
    interestingData.url = addressSectionInfo.url;

    // "Last Sold Price"
    interestingData.latestPriceInfo = addressSectionInfo.latestPriceInfo;

    return interestingData;
}


async function getBelowTheFold(propertyId) {
    const r = await fetch(`https://www.redfin.com/stingray/api/home/details/belowTheFold?propertyId=${propertyId}&accessLevel=1`);
    const t = await r.text();
    const data = JSON.parse(t.replace('{}&&', ''));
    const payload = data.payload;

    const amenitiesInfo = payload.amenitiesInfo;

    // This has interesting stuff about heating / cooling, maybe solar?
    const amenityGroups = [].concat(...amenitiesInfo.superGroups.map(z => z.amenityGroups));
    interestingAmenityGroups = amenityGroups.filter((g) => g.groupTitle.includes('Heating') || g.groupTitle.includes('Cooling'));
    return { amenities: interestingAmenityGroups };
}

async function getDetails(propertyId) {
    const aboveR = getAboveTheFold(propertyId);
    const belowR = getBelowTheFold(propertyId);

    const [above, below] = await Promise.all([aboveR, belowR]);

    return { ...above, ...below };
}

async function seeAboutRateLimit(n) {
    // go descending order through N listings, making two requests each and sleeping a reasonable second between each.
    const pi = 5303414 - 1000;
    const listingsData = window.li || [];

    // Can get data out later by copying this variable
    window.li = listingsData;
    for (let i = 0; i < n; i++) {
        const listingData = await getDetails(pi - i);
        listingsData.push(listingData);
        await sleep(100);
    }
    return listingsData;
}