Created
June 19, 2019 00:40
-
-
Save arthurbailao/09dabce28bcb3090cbbab23c31a271cd to your computer and use it in GitHub Desktop.
Uber trips scraper
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const fetchRides = (offset = '0') => | |
fetch('https://riders.uber.com/api/getTripsForClient', { | |
method: 'POST', | |
body: JSON.stringify({ | |
limit: 50, | |
offset, | |
range: {fromTime: null, toTime: null}, }), headers: { | |
'x-csrf-token': document | |
.getElementById('__CSRF_TOKEN__') | |
.innerText.replace(/\\u0022/g, ''), | |
'content-type': 'application/json', | |
}, | |
}) | |
.then(response => response.json()) | |
.then(({data: {trips}}) => trips); | |
const fetchAllRides = (offset = '0', rides = []) => | |
fetchRides(offset).then( | |
({trips, pagingResult: {hasMore, nextCursor}}) => | |
hasMore | |
? fetchAllRides(nextCursor, rides.concat(trips)) | |
: rides.concat(trips), | |
); | |
const rides = await fetchAllRides() |
// get object prop by name
prop = (key = "", fallback = undefined) => (obj = {}) => obj[key] || fallback;
// filter by currencyCode field
filterByCurrency = (currency = "BRL") => (arr = []) =>
arr.filter(trip => trip.currencyCode === currency);
// sum clientFare fields by currencyCode
sumFares = (arr, currencyCode = "BRL") =>
filterByCurrency(currencyCode)(arr)
.map(prop("clientFare", 0))
.reduce((a, b) => a + b);
// generates a YYYY-MM date string from a Date object
yearMonth = date =>
date.getFullYear() + "-" + `${date.getUTCMonth() + 1}`.padStart(2, "0");
// adds the yearMonth date string to trips from the requestTime date
addYearMonthDate = (arr = []) =>
arr.map(ride => ({
...ride,
yearMonth: yearMonth(new Date(ride.requestTime))
}));
// group trips by YYYY-MM
groupByMonth = arr =>
addYearMonthDate(arr).reduce(
(acc, { yearMonth, ...obj }) => ({
...acc,
[yearMonth]: [...(acc[yearMonth] || []), obj]
}),
{}
);
USELESS_FIELDS = [
"clientUUID",
"marketplace",
"paymentProfileUUID",
"requestDevice",
"vehicleViewID"
];
// cleans unnecessary fields
cleanFields = (arr = []) =>
arr.map(trip => {
for (let i = 0, { length } = USELESS_FIELDS; i < length; i++) {
const field = USELESS_FIELDS[i];
delete trip[field];
}
return trip;
});
// helper functions to generate a csv from the trips data
generateCSVHeader = ([first = {}] = []) =>
Object.keys(first)
.map(key => `"${key}"`)
.join(",");
generateCSVRow = (trip = {}) =>
Object.values(trip)
.map(value => `"${value}"`)
.join(",");
generateCSVRows = (arr = []) => arr.map(generateCSVRow);
generateCSV = (arr = []) =>
[
generateCSVHeader(arr),
...generateCSVRows(arr)
].join("\n").trim();
// downloads a csv content as a file
downloadCSV = (csvContent = '') => {
const encodedUri = encodeURI("data:text/csv;charset=utf-8," + csvContent);
const link = document.createElement("a");
link.setAttribute("href", encodedUri);
link.setAttribute("download", "uber_trips.csv");
document.body.appendChild(link);
link.click();
};
leaving this here temporarily before I make an userscript to automatically export the data
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
rides.filter(x=> x.currencyCode=="BRL").map(x=> x.clientFare).reduce((x,y) => x + y)