Skip to content

Instantly share code, notes, and snippets.

@arthurbailao
Created June 19, 2019 00:40
Show Gist options
  • Save arthurbailao/09dabce28bcb3090cbbab23c31a271cd to your computer and use it in GitHub Desktop.
Save arthurbailao/09dabce28bcb3090cbbab23c31a271cd to your computer and use it in GitHub Desktop.
Uber trips scraper
const fetchRides = (offset = '0') =>
fetch('https://riders.uber.com/api/getTripsForClient', {
method: 'POST',
body: JSON.stringify({
limit: 50,
offset,
range: {fromTime: null, toTime: null}, }), headers: {
'x-csrf-token': document
.getElementById('__CSRF_TOKEN__')
.innerText.replace(/\\u0022/g, ''),
'content-type': 'application/json',
},
})
.then(response => response.json())
.then(({data: {trips}}) => trips);
const fetchAllRides = (offset = '0', rides = []) =>
fetchRides(offset).then(
({trips, pagingResult: {hasMore, nextCursor}}) =>
hasMore
? fetchAllRides(nextCursor, rides.concat(trips))
: rides.concat(trips),
);
const rides = await fetchAllRides()
@FabioRodrigues
Copy link

rides.filter(x=> x.currencyCode=="BRL").map(x=> x.clientFare).reduce((x,y) => x + y)

@uriell
Copy link

uriell commented Jul 14, 2019

// get object prop by name
prop = (key = "", fallback = undefined) => (obj = {}) => obj[key] || fallback;

// filter by currencyCode field
filterByCurrency = (currency = "BRL") => (arr = []) =>
  arr.filter(trip => trip.currencyCode === currency);

// sum clientFare fields by currencyCode
sumFares = (arr, currencyCode = "BRL") =>
  filterByCurrency(currencyCode)(arr)
    .map(prop("clientFare", 0))
    .reduce((a, b) => a + b);

// generates a YYYY-MM date string from a Date object
yearMonth = date =>
  date.getFullYear() + "-" + `${date.getUTCMonth() + 1}`.padStart(2, "0");

// adds the yearMonth date string to trips from the requestTime date
addYearMonthDate = (arr = []) =>
  arr.map(ride => ({
    ...ride,
    yearMonth: yearMonth(new Date(ride.requestTime))
  }));

// group trips by YYYY-MM
groupByMonth = arr =>
  addYearMonthDate(arr).reduce(
    (acc, { yearMonth, ...obj }) => ({
      ...acc,
      [yearMonth]: [...(acc[yearMonth] || []), obj]
    }),
    {}
  );

USELESS_FIELDS = [
  "clientUUID",
  "marketplace",
  "paymentProfileUUID",
  "requestDevice",
  "vehicleViewID"
];

// cleans unnecessary fields
cleanFields = (arr = []) =>
  arr.map(trip => {
    for (let i = 0, { length } = USELESS_FIELDS; i < length; i++) {
      const field = USELESS_FIELDS[i];

      delete trip[field];
    }

    return trip;
  });

// helper functions to generate a csv from the trips data
generateCSVHeader = ([first = {}] = []) =>
  Object.keys(first)
    .map(key => `"${key}"`)
    .join(",");
generateCSVRow = (trip = {}) =>
  Object.values(trip)
    .map(value => `"${value}"`)
    .join(",");
generateCSVRows = (arr = []) => arr.map(generateCSVRow);
generateCSV = (arr = []) =>
  [
    generateCSVHeader(arr),
    ...generateCSVRows(arr)
  ].join("\n").trim();

// downloads a csv content as a file
downloadCSV = (csvContent = '') => {
  const encodedUri = encodeURI("data:text/csv;charset=utf-8," + csvContent);
  const link = document.createElement("a");
  link.setAttribute("href", encodedUri);
  link.setAttribute("download", "uber_trips.csv");
  document.body.appendChild(link);

  link.click();
};

leaving this here temporarily before I make an userscript to automatically export the data

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment