Skip to content

Instantly share code, notes, and snippets.

@yxy
Created April 21, 2018 07:13
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save yxy/dc1cedd91284a2562373f0f09f1c2b64 to your computer and use it in GitHub Desktop.
Save yxy/dc1cedd91284a2562373f0f09f1c2b64 to your computer and use it in GitHub Desktop.
mongodb map reduce advantage usage.
const mapper = function() {
const value = {
country: this.country,
province: this.province,
city: this.city,
region: this.region,
wifi: this.wifi,
app_version: this.app_version,
os_version: this.os_version,
os: this.os,
network_type: this.network_type,
channel: this.channel,
referrer: this.referrer,
id: this.distinct_id,
};
const key = new Date(this.time);
key.setDate(key.getDate());
key.setHours(0);
key.setMinutes(0);
key.setSeconds(0);
key.setMilliseconds(0);
emit(key, value);
};
const reducer = function(key, values) {
const initial = { count: 0, unique: 0, avg: 0, ids: {} };
const f = function(id, { count, by_ids }) {
by_ids[id] = by_ids.hasOwnProperty(id) ? by_ids[id] + 1 : 1;
count++;
return { count, by_ids };
};
// 除了ID以外都要归约化
const initialState = {
summary: { count: 0, unique: 0, by_ids: {} },
};
const _mapper = (obj, key, id) => {
const data = obj ? obj : { count: 0, unique: 0, by_ids: {} };
return Object.assign(data, f(id, data));
};
const toKey = field => `by_${field}`;
return values.reduce(function(acc, curr) {
const { id } = curr;
const fields = Object.keys(curr).filter(key => key !== "id");
fields.forEach(field => {
// 初始化维度值
// key = by_$field
const key = toKey(field);
const value = curr[field];
// acc[by_$field] = {}
acc[key] = acc.hasOwnProperty(key) ? acc[key] : {};
// acc[by_$field][$value] = { {count: 0, unique: 0, by_ids: {}}}
Object.assign(acc[key], { [value]: _mapper(acc[key][value], value, id) });
});
return Object.assign(acc, { summary: f(id, acc.summary) });
}, initialState);
};
const finalize = function(key, value) {
const f = ({ count, by_ids }) => {
var unique = Object.keys(by_ids).length || 0;
var avg = 0;
if (count > 0) {
avg = unique / count;
}
return { avg, unique };
};
const byf = by => Object.keys(by).map(key => ({}));
const newValue = Object.keys(value)
.filter(field => field !== "summary")
.map(field => {
const v = Object.keys(value[field])
.map(key => ({
[key]: Object.assign(value[field][key], f(value[field][key])),
}))
.reduce((acc, curr) => Object.assign(acc, curr));
return { [field]: v };
})
.reduce((acc, curr) => Object.assign(acc, curr), {});
return Object.assign(newValue, {
summary: Object.assign(value.summary, f(value.summary)),
});
};
conn = new Mongo();
db = conn.getDB("spinach");
db.myresults.drop();
db.user_signup.mapReduce(mapper, reducer, {
out: "myresults",
finalize,
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment