Skip to content

Instantly share code, notes, and snippets.

@rhelmer

rhelmer/gist:b18f50f668b0d6336997c30f6727dcfd

Last active April 26, 2024 21:56
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save rhelmer/b18f50f668b0d6336997c30f6727dcfd to your computer and use it in GitHub Desktop.
Save rhelmer/b18f50f668b0d6336997c30f6727dcfd to your computer and use it in GitHub Desktop.
Download ZIP
WIP moving shield-addon-utils in-tree
Raw
gistfile1.diff
diff --git a/toolkit/components/extensions/ext-toolkit.json b/toolkit/components/extensions/ext-toolkit.json
--- a/toolkit/components/extensions/ext-toolkit.json
+++ b/toolkit/components/extensions/ext-toolkit.json
@@ -152,6 +152,14 @@
["runtime"]
]
},
+ "study": {
+ "url": "chrome://extensions/content/parent/ext-study.js",
+ "schema": "chrome://extensions/content/schemas/study.json",
+ "scopes": ["addon_parent"],
+ "paths": [
+ ["study"]
+ ]
+ },
"storage": {
"url": "chrome://extensions/content/parent/ext-storage.js",
"schema": "chrome://extensions/content/schemas/storage.json",
diff --git a/toolkit/components/extensions/jar.mn b/toolkit/components/extensions/jar.mn
--- a/toolkit/components/extensions/jar.mn
+++ b/toolkit/components/extensions/jar.mn
@@ -30,6 +30,7 @@ toolkit.jar:
content/extensions/parent/ext-proxy.js (parent/ext-proxy.js)
content/extensions/parent/ext-runtime.js (parent/ext-runtime.js)
content/extensions/parent/ext-storage.js (parent/ext-storage.js)
+ content/extensions/parent/ext-study.js (parent/ext-study.js)
content/extensions/parent/ext-tabs-base.js (parent/ext-tabs-base.js)
content/extensions/parent/ext-telemetry.js (parent/ext-telemetry.js)
content/extensions/parent/ext-theme.js (parent/ext-theme.js)
diff --git a/toolkit/components/extensions/parent/ext-study.js b/toolkit/components/extensions/parent/ext-study.js
new file mode 100644
--- /dev/null
+++ b/toolkit/components/extensions/parent/ext-study.js
@@ -0,0 +1,505 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+var {XPCOMUtils} = ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
+
+XPCOMUtils.defineLazyModuleGetters(this, {
+ DataPermissions: "resource://normandy/studies/DataPermissions.jsm",
+ Log: "resource://gre/modules/Log.jsm",
+ StudyUtils: "resource://normandy/studies/StudyUtils.jsm",
+ TestingOverrides: "resource://gre/modules/TestingOverrides.jsm",
+});
+
+const LOGGER_ID_BASE = "study.";
+
+const EventEmitter =
+ ExtensionCommon.EventEmitter || ExtensionUtils.EventEmitter;
+
+/**
+ * Event emitter to handle Events defined in the API
+ *
+ * - onReady
+ * - onEndStudy
+ *
+ */
+class StudyApiEventEmitter extends EventEmitter {
+ emitReady(studyInfo) {
+ this.emit("ready", studyInfo);
+ }
+
+ emitEndStudy(endingResponse) {
+ this.emit("endStudy", endingResponse);
+ }
+}
+this.study = class extends ExtensionAPI {
+ /**
+ * We don't need to override the constructor for other
+ * reasons than to clarify the class member "extension"
+ * being of type Extension
+ *
+ * @param {object} extension Extension
+ */
+ constructor(extension) {
+ super(extension);
+ /**
+ * @type Extension
+ */
+ this.extension = extension;
+ this.studyApiEventEmitter = new StudyApiEventEmitter();
+ this.logWarning("constructed!");
+ }
+
+ // TODO either find something in-tree or document this
+ makeWidgetId(id) {
+ id = id.toLowerCase();
+ return id.replace(/[^a-z0-9_-]/g, "_");
+ }
+
+ get logger() {
+ let id = this.id || "<unknown>";
+ return Log.repository.getLogger(LOGGER_ID_BASE + id);
+ }
+
+ logDebug(message) {
+ this._logMessage(message, "debug");
+ }
+
+ logInfo(message) {
+ this._logMessage(message, "info");
+ }
+
+ logWarning(message) {
+ this._logMessage(message, "warn");
+ }
+
+ logError(message) {
+ this._logMessage(message, "error");
+ }
+
+ _logMessage(message, severity) {
+ this.logger[severity](`Loading extension '${this.id}': ${message}`);
+ }
+
+ /**
+ * Extension Uninstall
+ * APIs that allocate any resources (e.g., adding elements to the browser’s
+ * user interface, setting up internal event listeners, etc.) must free
+ * these resources when the extension for which they are allocated is
+ * shut down.
+ *
+ * https://searchfox.org/mozilla-central/source/toolkit/components/extensions/parent/ext-protocolHandlers.js#46
+ *
+ * @param {string} shutdownReason one of the reasons
+ * @returns {undefined}
+ */
+ async onShutdown(shutdownReason) {
+ this.logWarning("possible uninstalling", shutdownReason);
+ if (
+ shutdownReason === "ADDON_UNINSTALL" ||
+ shutdownReason === "ADDON_DISABLE"
+ ) {
+ this.logWarning("definitely uninstall | disable", shutdownReason);
+ const anEndingAlias = "user-disable";
+ const endingResponse = await this.studyUtils.endStudy(anEndingAlias);
+ // See #194, getApi is already torn down, so cannot hear it.
+ await this.studyApiEventEmitter.emitEndStudy(endingResponse);
+ }
+ }
+
+ /**
+ * @param {object} context the add-on context
+ * @returns {object} api with study, studyDebug keys
+ */
+ getAPI(context) {
+ const {extension} = this;
+
+ // Make studyUtils available for onShutdown handler
+ const studyUtils = new StudyUtils();
+ this.studyUtils = studyUtils;
+
+ /* eslint no-shadow: off */
+ const {studyApiEventEmitter} = this;
+
+ // once. Used for pref naming, telemetry
+ this.studyUtils.setExtensionManifest(extension.manifest);
+ this.studyUtils._internals = this.studyUtils._createInternals();
+
+ // for add-on logging via browser.study.logger.log()
+ const widgetId = this.makeWidgetId(extension.manifest.applications.gecko.id);
+ // FIXME const addonLogger = createLogger(widgetId, `shieldStudy.logLevel`);
+
+ async function endStudy(anEndingAlias) {
+ // FIXME this.logWarning("called endStudy anEndingAlias");
+ const endingResponse = await studyUtils.endStudy(anEndingAlias);
+ studyApiEventEmitter.emitEndStudy(endingResponse);
+ }
+
+ return {
+ study: {
+ /** Attempt an setup/enrollment, with these effects:
+ *
+ * - sets 'studyType' as Shield or Pioneer
+ * - affects telemetry
+ * - watches for dataPermission changes that should *always*
+ * stop that kind of study
+ *
+ * - Use or choose variation
+ * - `testing.variation` if present
+ * - OR deterministicVariation
+ * for the studyType using `weightedVariations`
+ *
+ * - During firstRun[1] only:
+ * - set firstRunTimestamp pref value
+ * - send 'enter' ping
+ * - if `allowEnroll`, send 'install' ping
+ * - else endStudy("ineligible") and return
+ *
+ * - Every Run
+ * - setActiveExperiment(studySetup)
+ * - monitor shield | pioneer permission endings
+ * - suggests alarming if `expire` is set.
+ *
+ * Returns:
+ * - studyInfo object (see `getStudyInfo`)
+ *
+ * Telemetry Sent (First run only)
+ *
+ * - enter
+ * - install
+ *
+ * Fires Events
+ *
+ * (At most one of)
+ * - study:onReady OR
+ * - study:onEndStudy
+ *
+ * Preferences set
+ * - `shield.${runtime.id}.firstRunTimestamp`
+ *
+ * Note:
+ * 1. allowEnroll is ONLY used during first run (install)
+ *
+ * @param {Object<studySetup>} studySetup See API.md
+ * @returns {Object<studyInfo>} studyInfo. See studyInfo
+ **/
+ setup: async function setup(studySetup) {
+ // 0. testing overrides, if any
+ if (!studySetup.testing) {
+ studySetup.testing = {};
+ }
+
+ // Setup and sets the variation / _internals
+ // includes possible 'firstRun' handling.
+ await studyUtils.setup(studySetup);
+
+ // current studyInfo.
+ let studyInfo = this.studyUtils.info();
+
+ // Check if the user is eligible to run this study using the |isEligible|
+ // function when the study is initialized
+ if (studyInfo.isFirstRun) {
+ if (!studySetup.allowEnroll) {
+ this.logWarning("User is ineligible, ending study.");
+ // 1. uses studySetup.endings.ineligible.url if any,
+ // 2. sends UT for "ineligible"
+ // 3. then uninstalls add-on
+ await endStudy("ineligible");
+ return this.studyUtils.info();
+ }
+ }
+
+ if (studyInfo.delayInMinutes === 0) {
+ this.logWarning("encountered already expired study");
+ await endStudy("expired");
+ return studyUtils.info();
+ }
+
+ /*
+ * Adds the study to the active list of telemetry experiments,
+ * and sends the "installed" telemetry ping if applicable,
+ * if it's a firstRun
+ */
+ await studyUtils.startup();
+
+ // update what the study variation and other info is.
+ studyInfo = studyUtils.info();
+ this.logWarning(`api info: ${JSON.stringify(studyInfo)}`);
+ try {
+ studyApiEventEmitter.emitReady(studyInfo);
+ } catch (e) {
+ this.logError.error("browser.study.setup error");
+ this.logError.error(e);
+ }
+ return studyUtils.info();
+ },
+
+ /* Signal to browser.study that it should end.
+ *
+ * Usage scenarios:
+ * - add-ons defined
+ * - postive endings (tried feature)
+ * - negative endings (client clicked 'no thanks')
+ * - expiration / timeout (feature should last for 14 days then uninstall)
+ *
+ * Logic:
+ * - If study has already ended, do nothing.
+ * - Else: END
+ *
+ * END:
+ * - record internally that study is ended.
+ * - disable all methods that rely on configuration / setup.
+ * - clear all prefs stored by `browser.study`
+ * - fire telemetry pings for:
+ * - 'exit'
+ * - the ending, one of:
+ *
+ * "ineligible",
+ * "expired",
+ * "user-disable",
+ * "ended-positive",
+ * "ended-neutral",
+ * "ended-negative",
+ *
+ * - augment all ending urls with query urls
+ * - fire 'study:end' event to `browser.study.onEndStudy` handlers.
+ *
+ * Addon should then do
+ * - open returned urls
+ * - feature specific cleanup
+ * - uninstall the add-on
+ *
+ * Note:
+ * 1. calling this function multiple time is safe.
+ * `browser.study` will choose the first in.
+ * 2. the 'user-disable' case is handled above
+ * 3. throws if the endStudy fails
+ **/
+ endStudy,
+
+ /* current study configuration, including
+ * - variation
+ * - activeExperimentName
+ * - delayInMinutes
+ * - firstRunTimestamp
+ *
+ * But not:
+ * - telemetry clientId
+ *
+ * Throws ExtensionError if called before `browser.study.setup`
+ **/
+ getStudyInfo: async function getStudyInfo() {
+ // this.logWarning("called getStudyInfo ");
+ return studyUtils.info();
+ },
+
+ /* Object of current dataPermissions (shield enabled true/false, pioneer enabled true/false) */
+ getDataPermissions: async function getDataPermissions() {
+ return DataPermissions.getDataPermissions();
+ },
+
+ /** Send Telemetry using appropriate shield or pioneer methods.
+ *
+ * shield:
+ * - `shield-study-addon` ping, requires object string keys and string values
+ *
+ * pioneer:
+ * - TBD
+ *
+ * Note:
+ * - no conversions / coercion of data happens.
+ *
+ * Note:
+ * - undefined what happens if validation fails
+ * - undefined what happens when you try to send 'shield' from 'pioneer'
+ *
+ * TBD fix the parameters here.
+ *
+ * @param {Object} payload Non-nested object with key strings, and key values
+ * @returns {undefined}
+ */
+ sendTelemetry: async function sendTelemetry(payload) {
+ this.logWarning("called sendTelemetry payload");
+
+ function throwIfInvalid(obj) {
+ // Check: all keys and values must be strings,
+ for (const k in obj) {
+ if (typeof k !== "string") {
+ throw new ExtensionError(`key ${k} not a string`);
+ }
+ if (typeof obj[k] !== "string") {
+ throw new ExtensionError(`value ${k} ${obj[k]} not a string`);
+ }
+ }
+ return true;
+ }
+
+ throwIfInvalid(payload);
+ return this.studyUtils.telemetry(payload);
+ },
+
+ /** Calculate Telemetry using appropriate shield or pioneer methods.
+ *
+ * shield:
+ * - Calculate the size of a ping
+ *
+ * pioneer:
+ * - Calculate the size of a ping that has Pioneer encrypted data
+ *
+ * @param {Object} payload Non-nested object with key strings, and key values
+ * @returns {Promise<number>} The total size of the ping.
+ */
+ calculateTelemetryPingSize: async function calculateTelemetryPingSize(
+ payload,
+ ) {
+ return this.studyUtils.calculateTelemetryPingSize(payload);
+ },
+
+ /** Search locally stored telemetry pings using these fields (if set)
+ *
+ * n:
+ * if set, no more than `n` pings.
+ * type:
+ * Array of 'ping types' (e.g., main, crash, shield-study-addon) to filter
+ * mininumTimestamp:
+ * only pings after this timestamp.
+ * headersOnly:
+ * boolean. If true, only the 'headers' will be returned.
+ *
+ * Pings will be returned sorted by timestamp with most recent first.
+ *
+ * Usage scenarios:
+ * - enrollment / eligiblity using recent Telemetry behaviours or client environment
+ * - add-on testing scenarios
+ *
+ * @param {Object<query>} searchTelemetryQuery see above
+ * @returns {Array<sendTelemetry>} matchingPings
+ */
+ async searchSentTelemetry(searchTelemetryQuery) {
+ const {TelemetryArchive} = ChromeUtils.import(
+ "resource://gre/modules/TelemetryArchive.jsm",
+ {},
+ );
+ const {searchTelemetryArchive} = require("./telemetry.js");
+ return searchTelemetryArchive(TelemetryArchive, searchTelemetryQuery);
+ },
+
+ /* Using AJV, do jsonschema validation of an object. Can be used to validate your arguments, packets at client. */
+ validateJSON: async function validateJSON(someJson, jsonschema) {
+ this.logWarning("called validateJSON someJson, jsonschema");
+ return this.studyUtils.jsonschema.validate(someJson, jsonschema);
+ // return { valid: true, errors: [] };
+ },
+
+ /* Returns an object with the following keys:
+ variationName - to be able to test specific variations
+ firstRunTimestamp - to be able to test the expiration event
+ expired - to be able to test the behavior of an already expired study
+ The values are set by the corresponding preference under the `extensions.${widgetId}.test.*` preference branch. */
+ getTestingOverrides: async function getTestingOverrides() {
+ this.logError.info(
+ "The preferences that can be used to override study testing flags: ",
+ TestingOverrides.listPreferences(widgetId),
+ );
+ return TestingOverrides.getTestingOverrides(widgetId);
+ },
+
+ /**
+ * Schema.json `events`
+ *
+ * See https://firefox-source-docs.mozilla.org/toolkit/components/extensions/webextensions/events.html
+ */
+
+ /* Fires when the study is 'ready' for the feature to startup. */
+ onReady: new EventManager(context, "study:onReady", fire => {
+ const listener = (eventReference, studyInfo) => {
+ fire.async(studyInfo);
+ };
+ studyApiEventEmitter.once("ready", listener);
+ return () => {
+ studyApiEventEmitter.off("ready", listener);
+ };
+ }).api(),
+
+ /* Listen for when the study wants to end.
+ *
+ * Act on it by
+ * - opening surveyUrls
+ * - tearing down your feature
+ * - uninstalling the add-on
+ */
+ onEndStudy: new EventManager(context, "study:onEndStudy", fire => {
+ const listener = (eventReference, ending) => {
+ fire.async(ending);
+ };
+ studyApiEventEmitter.on("endStudy", listener);
+ return () => {
+ studyApiEventEmitter.off("endStudy", listener);
+ };
+ }).api(),
+
+ logger: {
+ /* Corresponds to console.info */
+ info: async function info(values) {
+ this.logInfo(values);
+ },
+
+ /* Corresponds to console.log */
+ log: async function log(values) {
+ this.logInfo(values);
+ },
+
+ /* Corresponds to console.debug */
+ debug: async function debug(values) {
+ this.logDebug(values);
+ },
+
+ /* Corresponds to console.warn */
+ warn: async function warn(values) {
+ this.logWarn(values);
+ },
+
+ /* Corresponds to console.error */
+ error: async function error(values) {
+ this.logError(values);
+ },
+ },
+ },
+
+ studyDebug: {
+ throwAnException(message) {
+ throw new ExtensionError(message);
+ },
+
+ async throwAnExceptionAsync(message) {
+ throw new ExtensionError(message);
+ },
+
+ async setActive() {
+ return this.studyUtils.setActive();
+ },
+
+ async startup({reason}) {
+ return this.studyUtils.startup({reason});
+ },
+
+ async setFirstRunTimestamp(timestamp) {
+ return this.studyUtils.setFirstRunTimestamp(timestamp);
+ },
+
+ async reset() {
+ return this.studyUtils.reset();
+ },
+
+ async getInternals() {
+ return this.studyUtils._internals;
+ },
+
+ getInternalTestingOverrides: async function getInternalTestingOverrides() {
+ return TestingOverrides.getInternalTestingOverrides(widgetId);
+ },
+ },
+ };
+ }
+};
diff --git a/toolkit/components/extensions/schemas/jar.mn b/toolkit/components/extensions/schemas/jar.mn
--- a/toolkit/components/extensions/schemas/jar.mn
+++ b/toolkit/components/extensions/schemas/jar.mn
@@ -31,6 +31,7 @@ toolkit.jar:
content/extensions/schemas/privacy.json
content/extensions/schemas/runtime.json
content/extensions/schemas/storage.json
+ content/extensions/schemas/study.json
content/extensions/schemas/telemetry.json
content/extensions/schemas/test.json
content/extensions/schemas/theme.json
diff --git a/toolkit/components/extensions/schemas/study.json b/toolkit/components/extensions/schemas/study.json
new file mode 100644
--- /dev/null
+++ b/toolkit/components/extensions/schemas/study.json
@@ -0,0 +1,958 @@
+[
+ {
+ "namespace": "study",
+ "description": "Interface for Shield and Pioneer studies.",
+ "apiVersion": 5,
+ "types": [
+ {
+ "id": "NullableString",
+ "$schema": "http://json-schema.org/draft-04/schema",
+ "oneOf": [
+ {
+ "type": "null"
+ },
+ {
+ "type": "string"
+ }
+ ],
+ "choices": [
+ {
+ "type": "null"
+ },
+ {
+ "type": "string"
+ }
+ ],
+ "testcases": [null, "a string"]
+ },
+ {
+ "id": "NullableBoolean",
+ "$schema": "http://json-schema.org/draft-04/schema",
+ "oneOf": [
+ {
+ "type": "null"
+ },
+ {
+ "type": "boolean"
+ }
+ ],
+ "choices": [
+ {
+ "type": "null"
+ },
+ {
+ "type": "boolean"
+ }
+ ],
+ "testcases": [null, true, false],
+ "failcases": ["1234567890", "foo", []]
+ },
+ {
+ "id": "NullableInteger",
+ "$schema": "http://json-schema.org/draft-04/schema",
+ "oneOf": [
+ {
+ "type": "null"
+ },
+ {
+ "type": "integer"
+ }
+ ],
+ "choices": [
+ {
+ "type": "null"
+ },
+ {
+ "type": "integer"
+ }
+ ],
+ "testcases": [null, 1234567890],
+ "failcases": ["1234567890", []]
+ },
+ {
+ "id": "NullableNumber",
+ "$schema": "http://json-schema.org/draft-04/schema",
+ "oneOf": [
+ {
+ "type": "null"
+ },
+ {
+ "type": "number"
+ }
+ ],
+ "choices": [
+ {
+ "type": "null"
+ },
+ {
+ "type": "number"
+ }
+ ],
+ "testcases": [null, 1234567890, 1234567890.123],
+ "failcases": ["1234567890", "1234567890.123", []]
+ },
+ {
+ "id": "studyTypesEnum",
+ "$schema": "http://json-schema.org/draft-04/schema",
+ "type": "string",
+ "enum": ["shield", "pioneer"],
+ "testcases": ["shield", "pioneer"],
+ "failcases": ["foo"]
+ },
+ {
+ "id": "weightedVariationObject",
+ "$schema": "http://json-schema.org/draft-04/schema",
+ "type": "object",
+ "properties": {
+ "name": {
+ "type": "string"
+ },
+ "weight": {
+ "type": "number",
+ "minimum": 0
+ }
+ },
+ "required": ["name", "weight"],
+ "testcase": {
+ "name": "feature-active",
+ "weight": 1.5
+ }
+ },
+ {
+ "id": "weightedVariationsArray",
+ "$schema": "http://json-schema.org/draft-04/schema",
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "name": {
+ "type": "string"
+ },
+ "weight": {
+ "type": "number",
+ "minimum": 0
+ }
+ },
+ "required": ["name", "weight"]
+ },
+ "testcase": [
+ {
+ "name": "feature-active",
+ "weight": 1.5
+ },
+ {
+ "name": "feature-inactive",
+ "weight": 1.5
+ }
+ ]
+ },
+ {
+ "id": "anEndingRequest",
+ "$schema": "http://json-schema.org/draft-04/schema",
+ "type": "object",
+ "properties": {
+ "fullname": {
+ "$ref": "NullableString",
+ "optional": true
+ },
+ "category": {
+ "oneOf": [
+ {
+ "type": "null"
+ },
+ {
+ "type": "string",
+ "enum": ["ended-positive", "ended-neutral", "ended-negative"]
+ }
+ ],
+ "choices": [
+ {
+ "type": "null"
+ },
+ {
+ "type": "string",
+ "enum": ["ended-positive", "ended-neutral", "ended-negative"]
+ }
+ ],
+ "optional": true
+ },
+ "baseUrls": {
+ "oneOf": [
+ {
+ "type": "null"
+ },
+ {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ ],
+ "choices": [
+ {
+ "type": "null"
+ },
+ {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ ],
+ "optional": true,
+ "default": []
+ },
+ "exacturls": {
+ "oneOf": [
+ {
+ "type": "null"
+ },
+ {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ ],
+ "choices": [
+ {
+ "type": "null"
+ },
+ {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ ],
+ "optional": "true\ndefault: []"
+ }
+ },
+ "additionalProperties": true,
+ "testcases": [
+ {
+ "baseUrls": ["some.url"],
+ "fullname": "anEnding",
+ "category": "ended-positive"
+ },
+ {},
+ {
+ "baseUrls": ["some.url"]
+ },
+ {
+ "baseUrls": [],
+ "fullname": null,
+ "category": null
+ }
+ ],
+ "failcases": [
+ {
+ "baseUrls": null,
+ "category": "not okay"
+ }
+ ]
+ },
+ {
+ "id": "onEndStudyResponse",
+ "$schema": "http://json-schema.org/draft-04/schema",
+ "type": "object",
+ "properties": {
+ "fields": {
+ "type": "object",
+ "additionalProperties": true
+ },
+ "urls": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ {
+ "id": "studyInfoObject",
+ "$schema": "http://json-schema.org/draft-04/schema",
+ "type": "object",
+ "additionalProperties": true,
+ "properties": {
+ "variation": {
+ "$ref": "weightedVariationObject"
+ },
+ "firstRunTimestamp": {
+ "$ref": "NullableInteger"
+ },
+ "activeExperimentName": {
+ "type": "string"
+ },
+ "delayInMinutes": {
+ "$ref": "NullableNumber"
+ },
+ "isFirstRun": {
+ "type": "boolean"
+ }
+ },
+ "required": [
+ "variation",
+ "firstRunTimestamp",
+ "activeExperimentName",
+ "isFirstRun"
+ ]
+ },
+ {
+ "id": "dataPermissionsObject",
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "shield": {
+ "type": "boolean"
+ },
+ "pioneer": {
+ "type": "boolean"
+ }
+ },
+ "required": ["shield", "pioneer"]
+ },
+ {
+ "id": "studySetup",
+ "$schema": "http://json-schema.org/draft-04/schema",
+ "type": "object",
+ "properties": {
+ "activeExperimentName": {
+ "type": "string"
+ },
+ "studyType": {
+ "$ref": "studyTypesEnum"
+ },
+ "expire": {
+ "type": "object",
+ "properties": {
+ "days": {
+ "type": "integer"
+ }
+ },
+ "optional": true,
+ "additionalProperties": false
+ },
+ "endings": {
+ "type": "object",
+ "additionalProperties": {
+ "$ref": "anEndingRequest"
+ }
+ },
+ "weightedVariations": {
+ "$ref": "weightedVariationsArray"
+ },
+ "telemetry": {
+ "type": "object",
+ "properties": {
+ "send": {
+ "type": "boolean"
+ },
+ "removeTestingFlag": {
+ "type": "boolean"
+ },
+ "internalTelemetryArchive": {
+ "optional": true,
+ "$ref": "NullableBoolean"
+ }
+ }
+ },
+ "testing": {
+ "type": "object",
+ "properties": {
+ "variationName": {
+ "$ref": "NullableString",
+ "optional": true
+ },
+ "firstRunTimestamp": {
+ "$ref": "NullableInteger",
+ "optional": true
+ },
+ "expired": {
+ "choices": [
+ {
+ "type": "null"
+ },
+ {
+ "type": "boolean"
+ }
+ ],
+ "oneOf": [
+ {
+ "type": "null"
+ },
+ {
+ "type": "boolean"
+ }
+ ],
+ "optional": true
+ }
+ },
+ "additionalProperties": false,
+ "optional": true
+ }
+ },
+ "required": [
+ "activeExperimentName",
+ "studyType",
+ "endings",
+ "weightedVariations",
+ "telemetry"
+ ],
+ "additionalProperties": true,
+ "testcases": [
+ {
+ "activeExperimentName": "aStudy",
+ "studyType": "shield",
+ "expire": {
+ "days": 10
+ },
+ "endings": {
+ "anEnding": {
+ "baseUrls": ["some.url"]
+ }
+ },
+ "weightedVariations": [
+ {
+ "name": "feature-active",
+ "weight": 1.5
+ }
+ ],
+ "telemetry": {
+ "send": false,
+ "removeTestingFlag": false
+ }
+ },
+ {
+ "activeExperimentName": "aStudy",
+ "studyType": "shield",
+ "expire": {
+ "days": 10
+ },
+ "endings": {
+ "anEnding": {
+ "baseUrls": ["some.url"]
+ }
+ },
+ "weightedVariations": [
+ {
+ "name": "feature-active",
+ "weight": 1.5
+ }
+ ],
+ "telemetry": {
+ "send": false,
+ "removeTestingFlag": false,
+ "internalTelemetryArchive": false
+ },
+ "testing": {
+ "variationName": "something",
+ "firstRunTimestamp": 1234567890,
+ "expired": true
+ }
+ },
+ {
+ "activeExperimentName": "aStudy",
+ "studyType": "pioneer",
+ "endings": {
+ "anEnding": {
+ "baseUrls": ["some.url"]
+ }
+ },
+ "weightedVariations": [
+ {
+ "name": "feature-active",
+ "weight": 1.5
+ }
+ ],
+ "telemetry": {
+ "send": false,
+ "removeTestingFlag": true,
+ "internalTelemetryArchive": true
+ },
+ "testing": {
+ "variationName": "something",
+ "firstRunTimestamp": 1234567890,
+ "expired": true
+ }
+ },
+ {
+ "activeExperimentName":
+ "shield-utils-test-addon@shield.mozilla.org",
+ "studyType": "shield",
+ "telemetry": {
+ "send": true,
+ "removeTestingFlag": false
+ },
+ "endings": {
+ "user-disable": {
+ "baseUrls": ["http://www.example.com/?reason=user-disable"]
+ },
+ "ineligible": {
+ "baseUrls": ["http://www.example.com/?reason=ineligible"]
+ },
+ "expired": {
+ "baseUrls": ["http://www.example.com/?reason=expired"]
+ },
+ "some-study-defined-ending": {
+ "category": "ended-neutral"
+ },
+ "some-study-defined-ending-with-survey-url": {
+ "baseUrls": [
+ "http://www.example.com/?reason=some-study-defined-ending-with-survey-url"
+ ],
+ "category": "ended-negative"
+ }
+ },
+ "weightedVariations": [
+ {
+ "name": "feature-active",
+ "weight": 1.5
+ },
+ {
+ "name": "feature-passive",
+ "weight": 1.5
+ },
+ {
+ "name": "control",
+ "weight": 1
+ }
+ ],
+ "expire": {
+ "days": 14
+ },
+ "testing": {},
+ "allowEnroll": true
+ }
+ ]
+ },
+ {
+ "id": "telemetryPayload",
+ "$schema": "http://json-schema.org/draft-04/schema",
+ "type": "object",
+ "additionalProperties": true,
+ "testcase": {
+ "foo": "bar"
+ }
+ },
+ {
+ "id": "searchTelemetryQuery",
+ "$schema": "http://json-schema.org/draft-04/schema",
+ "type": "object",
+ "properties": {
+ "type": {
+ "type": ["array"],
+ "items": {
+ "type": "string"
+ },
+ "optional": true
+ },
+ "n": {
+ "type": "integer",
+ "optional": true
+ },
+ "minimumTimestamp": {
+ "type": "number",
+ "optional": true
+ },
+ "headersOnly": {
+ "type": "boolean",
+ "optional": true
+ }
+ },
+ "additionalProperties": false,
+ "testcase": {
+ "type": ["shield-study-addon", "shield-study"],
+ "n": 100,
+ "minimumTimestamp": 1523968204184,
+ "headersOnly": false
+ }
+ },
+ {
+ "id": "anEndingAnswer",
+ "$schema": "http://json-schema.org/draft-04/schema",
+ "type": "object",
+ "additionalProperties": true
+ }
+ ],
+ "functions": [
+ {
+ "name": "setup",
+ "type": "function",
+ "async": true,
+ "description":
+ "Attempt an setup/enrollment, with these effects:\n\n- sets 'studyType' as Shield or Pioneer\n - affects telemetry\n - (5.2+ TODO) watches for dataPermission changes that should *always*\n stop that kind of study\n\n- Use or choose variation\n - `testing.variation` if present\n - OR (internal) deterministicVariation\n from `weightedVariations`\n based on hash of\n\n - activeExperimentName\n - clientId\n\n- During firstRun[1] only:\n - set firstRunTimestamp pref value\n - send 'enter' ping\n - if `allowEnroll`, send 'install' ping\n - else endStudy(\"ineligible\") and return\n\n- Every Run\n - setActiveExperiment(studySetup)\n - monitor shield | pioneer permission endings\n - suggests alarming if `expire` is set.\n\nReturns:\n- studyInfo object (see `getStudyInfo`)\n\nTelemetry Sent (First run only)\n\n - enter\n - install\n\nFires Events\n\n(At most one of)\n- study:onReady OR\n- study:onEndStudy\n\nPreferences set\n- `shield.${runtime.id}.firstRunTimestamp`\n\nNote:\n1. allowEnroll is ONLY used during first run (install)\n",
+ "parameters": [
+ {
+ "name": "studySetup",
+ "$ref": "studySetup"
+ }
+ ],
+ "returns": [
+ {
+ "$ref": "studyInfoObject"
+ }
+ ]
+ },
+ {
+ "name": "endStudy",
+ "type": "function",
+ "async": true,
+ "defaultReturn": {
+ "urls": ["url1", "url2"],
+ "endingName": "some-reason"
+ },
+ "description":
+ "Signal to browser.study that it should end.\n\nUsage scenarios:\n- add-ons defined\n - positive endings (tried feature)\n - negative endings (client clicked 'no thanks')\n - expiration / timeout (feature should last for 14 days then uninstall)\n\nLogic:\n- If study has already ended, do nothing.\n- Else: END\n\nEND:\n- record internally that study is ended.\n- disable all methods that rely on configuration / setup.\n- clear all prefs stored by `browser.study`\n- fire telemetry pings for:\n - 'exit'\n - the ending, one of:\n\n \"ineligible\",\n \"expired\",\n \"user-disable\",\n \"ended-positive\",\n \"ended-neutral\",\n \"ended-negative\",\n\n- augment all ending URLs with query URLs\n- fire 'study:end' event to `browser.study.onEndStudy` handlers.\n\nAdd-on should then do\n- open returned URLs\n- feature specific cleanup\n- uninstall the add-on\n\nNote:\n1. calling this function multiple time is safe.\n`browser.study` will choose the\n",
+ "parameters": [
+ {
+ "name": "anEndingAlias",
+ "type": "string"
+ }
+ ],
+ "returns": [
+ {
+ "$ref": "anEndingAnswer"
+ }
+ ]
+ },
+ {
+ "name": "getStudyInfo",
+ "type": "function",
+ "async": true,
+ "description":
+ "current study configuration, including\n- variation\n- activeExperimentName\n- delayInMinutes\n- firstRunTimestamp\n- isFirstRun\n\nBut not:\n- telemetry clientId\n\nThrows Error if called before `browser.study.setup`\n",
+ "defaultReturn": {
+ "variation": "styleA",
+ "firstRunTimestamp": 1523968204184,
+ "activeExperimentName": "some experiment",
+ "delayInMinutes": 12
+ },
+ "parameters": [],
+ "returns": [
+ {
+ "$ref": "studyInfoObject"
+ }
+ ]
+ },
+ {
+ "name": "getDataPermissions",
+ "type": "function",
+ "async": true,
+ "description":
+ "Object of current dataPermissions (shield enabled true/false, pioneer enabled true/false)",
+ "defaultReturn": {
+ "shield": true,
+ "pioneer": false
+ },
+ "parameters": [],
+ "returns": [
+ {
+ "$ref": "dataPermissionsObject"
+ }
+ ]
+ },
+ {
+ "name": "sendTelemetry",
+ "type": "function",
+ "description":
+ "Send Telemetry using appropriate shield or pioneer methods.\n\nNote: The payload must adhere to the `data.attributes` property in the [`shield-study-addon`](https://github.com/mozilla-services/mozilla-pipeline-schemas/blob/dev/templates/include/telemetry/shieldStudyAddonPayload.3.schema.json) schema. That is, it must be a flat object with string keys and string values.\n\nNote:\n- no conversions / coercion of data happens.\n- undefined what happens if validation fails\n\nTBD fix the parameters here.\n",
+ "async": true,
+ "parameters": [
+ {
+ "name": "payload",
+ "$ref": "telemetryPayload"
+ }
+ ],
+ "defaultReturn": "undefined",
+ "returns": null
+ },
+ {
+ "name": "calculateTelemetryPingSize",
+ "type": "function",
+ "description":
+ "Calculate Telemetry using appropriate shield or pioneer methods.\n\nshield:\n- Calculate the size of a ping\n\npioneer:\n- Calculate the size of a ping that has Pioneer encrypted data\n",
+ "async": true,
+ "parameters": [
+ {
+ "name": "payload",
+ "$ref": "telemetryPayload"
+ }
+ ],
+ "defaultReturn": "undefined",
+ "returns": [
+ {
+ "type": "number"
+ }
+ ]
+ },
+ {
+ "name": "searchSentTelemetry",
+ "type": "function",
+ "async": true,
+ "description":
+ "Search locally stored telemetry pings using these fields (if set)\n\nn:\n if set, no more than `n` pings.\ntype:\n Array of 'ping types' (e.g., main, crash, shield-study-addon) to filter\nminimumTimestamp:\n only pings after this timestamp.\nheadersOnly:\n boolean. If true, only the 'headers' will be returned.\n\nPings will be returned sorted by timestamp with most recent first.\n\nUsage scenarios:\n- enrollment / eligiblity using recent Telemetry behaviours or client environment\n- add-on testing scenarios\n",
+ "defaultReturn": [
+ {
+ "pingType": "main"
+ }
+ ],
+ "parameters": [
+ {
+ "name": "searchTelemetryQuery",
+ "$ref": "searchTelemetryQuery"
+ }
+ ]
+ },
+ {
+ "name": "getTestingOverrides",
+ "type": "function",
+ "async": true,
+ "description":
+ "Returns an object with the following keys:\n variationName - to be able to test specific variations\n firstRunTimestamp - to be able to test the expiration event\n expired - to be able to test the behavior of an already expired study\nUsed to override study testing flags in getStudySetup().\nThe values are set by the corresponding preference under the `extensions.${widgetId}.test.*` preference branch.\n",
+ "parameters": []
+ },
+ {
+ "name": "validateJSON",
+ "type": "function",
+ "async": true,
+ "defaultReturn": {
+ "valid": true,
+ "errors": []
+ },
+ "description":
+ "Using AJV, do jsonschema validation of an object. Can be used to validate your arguments, packets at client.",
+ "parameters": [
+ {
+ "name": "someJson",
+ "type": "object",
+ "additionalProperties": true
+ },
+ {
+ "name": "jsonschema",
+ "type": "object",
+ "descripton": "a valid jsonschema object",
+ "additionalProperties": true
+ }
+ ],
+ "returns": [
+ {
+ "type": "object"
+ },
+ {
+ "parameters": null,
+ "valid": [
+ {
+ "type": "boolean"
+ }
+ ],
+ "errors": [
+ {
+ "type": "array"
+ }
+ ]
+ }
+ ]
+ }
+ ],
+ "events": [
+ {
+ "name": "onReady",
+ "type": "function",
+ "defaultReturn": {
+ "variation": "styleA",
+ "firstRunTimestamp": 1523968204184
+ },
+ "description":
+ "Fires when the study is 'ready' for the feature to startup.",
+ "parameters": [
+ {
+ "name": "studyInfo",
+ "type": "object"
+ }
+ ]
+ },
+ {
+ "name": "onEndStudy",
+ "type": "function",
+ "defaultReturn": {
+ "urls": [],
+ "reason": "some-reason"
+ },
+ "description":
+ "Listen for when the study wants to end.\n\nAct on it by\n- opening surveyUrls\n- tearing down your feature\n- uninstalling the add-on\n",
+ "parameters": [
+ {
+ "name": "ending",
+ "type": "object"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "namespace": "study.logger",
+ "description":
+ "For study developers to be able to log messages which are hidden by default but can\nbe displayed via a preference (not currently possible with avoid console.{info,log,debug,warn,error}).\nLog messages will be prefixed with the add-on's widget id and the log level is controlled by the\n`shieldStudy.logLevel` preference.\nNote that since there is no way to handle an arbitrarily variable number of arguments in the schema,\nall values to log needs to be sent as a single variable.\nUsage example: await browser.study.logger.log(\"foo\");\nUsage example (multiple things to log): await browser.study.logger.log([\"foo\", bar]);\n",
+ "functions": [
+ {
+ "name": "info",
+ "type": "function",
+ "async": true,
+ "description": "Corresponds to console.info",
+ "parameters": [
+ {
+ "name": "values",
+ "type": "any"
+ }
+ ]
+ },
+ {
+ "name": "log",
+ "type": "function",
+ "async": true,
+ "description": "Corresponds to console.log",
+ "parameters": [
+ {
+ "name": "values",
+ "type": "any"
+ }
+ ]
+ },
+ {
+ "name": "debug",
+ "type": "function",
+ "async": true,
+ "description": "Corresponds to console.debug",
+ "parameters": [
+ {
+ "name": "values",
+ "type": "any"
+ }
+ ]
+ },
+ {
+ "name": "warn",
+ "type": "function",
+ "async": true,
+ "description": "Corresponds to console.warn",
+ "parameters": [
+ {
+ "name": "values",
+ "type": "any"
+ }
+ ]
+ },
+ {
+ "name": "error",
+ "type": "function",
+ "async": true,
+ "description": "Corresponds to console.error",
+ "parameters": [
+ {
+ "name": "values",
+ "type": "any"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "namespace": "studyDebug",
+ "description": "Interface for Test Utilities",
+ "apiVersion": 5,
+ "functions": [
+ {
+ "name": "throwAnException",
+ "type": "function",
+ "description":
+ "Throws an exception from a privileged function - for making sure that we can catch these in our web extension",
+ "async": false,
+ "parameters": [
+ {
+ "name": "message",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "throwAnExceptionAsync",
+ "type": "function",
+ "description":
+ "Throws an exception from a privileged async function - for making sure that we can catch these in our web extension",
+ "async": true,
+ "parameters": [
+ {
+ "name": "message",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "firstSeen",
+ "type": "function",
+ "async": true,
+ "description": "",
+ "parameters": []
+ },
+ {
+ "name": "setActive",
+ "type": "function",
+ "async": true,
+ "description": "",
+ "parameters": []
+ },
+ {
+ "name": "startup",
+ "type": "function",
+ "async": true,
+ "description": "",
+ "parameters": [
+ {
+ "name": "details",
+ "type": "object",
+ "additionalProperties": true
+ }
+ ]
+ },
+ {
+ "name": "setFirstRunTimestamp",
+ "type": "function",
+ "async": true,
+ "description":
+ "Set the pref for firstRunTimestamp, to simulate:\n- 2nd run\n- other useful tests around expiration and states.\n",
+ "parameters": [
+ {
+ "name": "timestamp",
+ "type": "number",
+ "minimum": 1
+ }
+ ]
+ },
+ {
+ "name": "reset",
+ "type": "function",
+ "async": true,
+ "description":
+ "\nReset the studyUtils _internals, for debugging purposes.\n",
+ "parameters": []
+ },
+ {
+ "name": "getInternals",
+ "type": "function",
+ "async": true,
+ "description":
+ "Return `_internals` of the studyUtils object.\n\nUse this for debugging state.\n\nAbout `this._internals`:\n- variation: (chosen variation, `setup` )\n- isEnding: bool `endStudy`\n- isSetup: bool `setup`\n- isFirstRun: bool `setup`, based on pref\n- studySetup: bool `setup` the config\n- seenTelemetry: array of seen telemetry. Fully populated only if studySetup.telemetry.internalTelemetryArchive is true\n- prefs: object of all created prefs and their names\n",
+ "parameters": []
+ },
+ {
+ "name": "getInternalTestingOverrides",
+ "type": "function",
+ "async": true,
+ "description":
+ "Returns an object with the following keys:\n studyType - to be able to test add-ons with different studyType configurations\nUsed to override study testing flags in getStudySetup().\nThe values are set by the corresponding preference under the `extensions.${widgetId}.test.*` preference branch.\n",
+ "parameters": []
+ }
+ ]
+ }
+]
diff --git a/toolkit/components/normandy/jar.mn b/toolkit/components/normandy/jar.mn
--- a/toolkit/components/normandy/jar.mn
+++ b/toolkit/components/normandy/jar.mn
@@ -10,6 +10,9 @@ toolkit.jar:
res/normandy/actions/ (./actions/*.jsm)
res/normandy/actions/schemas/index.js (./actions/schemas/index.js)
+% resource normandy-studies %res/normandy/studies/
+ res/normandy/studies (./studies/*)
+
% resource normandy-content %res/normandy/content/ contentaccessible=yes
res/normandy/content/ (./content/*)
diff --git a/toolkit/components/normandy/studies/DataPermissions.jsm b/toolkit/components/normandy/studies/DataPermissions.jsm
new file mode 100644
--- /dev/null
+++ b/toolkit/components/normandy/studies/DataPermissions.jsm
@@ -0,0 +1,39 @@
+const { Services } = ChromeUtils.import(
+ "resource://gre/modules/Services.jsm",
+ {},
+);
+const { AddonManager } = ChromeUtils.import(
+ "resource://gre/modules/AddonManager.jsm",
+ {},
+);
+
+/**
+ * Checks to see if SHIELD is enabled for a user.
+ *
+ * @returns {Boolean}
+ * A boolean to indicate SHIELD opt-in status.
+ */
+export function isShieldEnabled() {
+ return Services.prefs.getBoolPref("app.shield.optoutstudies.enabled", true);
+}
+
+/**
+ * Checks to see if the user has opted in to Pioneer. This is
+ * done by checking that the opt-in addon is installed and active.
+ *
+ * @returns {Boolean}
+ * A boolean to indicate opt-in status.
+ */
+export async function isUserOptedInToPioneer() {
+ const addon = await AddonManager.getAddonByID("pioneer-opt-in@mozilla.org");
+ return isShieldEnabled() && addon !== null && addon.isActive;
+}
+
+export async function getDataPermissions() {
+ const shield = isShieldEnabled();
+ const pioneer = await isUserOptedInToPioneer();
+ return {
+ shield,
+ pioneer,
+ };
+}
diff --git a/toolkit/components/normandy/studies/GetPingSize.jsm b/toolkit/components/normandy/studies/GetPingSize.jsm
new file mode 100644
--- /dev/null
+++ b/toolkit/components/normandy/studies/GetPingSize.jsm
@@ -0,0 +1,23 @@
+
+var EXPORTED_SYMBOLS = ["GetPingSize"];
+
+class GetPingSize {
+ /**
+ * Calculate the size of a ping.
+ *
+ * @param {Object} payload
+ * The data payload of the ping.
+ *
+ * @returns {Number}
+ * The total size of the ping.
+ */
+ getPingSize(payload) {
+ const converter = Cc[
+ "@mozilla.org/intl/scriptableunicodeconverter"
+ ].createInstance(Ci.nsIScriptableUnicodeConverter);
+ converter.charset = "UTF-8";
+ let utf8Payload = converter.ConvertFromUnicode(JSON.stringify(payload));
+ utf8Payload += converter.Finish();
+ return utf8Payload.length;
+ }
+}
\ No newline at end of file
diff --git a/toolkit/components/normandy/studies/MakeWidgetId.jsm b/toolkit/components/normandy/studies/MakeWidgetId.jsm
new file mode 100644
--- /dev/null
+++ b/toolkit/components/normandy/studies/MakeWidgetId.jsm
@@ -0,0 +1,6 @@
+function makeWidgetId(id) {
+ id = id.toLowerCase();
+ return id.replace(/[^a-z0-9_-]/g, "_");
+}
+
+export default makeWidgetId;
diff --git a/toolkit/components/normandy/studies/Sampling.jsm b/toolkit/components/normandy/studies/Sampling.jsm
new file mode 100644
--- /dev/null
+++ b/toolkit/components/normandy/studies/Sampling.jsm
@@ -0,0 +1,103 @@
+/** Sampling utilies, including hashing functions */
+
+const { Services } = ChromeUtils.import(
+ "resource://gre/modules/Services.jsm",
+ {},
+);
+const { TextEncoder } = Cu.getGlobalForObject(Services);
+
+Cu.importGlobalProperties(["crypto"]);
+
+var EXPORTED_SYMBOLS = ["Sampling"];
+
+class Sampling {
+/**
+ * Given sample weights (weightedVariations) and a particular position
+ * (fraction), return a variation. If no fraction given, return a variation
+ * at random fraction proportional to the weightVariations object
+ * @param {Object[]} weightedVariations - the array of branch name:weight pairs
+ * used to randomly assign the user to a branch
+ * @param {Number} fraction - a number (0 <= fraction < 1)
+ * @returns {Object} - the variation object in weightedVariations for the given
+ * fraction
+ *
+ */
+ chooseWeighted(weightedVariations, fraction = Math.random()) {
+ /*
+ weightedVariations, list of:
+ {
+ name: string of any length
+ weight: float >= 0
+ }
+ */
+
+ const weights = weightedVariations.map(x => x.weight || 1);
+ const partial = this.cumsum(weights);
+ const total = weights.reduce((a, b) => a + b);
+ for (let ii = 0; ii < weightedVariations.length; ii++) {
+ if (fraction <= partial[ii] / total) {
+ return weightedVariations[ii];
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Converts a string into a fraction (0 <= fraction < 1) based on the first
+ * X bits of its sha256 hexadecimal representation
+ * Note: Salting (adding the study name to the telemetry clientID) ensures
+ * that the same user gets a different bucket/hash for each study.
+ * Hashing of the salted string ensures uniform hashing; i.e. that every
+ * bucket/variation gets filled.
+ * @param {string} saltedString - a salted string used to create a hash for
+ * the user
+ * @param {Number} bits - The first number of bits to use in the sha256 hex
+ * representation
+ * @returns {Number} - a fraction (0 <= fraction < 1)
+ */
+ async hashFraction(saltedString, bits = 12) {
+ const hash = await this.sha256(saltedString);
+ return parseInt(hash.substr(0, bits), 16) / Math.pow(16, bits);
+ }
+
+ /**
+ * Converts a string into its sha256 hexadecimal representation.
+ * Note: This is ultimately used to make a hash of the user's telemetry clientID
+ * and the study name.
+ * @param {string} message - The message to convert.
+ * @returns {string} - a hexadecimal, 256-bit hash
+ */
+ async sha256(message) {
+ // encode as UTF-8
+ const msgBuffer = new TextEncoder("utf-8").encode(message);
+ // hash the message
+ const hashBuffer = await crypto.subtle.digest("SHA-256", msgBuffer);
+ // convert ArrayBuffer to Array
+ const hashArray = Array.from(new Uint8Array(hashBuffer));
+ // convert bytes to hex string
+ const hashHex = hashArray
+ .map(b => ("00" + b.toString(16)).slice(-2))
+ .join("");
+ return hashHex;
+ }
+
+ /**
+ * Converts an array of length N into a cumulative sum array of length N,
+ * where n_i = sum(array.slice(0,i)) i.e. each element is the sum of all
+ * elements up to and including that element
+ * This is ultimately used for turning sample weights (AKA weightedVariations)
+ * into right hand limits (>= X) to deterministically select which variation
+ * a user receives.
+ * @example [.25,.3,.45] => [.25,.55,1.0]; if a user's sample weight were .25,
+ * they would fall into the left-most bucket
+ * @param {Number[]} arr - An array of sample weights (0 <= sample weight < 1)
+ * @returns {Number[]} - A cumulative sum array of sample weights
+ * (0 <= sample weight <= 1)
+ */
+ cumsum(arr) {
+ return arr.reduce(function(r, c, i) {
+ r.push((r[i - 1] || 0) + c);
+ return r;
+ }, []);
+ }
+};
diff --git a/toolkit/components/normandy/studies/StudyUtils.jsm b/toolkit/components/normandy/studies/StudyUtils.jsm
new file mode 100644
--- /dev/null
+++ b/toolkit/components/normandy/studies/StudyUtils.jsm
@@ -0,0 +1,749 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+var {XPCOMUtils} = ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
+
+XPCOMUtils.defineLazyModuleGetters(this, {
+ Log: "resource://gre/modules/Log.jsm",
+ PioneerStudyType: "resource://normandy/studies/studyTypes/Pioneer.jsm",
+ Sampling: "resource://normandy/studies/Sampling.jsm",
+ ShieldStudyType: "resource://normandy/studies/studyTypes/Shield.jsm",
+});
+
+const LOGGER_ID_BASE = "study.utils";
+
+var EXPORTED_SYMBOLS = ["StudyUtils"];
+
+
+/*
+* Supports the `browser.study` webExtensionExperiment api.
+*
+* - Conversion of v4 "StudyUtils.jsm".
+* - Contains the 'dangerous' code.
+* - Creates and exports the `studyUtils` singleton
+* - does all the actuall privileged work including Telemetry
+*
+* See API.md at:
+* https://github.com/mozilla/shield-studies-addon-utils/blob/develop/docs/api.md
+*
+* Note: There are a number of methods that won't work if the
+* setup method has not executed (they perform a check with the
+* `throwIfNotSetup` method). The setup method ensures that the
+* studySetup data passed in is valid per the studySetup schema.
+*
+* Tests for this module are at /test-addon.
+*/
+
+// FIXME
+const UTILS_VERSION = "1.0";
+const PACKET_VERSION = 3;
+
+const { Services } = ChromeUtils.import(
+ "resource://gre/modules/Services.jsm",
+ {},
+);
+const { Preferences } = ChromeUtils.import(
+ "resource://gre/modules/Preferences.jsm",
+ {},
+);
+
+// Cu.importGlobalProperties(["URL", "URLSearchParams"]);
+
+const { ExtensionUtils } = ChromeUtils.import(
+ "resource://gre/modules/ExtensionUtils.jsm",
+ {},
+);
+// eslint-disable-next-line no-undef
+const { ExtensionError } = ExtensionUtils;
+
+// telemetry utils
+const { TelemetryEnvironment } = ChromeUtils.import(
+ "resource://gre/modules/TelemetryEnvironment.jsm",
+ null,
+);
+
+/** Simple spread/rest based merge, using Object.assign.
+ *
+ * Right-most overrides, top level only, by full value replacement.
+ *
+ * Note: Unlike deep merges might not handle symbols and other things.
+ *
+ * @param {...Object} sources - 1 or more sources
+ * @returns {Object} - the resulting merged object
+ */
+function merge(...sources) {
+ return Object.assign({}, ...sources);
+}
+
+/**
+ * Appends a query string to a url.
+ * @param {string} url - a base url to append; must be static (data) or external
+ * @param {Object} args - query arguments, one or more object literal used to
+ * build a query string
+ *
+ * @returns {string} - an absolute url appended with a query string
+ */
+function mergeQueryArgs(url, ...args) {
+ const U = new URL(url);
+ // get the query string already attached to url, if it exists
+ let q = U.search || "?";
+ // create an interface to interact with the query string
+ q = new URLSearchParams(q);
+ const merged = merge({}, ...args);
+ // Set each search parameter in "merged" to its value in the query string,
+ // building up the query string one search parameter at a time.
+ Object.keys(merged).forEach(k => {
+ q.set(k, merged[k]);
+ });
+ // append our new query string to the URL object made with "url"
+ U.search = q.toString();
+ // return the full url, with the appended query string
+ return U.toString();
+}
+
+/**
+ * Class representing utilities singleton for shield studies.
+ */
+class StudyUtils {
+ /**
+ * Create a StudyUtils instance to power the `browser.study` API
+ *
+ * About `this._internals`:
+ * - variation: (chosen variation, `setup` )
+ * - isEnding: bool `endStudy`
+ * - isSetup: bool `setup`
+ * - isFirstRun: bool `setup`, based on pref
+ * - studySetup: bool `setup` the config
+ * - seenTelemetry: array of seen telemetry. Fully populated only if studySetup.telemetry.internalTelemetryArchive is true
+ * - prefs: object of all created prefs and their names
+ * - endingRequested: string of ending name
+ * - endingReturns: object with useful ending instructions
+ *
+ * Returned by `studyDebug.getInternals()` for testing
+ * Reset by `studyDebug.reset` and `studyUtils.reset`
+ *
+ * About: `this._extensionManifest`
+ * - mirrors the extensionManifest at the time of api creation
+ * - used by uninstall, and to name the firstRunTimestamp pref
+ *
+ */
+ constructor() {
+ // Expose sampling methods onto the exported studyUtils singleton
+ this.sampling = new Sampling();
+
+ this._extensionManifest = {};
+
+ // internals, also used by `studyDebug.getInternals()`
+ // either setup() or reset() will create, using extensionManifest
+ this._internals = {};
+ }
+
+ get logger() {
+ let id = this.id || "<unknown>";
+ return Log.repository.getLogger(LOGGER_ID_BASE + id);
+ }
+
+ logDebug(message) {
+ this._logMessage(message, "debug");
+ }
+
+ logInfo(message) {
+ this._logMessage(message, "info");
+ }
+
+ logWarning(message) {
+ this._logMessage(message, "warn");
+ }
+
+ logError(message) {
+ this._logMessage(message, "error");
+ }
+
+ _logMessage(message, severity) {
+ this.logger[severity](`Loading extension '${this.id}': ${message}`);
+ }
+
+ // TODO either find something in-tree or document this
+ makeWidgetId(id) {
+ id = id.toLowerCase();
+ return id.replace(/[^a-z0-9_-]/g, "_");
+ }
+
+ _createInternals() {
+ if (!this._extensionManifest) {
+ throw new ExtensionError(
+ "_createInternals needs `setExtensionManifest`. This should be done by `getApi`.",
+ );
+ }
+
+ const widgetId = this.makeWidgetId(
+ this._extensionManifest.applications.gecko.id,
+ );
+
+ const internals = {
+ widgetId,
+ variation: undefined,
+ studySetup: undefined,
+ isFirstRun: false,
+ isSetup: false,
+ isEnding: false,
+ isEnded: false,
+ seenTelemetry: [],
+ prefs: {
+ firstRunTimestamp: `shield.${widgetId}.firstRunTimestamp`,
+ },
+ endingRequested: undefined,
+ endingReturned: undefined,
+ };
+ Object.seal(internals);
+ return internals;
+ }
+
+ /**
+ * Checks if the StudyUtils.setup method has been called
+ * @param {string} name - the name of a StudyUtils method
+ * @returns {void}
+ */
+ throwIfNotSetup(name = "unknown") {
+ if (!this._internals.isSetup)
+ throw new ExtensionError(
+ name + ": this method can't be used until `setup` is called",
+ );
+ }
+
+ /**
+ * Validates the studySetup object passed in from the add-on.
+ * @param {Object} studySetup - the studySetup object, see schema.studySetup.json
+ * @returns {StudyUtils} - the StudyUtils class instance
+ */
+ async setup(studySetup) {
+ if (!this._internals) {
+ throw new ExtensionError("StudyUtils internals are not initiated");
+ }
+
+ this.logInfo(`setting up! -- ${JSON.stringify(studySetup)}`);
+
+ if (this._internals.isSetup) {
+ throw new ExtensionError("StudyUtils is already setup");
+ }
+ this._internals.studySetup = studySetup;
+
+ // Different study types treat data and configuration differently
+ if (studySetup.studyType === "shield") {
+ this.studyType = new ShieldStudyType(this);
+ }
+ if (studySetup.studyType === "pioneer") {
+ this.studyType = new PioneerStudyType(this);
+ }
+
+ function getVariationByName(name, variations) {
+ if (!name) return null;
+ const chosen = variations.filter(x => x.name === name)[0];
+ if (!chosen) {
+ throw new ExtensionError(
+ `setup error: testing.variationName "${name}" not in ${JSON.stringify(
+ variations,
+ )}`,
+ );
+ }
+ return chosen;
+ }
+ // variation: decide and set
+ const variation =
+ getVariationByName(
+ studySetup.testing.variationName,
+ studySetup.weightedVariations,
+ ) ||
+ (await this._deterministicVariation(
+ studySetup.activeExperimentName,
+ studySetup.weightedVariations,
+ ));
+ this.logDebug(`setting up: variation ${variation.name}`);
+
+ this._internals.variation = variation;
+ this._internals.isSetup = true;
+
+ // isFirstRun? ever seen before?
+ const firstRunTimestamp = this.getFirstRunTimestamp();
+ // 'firstSeen' is the first telemetry we attempt to send. needs 'isSetup'
+ if (firstRunTimestamp !== null) {
+ this._internals.isFirstRun = false;
+ } else {
+ // 'enter' telemetry, and firstSeen
+ await studyUtils.firstSeen();
+ }
+
+ // Note: is allowed to enroll is handled at API.
+ // FIXME: 5.1 maybe do it here?
+ return this;
+ }
+
+ /**
+ * Resets the state of the study. Suggested use is for testing.
+ * @returns {Object} internals internals
+ */
+ reset() {
+ this._internals = this._createInternals();
+ this.studyType = null;
+ this.resetFirstRunTimestamp();
+ }
+
+ /**
+ * Gets the variation for the StudyUtils instance.
+ * @returns {Object} - the study variation for this user
+ */
+ getVariation() {
+ this.throwIfNotSetup("getvariation");
+ this.logDebug(
+ `getVariation: ${JSON.stringify(this._internals.variation)}`,
+ );
+ return this._internals.variation;
+ }
+
+ setExtensionManifest(extensionManifest) {
+ this._extensionManifest = extensionManifest;
+ }
+
+ /**
+ * @returns {any} the firstRunTimestamp as a number in case the preference is set, or null if the preference is not set
+ */
+ getFirstRunTimestamp() {
+ if (
+ typeof this._internals.studySetup.testing.firstRunTimestamp !==
+ "undefined" &&
+ this._internals.studySetup.testing.firstRunTimestamp !== null
+ ) {
+ return Number(this._internals.studySetup.testing.firstRunTimestamp);
+ }
+ const firstRunTimestampPreferenceValue = Services.prefs.getStringPref(
+ this._internals.prefs.firstRunTimestamp,
+ null,
+ );
+ return firstRunTimestampPreferenceValue !== null
+ ? Number(firstRunTimestampPreferenceValue)
+ : null;
+ }
+
+ setFirstRunTimestamp(timestamp) {
+ const pref = this._internals.prefs.firstRunTimestamp;
+ return Services.prefs.setStringPref(pref, "" + timestamp);
+ }
+
+ resetFirstRunTimestamp() {
+ const pref = this._internals.prefs.firstRunTimestamp;
+ Preferences.reset(pref);
+ }
+
+ /** Calculate time left in study given `studySetup.expire.days` and firstRunTimestamp
+ *
+ * Safe to use with `browser.alarms.create{ delayInMinutes, }`
+ *
+ * A value of 0 means "the past / now".
+ *
+ * @return {Number} delayInMinutes Either the time left or Number.MAX_SAFE_INTEGER
+ */
+ getDelayInMinutes() {
+ if (this._internals.studySetup.testing.expired === true) {
+ return 0;
+ }
+ const toMinutes = 1 / (1000 * 60);
+ const days = this._internals.studySetup.expire.days;
+ let delayInMs = Number.MAX_SAFE_INTEGER; // approx 286,000 years
+ if (days) {
+ // days in ms
+ const ms = days * 86400 * 1000;
+ const firstrun = this.getFirstRunTimestamp();
+ if (firstrun === null) {
+ return null;
+ }
+ delayInMs = Math.max(firstrun + ms - Date.now(), 0);
+ }
+ return delayInMs * toMinutes;
+ }
+
+ /**
+ * Gets the telemetry client ID for the user.
+ * @returns {string} - the telemetry client ID
+ */
+ async getTelemetryId() {
+ return this.studyType.getTelemetryId();
+ }
+
+ /**
+ * Gets the Shield recipe client ID.
+ * @returns {string} - the Shield recipe client ID.
+ */
+ getShieldId() {
+ const key = "extensions.shield-recipe-client.user_id";
+ return Services.prefs.getStringPref(key, "");
+ }
+
+ /**
+ * Packages information about the study into an object.
+ * @returns {Object} - study information, see schema.studySetup.json
+ */
+ info() {
+ this.logDebug("getting info");
+ this.throwIfNotSetup("info");
+
+ const studyInfo = {
+ activeExperimentName: this._internals.studySetup.activeExperimentName,
+ isFirstRun: this._internals.isFirstRun,
+ firstRunTimestamp: this.getFirstRunTimestamp(),
+ variation: this.getVariation(),
+ shieldId: this.getShieldId(),
+ delayInMinutes: this.getDelayInMinutes(),
+ };
+ const now = new Date();
+ const diff = Number(now) - studyInfo.firstRunTimestamp;
+ this.logDebug(
+ "Study info date information: now, new Date(firstRunTimestamp), firstRunTimestamp, diff (in minutes), delayInMinutes",
+ now,
+ new Date(studyInfo.firstRunTimestamp),
+ studyInfo.firstRunTimestamp,
+ diff / 1000 / 60,
+ studyInfo.delayInMinutes,
+ );
+ return studyInfo;
+ }
+
+ /**
+ * Get the telemetry configuration for the study.
+ * @returns {Object} - the telemetry configuration, see schema.studySetup.json
+ */
+ get telemetryConfig() {
+ this.throwIfNotSetup("telemetryConfig");
+ return this._internals.studySetup.telemetry;
+ }
+
+ /**
+ * Deterministically selects and returns the study variation for the user.
+ * @param {string} activeExperimentName name to use as part of the hash
+ * @param {Object[]} weightedVariations - see schema.weightedVariations.json
+ * @param {Number} fraction - a number (0 <= fraction < 1); can be set explicitly for testing
+ * @returns {Object} - the study variation for this user
+ */
+ async _deterministicVariation(
+ activeExperimentName,
+ weightedVariations,
+ fraction = null,
+ ) {
+ // this is the standard arm choosing method, used by both shield and pioneer studies
+ if (fraction === null) {
+ // hash the studyName and telemetryId to get the same branch every time.
+ const clientId = await this.getTelemetryId();
+ fraction = await this.sampling.hashFraction(
+ activeExperimentName + clientId,
+ 12,
+ );
+ }
+ this.logDebug(`_deterministicVariation`, weightedVariations);
+ return this.sampling.chooseWeighted(weightedVariations, fraction);
+ }
+
+ /**
+ * Sends an 'enter' telemetry ping for the study; should be called on add-on
+ * startup for the reason ADDON_INSTALL. For more on study states like 'enter'
+ * see ABOUT.md at github.com/mozilla/shield-studies-addon-template
+ *
+ * Side effects:
+ * - sends 'enter'
+ * - sets this._internals.prefs.firstRunTimestamp to Date.now()
+ *
+ * @returns {void}
+ */
+ async firstSeen() {
+ this.throwIfNotSetup("firstSeen uses telemetry.");
+ this.logDebug(`attempting firstSeen`);
+ this._internals.isFirstRun = true;
+ await this._telemetry({ study_state: "enter" }, "shield-study");
+ this.setFirstRunTimestamp(Date.now());
+ }
+
+ /**
+ * Marks the study's telemetry pings as being part of this experimental
+ * cohort in a way that downstream data pipeline tools
+ * (like ExperimentsViewer) can use it.
+ * @returns {void}
+ */
+ setActive() {
+ this.throwIfNotSetup("setActive uses telemetry.");
+ const info = this.info();
+ this.logDebug(
+ "marking TelemetryEnvironment",
+ info.activeExperimentName,
+ info.variation.name,
+ );
+ TelemetryEnvironment.setExperimentActive(
+ info.activeExperimentName,
+ info.variation.name,
+ );
+ }
+
+ /**
+ * Removes the study from the active list of telemetry experiments
+ * @returns {void}
+ */
+ unsetActive() {
+ this.throwIfNotSetup("unsetActive uses telemetry.");
+ const info = this.info();
+ this.logDebug(
+ "unmarking TelemetryEnvironment",
+ info.activeExperimentName,
+ info.variation.name,
+ );
+ TelemetryEnvironment.setExperimentInactive(info.activeExperimentName);
+ }
+
+ /**
+ * Adds the study to the active list of telemetry experiments and sends the
+ * "installed" telemetry ping if applicable
+ * @param {string} reason - The reason the add-on has started up
+ * @returns {void}
+ */
+ async startup() {
+ this.throwIfNotSetup("startup");
+ const isFirstRun = this._internals.isFirstRun;
+ this.logDebug(`startup. setting active. isFirstRun? ${isFirstRun}`);
+ this.setActive();
+ if (isFirstRun) {
+ await this._telemetry({ study_state: "installed" }, "shield-study");
+ }
+ }
+
+ /**
+ * Ends the study:
+ * - Removes the study from the active list of telemetry experiments
+ * - Sends a telemetry ping about the nature of the ending
+ * (positive, neutral, negative)
+ * - Sends an exit telemetry ping
+ * @param {string} endingName - The reason the study is ending, see
+ * schema.studySetup.json
+ * @returns {Object} endingReturned _internals.endingReturned
+ */
+ async endStudy(endingName) {
+ this.throwIfNotSetup("endStudy");
+
+ // also handle default endings.
+ const alwaysHandle = ["ineligible", "expired", "user-disable"];
+ let ending = this._internals.studySetup.endings[endingName];
+ if (!ending) {
+ // a 'no-action' ending is okay for the 'always handle'
+ if (alwaysHandle.includes(endingName)) {
+ ending = {};
+ } else {
+ throw new ExtensionError(`${endingName} isn't known ending`);
+ }
+ }
+
+ // throw if already ending
+ if (this._internals.isEnding) {
+ this.logDebug("endStudy, already ending!");
+ throw new ExtensionError(
+ `endStudy, requested: ${endingName}, but already ending ${
+ this._internals.endingRequested
+ }`,
+ );
+ }
+
+ // if not already ending, claim it. We are first!
+ this._internals.isEnding = true;
+ this._internals.endingRequested = endingName;
+
+ this.logDebug(`endStudy ${endingName}`);
+ await this.unsetActive();
+
+ // do the work to end the studyUtils involvement
+
+ // 1. Telemetry for ending
+ const { fullname } = ending;
+ let finalName = endingName;
+ switch (endingName) {
+ // handle the 'formal' endings (defined in parquet)
+ case "ineligible":
+ case "expired":
+ case "user-disable":
+ case "ended-positive":
+ case "ended-neutral":
+ case "ended-negative":
+ await this._telemetry(
+ {
+ study_state: endingName,
+ study_state_fullname: fullname || endingName,
+ },
+ "shield-study",
+ );
+ break;
+ default:
+ (finalName = ending.category || "ended-neutral");
+ // call all 'unknowns' as "ended-neutral"
+ await this._telemetry(
+ {
+ study_state: finalName,
+ study_state_fullname: endingName,
+ },
+ "shield-study",
+ );
+ break;
+ }
+ await this._telemetry({ study_state: "exit" }, "shield-study");
+
+ // 2. create ending instructions for the webExt to use
+ const out = {
+ shouldUninstall: true,
+ urls: [],
+ endingName,
+ };
+
+ // baseUrls: needs to be appended with query arguments before use,
+ // exactUrls: used as is
+ const { baseUrls, exactUrls } = ending;
+ if (exactUrls) {
+ out.urls.push(...exactUrls);
+ }
+ const qa = await this.endingQueryArgs();
+ qa.reason = finalName;
+ qa.fullreason = endingName;
+
+ if (baseUrls) {
+ for (const baseUrl of baseUrls) {
+ const fullUrl = mergeQueryArgs(baseUrl, qa);
+ out.urls.push(fullUrl);
+ }
+ }
+
+ out.queryArgs = qa;
+
+ // 3. Temporarily store information about the ending for test purposes
+ this._internals.endingReturned = out;
+ this._internals.isEnded = true; // done!
+
+ // 4. Make sure that future add-on installations are treated as new studies rather than a continuation of the previous one
+ this.resetFirstRunTimestamp();
+
+ return out;
+ }
+
+ /**
+ * Builds an object whose properties are query arguments that can be
+ * appended to a study ending url
+ * @returns {Object} - the query arguments for the study
+ */
+ async endingQueryArgs() {
+ this.throwIfNotSetup("endingQueryArgs");
+ const info = this.info();
+ const who = await this.getTelemetryId();
+ const queryArgs = {
+ shield: PACKET_VERSION,
+ study: info.activeExperimentName,
+ variation: info.variation.name,
+ updateChannel: Services.appinfo.defaultUpdateChannel,
+ fxVersion: Services.appinfo.version,
+ addon: this._extensionManifest.version, // addon version
+ who, // telemetry clientId
+ };
+ queryArgs.testing = Number(!this.telemetryConfig.removeTestingFlag);
+ return queryArgs;
+ }
+
+ /**
+ * Validates and submits telemetry pings from StudyUtils.
+ * @param {Object} data - the data to send as part of the telemetry packet
+ * @param {string} bucket - the type of telemetry packet to be sent
+ * @returns {Promise|boolean} - A promise that resolves with the ping id
+ * once the ping is stored or sent, or false if
+ * - there is a validation error,
+ * - the packet is of type "shield-study-error"
+ * - the study's telemetryConfig.send is set to false
+ */
+ async _telemetry(data, bucket = "shield-study-addon") {
+ this.throwIfNotSetup("_telemetry");
+ this.logDebug(`telemetry in: ${bucket} ${JSON.stringify(data)}`);
+ const info = this.info();
+ this.logDebug(`telemetry INFO: ${JSON.stringify(info)}`);
+
+ const payload = {
+ version: PACKET_VERSION,
+ study_name: info.activeExperimentName,
+ branch: info.variation.name,
+ addon_version: this._extensionManifest.version,
+ shield_version: UTILS_VERSION,
+ type: bucket,
+ data,
+ testing: !this.telemetryConfig.removeTestingFlag,
+ };
+
+ this.logDebug(`telemetry: ${JSON.stringify(payload)}`);
+
+ let pingId;
+
+ // during development, don't actually send
+ if (!this.telemetryConfig.send) {
+ this.logDebug("NOT sending. `telemetryConfig.send` is false");
+ pingId = false;
+ } else {
+ pingId = await this.studyType.sendTelemetry(bucket, payload);
+ }
+
+ // Store a copy of the ping if it's a shield-study or error ping, which are few in number, or if we have activated the internal telemetry archive configuration
+ if (
+ bucket === "shield-study" ||
+ bucket === "shield-study-error" ||
+ this.telemetryConfig.internalTelemetryArchive
+ ) {
+ this._internals.seenTelemetry.push({ id: pingId, payload });
+ }
+
+ return pingId;
+ }
+
+ /**
+ * Validates and submits telemetry pings from the add-on; mostly from
+ * webExtension messages.
+ * @param {Object} payload - the data to send as part of the telemetry packet
+ * @returns {Promise|boolean} - see StudyUtils._telemetry
+ */
+ async telemetry(payload) {
+ this.throwIfNotSetup("telemetry");
+ this.logDebug(`telemetry ${JSON.stringify(payload)}`);
+ const toSubmit = {
+ attributes: payload,
+ };
+ return this._telemetry(toSubmit, "shield-study-addon");
+ }
+
+ /**
+ * Submits error report telemetry pings.
+ * @param {Object} errorReport - the error report, see StudyUtils._telemetry
+ * @returns {Promise|boolean} - see StudyUtils._telemetry
+ */
+ telemetryError(errorReport) {
+ return this._telemetry(errorReport, "shield-study-error");
+ }
+
+ /** Calculate Telemetry using appropriate shield or pioneer methods.
+ *
+ * shield:
+ * - Calculate the size of a ping
+ *
+ * pioneer:
+ * - Calculate the size of a ping that has Pioneer encrypted data
+ *
+ * @param {Object} payload Non-nested object with key strings, and key values
+ * @returns {Promise<Number>} The total size of the ping.
+ */
+ async calculateTelemetryPingSize(payload) {
+ this.throwIfNotSetup("calculateTelemetryPingSize");
+ const toSubmit = {
+ attributes: payload,
+ };
+ return this.studyType.getPingSize(toSubmit, "shield-study-addon");
+ }
+}
+
+// TODO, use the usual es6 exports
+// Actually create the singleton.
+const studyUtils = new StudyUtils();
+this.studyUtils = studyUtils;
diff --git a/toolkit/components/normandy/studies/Telemetry.jsm b/toolkit/components/normandy/studies/Telemetry.jsm
new file mode 100644
--- /dev/null
+++ b/toolkit/components/normandy/studies/Telemetry.jsm
@@ -0,0 +1,58 @@
+/* eslint-env node */
+
+// TODO, eventually remove this. It's used by the Template testing, for now.
+
+// TODO, making this a separate file means that we have to pass the error from the other compartment.
+
+/**
+ * Returns array of pings of type `type` in reverse sorted order by timestamp
+ * first element is most recent ping
+ *
+ * searchTelemetryQuery
+ * - type: string or array of ping types
+ * - n: positive integer. at most n pings.
+ * - timestamp: only pings after this timestamp.
+ * - headersOnly: boolean, just the 'headers' for the pings, not the full bodies.
+ *
+ * TODO: Fix shortcoming:
+ * Some pings are sent immediately after one another and it's
+ * original sending order is not reflected by the return of
+ * TelemetryArchive.promiseArchivedPingList
+ * Thus, we can currently only test that the last two pings are the
+ * correct ones but not that their order is correct
+ *
+ *
+ * @param {Object<backstagePass>} TelemetryArchive from TelemetryArchive.jsm
+ * @param {ObjectsearchTelemetryQuery} searchTelemetryQuery See searchSentTelemetry
+ *
+ * @returns {Array} Array of found Telemetry Pings
+ */
+async function searchTelemetryArchive(TelemetryArchive, searchTelemetryQuery) {
+ let { type } = searchTelemetryQuery;
+ const { n, timestamp, headersOnly } = searchTelemetryQuery;
+ // {type, id, timestampCreated}
+ let pings = await TelemetryArchive.promiseArchivedPingList();
+ if (type && !Array.isArray(type)) {
+ type = [type];
+ }
+ if (type) pings = pings.filter(p => type.includes(p.type));
+
+ if (timestamp) pings = pings.filter(p => p.timestampCreated > timestamp);
+
+ if (pings.length === 0) {
+ return Promise.resolve([]);
+ }
+
+ pings.sort((a, b) => b.timestampCreated - a.timestampCreated);
+
+ if (n) pings = pings.slice(0, n);
+ const pingData = headersOnly
+ ? pings
+ : pings.map(ping => TelemetryArchive.promiseArchivedPingById(ping.id));
+
+ return Promise.all(pingData);
+}
+
+module.exports = {
+ searchTelemetryArchive,
+};
diff --git a/toolkit/components/normandy/studies/TestingOverrides.jsm b/toolkit/components/normandy/studies/TestingOverrides.jsm
new file mode 100644
--- /dev/null
+++ b/toolkit/components/normandy/studies/TestingOverrides.jsm
@@ -0,0 +1,34 @@
+const { Preferences } = ChromeUtils.import(
+ "resource://gre/modules/Preferences.jsm",
+ {},
+);
+
+export function getTestingOverrides(widgetId) {
+ const testingOverrides = {};
+ testingOverrides.variationName =
+ Preferences.get(`extensions.${widgetId}.test.variationName`) || null;
+ const firstRunTimestamp = Preferences.get(
+ `extensions.${widgetId}.test.firstRunTimestamp`,
+ );
+ testingOverrides.firstRunTimestamp = firstRunTimestamp
+ ? Number(firstRunTimestamp)
+ : null;
+ testingOverrides.expired =
+ Preferences.get(`extensions.${widgetId}.test.expired`) || null;
+ return testingOverrides;
+}
+
+export function listPreferences(widgetId) {
+ return [
+ `extensions.${widgetId}.test.variationName`,
+ `extensions.${widgetId}.test.firstRunTimestamp`,
+ `extensions.${widgetId}.test.expired`,
+ ];
+}
+
+export function getInternalTestingOverrides(widgetId) {
+ const internalTestingOverrides = {};
+ internalTestingOverrides.studyType =
+ Preferences.get(`extensions.${widgetId}.test.studyType`) || null;
+ return internalTestingOverrides;
+}
diff --git a/toolkit/components/normandy/studies/studyTypes/Pioneer.jsm b/toolkit/components/normandy/studies/studyTypes/Pioneer.jsm
new file mode 100644
--- /dev/null
+++ b/toolkit/components/normandy/studies/studyTypes/Pioneer.jsm
@@ -0,0 +1,344 @@
+/* eslint-env commonjs */
+/* eslint no-unused-vars: ["error", { "varsIgnorePattern": "(Pioneer)" }]*/
+
+import { utilsLogger } from "../logger";
+import * as dataPermissions from "../dataPermissions";
+import { getPingSize } from "../getPingSize";
+
+const { Services } = ChromeUtils.import(
+ "resource://gre/modules/Services.jsm",
+ {},
+);
+const { TelemetryController } = ChromeUtils.import(
+ "resource://gre/modules/TelemetryController.jsm",
+ {},
+);
+
+const { generateUUID } = Cc["@mozilla.org/uuid-generator;1"].getService(
+ Ci.nsIUUIDGenerator,
+);
+
+import {
+ setCrypto as joseSetCrypto,
+ Jose,
+ JoseJWE,
+} from "jose-jwe-jws/dist/jose-commonjs.js";
+
+// The public keys used for encryption
+import * as PUBLIC_KEYS from "./pioneer.public_keys.json";
+
+const PIONEER_ID_PREF = "extensions.pioneer.cachedClientID";
+
+const EVENTS = {
+ INELIGIBLE: "ineligible",
+ EXPIRED: "expired",
+ USER_DISABLE: "user-disable",
+ ENDED_POSITIVE: "ended-positive",
+ ENDED_NEUTRAL: "ended-neutral",
+ ENDED_NEGATIVE: "ended-negative",
+};
+
+// Make crypto available and make jose use it.
+Cu.importGlobalProperties(["crypto"]);
+joseSetCrypto(crypto);
+
+/**
+ * @typedef {Object} Config
+ * @property {String} studyName
+ * Unique name of the study.
+ *
+ * @property {String?} telemetryEnv
+ * Optional. Which telemetry environment to send data to. Should be
+ * either ``"prod"`` or ``"stage"``. Defaults to ``"prod"``.
+ */
+
+/**
+ * Utilities for making Pioneer Studies.
+ */
+class PioneerUtils {
+ /**
+ * @param {Config} config Object with Pioneer-related configuration as specified above
+ */
+ constructor(config) {
+ this.config = config;
+ this.encrypter = null;
+ this._logger = null;
+ }
+
+ /**
+ * @returns {Object} A public key
+ */
+ getPublicKey() {
+ const env = this.config.telemetryEnv || "prod";
+ return PUBLIC_KEYS[env];
+ }
+
+ /**
+ * @returns {void}
+ */
+ setupEncrypter() {
+ if (this.encrypter === null) {
+ const pk = this.getPublicKey();
+ const rsa_key = Jose.Utils.importRsaPublicKey(pk.key, "RSA-OAEP");
+ const cryptographer = new Jose.WebCryptographer();
+ this.encrypter = new JoseJWE.Encrypter(cryptographer, rsa_key);
+ }
+ }
+
+ /**
+ * @returns {String} Unique ID for a Pioneer user.
+ */
+ getPioneerId() {
+ let id = Services.prefs.getCharPref(PIONEER_ID_PREF, "");
+
+ if (!id) {
+ // generateUUID adds leading and trailing "{" and "}". strip them off.
+ id = generateUUID()
+ .toString()
+ .slice(1, -1);
+ Services.prefs.setCharPref(PIONEER_ID_PREF, id);
+ }
+
+ return id;
+ }
+
+ /**
+ * @private
+ * @param {String} data The data to encrypt
+ * @returns {String} The encrypted data
+ */
+ async encryptData(data) {
+ this.setupEncrypter();
+ return this.encrypter.encrypt(data);
+ }
+
+ /**
+ * Constructs a payload object with encrypted data.
+ *
+ * @param {String} schemaName
+ * The name of the schema to be used for validation.
+ *
+ * @param {int} schemaVersion
+ * The version of the schema to be used for validation.
+ *
+ * @param {Object} data
+ * An object containing data to be encrypted and submitted.
+ *
+ * @returns {Object}
+ * A Telemetry payload object with the encrypted data.
+ */
+ async buildEncryptedPayload(schemaName, schemaVersion, data) {
+ const pk = this.getPublicKey();
+
+ return {
+ encryptedData: await this.encryptData(JSON.stringify(data)),
+ encryptionKeyId: pk.id,
+ pioneerId: this.getPioneerId(),
+ studyName: this.config.studyName,
+ schemaName,
+ schemaVersion,
+ };
+ }
+
+ /**
+ * Encrypts the given data and submits a properly formatted
+ * Pioneer ping to Telemetry.
+ *
+ * @param {String} schemaName
+ * The name of the schema to be used for validation.
+ *
+ * @param {int} schemaVersion
+ * The version of the schema to be used for validation.
+ *
+ * @param {Object} data
+ * A object containing data to be encrypted and submitted.
+ *
+ * @param {Object} options
+ * An object with additional options for the function.
+ *
+ * @param {Boolean} options.force
+ * A boolean to indicate whether to force submission of the ping.
+ *
+ * @returns {String}
+ * The ID of the ping that was submitted
+ */
+ async submitEncryptedPing(schemaName, schemaVersion, data, options = {}) {
+ // If the user is no longer opted in we should not be submitting pings.
+ const isUserOptedIn = await dataPermissions.isUserOptedInToPioneer();
+ if (!isUserOptedIn && !options.force) {
+ return null;
+ }
+
+ const payload = await this.buildEncryptedPayload(
+ schemaName,
+ schemaVersion,
+ data,
+ );
+
+ const telOptions = {
+ addClientId: true,
+ addEnvironment: true,
+ };
+
+ return TelemetryController.submitExternalPing(
+ "pioneer-study",
+ payload,
+ telOptions,
+ );
+ }
+
+ /**
+ * Gets an object that is a mapping of all the available events.
+ *
+ * @returns {Object}
+ * An object with all the available events.
+ */
+ getAvailableEvents() {
+ return EVENTS;
+ }
+
+ /**
+ * Submits an encrypted event ping.
+ *
+ * @param {String} eventId
+ * The ID of the event that occured.
+ *
+ * @param {Object} options
+ * An object of options to be passed through to submitEncryptedPing
+ *
+ * @returns {String}
+ * The ID of the event ping that was submitted.
+ */
+ async submitEventPing(eventId, options = {}) {
+ if (!Object.values(EVENTS).includes(eventId)) {
+ throw new Error("Invalid event ID.");
+ }
+ return this.submitEncryptedPing("event", 1, { eventId }, options);
+ }
+}
+
+class PioneerStudyType {
+ /**
+ * @param {object} studyUtils The studyUtils instance from where this class was instantiated
+ */
+ constructor(studyUtils) {
+ const studySetup = studyUtils._internals.studySetup;
+ const Config = {
+ studyName: studySetup.activeExperimentName,
+ telemetryEnv: studySetup.telemetry.removeTestingFlag ? "prod" : "stage",
+ };
+ this.pioneerUtils = new PioneerUtils(Config);
+ this.schemaVersion = 3; // Corresponds to the schema versions used in https://github.com/mozilla-services/mozilla-pipeline-schemas/tree/dev/templates/telemetry/shield-study (and the shield-study-addon, shield-study-error equivalents)
+ }
+
+ /**
+ * @returns {Promise<String>} The ID of the event ping that was submitted.
+ */
+ async notifyNotEligible() {
+ return this.notifyEndStudy(this.EVENTS.INELIGIBLE);
+ }
+
+ /**
+ * @param {String?} eventId The ID of the event that occured.
+ * @returns {Promise<String>} The ID of the event ping that was submitted.
+ */
+ async notifyEndStudy(eventId = EVENTS.ENDED_NEUTRAL) {
+ return this.pioneerUtils.submitEventPing(eventId, { force: true });
+ }
+
+ /**
+ * @returns {Promise<String>} Unique ID for a Pioneer user.
+ */
+ async getTelemetryId() {
+ return this.pioneerUtils.getPioneerId();
+ }
+
+ /**
+ * @param {String} bucket The type of telemetry payload
+ * @param {Object} payload The telemetry payload
+ * @returns {Promise<String>} The ID of the ping that was submitted
+ */
+ async sendTelemetry(bucket, payload) {
+ const schemaName = bucket;
+ return this._telemetry(schemaName, this.schemaVersion, payload);
+ }
+
+ /**
+ * Encrypts the given data and submits a properly formatted
+ * Pioneer ping to Telemetry.
+ *
+ * @param {String} schemaName
+ * The name of the schema to be used for validation.
+ *
+ * @param {int} schemaVersion
+ * The version of the schema to be used for validation.
+ *
+ * @param {Object} payload
+ * A object containing data to be encrypted and submitted.
+ *
+ * @returns {Promise<String>} The ID of the ping that was submitted
+ * @private
+ */
+ async _telemetry(schemaName, schemaVersion, payload) {
+ const pingId = await this.pioneerUtils.submitEncryptedPing(
+ schemaName,
+ schemaVersion,
+ payload,
+ );
+ if (pingId) {
+ utilsLogger.debug(
+ "Pioneer Telemetry sent (encrypted)",
+ JSON.stringify(payload),
+ );
+ } else {
+ utilsLogger.debug(
+ "Pioneer Telemetry not sent due to privacy preferences",
+ JSON.stringify(payload),
+ );
+ }
+ return pingId;
+ }
+
+ /**
+ * Calculate the size of a ping.
+ *
+ * @param {String} bucket The type of telemetry payload
+ *
+ * @param {Object} payload
+ * The data payload of the ping.
+ *
+ * @returns {Promise<Number>}
+ * The total size of the ping.
+ */
+ async getPingSize(bucket, payload) {
+ const schemaName = bucket;
+ return this.getEncryptedPingSize(schemaName, this.schemaVersion, payload);
+ }
+
+ /**
+ * Calculate the size of a ping that has Pioneer encrypted data.
+ *
+ * @param {String} schemaName
+ * The name of the schema to be used for validation.
+ *
+ * @param {int} schemaVersion
+ * The version of the schema to be used for validation.
+ *
+ * @param {Object} data
+ * An object containing data to be encrypted and submitted.
+ *
+ * @returns {Promise<Number>}
+ * The total size of the ping.
+ */
+ async getEncryptedPingSize(schemaName, schemaVersion, data) {
+ return getPingSize(
+ await this.pioneerUtils.buildEncryptedPayload(
+ schemaName,
+ schemaVersion,
+ data,
+ ),
+ );
+ }
+}
+
+export default PioneerStudyType;
diff --git a/toolkit/components/normandy/studies/studyTypes/Shield.jsm b/toolkit/components/normandy/studies/studyTypes/Shield.jsm
new file mode 100644
--- /dev/null
+++ b/toolkit/components/normandy/studies/studyTypes/Shield.jsm
@@ -0,0 +1,63 @@
+
+const { GetPingSize } = ChromeUtils.import(
+ "resource://normandy/studies/GetPingSize.jsm",
+ {},
+);
+
+const { TelemetryController } = ChromeUtils.import(
+ "resource://gre/modules/TelemetryController.jsm",
+ null,
+);
+const { ClientID } = ChromeUtils.import(
+ "resource://gre/modules/ClientID.jsm",
+ {},
+);
+// ChromeUtils.import("resource://gre/modules/ExtensionUtils.jsm");
+
+// eslint-disable-next-line no-undef
+// const { ExtensionError } = ExtensionUtils;
+
+var EXPORTED_SYMBOLS = ["ShieldStudyType"];
+
+class ShieldStudyType {
+ /**
+ * @param {object} studyUtils The studyUtils instance from where this class was instantiated
+ */
+ constructor(studyUtils) {
+ // console.log("studyUtils", studyUtils);
+ }
+
+ /**
+ * @returns {Promise<String>} The telemetry client id
+ */
+ async getTelemetryId() {
+ return ClientID.getClientID();
+ }
+
+ /**
+ * @param {String} bucket The type of telemetry payload
+ * @param {Object} payload The telemetry payload
+ * @returns {Promise<String>} The ID of the ping that was submitted
+ */
+ async sendTelemetry(bucket, payload) {
+ const telOptions = { addClientId: true, addEnvironment: true };
+ return TelemetryController.submitExternalPing(bucket, payload, telOptions);
+ }
+
+ /**
+ * Calculate the size of a ping.
+ *
+ * @param {String} bucket The type of telemetry payload
+ *
+ * @param {Object} payload
+ * The data payload of the ping.
+ *
+ * @returns {Promise<Number>}
+ * The total size of the ping.
+ */
+ async getPingSize(bucket, payload) {
+ let pingSize = new GetPingSize();
+ return pingSize.getPingSize(payload);
+ }
+}
+
diff --git a/toolkit/components/normandy/studies/studyTypes/pioneer.public_keys.json b/toolkit/components/normandy/studies/studyTypes/pioneer.public_keys.json
new file mode 100644
--- /dev/null
+++ b/toolkit/components/normandy/studies/studyTypes/pioneer.public_keys.json
@@ -0,0 +1,20 @@
+{
+ "stage": {
+ "id": "pioneer-20170905",
+ "key": {
+ "e": "AQAB",
+ "kty": "RSA",
+ "n":
+ "3nI-DQ7NoUZCvT348Vi4JfGC1h6R3Qf_yXR0dKM5DmwsuQMxguce6sZ28GWQHJjgbdcs8nTuNQihyVtr9vLsoKUVSmPs_a3QEGXEhTpuTtm7cCb_7HyAlwGtysn2AsdElG8HsDFWlZmiDaHTrTmdLnuk-Z3GRg4nnA4xs4vvUuh0fCVIKoSMFyt3Tkc6IBWJ9X3XrDEbSPrghXV7Cu8LMK3Y4avy6rjEGjWXL-WqIPhiYJcBiFnCcqUCMPvdW7Fs9B36asc_2EQAM5d7BAiBwMjoosSyU6b4JGpI530c3xhqLbX00q1ePCG732cIwp0-bGWV_q0FpQX2M9cNv2Ax4Q"
+ }
+ },
+ "prod": {
+ "id": "pioneer-20170905",
+ "key": {
+ "e": "AQAB",
+ "kty": "RSA",
+ "n":
+ "_uqWswIJpR-cFdwwtNdAI_B_0sPIyQyBy6hiiQ0GKLF2k1PkN6RaxtbZK8v1_BriYtEgWn3hNzJNbKBWBMFtF5-8OfvxH-hgIIeDmRmeHmynLBBCDVf2HAZYaDXJiM7s6LBubDuoPDc3Ovoj287W7E4LgzsBS0wo3ARIwlKn6x0Dj5tu6CQ5r3t0GKZoSFkiVZA7nke-VC55nlDacIIYAqkMX0dzsBaCRmf2C5JJTP-K14iRLB5VFGZ_vnoZ-Wi1BGRV2TNRl3xl0lFJIcPklFpU3hsnRPiF4y7kenU6OIhJVQMqX1CtCF698k7SFCYJt7r1ymWJE-tv0ZwF9b1MFw"
+ }
+ }
+}
diff --git a/toolkit/components/normandy/test/unit/test_StudyUtils.js b/toolkit/components/normandy/test/unit/test_StudyUtils.js
new file mode 100644
--- /dev/null
+++ b/toolkit/components/normandy/test/unit/test_StudyUtils.js
@@ -0,0 +1,44 @@
+// TODO hook this up as an actual unit test, copy/paste into `about:debugging` console of an extension for now.
+
+const studyType = "shield";
+const studySetup = {
+ activeExperimentName: `shield-utils-test-addon@${studyType}.mozilla.org`,
+ studyType,
+ endings: {
+ ineligible: {
+ baseUrls: [
+ "https://qsurvey.mozilla.com/s3/Shield-Study-Example-Survey/?reason=ineligible",
+ ],
+ },
+ BrowserStudyApiEnding: {
+ baseUrls: [
+ "https://qsurvey.mozilla.com/s3/Shield-Study-Example-Survey/?reason=BrowserStudyApiEnding",
+ ],
+ },
+ },
+ telemetry: {
+ send: false,
+ removeTestingFlag: false,
+ internalTelemetryArchive: true,
+ },
+ logLevel: 10,
+ weightedVariations: [
+ {
+ name: "control",
+ weight: 1,
+ },
+ ],
+ expire: {
+ days: 14,
+ },
+ // Dynamic study configuration flags
+ allowEnroll: true,
+ testing: {},
+};
+
+// FIXME seeing Error: firstSeen uses telemetry.: this method can't be used until `setup` is called
+Promise.all([
+ browser.study.setup(studySetup),
+ browser.study.getStudyInfo(),
+ browser.study.endStudy("ineligible"),
+]).then(a => console.log(a));
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment