gigmail-http-observer.js

/*
 * Code is almost directly copied from firebug-http-observer.js in the Firebug
 * extension
 */

// Constants
const CLASS_ID = Components.ID("{40004b06-77ac-41ef-be0a-cd910679b86e}");
const CLASS_NAME = "GTDInbox Gmail HTTP Observer Service";
const CONTRACT_ID = "@gtdinbox.com/glgmail-http-observer;1";

const Cc = Components.classes;
const Ci = Components.interfaces;
const Cr = Components.results;

var observerService = Cc["@mozilla.org/observer-service;1"].getService(Ci.nsIObserverService);
var categoryManager = Cc["@mozilla.org/categorymanager;1"].getService(Ci.nsICategoryManager);


// HTTP Request Observer implementation
function HttpRequestObserver() {
    this.observers = [];
}

HttpRequestObserver.prototype = {
    initialize: function() {
        observerService.addObserver(this, "quit-application", false);
        observerService.addObserver(this, "http-on-modify-request", false);
        observerService.addObserver(this, "http-on-examine-response", false);
    },

    shutdown: function() {
        observerService.removeObserver(this, "quit-application");
        observerService.removeObserver(this, "http-on-modify-request");
        observerService.removeObserver(this, "http-on-examine-response");
    },

    /* nsIObserve */
    observe: function(subject, topic, data) {
        if (topic == "app-startup") {
            this.initialize();
            return;
        } else if (topic == "quit-application") {
            this.shutdown();
            return;
        }

        try {
            if (topic == "http-on-modify-request" || topic == "http-on-examine-response") {
                this.notifyObservers(subject, topic, data);
            }
        } catch (err) {}
    },

    /* nsIObserverService */
    addObserver: function(observer, topic, weak) {
        if (topic != "glgmail-http-event") throw Cr.NS_ERROR_INVALID_ARG;

        this.observers.push(observer);
    },

    removeObserver: function(observer, topic) {
        if (topic != "glgmail-http-event") throw Cr.NS_ERROR_INVALID_ARG;

        for (var i=0; i<this.observers.length; i++) {
            if (this.observers[i] == observer) {
                this.observers.splice(i, 1);
                break;
            }
        }
    },

    notifyObservers: function(subject, topic, data) {
        for(var i=0; i<this.observers.length; i++) this.observers[i].observe(subject, topic, data);
    },

    enumerateObservers: function(topic) { return null; },

	/* nsISupports */
	QueryInterface: function(iid) {
        if (iid.equals(Ci.nsISupports) || iid.equals(Ci.nsIObserverService) || iid.equals(Ci.nsIObserver)) {
 		    return this;
 		}

		throw Cr.NS_ERROR_NO_INTERFACE;
	}
}

function safeGetName(request) {
    try {
        return request.name;
    } catch (exc) {
        return null;
    }
}


// Service factory
var gHttpObserverSingleton = null;
var HttpRequestObserverFactory = {
    createInstance: function (outer, iid) {
        if (outer != null) throw Cr.NS_ERROR_NO_AGGREGATION;

        if (iid.equals(Ci.nsISupports) || iid.equals(Ci.nsIObserverService) || iid.equals(Ci.nsIObserver)) {
            if (!gHttpObserverSingleton) gHttpObserverSingleton = new HttpRequestObserver();

            return gHttpObserverSingleton.QueryInterface(iid);
        }

        throw Cr.NS_ERROR_NO_INTERFACE;
    },

	QueryInterface: function(iid) {
		if (iid.equals(Ci.nsISupports) || iid.equals(Ci.nsISupportsWeakReference) || iid.equals(Ci.nsIFactory)) return this;

		throw Cr.NS_ERROR_NO_INTERFACE;
	}
};


// Module implementation
var HttpRequestObserverModule = {
    registerSelf: function (compMgr, fileSpec, location, type) {
        compMgr = compMgr.QueryInterface(Ci.nsIComponentRegistrar);
        compMgr.registerFactoryLocation(CLASS_ID, CLASS_NAME, CONTRACT_ID, fileSpec, location, type);

        categoryManager.addCategoryEntry("app-startup", CLASS_NAME, "service," + CONTRACT_ID, true, true);
    },

    unregisterSelf: function(compMgr, fileSpec, location) {
        compMgr = compMgr.QueryInterface(Ci.nsIComponentRegistrar);
        compMgr.unregisterFactoryLocation(CLASS_ID, location);

        categoryManager.deleteCategoryEntry("app-startup", CLASS_NAME, true);
    },

    getClassObject: function (compMgr, cid, iid) {
        if (!iid.equals(Ci.nsIFactory)) throw Cr.NS_ERROR_NOT_IMPLEMENTED;
        if (cid.equals(CLASS_ID)) return HttpRequestObserverFactory;

        throw Cr.NS_ERROR_NO_INTERFACE;
    },

    canUnload: function(compMgr) {
        return true;
    }
};


function NSGetModule(compMgr, fileSpec) {
    return HttpRequestObserverModule;
}

glTrafficWatcher.js

/*
 * ***** BEGIN LICENSE BLOCK *****
 *   Version: MPL 1.1
 *
 * The contents of this file are subject to the Mozilla Public License Version
 * 1.1 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 * http://www.mozilla.org/MPL/
 * 
 * Software distributed under the License is distributed on an "AS IS" basis,
 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
 * for the specific language governing rights and limitations under the
 * License.
 *
 * The Original Code is GTDInbox.
 *
 * The Initial Developer of the Original Code is
 * Stephen Augenstein
 * Portions created by the Initial Developer are Copyright (C) 2009
 * the Initial Developer. All Rights Reserved.
 *
 * Contributor(s):
 * Andy Mitchell (andy@gtdinbox.com)
 * 
 * Originally developed for GTDInbox for Gmail. http://www.gtdinbox.com
 * Find out more about developing with this codebase at http://www.gtdinbox.com/development.htm
 * Anyone who utilises this code is required to include this license notice in all covered code.
 * 
 * ***** END LICENSE BLOCK ***** */


/**
 * Uses the glgmail-http-observer component to watch and extract all pertinent Gmail
 * traffic. If there is a listener for a specific type of traffic data, it will
 * track that traffic using a listener on the nsITraceableChannel and will dispatch
 * an event when the data has been collected and wrapped in a {@link glDataObject}.
 *
 * @class ?
 */
var glTrafficWatcher = giBase.extend({
    // Properties {
    /**
     * Whether the object has an event listener registered with glgmail-http-observer
     * @property {Boolean} ?
     */
    _registered: false,

    /**
     * glGmail object
     * @property {glGmail} ?
     */
    _glGmail: null,
    //}

    // Methods {
    /**
     * Starts logging for the given glGmail object context
     * @function ?
     * @param {glGmail} gmail - The glGmail object for the page
     */
    connect: function(gmail) {
        if(this._registered) return;

        giLogger.log('Connecting traffic watcher to page', 'glTrafficWatcher');

        if(!Components.interfaces.nsITraceableChannel) {
            throw new Error('glTrafficWatcher will not work without nsITraceableChannel (available in Firefox 3.0.4)');
        }

        this._glGmail = gmail;

        glTrafficWatcher.OBSERVER_SERVICE.addObserver(this, "glgmail-http-event", false);
        this._registered = true;
    },

    /**
     * Stops logging for the current glGmail object context
     * @function ?
     */
    shutdown: function() {
        if(!this._registered) return;

        giLogger.log('Disconnecting traffic watcher from page', 'glTrafficWatcher');

        glTrafficWatcher.OBSERVER_SERVICE.removeObserver(this, "glgmail-http-event");
        this._registered = false;

        this._removeAllListeners();
        this._glGmail = false;
    },

    /**
     * Called when glgmail-http-event triggered, it filters out unimportant
     * requests and then stores the data it needs. This method is required for
     * compliance with the nsIObserve interface.
     * @function ?
     * @param {nsIHttpChannel} subject - The request object
     * @param {String} topic - The "event" that has taken place
     * @param {Object} data - No clue
     */
    observe: function(subject, topic, data) {
        try {	// Put in try-catch to ensure that it can't possibly break anything - may be unnecessary
            if (!(subject instanceof Components.interfaces.nsIHttpChannel)) return;
    
            // Check if the request comes from the window being monitored by the current glGmail instance
            var win = glTrafficWatcher._getWindowForRequest(subject);
            if(!win) return;
            var rootWin = win;
            while(rootWin != this._glGmail.window && rootWin.parent && rootWin != rootWin.parent) rootWin = rootWin.parent;
            if(rootWin != this._glGmail.window) return;
    
            if(topic == 'http-on-examine-response') {
                this.onExamineResponse(subject);
            }
        } catch(e) {
            giLogger.warn(e, 'glTrafficWatcher.observe', this._glGmail);
        }
    },

    /**
     * Notifies listeners that the request has come in and hooks up TracingListener
     * if listeners exist for request completion.
     * @function ?
     * @param {nsIHttpChannel} request - The request object
     */
    onExamineResponse: function(request) {
        try { // Put in try-catch to ensure that it can't possibly break anything - may be unnecessary
            var url = request.URI.asciiSpec;
    
    
            // Notify listeners that the request came in if being listened for
            //if( this._glGmail.fb ) this._glGmail.fb.log('onExamineResponse: ' + this._urlEventName(url) + ", " + url); //DEBUG
            if(this._urlHasListeners(url)) {
                // Get post data and dispatch event
                this.dispatchEvent({
                    name: this._urlEventName(url),
                    url: url,
                    postData: glTrafficWatcher._getRequestPostData(request)
                });
            }
    
            if(this._urlHasListeners(url, true)) {
                // Set up tracing channel to grab response because it should be watched
                try {
                    request.QueryInterface(Components.interfaces.nsITraceableChannel);
                    var newListener = new glTrafficWatcher.TracingListener(this);
                    newListener.originalListener = request.setNewListener(newListener);
                } catch(e) {}
            }
        } catch(e) {
            giLogger.warn(e, 'glTrafficWatcher.onExamineResponse', this._glGmail);
        }
    },

    /**
     * Called by the {@link glTrafficWatcher.TracingListener TracingListener}
     * object when the request is stopped. This includes premature stopping and
     * stopping on completion. Dispatches an event containing a {@link glDataObject}
     * object that wraps the accumulated response data.
     * @function ?
     * @param {nsIHttpChannel} request - The request object
     * @param {String} responseData - The accumulated response data
     */
    stopRequest: function(request, responseData) {
        try {	// Put in try-catch to ensure that it can't possibly break anything - may be unnecessary
            if(!this._registered) return; // May still be requests in the queue after de-registering so just ignore them
    
    
            var url = null;
            try { url = request.name; } catch (e) { return; }
            
            // Check if event has listeners (to prevent glDataObject creation when unnecessary) {
            var eventName = this._urlEventName(url, true);
            //if( this._glGmail.fb ) this._glGmail.fb.log('stopRequest: ' + eventName + '\n' + url); //DEBUG UTF8
            if(!eventName) return;
            if(!this.hasEventListener(eventName)) return;
            //if( this._glGmail.fb ) this._glGmail.fb.log('process stop request ' + eventName);
            //}
    
            
            // Extract the response data and turn it into a glDataObject {
            var data = null;
            try {
                var responseType = /[?&]rt=([^&]+)/.exec(url)[1];
                switch(responseType) {
                    case 'j':
                        data = glDataObject.createFromJSResponse(responseData, url, this._glGmail);
                        break;
                    case 'h':
                        data = glDataObject.createFromHTMLResponse(responseData, url, this._glGmail);
                        break;
                    default:
                        throw new Error('Invalid response type for url "'+url+'": "'+responseType+'"');
                }
            } catch(e) {
                giLogger.warn(e, 'glTrafficWatcher.stopRequest[inner data extraction]', this._glGmail);
                return;
            }
            //}
            
            // Notify listeners that the request is completed
            this.dispatchEvent({
                name: eventName,
                url: url,
                responseData: data
            });
    
        } catch(e) {
            giLogger.warn(e, 'glTrafficWatcher.stopRequest', this._glGmail);
        }
    },

    /**
     * Given a URL, determines whether or not there are any listeners for the
     * event dispatched for that URL.
     * @function {Boolean} ?
     * @param {String} url - The url to check
     * @param {Boolean} completed - Whether the request data has been received, or whether this is a notification of the request
     * @return Whether the URL should have its traffic watched
     */
    _urlHasListeners: function(url, completed) {
        if(typeof completed == 'undefined')  completed = false; // Default completed to false


        // Pre filter based on url
        if(!/[?&]view=/.test(url)) return false;
        if(!/[?&]rt=/.test(url)) return false;

        // Check if has event listeners registered for it
        var eventName = this._urlEventName(url, completed);
        if(!eventName) return false;
        if(!this.hasEventListener(eventName)) return false;


        // Passed all checks so return true
        return true;
    },

    /**
     * Returns the event name for a given URL. Used to make it easier to reduce
     * processing of all requests by only processing URLs that are being listened
     * for. Currently only returns event names for a small amount of the traffic.
     * @function {String} ?
     * @param {String} url - The url to check
     * @param {Boolean} completed - Whether the request data has been received, or whether this is a notification of the request
     * @return The event name - false if no event for the url
     */
    _urlEventName: function(url, completed) {
        if(typeof completed == 'undefined')  completed = false;
        var eventName = false;
        try {
            var viewType = /[?&]view=([^&]+)/.exec(url)[1];
            switch(viewType) {
                case 'up':
                    var actionType = /[?&]act=([^&]+)/.exec(url)[1];
                    switch(actionType) {
                        case 'sm':
                            eventName = 'send_message_request'+(completed?'_completed':'');
                            break;
                        default:
                            eventName = 'action_request'+(completed?'_completed':'');
                            break;
                    }
                    break;
                case 'tl':
                    var m = /[?&]act=([^&]+)/.exec(url);
                    if( m ) {
                        var actionType = m[1];
                        switch(actionType) {
                            case 'sm':
                                eventName = 'send_message_request'+(completed?'_completed':'');
                                break;
                            default:
                                eventName = 'action_request'+(completed?'_completed':'');
                                break;
                        }
                    } else {
                        eventName = 'thread_list_request'+(completed?'_completed':'');
                    }
                    break;
                // TODO extract text from base machine
                // Dispatched when a new message arrives; and when load a new thread list.
                // Note Gmail appears to pre-cache conversations (in ThreadList) so no need to download them when it loads a message.
                case 'cv':
                    if( /[?&]t=[^&]/.test(url) ) {
                        eventName = 'conversation_request'+(completed?'_completed':'');
                    }
                    break;
                
            }
        } catch(e) {
            giLogger.log('Error calculating event name: '+e, 'glTrafficWatcher');
        }

        return eventName;
    },

    // Required by nsISupports
	QueryInterface: function(iid) {
        if (iid.equals(Components.interfaces.nsISupports) || iid.equals(Components.interfaces.nsIObserver)) {
                return this;
            }

            throw Components.results.NS_ERROR_NO_INTERFACE;
	},
    //}
},{
    // Static Properties {
    

    /**
     * Dispatched when a send message request is detected
     * @event ?
     */
    SEND_MESSAGE_REQUEST: 'send_message_request',

    /**
     * Dispatched when a send message request is completed
     * @event ?
     */
    SEND_MESSAGE_REQUEST_COMPLETED: 'send_message_request_completed',

    /**
     * Dispatched when an action performing request is detected
     * @event ?
     */
    ACTION_REQUEST: 'action_request',

    /**
     * Dispatched when an action performing request is completed
     * @event ?
     */
    ACTION_REQUEST_COMPLETED: 'action_request_completed',

    /**
     * Dispatched when a thread list request is detected
     * @event ?
     */
    THREAD_LIST_REQUEST: 'thread_list_request',

    /**
     * Dispatched when a thread list request is completed
     * @event ?
     */
    THREAD_LIST_REQUEST_COMPLETED: 'thread_list_request_completed',

    /**
     * Dispatched when a conversation request is detected
     * @event ?
     */
    CONVERSATION_REQUEST: 'conversation_request',

    /**
     * Dispatched when a conversation request is completed
     * @event ?
     */
    CONVERSATION_REQUEST_COMPLETED: 'conversation_request_completed',

    /**
     * A reference to the glgmail http observer service
     * @property ?
     */
    OBSERVER_SERVICE: Components.classes["@gtdinbox.com/glgmail-http-observer;1"].getService(Components.interfaces["nsIObserverService"]),
    //}

    // Static Methods {
    /**
     * Extract post data from a request if possible
     * @function {static String} ?
     * @param {nsIHttpChannel} request - The request
     * @return The extracted post data if any
     */
    _getRequestPostData: function(request) {
        var postData = '';
        try {
            var is = request.QueryInterface(Components.interfaces.nsIUploadChannel).uploadStream;
            if (is) {
                var ss = is.QueryInterface(Components.interfaces.nsISeekableStream);
                if (ss) ss.seek(Components.interfaces.nsISeekableStream.NS_SEEK_SET, 0);

                var binaryInputStream = Components.classes["@mozilla.org/binaryinputstream;1"].createInstance(Components.interfaces.nsIBinaryInputStream);
                binaryInputStream.setInputStream(is);
                var segments = [];
                for (var count = is.available(); count; count = is.available())
                    segments.push(binaryInputStream.readBytes(count));
                postData = segments.join("");
                if(postData) {
                    var conv = Components.classes["@mozilla.org/intl/scriptableunicodeconverter"].getService(Components.interfaces.nsIScriptableUnicodeConverter);
                    conv.charset = "UTF-8";
                    postData = conv.ConvertToUnicode(postData);
                }

                if (ss) ss.seek(Components.interfaces.nsISeekableStream.NS_SEEK_SET, 0);
            }
        } catch(e) {}
        return postData;
    },

    _getWindowForRequest: function(request) {
        // Copied from Firebug
        var webProgress = glTrafficWatcher._getRequestWebProgress(request);

        try {
            if (webProgress) return webProgress.DOMWindow;
        } catch (e) {}

        return null;
    },

    _getRequestWebProgress: function(request) {
        // Copied from Firebug
        try {
            if (request.notificationCallbacks) {
                return request.notificationCallbacks.getInterface(Components.interfaces.nsIWebProgress);
            }
        } catch (e) {}

        try {
            if (request.loadGroup && request.loadGroup.groupObserver) {
                return request.loadGroup.groupObserver.QueryInterface(Components.interfaces.nsIWebProgress);
            }
        } catch (e) {}

        return null;
    }
    //}
});
glTrafficWatcher.implement(giEventDispatcher); // Add event dispatching support

/**
 * This object implements nsIStreamListener interface and extracts response data
 * from channels being listened to and sends it to the associated glTrafficWatcher.
 * @class glTrafficWatcher.TracingListener
 */
glTrafficWatcher.TracingListener = giBase.extend({
    // Properties {
    /**
     * The next listener in the chain if multiple listeners registered with the same request
     * @property {Object} ?
     */
    originalListener: null,

    /**
     * The trafficWatcher object it is associated with
     * @property {glTrafficWatcher} ?
     */
    _trafficWatcher: null,

    /**
     * An array of data chunks
     * @property {String[]} ?
     */
    _receivedData: null,
    //}

    /**
     * The constructor.
     * @constructor ?
     * @param {glTrafficWatcher} trafficWatcher - The watcher it's associated with
     */
    constructor: function(trafficWatcher) {
        this._trafficWatcher = trafficWatcher;
        this._receivedData = [];
    },

    // Methods {
    onCollectData: function(request, inputStream, offset, count) {
        // Put in try-catch to ensure that it can't possibly break anything - may be unnecessary
        try {
            var binaryInputStream = Components.classes["@mozilla.org/binaryinputstream;1"].createInstance(Components.interfaces.nsIBinaryInputStream);
            var storageStream = Components.classes["@mozilla.org/storagestream;1"].createInstance(Components.interfaces.nsIStorageStream);
            var binaryOutputStream = Components.classes["@mozilla.org/binaryoutputstream;1"].createInstance(Components.interfaces.nsIBinaryOutputStream);

            binaryInputStream.setInputStream(inputStream);
            storageStream.init(8192, count, null);
            binaryOutputStream.setOutputStream(storageStream.getOutputStream(0));

            // Copy received data as they come.
            var data = binaryInputStream.readBytes(count);
            this._receivedData.push(data);

            binaryOutputStream.writeBytes(data, count);

            return storageStream.newInputStream(0);
        } catch(e) {
            giLogger.log('Exception caught in onCollectData: '+e, 'glTrafficWatcher.TracingListener');
        }

        return null;
    },

    /* nsIStreamListener */
    onDataAvailable: function(request, requestContext, inputStream, offset, count) {
        try {
            var newStream = this.onCollectData(request, inputStream, offset, count);
            if (newStream) inputStream = newStream;

        
            if (this.originalListener) this.originalListener.onDataAvailable(request, requestContext, inputStream, offset, count);
        } catch(e) {
            giLogger.log('Exception caught in onDataAvailable: '+e, 'glTrafficWatcher.TracingListener');
        }
    },

    onStartRequest: function(request, requestContext) {
        try {
            if (this.originalListener) this.originalListener.onStartRequest(request, requestContext);
        } catch(e) {
            giLogger.log('Exception caught in onStartRequest: '+e, 'glTrafficWatcher.TracingListener');
        }
    },

    onStopRequest: function(request, requestContext, statusCode) {
        try {
            this._trafficWatcher.stopRequest(request, this._receivedData.join());

            if (this.originalListener) this.originalListener.onStopRequest(request, requestContext, statusCode);
        } catch(e) {
            giLogger.log('Exception caught in onStopRequest: '+e, 'glTrafficWatcher.TracingListener');
        }
    },

    /* nsISupports */
    QueryInterface: function(iid) {
        if (iid.equals(Components.interfaces.nsIStreamListener) || iid.equals(Components.interfaces.nsISupportsWeakReference) || iid.equals(Components.interfaces.nsISupports)) {
            return this;
        }

        throw Components.results.NS_NOINTERFACE;
    }
    //}
});