Backup Brome Blog

Gmail inbox automation snippet

```python
import httplib2
import os
import argparse
import base64
import email
import re

from apiclient import errors
from apiclient import discovery
import oauth2client
from oauth2client import client
from oauth2client import tools

class GmailInbox(object):
    #https://developers.google.com/gmail/api/quickstart/python

    def __init__(self, **kwargs):
        self.secret_file = kwargs.get('secret_file')

        self.configure()

    def list_labels(self):
        results = self.service.users().labels().list(userId=self.user_id).execute()
        labels = results.get('labels', [])

        if not labels:
            print 'No labels found.'
        else:
            print 'Labels:'
            for label in labels:
                print 'name: %s; id: %'%(label['name'], label['id'])

    def configure(self):
        self.credentials = self.get_credentials()
        self.http = self.credentials.authorize(httplib2.Http())
        self.service = discovery.build('gmail', 'v1', http=self.http)
        self.user_id = 'me'

    def get_credentials(self):
        #flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()
        flags = None
        scopes = 'https://mail.google.com/'
        application_name = 'EmailW'

        credential_dir = os.path.join(
            '/path/to/', #CHANGEME
            'config',
            '.credentials'
        )
        if not os.path.exists(credential_dir):
            os.makedirs(credential_dir)

        credential_path = os.path.join(credential_dir, 'emailw.json')

        store = oauth2client.file.Storage(credential_path)
        credentials = store.get()
        if not credentials or credentials.invalid:
            flow = client.flow_from_clientsecrets(self.secret_file, scopes)
            flow.user_agent = application_name
            credentials = tools.run_flow(flow, store, flags)

        return credentials

    def list_message(self, query=''):
      try:
        response = self.service.users().messages().list(userId=self.user_id,
                                                   q=query).execute()
        messages = []
        if 'messages' in response:
          messages.extend(response['messages'])

        while 'nextPageToken' in response:
          page_token = response['nextPageToken']
          response = service.users().messages().list(userId=user_id, q=query,
                                             pageToken=page_token).execute()
          messages.extend(response['messages'])

        return messages
      except errors.HttpError, error:
        print 'An error occurred: %s' % error

    def get_message_body_text(self, msg):
      try:
        message = self.service.users().messages().get(userId=self.user_id, id=msg['id'], format='raw').execute()

        msg_str = base64.urlsafe_b64decode(message['raw'].encode('ASCII'))

        mime_msg = email.message_from_string(msg_str)

        return mime_msg.as_string()
      except errors.HttpError, error:
        print 'An error occurred: %s' % error

    def list_message_in_label(self, label):
        label_ids = [label]

        try:
            response = self.service.users().messages().list(
                                                    userId=self.user_id,
                                                    labelIds=label_ids
                                                    ).execute()
            messages = []
            if 'messages' in response:
              messages.extend(response['messages'])

            while 'nextPageToken' in response:
              page_token = response['nextPageToken']
              response = self.service.users().messages().list(
                                                        userId=self.user_id,
                                                         labelIds=label_ids,
                                                         pageToken=page_token
                                                    ).execute()
              messages.extend(response['messages'])

            return messages
        except errors.HttpError, error:
            print 'An error occurred: %s' % error

    def delete_all_message_in_label(self, label):
        msgs = self.list_message_in_label(label)
        if msgs:
            for msg in msgs:
                try:
                    self.service.users().messages().delete(userId=self.user_id, id=msg['id']).execute()
                    print 'Message with id: %s deleted successfully.' % msg['id']
                except errors.HttpError, error:
                    print 'An error occurred: %s' % error
        else:
            print 'No message to delete'
            
	#Example of an useful function: recover a token
    def get_password_recovery_token(self, timeout = 10):
        print "Getting the password recovery token"
        token = False

        for i in range(timeout):
            msg = self.list_message('change your password')

            if msg:
                msg = msg[0]

                text = self.get_message_body_text(msg).replace('\r\n', '').replace('=0A', '').replace('=', '')
                try:
                    token = re.search("{your regex}", text).group(0)
                    print "Token found: %s"%token
                except (AttributeError, IndexError):
                    pass

                self.delete_all_message_in_label('Label_2')
                break
            else:
                sleep(1)
                print "Recovery password message still not received..."

        return token
```

Selenium EC2 Grid Hub Installation

# GRID HUB INSTALLATION ON AWS

* Launch an ec2 m3.xlarge Ubuntu server LTS
* Add a 100GiB EBS volume (if you want to enable the video capture) (optional)
* Configure the security group
 * SSH TCP 22 0.0.0.0/0
 * Custom TCP Rule TCP 5000 0.0.0.0/0 #Webserver port
 * Custom TCP Rule TCP  4444  0.0.0.0/0 #Selenium hub port

# Connect to the instance

## Install the dependency packages

```bash
$ sudo apt-add-repository ppa:webupd8team/java
$ sudo apt-get update
$ sudo apt-get install python-pip
$ sudo apt-get install git
$ sudo apt-get install xclip
$ sudo apt-get install python-dev
$ sudo apt-get install libxml2-dev libxslt1-dev
$ sudo apt-get install python-lxml zlib1g-dev
$ sudo apt-get install python2.7-mysqldb
$ sudo apt-get install oracle-java8-installer
```
## Install and configure MySQL server
```bash
$ sudo apt-get install mysql-server
$ sudo mysql_secure_installation
```

## Configure the timezone
```bash
$ sudo dpkg-reconfigure tzdata
```

## Install brome and configure your project
```bash
$ sudo pip install brome
$ git clone $YOUR_PROJECT
# CONFIGURE YOUR PROJECT
$ ./bro admin --create-database
```

## Configure vnc and boto
```bash
$ mkdir /home/ubuntu/.vnc
$ echo '$VNC-PASSWORD' > /home/ubuntu/.vnc/passwd
```
* Follow the boto configuration instruction http://boto.readthedocs.org/en/latest/boto_config_tut.html
```bash
$ ./bro webserver #TEST IF IT IS WORKING
$ nohup ./bro webserver -t &
$ exit
```

## Transfert your private key into the hub server
```bash
$ scp -i "$KEY-NAME.pem" -r "$KEY-NAME.pem" ubuntu@$INSTANCE-IP:/home/ubuntu/.ssh/
```

* Create an Amazon Machine Image (AMI) (optional)

Selenium EC2 Grid Node Installation

# GRID NODE INSTALLATION ON AWS

* Launch an ec2 t2.micro Ubuntu server LTS
* Configure the security group
 * SSH TCP 22 0.0.0.0/0
 * Custom TCP Rule TCP 5555 0.0.0.0/0 #Selenium node port
 * Custom TCP Rule TCP 5900 0.0.0.0/0 #Vnc

# Connect to the instance

## Install the necessary packages
```bash
$ sudo apt-add-repository ppa:webupd8team/java
$ sudo apt-get update
$ sudo apt-get install xclip
$ sudo apt-get install unzip
$ sudo apt-get install xvfb
$ sudo apt-get install oracle-java8-installer
$ sudo apt-get install x11vnc
$ sudo apt-get install dbus-x11
$ sudo apt-get install firefox
```

## Configure x11vnc
```bash
$ x11vnc -storepasswd
$ sudo cp ~/.vnc/passwd /etc/x11vnc.pass
$ sudo vim /etc/init.d/x11vnc
```

Paste this into `/etc/init.d/x11vnc`.
```
#!/bin/sh
#
# /etc/init.d/x11vnc
#
### BEGIN INIT INFO
# Provides:          x11vnc server
# Required-Start:    xdm
# Should-Start:
# Required-Stop:
# Should-Stop:
# Default-Start:     5
# Default-Stop:      0 1 2 6
# Short-Description:
# Description:       Start or stop vnc server
### END INIT INFO
case "$1" in
    start)
        echo "Starting x11vnc"
        Xvfb :0 -screen 0, 1920x1080x24 &
        sleep 4
        DISPLAY=:0 /usr/bin/x11vnc -rfbport 5900 -rfbauth /etc/x11vnc.pass -o /var/log/x11vnc.log -forever  -bg
        exit 0
        ;;
    *)
        echo "Usage: ${SERVICE} {start}"
        exit 1
        ;;
esac
```

```bash
$ sudo chmod +x /etc/init.d/x11vnc
$ sudo update-rc.d x11vnc defaults 
```

Reboot to test if vnc is working.

## Install selenium dependencies

* Install chrome http://askubuntu.com/questions/510056/how-to-install-google-chrome/510186#510186
* Download the Selenium Standalone Server http://www.seleniumhq.org/download/
```bash
$ wget $SELENIUM-SERVER-URL
$ mv $SELENIUM-DOWNLOADED-FILE selenium-server.jar
```
* Download the chromedriver https://sites.google.com/a/chromium.org/chromedriver/downloads
```bash
$ wget http://chromedriver.storage.googleapis.com/$CHROMEDRIVER-VERSION/chromedriver_linux64.zip
$ unzip chromedriver_linux64.zip
$ sudo ln -s /path/to/chromedriver /usr/local/bin/
$ chromedriver -v #make sure it work
```

## Install recordscreen.py (optional)

Reference: http://www.davidrevoy.com/article65/recordscreen-py-video-and-audio-capture-from-terminal-for-ubuntu-12-04
```bash
sudo apt-get install libav-tools x11-apps
```

Start recording bash script
```bash
#!/bin/bash

LOG_FILE=`echo $1 | cut -d . -f 1`_avconv.log
DISPLAY=:0 nohup avconv -f x11grab -r 15 -s '1920x1080' -i :0.0+0,0 -vcodec libx264 -preset ultrafast -g 15 -crf 0 -pix_fmt yuv444p -threads 1 $1 > $LOG_FILE 2>&1 &
DISPLAY=:0 nohup xclock -d -update 1 > /dev/null 2>&1 &
```

Stop recording bash script
```bash
#!/bin/bash

pkill -f "avconv"
pkill -f "xclock"

for i in `seq 1 60`;
do
        RET=`pidof avconv`
        echo "$RET"
        if [ "$RET" ]; then
                break;
        fi

        sleep 1;
done

echo "Done!"
```

* Create an Amazon Machine Image (AMI) (optional)

Selenium Ubuntu Node Configuration on VirtualBox

# Node machine

Install Ubuntu on Virtualbox
----------------------------

[Install Ubuntu on Virtualbox](http://askubuntu.com/questions/142549/how-to-install-ubuntu-on-virtualbox)
Install the virtualbox guest additions.

Note: 
    - make sure your network adapter is 'Bridged Adapter'
    - sshd must be up and running: https://help.ubuntu.com/community/SSH/OpenSSH/Configuring
    - enable the automatic login for the user: http://askubuntu.com/a/568852
    - allow root to login via ssh: http://askubuntu.com/a/489034 and http://askubuntu.com/a/155280

Install java (oracle)
---------------------

http://askubuntu.com/questions/48468/how-do-i-install-java/106496#106496

Download selenium server standalone jar
---------------------------------------

http://www.seleniumhq.org/download/

Browsers
--------

* [Chrome](http://askubuntu.com/questions/510056/how-to-install-google-chrome/510186#510186)
* Firefox: nothing to install since firefox driver is shipped with selenium
* [PhantomJS](http://stackoverflow.com/a/14267295)

Chromedriver
------------

Get the latest chromedriver version: https://sites.google.com/a/chromium.org/chromedriver/downloads

```
$ wget http://chromedriver.storage.googleapis.com/{VERSION}/chromedriver_linux64.zip
$ unzip chromedriver_linux64.zip
$ sudo ln -s /path/to/chromedriver /usr/local/bin/
$ chromedriver -v #make sure it work
```

Configure VBoxControl
---------------------

```
$ cd ~
$ touch setproperty.sh
$ chmod +x setproperty.sh
$ vim setproperty.sh
```

Add these lines:
```
VBoxControl guestproperty set wait_until_ready ready
exit 0
```

```
$ sudo chmod +x /etc/rc.local
$ sudo vim /etc/rc.local
```

Add this line:
```
bash /home/{user}/setproperty.sh
```

Install and configure x11vnc [optional]
---------------------------------------

If you want to record the session install x11vnc:

```
$ sudo apt-get install x11vnc
$ x11vnc -storepasswd
$ sudo cp ~/.vnc/passwd /etc/x11vnc.pass
$ sudo vim /etc/init/x11vnc.conf
```

Add these lines to the /etc/init/x11vnc.conf:
```
start on login-session-start
script
x11vnc -display :0 -auth /var/run/lightdm/root/:0 -forever -bg -o /var/log/x11vnc.log -rfbauth /etc/x11vnc.pass -rfbport 5900
end script
```

If you want to test your x11vnc installation `reboot` first.

# Hub machine

Brome browser config
--------------------

Add these lines in your /path/to/project/config/browsers_config.yml
```
example-vbox:
  vbname: 'ubuntu-node'
  browserName: 'firefox' # | 'chrome' | 'phantomjs'
  platform: 'LINUX'
  version: '{browser_version}' #Changeme only for reference purpose
  "browser:maximize_window": true #optional
  record_session: true #optional
  available_in_webserver: true
  hub_ip: 'localhost' #this will be change automatically by the brome runner
  username: '{username}' #Changeme
  password: '{password}' #Changeme
  launch: true
  terminate: true
  vbox_type: 'gui' # | 'headless'
  nb_browser_by_instance: 1
  selenium_path: '/path/to/selenium-server.jar' #Changeme
  selenium_command: "DISPLAY=:0 nohup java -jar {selenium_path} -role node -hub http://{hub_ip}:4444/grid/register -browser browserName={browserName},maxInstances={nb_browser_by_instance},platform={platform} > node.log 2>&1 &"
```

Brome config
------------

Make sure you have set up your grid config in the brome.yml

```
#/path/to/project/config/brome.yml
[...]
grid_runner:
  kill_selenium_server: true
  max_running_time: 7200
  selenium_hub_config: '/path/to/hub-config.json' #optional
  selenium_server_command: 'java -jar {selenium_server_jar_path}
    -role hub -hubConfig {selenium_hub_config} -DPOOL_MAX 512 &'
  selenium_server_ip: 'localhost'
  selenium_server_jar_path: '/path/to/selenium-server-standalone.jar'
  selenium_server_port: 4444
  start_selenium_server: true
[...]
```

Sample hub-config.json:
```
{
  "host": null,
  "port": 4444,
  "newSessionWaitTimeout": -1,
  "servlets" : [],
  "prioritizer": null,
  "capabilityMatcher": "org.openqa.grid.internal.utils.DefaultCapabilityMatcher",
  "throwOnCapabilityNotPresent": true,
  "nodePolling": 5000,
  "cleanUpCycle": 5000,
  "timeout": 30000,
  "browserTimeout": 90000,
  "maxSession": 30,
  "jettyMaxThreads":-1
}
```

Create the vnc password file [optional]
---------------------------------------

```
$ cd ~/.vnc/
$ vim passwd #add the password in plain text
```

Launch the instance
-------------------

```
$ ./bro run -r 'example-vbox'
```