backfill.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from socorro.external.postgresql.base import PostgreSQLBase
from socorro.lib import external_common
import socorro.database.database as db
from socorro.external import MissingOrBadArgumentError


#==============================================================================
class Backfill(PostgreSQLBase):

    #--------------------------------------------------------------------------
    def get(self, **kwargs):
  
        self.connection = self.database.connection()
        cursor = self.connection.cursor()

        filters = [
            ("kind_of_backfill", None, "str"),
            ("start_date", None, "date"),
            ("end_date", None, "date"),
            ("check_data", None, "str"),
        ]

        params = external_common.parse_arguments(filters, kwargs)

        if not params.kind_of_backfill:
            raise MissingOrBadArgumentError(
                "Mandatory parameter 'kind_of_backfill' is missing or empty"
            )

        query = "SELECT backfill_%(kind)s" %{"kind": kwargs['kind_of_backfill']}

        parameters = kwargs
        parameters.pop('kind_of_backfill')
        parameters = str(kwargs.keys())
        parameters = parameters.replace("'", "").replace("[", "(%(").replace("]",")s)").replace(", ", ")s, %(")
        if kwargs != {}:
            query += parameters
            print query % kwargs
        else:
            query += "()"

        error_message = "Failed to retrieve backfill "+params.kind_of_backfill+" from PostgreSQL"
        if kwargs != {}:
            results = self.query(query, kwargs, error_message=error_message)
        else:
            results = self.query(query, error_message=error_message)
        
        return results

running

(socorro-virtualenv)gabi@ubuntu:~/Desktop/CODE/src/socorro$ nosetests socorro/unittest/external/postgresql/test_backfill.py -s
SELECT backfill_adu(2013-08-14)
SELECT backfill_all_dups(2013-08-14, 2013-08-15)
SELECT backfill_build_adu(2013-08-14)
SELECT backfill_correlations(2013-08-14)
SELECT backfill_reports_clean(2013-08-14, 2013-08-15)
SELECT backfill_explosiveness(2013-08-14)
SELECT backfill_home_page_graph(2013-08-14)
SELECT backfill_crashes_by_user_build(2013-08-14)
SELECT backfill_matviews(2013-08-14, 2013-08-15, false)
SELECT backfill_nightly_builds(2013-08-14)
SELECT backfill_tcbs_build(2013-08-14)
SELECT backfill_exploitability(2013-08-14)
SELECT backfill_tcbs(2013-08-14)
SELECT backfill_rank_compare(2013-08-14)
SELECT backfill_crashes_by_user(2013-08-14)
SELECT backfill_hang_report(2013-08-14)
SELECT backfill_one_day(2013-08-15)
SELECT backfill_home_page_graph_build(2013-08-14)
SELECT backfill_signature_summary(2013-08-14)
.
----------------------------------------------------------------------
Ran 1 test in 32.841s

OK

test_backfill.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import datetime
from socorro.external.postgresql.backfill import Backfill
from nose.plugins.attrib import attr
from .unittestbase import PostgreSQLTestCase
from socorro.lib import datetimeutil
from socorro.external.postgresql import fakedata
from socorro.external import MissingOrBadArgumentError
from socorro.external.postgresql import setupdb_app
import socorro.database.database as db
import cStringIO

#==============================================================================
@attr(integration='postgres')
class TestBackfill(PostgreSQLTestCase):
    """Simple test of backfill calling"""
                      
    #--------------------------------------------------------------------------
    def setUp(self):
        """ Populate product_info table with fake data """
        super(TestBackfill, self).setUp()

        cursor = self.connection.cursor()

        self.now = datetimeutil.utc_now()
        now = self.now.date()
        yesterday = now - datetime.timedelta(days=1)
        lastweek = now - datetime.timedelta(days=7)

        raw =  fakedata.RawADU()
        row = raw.generate_rows().next()

        self.tables = []
        start_date = end_date = None

        for table in fakedata.tables:
            table = table(days=1)

            if start_date:
                if  start_date > table.start_date:
                    start_date = table.start_date
            else:
                start_date = table.start_date

            if end_date:
                if end_date < table.start_date:
                    end_date = table.end_date
            else:
                end_date = table.end_date

            table_name = table.table
            columns =  table.columns

            for rows in table.generate_rows():            
              data = dict(zip(columns, rows))
              columns_ = str(columns).replace("[", "(").replace("]", ")").replace("'", "")
              values = str(columns_).replace("(", "(%(").replace(")", ")s);").replace(", ", ")s, %(")
              self.tables.append(table_name)

              query = "INSERT INTO %(table_name)s " % {'table_name': table_name}
              query = query + columns_ +" VALUES " + values
              cursor.execute(query, data)
              self.connection.commit()
                          
    #--------------------------------------------------------------------------
    def tearDown(self):
        """ Cleanup the database, delete tables and functions """

        tables = str(self.tables).replace("'", "").replace("[", "").replace("]", "")
        cursor = self.connection.cursor()
        cursor.execute("TRUNCATE "+ tables + " CASCADE;")
        self.connection.commit()
        self.connection.close()

        super(TestBackfill, self).tearDown()

   #--------------------------------------------------------------------------
    def test_get(self):

        backfill = Backfill(config=self.config)

        self.now = datetimeutil.utc_now()
        now = self.now.date()
        yesterday = now - datetime.timedelta(days=1)

        #......................................................................
        # Test Raise error if parameter is not passed
        params = {"kind_of_backfill": ''}
        self.assertRaises(MissingOrBadArgumentError,
                          backfill.get,
                          **params)

        #......................................................................
        # Test backfill_adu(updateday date) RETURNS boolean
        params = {
            "kind_of_backfill": 'adu',
            "start_date": yesterday,
        }
        result = backfill.get(**params)
        self.assertTrue(result[0][0])
        
        #......................................................................
        # Test backfill_all_dups(start_date timestamp without time zone, 
        # end_date timestamp without time zone) RETURNS boolean
        params = {
            "kind_of_backfill": 'all_dups',
            "start_date": yesterday,
            "end_date": now,
        }
        result = backfill.get(**params)
        self.assertTrue(result[0][0])

        #......................................................................
        # Test backfill_build_adu(updateday date) RETURNS boolean
        params = {
            "kind_of_backfill": 'build_adu',
            "start_date": yesterday,
        }
        result = backfill.get(**params)
        self.assertTrue(result[0][0])

        #......................................................................
        # Test backfill_correlations(updateday date) RETURNS boolean
        params = {
            "kind_of_backfill": 'correlations',
            "start_date": yesterday,
        }
        result = backfill.get(**params)
        self.assertTrue(result[0][0])

        #......................................................................
        # Test  backfill_reports_clean(begin_time timestamp with time zone, 
        # end_time timestamp with time zone DEFAULT NULL::timestamp with 
        # time zone) RETURNS boolean 
        params = {
            "kind_of_backfill": 'reports_clean',
            "start_date": yesterday,
            "end_date": now,
        }
        result = backfill.get(**params)
        self.assertTrue(result[0][0])
        
        #......................................................................
        # Test backfill_explosiveness(updateday date) RETURNS boolean
        params = {
            "kind_of_backfill": 'explosiveness',
            "update_date": yesterday,
        }
        result = backfill.get(**params)
        self.assertTrue(result[0][0])
        
        #......................................................................
        # Test backfill_home_page_graph(updateday date, check_period interval 
        # DEFAULT '01:00:00'::interval) RETURNS boolean
        params = {
            "kind_of_backfill": 'home_page_graph',
            "update_date": yesterday,
        }
        result = backfill.get(**params)
        self.assertTrue(result[0][0])
        
        '''        
        #......................................................................
        # Test backfill_weekly_report_partitions(startweek DATE, endweek DATE, 
        # tablename TEXT) RETURNS boolean
        params = {
            "kind_of_backfill": 'weekly_report_partitions',
            "start_date": yesterday,
            "end_date": now,
            "tablename": "raw_adu",
        }
        result = backfill.get(**params)
        self.assertTrue(result[0][0])
        '''
        #......................................................................
        # Test backfill_crashes_by_user_build(updateday date, check_period 
        # interval DEFAULT '01:00:00'::interval) RETURNS boolean
        params = {
            "kind_of_backfill": 'crashes_by_user_build',
            "update_date": yesterday,
        }
        result = backfill.get(**params)
        self.assertTrue(result[0][0])
        
        #......................................................................
        # Test backfill_matviews(firstday date, lastday date DEFAULT 
        # NULL::date, reportsclean boolean DEFAULT true, check_period interval 
        # DEFAULT '01:00:00'::interval) RETURNS boolean
        params = {
            "kind_of_backfill": 'matviews',
            "start_date": yesterday,
            "end_date": now,
            "check_data": 'false',
        }
        result = backfill.get(**params)
        self.assertTrue(result[0][0])

        #......................................................................
        # Test backfill_nightly_builds(updateday date) RETURNS boolean
        params = {
            "kind_of_backfill": 'nightly_builds',
            "update_date": yesterday,
        }
        result = backfill.get(**params)
        self.assertTrue(result[0][0])
        
        #......................................................................
        # Test backfill_tcbs_build(updateday date, check_period interval 
        # DEFAULT '01:00:00'::interval) RETURNS boolean
        params = {
            "kind_of_backfill": 'tcbs_build',
            "update_date": yesterday,
        }
        result = backfill.get(**params)
        self.assertTrue(result[0][0])

        #......................................................................
        # Test backfill_exploitability(updateday date) RETURNS boolean
        params = {
            "kind_of_backfill": 'exploitability',
            "update_date": yesterday,
        }
        result = backfill.get(**params)
        self.assertTrue(result[0][0])

        #......................................................................
        # Test backfill_tcbs(updateday date, check_period interval DEFAULT 
        # '01:00:00'::interval) RETURNS boolean
        params = {
            "kind_of_backfill": 'tcbs',
            "update_date": yesterday,
        }
        result = backfill.get(**params)
        self.assertTrue(result[0][0])

        #......................................................................
        # Test backfill_rank_compare(updateday date DEFAULT NULL::date) 
        # RETURNS boolean
        params = {
            "kind_of_backfill": 'rank_compare',
            "update_date": yesterday,
        }
        result = backfill.get(**params)
        self.assertTrue(result[0][0])
        '''        
        #......................................................................
        # Test backfill_reports_duplicates(start_time timestamp without 
        # time zone, end_time timestamp without time zone) RETURNS integer
        params = {
            "kind_of_backfill": 'reports_duplicates',
            "start_date": yesterday,
            "end_date": now,
        }
        result = backfill.get(**params)
        self.assertTrue(result[0][0])
        '''
        #......................................................................
        # Test backfill_crashes_by_user(updateday date, check_period 
        # interval DEFAULT '01:00:00'::interval) RETURNS boolean
        params = {
            "kind_of_backfill": 'crashes_by_user',
            "update_date": yesterday,
        }
        result = backfill.get(**params)
        self.assertTrue(result[0][0])

        #......................................................................
        # Test backfill_hang_report(backfilldate date) RETURNS boolean
        params = {
            "kind_of_backfill": 'hang_report',
            "backfilldate": yesterday,
        }
        result = backfill.get(**params)
        self.assertTrue(result[0][0])
        '''
        #......................................................................
        # Test backfill_one_day() RETURNS text
        params = {
            "kind_of_backfill": 'one_day',
        }
        result = backfill.get(**params)
        self.assertTrue(result[0][0])
        '''
        
        #......................................................................
        # Test backfill_one_day(bkdate date) RETURNS text
        params = {
            "kind_of_backfill": 'one_day',
            "bkdate": now,
        }
        result = backfill.get(**params)
        self.assertEqual(result, [('done',)])
        '''
        #......................................................................
        # Test backfill_daily_crashes(updateday date) RETURNS boolean
        params = {
            "kind_of_backfill": 'daily_crashes',
            "update_date": now,
        }
        result = backfill.get(**params)
        self.assertTrue(result[0][0])
        
        #......................................................................
        # Test backfill_signature_counts(begindate date, enddate date) RETURNS boolean
        params = {
            "kind_of_backfill": 'signature_counts',
            "start_date": yesterday,
            "end_date": now,
        }
        result = backfill.get(**params)
        self.assertTrue(result[0][0])
        '''

        #......................................................................
        # Test backfill_home_page_graph_build(updateday date, check_period 
        # interval DEFAULT '01:00:00'::interval) RETURNS boolean
        params = {
            "kind_of_backfill": 'home_page_graph_build',
            "update_date": yesterday,
        }
        result = backfill.get(**params)
        self.assertTrue(result[0][0])

        #......................................................................
        # Test backfill_signature_summary(updateday date) RETURNS boolean
        params = {
            "kind_of_backfill": 'signature_summary',
            "update_date": yesterday,
        }
        result = backfill.get(**params)
        self.assertTrue(result[0][0])