Skip to content

Instantly share code, notes, and snippets.

@GabiThume
Last active Dec 21, 2015
Embed
What would you like to do?
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from socorro.external.postgresql.base import PostgreSQLBase
from socorro.lib import external_common
import socorro.database.database as db
from socorro.external import MissingOrBadArgumentError
#==============================================================================
class Backfill(PostgreSQLBase):
#--------------------------------------------------------------------------
def get(self, **kwargs):
self.connection = self.database.connection()
cursor = self.connection.cursor()
filters = [
("kind_of_backfill", None, "str"),
("start_date", None, "date"),
("end_date", None, "date"),
("check_data", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.kind_of_backfill:
raise MissingOrBadArgumentError(
"Mandatory parameter 'kind_of_backfill' is missing or empty"
)
query = "SELECT backfill_%(kind)s" %{"kind": kwargs['kind_of_backfill']}
parameters = kwargs
parameters.pop('kind_of_backfill')
parameters = str(kwargs.keys())
parameters = parameters.replace("'", "").replace("[", "(%(").replace("]",")s)").replace(", ", ")s, %(")
if kwargs != {}:
query += parameters
print query % kwargs
else:
query += "()"
error_message = "Failed to retrieve backfill "+params.kind_of_backfill+" from PostgreSQL"
if kwargs != {}:
results = self.query(query, kwargs, error_message=error_message)
else:
results = self.query(query, error_message=error_message)
return results
(socorro-virtualenv)gabi@ubuntu:~/Desktop/CODE/src/socorro$ nosetests socorro/unittest/external/postgresql/test_backfill.py -s
SELECT backfill_adu(2013-08-14)
SELECT backfill_all_dups(2013-08-14, 2013-08-15)
SELECT backfill_build_adu(2013-08-14)
SELECT backfill_correlations(2013-08-14)
SELECT backfill_reports_clean(2013-08-14, 2013-08-15)
SELECT backfill_explosiveness(2013-08-14)
SELECT backfill_home_page_graph(2013-08-14)
SELECT backfill_crashes_by_user_build(2013-08-14)
SELECT backfill_matviews(2013-08-14, 2013-08-15, false)
SELECT backfill_nightly_builds(2013-08-14)
SELECT backfill_tcbs_build(2013-08-14)
SELECT backfill_exploitability(2013-08-14)
SELECT backfill_tcbs(2013-08-14)
SELECT backfill_rank_compare(2013-08-14)
SELECT backfill_crashes_by_user(2013-08-14)
SELECT backfill_hang_report(2013-08-14)
SELECT backfill_one_day(2013-08-15)
SELECT backfill_home_page_graph_build(2013-08-14)
SELECT backfill_signature_summary(2013-08-14)
.
----------------------------------------------------------------------
Ran 1 test in 32.841s
OK
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import datetime
from socorro.external.postgresql.backfill import Backfill
from nose.plugins.attrib import attr
from .unittestbase import PostgreSQLTestCase
from socorro.lib import datetimeutil
from socorro.external.postgresql import fakedata
from socorro.external import MissingOrBadArgumentError
from socorro.external.postgresql import setupdb_app
import socorro.database.database as db
import cStringIO
#==============================================================================
@attr(integration='postgres')
class TestBackfill(PostgreSQLTestCase):
"""Simple test of backfill calling"""
#--------------------------------------------------------------------------
def setUp(self):
""" Populate product_info table with fake data """
super(TestBackfill, self).setUp()
cursor = self.connection.cursor()
self.now = datetimeutil.utc_now()
now = self.now.date()
yesterday = now - datetime.timedelta(days=1)
lastweek = now - datetime.timedelta(days=7)
raw = fakedata.RawADU()
row = raw.generate_rows().next()
self.tables = []
start_date = end_date = None
for table in fakedata.tables:
table = table(days=1)
if start_date:
if start_date > table.start_date:
start_date = table.start_date
else:
start_date = table.start_date
if end_date:
if end_date < table.start_date:
end_date = table.end_date
else:
end_date = table.end_date
table_name = table.table
columns = table.columns
for rows in table.generate_rows():
data = dict(zip(columns, rows))
columns_ = str(columns).replace("[", "(").replace("]", ")").replace("'", "")
values = str(columns_).replace("(", "(%(").replace(")", ")s);").replace(", ", ")s, %(")
self.tables.append(table_name)
query = "INSERT INTO %(table_name)s " % {'table_name': table_name}
query = query + columns_ +" VALUES " + values
cursor.execute(query, data)
self.connection.commit()
#--------------------------------------------------------------------------
def tearDown(self):
""" Cleanup the database, delete tables and functions """
tables = str(self.tables).replace("'", "").replace("[", "").replace("]", "")
cursor = self.connection.cursor()
cursor.execute("TRUNCATE "+ tables + " CASCADE;")
self.connection.commit()
self.connection.close()
super(TestBackfill, self).tearDown()
#--------------------------------------------------------------------------
def test_get(self):
backfill = Backfill(config=self.config)
self.now = datetimeutil.utc_now()
now = self.now.date()
yesterday = now - datetime.timedelta(days=1)
#......................................................................
# Test Raise error if parameter is not passed
params = {"kind_of_backfill": ''}
self.assertRaises(MissingOrBadArgumentError,
backfill.get,
**params)
#......................................................................
# Test backfill_adu(updateday date) RETURNS boolean
params = {
"kind_of_backfill": 'adu',
"start_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_all_dups(start_date timestamp without time zone,
# end_date timestamp without time zone) RETURNS boolean
params = {
"kind_of_backfill": 'all_dups',
"start_date": yesterday,
"end_date": now,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_build_adu(updateday date) RETURNS boolean
params = {
"kind_of_backfill": 'build_adu',
"start_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_correlations(updateday date) RETURNS boolean
params = {
"kind_of_backfill": 'correlations',
"start_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_reports_clean(begin_time timestamp with time zone,
# end_time timestamp with time zone DEFAULT NULL::timestamp with
# time zone) RETURNS boolean
params = {
"kind_of_backfill": 'reports_clean',
"start_date": yesterday,
"end_date": now,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_explosiveness(updateday date) RETURNS boolean
params = {
"kind_of_backfill": 'explosiveness',
"update_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_home_page_graph(updateday date, check_period interval
# DEFAULT '01:00:00'::interval) RETURNS boolean
params = {
"kind_of_backfill": 'home_page_graph',
"update_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
'''
#......................................................................
# Test backfill_weekly_report_partitions(startweek DATE, endweek DATE,
# tablename TEXT) RETURNS boolean
params = {
"kind_of_backfill": 'weekly_report_partitions',
"start_date": yesterday,
"end_date": now,
"tablename": "raw_adu",
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
'''
#......................................................................
# Test backfill_crashes_by_user_build(updateday date, check_period
# interval DEFAULT '01:00:00'::interval) RETURNS boolean
params = {
"kind_of_backfill": 'crashes_by_user_build',
"update_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_matviews(firstday date, lastday date DEFAULT
# NULL::date, reportsclean boolean DEFAULT true, check_period interval
# DEFAULT '01:00:00'::interval) RETURNS boolean
params = {
"kind_of_backfill": 'matviews',
"start_date": yesterday,
"end_date": now,
"check_data": 'false',
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_nightly_builds(updateday date) RETURNS boolean
params = {
"kind_of_backfill": 'nightly_builds',
"update_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_tcbs_build(updateday date, check_period interval
# DEFAULT '01:00:00'::interval) RETURNS boolean
params = {
"kind_of_backfill": 'tcbs_build',
"update_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_exploitability(updateday date) RETURNS boolean
params = {
"kind_of_backfill": 'exploitability',
"update_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_tcbs(updateday date, check_period interval DEFAULT
# '01:00:00'::interval) RETURNS boolean
params = {
"kind_of_backfill": 'tcbs',
"update_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_rank_compare(updateday date DEFAULT NULL::date)
# RETURNS boolean
params = {
"kind_of_backfill": 'rank_compare',
"update_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
'''
#......................................................................
# Test backfill_reports_duplicates(start_time timestamp without
# time zone, end_time timestamp without time zone) RETURNS integer
params = {
"kind_of_backfill": 'reports_duplicates',
"start_date": yesterday,
"end_date": now,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
'''
#......................................................................
# Test backfill_crashes_by_user(updateday date, check_period
# interval DEFAULT '01:00:00'::interval) RETURNS boolean
params = {
"kind_of_backfill": 'crashes_by_user',
"update_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_hang_report(backfilldate date) RETURNS boolean
params = {
"kind_of_backfill": 'hang_report',
"backfilldate": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
'''
#......................................................................
# Test backfill_one_day() RETURNS text
params = {
"kind_of_backfill": 'one_day',
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
'''
#......................................................................
# Test backfill_one_day(bkdate date) RETURNS text
params = {
"kind_of_backfill": 'one_day',
"bkdate": now,
}
result = backfill.get(**params)
self.assertEqual(result, [('done',)])
'''
#......................................................................
# Test backfill_daily_crashes(updateday date) RETURNS boolean
params = {
"kind_of_backfill": 'daily_crashes',
"update_date": now,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_signature_counts(begindate date, enddate date) RETURNS boolean
params = {
"kind_of_backfill": 'signature_counts',
"start_date": yesterday,
"end_date": now,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
'''
#......................................................................
# Test backfill_home_page_graph_build(updateday date, check_period
# interval DEFAULT '01:00:00'::interval) RETURNS boolean
params = {
"kind_of_backfill": 'home_page_graph_build',
"update_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_signature_summary(updateday date) RETURNS boolean
params = {
"kind_of_backfill": 'signature_summary',
"update_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment