Skip to content

Instantly share code, notes, and snippets.

@GabiThume

GabiThume/backfill.py

Created August 19, 2013 18:45
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save GabiThume/6272593 to your computer and use it in GitHub Desktop.
Save GabiThume/6272593 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
backfill.py
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import datetime
from socorro.external.postgresql.backfill import Backfill
from nose.plugins.attrib import attr
from .unittestbase import PostgreSQLTestCase
from socorro.lib import datetimeutil
from socorro.external.postgresql import fakedata
from socorro.external import MissingOrBadArgumentError
from socorro.external.postgresql import setupdb_app
import socorro.database.database as db
import cStringIO
#==============================================================================
@attr(integration='postgres')
class TestBackfill(PostgreSQLTestCase):
"""Tests the calling of all backfill functions"""
#--------------------------------------------------------------------------
def setUp(self):
""" Populate tables with fake data """
super(TestBackfill, self).setUp()
cursor = self.connection.cursor()
self.tables = []
for table in fakedata.tables:
table = table(days=1)
table_name = table.table
table_columns = table.columns
values = str(tuple(["%("+i+")s" for i in table_columns]))
columns = str(tuple(table_columns))
self.tables.append(table_name)
for rows in table.generate_rows():
data = dict(zip(table_columns, rows))
query = "INSERT INTO %(table)s " % {'table': table_name}
query = query + columns.replace("'", "").replace(",)", ")")
query = query + " VALUES "
query = query + values.replace(",)", ")").replace("'", "")
cursor.execute(query, data)
self.connection.commit()
#--------------------------------------------------------------------------
def tearDown(self):
""" Cleanup the database, delete tables and functions """
cursor = self.connection.cursor()
tables = str(self.tables).replace("[", "").replace("]", "")
cursor.execute("TRUNCATE "+ tables.replace("'", "") + " CASCADE;")
self.connection.commit()
self.connection.close()
super(TestBackfill, self).tearDown()
#--------------------------------------------------------------------------
def test_get(self):
backfill = Backfill(config=self.config)
self.now = datetimeutil.utc_now()
now = self.now.date()
yesterday = now - datetime.timedelta(days=1)
lastweek = now - datetime.timedelta(days=7)
#......................................................................
# Test Raise error if parameter is not passed
params = {"kind_of_backfill": ''}
self.assertRaises(MissingOrBadArgumentError, backfill.get, **params)
#......................................................................
# Test backfill_adu(updateday date) RETURNS boolean
params = {
"kind_of_backfill": 'adu',
"start_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_all_dups(start_date timestamp without time zone,
# end_date timestamp without time zone) RETURNS boolean
params = {
"kind_of_backfill": 'all_dups',
"start_date": yesterday,
"end_date": now,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_build_adu(updateday date) RETURNS boolean
params = {
"kind_of_backfill": 'build_adu',
"start_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_correlations(updateday date) RETURNS boolean
params = {
"kind_of_backfill": 'correlations',
"start_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
'''
#......................................................................
# Test backfill_reports_clean(begin_time timestamp with time zone,
# end_time timestamp with time zone DEFAULT NULL::timestamp with
# time zone) RETURNS boolean
params = {
"kind_of_backfill": 'reports_clean',
"start_date": yesterday,
"end_date": now,
}
result = backfill.get(**params)
#self.assertTrue(result[0][0])
'''
#......................................................................
# Test backfill_explosiveness(updateday date) RETURNS boolean
params = {
"kind_of_backfill": 'explosiveness',
"update_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_home_page_graph(updateday date, check_period interval
# DEFAULT '01:00:00'::interval) RETURNS boolean
params = {
"kind_of_backfill": 'home_page_graph',
"update_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_weekly_report_partitions(startweek DATE, endweek DATE,
# tablename TEXT) RETURNS boolean
params = {
"kind_of_backfill": 'weekly_report_partitions',
"start_date": '2012-01-02',
"end_date": '2013-03-04',
"tablename": 'raw_crashes',
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_crashes_by_user_build(updateday date, check_period
# interval DEFAULT '01:00:00'::interval) RETURNS boolean
params = {
"kind_of_backfill": 'crashes_by_user_build',
"update_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_matviews(firstday date, lastday date DEFAULT
# NULL::date, reportsclean boolean DEFAULT true, check_period interval
# DEFAULT '01:00:00'::interval) RETURNS boolean
params = {
"kind_of_backfill": 'matviews',
"start_date": yesterday,
"end_date": now,
"check_data": 'false',
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_nightly_builds(updateday date) RETURNS boolean
params = {
"kind_of_backfill": 'nightly_builds',
"update_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_tcbs_build(updateday date, check_period interval
# DEFAULT '01:00:00'::interval) RETURNS boolean
params = {
"kind_of_backfill": 'tcbs_build',
"update_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_exploitability(updateday date) RETURNS boolean
params = {
"kind_of_backfill": 'exploitability',
"update_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_tcbs(updateday date, check_period interval DEFAULT
# '01:00:00'::interval) RETURNS boolean
params = {
"kind_of_backfill": 'tcbs',
"update_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_rank_compare(updateday date DEFAULT NULL::date)
# RETURNS boolean
params = {
"kind_of_backfill": 'rank_compare',
"update_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
'''
#......................................................................
# Test backfill_reports_duplicates(start_time timestamp without
# time zone, end_time timestamp without time zone) RETURNS integer
params = {
"kind_of_backfill": 'reports_duplicates',
"start_date": yesterday,
"end_date": now,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
'''
#......................................................................
# Test backfill_crashes_by_user(updateday date, check_period
# interval DEFAULT '01:00:00'::interval) RETURNS boolean
params = {
"kind_of_backfill": 'crashes_by_user',
"update_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_hang_report(backfilldate date) RETURNS boolean
params = {
"kind_of_backfill": 'hang_report',
"backfilldate": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_one_day(bkdate date) RETURNS text
params = {# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from socorro.external.postgresql.base import PostgreSQLBase
from socorro.external import MissingOrBadArgumentError
from socorro.lib import external_common
#==============================================================================
class Backfill(PostgreSQLBase):
#--------------------------------------------------------------------------
def get(self, **kwargs):
filters = [
("kind_of_backfill", None, "str"),
("start_date", None, "date"),
("end_date", None, "date"),
("update_date", None, "date"),
("tablename", None, "str"),
("check_data", None, "boolean"),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.kind_of_backfill:
raise MissingOrBadArgumentError(
"Mandatory parameter 'kind_of_backfill' is missing or empty"
)
query = "SELECT backfill_%(kind)s " \
% {"kind": params.kind_of_backfill}
if len(kwargs) <= 1:
query = query + "()"
else:
if kwargs['kind_of_backfill'] == 'weekly_report_partitions':
weekly_report_partitions = ['start_date', 'end_date', 'tablename']
query_params = [(i, params[i]) for i in weekly_report_partitions]
params = [i[1] for i in query_params]
query = query + str(tuple(params))
else:
params = kwargs.copy()
params.pop('kind_of_backfill')
params = str(params.keys())
params = params.replace("'", "").replace("[", "(%(")
params = params.replace("]", ")s)").replace(", ", ")s, %(")
query += params
error_message = "Failed to retrieve backfill %s from PostgreSQL"
error_message = error_message % kwargs['kind_of_backfill']
if kwargs == {}:
results = self.query(query, error_message=error_message)
print query
else:
results = self.query(query, kwargs, error_message=error_message)
print query % kwargs
return results
"kind_of_backfill": 'one_day',
"bkdate": now,
}
result = backfill.get(**params)
self.assertEqual(result, [('done',)])
#......................................................................
# Test backfill_daily_crashes(updateday date) RETURNS boolean -- "last_backfill_temp"
cursor = self.connection.cursor()
#cursor.execute("""
# CREATE TABLE daily_crashes
# (count INTEGER, report_type VARCHAR(50),
# productdims_id INTEGER, os_short_name VARCHAR(50),
# adu_day DATE)
# """
self.tables.append('daily_crashes')
self.connection.commit()
params = {
"kind_of_backfill": 'daily_crashes',
"update_date": now,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
'''
#......................................................................
# Test backfill_signature_counts(begindate date, enddate date) RETURNS boolean -- os_signature_counts
params = {
"kind_of_backfill": 'signature_counts',
"start_date": yesterday,
"end_date": now,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
'''
#......................................................................
# Test backfill_home_page_graph_build(updateday date, check_period
# interval DEFAULT '01:00:00'::interval) RETURNS boolean
params = {
"kind_of_backfill": 'home_page_graph_build',
"update_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
#......................................................................
# Test backfill_signature_summary(updateday date) RETURNS boolean
params = {
"kind_of_backfill": 'signature_summary',
"update_date": yesterday,
}
result = backfill.get(**params)
self.assertTrue(result[0][0])
Raw
gistfile1.txt
SELECT backfill_adu (2013-08-18)
SELECT backfill_all_dups (2013-08-18, 2013-08-19)
SELECT backfill_build_adu (2013-08-18)
SELECT backfill_correlations (2013-08-18)
SELECT backfill_explosiveness (2013-08-18)
SELECT backfill_home_page_graph (2013-08-18)
SELECT backfill_weekly_report_partitions ('2012-01-02', '2013-03-04', 'raw_crashes')
SELECT backfill_crashes_by_user_build (2013-08-18)
SELECT backfill_matviews (2013-08-18, 2013-08-19, false)
SELECT backfill_nightly_builds (2013-08-18)
SELECT backfill_tcbs_build (2013-08-18)
SELECT backfill_exploitability (2013-08-18)
SELECT backfill_tcbs (2013-08-18)
SELECT backfill_rank_compare (2013-08-18)
SELECT backfill_crashes_by_user (2013-08-18)
SELECT backfill_hang_report (2013-08-18)
SELECT backfill_one_day (2013-08-19)
SELECT backfill_daily_crashes (2013-08-19)
SELECT backfill_home_page_graph_build (2013-08-18)
SELECT backfill_signature_summary (2013-08-18)
.
----------------------------------------------------------------------
Ran 1 test in 31.336s
OK
Raw
test_backfill.py
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from socorro.external.postgresql.base import PostgreSQLBase
from socorro.external import MissingOrBadArgumentError
from socorro.lib import external_common
#==============================================================================
class Backfill(PostgreSQLBase):
#--------------------------------------------------------------------------
def get(self, **kwargs):
filters = [
("kind_of_backfill", None, "str"),
("start_date", None, "date"),
("end_date", None, "date"),
("update_date", None, "date"),
("tablename", None, "str"),
("check_data", None, "boolean"),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.kind_of_backfill:
raise MissingOrBadArgumentError(
"Mandatory parameter 'kind_of_backfill' is missing or empty"
)
query = "SELECT backfill_%(kind)s " \
% {"kind": params.kind_of_backfill}
if len(kwargs) <= 1:
query = query + "()"
else:
if kwargs['kind_of_backfill'] == 'weekly_report_partitions':
weekly_report_partitions = ['start_date', 'end_date', 'tablename']
query_params = [(i, params[i]) for i in weekly_report_partitions]
params = [i[1] for i in query_params]
query = query + str(tuple(params))
else:
params = kwargs.copy()
params.pop('kind_of_backfill')
params = str(params.keys())
params = params.replace("'", "").replace("[", "(%(")
params = params.replace("]", ")s)").replace(", ", ")s, %(")
query += params
error_message = "Failed to retrieve backfill %s from PostgreSQL"
error_message = error_message % kwargs['kind_of_backfill']
if kwargs == {}:
results = self.query(query, error_message=error_message)
print query
else:
results = self.query(query, kwargs, error_message=error_message)
print query % kwargs
return results
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment