Skip to content

Instantly share code, notes, and snippets.

/gist:596234

Created September 24, 2010 23:33
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save anonymous/596234 to your computer and use it in GitHub Desktop.
Save anonymous/596234 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
gistfile1.txt
Index: WebKitTools/ChangeLog
===================================================================
--- WebKitTools/ChangeLog (revision 68237)
+++ WebKitTools/ChangeLog (working copy)
@@ -1,3 +1,12 @@
+2010-09-24 Gabor Rapcsanyi <rgabor@inf.u-szeged.hu>
+
+ Reviewed by NOBODY (OOPS!).
+
+ [NRWT] Put the http and websocket tests to end of the test list.
+ https://bugs.webkit.org/show_bug.cgi?id=
+
+ * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
+
2010-09-23 Tony Chang <tony@chromium.org>
Unreviewed, rolling out r68232.
Index: WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
===================================================================
--- WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py (revision 68237)
+++ WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py (working copy)
@@ -1,5 +1,6 @@
#!/usr/bin/env python
# Copyright (C) 2010 Google Inc. All rights reserved.
+# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
@@ -62,6 +63,7 @@
import sys
import time
import traceback
+import threading
from layout_package import dump_render_tree_thread
from layout_package import json_layout_results_generator
@@ -69,6 +71,7 @@
from layout_package import test_expectations
from layout_package import test_failures
from layout_package import test_results_uploader
+from layout_package import http_lock
from test_types import image_diff
from test_types import text_diff
from test_types import test_type_base
@@ -273,6 +276,9 @@
self._result_queue = Queue.Queue()
self._retrying = False
+ self._thread_lock = threading.Lock()
+ self._http_lock = http_lock.HttpLock()
+
def collect_tests(self, args, last_unexpected_results):
"""Find all the files to test.
@@ -510,48 +516,53 @@
Return:
The Queue of lists of TestInfo objects.
"""
+ test_lists = []
+ test_lists_back = []
if (self._options.experimental_fully_parallel or
self._is_single_threaded()):
- filename_queue = Queue.Queue()
+ tests = []
+ tests_back = []
for test_file in test_files:
- filename_queue.put(
- ('.', [self._get_test_info_for_file(test_file)]))
- return filename_queue
+ splitted_dir_path = self._get_dir_for_test_file(test_file).split(os.sep)
+ if 'http' in splitted_dir_path or 'websocket' in splitted_dir_path:
+ tests_back.append(self._get_test_info_for_file(test_file))
+ else:
+ tests.append(self._get_test_info_for_file(test_file))
+ if tests:
+ test_lists = [(".", tests)]
+ if tests_back:
+ test_lists_back = [("http", tests_back)]
+ else:
+ tests_by_dir = {}
+ for test_file in test_files:
+ directory = self._get_dir_for_test_file(test_file)
+ tests_by_dir.setdefault(directory, [])
+ tests_by_dir[directory].append(
+ self._get_test_info_for_file(test_file))
+ # Sort by the number of tests in the dir so that the ones with the
+ # most tests get run first in order to maximize parallelization.
+ # Number of tests is a good enough, but not perfect, approximation
+ # of how long that set of tests will take to run. We can't just use
+ # a PriorityQueue until we move # to Python 2.6.
+ for directory in tests_by_dir:
+ test_list = tests_by_dir[directory]
+ # Keep the tests in alphabetical order.
+ # TODO: Remove once tests are fixed so they can be run in any
+ # order.
+ test_list.reverse()
+ test_list_tuple = (directory, test_list)
+ splitted_dir_path = directory.split(os.sep)
+ if 'http' in splitted_dir_path or 'websocket' in splitted_dir_path:
+ test_lists_back.append(test_list_tuple)
+ else:
+ test_lists.append(test_list_tuple)
+ test_lists.sort(lambda a, b: cmp(len(b[1]), len(a[1])))
+ # Put the http and websocket tests to end of the list
+ # for http locking.
+ if test_lists_back:
+ test_lists += test_lists_back
- tests_by_dir = {}
- for test_file in test_files:
- directory = self._get_dir_for_test_file(test_file)
- tests_by_dir.setdefault(directory, [])
- tests_by_dir[directory].append(
- self._get_test_info_for_file(test_file))
-
- # Sort by the number of tests in the dir so that the ones with the
- # most tests get run first in order to maximize parallelization.
- # Number of tests is a good enough, but not perfect, approximation
- # of how long that set of tests will take to run. We can't just use
- # a PriorityQueue until we move # to Python 2.6.
- test_lists = []
- http_tests = None
- for directory in tests_by_dir:
- test_list = tests_by_dir[directory]
- # Keep the tests in alphabetical order.
- # TODO: Remove once tests are fixed so they can be run in any
- # order.
- test_list.reverse()
- test_list_tuple = (directory, test_list)
- if directory == 'LayoutTests' + os.sep + 'http':
- http_tests = test_list_tuple
- else:
- test_lists.append(test_list_tuple)
- test_lists.sort(lambda a, b: cmp(len(b[1]), len(a[1])))
-
- # Put the http tests first. There are only a couple hundred of them,
- # but each http test takes a very long time to run, so sorting by the
- # number of tests doesn't accurately capture how long they take to run.
- if http_tests:
- test_lists.insert(0, http_tests)
-
filename_queue = Queue.Queue()
for item in test_lists:
filename_queue.put(item)
@@ -597,7 +608,7 @@
test_args = self._get_test_args(i)
thread = dump_render_tree_thread.TestShellThread(self._port,
self._options, filename_queue, self._result_queue,
- test_types, test_args)
+ test_types, test_args, self._thread_lock, self._http_lock)
if self._is_single_threaded():
thread.run_in_main_thread(self, result_summary)
else:
@@ -696,6 +707,10 @@
"""Returns whether the test runner needs an HTTP server."""
return self._contains_tests(self.HTTP_SUBDIR)
+ def needs_websocket(self):
+ """Returns whether the test runner needs a WEBSOCKET server."""
+ return self._contains_tests(self.WEBSOCKET_SUBDIR)
+
def set_up_run(self):
"""Configures the system to be ready to run tests.
@@ -728,14 +743,16 @@
if not result_summary:
return None
- if self.needs_http():
- self._printer.print_update('Starting HTTP server ...')
- self._port.start_http_server()
+ # Do not start when http lock enabled
+ if not self._options.wait_for_httpd:
+ if self.needs_http():
+ self._printer.print_update('Starting HTTP server ...')
+ self._port.start_http_server()
- if self._contains_tests(self.WEBSOCKET_SUBDIR):
- self._printer.print_update('Starting WebSocket server ...')
- self._port.start_websocket_server()
- # self._websocket_secure_server.Start()
+ if self.needs_websocket():
+ self._printer.print_update('Starting WebSocket server ...')
+ self._port.start_websocket_server()
+ # self._websocket_secure_server.Start()
return result_summary
@@ -789,6 +806,8 @@
sys.stdout.flush()
sys.stderr.flush()
+ self.clean_up_run()
+
self._printer.print_one_line_summary(result_summary.total,
result_summary.expected,
result_summary.unexpected)
@@ -832,6 +851,8 @@
self._port.stop_websocket_server()
_log.debug("stopping helper")
self._port.stop_helper()
+ _log.debug("clean up http lock")
+ self._http_lock.cleanup_http_lock()
def update_summary(self, result_summary):
"""Update the summary and print results with any completed tests."""
@@ -1406,7 +1427,6 @@
result_summary = test_runner.set_up_run()
if result_summary:
num_unexpected_results = test_runner.run(result_summary)
- test_runner.clean_up_run()
_log.debug("Testing completed, Exit status: %d" %
num_unexpected_results)
finally:
@@ -1587,6 +1607,9 @@
optparse.make_option("--no-record-results", action="store_false",
default=True, dest="record_results",
help="Don't record the results."),
+ optparse.make_option("--wait-for-httpd", action="store_true",
+ default=False, dest="wait_for_httpd",
+ help="Wait for http locks."),
# old-run-webkit-tests also has HTTP toggle options:
# --[no-]http Run (or do not run) http tests
# (default: run)
Index: WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py
===================================================================
--- WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py (revision 68237)
+++ WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py (working copy)
@@ -62,6 +62,7 @@
base.Port.__init__(self, **kwargs)
self._cached_build_root = None
self._cached_apache_path = None
+ self.http_server_running = False
# FIXME: disable pixel tests until they are run by default on the
# build machines.
Index: WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py
===================================================================
--- WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py (revision 68237)
+++ WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py (working copy)
@@ -1,5 +1,6 @@
#!/usr/bin/env python
# Copyright (C) 2010 Google Inc. All rights reserved.
+# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
@@ -247,7 +248,7 @@
class TestShellThread(WatchableThread):
def __init__(self, port, options, filename_list_queue, result_queue,
- test_types, test_args):
+ test_types, test_args, thread_lock, http_server_lock):
"""Initialize all the local state for this DumpRenderTree thread.
Args:
@@ -276,6 +277,11 @@
self._num_tests = 0
self._start_time = 0
self._stop_time = 0
+ self._thread_lock = thread_lock
+ self._http_server_lock = http_server_lock
+ self._lock_wait_begin = None
+ self._lock_wait_end = None
+ self._lock_wait_time = 0
# Current directory of tests we're running.
self._current_dir = None
@@ -298,7 +304,8 @@
return self._test_results
def get_total_time(self):
- return max(self._stop_time - self._start_time, 0.0)
+ return max(self._stop_time - self._start_time - \
+ self._lock_wait_time, 0.0)
def get_num_tests(self):
return self._num_tests
@@ -337,6 +344,18 @@
do multi-threaded debugging."""
self._run(test_runner, result_summary)
+ def next_timeout(self):
+ """Return the time the test is supposed to finish by."""
+ if self._next_timeout:
+ return self._next_timeout + self._http_lock_wait_time()
+ return self._next_timeout
+
+ def _http_lock_wait_time(self):
+ """Return the time of waiting for http lock."""
+ if self._lock_wait_end == 0:
+ return time.time() - self._lock_wait_begin
+ return self._lock_wait_end - self._lock_wait_begin
+
def _run(self, test_runner, result_summary):
"""Main work entry point of the thread. Basically we pull urls from the
filename queue and run the tests until we run out of urls.
@@ -372,6 +391,28 @@
tests_run_file.close()
return
+ self._lock_wait_end = 0
+ self._lock_wait_begin = time.time()
+
+ splitted_path = self._current_dir.split(os.sep)
+ if self._options.wait_for_httpd and not self._port.http_server_running and \
+ ('http' in splitted_path or 'websocket' in splitted_path):
+ self._thread_lock.acquire()
+ try:
+ if not self._port.http_server_running:
+ self._http_server_lock.wait_for_httpd_lock()
+
+ self._port.start_http_server()
+ self._port.start_websocket_server()
+
+ self._port.http_server_running = True
+ if self._lock_wait_time == 0:
+ self._lock_wait_time = time.time() - self._lock_wait_begin
+ finally:
+ self._thread_lock.release()
+
+ self.lock_wait_end = time.time()
+
self._num_tests_in_current_dir = len(self._filename_list)
self._current_dir_start_time = time.time()
Index: WebKitTools/Scripts/webkitpy/layout_tests/layout_package/http_lock.py
===================================================================
--- WebKitTools/Scripts/webkitpy/layout_tests/layout_package/http_lock.py (revision 0)
+++ WebKitTools/Scripts/webkitpy/layout_tests/layout_package/http_lock.py (revision 0)
@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
+# Copyright (C) 2010 Andras Becsi (abecsi@inf.u-szeged.hu), University of Szeged
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF SZEGED ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL UNIVERSITY OF SZEGED OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import os
+import sys
+import time
+import fcntl
+import signal
+import glob
+
+
+class HttpLock:
+
+ def __init__(self):
+
+ signal.signal(signal.SIGINT, self._handle_interrupt)
+ signal.signal(signal.SIGTERM, self._handle_interrupt)
+
+ self._lock_path = "/tmp"
+ self._lock_file_prefix = "WebKitHttpd.lock."
+ self._lock_file_path_prefix = os.path.join(self._lock_path, self._lock_file_prefix)
+ self._guard_lock_file = os.path.join(self._lock_path, "WebKit.lock")
+ self._my_lock_file_name = ""
+ self._wait_begin_time = None
+ self._wait_time = None
+
+ def _handle_interrupt(self, signum, stack):
+ self.cleanup_http_lock()
+ sys.exit(signum)
+
+ def cleanup_http_lock(self):
+ if os.path.exists(self._my_lock_file_name):
+ os.unlink(self._my_lock_file_name)
+
+ def _extract_lock_number(self, lock_file_name):
+ if not os.path.exists(lock_file_name):
+ return -1
+ prefix_length = len(self._lock_file_path_prefix)
+ return int(lock_file_name[prefix_length:])
+
+ def _lock_file_list(self):
+ lock_list = glob.glob(self._lock_file_path_prefix + '*')
+ lock_list.sort(key=self._extract_lock_number)
+ return lock_list
+
+ def _next_lock_number(self):
+ lock_list = self._lock_file_list()
+ if not lock_list:
+ return 0
+ return self._extract_lock_number(lock_list[-1]) + 1
+
+ def _check_pid(self, current_pid):
+ try:
+ os.kill(current_pid, 0)
+ except OSError:
+ return False
+ else:
+ return True
+
+ def _curent_lock_pid(self):
+ lock_list = self._lock_file_list()
+ if not lock_list:
+ return
+ try:
+ current_lock_file = open(lock_list[0], 'r')
+ current_pid = current_lock_file.readline()
+ current_lock_file.close()
+ if not current_pid or \
+ not (sys.platform in ('darwin', 'linux2') and self._check_pid(int(current_pid))):
+ os.unlink(lock_list[0])
+ return
+ except IOError, OSError:
+ return
+ return int(current_pid)
+
+ def _create_lock_file(self):
+ while(True):
+ try:
+ sequential_guard_lock = os.open(self._guard_lock_file, os.O_CREAT | os.O_NONBLOCK | os.O_EXCL)
+
+ self._my_lock_file_name = self._lock_file_path_prefix + str(self._next_lock_number())
+ lock_file = open(self._my_lock_file_name, 'w')
+ lock_file.write(str(os.getpid()))
+ lock_file.close()
+ os.close(sequential_guard_lock)
+ os.unlink(self._guard_lock_file)
+ break
+ except OSError:
+ pass
+
+ def wait_for_httpd_lock(self):
+ self._create_lock_file()
+ while self._curent_lock_pid() != os.getpid():
+ time.sleep(1)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment