Dendrite SyTest

01http-server.pl.patch

diff --git a/tests/01http-server.pl b/tests/01http-server.pl
index eb60bf8b..d8f099b8 100644
--- a/tests/01http-server.pl
+++ b/tests/01http-server.pl
@@ -47,6 +47,9 @@ sub start_test_server_ssl {
    );
 }
 
+# List of Awaiter structs
+my @pending_awaiters;
+
 our $TEST_SERVER_INFO = fixture(
    requires => [],
 
@@ -59,6 +62,8 @@ our $TEST_SERVER_INFO = fixture(
       my $http_client;
       my $server_info;
 
+      @pending_awaiters = ();
+
       start_test_server_ssl( $http_server )->then( sub {
          my ( $listener ) = @_;
          my $sockport = $listener->read_handle->sockport;
@@ -135,9 +140,6 @@ our $TEST_SERVER_INFO = fixture(
    },
 );
 
-# List of Awaiter structs
-my @pending_awaiters;
-
 package SyTest::HTTPServer {
    use base qw( Net::Async::HTTP::Server );
 

03_unread_count.pl.patch

diff --git a/tests/61push/09_notifications_api.pl b/tests/61push/09_notifications_api.pl
index 3718e24a..2a161426 100644
--- a/tests/61push/09_notifications_api.pl
+++ b/tests/61push/09_notifications_api.pl
@@ -18,12 +18,14 @@ test "Notifications can be viewed with GET /notifications",
 
          matrix_join_room( $user2, $room_id );
       })->then( sub {
-         matrix_send_room_text_message( $user2, $room_id,
+         matrix_send_room_text_message_synced( $user2, $room_id,
             body => "Test message 1",
          );
       })->then( sub {
          my ( $event_id ) = @_;
 
+         log_if_fail( "Sent Test message 1" );
+
          # We need to send a read receipt before the server will start
          # calculating notifications.
          matrix_advance_room_receipt( $user1, $room_id, "m.read" => $event_id );
@@ -37,8 +39,10 @@ test "Notifications can be viewed with GET /notifications",
          # push rules and 2) we need to call /notifications repeatedly as it
          # might take a while for messages to propagate.
 
+         log_if_fail( "Advanced room receipt 1" );
+
          retry_until_success {
-            matrix_send_room_text_message( $user2, $room_id,
+            matrix_send_room_text_message_synced( $user2, $room_id,
                body => "Test message 2",
             )->then(sub {
                do_request_json_for( $user1,
@@ -63,6 +67,8 @@ test "Notifications can be viewed with GET /notifications",
       })->then( sub {
          my ( $notif ) = @_;
 
+         log_if_fail( "Sent Test message 2" );
+
          # Check the notif has the expected keys
          assert_json_keys( $notif, qw( room_id actions event read ts ) );
          assert_ok( exists $notif->{profile_tag}, "profile_tag defined" );
@@ -70,13 +76,15 @@ test "Notifications can be viewed with GET /notifications",
 
          # Now we send a message and advance the read receipt up until that
          # point, and test that notifications becomes empty
-         matrix_send_room_text_message( $user2, $room_id,
+         matrix_send_room_text_message_synced( $user2, $room_id,
             body => "Test message 3",
          );
       })->then( sub {
          my ( $event_id ) = @_;
 
-         matrix_advance_room_receipt( $user1, $room_id, "m.read" => $event_id );
+         log_if_fail( "Sent Test message 3" );
+
+         matrix_advance_room_receipt_synced( $user1, $room_id, "m.read" => $event_id );
       })->then( sub {
          retry_until_success {
             do_request_json_for( $user1,

base.go.patch

diff --git a/setup/base.go b/setup/base.go
index 5778235c..70046cde 100644
--- a/setup/base.go
+++ b/setup/base.go
@@ -175,6 +175,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo
        }
        transport := *http.DefaultTransport.(*http.Transport)
        if cfg.Global.HTTPClientTLSInsecureSkipVerify {
+               transport.DisableKeepAlives = true
                transport.TLSClientConfig = &tls.Config{
                        InsecureSkipVerify: true,
                }

Dendrite.pm.patch

diff --git a/lib/SyTest/Homeserver/Dendrite.pm b/lib/SyTest/Homeserver/Dendrite.pm
index f65d3077..ccce71f8 100644
--- a/lib/SyTest/Homeserver/Dendrite.pm
+++ b/lib/SyTest/Homeserver/Dendrite.pm
@@ -102,6 +102,9 @@ sub _get_config
          private_key => $self->{paths}{matrix_key},
          presence_enabled => $JSON::true,
 
+         # The Pushgateway uses HTTPS.
+         http_client_tls_insecure_skip_verify_for_testing => $JSON::true,
+
          kafka => {
             use_naffka => $JSON::true,
             naffka_database => {

dendrite_sytest.sh.patch

diff --git a/scripts/dendrite_sytest.sh b/scripts/dendrite_sytest.sh
index de4db739..5426b288 100755
--- a/scripts/dendrite_sytest.sh
+++ b/scripts/dendrite_sytest.sh
@@ -61,12 +61,14 @@ fi
 
 # Check for new tests to be added to the test whitelist
 /src/show-expected-fail-tests.sh /logs/results.tap /src/sytest-whitelist \
-    /src/sytest-blacklist > /work/show_expected_fail_tests_output.txt || TEST_STATUS=$?
+    /src/sytest-blacklist | tee /work/show_expected_fail_tests_output.txt || TEST_STATUS=$?
 
 echo >&2 "--- Copying assets"
 
 # Copy out the logs
 rsync -r --ignore-missing-args --min-size=1B -av /work/server-0 /work/server-1 /logs --include "*/" --include="*.log.*" --include="*.log" --exclude="*"
+find /logs | xargs -r chmod go+r
+find /logs -type d | xargs -r chmod go+x
 
 # Generate annotate.md. This is Buildkite-specific.
 if [ -n "$BUILDKITE_LABEL" ] && [ $TEST_STATUS -ne 0 ]; then
 

run-sytest.sh

#!/bin/bash
#
# Runs SyTest either from Docker Hub, or from ../sytest. If it's run
# locally, the Docker image is rebuilt first.
#
# Logs are stored in ../sytestout/logs.

set -e
set -o pipefail

main() {
    local tag=buster
    local base_image=debian:$tag
    local runargs=()

    cd "$(dirname "$0")"

    if [ -d ../sytest ]; then
        local tmpdir
        tmpdir="$(mktemp -d --tmpdir run-systest.XXXXXXXXXX)"
        trap "rm -r '$tmpdir'" EXIT

        if [ -z "$DISABLE_BUILDING_SYTEST" ]; then
            echo "Re-building ../sytest Docker images..."

            local status
            (
                cd ../sytest

                docker build -f docker/base.Dockerfile --build-arg BASE_IMAGE="$base_image" --tag matrixdotorg/sytest:"$tag" .
                docker build -f docker/dendrite.Dockerfile --build-arg SYTEST_IMAGE_TAG="$tag" --tag matrixdotorg/sytest-dendrite:latest .
            ) &>"$tmpdir/buildlog" || status=$?
            if (( status != 0 )); then
                # Docker is very verbose, and we don't really care about
                # building SyTest. So we accumulate and only output on
                # failure.
                cat "$tmpdir/buildlog" >&2
                return $status
            fi
        fi

        runargs+=( -v "$PWD/../sytest:/sytest:ro" )
    fi

    local sytestout=$PWD/../sytestout
    mkdir -p "$sytestout"/{logs,cache/go-build,cache/go-pkg}
    docker run \
           --rm \
           --name "sytest-dendrite-${LOGNAME}" \
           -e LOGS_USER=$(id -u) \
           -e LOGS_GROUP=$(id -g) \
           -v "$PWD:/src/:ro" \
           -v "$sytestout/logs:/logs/" \
           -v "$sytestout/cache/go-build:/root/.cache/go-build" \
           -v "$sytestout/cache/go-pkg:/gopath/pkg" \
           "${runargs[@]}" \
           matrixdotorg/sytest-dendrite:latest "$@"
}

main "$@"