Skip to content

Instantly share code, notes, and snippets.

@torarnv
Created August 5, 2021 10:09
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save torarnv/9e0f0669f853fbe0f8f6b421512a718f to your computer and use it in GitHub Desktop.
Save torarnv/9e0f0669f853fbe0f8f6b421512a718f to your computer and use it in GitHub Desktop.
diff --git c/tests/auto/testlib/selftests/expected_expectfail.junitxml w/tests/auto/testlib/selftests/expected_expectfail.junitxml
index a7b3f48ec43..94fa4476596 100644
--- c/tests/auto/testlib/selftests/expected_expectfail.junitxml
+++ w/tests/auto/testlib/selftests/expected_expectfail.junitxml
@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8" ?>
-<testsuite name="tst_ExpectFail" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="24" failures="6" errors="0" skipped="0" time="@TEST_DURATION@">
+<testsuite name="tst_ExpectFail" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="27" failures="13" errors="0" skipped="0" time="@TEST_DURATION@">
<properties>
<property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
<property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
@@ -51,7 +51,20 @@
<testcase name="xfailWithoutVerify(second row)" classname="tst_ExpectFail" time="@TEST_DURATION@">
<failure type="fail" message="QEXPECT_FAIL was called without any subsequent verification statements"/>
</testcase>
- <testcase name="xpass" classname="tst_ExpectFail" time="@TEST_DURATION@">
+ <testcase name="xpassAndAbort" classname="tst_ExpectFail" time="@TEST_DURATION@">
+ <failure type="xpass" message="&apos;true&apos; returned TRUE unexpectedly. ()"/>
+ </testcase>
+ <testcase name="xpassAndContinueThenFail" classname="tst_ExpectFail" time="@TEST_DURATION@">
+ <failure type="xpass" message="&apos;true&apos; returned TRUE unexpectedly. ()"/>
+ <failure type="fail" message="&apos;false&apos; returned FALSE. (Failure after xpass)"/>
+ </testcase>
+ <testcase name="xpassAndContinueThenPass" classname="tst_ExpectFail" time="@TEST_DURATION@">
+ <failure type="xpass" message="&apos;true&apos; returned TRUE unexpectedly. ()"/>
+ </testcase>
+ <testcase name="xpassAndContinueThenXPass" classname="tst_ExpectFail" time="@TEST_DURATION@">
+ <failure type="xpass" message="&apos;true&apos; returned TRUE unexpectedly. ()"/>
+ <failure type="xpass" message="&apos;true&apos; returned TRUE unexpectedly. ()"/>
+ <failure type="xpass" message="&apos;true&apos; returned TRUE unexpectedly. ()"/>
<failure type="xpass" message="&apos;true&apos; returned TRUE unexpectedly. ()"/>
</testcase>
<testcase name="xpassDataDrivenWithQVerify(XPass)" classname="tst_ExpectFail" time="@TEST_DURATION@">
diff --git c/tests/auto/testlib/selftests/expected_expectfail.lightxml w/tests/auto/testlib/selftests/expected_expectfail.lightxml
index 7c91656b89a..3ed29799eee 100644
--- c/tests/auto/testlib/selftests/expected_expectfail.lightxml
+++ w/tests/auto/testlib/selftests/expected_expectfail.lightxml
@@ -126,7 +126,37 @@
</Incident>
<Duration msecs="0"/>
</TestFunction>
-<TestFunction name="xpass">
+<TestFunction name="xpassAndAbort">
+<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
+ <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description>
+</Incident>
+ <Duration msecs="0"/>
+</TestFunction>
+<TestFunction name="xpassAndContinueThenFail">
+<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
+ <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
+ <Description><![CDATA['false' returned FALSE. (Failure after xpass)]]></Description>
+</Incident>
+ <Duration msecs="0"/>
+</TestFunction>
+<TestFunction name="xpassAndContinueThenPass">
+<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
+ <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description>
+</Incident>
+ <Duration msecs="0"/>
+</TestFunction>
+<TestFunction name="xpassAndContinueThenXPass">
+<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
+ <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description>
+</Incident>
+<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
+ <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description>
+</Incident>
+<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
+ <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description>
+</Incident>
<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description>
</Incident>
diff --git c/tests/auto/testlib/selftests/expected_expectfail.tap w/tests/auto/testlib/selftests/expected_expectfail.tap
index 2ad422b7f9a..676c5c97018 100644
--- c/tests/auto/testlib/selftests/expected_expectfail.tap
+++ w/tests/auto/testlib/selftests/expected_expectfail.tap
@@ -84,13 +84,31 @@ not ok 18 - xfailWithoutVerify(second row)
---
# QEXPECT_FAIL was called without any subsequent verification statements
...
-ok 19 - xpass() # TODO 'true' returned TRUE unexpectedly. ()
-ok 20 - xpassDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. ()
-ok 21 - xpassDataDrivenWithQVerify(Pass)
-ok 22 - xpassDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly.
-ok 23 - xpassDataDrivenWithQCompare(Pass)
-ok 24 - cleanupTestCase()
-1..24
-# tests 24
+ok 19 - xpassAndAbort() # TODO 'true' returned TRUE unexpectedly. ()
+ok 20 - xpassAndContinueThenFail() # TODO 'true' returned TRUE unexpectedly. ()
+not ok 21 - xpassAndContinueThenFail()
+ ---
+ type: QVERIFY
+ message: Failure after xpass
+ wanted: true (false)
+ found: false (false)
+ expected: true (false)
+ actual: false (false)
+ at: tst_ExpectFail::xpassAndContinueThenFail() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
+ file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+ line: 0
+ ...
+ok 22 - xpassAndContinueThenPass() # TODO 'true' returned TRUE unexpectedly. ()
+ok 23 - xpassAndContinueThenXPass() # TODO 'true' returned TRUE unexpectedly. ()
+ok 24 - xpassAndContinueThenXPass() # TODO 'true' returned TRUE unexpectedly. ()
+ok 25 - xpassAndContinueThenXPass() # TODO 'true' returned TRUE unexpectedly. ()
+ok 26 - xpassAndContinueThenXPass() # TODO 'true' returned TRUE unexpectedly. ()
+ok 27 - xpassDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. ()
+ok 28 - xpassDataDrivenWithQVerify(Pass)
+ok 29 - xpassDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly.
+ok 30 - xpassDataDrivenWithQCompare(Pass)
+ok 31 - cleanupTestCase()
+1..31
+# tests 31
# pass 18
-# fail 6
+# fail 13
diff --git c/tests/auto/testlib/selftests/expected_expectfail.teamcity w/tests/auto/testlib/selftests/expected_expectfail.teamcity
index cac2c344910..2cd84b1c759 100644
--- c/tests/auto/testlib/selftests/expected_expectfail.teamcity
+++ w/tests/auto/testlib/selftests/expected_expectfail.teamcity
@@ -47,9 +47,26 @@
##teamcity[testStarted name='xfailWithoutVerify(second row)' flowId='tst_ExpectFail']
##teamcity[testFailed name='xfailWithoutVerify(second row)' message='Failure!' details='QEXPECT_FAIL was called without any subsequent verification statements' flowId='tst_ExpectFail']
##teamcity[testFinished name='xfailWithoutVerify(second row)' flowId='tst_ExpectFail']
-##teamcity[testStarted name='xpass()' flowId='tst_ExpectFail']
-##teamcity[testFailed name='xpass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail']
-##teamcity[testFinished name='xpass()' flowId='tst_ExpectFail']
+##teamcity[testStarted name='xpassAndAbort()' flowId='tst_ExpectFail']
+##teamcity[testFailed name='xpassAndAbort()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail']
+##teamcity[testFinished name='xpassAndAbort()' flowId='tst_ExpectFail']
+##teamcity[testStarted name='xpassAndContinueThenFail()' flowId='tst_ExpectFail']
+##teamcity[testFailed name='xpassAndContinueThenFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail']
+##teamcity[testFinished name='xpassAndContinueThenFail()' flowId='tst_ExpectFail']
+##teamcity[testFailed name='xpassAndContinueThenFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'false|' returned FALSE. (Failure after xpass)' flowId='tst_ExpectFail']
+##teamcity[testFinished name='xpassAndContinueThenFail()' flowId='tst_ExpectFail']
+##teamcity[testStarted name='xpassAndContinueThenPass()' flowId='tst_ExpectFail']
+##teamcity[testFailed name='xpassAndContinueThenPass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail']
+##teamcity[testFinished name='xpassAndContinueThenPass()' flowId='tst_ExpectFail']
+##teamcity[testStarted name='xpassAndContinueThenXPass()' flowId='tst_ExpectFail']
+##teamcity[testFailed name='xpassAndContinueThenXPass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail']
+##teamcity[testFinished name='xpassAndContinueThenXPass()' flowId='tst_ExpectFail']
+##teamcity[testFailed name='xpassAndContinueThenXPass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail']
+##teamcity[testFinished name='xpassAndContinueThenXPass()' flowId='tst_ExpectFail']
+##teamcity[testFailed name='xpassAndContinueThenXPass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail']
+##teamcity[testFinished name='xpassAndContinueThenXPass()' flowId='tst_ExpectFail']
+##teamcity[testFailed name='xpassAndContinueThenXPass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail']
+##teamcity[testFinished name='xpassAndContinueThenXPass()' flowId='tst_ExpectFail']
##teamcity[testStarted name='xpassDataDrivenWithQVerify(XPass)' flowId='tst_ExpectFail']
##teamcity[testFailed name='xpassDataDrivenWithQVerify(XPass)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail']
##teamcity[testFinished name='xpassDataDrivenWithQVerify(XPass)' flowId='tst_ExpectFail']
diff --git c/tests/auto/testlib/selftests/expected_expectfail.txt w/tests/auto/testlib/selftests/expected_expectfail.txt
index 75d41e5733d..13f32f94f08 100644
--- c/tests/auto/testlib/selftests/expected_expectfail.txt
+++ w/tests/auto/testlib/selftests/expected_expectfail.txt
@@ -42,7 +42,21 @@ XFAIL : tst_ExpectFail::xfailOnAnyRow(second row) This test should xfail
PASS : tst_ExpectFail::xfailOnAnyRow(second row)
FAIL! : tst_ExpectFail::xfailWithoutVerify(first row) QEXPECT_FAIL was called without any subsequent verification statements
FAIL! : tst_ExpectFail::xfailWithoutVerify(second row) QEXPECT_FAIL was called without any subsequent verification statements
-XPASS : tst_ExpectFail::xpass() 'true' returned TRUE unexpectedly. ()
+XPASS : tst_ExpectFail::xpassAndAbort() 'true' returned TRUE unexpectedly. ()
+ Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)]
+XPASS : tst_ExpectFail::xpassAndContinueThenFail() 'true' returned TRUE unexpectedly. ()
+ Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)]
+FAIL! : tst_ExpectFail::xpassAndContinueThenFail() 'false' returned FALSE. (Failure after xpass)
+ Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)]
+XPASS : tst_ExpectFail::xpassAndContinueThenPass() 'true' returned TRUE unexpectedly. ()
+ Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)]
+XPASS : tst_ExpectFail::xpassAndContinueThenXPass() 'true' returned TRUE unexpectedly. ()
+ Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)]
+XPASS : tst_ExpectFail::xpassAndContinueThenXPass() 'true' returned TRUE unexpectedly. ()
+ Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)]
+XPASS : tst_ExpectFail::xpassAndContinueThenXPass() 'true' returned TRUE unexpectedly. ()
+ Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)]
+XPASS : tst_ExpectFail::xpassAndContinueThenXPass() 'true' returned TRUE unexpectedly. ()
Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)]
XPASS : tst_ExpectFail::xpassDataDrivenWithQVerify(XPass) 'true' returned TRUE unexpectedly. ()
Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)]
@@ -51,5 +65,5 @@ XPASS : tst_ExpectFail::xpassDataDrivenWithQCompare(XPass) QCOMPARE(1, 1) retur
Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)]
PASS : tst_ExpectFail::xpassDataDrivenWithQCompare(Pass)
PASS : tst_ExpectFail::cleanupTestCase()
-Totals: 18 passed, 6 failed, 0 skipped, 0 blacklisted, 0ms
+Totals: 18 passed, 13 failed, 0 skipped, 0 blacklisted, 0ms
********* Finished testing of tst_ExpectFail *********
diff --git c/tests/auto/testlib/selftests/expected_expectfail.xml w/tests/auto/testlib/selftests/expected_expectfail.xml
index ccfd942c92f..42492bd6c1a 100644
--- c/tests/auto/testlib/selftests/expected_expectfail.xml
+++ w/tests/auto/testlib/selftests/expected_expectfail.xml
@@ -128,7 +128,37 @@
</Incident>
<Duration msecs="0"/>
</TestFunction>
-<TestFunction name="xpass">
+<TestFunction name="xpassAndAbort">
+<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
+ <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description>
+</Incident>
+ <Duration msecs="0"/>
+</TestFunction>
+<TestFunction name="xpassAndContinueThenFail">
+<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
+ <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
+ <Description><![CDATA['false' returned FALSE. (Failure after xpass)]]></Description>
+</Incident>
+ <Duration msecs="0"/>
+</TestFunction>
+<TestFunction name="xpassAndContinueThenPass">
+<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
+ <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description>
+</Incident>
+ <Duration msecs="0"/>
+</TestFunction>
+<TestFunction name="xpassAndContinueThenXPass">
+<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
+ <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description>
+</Incident>
+<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
+ <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description>
+</Incident>
+<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
+ <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description>
+</Incident>
<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0">
<Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description>
</Incident>
diff --git c/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp w/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
index fb6300b4a8e..fa5b4a4e9a4 100644
--- c/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+++ w/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
@@ -51,7 +51,10 @@ private slots:
void xfailOnAnyRow() const;
void xfailWithoutVerify_data() const;
void xfailWithoutVerify() const;
- void xpass() const;
+ void xpassAndAbort() const;
+ void xpassAndContinueThenFail() const;
+ void xpassAndContinueThenPass() const;
+ void xpassAndContinueThenXPass() const;
void xpassDataDrivenWithQVerify_data() const;
void xpassDataDrivenWithQVerify() const;
void xpassDataDrivenWithQCompare_data() const;
@@ -209,7 +212,7 @@ void tst_ExpectFail::xfailWithoutVerify() const
QEXPECT_FAIL("", "This expected failure should be ignored", Abort);
}
-void tst_ExpectFail::xpass() const
+void tst_ExpectFail::xpassAndAbort() const
{
QEXPECT_FAIL("", "This test should xpass", Abort);
QVERIFY(true);
@@ -219,6 +222,40 @@ void tst_ExpectFail::xpass() const
QVERIFY2(false, "This should not be reached");
}
+void tst_ExpectFail::xpassAndContinueThenFail() const
+{
+ QEXPECT_FAIL("", "This test should xpass", Continue);
+ QVERIFY(true);
+
+ QVERIFY2(false, "Failure after xpass");
+}
+
+void tst_ExpectFail::xpassAndContinueThenPass() const
+{
+ QEXPECT_FAIL("", "This test should xpass", Continue);
+ QVERIFY(true);
+
+ QVERIFY2(true, "Pass after xpass");
+ QVERIFY2(true, "Pass after xpass");
+ QVERIFY2(true, "Pass after xpass");
+}
+
+void tst_ExpectFail::xpassAndContinueThenXPass() const
+{
+ QEXPECT_FAIL("", "This test should xpass", Continue);
+ QVERIFY(true);
+
+ QEXPECT_FAIL("", "This test should xpass", Continue);
+ QVERIFY(true);
+
+ QEXPECT_FAIL("", "This test should xpass", Continue);
+ QVERIFY(true);
+
+ QEXPECT_FAIL("", "This test should xpass", Continue);
+ QVERIFY(true);
+}
+
+
void tst_ExpectFail::xpassDataDrivenWithQVerify_data() const
{
QTest::addColumn<bool>("shouldXPass");
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment