Test Time Bombs lightning talk Wojciech Bulaty 18 Aug 2015
Me (Wojtek) 9 years of software development 5+ years of TDD, pair programming, etc.
Problem ● Problem – One failing test and the build is red – Long running builds? Harder to notice new failing tests! – Test failing for a long time (temporary integration issues?) ● Possible solutions – Suppress a failing test for a day? (green build again) – Set a reminder to look at a test next time somebody sees it? (red for a month)
Specific example ● Integration test failing because of network issues
2. Me (Wojtek)
9 years of software development
5+ years of TDD, pair programming, etc.
3. Problem
● Problem
– One failing test and the build is red
– Long running builds? Harder to notice new failing tests!
– Test failing for a long time (temporary integration issues?)
● Possible solutions
– Suppress a failing test for a day? (green build again)
– Set a reminder to look at a test next time somebody sees it? (red
for a month)
5. Common solutions
● Supress the test by @Ignore
– Who will un-ignore it?
– When?
● //TODO comments never get actioned
● Card on the board
● JIRA ticket
● Mute tests in TeamCity (local build remain red,
cannot check in)
7. package wb;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.Clock;
import java.util.Date;
import static java.lang.String.format;
public class Timebomb {
private final Clock clock;
Timebomb(Clock
public static Tim
ebomb timebomb() {
return new Timebomb(Clock.systemUTC());
}
public boolean blowUpAfter(int year, int month, int dayOfMonth, String explanation) {
try {
blowUpAfter(new SimpleDateFormat("yyyy-MM-dd").parse(format("%s-%s-%s", year, month, dayOfMonth)), explanation);
} catch (ParseException e) {
throw new RuntimeException(e);
}
return true;
}
public boolean blowUpAfter(Date dateToBlowUp, String explanation) {
Date now = new Date(clock.millis());
if(now.compareTo(dateToBlowUp)>0) {
throw new AssertionError("Requested timebomb, "+ explanation);
}
return true;
}
8. package wb;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.Clock;
import java.util.Date;
import static java.lang.String.format;
public class Timebomb {
private final Clock clock;
Timebolock = clock;
}
public static Timebomb timebomb() {
return new Timebomb(Clock.systemUTC());
}
public boolean blowUpAfter(int year, int month, int dayOfMonth, String explanation) {
try {
blowUpAfter(new SimpleDateFormat("yyyy-MM-dd").parse(format("%s-%s-%s", year, month, dayOfMonth)), explanation);
} catch (ParseException e) {
throw new RuntimeException(e);
}
return true;
}
public boolean blowUpAfter(Date dateToBlowUp, String explanation) {
Date now = new Date(clock.millis());
if(now.compareTo(dateToBlowUp)>0) {
throw new AssertionError("Requested timebomb, "+ explanation);
}
return true;
}
9. package wb;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.Clock;
import java.util.Date;
import static java.lang.String.format;
public class Timebomb {
private final Clock clock;
Timebomb(Clock clock) {
this.clock
public static Timebomb timebomb() {
return n
ew Timebomb(Clock.systemUTC());
}
public boolean blowUpAfter(int year, int month, int dayOfMonth, String explanation) {
try {
blowUpAfter(new SimpleDateFormat("yyyy-MM-dd").parse(format("%s-%s-%s", year, month, dayOfMonth)), explanation);
} catch (ParseException e) {
throw new RuntimeException(e);
}
return true;
}
public boolean blowUpAfter(Date dateToBlowUp, String explanation) {
Date now = new Date(clock.millis());
if(now.compareTo(dateToBlowUp)>0) {
throw new AssertionError("Requested timebomb, "+ explanation);
}
return false;
}
10. package wb;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.Clock;
import java.util.Date;
import static java.lang.String.format;
public class Timebomb {
private final Clock clock;
Timebo
m
public static Tim
ebomb timebomb() {
return new Timebomb(Clock.systemUTC());
}
public boolean blowUpAfter(int year, int month, int dayOfMonth, String explanation) {
try { explanation
);
} catch (ParseException e) {
throw new RuntimeException(e);
}
return true;
}
public boolean blowUpAfter(Date dateToBlowUp, String explanation) {
Date now = new Date(clock.millis());
if(now.compareTo(dateToBlowUp)>0) {
throw new AssertionError("Requested timebomb, "+ explanation);
}
return false;
11. package wb;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.Clock;
import java.util.Date;
import static java.lang.String.format;
public class Timebomb {
private final Clock clock;
Timebomb(Clock clock) {
this.cl
}
public static Timebomb timebomb() {
return new Timebomb(Clock.systemUTC());
}
public boolean blowUpAfter(int year, int month, int dayOfMonth, String explanation) {
try {
return blowUpAfter(new SimpleDateFormat("yyyy-MM-dd").parse(format("%s-%s-%s", year, month, dayOfMonth)), explanation);
} catch (ParseException e) {
throw new RuntimeException(e);
}
}
public boolean blowUpAfter(Date dateToBlowUp, String explanation) {
Date now = new Date(clock.millis());
if(now.compareTo(dateToBlowUp)>0) {
throw new AssertionError("Requested timebomb, "+ explanation);
}
return false;
}
12. package wb;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.Clock;
import java.util.Date;
import static java.lang.String.format;
public class Timebomb {
private final Clock clock;
Timebomb(Clock clock) {
this.clock = clock;
}
public static Timebomb timebomb() {
return new Timebomb(Clock.systemUTC());
}
public boolean blowUpAfter(int year, int month, int dayOfMonth, String explanation) {
try {
return blowUpAfter(new SimpleDateFormat("yyyy-MM-dd").parse(format("%s-%s-%s", year, month, dayOfMonth)), explanation);
} catch (ParseException e) {
throw new RuntimeException(e);
}
}
public boolean blowUpAfter(Date dateToBlowUp, String explanation) {
Date now = new Date(clock.millis());
if(now.compareTo(dateToBlowUp)>0) {
throw new AssertionError("Requested timebomb, "+ explanation);
}
return false;
}
13. import org.junit.Test;
import wb.TestData;
import java.nio.file.Path;
import java.nio.file.Paths;
import static wb.Timebomb.timebomb;
public class SftpClientIntegrationTest {
private Path localPath = Paths.get("/testing/testFile1");
private Path remotePath = Paths.get("/testing/testFile1");
private String hostname = "aHost.mycompany.com";
@Test
public void downloadsAFile() throws Exception {
if(timebomb().blowUpAfter(2015, 8, 7, "chase with Paul from operations the network connectivity")) {
new SftpClient(hostname).download(remotePath, localPath);
}
}
@Test
public void supportsAllSftpServers() throws Exception {
String onlyWorkingTestSite = "testEnvUK";
timebomb().blowUpAfter(2015, 9, 1, "chase with Mike from SA team about the test environments other than " +
onlyWorkingTestSite);
TestData.allSftpServers()
.stream().filter(hostname -> hostname.startsWith(onlyWorkingTestSite))
.forEach(hostname -> new SftpClient(hostname).listDirectory(remotePath));
}
14. import org.junit.Test;
import wb.TestData;
import java.nio.file.Path;
import java.nio.file.Paths;
import static wb.Timebomb.timebomb;
public class SftpClientIntegrationTest {
private Path localPath = Paths.get("/testing/testFile1");
private Path remotePath = Paths.get("/testing/testFile1");
private String hostname = "aHost.mycompany.com";
@Test
public void downloadsAFile() throws Exception {
if(timebomb().blowUpAfter(2015, 8, 7, "chase with Paul from operations the network connectivity")) {
new SftpClient(hostname).download(remotePath, localPath);
}
}
@Test
public void supportsAllSftpServers() throws Exception {
String onlyWorkingTestSite = "testEnvUK";
timebomb().blowUpAfter(2015, 9, 1, "chase with Mike from SA team about the test environments other than " +
onlyWorkingTestSite);
TestData.allSftpServers()
.stream().filter(hostname -> hostname.startsWith(onlyWorkingTestSite))
.forEach(hostname -> new SftpClient(hostname).listDirectory(remotePath));
}
15. import org.junit.Test;
import wb.TestData;
import java.nio.file.Path;
import java.nio.file.Paths;
import static wb.Timebomb.timebomb;
public class SftpClientIntegrationTest {
private Path localPath = Paths.get("/testing/testFile1");
private Path remotePath = Paths.get("/testing/testFile1");
private String hostname = "aHost.mycompany.com";
@Test
public void downloadsAFile() throws Exception {
if(timebomb().blowUpAfter(2015, 8, 7, "chase with Paul from operations the network connectivity")) {
new SftpClient(hostname).download(remotePath, localPath);
}
}
@Test
public void supportsAllSftpServers() throws Exception {
String onlyWorkingTestSite = "testEnvUK";
timebomb().blowUpAfter(2015, 9, 1, "chase with Mike from SA team about the test environments other than " +
onlyWorkingTestSite);
TestData.allSftpServers()
.stream().filter(hostname -> hostname.startsWith(onlyWorkingTestSite))
.forEach(hostname -> new SftpClient(hostname).listDirectory(remotePath));
}
16. import org.junit.Test;
import wb.TestData;
import java.nio.file.Path;
import java.nio.file.Paths;
import static wb.Timebomb.timebomb;
public class SftpClientIntegrationTest {
private Path localPath = Paths.get("/testing/testFile1");
private Path remotePath = Paths.get("/testing/testFile1");
private String hostname = "aHost.mycompany.com";
@Test
public void downloadsAFile() throws Exception {
if(timebomb().blowUpAfter(2015, 8, 7, "chase with Paul from operations the network connectivity")) {
new SftpClient(hostname).download(remotePath, localPath);
}
}
@Test
public void supportsAllSftpServers() throws Exception {
String onlyWorkingTestSite = "testEnvUK";
timebomb().blowUpAfter(2015, 9, 1, "chase with Mike from SA team about the test environments other than " +
onlyWorkingTestSite);
TestData.allSftpServers()
.stream().filter(hostname -> hostname.startsWith(onlyWorkingTestSite))
.forEach(hostname -> new SftpClient(hostname).listDirectory(remotePath));
}
17. import org.junit.Test;
import wb.TestData;
import java.nio.file.Path;
import java.nio.file.Paths;
import static wb.Timebomb.timebomb;
public class SftpClientIntegrationTest {
private Path localPath = Paths.get("/testing/testFile1");
private Path remotePath = Paths.get("/testing/testFile1");
private String hostname = "aHost.mycompany.com";
@Test
public void downloadsAFile() throws Exception {
if(timebomb().blowUpAfter(2015, 8, 7, "chase with Paul from operations the network connectivity")) {
new SftpClient(hostname).download(remotePath, localPath);
}
}
@Test
public void supportsAllSftpServers() throws Exception {
String onlyWorkingTestSite = "testEnvUK";
timebomb().blowUpAfter(2015, 9, 1, "chase with Mike from SA team about the test environments other than " +
onlyWorkingTestSite);
TestData.allSftpServers()
.stream().filter(hostname -> hostname.startsWith(onlyWorkingTestSite))
.forEach(hostname -> new SftpClient(hostname).listDirectory(remotePath));
}
18. import org.junit.Test;
import wb.TestData;
import java.nio.file.Path;
import java.nio.file.Paths;
import static wb.Timebomb.timebomb;
public class SftpClientIntegrationTest {
private Path localPath = Paths.get("/testing/testFile1");
private Path remotePath = Paths.get("/testing/testFile1");
private String hostname = "aHost.mycompany.com";
@Test
public void downloadsAFile() throws Exception {
if(timebomb().blowUpAfter(2015, 8, 7, "chase with Paul from operations the network connectivity")) {
new SftpClient(hostname).download(remotePath, localPath);
}
}
@Test
public void supportsAllSftpServers() throws Exception {
String onlyWorkingTestSite = "testEnvUK";
timebomb().blowUpAfter(2015, 9, 1, "chase with Mike from SA team about the test environments other than " +
onlyWorkingTestSite);
TestData.allSftpServers()
.stream().filter(hostname -> hostname.startsWith(onlyWorkingTestSite))
.forEach(hostname -> new SftpClient(hostname).listDirectory(remotePath));
}
19. import org.junit.Test;
import wb.TestData;
import java.nio.file.Path;
import java.nio.file.Paths;
import static wb.Timebomb.timebomb;
public class SftpClientIntegrationTest {
private Path localPath = Paths.get("/testing/testFile1");
private Path remotePath = Paths.get("/testing/testFile1");
private String hostname = "aHost.mycompany.com";
@Test
public void downloadsAFile() throws Exception {
if(timebomb().blowUpAfter(2015, 8, 7, "chase with Paul from operations the network connectivity")) {
new SftpClient(hostname).download(remotePath, localPath);
}
}
@Test
public void supportsAllSftpServers() throws Exception {
String onlyWorkingTestSite = "testEnvUK";
timebomb().blowUpAfter(2015, 9, 1, "chase with Mike from SA team about the test environments other than " +
onlyWorkingTestSite);
TestData.allSftpServers()
.stream().filter(hostname -> hostname.startsWith(onlyWorkingTestSite))
.forEach(hostname -> new SftpClient(hostname).listDirectory(remotePath));
}
20. import org.junit.Test;
import wb.TestData;
import java.nio.file.Path;
import java.nio.file.Paths;
import static wb.Timebomb.timebomb;
public class SftpClientIntegrationTest {
private Path localPath = Paths.get("/testing/testFile1");
private Path remotePath = Paths.get("/testing/testFile1");
private String hostname = "aHost.mycompany.com";
@Test
public void downloadsAFile() throws Exception {
if(timebomb().blowUpAfter(2015, 8, 7, "chase with Paul from operations the network connectivity")) {
new SftpClient(hostname).download(remotePath, localPath);
}
}
@Test
public void supportsAllSftpServers() throws Exception {
String onlyWorkingTestSite = "testEnvUK";
timebomb().blowUpAfter(2015, 9, 1, "chase with Mike from SA team about the test environments other than " +
onlyWorkingTestSite);
TestData.allSftpServers()
.stream().filter(hostname -> hostname.startsWith(onlyWorkingTestSite))
.forEach(hostname -> new SftpClient(hostname).listDirectory(remotePath));
}
21. import org.junit.Test;
import wb.TestData;
import java.nio.file.Path;
import java.nio.file.Paths;
import static wb.Timebomb.timebomb;
public class SftpClientIntegrationTest {
private Path localPath = Paths.get("/testing/testFile1");
private Path remotePath = Paths.get("/testing/testFile1");
private String hostname = "aHost.mycompany.com";
@Test
public void downloadsAFile() throws Exception {
if(timebomb().blowUpAfter(2015, 8, 7, "chase with Paul from operations the network connectivity")) {
new SftpClient(hostname).download(remotePath, localPath);
}
}
@Test
public void supportsAllSftpServers() throws Exception {
String onlyWorkingTestSite = "testEnvUK";
timebomb().blowUpAfter(2015, 9, 1, "chase with Mike from SA team about the test environments other than " +
onlyWorkingTestSite);
TestData.allSftpServers()
.stream().filter(hostname -> hostname.startsWith(onlyWorkingTestSite))
.forEach(hostname -> new SftpClient(hostname).listDirectory(remotePath));
}
22. Tradeoffs
● Will not see if the test is flaky for the period of
time.
● If application is not build/test not run for a long
time, the timebomb will not go off.
23. Credits & Thanks
● Not my idea
● Common practice within a team I used to work
with in the past
An integration test is failing, because the network is down. We cannot do anything about it. Is it supposed to be up in 3 days again, according to the networks team. That means our build will be red for the next 3 days, because of this test. If an other test starts failing we might not even notice it, because the build is already red.