Merge branch 'multiple_reruns' into 'master'

Add support for multiple reruns on failed runs

See merge request integration/nightlyjobs!16

src/org/moodle/ci/Task.groovy

@@ -126,6 +126,13 @@ class Task {
      */
     protected String runnerVersion = STABLE
 
+    /**
+     * The number of reruns to perform with failed runs.
+     *
+     * Defaults to 1 for all our jobs if not specified via Task param. 0 means no reruns.
+     */
+     protected Integer numReruns = 1
+
     /**
      * The URL of the repository to fetch from.
      *
@@ -186,6 +193,21 @@ class Task {
         throw new Exception("I don't know what PHP Version to test against")
     }
 
+    /**
+     * The number of reruns to perform with failed runs
+     *
+     * Forced to be between 0 (no reruns) and 9.
+     */
+    def getNumReruns() {
+        if (numReruns < 0) {
+            return 1
+        }
+        if (numReruns > 9) {
+            return 9
+        }
+        return numReruns
+    }
+
     def getPathToRunner(env, steps) {
         def runner
         if (pathToRunner) {

test/org/moodle/ci/TaskSpec.groovy

@@ -57,6 +57,65 @@ class TaskSpec extends Specification {
         assert output == "MDL-12345-34: phpunit pgsql (PHP7.0)"
     }
 
+    def "getNumReruns() with no numReruns"() {
+        given:
+
+        when:
+        def task = new Task()
+        def returns = task.getNumReruns()
+
+        then:
+        assert returns == 1
+    }
+
+    def "getNumReruns() with numReruns"() {
+        given:
+        def numReruns = 3
+
+        when:
+        def task = new Task(numReruns : numReruns)
+        def returns = task.getNumReruns()
+
+        then:
+        assert returns == 3
+    }
+
+    def "getNumReruns() with zero numReruns"() {
+        given:
+        def numReruns = 0
+
+        when:
+        def task = new Task(numReruns : numReruns)
+        def returns = task.getNumReruns()
+
+        then:
+        assert returns == 0
+    }
+
+    def "getNumReruns() with low numReruns"() {
+        given:
+        def numReruns = -1
+
+        when:
+        def task = new Task(numReruns : numReruns)
+        def returns = task.getNumReruns()
+
+        then:
+        assert returns == 1
+    }
+
+    def "getNumReruns() with high numReruns"() {
+        given:
+        def numReruns = 999
+
+        when:
+        def task = new Task(numReruns : numReruns)
+        def returns = task.getNumReruns()
+
+        then:
+        assert returns == 9
+    }
+
     def "getMobileVersion() with no version"() {
         given:
         def mobileVersion = null

vars/runTask.groovy

@@ -18,6 +18,7 @@ def call(org.moodle.ci.Task task) {
             TESTSUITE = "${task.testsuite}"
             BEHAT_SUITE = "${task.suite}"
             BEHAT_TIMING_FILENAME = "${task.timingFileName}"
+            BEHAT_NUM_RERUNS = "${task.numReruns}"
         }
 
         stages {
@@ -81,9 +82,19 @@ def call(org.moodle.ci.Task task) {
         post {
             always {
                 // Process the jUnit test results.
-                junit allowEmptyResults: true, testDataPublishers: [[$class: 'ClaimTestDataPublisher']], testResults: "${env.BUILD_ID}/*.junit/*.xml,${env.BUILD_ID}/*.junit"
+                // By default only process last-rerun junit files. Those are the final, consistent failures.
+                script {
+                    def jFiles = "${env.BUILD_ID}/*rerun${task.numReruns}.junit/*.xml"
+                    if (task.numReruns == 0) {
+                        // Without reruns, let's process the main junit logs.
+                        jFiles = "${env.BUILD_ID}/*.junit/*.xml"
+                    }
+                    junit allowEmptyResults: true,
+                        testDataPublishers: [[$class: 'ClaimTestDataPublisher']],
+                        testResults: "${jFiles}"
+                }
 
-                // Keep a copy of the artifacts.
+                // Keep a copy of the junit artifacts into s3 bucket.
                 s3Upload consoleLogLevel: 'INFO',
                     dontSetBuildResultOnFailure: false,
                     dontWaitForConcurrentBuildCompletion: false,
@@ -98,7 +109,7 @@ def call(org.moodle.ci.Task task) {
                             noUploadOnFailure: false,
                             selectedRegion: 'eu-west-1',
                             showDirectlyInBrowser: true,
-                            sourceFile: "${env.BUILD_ID}/*.junit/*.xml,${env.BUILD_ID}/*.junit",
+                            sourceFile: "${env.BUILD_ID}/*.junit/*.xml",
                             storageClass: 'STANDARD',
                             uploadFromSlave: true,
                             useServerSideEncryption: false
@@ -109,7 +120,7 @@ def call(org.moodle.ci.Task task) {
                     userMetadata: []
 
                 // Keep all artifacts in the build directory, but don't store the junit files. These are in S3 anyway.
-                archiveArtifacts allowEmptyArchive: true, artifacts: "${env.BUILD_ID}/**", excludes: "${env.BUILD_ID}/*.junit/*.xml,${env.BUILD_ID}/*.junit"
+                archiveArtifacts allowEmptyArchive: true, artifacts: "${env.BUILD_ID}/**", excludes: "${env.BUILD_ID}/*.junit/*.xml"
 
                 // Clean the workspace. No point storing all of this.
                 cleanWs deleteDirs: true, notFailBuild: true