Merge pull request SCons#3104 from dmoody256/JobsTestsParallelFix

JobTests.py parallel issue fix

Author: bdbaddog

.travis.yml

@@ -14,12 +14,7 @@ jobs:
   include:
     - &test_job
       stage: Test
-      script: 
-      # WORKAROUND: attempt to retry JobTests.py if it fails and then continue if it passes, if it fails ten times
-      # then it is a real failure not related to intermittent travis failures
-        - n=0; while [[ $n -lt 10 ]]; do python runtest.py src/engine/SCons/JobTests.py && break; n=$((n+1)); done; if [ "$n" -gt "9" ]; then false; fi
-        - echo "src/engine/SCons/JobTests.py" > exclude_jobtest
-        - python runtest.py -a --exclude-list exclude_jobtest || if [[ $? == 2 ]]; then true; else false; fi
+      script: python runtest.py -a || if [[ $? == 2 ]]; then true; else false; fi
       before_script: skip
       after_success: skip
       python: 2.7
@@ -71,14 +66,8 @@ jobs:
         - echo "parallel = True" >> .coveragerc
         - printf "omit =\n\t*Tests.py\n\tsrc/test_*\n\tsrc/setup.py\n\n" >> .coveragerc
         - echo "[path] = $PWD" >> .coveragerc
-      # Not including this workaround in the coverage report, because it will result
-      # in constantly changing coverage reports depending on the number of times
-      # the JobTests.py had to run to pass
-      # TODO: figure out how to cover JobTests.py
-      # - n=0; while [[ $n -lt 10 ]]; do coverage run --rcfile=$PWD/.coveragerc runtest.py src/engine/SCons/JobTests.py && break; n=$((n+1)); done; if [ "$n" -gt "9" ]; then false; fi
-      # exclude JobTest.py becuase we already ran that
-        - echo "src/engine/SCons/JobTests.py" > exclude_jobtest
-        - python runtest.py -l -a --exclude-list exclude_jobtest > all_tests
+        # get a list of all the tests to split them up
+        - python runtest.py -l -a > all_tests
         - let "start = ($(wc -l < all_tests) / ${TOTAL_BUILD_JOBS}) * (${BUILD_JOB_NUM} - 1)"; true;
         - let "end = ($(wc -l < all_tests) / ${TOTAL_BUILD_JOBS}) * ${BUILD_JOB_NUM}"
         - if (( ${BUILD_JOB_NUM} == ${TOTAL_BUILD_JOBS} )); then end=$(wc -l < all_tests); fi

src/engine/SCons/JobTests.py

@@ -52,7 +52,7 @@ def get_cpu_nums():
     return 1 # Default
 
 # a large number
-num_sines = 10000
+num_sines = 500
 
 # how many parallel jobs to perform for the test
 num_jobs = get_cpu_nums()*2
@@ -105,7 +105,17 @@ def execute(self):
         self.taskmaster.begin_list.append(self.i)
         self.taskmaster.guard.release()
 
+        # while task is executing, represent this in the parallel_list
+        # and then turn it off
+        self.taskmaster.parallel_list[self.i] = 1
         self._do_something()
+        self.taskmaster.parallel_list[self.i] = 0
+
+        # check if task was executing while another was also executing
+        for j in range(1, self.taskmaster.num_tasks):
+            if(self.taskmaster.parallel_list[j+1] == 1):
+                self.taskmaster.found_parallel = True
+                break
 
         self.was_executed = 1
 
@@ -132,10 +142,13 @@ def failed(self):
     def postprocess(self):
         self.taskmaster.num_postprocessed = self.taskmaster.num_postprocessed + 1
 
+    def exception_set(self):
+        pass
+
 class RandomTask(Task):
     def _do_something(self):
         # do something that will take some random amount of time:
-        for i in range(random.randrange(0, num_sines, 1)):
+        for i in range(random.randrange(0, 100 + num_sines, 1)):
             x = math.sin(i)
         time.sleep(0.01)
 
@@ -190,7 +203,10 @@ def __init__(self, n, test_case, Task):
         self.num_executed = 0
         self.num_failed = 0
         self.num_postprocessed = 0
+        self.parallel_list = [0] * (n+1)
+        self.found_parallel = False
         self.Task = Task
+        
         # 'guard' guards 'task_begin_list' and 'task_end_list'
         try:
             import threading
@@ -222,12 +238,7 @@ def all_tasks_are_postprocessed(self):
 
     def tasks_were_serial(self):
         "analyze the task order to see if they were serial"
-        serial = 1 # assume the tasks were serial
-        for i in range(num_tasks):
-            serial = serial and (self.begin_list[i]
-                                 == self.end_list[i]
-                                 == (i + 1))
-        return serial
+        return not self.found_parallel
 
     def exception_set(self):
         pass
@@ -271,7 +282,7 @@ def runTest(self):
 
         class SleepTask(Task):
             def _do_something(self):
-                time.sleep(0.1)
+                time.sleep(0.01)
 
         global SaveThreadPool
         SaveThreadPool = SCons.Job.ThreadPool
@@ -281,7 +292,7 @@ def put(self, task):
                 ThreadPoolCallList.append('put(%s)' % task.i)
                 return SaveThreadPool.put(self, task)
             def get(self):
-                time.sleep(0.5)
+                time.sleep(0.05)
                 result = SaveThreadPool.get(self)
                 ThreadPoolCallList.append('get(%s)' % result[0].i)
                 return result