diff --git a/docs/config_reference.rst b/docs/config_reference.rst index 90bb95277..af808c1d7 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -347,6 +347,14 @@ System Partition Configuration .. versionadded:: 4.10 The ``pbspro`` scheduler is added. + .. versionchanged:: 4.11 + Tests whose Slurm job has expired are now skipped instead of failed. + + .. note:: + If a job submitted through the ``slurm`` backend ends up in the ``DEADLINE`` state, i.e., it did not start before the deadline set with ``--deadline``, the associated test will be skipped and not failed. + This applies to the ``slurm`` backend only; + the ``squeue`` backend does not treat expired jobs specially, because once the job is out of the queue, its state cannot be retrieved. + .. note:: The Slurm-based backends unset all ``SBATCH_*`` environment variables before submitting a job. This is done to avoid environment variables bypassing ReFrame's configuration. diff --git a/reframe/core/schedulers/slurm.py b/reframe/core/schedulers/slurm.py index 5892a936f..c25cb4089 100644 --- a/reframe/core/schedulers/slurm.py +++ b/reframe/core/schedulers/slurm.py @@ -20,7 +20,8 @@ from reframe.core.backends import register_scheduler from reframe.core.exceptions import (SpawnedProcessError, JobBlockedError, - JobSchedulerError) + JobSchedulerError, + SkipTestError) from reframe.utility import nodelist_abbrev, seconds_to_hms @@ -545,6 +546,12 @@ def poll(self, *jobs): job._state = ','.join(m.group('state') for m in jobarr_info) if slurm_state_completed(job.state): + # If the job has hit its DEADLINE, skip the test + if job.state == 'DEADLINE': + job._exception = SkipTestError( + f'the associated job {job.jobid} has expired' + ) + # Since Slurm exitcodes are positive take the maximum one job._exitcode = max( int(m.group('exitcode')) for m in jobarr_info