From 94c5068bca056864515aa9c0f48bcaac397da6df Mon Sep 17 00:00:00 2001 From: Jeffrey Thiessen Date: Wed, 13 Dec 2023 13:22:16 -0600 Subject: [PATCH 1/5] first cut --- .../irida/service/SequencingObjectProcessingService.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectProcessingService.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectProcessingService.java index 3970d1e3175..8925d3b0f0b 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectProcessingService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectProcessingService.java @@ -72,6 +72,12 @@ public synchronized void findFilesToProcess() { while (queueSpace > 0 && iterator.hasNext()) { SequencingObject sequencingObject = iterator.next(); + if (!sequencingObject.getSequencingRun().getUploadStatus().equals(SequencingRunUploadStatus.COMPLETE)) { + logger.trace("Skipping file " + sequencingObject.getId() + " as is not on a COMPLETE sequencing run.") + continue; + // must test this works on files uploaded via API and via Web + } + logger.trace("File processor " + machineString + " is processing file " + sequencingObject.getId()); try { From f7a19424b31ca7fd6c8862b521836f4b84531d5b Mon Sep 17 00:00:00 2001 From: Jeffrey Thiessen Date: Wed, 13 Dec 2023 16:55:07 -0600 Subject: [PATCH 2/5] fix working --- .../service/SequencingObjectProcessingService.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectProcessingService.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectProcessingService.java index 8925d3b0f0b..eac12842fab 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectProcessingService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectProcessingService.java @@ -1,6 +1,7 @@ package ca.corefacility.bioinformatics.irida.service; import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject; +import ca.corefacility.bioinformatics.irida.model.enums.SequencingRunUploadStatus; import ca.corefacility.bioinformatics.irida.processing.FileProcessingChain; import ca.corefacility.bioinformatics.irida.repositories.sequencefile.SequencingObjectRepository; import ca.corefacility.bioinformatics.irida.service.impl.processor.SequenceFileProcessorLauncher; @@ -71,11 +72,13 @@ public synchronized void findFilesToProcess() { while (queueSpace > 0 && iterator.hasNext()) { SequencingObject sequencingObject = iterator.next(); - - if (!sequencingObject.getSequencingRun().getUploadStatus().equals(SequencingRunUploadStatus.COMPLETE)) { - logger.trace("Skipping file " + sequencingObject.getId() + " as is not on a COMPLETE sequencing run.") +// logger.info("HERE*************************************"); + if ( + (sequencingObject.getSequencingRun() != null) && + (!sequencingObject.getSequencingRun().getUploadStatus().equals(SequencingRunUploadStatus.COMPLETE)) + ) { + logger.trace("Skipping file " + sequencingObject.getId() + " as is not on a COMPLETE sequencing run."); continue; - // must test this works on files uploaded via API and via Web } logger.trace("File processor " + machineString + " is processing file " + sequencingObject.getId()); From 50a1b97644b398c28f26fe7b8c94adf7e10e8343 Mon Sep 17 00:00:00 2001 From: Jeffrey Thiessen Date: Wed, 13 Dec 2023 16:58:28 -0600 Subject: [PATCH 3/5] remove comment --- .../irida/service/SequencingObjectProcessingService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectProcessingService.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectProcessingService.java index eac12842fab..c2334970a43 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectProcessingService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectProcessingService.java @@ -72,7 +72,7 @@ public synchronized void findFilesToProcess() { while (queueSpace > 0 && iterator.hasNext()) { SequencingObject sequencingObject = iterator.next(); -// logger.info("HERE*************************************"); + if ( (sequencingObject.getSequencingRun() != null) && (!sequencingObject.getSequencingRun().getUploadStatus().equals(SequencingRunUploadStatus.COMPLETE)) From 173053fd14aa9d68c94b767597634a31be07c2b6 Mon Sep 17 00:00:00 2001 From: Jeffrey Thiessen Date: Wed, 13 Dec 2023 17:43:06 -0600 Subject: [PATCH 4/5] Use list filter instead of if inside loop --- .../service/SequencingObjectProcessingService.java | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectProcessingService.java b/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectProcessingService.java index c2334970a43..211b53c0d54 100644 --- a/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectProcessingService.java +++ b/src/main/java/ca/corefacility/bioinformatics/irida/service/SequencingObjectProcessingService.java @@ -67,20 +67,18 @@ public synchronized void findFilesToProcess() { List toProcess = sequencingObjectRepository .getSequencingObjectsWithProcessingState(SequencingObject.ProcessingState.UNPROCESSED); + // filter out sequencing objects on a SequencingRun that is not in a COMPLETE state + toProcess.removeIf(seqObj -> ( + (seqObj.getSequencingRun() != null) && + (!seqObj.getSequencingRun().getUploadStatus().equals(SequencingRunUploadStatus.COMPLETE)) + )); + // individually loop through and mark the ones we're going to process. Looping individually so 2 processes are less likely to write at the same time. Iterator iterator = toProcess.iterator(); while (queueSpace > 0 && iterator.hasNext()) { SequencingObject sequencingObject = iterator.next(); - if ( - (sequencingObject.getSequencingRun() != null) && - (!sequencingObject.getSequencingRun().getUploadStatus().equals(SequencingRunUploadStatus.COMPLETE)) - ) { - logger.trace("Skipping file " + sequencingObject.getId() + " as is not on a COMPLETE sequencing run."); - continue; - } - logger.trace("File processor " + machineString + " is processing file " + sequencingObject.getId()); try { From 997fa5e7c1bfec742897296373456bc2ad7555ac Mon Sep 17 00:00:00 2001 From: Jeffrey Thiessen Date: Thu, 14 Dec 2023 10:47:28 -0600 Subject: [PATCH 5/5] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index da709766cc5..0d91a20431f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ # [23.10.1] - 2023/12/07 * [UI]: Fixed issue where filter inputs required focus when they are opened on the project samples page. See [PR 1503](https://github.com/phac-nml/irida/pull/1503) * [UI]: Fixed bug preventing sorting and paging on the project members page. See [PR 1504](https://github.com/phac-nml/irida/pull/1504) +* [Developer]: Fixed race condition where FastQC file processor would start on incomplete file by filtering out files on non complete sequencing runs. [PR 1506](https://github.com/phac-nml/irida/pull/1506) ## [23.10] - 2023/10/15 * [Developer]: Added functionality to delete sequence files from file system when a sequence run is removed. [See PR 1468](https://github.com/phac-nml/irida/pull/1468)